mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 06:36:55 +01:00
Add machine-readable review contracts (#182)
Generate a deterministic action-skill index from frontmatter, publish structural schemas for orchestration and findings, and validate flat review composition in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: dayland <dayland@microsoft.com> Copilot-Session: 76eb42c4-2acd-4f9c-a898-f4a44f9d46f7
This commit is contained in:
parent
852a676285
commit
b74967bc5b
8 changed files with 803 additions and 3 deletions
240
.github/scripts/Test-SkillIndex.ps1
vendored
Normal file
240
.github/scripts/Test-SkillIndex.ps1
vendored
Normal file
|
|
@ -0,0 +1,240 @@
|
||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Validates the BCQuality action-skill index generator and shared schemas.
|
||||||
|
#>
|
||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[string] $Root = (Resolve-Path (Join-Path -Path $PSScriptRoot -ChildPath '..' '..'))
|
||||||
|
)
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = 'Stop'
|
||||||
|
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||||
|
|
||||||
|
function Assert-ThrowsLike {
|
||||||
|
param(
|
||||||
|
[scriptblock] $Action,
|
||||||
|
[string] $Pattern
|
||||||
|
)
|
||||||
|
|
||||||
|
try {
|
||||||
|
& $Action
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
if ($_.Exception.Message -like $Pattern) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
throw "Expected error like '$Pattern', received: $($_.Exception.Message)"
|
||||||
|
}
|
||||||
|
throw "Expected error like '$Pattern', but no error was thrown."
|
||||||
|
}
|
||||||
|
|
||||||
|
$generator = Join-Path $Root 'tools/Build-SkillIndex.ps1'
|
||||||
|
$indexSchema = Join-Path $Root 'schemas/skill-index.schema.json'
|
||||||
|
$reportSchema = Join-Path $Root 'schemas/findings-report.schema.json'
|
||||||
|
foreach ($path in $generator, $indexSchema, $reportSchema) {
|
||||||
|
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
||||||
|
throw "Required contract file not found: $path"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("skillindex_" + [guid]::NewGuid().ToString('N'))
|
||||||
|
New-Item -ItemType Directory -Path $tmp -Force | Out-Null
|
||||||
|
try {
|
||||||
|
$first = Join-Path $tmp 'first.json'
|
||||||
|
$second = Join-Path $tmp 'second.json'
|
||||||
|
& $generator -BCQualityRoot $Root -IndexPath $first | Out-Null
|
||||||
|
& $generator -BCQualityRoot $Root -IndexPath $second | Out-Null
|
||||||
|
|
||||||
|
$normalize = {
|
||||||
|
param([string] $Path)
|
||||||
|
return ((Get-Content -LiteralPath $Path -Raw) -replace '"generatedAt":"[^"]*"', '"generatedAt":"<timestamp>"')
|
||||||
|
}
|
||||||
|
if ((& $normalize $first) -ne (& $normalize $second)) {
|
||||||
|
throw 'Skill index is not deterministic beyond generatedAt.'
|
||||||
|
}
|
||||||
|
|
||||||
|
$raw = Get-Content -LiteralPath $first -Raw
|
||||||
|
if (-not ($raw | Test-Json -SchemaFile $indexSchema -ErrorAction Stop)) {
|
||||||
|
throw 'Generated skill index does not satisfy schemas/skill-index.schema.json.'
|
||||||
|
}
|
||||||
|
|
||||||
|
$index = $raw | ConvertFrom-Json
|
||||||
|
$skills = @($index.skills)
|
||||||
|
if ($index.skillCount -ne $skills.Count) {
|
||||||
|
throw "skillCount is $($index.skillCount), but the index contains $($skills.Count) records."
|
||||||
|
}
|
||||||
|
|
||||||
|
$paths = @($skills.path)
|
||||||
|
$duplicates = @($paths | Group-Object | Where-Object Count -gt 1)
|
||||||
|
if ($duplicates.Count) {
|
||||||
|
throw "Duplicate skill paths: $($duplicates.Name -join ', ')"
|
||||||
|
}
|
||||||
|
foreach ($path in $paths) {
|
||||||
|
if (-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
||||||
|
throw "Indexed skill does not exist: $path"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$expectedLeaves = @(
|
||||||
|
'microsoft/skills/review/al-performance-review.md',
|
||||||
|
'microsoft/skills/review/al-security-review.md',
|
||||||
|
'microsoft/skills/review/al-privacy-review.md',
|
||||||
|
'microsoft/skills/review/al-upgrade-review.md',
|
||||||
|
'microsoft/skills/review/al-style-review.md',
|
||||||
|
'microsoft/skills/review/al-ui-review.md',
|
||||||
|
'microsoft/skills/review/al-error-handling-review.md',
|
||||||
|
'microsoft/skills/review/al-events-review.md',
|
||||||
|
'microsoft/skills/review/al-interfaces-review.md',
|
||||||
|
'microsoft/skills/review/al-breaking-changes-review.md',
|
||||||
|
'microsoft/skills/review/al-web-services-review.md',
|
||||||
|
'microsoft/skills/review/al-testing-review.md',
|
||||||
|
'microsoft/skills/review/al-data-modeling-review.md',
|
||||||
|
'microsoft/skills/review/al-query-review.md',
|
||||||
|
'microsoft/skills/review/al-appsource-review.md',
|
||||||
|
'microsoft/skills/review/al-telemetry-review.md'
|
||||||
|
)
|
||||||
|
$review = @($skills | Where-Object id -eq 'al-code-review')
|
||||||
|
if ($review.Count -ne 1) {
|
||||||
|
throw "Expected exactly one al-code-review record, found $($review.Count)."
|
||||||
|
}
|
||||||
|
if ((@($review[0].subSkills) -join "`n") -cne ($expectedLeaves -join "`n")) {
|
||||||
|
throw 'al-code-review subSkills did not preserve the declared 16-leaf order.'
|
||||||
|
}
|
||||||
|
foreach ($leafPath in $expectedLeaves) {
|
||||||
|
$leaf = @($skills | Where-Object path -ceq $leafPath)
|
||||||
|
if ($leaf.Count -ne 1 -or @($leaf[0].subSkills).Count -ne 0) {
|
||||||
|
throw "Expected '$leafPath' to resolve to exactly one leaf action skill."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$minimalReport = @{
|
||||||
|
skill = @{ id = 'al-style-review'; version = 1 }
|
||||||
|
outcome = 'completed'
|
||||||
|
summary = @{
|
||||||
|
counts = @{ blocker = 0; major = 0; minor = 0; info = 0 }
|
||||||
|
coverage = @{ 'worklist-size' = 0; 'items-evaluated' = 0 }
|
||||||
|
}
|
||||||
|
findings = @()
|
||||||
|
suppressed = @()
|
||||||
|
} | ConvertTo-Json -Depth 8
|
||||||
|
if (-not ($minimalReport | Test-Json -SchemaFile $reportSchema -ErrorAction Stop)) {
|
||||||
|
throw 'Minimal findings report does not satisfy schemas/findings-report.schema.json.'
|
||||||
|
}
|
||||||
|
|
||||||
|
$reviewSkillText = Get-Content -LiteralPath (
|
||||||
|
Join-Path -Path $Root -ChildPath 'microsoft/skills/review/al-code-review.md'
|
||||||
|
) -Raw
|
||||||
|
$reportExamples = [regex]::Matches($reviewSkillText, '(?s)```json\s*(\{.*?\})\s*```')
|
||||||
|
if ($reportExamples.Count -ne 2) {
|
||||||
|
throw "Expected two al-code-review JSON examples, found $($reportExamples.Count)."
|
||||||
|
}
|
||||||
|
foreach ($example in $reportExamples) {
|
||||||
|
if (-not ($example.Groups[1].Value | Test-Json -SchemaFile $reportSchema -ErrorAction Stop)) {
|
||||||
|
throw 'An al-code-review output example does not satisfy schemas/findings-report.schema.json.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$fixtureRoot = Join-Path -Path $tmp -ChildPath 'fixture'
|
||||||
|
$fixtureSkills = Join-Path -Path $fixtureRoot -ChildPath 'microsoft/skills/review'
|
||||||
|
New-Item -ItemType Directory -Path $fixtureSkills -Force | Out-Null
|
||||||
|
$leaf = @'
|
||||||
|
---
|
||||||
|
kind: action-skill
|
||||||
|
id: al-leaf-review
|
||||||
|
version: 1
|
||||||
|
title: Leaf
|
||||||
|
description: Test leaf.
|
||||||
|
inputs: [file-path]
|
||||||
|
outputs: [findings-report]
|
||||||
|
---
|
||||||
|
|
||||||
|
# Leaf
|
||||||
|
|
||||||
|
## Source
|
||||||
|
Source.
|
||||||
|
## Relevance
|
||||||
|
Relevance.
|
||||||
|
## Worklist
|
||||||
|
Worklist.
|
||||||
|
## Action
|
||||||
|
Action.
|
||||||
|
## Output
|
||||||
|
Output.
|
||||||
|
'@
|
||||||
|
Set-Content -LiteralPath (Join-Path $fixtureSkills 'al-leaf-review.md') -Value $leaf -Encoding utf8NoBOM
|
||||||
|
|
||||||
|
$duplicateSuper = @'
|
||||||
|
---
|
||||||
|
kind: action-skill
|
||||||
|
id: al-code-review
|
||||||
|
version: 1
|
||||||
|
title: Review
|
||||||
|
description: Test super-skill.
|
||||||
|
inputs: [file-path]
|
||||||
|
outputs: [findings-report]
|
||||||
|
sub-skills:
|
||||||
|
- microsoft/skills/review/al-leaf-review.md
|
||||||
|
- microsoft/skills/review/al-leaf-review.md
|
||||||
|
---
|
||||||
|
|
||||||
|
# Review
|
||||||
|
|
||||||
|
## Source
|
||||||
|
Source.
|
||||||
|
## Relevance
|
||||||
|
Relevance.
|
||||||
|
## Worklist
|
||||||
|
Worklist.
|
||||||
|
## Action
|
||||||
|
Action.
|
||||||
|
## Output
|
||||||
|
Output.
|
||||||
|
'@
|
||||||
|
$superPath = Join-Path $fixtureSkills 'al-code-review.md'
|
||||||
|
Set-Content -LiteralPath $superPath -Value $duplicateSuper -Encoding utf8NoBOM
|
||||||
|
Assert-ThrowsLike -Pattern '*duplicate sub-skill*' -Action {
|
||||||
|
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'invalid.json')
|
||||||
|
}
|
||||||
|
|
||||||
|
$nestedLeaf = $leaf.Replace('id: al-leaf-review', 'id: al-nested-review').Replace(
|
||||||
|
'outputs: [findings-report]',
|
||||||
|
"outputs: [findings-report]`nsub-skills:`n - microsoft/skills/review/al-leaf-review.md"
|
||||||
|
)
|
||||||
|
Set-Content -LiteralPath (Join-Path $fixtureSkills 'al-nested-review.md') -Value $nestedLeaf -Encoding utf8NoBOM
|
||||||
|
$nestedSuper = @'
|
||||||
|
---
|
||||||
|
kind: action-skill
|
||||||
|
id: al-code-review
|
||||||
|
version: 1
|
||||||
|
title: Review
|
||||||
|
description: Test super-skill.
|
||||||
|
inputs: [file-path]
|
||||||
|
outputs: [findings-report]
|
||||||
|
sub-skills:
|
||||||
|
- microsoft/skills/review/al-nested-review.md
|
||||||
|
---
|
||||||
|
|
||||||
|
# Review
|
||||||
|
|
||||||
|
## Source
|
||||||
|
Source.
|
||||||
|
## Relevance
|
||||||
|
Relevance.
|
||||||
|
## Worklist
|
||||||
|
Worklist.
|
||||||
|
## Action
|
||||||
|
Action.
|
||||||
|
## Output
|
||||||
|
Output.
|
||||||
|
'@
|
||||||
|
Set-Content -LiteralPath $superPath -Value $nestedSuper -Encoding utf8NoBOM
|
||||||
|
Assert-ThrowsLike -Pattern '*Nested super-skills are not supported*' -Action {
|
||||||
|
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'nested.json')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Output 'Skill-index check PASSED: deterministic, schema-valid, and all 16 review leaves preserved in order.'
|
||||||
41
.github/scripts/validate_frontmatter.py
vendored
41
.github/scripts/validate_frontmatter.py
vendored
|
|
@ -381,6 +381,20 @@ def validate_action_skill(path: Path, parsed: Parsed, report: Report) -> None:
|
||||||
bad = [x for x in ss if not x.endswith(".md")]
|
bad = [x for x in ss if not x.endswith(".md")]
|
||||||
if bad:
|
if bad:
|
||||||
report.error(path, "R20", f"sub-skills entries must end in '.md': {bad}", 1)
|
report.error(path, "R20", f"sub-skills entries must end in '.md': {bad}", 1)
|
||||||
|
non_canonical = [
|
||||||
|
x for x in ss
|
||||||
|
if "\\" in x or x.startswith("/") or ".." in Path(x).parts or x.startswith("./")
|
||||||
|
]
|
||||||
|
if non_canonical:
|
||||||
|
report.error(
|
||||||
|
path,
|
||||||
|
"R20",
|
||||||
|
f"sub-skills entries must be canonical repo-relative paths: {non_canonical}",
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
duplicates = sorted({x for x in ss if ss.count(x) > 1})
|
||||||
|
if duplicates:
|
||||||
|
report.error(path, "R20", f"sub-skills contains duplicate paths: {duplicates}", 1)
|
||||||
|
|
||||||
# R21 five required sections, in order, each exactly once
|
# R21 five required sections, in order, each exactly once
|
||||||
heads = [h for h, _ in headings_in_order(parsed.body)]
|
heads = [h for h, _ in headings_in_order(parsed.body)]
|
||||||
|
|
@ -565,7 +579,13 @@ class SkillRecord:
|
||||||
skill_id: str | None
|
skill_id: str | None
|
||||||
|
|
||||||
|
|
||||||
def validate_sub_skills_registry(path: Path, fm: dict[str, Any], root: Path, report: Report) -> None:
|
def validate_sub_skills_registry(
|
||||||
|
path: Path,
|
||||||
|
fm: dict[str, Any],
|
||||||
|
root: Path,
|
||||||
|
action_skills_by_path: dict[str, dict[str, Any]],
|
||||||
|
report: Report,
|
||||||
|
) -> None:
|
||||||
"""R26: a super-skill's declared `sub-skills` must exactly match the
|
"""R26: a super-skill's declared `sub-skills` must exactly match the
|
||||||
`al-*-review.md` leaf files present in the same directory (set equality,
|
`al-*-review.md` leaf files present in the same directory (set equality,
|
||||||
ordering-agnostic). This keeps the registered leaf list the single source
|
ordering-agnostic). This keeps the registered leaf list the single source
|
||||||
|
|
@ -598,6 +618,17 @@ def validate_sub_skills_registry(path: Path, fm: dict[str, Any], root: Path, rep
|
||||||
f"sub-skills entry is not a sibling 'al-*-review.md' leaf: {entry}", 1,
|
f"sub-skills entry is not a sibling 'al-*-review.md' leaf: {entry}", 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
for entry in ss:
|
||||||
|
leaf = action_skills_by_path.get(entry)
|
||||||
|
if leaf is None:
|
||||||
|
if (root / entry).exists():
|
||||||
|
report.error(path, "R26", f"sub-skills entry is not an action skill: {entry}", 1)
|
||||||
|
continue
|
||||||
|
if is_non_empty_list_of_str(leaf.get("sub-skills")):
|
||||||
|
report.error(path, "R26", f"nested super-skill is not permitted in v1 composition: {entry}", 1)
|
||||||
|
if leaf.get("outputs") != ["findings-report"]:
|
||||||
|
report.error(path, "R26", f"sub-skill must produce findings-report: {entry}", 1)
|
||||||
|
|
||||||
# Sibling leaves on disk that were never registered ('forgot to wire it up').
|
# Sibling leaves on disk that were never registered ('forgot to wire it up').
|
||||||
for leaf in sorted(leaves - declared):
|
for leaf in sorted(leaves - declared):
|
||||||
report.error(path, "R26", f"leaf not registered in sub-skills: {leaf}", 1)
|
report.error(path, "R26", f"leaf not registered in sub-skills: {leaf}", 1)
|
||||||
|
|
@ -670,9 +701,13 @@ def run(root: Path) -> Report:
|
||||||
others = [q.relative_to(root).as_posix() for q in paths if q != p]
|
others = [q.relative_to(root).as_posix() for q in paths if q != p]
|
||||||
report.error(p, "R24", f"skill id '{sid}' ({kind}) is not unique; also defined in: {others}")
|
report.error(p, "R24", f"skill id '{sid}' ({kind}) is not unique; also defined in: {others}")
|
||||||
|
|
||||||
# Fourth pass: R26 sub-skills registry matches leaf files on disk
|
# Fourth pass: R26 sub-skills registry matches compatible leaf files on disk
|
||||||
|
action_skills_by_path = {
|
||||||
|
path.relative_to(root).as_posix(): fm
|
||||||
|
for path, fm in action_skill_fms
|
||||||
|
}
|
||||||
for path, fm in action_skill_fms:
|
for path, fm in action_skill_fms:
|
||||||
validate_sub_skills_registry(path, fm, root, report)
|
validate_sub_skills_registry(path, fm, root, action_skills_by_path, report)
|
||||||
|
|
||||||
return report
|
return report
|
||||||
|
|
||||||
|
|
|
||||||
18
.github/workflows/skill-index.yml
vendored
Normal file
18
.github/workflows/skill-index.yml
vendored
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
name: Validate skill index and report schemas
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
validate-contract:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Check out repository
|
||||||
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
|
|
||||||
|
- name: Validate skill-index generator and schemas
|
||||||
|
shell: pwsh
|
||||||
|
run: ./.github/scripts/Test-SkillIndex.ps1 -Root .
|
||||||
|
|
@ -41,6 +41,11 @@ An orchestrator invokes this skill with a `pr-diff`, `file-path`, or `folder-pat
|
||||||
|
|
||||||
The sub-skills invoked by this skill are those listed in frontmatter `sub-skills`. Additional leaf skills are added by updating the `sub-skills` list. The skill does not discover sub-skills implicitly.
|
The sub-skills invoked by this skill are those listed in frontmatter `sub-skills`. Additional leaf skills are added by updating the `sub-skills` list. The skill does not discover sub-skills implicitly.
|
||||||
|
|
||||||
|
Hosts that orchestrate leaves mechanically SHOULD run
|
||||||
|
`tools/Build-SkillIndex.ps1` and resolve this skill by `id: al-code-review`.
|
||||||
|
The generated `subSkills` array preserves the frontmatter order and avoids
|
||||||
|
host-specific Markdown parsing.
|
||||||
|
|
||||||
## Relevance
|
## Relevance
|
||||||
|
|
||||||
A sub-skill is relevant when both of the following hold:
|
A sub-skill is relevant when both of the following hold:
|
||||||
|
|
|
||||||
159
schemas/findings-report.schema.json
Normal file
159
schemas/findings-report.schema.json
Normal file
|
|
@ -0,0 +1,159 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"$id": "https://github.com/microsoft/BCQuality/schemas/findings-report.schema.json",
|
||||||
|
"title": "BCQuality findings report",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["skill", "outcome", "summary", "findings", "suppressed"],
|
||||||
|
"properties": {
|
||||||
|
"skill": { "$ref": "#/definitions/skillReference" },
|
||||||
|
"outcome": {
|
||||||
|
"enum": ["completed", "not-applicable", "no-knowledge", "partial", "failed"]
|
||||||
|
},
|
||||||
|
"outcome-reason": { "type": "string", "minLength": 1 },
|
||||||
|
"summary": { "$ref": "#/definitions/summary" },
|
||||||
|
"findings": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/finding" }
|
||||||
|
},
|
||||||
|
"suppressed": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/suppressed" }
|
||||||
|
},
|
||||||
|
"sub-results": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#" }
|
||||||
|
},
|
||||||
|
"skipped-sub-skills": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/skippedSubSkill" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"allOf": [
|
||||||
|
{
|
||||||
|
"if": {
|
||||||
|
"properties": {
|
||||||
|
"outcome": { "enum": ["partial", "failed"] }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"then": { "required": ["outcome-reason"] }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"if": {
|
||||||
|
"properties": {
|
||||||
|
"outcome": { "enum": ["not-applicable", "no-knowledge", "failed"] }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"then": {
|
||||||
|
"properties": {
|
||||||
|
"findings": { "maxItems": 0 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"definitions": {
|
||||||
|
"skillReference": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["id", "version"],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" },
|
||||||
|
"version": { "type": "integer", "minimum": 1 }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"counts": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["blocker", "major", "minor", "info"],
|
||||||
|
"properties": {
|
||||||
|
"blocker": { "type": "integer", "minimum": 0 },
|
||||||
|
"major": { "type": "integer", "minimum": 0 },
|
||||||
|
"minor": { "type": "integer", "minimum": 0 },
|
||||||
|
"info": { "type": "integer", "minimum": 0 }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"coverage": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["worklist-size", "items-evaluated"],
|
||||||
|
"properties": {
|
||||||
|
"worklist-size": { "type": "integer", "minimum": 0 },
|
||||||
|
"items-evaluated": { "type": "integer", "minimum": 0 }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"summary": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["counts", "coverage"],
|
||||||
|
"properties": {
|
||||||
|
"counts": { "$ref": "#/definitions/counts" },
|
||||||
|
"coverage": { "$ref": "#/definitions/coverage" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"reference": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["path"],
|
||||||
|
"properties": {
|
||||||
|
"path": { "type": "string", "minLength": 1, "pattern": "^[^\\\\]+$" },
|
||||||
|
"sha": { "type": "string", "pattern": "^[a-fA-F0-9]{40}$" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"location": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["file", "line"],
|
||||||
|
"properties": {
|
||||||
|
"file": { "type": "string", "minLength": 1, "pattern": "^[^\\\\]+$" },
|
||||||
|
"line": { "type": "integer", "minimum": 1 },
|
||||||
|
"range": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["start-line", "end-line"],
|
||||||
|
"properties": {
|
||||||
|
"start-line": { "type": "integer", "minimum": 1 },
|
||||||
|
"end-line": { "type": "integer", "minimum": 1 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"finding": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["id", "severity", "message", "references", "confidence"],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "minLength": 1 },
|
||||||
|
"severity": { "enum": ["blocker", "major", "minor", "info"] },
|
||||||
|
"message": { "type": "string", "minLength": 1 },
|
||||||
|
"location": { "$ref": "#/definitions/location" },
|
||||||
|
"references": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/reference" }
|
||||||
|
},
|
||||||
|
"confidence": { "enum": ["high", "medium", "low"] },
|
||||||
|
"from-sub-skill": { "type": "string", "minLength": 1 },
|
||||||
|
"domain": { "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" },
|
||||||
|
"suggested-code": { "type": "string", "minLength": 1 },
|
||||||
|
"suggested-code-omission-reason": { "type": "string", "minLength": 1 }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"suppressed": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["reference", "reason"],
|
||||||
|
"properties": {
|
||||||
|
"reference": { "$ref": "#/definitions/reference" },
|
||||||
|
"reason": { "enum": ["layer-precedence", "configuration"] }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"skippedSubSkill": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["skill", "reason"],
|
||||||
|
"properties": {
|
||||||
|
"skill": { "$ref": "#/definitions/skillReference" },
|
||||||
|
"reason": { "enum": ["configuration", "not-applicable"] }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
84
schemas/skill-index.schema.json
Normal file
84
schemas/skill-index.schema.json
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"$id": "https://github.com/microsoft/BCQuality/schemas/skill-index.schema.json",
|
||||||
|
"title": "BCQuality action-skill index",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["version", "generatedAt", "skillCount", "sourceSnapshot", "skills"],
|
||||||
|
"properties": {
|
||||||
|
"version": { "const": 1 },
|
||||||
|
"generatedAt": { "type": "string", "format": "date-time" },
|
||||||
|
"skillCount": { "type": "integer", "minimum": 0 },
|
||||||
|
"sourceSnapshot": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
||||||
|
"skills": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/skill" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"definitions": {
|
||||||
|
"stringArray": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string", "minLength": 1 }
|
||||||
|
},
|
||||||
|
"skill": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"path",
|
||||||
|
"layer",
|
||||||
|
"id",
|
||||||
|
"version",
|
||||||
|
"title",
|
||||||
|
"description",
|
||||||
|
"inputs",
|
||||||
|
"outputs",
|
||||||
|
"filters",
|
||||||
|
"subSkills",
|
||||||
|
"sourceSha256"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"path": { "type": "string", "pattern": "^(microsoft|community|custom)/skills/.+\\.md$" },
|
||||||
|
"layer": { "enum": ["microsoft", "community", "custom"] },
|
||||||
|
"id": { "type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" },
|
||||||
|
"version": { "type": "integer", "minimum": 1 },
|
||||||
|
"title": { "type": "string", "minLength": 1 },
|
||||||
|
"description": { "type": "string", "minLength": 1 },
|
||||||
|
"inputs": { "$ref": "#/definitions/stringArray" },
|
||||||
|
"outputs": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 1,
|
||||||
|
"maxItems": 1,
|
||||||
|
"items": { "const": "findings-report" }
|
||||||
|
},
|
||||||
|
"filters": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["bc-version", "technologies", "countries", "application-area"],
|
||||||
|
"properties": {
|
||||||
|
"bc-version": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"oneOf": [
|
||||||
|
{ "type": "integer", "minimum": 1 },
|
||||||
|
{ "type": "string", "pattern": "^(all|[1-9][0-9]*\\.\\.[1-9][0-9]*|[1-9][0-9]*\\.\\.)$" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"technologies": { "$ref": "#/definitions/stringArray" },
|
||||||
|
"countries": { "$ref": "#/definitions/stringArray" },
|
||||||
|
"application-area": { "$ref": "#/definitions/stringArray" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"subSkills": {
|
||||||
|
"type": "array",
|
||||||
|
"uniqueItems": true,
|
||||||
|
"items": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^(microsoft|community|custom)/skills/.+\\.md$"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sourceSha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
12
skills/do.md
12
skills/do.md
|
|
@ -106,6 +106,12 @@ Every action skill MUST contain these five sections, in order:
|
||||||
|
|
||||||
Every action skill emits a single JSON document that conforms to this schema:
|
Every action skill emits a single JSON document that conforms to this schema:
|
||||||
|
|
||||||
|
The machine-readable structural schema is
|
||||||
|
[`schemas/findings-report.schema.json`](../schemas/findings-report.schema.json).
|
||||||
|
The rules below remain authoritative for semantic checks that JSON Schema
|
||||||
|
cannot perform by itself, including summary arithmetic, reference existence,
|
||||||
|
source-scope locations, and article-body retrieval.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"skill": { "id": "string", "version": 1 },
|
"skill": { "id": "string", "version": 1 },
|
||||||
|
|
@ -349,6 +355,12 @@ the declared worklist order, and wait for every invocation to finish before
|
||||||
performing any super-skill self-review or final rollup. Scheduling MUST NOT
|
performing any super-skill self-review or final rollup. Scheduling MUST NOT
|
||||||
change relevance, coverage, failure, reference-integrity, or output semantics.
|
change relevance, coverage, failure, reference-integrity, or output semantics.
|
||||||
|
|
||||||
|
Orchestrators SHOULD generate `skill-index.json` with
|
||||||
|
`tools/Build-SkillIndex.ps1` and consume the super-skill's ordered `subSkills`
|
||||||
|
from that index instead of parsing Markdown. Action-skill frontmatter remains
|
||||||
|
the source of truth; the generated index conforms to
|
||||||
|
`schemas/skill-index.schema.json`.
|
||||||
|
|
||||||
### Section interpretation for super-skills
|
### Section interpretation for super-skills
|
||||||
|
|
||||||
The five required sections still apply. Their meaning shifts from knowledge files to sub-skills:
|
The five required sections still apply. Their meaning shifts from knowledge files to sub-skills:
|
||||||
|
|
|
||||||
247
tools/Build-SkillIndex.ps1
Normal file
247
tools/Build-SkillIndex.ps1
Normal file
|
|
@ -0,0 +1,247 @@
|
||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Builds the machine-readable BCQuality action-skill index.
|
||||||
|
|
||||||
|
.DESCRIPTION
|
||||||
|
Action-skill frontmatter remains the source of truth. This script emits the
|
||||||
|
versioned JSON contract orchestrators consume so they do not need to parse
|
||||||
|
Markdown or duplicate composition rules.
|
||||||
|
|
||||||
|
.PARAMETER BCQualityRoot
|
||||||
|
BCQuality repository or filtered content root.
|
||||||
|
|
||||||
|
.PARAMETER IndexPath
|
||||||
|
Output path. Defaults to <BCQualityRoot>/skill-index.json.
|
||||||
|
|
||||||
|
.OUTPUTS
|
||||||
|
Returns the number of indexed action skills.
|
||||||
|
#>
|
||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[string] $BCQualityRoot,
|
||||||
|
[string] $IndexPath
|
||||||
|
)
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
|
if (-not $BCQualityRoot) {
|
||||||
|
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
|
||||||
|
}
|
||||||
|
if (-not (Test-Path -LiteralPath $BCQualityRoot -PathType Container)) {
|
||||||
|
throw "BCQuality root not found: $BCQualityRoot"
|
||||||
|
}
|
||||||
|
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
|
||||||
|
if (-not $IndexPath) {
|
||||||
|
$IndexPath = Join-Path $BCQualityRoot 'skill-index.json'
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-RelativePath {
|
||||||
|
param([string] $Root, [string] $Full)
|
||||||
|
|
||||||
|
return ($Full.Substring($Root.Length).TrimStart([char]'/', [char]'\') -replace '\\', '/')
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-Sha256 {
|
||||||
|
param([byte[]] $Bytes)
|
||||||
|
|
||||||
|
$sha = [Security.Cryptography.SHA256]::Create()
|
||||||
|
try {
|
||||||
|
return ([BitConverter]::ToString($sha.ComputeHash($Bytes)) -replace '-', '').ToLowerInvariant()
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
$sha.Dispose()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-ValueSha256 {
|
||||||
|
param([Parameter(Mandatory)] $Value)
|
||||||
|
|
||||||
|
return Get-Sha256 -Bytes ([Text.Encoding]::UTF8.GetBytes(
|
||||||
|
(ConvertTo-Json -InputObject $Value -Depth 12 -Compress)
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
function ConvertFrom-SkillFrontmatter {
|
||||||
|
param(
|
||||||
|
[string] $Path,
|
||||||
|
[string] $Text
|
||||||
|
)
|
||||||
|
|
||||||
|
$lines = [regex]::Split($Text.TrimStart([char]0xfeff), '\r\n|\n|\r')
|
||||||
|
if ($lines.Count -lt 3 -or $lines[0].Trim() -ne '---') {
|
||||||
|
throw [IO.InvalidDataException]::new("Missing frontmatter in '$Path'.")
|
||||||
|
}
|
||||||
|
|
||||||
|
$end = -1
|
||||||
|
for ($i = 1; $i -lt $lines.Count; $i++) {
|
||||||
|
if ($lines[$i].Trim() -eq '---') {
|
||||||
|
$end = $i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($end -lt 0) {
|
||||||
|
throw [IO.InvalidDataException]::new("Unterminated frontmatter in '$Path'.")
|
||||||
|
}
|
||||||
|
|
||||||
|
$frontmatter = [ordered]@{}
|
||||||
|
for ($i = 1; $i -lt $end; $i++) {
|
||||||
|
$line = $lines[$i]
|
||||||
|
if ($line -notmatch '^([a-zA-Z][\w-]*)\s*:\s*(.*)$') {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
$key = $Matches[1]
|
||||||
|
$value = $Matches[2].Trim()
|
||||||
|
if ($value -eq '') {
|
||||||
|
$items = [System.Collections.Generic.List[string]]::new()
|
||||||
|
while ($i + 1 -lt $end -and $lines[$i + 1] -match '^\s+-\s+(.+?)\s*$') {
|
||||||
|
$i++
|
||||||
|
$items.Add($Matches[1].Trim().Trim('"', "'")) | Out-Null
|
||||||
|
}
|
||||||
|
$frontmatter[$key] = @($items)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($value -match '^\[(.*)\]$') {
|
||||||
|
$inner = $Matches[1].Trim()
|
||||||
|
$values = [System.Collections.Generic.List[object]]::new()
|
||||||
|
if ($inner) {
|
||||||
|
foreach ($item in $inner -split '\s*,\s*') {
|
||||||
|
$normalized = $item.Trim().Trim('"', "'")
|
||||||
|
$number = 0
|
||||||
|
if ($key -eq 'bc-version' -and [int]::TryParse($normalized, [ref]$number)) {
|
||||||
|
$values.Add($number) | Out-Null
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
$values.Add($normalized) | Out-Null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$frontmatter[$key] = [object[]]@($values)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
$frontmatter[$key] = $value.Trim('"', "'")
|
||||||
|
}
|
||||||
|
|
||||||
|
return $frontmatter
|
||||||
|
}
|
||||||
|
|
||||||
|
$records = [System.Collections.Generic.List[object]]::new()
|
||||||
|
$recordsByPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||||
|
$sourceManifest = [System.Collections.Generic.List[object]]::new()
|
||||||
|
|
||||||
|
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||||
|
$skillsRoot = Join-Path $BCQualityRoot (Join-Path $layer 'skills')
|
||||||
|
if (-not (Test-Path -LiteralPath $skillsRoot -PathType Container)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($file in Get-ChildItem -LiteralPath $skillsRoot -Recurse -File -Filter '*.md' | Sort-Object FullName) {
|
||||||
|
$bytes = [IO.File]::ReadAllBytes($file.FullName)
|
||||||
|
try {
|
||||||
|
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||||
|
}
|
||||||
|
catch [Text.DecoderFallbackException] {
|
||||||
|
throw [IO.InvalidDataException]::new("Invalid UTF-8 in '$($file.FullName)'.", $_.Exception)
|
||||||
|
}
|
||||||
|
|
||||||
|
$frontmatter = ConvertFrom-SkillFrontmatter -Path $file.FullName -Text $text
|
||||||
|
if ($frontmatter['kind'] -ne 'action-skill') {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($required in 'id', 'version', 'title', 'description', 'inputs', 'outputs') {
|
||||||
|
if (-not $frontmatter.Contains($required) -or $null -eq $frontmatter[$required] -or
|
||||||
|
([string]$frontmatter[$required]).Trim() -eq '') {
|
||||||
|
throw [IO.InvalidDataException]::new(
|
||||||
|
"Action skill '$($file.FullName)' is missing required frontmatter '$required'."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$path = Get-RelativePath -Root $BCQualityRoot -Full $file.FullName
|
||||||
|
$sourceSha256 = Get-Sha256 -Bytes $bytes
|
||||||
|
$version = 0
|
||||||
|
if (-not [int]::TryParse([string]$frontmatter['version'], [ref]$version) -or $version -le 0) {
|
||||||
|
throw [IO.InvalidDataException]::new("Action skill '$path' has an invalid version.")
|
||||||
|
}
|
||||||
|
|
||||||
|
$subSkills = @()
|
||||||
|
if ($frontmatter.Contains('sub-skills')) {
|
||||||
|
$subSkills = @($frontmatter['sub-skills'])
|
||||||
|
if (-not $subSkills.Count) {
|
||||||
|
throw [IO.InvalidDataException]::new("Super-skill '$path' has an empty sub-skills list.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$record = [pscustomobject][ordered]@{
|
||||||
|
path = $path
|
||||||
|
layer = $layer
|
||||||
|
id = [string]$frontmatter['id']
|
||||||
|
version = $version
|
||||||
|
title = [string]$frontmatter['title']
|
||||||
|
description = [string]$frontmatter['description']
|
||||||
|
inputs = [string[]]@($frontmatter['inputs'])
|
||||||
|
outputs = [string[]]@($frontmatter['outputs'])
|
||||||
|
filters = [ordered]@{
|
||||||
|
'bc-version' = [object[]]$(if ($frontmatter.Contains('bc-version')) { $frontmatter['bc-version'] })
|
||||||
|
technologies = [string[]]$(if ($frontmatter.Contains('technologies')) { $frontmatter['technologies'] })
|
||||||
|
countries = [string[]]$(if ($frontmatter.Contains('countries')) { $frontmatter['countries'] })
|
||||||
|
'application-area' = [string[]]$(if ($frontmatter.Contains('application-area')) { $frontmatter['application-area'] })
|
||||||
|
}
|
||||||
|
subSkills = [string[]]$subSkills
|
||||||
|
sourceSha256 = $sourceSha256
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not $recordsByPath.TryAdd($path, $record)) {
|
||||||
|
throw "Duplicate action-skill path: $path"
|
||||||
|
}
|
||||||
|
$records.Add($record) | Out-Null
|
||||||
|
$sourceManifest.Add([ordered]@{ path = $path; sha256 = $sourceSha256 }) | Out-Null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$ids = @($records | Group-Object id | Where-Object Count -gt 1)
|
||||||
|
if ($ids.Count) {
|
||||||
|
throw "Duplicate action-skill IDs: $($ids.Name -join ', ')"
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($record in $records) {
|
||||||
|
$seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||||
|
foreach ($subSkillPath in @($record.subSkills)) {
|
||||||
|
if (-not $seen.Add($subSkillPath)) {
|
||||||
|
throw "Super-skill '$($record.path)' declares duplicate sub-skill '$subSkillPath'."
|
||||||
|
}
|
||||||
|
if (-not $recordsByPath.ContainsKey($subSkillPath)) {
|
||||||
|
throw "Super-skill '$($record.path)' references missing action skill '$subSkillPath'."
|
||||||
|
}
|
||||||
|
|
||||||
|
$leaf = $recordsByPath[$subSkillPath]
|
||||||
|
if (@($leaf.subSkills).Count) {
|
||||||
|
throw "Nested super-skills are not supported: '$($record.path)' references '$subSkillPath'."
|
||||||
|
}
|
||||||
|
if (@($leaf.outputs).Count -ne 1 -or $leaf.outputs[0] -ne 'findings-report') {
|
||||||
|
throw "Sub-skill '$subSkillPath' must produce findings-report."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$index = [ordered]@{
|
||||||
|
version = 1
|
||||||
|
generatedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||||
|
skillCount = $records.Count
|
||||||
|
sourceSnapshot = Get-ValueSha256 -Value @($sourceManifest)
|
||||||
|
skills = @($records)
|
||||||
|
}
|
||||||
|
|
||||||
|
$parent = Split-Path -Parent $IndexPath
|
||||||
|
if ($parent -and -not (Test-Path -LiteralPath $parent)) {
|
||||||
|
New-Item -ItemType Directory -Path $parent -Force | Out-Null
|
||||||
|
}
|
||||||
|
Set-Content -LiteralPath $IndexPath -Value (
|
||||||
|
ConvertTo-Json -InputObject $index -Depth 12 -Compress
|
||||||
|
) -Encoding utf8NoBOM
|
||||||
|
|
||||||
|
return $records.Count
|
||||||
Loading…
Add table
Add a link
Reference in a new issue