mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Strengthen review contracts and add AL reliability guidance (#196)
* Strengthen review contracts and HTTP guidance - add outbound HttpClient transport and HTTP status review rules with paired fixtures`n- resolve layered action-skill overrides deterministically across enabled layers`n- validate findings reports and enforce measurable changed-fixture coverage * Add data handling and test isolation guidance - add SCM guidance for deriving base quantities through line unit-of-measure validation`n- add security guidance for parameterizing SetFilter with external text`n- add test isolation guidance for resetting per-test state before initialization guards`n- add web-service guidance for JSON null handling and invariant standard format 9`n- route and cover all five rules with paired evaluation fixtures * Fix findings report rollup validation * Validate findings report rollups * Enforce merged finding identity * Fix locationless finding deduplication * Reject conflicting merged corrections * Detect conflicting leaf corrections * Route HTTP error checks to canonical web-services knowledge Let the Error Handling leaf conditionally retrieve the existing HTTP owner articles, preserving applicability and exact-path provenance. Add deterministic source-contract and retrieval regressions without duplicating knowledge rules. Copilot-Session-Id: a92a7788-103e-4651-9b84-19e34caffb94 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: wenjiefan <wenjiefan@microsoft.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Jesper Schulz-Wedde <jesper.schulzwedde@microsoft.com>
This commit is contained in:
Stefano Demiliani
GitHub
parent
130d5de6c4
commit
4287233f80
41 changed files with 1974 additions and 38 deletions
79
.github/scripts/Test-SkillIndex.ps1
vendored
79
.github/scripts/Test-SkillIndex.ps1
vendored
|
|
@ -30,9 +30,10 @@ function Assert-ThrowsLike {
|
|||
}
|
||||
|
||||
$generator = Join-Path $Root 'tools/Build-SkillIndex.ps1'
|
||||
$resolver = Join-Path $Root 'tools/Resolve-SkillWorklist.ps1'
|
||||
$indexSchema = Join-Path $Root 'schemas/skill-index.schema.json'
|
||||
$reportSchema = Join-Path $Root 'schemas/findings-report.schema.json'
|
||||
foreach ($path in $generator, $indexSchema, $reportSchema) {
|
||||
foreach ($path in $generator, $resolver, $indexSchema, $reportSchema) {
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
||||
throw "Required contract file not found: $path"
|
||||
}
|
||||
|
|
@ -235,9 +236,83 @@ Output.
|
|||
Assert-ThrowsLike -Pattern '*Nested super-skills are not supported*' -Action {
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'nested.json')
|
||||
}
|
||||
|
||||
Remove-Item -LiteralPath (Join-Path $fixtureSkills 'al-nested-review.md') -Force
|
||||
$validSuper = @'
|
||||
---
|
||||
kind: action-skill
|
||||
id: al-code-review
|
||||
version: 1
|
||||
title: Review
|
||||
description: Test super-skill.
|
||||
inputs: [file-path]
|
||||
outputs: [findings-report]
|
||||
sub-skills:
|
||||
- microsoft/skills/review/al-leaf-review.md
|
||||
---
|
||||
|
||||
# Review
|
||||
|
||||
## Source
|
||||
Source.
|
||||
## Relevance
|
||||
Relevance.
|
||||
## Worklist
|
||||
Worklist.
|
||||
## Action
|
||||
Action.
|
||||
## Output
|
||||
Output.
|
||||
'@
|
||||
Set-Content -LiteralPath $superPath -Value $validSuper -Encoding utf8NoBOM
|
||||
|
||||
$customSkills = Join-Path -Path $fixtureRoot -ChildPath 'custom/skills/review'
|
||||
New-Item -ItemType Directory -Path $customSkills -Force | Out-Null
|
||||
$customLeaf = $leaf.Replace('title: Leaf', 'title: Custom Leaf')
|
||||
Set-Content -LiteralPath (Join-Path $customSkills 'custom-leaf-review.md') -Value $customLeaf -Encoding utf8NoBOM
|
||||
|
||||
$layeredPath = Join-Path $tmp 'layered.json'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $layeredPath | Out-Null
|
||||
$layeredIndex = Get-Content -LiteralPath $layeredPath -Raw | ConvertFrom-Json
|
||||
$layeredLeaves = @($layeredIndex.skills | Where-Object id -eq 'al-leaf-review')
|
||||
if ($layeredLeaves.Count -ne 2) {
|
||||
throw "Expected both layered al-leaf-review implementations, found $($layeredLeaves.Count)."
|
||||
}
|
||||
if ((@($layeredLeaves.layer | Sort-Object) -join ',') -cne 'custom,microsoft') {
|
||||
throw 'Layered al-leaf-review implementations did not preserve custom and microsoft records.'
|
||||
}
|
||||
|
||||
$resolved = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
|
||||
'microsoft/skills/review/al-code-review.md'
|
||||
)
|
||||
if ($resolved.subSkills.Count -ne 1 -or
|
||||
$resolved.subSkills[0].path -cne 'custom/skills/review/custom-leaf-review.md') {
|
||||
throw 'The custom implementation did not win the layered leaf slot.'
|
||||
}
|
||||
|
||||
$microsoftOnly = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
|
||||
'microsoft/skills/review/al-code-review.md'
|
||||
) -EnabledLayers microsoft
|
||||
if ($microsoftOnly.subSkills.Count -ne 1 -or
|
||||
$microsoftOnly.subSkills[0].path -cne 'microsoft/skills/review/al-leaf-review.md') {
|
||||
throw 'Disabling the custom layer did not fall back to the Microsoft implementation.'
|
||||
}
|
||||
|
||||
$customDisabled = & $resolver -BCQualityRoot $fixtureRoot -IndexPath $layeredPath -SuperSkillPath (
|
||||
'microsoft/skills/review/al-code-review.md'
|
||||
) -DisabledSkills 'custom/skills/review/custom-leaf-review.md'
|
||||
if ($customDisabled.subSkills.Count -ne 1 -or
|
||||
$customDisabled.subSkills[0].path -cne 'microsoft/skills/review/al-leaf-review.md') {
|
||||
throw 'Disabling the custom implementation did not fall back to Microsoft.'
|
||||
}
|
||||
|
||||
Set-Content -LiteralPath (Join-Path $customSkills 'duplicate-leaf-review.md') -Value $customLeaf -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*Duplicate action-skill IDs within a layer: custom:al-leaf-review*' -Action {
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath (Join-Path $tmp 'duplicate-layer.json')
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
Write-Output "Skill-index check PASSED: deterministic, schema-valid, and all $($expectedLeaves.Count) review leaves preserved in order."
|
||||
Write-Output "Skill-index check PASSED: deterministic, schema-valid, layered overrides resolved, and all $($expectedLeaves.Count) review leaves preserved in order."
|
||||
|
|
|
|||
8
.github/scripts/validate_frontmatter.py
vendored
8
.github/scripts/validate_frontmatter.py
vendored
|
|
@ -688,12 +688,16 @@ def run(root: Path) -> Report:
|
|||
if domain_dir.is_dir():
|
||||
validate_samples_in_domain(domain_dir, root, report)
|
||||
|
||||
# Third pass: R24 unique ids within kind
|
||||
# Third pass: R24 unique ids within kind. Layered action-skill overrides
|
||||
# may share an id, but two definitions in one layer are ambiguous.
|
||||
by_kind: dict[str, dict[str, list[Path]]] = {}
|
||||
for rec in skill_records:
|
||||
if rec.skill_id is None:
|
||||
continue
|
||||
by_kind.setdefault(rec.kind, {}).setdefault(rec.skill_id, []).append(rec.path)
|
||||
scope = rec.kind
|
||||
if rec.kind == "action-skill":
|
||||
scope = f"{rec.kind}:{rec.path.relative_to(root).parts[0]}"
|
||||
by_kind.setdefault(scope, {}).setdefault(rec.skill_id, []).append(rec.path)
|
||||
for kind, by_id in by_kind.items():
|
||||
for sid, paths in by_id.items():
|
||||
if len(paths) > 1:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue