mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Close AL development contract gaps
Bound post-implementation review rounds, expose output kinds in Entry dispatch, enforce capability coverage, map BCFIX-HANDOFF v1, clarify no-knowledge behavior, and reject repository-escaping skill paths. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 638b66d2-9f06-4f60-8781-808709e1485c
This commit is contained in:
Jesper Schulz-Wedde
parent
56b80e6dcf
commit
f6fca1d56d
15 changed files with 356 additions and 52 deletions
|
|
@ -52,6 +52,22 @@ if ($manifest.version -ne 1) {
|
|||
if ($capabilityManifest.version -ne 1) {
|
||||
$problems.Add("Unsupported capability manifest version: $($capabilityManifest.version)") | Out-Null
|
||||
}
|
||||
$minimumFixtureCoverage = if ($capabilityManifest.PSObject.Properties.Name -contains 'minimumFixtureCoverage') {
|
||||
[double]$capabilityManifest.minimumFixtureCoverage
|
||||
} else {
|
||||
-1
|
||||
}
|
||||
if ($minimumFixtureCoverage -lt 0 -or $minimumFixtureCoverage -gt 1) {
|
||||
$problems.Add("minimumFixtureCoverage must be between 0 and 1.") | Out-Null
|
||||
}
|
||||
$maximumReviewRounds = if ($manifest.PSObject.Properties.Name -contains 'maximumReviewRounds') {
|
||||
[int]$manifest.maximumReviewRounds
|
||||
} else {
|
||||
0
|
||||
}
|
||||
if ($maximumReviewRounds -le 0) {
|
||||
$problems.Add("maximumReviewRounds must be a positive integer.") | Out-Null
|
||||
}
|
||||
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||
$threshold = [double]$manifest.$thresholdName
|
||||
if ($threshold -lt 0 -or $threshold -gt 1) {
|
||||
|
|
@ -62,6 +78,12 @@ foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecisio
|
|||
$skillPath = [string]$manifest.skill
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
|
||||
$problems.Add("Development skill does not exist: $skillPath") | Out-Null
|
||||
} else {
|
||||
$skillText = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
||||
$limitMatch = [regex]::Match($skillText, '(?m)^quality-round-limit:\s*(\d+)\s*$')
|
||||
if (-not $limitMatch.Success -or [int]$limitMatch.Groups[1].Value -ne $maximumReviewRounds) {
|
||||
$problems.Add("maximumReviewRounds must match the development skill quality-round-limit.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$validChecks = @('compile', 'tests', 'review')
|
||||
|
|
@ -173,6 +195,16 @@ foreach ($case in @($manifest.cases)) {
|
|||
}
|
||||
}
|
||||
|
||||
$fixtureBackedCount = @(
|
||||
$capabilityManifest.capabilities |
|
||||
Where-Object status -in @('fixture', 'validated')
|
||||
).Count
|
||||
$capabilityCount = @($capabilityManifest.capabilities).Count
|
||||
$fixtureCoverage = if ($capabilityCount) { $fixtureBackedCount / $capabilityCount } else { 0.0 }
|
||||
if ($fixtureCoverage -lt $minimumFixtureCoverage) {
|
||||
$problems.Add("Fixture-backed capability coverage $fixtureCoverage is below minimumFixtureCoverage $minimumFixtureCoverage.") | Out-Null
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Development fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
|
|
@ -269,6 +301,11 @@ if ($PrepareDirectory) {
|
|||
findings = @()
|
||||
suppressed = @()
|
||||
}
|
||||
'review-rounds' = @([ordered]@{
|
||||
round = 1
|
||||
outcome = 'clean | fixing | stalled | limit-reached'
|
||||
'gating-finding-ids' = @()
|
||||
})
|
||||
suppressed = @()
|
||||
remaining = @()
|
||||
}
|
||||
|
|
@ -303,7 +340,7 @@ if ($ResultsDirectory) {
|
|||
continue
|
||||
}
|
||||
$report = $result.implementationReport
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'plan', 'knowledge', 'changes', 'validation', 'review', 'suppressed', 'remaining')) {
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'plan', 'knowledge', 'changes', 'validation', 'review', 'review-rounds', 'suppressed', 'remaining')) {
|
||||
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
|
||||
$failures.Add("$($case.id): implementation report is missing '$requiredField'.") | Out-Null
|
||||
}
|
||||
|
|
@ -492,6 +529,41 @@ if ($ResultsDirectory) {
|
|||
$failures.Add("$($case.id): post-implementation review has $($gatingFindings.Count) gating finding(s).") | Out-Null
|
||||
}
|
||||
}
|
||||
[object[]]$reviewRounds = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'review-rounds') {
|
||||
$reviewRounds = @($report.'review-rounds')
|
||||
}
|
||||
if (-not $reviewRounds.Count -or $reviewRounds.Count -gt $maximumReviewRounds) {
|
||||
$failures.Add("$($case.id): review-rounds must contain 1..$maximumReviewRounds entries.") | Out-Null
|
||||
} else {
|
||||
for ($index = 0; $index -lt $reviewRounds.Count; $index++) {
|
||||
$round = $reviewRounds[$index]
|
||||
$roundNumber = if ($round.PSObject.Properties.Name -contains 'round') { [int]$round.round } else { 0 }
|
||||
$roundOutcome = if ($round.PSObject.Properties.Name -contains 'outcome') { [string]$round.outcome } else { '' }
|
||||
if ($roundNumber -ne ($index + 1)) {
|
||||
$failures.Add("$($case.id): review round numbering is not contiguous.") | Out-Null
|
||||
}
|
||||
if ($roundOutcome -notin @('clean', 'fixing', 'stalled', 'limit-reached')) {
|
||||
$failures.Add("$($case.id): review round $roundNumber has invalid outcome '$roundOutcome'.") | Out-Null
|
||||
}
|
||||
[object[]]$roundGatingIds = @()
|
||||
if ($round.PSObject.Properties.Name -contains 'gating-finding-ids') {
|
||||
$roundGatingIds = @($round.'gating-finding-ids')
|
||||
}
|
||||
if ($roundOutcome -eq 'clean' -and $roundGatingIds.Count) {
|
||||
$failures.Add("$($case.id): clean review round $roundNumber must have no gating IDs.") | Out-Null
|
||||
}
|
||||
if ($roundOutcome -in @('fixing', 'stalled', 'limit-reached') -and -not $roundGatingIds.Count) {
|
||||
$failures.Add("$($case.id): review round $roundNumber outcome '$roundOutcome' requires gating IDs.") | Out-Null
|
||||
}
|
||||
if (@($roundGatingIds | Sort-Object -Unique).Count -ne $roundGatingIds.Count) {
|
||||
$failures.Add("$($case.id): review round $roundNumber has duplicate gating IDs.") | Out-Null
|
||||
}
|
||||
}
|
||||
if ([string]$reviewRounds[-1].outcome -ne 'clean') {
|
||||
$failures.Add("$($case.id): completed implementation must end with a clean review round.") | Out-Null
|
||||
}
|
||||
}
|
||||
[object[]]$remaining = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'remaining') {
|
||||
$remaining = @($report.remaining)
|
||||
|
|
@ -508,9 +580,5 @@ if ($ResultsDirectory) {
|
|||
}
|
||||
Write-Host "Development fixture scoring PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
} else {
|
||||
$fixtureBackedCapabilities = @(
|
||||
$capabilityManifest.capabilities |
|
||||
Where-Object status -in @('fixture', 'validated')
|
||||
).Count
|
||||
Write-Host "Development fixture validation PASSED: $(@($manifest.cases).Count) cases; $fixtureBackedCapabilities of $($capabilityIds.Count) capabilities have fixtures."
|
||||
Write-Host "Development fixture validation PASSED: $(@($manifest.cases).Count) cases; $fixtureBackedCount of $($capabilityIds.Count) capabilities have fixtures (minimum $minimumFixtureCoverage)."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,22 @@ function Get-ModelCaseId {
|
|||
}
|
||||
}
|
||||
|
||||
function Get-PlanRequest {
|
||||
param([object] $Plan)
|
||||
|
||||
if ($Plan.PSObject.Properties.Name -contains 'request' -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Plan.request)) {
|
||||
return [string]$Plan.request
|
||||
}
|
||||
if ($Plan.PSObject.Properties.Name -contains 'format' -and
|
||||
[string]$Plan.format -eq 'BCFIX-HANDOFF' -and
|
||||
$Plan.PSObject.Properties.Name -contains 'nextStep') {
|
||||
$issue = if ($Plan.PSObject.Properties.Name -contains 'issue') { [string]$Plan.issue } else { 'unknown' }
|
||||
return "Continue BCFIX issue #${issue}: $($Plan.nextStep)"
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
if ($manifest.version -ne 1) {
|
||||
$problems.Add("Unsupported guidance fixture version: $($manifest.version)") | Out-Null
|
||||
}
|
||||
|
|
@ -64,11 +80,49 @@ foreach ($case in @($manifest.cases)) {
|
|||
continue
|
||||
}
|
||||
$plan = $case.'development-plan'
|
||||
if ($validKinds -notcontains [string]$plan.kind) {
|
||||
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
|
||||
[string]$case.expectedKind
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($validKinds -notcontains $expectedKind) {
|
||||
$problems.Add("${id}: expectedKind is invalid.") | Out-Null
|
||||
}
|
||||
$isBcfixHandoff = (
|
||||
$plan.PSObject.Properties.Name -contains 'format' -and
|
||||
[string]$plan.format -eq 'BCFIX-HANDOFF'
|
||||
)
|
||||
if ($isBcfixHandoff) {
|
||||
$requiredHandoffFields = @(
|
||||
'version', 'issue', 'phase', 'status', 'baton', 'rootCause',
|
||||
'harnessMap', 'iterationsUsed', 'filesCommitted', 'lastTestResult',
|
||||
'deadEnds', 'nextStep'
|
||||
)
|
||||
foreach ($field in $requiredHandoffFields) {
|
||||
if ($plan.PSObject.Properties.Name -notcontains $field) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF is missing '$field'.") | Out-Null
|
||||
}
|
||||
}
|
||||
$handoffVersion = if ($plan.PSObject.Properties.Name -contains 'version') { [int]$plan.version } else { 0 }
|
||||
$handoffPhase = if ($plan.PSObject.Properties.Name -contains 'phase') { [string]$plan.phase } else { '' }
|
||||
$handoffStatus = if ($plan.PSObject.Properties.Name -contains 'status') { [string]$plan.status } else { '' }
|
||||
if ($handoffVersion -ne 1) {
|
||||
$problems.Add("${id}: only BCFIX-HANDOFF version 1 is supported.") | Out-Null
|
||||
}
|
||||
if ($handoffPhase -notin @('plan', 'baseline', 'implement', 'pr')) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF phase is invalid.") | Out-Null
|
||||
}
|
||||
if ($handoffStatus -notin @('in-progress', 'paused', 'done')) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF status is invalid.") | Out-Null
|
||||
}
|
||||
} elseif (
|
||||
$plan.PSObject.Properties.Name -notcontains 'kind' -or
|
||||
$validKinds -notcontains [string]$plan.kind
|
||||
) {
|
||||
$problems.Add("${id}: development-plan.kind is invalid.") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace([string]$plan.request)) {
|
||||
$problems.Add("${id}: development-plan.request is required.") | Out-Null
|
||||
if ([string]::IsNullOrWhiteSpace((Get-PlanRequest $plan))) {
|
||||
$problems.Add("${id}: development-plan must provide request or BCFIX nextStep.") | Out-Null
|
||||
}
|
||||
|
||||
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||
|
|
@ -125,7 +179,7 @@ if ($PrepareDirectory) {
|
|||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = 'knowledge-index.json'
|
||||
'task-context' = [ordered]@{
|
||||
goal = [string]$case.'development-plan'.request
|
||||
goal = Get-PlanRequest $case.'development-plan'
|
||||
'inputs-available' = @('development-plan', 'repository')
|
||||
technologies = @($case.context.technologies)
|
||||
countries = @($case.context.countries)
|
||||
|
|
@ -216,7 +270,7 @@ if ($ResultsDirectory) {
|
|||
$report.PSObject.Properties.Name -contains 'summary' -and
|
||||
$report.summary.PSObject.Properties.Name -contains 'kind'
|
||||
) { [string]$report.summary.kind } else { '' }
|
||||
if ($kind -ne [string]$case.'development-plan'.kind) {
|
||||
if ($kind -ne [string]$case.expectedKind) {
|
||||
$failures.Add("$($case.id): summary.kind '$kind' does not match the plan.") | Out-Null
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue