mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-07 07:36:54 +01:00
Narrow AL development to read-only plan guidance
Retain shared knowledge enrichment and review guidance; defer standalone implementation and source-ingestion tracking. Add runner-owned baseline evidence, contract regressions, and explicit consumer/pilot boundaries. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Jesper Schulz-Wedde
parent
f6fca1d56d
commit
1deac52a53
26 changed files with 1486 additions and 11037 deletions
|
|
@ -1,352 +1,215 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates, prepares, and scores read-only AL development-guidance fixtures.
|
||||
.DESCRIPTION
|
||||
Capture runner-owned evidence BEFORE invoking an agent:
|
||||
-CaptureBaseline -WorkspaceMapPath <json> -BaselinePath <new-json>
|
||||
The workspace map is {"manifest-case-id":"absolute-standalone-git-root",...}.
|
||||
Score AFTER the agent finishes:
|
||||
-BaselinePath <json> -BaselineSha256 <runner-retained-digest> -ResultsDirectory <directory>
|
||||
Preparation (-PrepareDirectory) is independent; supply -WorkspaceMapPath to
|
||||
include runner-selected target paths. Never derive target paths from results.
|
||||
|
||||
Baseline, map, prepared requests, and results must be outside all targets and
|
||||
the knowledge checkout. Keep the baseline runner-only; retain the printed
|
||||
SHA256 and pass -BaselineSha256 when scoring to detect baseline tampering.
|
||||
Capture never overwrites an existing baseline. The runner must protect this
|
||||
script, the baseline/digest and its invocation from the agent.
|
||||
|
||||
This compares before/after evidence, NOT an OS sandbox or a write monitor.
|
||||
It cannot detect reverted transient writes, prove that articles were opened,
|
||||
or validate semantic faithfulness of prose. Files, directories, hashes,
|
||||
stable metadata, Git HEAD/refs/index and ignored/untracked files are compared.
|
||||
Links/reparse points, hard links, alternate data streams, external Git
|
||||
storage in targets, submodules and sparse checkouts are rejected rather than
|
||||
followed. Run in quiescent repositories. The knowledge checkout may itself
|
||||
be a linked Git worktree; its Git storage identity is recorded explicitly.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
[string] $Root = (Join-Path $PSScriptRoot '..'),
|
||||
[string] $ManifestPath,
|
||||
[string] $PrepareDirectory,
|
||||
[string] $ResultsDirectory
|
||||
[string] $ResultsDirectory,
|
||||
[switch] $CaptureBaseline,
|
||||
[string] $BaselinePath,
|
||||
[string] $BaselineSha256,
|
||||
[string] $WorkspaceMapPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $ManifestPath) {
|
||||
$ManifestPath = Join-Path $Root 'evaluation/development-guidance-fixtures.json'
|
||||
}
|
||||
|
||||
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
|
||||
$problems = [System.Collections.Generic.List[string]]::new()
|
||||
$validKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
||||
|
||||
function Get-ModelCaseId {
|
||||
param([string] $ManifestId)
|
||||
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
|
||||
$hash = $sha.ComputeHash($bytes)
|
||||
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
|
||||
return "case-$token"
|
||||
} finally {
|
||||
$sha.Dispose()
|
||||
try {
|
||||
if ($CaptureBaseline -and ($ResultsDirectory -or $PrepareDirectory)) {
|
||||
throw 'CaptureBaseline is a separate pre-run operation.'
|
||||
}
|
||||
}
|
||||
|
||||
function Get-PlanRequest {
|
||||
param([object] $Plan)
|
||||
|
||||
if ($Plan.PSObject.Properties.Name -contains 'request' -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Plan.request)) {
|
||||
return [string]$Plan.request
|
||||
if (($CaptureBaseline -or $ResultsDirectory) -and -not $BaselinePath) {
|
||||
throw 'Capture and scoring require BaselinePath.'
|
||||
}
|
||||
if ($Plan.PSObject.Properties.Name -contains 'format' -and
|
||||
[string]$Plan.format -eq 'BCFIX-HANDOFF' -and
|
||||
$Plan.PSObject.Properties.Name -contains 'nextStep') {
|
||||
$issue = if ($Plan.PSObject.Properties.Name -contains 'issue') { [string]$Plan.issue } else { 'unknown' }
|
||||
return "Continue BCFIX issue #${issue}: $($Plan.nextStep)"
|
||||
if ($ResultsDirectory -and -not $BaselineSha256) {
|
||||
throw 'Scoring requires the runner-retained pre-run BaselineSha256.'
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
if ($manifest.version -ne 1) {
|
||||
$problems.Add("Unsupported guidance fixture version: $($manifest.version)") | Out-Null
|
||||
}
|
||||
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||
$threshold = [double]$manifest.$thresholdName
|
||||
if ($threshold -lt 0 -or $threshold -gt 1) {
|
||||
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
|
||||
if ($CaptureBaseline -and -not $WorkspaceMapPath) {
|
||||
throw 'Capture requires a runner-owned WorkspaceMapPath.'
|
||||
}
|
||||
}
|
||||
|
||||
$skillPath = [string]$manifest.skill
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
|
||||
$problems.Add("Guidance skill does not exist: $skillPath") | Out-Null
|
||||
}
|
||||
|
||||
$seenIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$id = [string]$case.id
|
||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
|
||||
} elseif (-not $seenIds.Add($id)) {
|
||||
$problems.Add("Duplicate fixture id: $id") | Out-Null
|
||||
if ($ResultsDirectory -and ($WorkspaceMapPath -or $PrepareDirectory)) {
|
||||
throw 'Scoring uses only the recorded workspace map; run preparation separately.'
|
||||
}
|
||||
if ($case.PSObject.Properties.Name -notcontains 'development-plan') {
|
||||
$problems.Add("${id}: development-plan is required.") | Out-Null
|
||||
continue
|
||||
$Root = Get-GuidanceSafePath $Root -Directory
|
||||
if (-not $ManifestPath) { $ManifestPath = Join-Path $Root 'evaluation\development-guidance-fixtures.json' }
|
||||
$ManifestPath = Get-GuidanceSafePath $ManifestPath -File
|
||||
$manifest = Read-GuidanceJson $ManifestPath
|
||||
Assert-GuidanceManifest $manifest $Root
|
||||
$caseIds = @($manifest.cases | ForEach-Object { $_.id })
|
||||
$workspaces = [ordered]@{}
|
||||
$baseline = $null
|
||||
if ($WorkspaceMapPath) {
|
||||
$WorkspaceMapPath = Get-GuidanceSafePath $WorkspaceMapPath -File
|
||||
$map = Read-GuidanceJson $WorkspaceMapPath
|
||||
Assert-GuidanceObject $map 'workspace map'
|
||||
if ($map.Count -ne $caseIds.Count) { throw 'Workspace map must bind exactly every manifest case.' }
|
||||
foreach ($id in $caseIds) {
|
||||
Assert-GuidanceString $map[$id] 'workspace map value'
|
||||
if (-not [IO.Path]::IsPathFullyQualified($map[$id])) { throw 'Workspace roots must be absolute.' }
|
||||
$workspaces[$id] = Get-GuidanceSafePath $map[$id] -Directory
|
||||
}
|
||||
}
|
||||
$plan = $case.'development-plan'
|
||||
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
|
||||
[string]$case.expectedKind
|
||||
} else {
|
||||
''
|
||||
if ($ResultsDirectory) {
|
||||
$BaselinePath = Get-GuidanceSafePath $BaselinePath -File
|
||||
if ($BaselineSha256 -cnotmatch '^[0-9A-Fa-f]{64}$') {
|
||||
throw 'BaselineSha256 must be a SHA256 digest.'
|
||||
}
|
||||
if ((Get-GuidanceHash $BaselinePath) -ne $BaselineSha256) {
|
||||
throw 'Runner baseline digest mismatch.'
|
||||
}
|
||||
$baseline = Read-GuidanceJson $BaselinePath
|
||||
Assert-GuidanceObject $baseline 'baseline' @('version', 'kind', 'root', 'manifestPath', 'manifestSha256', 'workspaces', 'rootSnapshot')
|
||||
if ($baseline.version -ne 1 -or $baseline.kind -cne 'bcquality-guidance-runner-baseline') {
|
||||
throw 'Unsupported runner baseline.'
|
||||
}
|
||||
if ($baseline.root -cne $Root -or $baseline.manifestPath -cne $ManifestPath -or
|
||||
$baseline.manifestSha256 -ne (Get-GuidanceHash $ManifestPath)) {
|
||||
throw 'Runner baseline does not match the knowledge root or manifest.'
|
||||
}
|
||||
Assert-GuidanceObject $baseline.workspaces 'baseline workspaces'
|
||||
if ($baseline.workspaces.Count -ne $caseIds.Count) { throw 'Runner baseline case set mismatch.' }
|
||||
foreach ($id in $caseIds) {
|
||||
$entry = $baseline.workspaces[$id]
|
||||
Assert-GuidanceObject $entry 'baseline workspace entry' @('caseId', 'root', 'snapshot')
|
||||
if ($entry.caseId -cne (Get-GuidanceCaseId $id)) { throw 'Runner baseline case identity mismatch.' }
|
||||
$workspaces[$id] = Get-GuidanceSafePath $entry.root -Directory
|
||||
}
|
||||
}
|
||||
if ($validKinds -notcontains $expectedKind) {
|
||||
$problems.Add("${id}: expectedKind is invalid.") | Out-Null
|
||||
}
|
||||
$isBcfixHandoff = (
|
||||
$plan.PSObject.Properties.Name -contains 'format' -and
|
||||
[string]$plan.format -eq 'BCFIX-HANDOFF'
|
||||
)
|
||||
if ($isBcfixHandoff) {
|
||||
$requiredHandoffFields = @(
|
||||
'version', 'issue', 'phase', 'status', 'baton', 'rootCause',
|
||||
'harnessMap', 'iterationsUsed', 'filesCommitted', 'lastTestResult',
|
||||
'deadEnds', 'nextStep'
|
||||
)
|
||||
foreach ($field in $requiredHandoffFields) {
|
||||
if ($plan.PSObject.Properties.Name -notcontains $field) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF is missing '$field'.") | Out-Null
|
||||
$protectedRoots = @($Root) + @($workspaces.Values)
|
||||
for ($i = 0; $i -lt $protectedRoots.Count; $i++) {
|
||||
for ($j = $i + 1; $j -lt $protectedRoots.Count; $j++) {
|
||||
if ((Test-GuidanceWithin $protectedRoots[$i] $protectedRoots[$j]) -or
|
||||
(Test-GuidanceWithin $protectedRoots[$j] $protectedRoots[$i])) {
|
||||
throw 'Knowledge checkout and case workspaces must be distinct, non-overlapping roots.'
|
||||
}
|
||||
}
|
||||
$handoffVersion = if ($plan.PSObject.Properties.Name -contains 'version') { [int]$plan.version } else { 0 }
|
||||
$handoffPhase = if ($plan.PSObject.Properties.Name -contains 'phase') { [string]$plan.phase } else { '' }
|
||||
$handoffStatus = if ($plan.PSObject.Properties.Name -contains 'status') { [string]$plan.status } else { '' }
|
||||
if ($handoffVersion -ne 1) {
|
||||
$problems.Add("${id}: only BCFIX-HANDOFF version 1 is supported.") | Out-Null
|
||||
}
|
||||
if ($handoffPhase -notin @('plan', 'baseline', 'implement', 'pr')) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF phase is invalid.") | Out-Null
|
||||
}
|
||||
if ($handoffStatus -notin @('in-progress', 'paused', 'done')) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF status is invalid.") | Out-Null
|
||||
}
|
||||
} elseif (
|
||||
$plan.PSObject.Properties.Name -notcontains 'kind' -or
|
||||
$validKinds -notcontains [string]$plan.kind
|
||||
) {
|
||||
$problems.Add("${id}: development-plan.kind is invalid.") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace((Get-PlanRequest $plan))) {
|
||||
$problems.Add("${id}: development-plan must provide request or BCFIX nextStep.") | Out-Null
|
||||
}
|
||||
|
||||
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
|
||||
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
|
||||
}
|
||||
foreach ($reference in $expectedKnowledge) {
|
||||
$reference = [string]$reference
|
||||
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
|
||||
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
|
||||
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
|
||||
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
|
||||
foreach ($artifact in @($BaselinePath, $WorkspaceMapPath, $PrepareDirectory, $ResultsDirectory)) {
|
||||
if ($artifact) {
|
||||
$safeArtifact = Get-GuidanceSafePath $artifact -AllowMissing
|
||||
foreach ($protectedRoot in $protectedRoots) {
|
||||
if (Test-GuidanceWithin $safeArtifact $protectedRoot) {
|
||||
throw 'Runner artifacts must be outside target workspaces and the knowledge checkout.'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Development guidance fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
if ($CaptureBaseline) {
|
||||
$BaselinePath = Get-GuidanceSafePath $BaselinePath -AllowMissing
|
||||
if (Test-Path -LiteralPath $BaselinePath) { throw 'Baseline already exists; capture never overwrites evidence.' }
|
||||
$snapshots = [ordered]@{}
|
||||
foreach ($id in $caseIds) {
|
||||
$snapshots[$id] = [ordered]@{
|
||||
caseId = Get-GuidanceCaseId $id
|
||||
root = $workspaces[$id]
|
||||
snapshot = Get-GuidanceSnapshot $workspaces[$id] -Target
|
||||
}
|
||||
}
|
||||
$record = [ordered]@{
|
||||
kind = 'bcquality-guidance-runner-baseline'
|
||||
version = 1
|
||||
root = $Root
|
||||
manifestPath = $ManifestPath
|
||||
manifestSha256 = Get-GuidanceHash $ManifestPath
|
||||
rootSnapshot = Get-GuidanceSnapshot $Root
|
||||
workspaces = $snapshots
|
||||
}
|
||||
Write-GuidanceNewJson $BaselinePath $record
|
||||
Write-Host "Guidance baseline captured. Runner SHA256: $(Get-GuidanceHash $BaselinePath)"
|
||||
} elseif ($PrepareDirectory) {
|
||||
$PrepareDirectory = Get-GuidanceSafePath $PrepareDirectory -AllowMissing
|
||||
if ((Test-Path -LiteralPath $PrepareDirectory) -and
|
||||
@(Get-ChildItem -LiteralPath $PrepareDirectory -Force).Count) {
|
||||
throw 'PrepareDirectory must be new or empty; existing requests/evidence are never overwritten.'
|
||||
}
|
||||
[IO.Directory]::CreateDirectory($PrepareDirectory) | Out-Null
|
||||
# The index builder walks recursively; reject linked corpus paths first.
|
||||
Assert-GuidanceTree $Root
|
||||
& (Join-Path $Root 'tools\Build-KnowledgeIndex.ps1') -BCQualityRoot $Root `
|
||||
-IndexPath (Join-Path $PrepareDirectory 'knowledge-index.json') | Out-Null
|
||||
$skillInstructions = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $manifest.skill))
|
||||
foreach ($case in $manifest.cases) {
|
||||
$modelId = Get-GuidanceCaseId $case.id
|
||||
$request = [ordered]@{
|
||||
protocol = 'Run the supplied read-only skill on the runner-assigned repository and existing plan. Return only caseId and guidanceReport. The runner captures evidence before invocation; do not capture or modify it. Do not create artifacts in the target or knowledge checkout.'
|
||||
caseId = $modelId
|
||||
skill = $manifest.skill
|
||||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||
knowledgeRoot = $Root
|
||||
'task-context' = [ordered]@{
|
||||
goal = Get-GuidancePlanRequest $case.'development-plan'
|
||||
'inputs-available' = @('development-plan', 'repository')
|
||||
'bc-version' = $case.context.'bc-version'
|
||||
technologies = $case.context.technologies
|
||||
countries = $case.context.countries
|
||||
'application-area' = $case.context.'application-area'
|
||||
}
|
||||
'development-plan' = $case.'development-plan'
|
||||
resultSchema = Get-GuidanceResultSchema $modelId
|
||||
}
|
||||
if ($workspaces.Count) { $request.repository = $workspaces[$case.id] }
|
||||
Write-GuidanceNewJson (Join-Path $PrepareDirectory "request-$modelId.json") $request
|
||||
}
|
||||
Write-Host "Development guidance preparation PASSED: $($caseIds.Count) case(s)."
|
||||
} elseif ($ResultsDirectory) {
|
||||
$ResultsDirectory = Get-GuidanceSafePath $ResultsDirectory -Directory
|
||||
$failures = [Collections.Generic.List[string]]::new()
|
||||
if (-not (Test-GuidanceSnapshotEqual $baseline.rootSnapshot (Get-GuidanceSnapshot $Root))) {
|
||||
$failures.Add('Knowledge checkout identity/content changed after baseline capture.')
|
||||
}
|
||||
foreach ($case in $manifest.cases) {
|
||||
$id = $case.id
|
||||
try {
|
||||
if (-not (Test-GuidanceSnapshotEqual $baseline.workspaces[$id].snapshot `
|
||||
(Get-GuidanceSnapshot $workspaces[$id] -Target))) {
|
||||
throw 'Target repository identity/content changed after baseline capture.'
|
||||
}
|
||||
$resultPath = Get-GuidanceSafePath (Join-Path $ResultsDirectory "result-$(Get-GuidanceCaseId $id).json") -File
|
||||
$result = Read-GuidanceJson $resultPath
|
||||
Assert-GuidanceResult $result $case $manifest $Root $workspaces[$id]
|
||||
} catch {
|
||||
# Only evaluator-authored diagnostics are printed, never model or file contents.
|
||||
$failures.Add("${id}: $(Get-GuidanceDiagnostic $_)")
|
||||
}
|
||||
}
|
||||
if ($failures.Count) {
|
||||
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):"
|
||||
$failures | ForEach-Object { Write-Host " - $_" }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Development guidance scoring PASSED: $($caseIds.Count) case(s)."
|
||||
} else {
|
||||
Write-Host "Development guidance fixture validation PASSED: $($caseIds.Count) case(s)."
|
||||
}
|
||||
} catch {
|
||||
Write-Host "Development guidance FAILED: $(Get-GuidanceDiagnostic $_)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ($PrepareDirectory) {
|
||||
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-guidance-evaluation'
|
||||
if (Test-Path -LiteralPath $PrepareDirectory) {
|
||||
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
|
||||
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
|
||||
throw "PrepareDirectory is not empty and is not a BCQuality development-guidance evaluation directory: $PrepareDirectory"
|
||||
}
|
||||
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
|
||||
Get-ChildItem -LiteralPath $PrepareDirectory -File |
|
||||
Where-Object {
|
||||
$_.Name -eq 'knowledge-index.json' -or
|
||||
$_.Name -like 'request-*.json' -or
|
||||
$_.Name -like 'result-*.json'
|
||||
} |
|
||||
Remove-Item -Force
|
||||
}
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
|
||||
}
|
||||
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development-guidance evaluation directory' -Encoding UTF8
|
||||
|
||||
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
||||
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
[ordered]@{
|
||||
protocol = 'Run the supplied AL development-plan skill read-only in a clean fixture repository. Return only guidanceReport using the supplied schema and retain the repository for read-only verification.'
|
||||
caseId = $modelId
|
||||
skill = $skillPath
|
||||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = 'knowledge-index.json'
|
||||
'task-context' = [ordered]@{
|
||||
goal = Get-PlanRequest $case.'development-plan'
|
||||
'inputs-available' = @('development-plan', 'repository')
|
||||
technologies = @($case.context.technologies)
|
||||
countries = @($case.context.countries)
|
||||
'application-area' = @($case.context.'application-area')
|
||||
}
|
||||
'development-plan' = $case.'development-plan'
|
||||
resultSchema = [ordered]@{
|
||||
caseId = $modelId
|
||||
workspaceRoot = 'absolute path to the retained clean fixture repository'
|
||||
guidanceReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
||||
'outcome-reason' = 'required for partial or failed'
|
||||
summary = [ordered]@{
|
||||
request = 'planned change'
|
||||
kind = 'feature | bug | refactor | upgrade | maintenance'
|
||||
candidates = 0
|
||||
selected = 0
|
||||
}
|
||||
context = [ordered]@{
|
||||
'bc-version' = 'resolved target or unknown'
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
unknown = @()
|
||||
}
|
||||
knowledge = @([ordered]@{
|
||||
path = 'repo-relative article path'
|
||||
sha = 'optional commit sha'
|
||||
'used-for' = 'plan decision'
|
||||
constraints = @('faithful normative constraint')
|
||||
'sample-paths' = @()
|
||||
})
|
||||
'validation-considerations' = @([ordered]@{
|
||||
id = 'stable id'
|
||||
reason = 'why evidence is needed'
|
||||
evidence = 'evidence implementation should obtain'
|
||||
})
|
||||
suppressed = @()
|
||||
unresolved = @()
|
||||
}
|
||||
}
|
||||
} | ConvertTo-Json -Depth 15 |
|
||||
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
|
||||
}
|
||||
}
|
||||
|
||||
if ($ResultsDirectory) {
|
||||
$failures = [System.Collections.Generic.List[string]]::new()
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
|
||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): missing result file.") | Out-Null
|
||||
continue
|
||||
}
|
||||
try {
|
||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||
} catch {
|
||||
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
|
||||
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'guidanceReport') {
|
||||
$failures.Add("$($case.id): guidanceReport is missing.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$report = $result.guidanceReport
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
|
||||
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
|
||||
$failures.Add("$($case.id): guidance report is missing '$requiredField'.") | Out-Null
|
||||
}
|
||||
}
|
||||
$skillId = if (
|
||||
$report.PSObject.Properties.Name -contains 'skill' -and
|
||||
$report.skill.PSObject.Properties.Name -contains 'id'
|
||||
) { [string]$report.skill.id } else { '' }
|
||||
if ($skillId -ne 'al-development-plan') {
|
||||
$failures.Add("$($case.id): guidance skill is '$skillId'.") | Out-Null
|
||||
}
|
||||
if ($report.PSObject.Properties.Name -notcontains 'outcome' -or [string]$report.outcome -ne 'completed') {
|
||||
$failures.Add("$($case.id): guidance outcome is not completed.") | Out-Null
|
||||
}
|
||||
$kind = if (
|
||||
$report.PSObject.Properties.Name -contains 'summary' -and
|
||||
$report.summary.PSObject.Properties.Name -contains 'kind'
|
||||
) { [string]$report.summary.kind } else { '' }
|
||||
if ($kind -ne [string]$case.expectedKind) {
|
||||
$failures.Add("$($case.id): summary.kind '$kind' does not match the plan.") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$knowledgeEntries = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'knowledge') {
|
||||
$knowledgeEntries = @($report.knowledge)
|
||||
}
|
||||
$usedKnowledge = @()
|
||||
foreach ($entry in $knowledgeEntries) {
|
||||
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
|
||||
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
|
||||
[object[]]$constraints = @()
|
||||
if ($entry.PSObject.Properties.Name -contains 'constraints') {
|
||||
$constraints = @($entry.constraints)
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($path) -or
|
||||
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): invalid knowledge path '$path'.") | Out-Null
|
||||
} else {
|
||||
$usedKnowledge += $path
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($usedFor) -or -not $constraints.Count) {
|
||||
$failures.Add("$($case.id): '$path' lacks used-for or constraints.") | Out-Null
|
||||
}
|
||||
if ($entry.PSObject.Properties.Name -contains 'sample-paths') {
|
||||
foreach ($samplePath in @($entry.'sample-paths')) {
|
||||
$samplePath = [string]$samplePath
|
||||
if ([string]::IsNullOrWhiteSpace($samplePath) -or
|
||||
-not (Test-Path -LiteralPath (Join-Path $Root $samplePath) -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): invalid sample path '$samplePath'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$required = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
|
||||
$matched = @($required | Where-Object { $usedKnowledge -contains $_ }).Count
|
||||
$recall = if ($required.Count) { $matched / $required.Count } else { 1.0 }
|
||||
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
|
||||
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
|
||||
}
|
||||
$accepted = @(
|
||||
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
|
||||
ForEach-Object { [string]$_ } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
$acceptedUsed = @($usedKnowledge | Where-Object { $accepted -contains $_ }).Count
|
||||
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
|
||||
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
|
||||
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
|
||||
}
|
||||
|
||||
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') {
|
||||
[string]$result.workspaceRoot
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
|
||||
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
|
||||
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
|
||||
} else {
|
||||
$status = @(& git -C $workspaceRoot status --porcelain)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
|
||||
} elseif ($status.Count) {
|
||||
$failures.Add("$($case.id): guidance skill changed the target repository.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($failures.Count) {
|
||||
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
|
||||
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Development guidance scoring PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
} else {
|
||||
Write-Host "Development guidance fixture validation PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue