mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Narrow AL development to read-only plan guidance
Retain shared knowledge enrichment and review guidance; defer standalone implementation and source-ingestion tracking. Add runner-owned baseline evidence, contract regressions, and explicit consumer/pilot boundaries. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Jesper Schulz-Wedde
parent
f6fca1d56d
commit
1deac52a53
26 changed files with 1486 additions and 11037 deletions
496
tools/DevelopmentGuidance.Evidence.ps1
Normal file
496
tools/DevelopmentGuidance.Evidence.ps1
Normal file
|
|
@ -0,0 +1,496 @@
|
|||
# Helpers for Test-DevelopmentGuidanceFixtures.ps1 and its deterministic regressions.
|
||||
function Get-GuidanceDiagnostic {
|
||||
param($ErrorRecord)
|
||||
if ($ErrorRecord.Exception -is [Management.Automation.RuntimeException] -and
|
||||
$ErrorRecord.FullyQualifiedErrorId -eq $ErrorRecord.Exception.Message) {
|
||||
return $ErrorRecord.Exception.Message
|
||||
}
|
||||
return 'Evidence or report could not be read safely (missing, malformed, inaccessible or unsupported state).'
|
||||
}
|
||||
|
||||
function Assert-GuidanceObject {
|
||||
param($Value, [string] $Label, [string[]] $Fields = @())
|
||||
if ($Value -isnot [Collections.IDictionary]) { throw "$Label must be a JSON object." }
|
||||
foreach ($field in $Fields) {
|
||||
if (-not $Value.Contains($field)) { throw "$Label is missing required field '$field'." }
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-GuidanceString {
|
||||
param($Value, [string] $Label)
|
||||
if ($Value -isnot [string] -or [string]::IsNullOrWhiteSpace($Value)) { throw "$Label must be a non-empty string." }
|
||||
}
|
||||
|
||||
function Assert-GuidanceArray {
|
||||
param($Value, [string] $Label, [switch] $Strings, [switch] $NonEmpty)
|
||||
if ($Value -isnot [array]) { throw "$Label must be a JSON array." }
|
||||
if ($NonEmpty -and $Value.Count -eq 0) { throw "$Label must not be empty." }
|
||||
if ($Strings) { foreach ($item in $Value) { Assert-GuidanceString $item "$Label entry" } }
|
||||
}
|
||||
|
||||
function Assert-GuidanceInteger {
|
||||
param($Value, [string] $Label)
|
||||
if (($Value -isnot [long] -and $Value -isnot [int] -and $Value -isnot [bigint]) -or $Value -lt 0) {
|
||||
throw "$Label must be a non-negative integer."
|
||||
}
|
||||
}
|
||||
|
||||
function Read-GuidanceJson {
|
||||
param([string] $Path)
|
||||
function Assert-JsonMembers($Element) {
|
||||
if ($Element.ValueKind -eq [Text.Json.JsonValueKind]::Object) {
|
||||
$names = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($property in $Element.EnumerateObject()) {
|
||||
if (-not $names.Add($property.Name)) { throw 'JSON contains duplicate or case-ambiguous members.' }
|
||||
Assert-JsonMembers $property.Value
|
||||
}
|
||||
} elseif ($Element.ValueKind -eq [Text.Json.JsonValueKind]::Array) {
|
||||
foreach ($item in $Element.EnumerateArray()) { Assert-JsonMembers $item }
|
||||
}
|
||||
}
|
||||
try {
|
||||
if ((Get-Item -LiteralPath $Path -Force).Length -gt 32MB) { throw 'Oversized JSON.' }
|
||||
$text = [IO.File]::ReadAllText($Path)
|
||||
$document = [Text.Json.JsonDocument]::Parse($text)
|
||||
try { Assert-JsonMembers $document.RootElement } finally { $document.Dispose() }
|
||||
return ConvertFrom-Json -InputObject $text -AsHashtable -Depth 64 -NoEnumerate
|
||||
} catch { throw 'Input is not readable, strict JSON with unique members.' }
|
||||
}
|
||||
|
||||
function Get-GuidanceHash {
|
||||
param([string] $Path)
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash
|
||||
}
|
||||
|
||||
function Get-GuidanceCaseId {
|
||||
param([string] $Id)
|
||||
$bytes = [Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($Id))
|
||||
return "case-$([Convert]::ToHexString($bytes).Substring(0, 8).ToLowerInvariant())"
|
||||
}
|
||||
|
||||
function Test-GuidanceWithin {
|
||||
param([string] $Path, [string] $Parent)
|
||||
$comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal }
|
||||
$prefix = $Parent.TrimEnd([IO.Path]::DirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar
|
||||
return $Path.Equals($Parent, $comparison) -or $Path.StartsWith($prefix, $comparison)
|
||||
}
|
||||
|
||||
function Assert-GuidanceItem {
|
||||
param($Item)
|
||||
if (($Item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $Item.LinkType -or $Item.LinkTarget) {
|
||||
throw 'Links, junctions, hard links and reparse points are not supported.'
|
||||
}
|
||||
if (-not $Item.PSIsContainer -and $IsWindows) {
|
||||
$streams = @(Get-Item -LiteralPath $Item.FullName -Stream '*' -Force -ErrorAction Stop)
|
||||
if (@($streams | Where-Object Stream -ne ':$DATA').Count) {
|
||||
throw 'Alternate data streams are not supported.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-GuidanceSafePath {
|
||||
param([string] $Path, [switch] $AllowMissing, [switch] $Directory, [switch] $File)
|
||||
Assert-GuidanceString $Path 'Filesystem path'
|
||||
$full = [IO.Path]::GetFullPath($Path)
|
||||
$driveRoot = [IO.Path]::GetPathRoot($full)
|
||||
if ($IsWindows -and ($driveRoot.StartsWith('\\') -or $full.Substring($driveRoot.Length).Contains(':'))) {
|
||||
throw 'Network paths and alternate stream paths are not supported.'
|
||||
}
|
||||
$cursor = $driveRoot
|
||||
$components = $full.Substring($driveRoot.Length).Split([IO.Path]::DirectorySeparatorChar, [StringSplitOptions]::RemoveEmptyEntries)
|
||||
foreach ($component in $components) {
|
||||
if ($component -match '[\. ]$') { throw 'Ambiguous filesystem path components are not supported.' }
|
||||
$cursor = Join-Path $cursor $component
|
||||
# Get-Item sees dangling links that Test-Path may treat as missing.
|
||||
$item = Get-Item -LiteralPath $cursor -Force -ErrorAction SilentlyContinue
|
||||
if ($null -ne $item) { Assert-GuidanceItem $item }
|
||||
elseif (-not $AllowMissing) { throw 'Required filesystem path is missing.' }
|
||||
}
|
||||
if ($Directory -and -not (Test-Path -LiteralPath $full -PathType Container)) { throw 'Required directory is missing.' }
|
||||
if ($File -and -not (Test-Path -LiteralPath $full -PathType Leaf)) { throw 'Required file is missing.' }
|
||||
return $full.TrimEnd([IO.Path]::DirectorySeparatorChar)
|
||||
}
|
||||
|
||||
function Assert-GuidanceTree {
|
||||
param([string] $Root)
|
||||
$pending = [Collections.Generic.Stack[string]]::new()
|
||||
$pending.Push($Root)
|
||||
while ($pending.Count) {
|
||||
foreach ($item in Get-ChildItem -LiteralPath $pending.Pop() -Force) {
|
||||
Assert-GuidanceItem $item
|
||||
if ($item.PSIsContainer) { $pending.Push($item.FullName) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-GuidanceGit {
|
||||
param([string] $Root, [string[]] $Arguments, [switch] $RawOutput)
|
||||
$start = [Diagnostics.ProcessStartInfo]::new('git')
|
||||
$start.UseShellExecute = $false
|
||||
$start.RedirectStandardOutput = $true
|
||||
$start.RedirectStandardError = $true
|
||||
foreach ($variable in @('GIT_DIR', 'GIT_WORK_TREE', 'GIT_COMMON_DIR', 'GIT_INDEX_FILE',
|
||||
'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_CONFIG',
|
||||
'GIT_CONFIG_COUNT', 'GIT_CONFIG_PARAMETERS', 'GIT_NAMESPACE')) {
|
||||
$null = $start.Environment.Remove($variable)
|
||||
}
|
||||
$start.Environment['GIT_CONFIG_NOSYSTEM'] = '1'
|
||||
$start.Environment['GIT_CONFIG_GLOBAL'] = ''
|
||||
$start.Environment['GIT_TERMINAL_PROMPT'] = '0'
|
||||
# In particular, do not execute a repository-supplied fsmonitor hook on reads.
|
||||
foreach ($argument in (@('--no-optional-locks', '-c', 'core.fsmonitor=false', '-C', $Root) + $Arguments)) {
|
||||
$start.ArgumentList.Add($argument)
|
||||
}
|
||||
$process = [Diagnostics.Process]::Start($start)
|
||||
try {
|
||||
$output = $process.StandardOutput.ReadToEndAsync()
|
||||
$errors = $process.StandardError.ReadToEndAsync()
|
||||
$process.WaitForExit()
|
||||
$null = $errors.GetAwaiter().GetResult()
|
||||
if ($process.ExitCode -ne 0) { throw 'Git evidence could not be read.' }
|
||||
$text = $output.GetAwaiter().GetResult()
|
||||
if ($RawOutput) { return $text }
|
||||
return $text.TrimEnd("`r", "`n")
|
||||
} finally { $process.Dispose() }
|
||||
}
|
||||
|
||||
function Get-GuidanceSnapshot {
|
||||
param([string] $Root, [switch] $Target)
|
||||
$Root = Get-GuidanceSafePath $Root -Directory
|
||||
$dotGit = Get-GuidanceSafePath (Join-Path $Root '.git')
|
||||
if (Test-Path -LiteralPath $dotGit -PathType Container) {
|
||||
$gitDir = $dotGit
|
||||
} else {
|
||||
if ($Target) { throw 'Target workspaces must be standalone repositories with internal Git storage.' }
|
||||
$pointer = [IO.File]::ReadAllText($dotGit)
|
||||
if ($pointer -notmatch '\Agitdir: ([^\r\n]+)\r?\n?\z') { throw 'Unsupported Git worktree pointer.' }
|
||||
$gitPath = $Matches[1]
|
||||
if (-not [IO.Path]::IsPathFullyQualified($gitPath)) { $gitPath = Join-Path $Root $gitPath }
|
||||
$gitDir = Get-GuidanceSafePath $gitPath -Directory
|
||||
}
|
||||
$commonDir = $gitDir
|
||||
$commonPointer = Get-GuidanceSafePath (Join-Path $gitDir 'commondir') -AllowMissing
|
||||
if (Test-Path -LiteralPath $commonPointer) {
|
||||
$commonPath = [IO.File]::ReadAllText($commonPointer).Trim()
|
||||
if (-not [IO.Path]::IsPathFullyQualified($commonPath)) { $commonPath = Join-Path $gitDir $commonPath }
|
||||
$commonDir = Get-GuidanceSafePath $commonPath -Directory
|
||||
}
|
||||
if ($Target -and ($gitDir -cne (Join-Path $Root '.git') -or $commonDir -cne $gitDir)) {
|
||||
throw 'Target workspaces must be standalone repositories with internal Git storage.'
|
||||
}
|
||||
$files = [Collections.Generic.List[object]]::new()
|
||||
$pending = [Collections.Generic.Stack[string]]::new()
|
||||
$pending.Push($Root)
|
||||
while ($pending.Count) {
|
||||
foreach ($item in @(Get-ChildItem -LiteralPath $pending.Pop() -Force | Sort-Object Name -CaseSensitive)) {
|
||||
Assert-GuidanceItem $item
|
||||
$relative = [IO.Path]::GetRelativePath($Root, $item.FullName).Replace('\', '/')
|
||||
$entry = [ordered]@{
|
||||
path = $relative
|
||||
kind = if ($item.PSIsContainer) { 'directory' } else { 'file' }
|
||||
attributes = [int]$item.Attributes
|
||||
unixMode = [int]$item.UnixFileMode
|
||||
creationUtcTicks = $item.CreationTimeUtc.Ticks
|
||||
}
|
||||
if ($item.PSIsContainer) {
|
||||
$pending.Push($item.FullName)
|
||||
} else {
|
||||
$entry.length = $item.Length
|
||||
$entry.lastWriteUtcTicks = $item.LastWriteTimeUtc.Ticks
|
||||
$entry.sha256 = Get-GuidanceHash $item.FullName
|
||||
}
|
||||
$files.Add($entry)
|
||||
}
|
||||
}
|
||||
# Linked knowledge worktrees have explicitly identified Git metadata outside
|
||||
# the content root. Record its meaningful configuration as well as HEAD/refs.
|
||||
$gitMetadata = [ordered]@{}
|
||||
foreach ($storage in @($gitDir, $commonDir) | Sort-Object -Unique) {
|
||||
foreach ($name in @('HEAD', 'commondir', 'config', 'config.worktree', 'packed-refs', 'refs', 'objects', 'index', 'info\exclude', 'shallow')) {
|
||||
$path = Get-GuidanceSafePath (Join-Path $storage $name) -AllowMissing
|
||||
if ((Test-Path -LiteralPath $path -PathType Container) -and -not (Test-GuidanceWithin $path $Root)) {
|
||||
Assert-GuidanceTree $path
|
||||
}
|
||||
if (Test-Path -LiteralPath $path -PathType Leaf) {
|
||||
$gitMetadata[$path] = Get-GuidanceHash $path
|
||||
}
|
||||
if ($name -in @('config', 'config.worktree') -and (Test-Path -LiteralPath $path) -and
|
||||
[IO.File]::ReadAllText($path) -match '(?im)^\s*\[include(?:If)?\b') {
|
||||
throw 'External Git configuration includes are not supported.'
|
||||
}
|
||||
}
|
||||
foreach ($name in @('objects\info\alternates', 'objects\info\http-alternates')) {
|
||||
if (Test-Path -LiteralPath (Join-Path $storage $name)) { throw 'External Git object stores are not supported.' }
|
||||
}
|
||||
}
|
||||
$top = Get-GuidanceSafePath (Invoke-GuidanceGit $Root @('rev-parse', '--show-toplevel')) -Directory
|
||||
if ($top -cne $Root) { throw 'Evidence requires the exact Git worktree root, not a subdirectory.' }
|
||||
$indexPath = Get-GuidanceSafePath (Join-Path $gitDir 'index') -AllowMissing
|
||||
$tracked = Invoke-GuidanceGit $Root @('ls-files', '--stage')
|
||||
if ($tracked -match '(?m)^160000 ') { throw 'Submodule workspaces are not supported.' }
|
||||
if ((Invoke-GuidanceGit $Root @('ls-files', '-t')) -match '(?m)^S ') { throw 'Sparse workspaces are not supported.' }
|
||||
$head = Invoke-GuidanceGit $Root @('rev-parse', '--verify', 'HEAD')
|
||||
$headFile = Get-GuidanceSafePath (Join-Path $gitDir 'HEAD') -File
|
||||
# Access times, Git status refreshes, and index-builder generatedAt are not evidence.
|
||||
return [ordered]@{
|
||||
version = 1
|
||||
root = $Root
|
||||
rootCreationUtcTicks = (Get-Item -LiteralPath $Root -Force).CreationTimeUtc.Ticks
|
||||
gitDir = $gitDir
|
||||
commonDir = $commonDir
|
||||
gitMetadata = $gitMetadata
|
||||
head = $head
|
||||
headFileSha256 = Get-GuidanceHash $headFile
|
||||
references = Invoke-GuidanceGit $Root @('for-each-ref', '--format=%(refname) %(objectname) %(symref)')
|
||||
indexPath = $indexPath
|
||||
indexSha256 = if (Test-Path -LiteralPath $indexPath) { Get-GuidanceHash $indexPath } else { $null }
|
||||
files = @($files | Sort-Object { $_.path } -CaseSensitive)
|
||||
}
|
||||
}
|
||||
|
||||
function Test-GuidanceSnapshotEqual {
|
||||
param($Before, $After)
|
||||
return ($Before | ConvertTo-Json -Depth 64 -Compress) -ceq ($After | ConvertTo-Json -Depth 64 -Compress)
|
||||
}
|
||||
|
||||
function Write-GuidanceNewJson {
|
||||
param([string] $Path, $Value)
|
||||
$parent = Split-Path -Parent $Path
|
||||
[IO.Directory]::CreateDirectory($parent) | Out-Null
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes(($Value | ConvertTo-Json -Depth 64))
|
||||
$stream = [IO.File]::Open($Path, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try { $stream.Write($bytes, 0, $bytes.Length) } finally { $stream.Dispose() }
|
||||
}
|
||||
|
||||
function Resolve-GuidanceReference {
|
||||
param([string] $Root, $Reference, [switch] $Knowledge, [string] $Article)
|
||||
Assert-GuidanceString $Reference 'Reference path'
|
||||
if ($Reference -cnotmatch '^[a-zA-Z0-9_-]+(?:/[a-zA-Z0-9_.-]+)+$' -or
|
||||
@($Reference.Split('/') | Where-Object { $_ -in @('.', '..') -or $_ -match '[\. ]$' }).Count) {
|
||||
throw 'Reference must be an unambiguous forward-slash repository-relative path without traversal.'
|
||||
}
|
||||
if ($Knowledge -and $Reference -cnotmatch '^(microsoft|community|custom)/knowledge/[a-z0-9-]+/(?:[a-z0-9-]+/)*[a-z0-9-]+\.md$') {
|
||||
throw 'Knowledge references must identify actual layered knowledge articles.'
|
||||
}
|
||||
if ($Article) {
|
||||
$stem = $Article.Substring(0, $Article.Length - 3)
|
||||
if ($Reference -cnotmatch ('^' + [regex]::Escape($stem) + '\.(good|bad)\.[a-zA-Z0-9]+$')) {
|
||||
throw 'Sample references must be good/bad siblings of their knowledge article.'
|
||||
}
|
||||
}
|
||||
$cursor = $Root
|
||||
foreach ($component in $Reference.Split('/')) {
|
||||
$cursor = Join-Path $cursor $component
|
||||
$cursor = Get-GuidanceSafePath $cursor
|
||||
if ((Get-Item -LiteralPath $cursor -Force).Name -cne $component) { throw 'Reference path casing must match the actual file.' }
|
||||
}
|
||||
if (-not (Test-GuidanceWithin $cursor $Root) -or -not (Test-Path -LiteralPath $cursor -PathType Leaf)) {
|
||||
throw 'Reference must resolve to an existing file inside the knowledge checkout.'
|
||||
}
|
||||
if ($Knowledge) {
|
||||
$text = [IO.File]::ReadAllText($cursor)
|
||||
if ($text -notmatch '(?s)^---\r?\n.*?\r?\n---' -or
|
||||
$text -notmatch '(?m)^domain:\s*\S+' -or $text -notmatch '(?m)^## (Best Practice|Anti Pattern)\s*$') {
|
||||
throw 'Knowledge reference does not contain a normative knowledge article.'
|
||||
}
|
||||
}
|
||||
return $cursor
|
||||
}
|
||||
|
||||
function Get-GuidancePlanRequest {
|
||||
param($Plan)
|
||||
if ($Plan -is [string]) { Assert-GuidanceString $Plan 'development-plan'; return $Plan }
|
||||
Assert-GuidanceObject $Plan 'development-plan' @('request')
|
||||
Assert-GuidanceString $Plan.request 'development-plan.request'
|
||||
return $Plan.request
|
||||
}
|
||||
|
||||
function Assert-GuidanceContext {
|
||||
param($Context)
|
||||
Assert-GuidanceObject $Context 'context' @('bc-version', 'technologies', 'countries', 'application-area', 'unknown')
|
||||
Assert-GuidanceString $Context.'bc-version' 'context.bc-version'
|
||||
foreach ($key in @('technologies', 'countries', 'application-area', 'unknown')) {
|
||||
Assert-GuidanceArray $Context[$key] "context.$key" -Strings
|
||||
if (@($Context[$key] | Sort-Object -Unique).Count -ne $Context[$key].Count) { throw "context.$key contains duplicates." }
|
||||
}
|
||||
foreach ($key in $Context.unknown) {
|
||||
if ($key -cnotin @('bc-version', 'technologies', 'countries', 'application-area')) { throw 'context.unknown contains an invalid dimension.' }
|
||||
}
|
||||
if ($Context.'bc-version' -eq 'unknown' -and 'bc-version' -cnotin $Context.unknown) {
|
||||
throw 'Unknown BC version must be recorded in context.unknown.'
|
||||
}
|
||||
foreach ($key in @('technologies', 'countries', 'application-area')) {
|
||||
if ((-not $Context[$key].Count -or 'unknown' -in $Context[$key]) -and $key -cnotin $Context.unknown) {
|
||||
throw 'Unavailable applicability dimensions must be recorded in context.unknown.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-GuidanceManifest {
|
||||
param($Manifest, [string] $Root)
|
||||
Assert-GuidanceObject $Manifest 'manifest' @('version', 'minimumKnowledgeRecall', 'minimumKnowledgePrecision', 'skill', 'cases')
|
||||
if ($Manifest.version -ne 1) { throw 'Unsupported guidance fixture manifest version.' }
|
||||
foreach ($name in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||
$value = $Manifest[$name]
|
||||
if (($value -isnot [double] -and $value -isnot [long] -and $value -isnot [int] -and $value -isnot [decimal]) -or
|
||||
$value -lt 0 -or $value -gt 1) { throw 'Manifest thresholds must be numbers between zero and one.' }
|
||||
}
|
||||
$null = Resolve-GuidanceReference $Root $Manifest.skill
|
||||
Assert-GuidanceArray $Manifest.cases 'manifest.cases' -NonEmpty
|
||||
$ids = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
$modelIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($case in $Manifest.cases) {
|
||||
Assert-GuidanceObject $case 'manifest case' @('id', 'expectedKind', 'expectedOutcome', 'development-plan', 'context', 'requiredKnowledge', 'optionalKnowledge', 'expectedUnknown', 'requiresUnresolved', 'requiresMaterialUnresolved')
|
||||
if ($case.id -isnot [string] -or $case.id -cnotmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') { throw 'Fixture id must be kebab-case.' }
|
||||
if (-not $ids.Add($case.id) -or -not $modelIds.Add((Get-GuidanceCaseId $case.id))) { throw 'Duplicate fixture or model case identity.' }
|
||||
if ($case.expectedKind -cnotin @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')) { throw 'Fixture expectedKind is invalid.' }
|
||||
if ($case.expectedOutcome -cnotin @('completed', 'not-applicable', 'no-knowledge', 'partial', 'failed')) { throw 'Fixture expectedOutcome is invalid.' }
|
||||
Assert-GuidanceString $case.expectedKind 'fixture.expectedKind'
|
||||
Assert-GuidanceString $case.expectedOutcome 'fixture.expectedOutcome'
|
||||
$null = Get-GuidancePlanRequest $case.'development-plan'
|
||||
Assert-GuidanceContext $case.context
|
||||
foreach ($name in @('requiredKnowledge', 'optionalKnowledge', 'expectedUnknown')) {
|
||||
Assert-GuidanceArray $case[$name] "fixture.$name" -Strings
|
||||
}
|
||||
if ($case.requiresUnresolved -isnot [bool]) { throw 'Fixture requiresUnresolved must be boolean.' }
|
||||
if ($case.requiresMaterialUnresolved -isnot [bool]) { throw 'Fixture requiresMaterialUnresolved must be boolean.' }
|
||||
if ($case.requiresMaterialUnresolved -and ($case.expectedOutcome -ne 'partial' -or -not $case.requiresUnresolved)) {
|
||||
throw 'Materially unresolved fixtures must expect partial and unresolved evidence.'
|
||||
}
|
||||
foreach ($key in $case.expectedUnknown) {
|
||||
if ($key -cnotin @('bc-version', 'technologies', 'countries', 'application-area')) { throw 'Fixture expectedUnknown is invalid.' }
|
||||
}
|
||||
$references = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||
if (@($references | Sort-Object -Unique).Count -ne $references.Count) { throw 'Fixture knowledge references contain duplicates.' }
|
||||
foreach ($reference in $references) { $null = Resolve-GuidanceReference $Root $reference -Knowledge }
|
||||
if ($case.expectedOutcome -in @('no-knowledge', 'not-applicable') -and $references.Count) {
|
||||
throw 'Empty-knowledge outcomes cannot require or accept knowledge.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-GuidanceResult {
|
||||
param($Result, $Case, $Manifest, [string] $Root, [string] $Workspace)
|
||||
Assert-GuidanceObject $Result 'result' @('caseId', 'guidanceReport')
|
||||
Assert-GuidanceString $Result.caseId 'result.caseId'
|
||||
if ($Result.caseId -cne (Get-GuidanceCaseId $Case.id)) { throw 'Result caseId mismatch.' }
|
||||
if ($Result.Contains('workspaceRoot')) {
|
||||
Assert-GuidanceString $Result.workspaceRoot 'result.workspaceRoot'
|
||||
if (-not [IO.Path]::IsPathFullyQualified($Result.workspaceRoot) -or
|
||||
(Get-GuidanceSafePath $Result.workspaceRoot -Directory) -cne $Workspace) {
|
||||
throw 'Result workspaceRoot disagrees with the runner binding.'
|
||||
}
|
||||
}
|
||||
$report = $Result.guidanceReport
|
||||
Assert-GuidanceObject $report 'guidanceReport' @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')
|
||||
Assert-GuidanceObject $report.skill 'skill' @('id', 'version')
|
||||
Assert-GuidanceString $report.skill.id 'skill.id'
|
||||
if ($report.skill.id -cne 'al-development-plan' -or $report.skill.version -isnot [long] -or $report.skill.version -ne 1) {
|
||||
throw 'Report skill identity/version is invalid.'
|
||||
}
|
||||
Assert-GuidanceString $report.outcome 'outcome'
|
||||
if ($report.outcome -cnotin @('completed', 'not-applicable', 'no-knowledge', 'partial', 'failed')) { throw 'Report outcome enum is invalid.' }
|
||||
if ($report.outcome -cne $Case.expectedOutcome) { throw 'Report outcome does not match fixture expectedOutcome.' }
|
||||
if ($report.outcome -in @('partial', 'failed') -or $report.Contains('outcome-reason')) {
|
||||
Assert-GuidanceString $report['outcome-reason'] 'outcome-reason'
|
||||
}
|
||||
Assert-GuidanceObject $report.summary 'summary' @('request', 'kind', 'candidates', 'selected')
|
||||
Assert-GuidanceString $report.summary.request 'summary.request'
|
||||
Assert-GuidanceString $report.summary.kind 'summary.kind'
|
||||
if ($report.summary.kind -cne $Case.expectedKind) { throw 'summary.kind does not match the intended change.' }
|
||||
Assert-GuidanceInteger $report.summary.candidates 'summary.candidates'
|
||||
Assert-GuidanceInteger $report.summary.selected 'summary.selected'
|
||||
Assert-GuidanceContext $report.context
|
||||
foreach ($name in @('knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
|
||||
Assert-GuidanceArray $report[$name] $name
|
||||
}
|
||||
Assert-GuidanceArray $report.unresolved 'unresolved' -Strings
|
||||
if ($report.summary.selected -ne $report.knowledge.Count -or $report.summary.selected -gt $report.summary.candidates) {
|
||||
throw 'Summary counts disagree with selected knowledge/candidates.'
|
||||
}
|
||||
if ($report.outcome -in @('no-knowledge', 'not-applicable') -and $report.knowledge.Count) { throw 'This outcome requires empty knowledge.' }
|
||||
if ($report.outcome -eq 'completed' -and -not $report.knowledge.Count) { throw 'Completed requires selected knowledge; empty evaluation is no-knowledge.' }
|
||||
if (($report.outcome -eq 'partial' -or $Case.requiresUnresolved) -and -not $report.unresolved.Count) {
|
||||
throw 'Partial/incomplete evaluation must explain unresolved gaps.'
|
||||
}
|
||||
foreach ($dimension in $Case.expectedUnknown) {
|
||||
if ($dimension -cnotin $report.context.unknown) { throw 'Expected unknown context was silently resolved.' }
|
||||
}
|
||||
foreach ($dimension in $report.context.unknown) {
|
||||
if (-not @($report.unresolved | Where-Object { $_ -match [regex]::Escape($dimension) }).Count) {
|
||||
throw 'Unknown dimensions require a corresponding unresolved explanation.'
|
||||
}
|
||||
}
|
||||
# Free-form unresolved text has no machine-readable materiality field in DO.
|
||||
# Known material fixture conditions are runner expectations, not model claims.
|
||||
if ($Case.requiresMaterialUnresolved -and $report.outcome -ne 'partial') { throw 'Material unknown guidance must remain partial.' }
|
||||
$used = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($entry in $report.knowledge) {
|
||||
Assert-GuidanceObject $entry 'knowledge entry' @('path', 'used-for', 'constraints', 'sample-paths')
|
||||
$null = Resolve-GuidanceReference $Root $entry.path -Knowledge
|
||||
if (-not $used.Add($entry.path)) { throw 'Duplicate knowledge reference.' }
|
||||
Assert-GuidanceString $entry.'used-for' 'knowledge.used-for'
|
||||
Assert-GuidanceArray $entry.constraints 'knowledge.constraints' -Strings -NonEmpty
|
||||
Assert-GuidanceArray $entry.'sample-paths' 'knowledge.sample-paths' -Strings
|
||||
if (@($entry.'sample-paths' | Sort-Object -Unique).Count -ne $entry.'sample-paths'.Count) { throw 'Duplicate sample reference.' }
|
||||
foreach ($sample in $entry.'sample-paths') { $null = Resolve-GuidanceReference $Root $sample -Article $entry.path }
|
||||
Assert-GuidanceReferenceSha $entry $Root
|
||||
}
|
||||
$validationIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($entry in $report.'validation-considerations') {
|
||||
Assert-GuidanceObject $entry 'validation consideration' @('id', 'reason', 'evidence')
|
||||
foreach ($key in @('id', 'reason', 'evidence')) { Assert-GuidanceString $entry[$key] "validation-considerations.$key" }
|
||||
if (-not $validationIds.Add($entry.id)) { throw 'Duplicate validation consideration id.' }
|
||||
}
|
||||
$suppressed = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($entry in $report.suppressed) {
|
||||
Assert-GuidanceObject $entry 'suppressed entry' @('reference', 'reason')
|
||||
Assert-GuidanceObject $entry.reference 'suppressed.reference' @('path')
|
||||
$null = Resolve-GuidanceReference $Root $entry.reference.path -Knowledge
|
||||
Assert-GuidanceReferenceSha $entry.reference $Root
|
||||
Assert-GuidanceString $entry.reason 'suppression reason'
|
||||
if ($entry.reason -cnotin @('layer-precedence', 'configuration')) { throw 'Suppression reason is invalid.' }
|
||||
if (-not $suppressed.Add($entry.reference.path) -or $used.Contains($entry.reference.path)) { throw 'Duplicate or selected suppressed reference.' }
|
||||
}
|
||||
$matched = @($Case.requiredKnowledge | Where-Object { $used.Contains($_) }).Count
|
||||
$recall = if ($Case.requiredKnowledge.Count) { $matched / $Case.requiredKnowledge.Count } else { 1.0 }
|
||||
$accepted = @($Case.requiredKnowledge) + @($Case.optionalKnowledge)
|
||||
$acceptedCount = @($used | Where-Object { $_ -cin $accepted }).Count
|
||||
$precision = if ($used.Count) { $acceptedCount / $used.Count } elseif (-not $Case.requiredKnowledge.Count) { 1.0 } else { 0.0 }
|
||||
if ($recall -lt $Manifest.minimumKnowledgeRecall) { throw 'Knowledge recall is below the manifest threshold.' }
|
||||
if ($precision -lt $Manifest.minimumKnowledgePrecision) { throw 'Knowledge precision is below the manifest threshold.' }
|
||||
}
|
||||
|
||||
function Assert-GuidanceReferenceSha {
|
||||
param($Entry, [string] $Root)
|
||||
if ($Entry.Contains('sha')) {
|
||||
if ($Entry.sha -isnot [string] -or $Entry.sha -cnotmatch '^([0-9a-fA-F]{40}|[0-9a-fA-F]{64})$') {
|
||||
throw 'Reference SHA must be a full commit object id.'
|
||||
}
|
||||
$head = Invoke-GuidanceGit $Root @('rev-parse', '--verify', 'HEAD')
|
||||
if ($Entry.sha -ne $head) { throw 'Reference SHA does not identify the recorded live checkout.' }
|
||||
$committed = Invoke-GuidanceGit $Root @('cat-file', 'blob', "$head`:$($Entry.path)") -RawOutput
|
||||
$live = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $Entry.path -Knowledge))
|
||||
if ($committed.Replace("`r`n", "`n") -cne $live.Replace("`r`n", "`n")) {
|
||||
throw 'Reference SHA content differs from the live knowledge article.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-GuidanceResultSchema {
|
||||
param([string] $CaseId)
|
||||
return [ordered]@{
|
||||
caseId = $CaseId
|
||||
guidanceReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
||||
'outcome-reason' = 'required for partial or failed'
|
||||
summary = [ordered]@{ request = 'planned intent'; kind = 'feature | bug | refactor | upgrade | maintenance'; candidates = 0; selected = 0 }
|
||||
context = [ordered]@{ 'bc-version' = 'resolved target or unknown'; technologies = @('al'); countries = @('w1'); 'application-area' = @('all'); unknown = @() }
|
||||
knowledge = @([ordered]@{ path = 'repo-relative knowledge article'; sha = 'optional full checkout commit id'; 'used-for' = 'plan decision'; constraints = @('faithful normative constraint'); 'sample-paths' = @() })
|
||||
'validation-considerations' = @([ordered]@{ id = 'stable id'; reason = 'why needed'; evidence = 'evidence implementation should obtain' })
|
||||
suppressed = @([ordered]@{ reference = [ordered]@{ path = 'suppressed knowledge path' }; reason = 'layer-precedence | configuration' })
|
||||
unresolved = @('Missing context/decision, affected candidate, and materiality; name unknown dimensions exactly.')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,584 +0,0 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates and prepares BCQuality AL development fixtures.
|
||||
|
||||
.DESCRIPTION
|
||||
Static validation checks fixture IDs, capability links, knowledge references,
|
||||
and the development skill. -PrepareDirectory emits opaque requests for model
|
||||
runs. -ResultsDirectory scores implementation reports produced by an
|
||||
external runner that performed the declared compile, test, and review checks.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
[string] $ManifestPath,
|
||||
[string] $CapabilitiesPath,
|
||||
[string] $PrepareDirectory,
|
||||
[string] $ResultsDirectory
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $ManifestPath) {
|
||||
$ManifestPath = Join-Path $Root 'evaluation/development-fixtures.json'
|
||||
}
|
||||
if (-not $CapabilitiesPath) {
|
||||
$CapabilitiesPath = Join-Path $Root 'coverage/development-capabilities.json'
|
||||
}
|
||||
|
||||
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
|
||||
$capabilityManifest = Get-Content -LiteralPath $CapabilitiesPath -Raw | ConvertFrom-Json
|
||||
$problems = [System.Collections.Generic.List[string]]::new()
|
||||
|
||||
function Get-ModelCaseId {
|
||||
param([string] $ManifestId)
|
||||
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
|
||||
$hash = $sha.ComputeHash($bytes)
|
||||
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
|
||||
return "case-$token"
|
||||
} finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
if ($manifest.version -ne 1) {
|
||||
$problems.Add("Unsupported development fixture version: $($manifest.version)") | Out-Null
|
||||
}
|
||||
if ($capabilityManifest.version -ne 1) {
|
||||
$problems.Add("Unsupported capability manifest version: $($capabilityManifest.version)") | Out-Null
|
||||
}
|
||||
$minimumFixtureCoverage = if ($capabilityManifest.PSObject.Properties.Name -contains 'minimumFixtureCoverage') {
|
||||
[double]$capabilityManifest.minimumFixtureCoverage
|
||||
} else {
|
||||
-1
|
||||
}
|
||||
if ($minimumFixtureCoverage -lt 0 -or $minimumFixtureCoverage -gt 1) {
|
||||
$problems.Add("minimumFixtureCoverage must be between 0 and 1.") | Out-Null
|
||||
}
|
||||
$maximumReviewRounds = if ($manifest.PSObject.Properties.Name -contains 'maximumReviewRounds') {
|
||||
[int]$manifest.maximumReviewRounds
|
||||
} else {
|
||||
0
|
||||
}
|
||||
if ($maximumReviewRounds -le 0) {
|
||||
$problems.Add("maximumReviewRounds must be a positive integer.") | Out-Null
|
||||
}
|
||||
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||
$threshold = [double]$manifest.$thresholdName
|
||||
if ($threshold -lt 0 -or $threshold -gt 1) {
|
||||
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$skillPath = [string]$manifest.skill
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
|
||||
$problems.Add("Development skill does not exist: $skillPath") | Out-Null
|
||||
} else {
|
||||
$skillText = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
||||
$limitMatch = [regex]::Match($skillText, '(?m)^quality-round-limit:\s*(\d+)\s*$')
|
||||
if (-not $limitMatch.Success -or [int]$limitMatch.Groups[1].Value -ne $maximumReviewRounds) {
|
||||
$problems.Add("maximumReviewRounds must match the development skill quality-round-limit.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$validChecks = @('compile', 'tests', 'review')
|
||||
$validInputKinds = @('auto', 'feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
||||
$validOutputKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
||||
$caseById = @{}
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$id = [string]$case.id
|
||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
|
||||
} elseif ($caseById.ContainsKey($id)) {
|
||||
$problems.Add("Duplicate fixture id: $id") | Out-Null
|
||||
} else {
|
||||
$caseById[$id] = $case
|
||||
}
|
||||
|
||||
if ($case.PSObject.Properties.Name -notcontains 'development-request') {
|
||||
$problems.Add("${id}: development-request is required.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$request = $case.'development-request'
|
||||
if ($validInputKinds -notcontains [string]$request.kind) {
|
||||
$problems.Add("${id}: development-request.kind must be one of $($validInputKinds -join ', ').") | Out-Null
|
||||
}
|
||||
$description = if ($request.PSObject.Properties.Name -contains 'description') {
|
||||
[string]$request.description
|
||||
} else {
|
||||
''
|
||||
}
|
||||
$planText = if ($request.PSObject.Properties.Name -contains 'plan') {
|
||||
[string]$request.plan
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($description) -and [string]::IsNullOrWhiteSpace($planText)) {
|
||||
$problems.Add("${id}: development-request requires description or plan.") | Out-Null
|
||||
}
|
||||
if ($request.PSObject.Properties.Name -notcontains 'acceptance-criteria' -or
|
||||
-not @($request.'acceptance-criteria').Count) {
|
||||
$problems.Add("${id}: development-request.acceptance-criteria must not be empty.") | Out-Null
|
||||
}
|
||||
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
|
||||
[string]$case.expectedKind
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($validOutputKinds -notcontains $expectedKind) {
|
||||
$problems.Add("${id}: expectedKind must be one of $($validOutputKinds -join ', ').") | Out-Null
|
||||
}
|
||||
if ([string]$request.kind -ne 'auto' -and [string]$request.kind -ne $expectedKind) {
|
||||
$problems.Add("${id}: explicit request kind '$($request.kind)' must equal expectedKind '$expectedKind'.") | Out-Null
|
||||
}
|
||||
|
||||
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
|
||||
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
|
||||
}
|
||||
foreach ($reference in $expectedKnowledge) {
|
||||
$reference = [string]$reference
|
||||
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
|
||||
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
|
||||
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
|
||||
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
|
||||
}
|
||||
}
|
||||
foreach ($check in @($case.requiredChecks)) {
|
||||
if ($validChecks -notcontains [string]$check) {
|
||||
$problems.Add("${id}: unsupported required check '$check'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$validCapabilityStatuses = @('planned', 'fixture', 'validated')
|
||||
$capabilityIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($capability in @($capabilityManifest.capabilities)) {
|
||||
$id = [string]$capability.id
|
||||
if (-not $capabilityIds.Add($id)) {
|
||||
$problems.Add("Duplicate capability id: $id") | Out-Null
|
||||
}
|
||||
if ($validCapabilityStatuses -notcontains [string]$capability.status) {
|
||||
$problems.Add("${id}: invalid capability status '$($capability.status)'.") | Out-Null
|
||||
}
|
||||
$fixtureIds = @($capability.fixtureIds)
|
||||
if ($capability.status -in @('fixture', 'validated') -and -not $fixtureIds.Count) {
|
||||
$problems.Add("${id}: status '$($capability.status)' requires at least one fixture.") | Out-Null
|
||||
}
|
||||
foreach ($fixtureId in $fixtureIds) {
|
||||
if (-not $caseById.ContainsKey([string]$fixtureId)) {
|
||||
$problems.Add("${id}: unknown fixture id '$fixtureId'.") | Out-Null
|
||||
} elseif (@($caseById[[string]$fixtureId].capabilities) -notcontains $id) {
|
||||
$problems.Add("${id}: fixture '$fixtureId' does not link back to the capability.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
foreach ($capabilityId in @($case.capabilities)) {
|
||||
if (-not $capabilityIds.Contains([string]$capabilityId)) {
|
||||
$problems.Add("$($case.id): unknown capability '$capabilityId'.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$capability = @(
|
||||
$capabilityManifest.capabilities |
|
||||
Where-Object id -eq ([string]$capabilityId)
|
||||
)[0]
|
||||
if (@($capability.fixtureIds) -notcontains [string]$case.id) {
|
||||
$problems.Add("$($case.id): capability '$capabilityId' does not link back to the fixture.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$fixtureBackedCount = @(
|
||||
$capabilityManifest.capabilities |
|
||||
Where-Object status -in @('fixture', 'validated')
|
||||
).Count
|
||||
$capabilityCount = @($capabilityManifest.capabilities).Count
|
||||
$fixtureCoverage = if ($capabilityCount) { $fixtureBackedCount / $capabilityCount } else { 0.0 }
|
||||
if ($fixtureCoverage -lt $minimumFixtureCoverage) {
|
||||
$problems.Add("Fixture-backed capability coverage $fixtureCoverage is below minimumFixtureCoverage $minimumFixtureCoverage.") | Out-Null
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Development fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ($PrepareDirectory) {
|
||||
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-evaluation'
|
||||
if (Test-Path -LiteralPath $PrepareDirectory) {
|
||||
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
|
||||
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
|
||||
throw "PrepareDirectory is not empty and is not a BCQuality development evaluation directory: $PrepareDirectory"
|
||||
}
|
||||
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
|
||||
Get-ChildItem -LiteralPath $PrepareDirectory -File |
|
||||
Where-Object {
|
||||
$_.Name -eq 'knowledge-index.json' -or
|
||||
$_.Name -like 'request-*.json' -or
|
||||
$_.Name -like 'result-*.json'
|
||||
} |
|
||||
Remove-Item -Force
|
||||
}
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
|
||||
}
|
||||
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development evaluation directory' -Encoding UTF8
|
||||
|
||||
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
||||
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
$request = [ordered]@{
|
||||
protocol = 'Run the supplied AL development skill in a fresh writable fixture repository. Persist the implementation, run real checks, and return only implementationReport using the supplied schema.'
|
||||
caseId = $modelId
|
||||
skill = $skillPath
|
||||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = 'knowledge-index.json'
|
||||
'task-context' = [ordered]@{
|
||||
goal = if ($case.'development-request'.PSObject.Properties.Name -contains 'description') {
|
||||
[string]$case.'development-request'.description
|
||||
} else {
|
||||
[string]$case.'development-request'.plan
|
||||
}
|
||||
'inputs-available' = @('development-request', 'repository')
|
||||
technologies = @($case.context.technologies)
|
||||
countries = @($case.context.countries)
|
||||
'application-area' = @($case.context.'application-area')
|
||||
}
|
||||
'development-request' = [ordered]@{
|
||||
kind = [string]$case.'development-request'.kind
|
||||
description = if ($case.'development-request'.PSObject.Properties.Name -contains 'description') {
|
||||
[string]$case.'development-request'.description
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
plan = if ($case.'development-request'.PSObject.Properties.Name -contains 'plan') {
|
||||
[string]$case.'development-request'.plan
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
'acceptance-criteria' = @($case.'development-request'.'acceptance-criteria')
|
||||
}
|
||||
resultSchema = [ordered]@{
|
||||
caseId = $modelId
|
||||
workspaceRoot = 'absolute path to the retained fixture repository'
|
||||
implementationReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development'; version = 1 }
|
||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
||||
'outcome-reason' = 'required for partial or failed'
|
||||
summary = [ordered]@{
|
||||
request = 'implemented change'
|
||||
'files-created' = 0
|
||||
'files-modified' = 0
|
||||
'files-deleted' = 0
|
||||
}
|
||||
plan = [ordered]@{
|
||||
kind = 'feature | bug | refactor | upgrade | maintenance'
|
||||
assumptions = @()
|
||||
decisions = @()
|
||||
objects = @()
|
||||
}
|
||||
knowledge = @([ordered]@{ path = 'repo-relative knowledge article path'; sha = 'optional commit sha'; 'used-for' = 'decision' })
|
||||
changes = @([ordered]@{ path = 'repo-relative changed file'; action = 'created | modified | deleted'; purpose = 'reason' })
|
||||
validation = @([ordered]@{ id = 'compile | tests | review'; command = 'command or quality-skill path'; status = 'passed | failed | not-run'; details = 'non-empty evidence' })
|
||||
review = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-code-review'; version = 1 }
|
||||
outcome = 'completed | partial | failed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 0; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 0; 'items-evaluated' = 0 }
|
||||
}
|
||||
findings = @()
|
||||
suppressed = @()
|
||||
}
|
||||
'review-rounds' = @([ordered]@{
|
||||
round = 1
|
||||
outcome = 'clean | fixing | stalled | limit-reached'
|
||||
'gating-finding-ids' = @()
|
||||
})
|
||||
suppressed = @()
|
||||
remaining = @()
|
||||
}
|
||||
}
|
||||
}
|
||||
$request | ConvertTo-Json -Depth 12 |
|
||||
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
|
||||
}
|
||||
}
|
||||
|
||||
if ($ResultsDirectory) {
|
||||
$failures = [System.Collections.Generic.List[string]]::new()
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
|
||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): missing result file.") | Out-Null
|
||||
continue
|
||||
}
|
||||
try {
|
||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||
} catch {
|
||||
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
|
||||
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'implementationReport') {
|
||||
$failures.Add("$($case.id): implementationReport is missing.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$report = $result.implementationReport
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'plan', 'knowledge', 'changes', 'validation', 'review', 'review-rounds', 'suppressed', 'remaining')) {
|
||||
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
|
||||
$failures.Add("$($case.id): implementation report is missing '$requiredField'.") | Out-Null
|
||||
}
|
||||
}
|
||||
$reportedSkillId = if (
|
||||
$report.PSObject.Properties.Name -contains 'skill' -and
|
||||
$report.skill.PSObject.Properties.Name -contains 'id'
|
||||
) {
|
||||
[string]$report.skill.id
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($reportedSkillId -ne 'al-development') {
|
||||
$failures.Add("$($case.id): implementation report skill is '$reportedSkillId'.") | Out-Null
|
||||
}
|
||||
$summaryRequest = if (
|
||||
$report.PSObject.Properties.Name -contains 'summary' -and
|
||||
$report.summary.PSObject.Properties.Name -contains 'request'
|
||||
) {
|
||||
[string]$report.summary.request
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($summaryRequest)) {
|
||||
$failures.Add("$($case.id): summary.request is missing or empty.") | Out-Null
|
||||
}
|
||||
$reportedKind = if (
|
||||
$report.PSObject.Properties.Name -contains 'plan' -and
|
||||
$report.plan.PSObject.Properties.Name -contains 'kind'
|
||||
) {
|
||||
[string]$report.plan.kind
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($reportedKind -ne [string]$case.expectedKind) {
|
||||
$failures.Add("$($case.id): plan.kind '$reportedKind' does not match expected mode '$($case.expectedKind)'.") | Out-Null
|
||||
}
|
||||
$outcome = if ($report.PSObject.Properties.Name -contains 'outcome') { [string]$report.outcome } else { '' }
|
||||
if ($outcome -ne 'completed') {
|
||||
$failures.Add("$($case.id): implementation outcome is '$outcome'.") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$knowledgeEntries = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'knowledge') {
|
||||
$knowledgeEntries = @($report.knowledge)
|
||||
}
|
||||
if (-not $knowledgeEntries.Count) {
|
||||
$failures.Add("$($case.id): implementation report contains no knowledge entries.") | Out-Null
|
||||
}
|
||||
$usedKnowledge = @(
|
||||
$knowledgeEntries |
|
||||
Where-Object { $_.PSObject.Properties.Name -contains 'path' } |
|
||||
ForEach-Object { [string]$_.path }
|
||||
)
|
||||
if (@($usedKnowledge | Sort-Object -Unique).Count -ne $usedKnowledge.Count) {
|
||||
$failures.Add("$($case.id): knowledge contains duplicate paths.") | Out-Null
|
||||
}
|
||||
foreach ($entry in $knowledgeEntries) {
|
||||
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
|
||||
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
|
||||
if ([string]::IsNullOrWhiteSpace($path) -or
|
||||
[System.IO.Path]::IsPathRooted($path) -or
|
||||
@($path -split '/|\\') -contains '..' -or
|
||||
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): knowledge entry has a missing or invalid path '$path'.") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($usedFor)) {
|
||||
$failures.Add("$($case.id): knowledge entry '$path' has no used-for explanation.") | Out-Null
|
||||
}
|
||||
}
|
||||
$requiredKnowledge = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
|
||||
$matched = @($requiredKnowledge | Where-Object { $usedKnowledge -contains $_ }).Count
|
||||
$recall = if ($requiredKnowledge.Count) { $matched / $requiredKnowledge.Count } else { 1.0 }
|
||||
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
|
||||
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
|
||||
}
|
||||
$acceptedKnowledge = @(
|
||||
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
|
||||
ForEach-Object { [string]$_ } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
$acceptedUsed = @($usedKnowledge | Where-Object { $acceptedKnowledge -contains $_ }).Count
|
||||
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
|
||||
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
|
||||
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$validationEntries = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'validation') {
|
||||
$validationEntries = @($report.validation)
|
||||
}
|
||||
foreach ($requiredCheck in @($case.requiredChecks)) {
|
||||
$check = @($validationEntries | Where-Object {
|
||||
$_.PSObject.Properties.Name -contains 'id' -and [string]$_.id -eq $requiredCheck
|
||||
})
|
||||
$checkStatus = if ($check.Count -eq 1 -and $check[0].PSObject.Properties.Name -contains 'status') {
|
||||
[string]$check[0].status
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($check.Count -ne 1 -or $checkStatus -ne 'passed') {
|
||||
$failures.Add("$($case.id): required check '$requiredCheck' did not pass exactly once.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$command = if ($check[0].PSObject.Properties.Name -contains 'command') { [string]$check[0].command } else { '' }
|
||||
$details = if ($check[0].PSObject.Properties.Name -contains 'details') { [string]$check[0].details } else { '' }
|
||||
if ([string]::IsNullOrWhiteSpace($command) -or [string]::IsNullOrWhiteSpace($details)) {
|
||||
$failures.Add("$($case.id): required check '$requiredCheck' lacks command or evidence details.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
[object[]]$changes = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'changes') {
|
||||
$changes = @($report.changes)
|
||||
}
|
||||
if (-not $changes.Count) {
|
||||
$failures.Add("$($case.id): implementation report contains no changed files.") | Out-Null
|
||||
}
|
||||
|
||||
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') { [string]$result.workspaceRoot } else { '' }
|
||||
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
|
||||
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
|
||||
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
|
||||
} else {
|
||||
& git -C $workspaceRoot rev-parse --is-inside-work-tree 2>$null | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$actualChanges = @(
|
||||
@(& git -C $workspaceRoot diff --name-only HEAD) +
|
||||
@(& git -C $workspaceRoot ls-files --others --exclude-standard) |
|
||||
Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } |
|
||||
ForEach-Object { ([string]$_).Replace('\', '/') } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add("$($case.id): unable to read the workspace diff.") | Out-Null
|
||||
} else {
|
||||
$reportedChanges = @(
|
||||
$changes |
|
||||
Where-Object { $_.PSObject.Properties.Name -contains 'path' } |
|
||||
ForEach-Object {
|
||||
$path = ([string]$_.path).Replace('\', '/')
|
||||
if ([System.IO.Path]::IsPathRooted($path) -or @($path -split '/') -contains '..') {
|
||||
$failures.Add("$($case.id): changed path is not repository-relative: $path") | Out-Null
|
||||
}
|
||||
$path
|
||||
} |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
foreach ($path in @($reportedChanges | Where-Object { $actualChanges -notcontains $_ })) {
|
||||
$failures.Add("$($case.id): reported change is absent from the worktree diff: $path") | Out-Null
|
||||
}
|
||||
foreach ($path in @($actualChanges | Where-Object { $reportedChanges -notcontains $_ })) {
|
||||
$failures.Add("$($case.id): worktree change is absent from the implementation report: $path") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($report.PSObject.Properties.Name -notcontains 'review') {
|
||||
$failures.Add("$($case.id): complete post-implementation review is missing.") | Out-Null
|
||||
} else {
|
||||
$review = $report.review
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'findings', 'suppressed')) {
|
||||
if ($review.PSObject.Properties.Name -notcontains $requiredField) {
|
||||
$failures.Add("$($case.id): review is missing '$requiredField'.") | Out-Null
|
||||
}
|
||||
}
|
||||
[object[]]$reviewFindings = @()
|
||||
if ($review.PSObject.Properties.Name -contains 'findings') {
|
||||
$reviewFindings = @($review.findings)
|
||||
}
|
||||
$reviewOutcome = if ($review.PSObject.Properties.Name -contains 'outcome') { [string]$review.outcome } else { '' }
|
||||
if ($reviewOutcome -ne 'completed') {
|
||||
$failures.Add("$($case.id): post-implementation review outcome is '$reviewOutcome'.") | Out-Null
|
||||
}
|
||||
$gatingFindings = @(
|
||||
$reviewFindings |
|
||||
Where-Object {
|
||||
$_.PSObject.Properties.Name -contains 'severity' -and
|
||||
[string]$_.severity -in @('blocker', 'major')
|
||||
}
|
||||
)
|
||||
if ($gatingFindings.Count) {
|
||||
$failures.Add("$($case.id): post-implementation review has $($gatingFindings.Count) gating finding(s).") | Out-Null
|
||||
}
|
||||
}
|
||||
[object[]]$reviewRounds = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'review-rounds') {
|
||||
$reviewRounds = @($report.'review-rounds')
|
||||
}
|
||||
if (-not $reviewRounds.Count -or $reviewRounds.Count -gt $maximumReviewRounds) {
|
||||
$failures.Add("$($case.id): review-rounds must contain 1..$maximumReviewRounds entries.") | Out-Null
|
||||
} else {
|
||||
for ($index = 0; $index -lt $reviewRounds.Count; $index++) {
|
||||
$round = $reviewRounds[$index]
|
||||
$roundNumber = if ($round.PSObject.Properties.Name -contains 'round') { [int]$round.round } else { 0 }
|
||||
$roundOutcome = if ($round.PSObject.Properties.Name -contains 'outcome') { [string]$round.outcome } else { '' }
|
||||
if ($roundNumber -ne ($index + 1)) {
|
||||
$failures.Add("$($case.id): review round numbering is not contiguous.") | Out-Null
|
||||
}
|
||||
if ($roundOutcome -notin @('clean', 'fixing', 'stalled', 'limit-reached')) {
|
||||
$failures.Add("$($case.id): review round $roundNumber has invalid outcome '$roundOutcome'.") | Out-Null
|
||||
}
|
||||
[object[]]$roundGatingIds = @()
|
||||
if ($round.PSObject.Properties.Name -contains 'gating-finding-ids') {
|
||||
$roundGatingIds = @($round.'gating-finding-ids')
|
||||
}
|
||||
if ($roundOutcome -eq 'clean' -and $roundGatingIds.Count) {
|
||||
$failures.Add("$($case.id): clean review round $roundNumber must have no gating IDs.") | Out-Null
|
||||
}
|
||||
if ($roundOutcome -in @('fixing', 'stalled', 'limit-reached') -and -not $roundGatingIds.Count) {
|
||||
$failures.Add("$($case.id): review round $roundNumber outcome '$roundOutcome' requires gating IDs.") | Out-Null
|
||||
}
|
||||
if (@($roundGatingIds | Sort-Object -Unique).Count -ne $roundGatingIds.Count) {
|
||||
$failures.Add("$($case.id): review round $roundNumber has duplicate gating IDs.") | Out-Null
|
||||
}
|
||||
}
|
||||
if ([string]$reviewRounds[-1].outcome -ne 'clean') {
|
||||
$failures.Add("$($case.id): completed implementation must end with a clean review round.") | Out-Null
|
||||
}
|
||||
}
|
||||
[object[]]$remaining = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'remaining') {
|
||||
$remaining = @($report.remaining)
|
||||
}
|
||||
if ($remaining.Count) {
|
||||
$failures.Add("$($case.id): completed report still lists remaining work.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
if ($failures.Count) {
|
||||
Write-Host "Development fixture scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
|
||||
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Development fixture scoring PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
} else {
|
||||
Write-Host "Development fixture validation PASSED: $(@($manifest.cases).Count) cases; $fixtureBackedCount of $($capabilityIds.Count) capabilities have fixtures (minimum $minimumFixtureCoverage)."
|
||||
}
|
||||
421
tools/Test-DevelopmentGuidanceEvaluator.ps1
Normal file
421
tools/Test-DevelopmentGuidanceEvaluator.ps1
Normal file
|
|
@ -0,0 +1,421 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Deterministic, offline regressions for the guidance evaluator (no model/AL run).
|
||||
.DESCRIPTION
|
||||
Creates only a uniquely named .guidance-evaluator-regression-* directory below
|
||||
the current checkout, with standalone Git repositories and sibling runner
|
||||
artifacts. Removes that exact directory in finally; never uses the OS temp
|
||||
directory or cleans a caller-provided repository.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
|
||||
$evaluator = Join-Path $PSScriptRoot 'Test-DevelopmentGuidanceFixtures.ps1'
|
||||
$sourceRoot = (Get-Item -LiteralPath (Join-Path $PSScriptRoot '..')).FullName
|
||||
$scratch = Join-Path $sourceRoot ".guidance-evaluator-regression-$([guid]::NewGuid().ToString('N'))"
|
||||
$root = Join-Path $scratch 'knowledge-checkout'
|
||||
$article = 'microsoft/knowledge/performance/pair-findset-with-next-loop.md'
|
||||
$otherArticle = 'microsoft/knowledge/performance/findset-true-applies-updlock-on-read.md'
|
||||
$sample = 'microsoft/knowledge/performance/pair-findset-with-next-loop.good.al'
|
||||
$tests = [Collections.Generic.List[string]]::new()
|
||||
$script:scenarioNumber = 0
|
||||
$script:reportScenario = $null
|
||||
|
||||
function Set-TestJson($Path, $Value) {
|
||||
[IO.File]::WriteAllText($Path, ($Value | ConvertTo-Json -Depth 64))
|
||||
}
|
||||
|
||||
function Invoke-TestGit([string] $Directory, [string[]] $Arguments) {
|
||||
$output = @(& git --no-optional-locks -C $Directory @Arguments 2>&1)
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Regression Git setup failed.' }
|
||||
}
|
||||
|
||||
function Initialize-TestRepository([string] $Directory) {
|
||||
[IO.Directory]::CreateDirectory($Directory) | Out-Null
|
||||
Invoke-TestGit $Directory @('init', '--quiet')
|
||||
Invoke-TestGit $Directory @('config', 'user.email', 'guidance-fixture@example.invalid')
|
||||
Invoke-TestGit $Directory @('config', 'user.name', 'Guidance fixture')
|
||||
Invoke-TestGit $Directory @('config', 'commit.gpgSign', 'false')
|
||||
Invoke-TestGit $Directory @('config', 'core.autocrlf', 'false')
|
||||
[IO.File]::WriteAllText((Join-Path $Directory 'app.json'), '{"name":"Synthetic AL fixture","application":"28.0.0.0"}')
|
||||
[IO.File]::WriteAllText((Join-Path $Directory 'tracked.al'), 'codeunit 50100 Example {}')
|
||||
[IO.File]::WriteAllText((Join-Path $Directory '.gitignore'), "ignored.txt`nignored-directory/`n")
|
||||
Invoke-TestGit $Directory @('add', '.')
|
||||
Invoke-TestGit $Directory @('commit', '--quiet', '-m', 'Synthetic fixture baseline')
|
||||
}
|
||||
|
||||
function Invoke-EvaluatorTest([string] $Name, [string[]] $Arguments, [bool] $ShouldPass, [string] $Diagnostic = '') {
|
||||
$output = @(& pwsh -NoProfile -File $evaluator @Arguments 2>&1) -join "`n"
|
||||
$code = $LASTEXITCODE
|
||||
if (($code -eq 0) -ne $ShouldPass -or $output -notmatch $(if ($ShouldPass) { 'PASSED|captured' } else { 'FAILED' })) {
|
||||
throw "Regression '$Name' unexpected exit $code. $output"
|
||||
}
|
||||
if ($Diagnostic -and $output -notmatch [regex]::Escape($Diagnostic)) {
|
||||
throw "Regression '$Name' missing diagnostic '$Diagnostic'. $output"
|
||||
}
|
||||
if ($output -match 'MODEL_SECRET_SENTINEL') { throw "Regression '$Name' leaked model content." }
|
||||
$tests.Add($Name)
|
||||
}
|
||||
|
||||
function New-TestScenario([string] $Outcome = 'completed', [switch] $Unknown, [switch] $SecondCase) {
|
||||
$script:scenarioNumber++
|
||||
$directory = Join-Path $scratch "scenario-$script:scenarioNumber"
|
||||
[IO.Directory]::CreateDirectory($directory) | Out-Null
|
||||
$workspace = Join-Path $directory 'target'
|
||||
Initialize-TestRepository $workspace
|
||||
[IO.File]::WriteAllText((Join-Path $workspace 'untracked.txt'), 'existing untracked content')
|
||||
[IO.File]::WriteAllText((Join-Path $workspace 'ignored.txt'), 'existing ignored content')
|
||||
[IO.Directory]::CreateDirectory((Join-Path $workspace 'ignored-directory')) | Out-Null
|
||||
[IO.File]::WriteAllText((Join-Path $workspace 'ignored-directory\child.txt'), 'ignored child')
|
||||
$results = Join-Path $directory 'results'
|
||||
[IO.Directory]::CreateDirectory($results) | Out-Null
|
||||
$hasKnowledge = $Outcome -in @('completed', 'partial')
|
||||
$case = [ordered]@{
|
||||
id = 'synthetic-case'
|
||||
expectedKind = 'bug'
|
||||
expectedOutcome = $Outcome
|
||||
expectedUnknown = @($(if ($Unknown) { 'bc-version' }))
|
||||
requiresUnresolved = ($Outcome -eq 'partial' -or $Unknown.IsPresent)
|
||||
requiresMaterialUnresolved = ($Outcome -eq 'partial')
|
||||
'development-plan' = [ordered]@{ kind = 'bug'; request = 'Iterate the supplied filtered record set.' }
|
||||
context = [ordered]@{
|
||||
'bc-version' = $(if ($Unknown) { 'unknown' } else { '28' })
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
unknown = @($(if ($Unknown) { 'bc-version' }))
|
||||
}
|
||||
requiredKnowledge = @($(if ($hasKnowledge) { $article }))
|
||||
optionalKnowledge = @()
|
||||
}
|
||||
$manifest = [ordered]@{
|
||||
version = 1
|
||||
skill = 'microsoft/skills/development/al-development-plan.md'
|
||||
minimumKnowledgeRecall = 1.0
|
||||
minimumKnowledgePrecision = 1.0
|
||||
cases = @($case)
|
||||
}
|
||||
$result = [ordered]@{
|
||||
caseId = Get-GuidanceCaseId $case.id
|
||||
guidanceReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||
outcome = $Outcome
|
||||
summary = [ordered]@{ request = 'Iterate all selected records.'; kind = 'bug'; candidates = [int]$hasKnowledge; selected = [int]$hasKnowledge }
|
||||
context = $case.context
|
||||
knowledge = @($(if ($hasKnowledge) {
|
||||
[ordered]@{ path = $article; 'used-for' = 'Choose the multi-record reader.'; constraints = @('Use FindSet when iterating with Next.'); 'sample-paths' = @($sample) }
|
||||
}))
|
||||
'validation-considerations' = @([ordered]@{ id = 'all-selected'; reason = 'Preserve selection.'; evidence = 'Test all selected records and an excluded record.' })
|
||||
suppressed = @()
|
||||
unresolved = @($(if ($Outcome -eq 'partial') {
|
||||
if ($Unknown) { 'bc-version is unknown and materially affects the candidate; clarify before completing guidance.' }
|
||||
else { 'The caller cardinality decision remains materially unresolved.' }
|
||||
} elseif ($Unknown) { 'bc-version is unknown but immaterial: selected loop guidance applies to all versions.' }))
|
||||
}
|
||||
}
|
||||
if ($Outcome -in @('partial', 'failed')) { $result.guidanceReport.'outcome-reason' = 'Fixture intentionally leaves evaluation incomplete.' }
|
||||
$manifestPath = Join-Path $directory 'manifest.json'
|
||||
$mapPath = Join-Path $directory 'workspace-map.json'
|
||||
$map = [ordered]@{ 'synthetic-case' = $workspace }
|
||||
if ($SecondCase) {
|
||||
$second = $case | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||
$second.id = 'second-case'
|
||||
$manifest.cases += $second
|
||||
$secondWorkspace = Join-Path $directory 'second-target'
|
||||
Initialize-TestRepository $secondWorkspace
|
||||
$map[$second.id] = $secondWorkspace
|
||||
$secondResult = $result | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||
$secondResult.caseId = Get-GuidanceCaseId $second.id
|
||||
Set-TestJson (Join-Path $results "result-$($secondResult.caseId).json") $secondResult
|
||||
}
|
||||
Set-TestJson $manifestPath $manifest
|
||||
Set-TestJson $mapPath $map
|
||||
$resultPath = Join-Path $results "result-$($result.caseId).json"
|
||||
Set-TestJson $resultPath $result
|
||||
$baselinePath = Join-Path $directory 'baseline.json'
|
||||
$captureArgs = @('-Root', $root, '-ManifestPath', $manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $mapPath, '-BaselinePath', $baselinePath)
|
||||
$output = @(& pwsh -NoProfile -File $evaluator @captureArgs 2>&1) -join "`n"
|
||||
if ($LASTEXITCODE -ne 0) { throw "Baseline setup failed. $output" }
|
||||
return [ordered]@{
|
||||
directory = $directory; workspace = $workspace; resultPath = $resultPath; result = $result
|
||||
results = $results; manifest = $manifest; manifestPath = $manifestPath; mapPath = $mapPath
|
||||
baselinePath = $baselinePath; captureArgs = $captureArgs
|
||||
scoreArgs = @('-Root', $root, '-ManifestPath', $manifestPath, '-ResultsDirectory', $results, '-BaselinePath', $baselinePath, '-BaselineSha256', (Get-GuidanceHash $baselinePath))
|
||||
}
|
||||
}
|
||||
|
||||
function Test-ReportMutation([string] $Name, [scriptblock] $Change, [string] $Diagnostic) {
|
||||
if ($null -eq $script:reportScenario) { $script:reportScenario = New-TestScenario }
|
||||
$scenario = $script:reportScenario
|
||||
$result = $scenario.result | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||
& $Change $result
|
||||
Set-TestJson $scenario.resultPath $result
|
||||
Invoke-EvaluatorTest $Name $scenario.scoreArgs $false $Diagnostic
|
||||
}
|
||||
|
||||
try {
|
||||
[IO.Directory]::CreateDirectory($root) | Out-Null
|
||||
foreach ($reference in @($article, $otherArticle, $sample, $otherArticle.Replace('.md', '.good.al'),
|
||||
'microsoft/skills/development/al-development-plan.md', 'tools/Build-KnowledgeIndex.ps1')) {
|
||||
$destination = Join-Path $root $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
|
||||
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
|
||||
}
|
||||
Initialize-TestRepository $root
|
||||
|
||||
$publicManifestPath = Join-Path $sourceRoot 'evaluation\development-guidance-fixtures.json'
|
||||
$publicManifest = Read-GuidanceJson $publicManifestPath
|
||||
$publicRoot = Join-Path $scratch 'public-fixture-checkout'
|
||||
$publicReferences = @($publicManifest.skill, 'tools/Build-KnowledgeIndex.ps1', 'evaluation/development-guidance-fixtures.json') +
|
||||
@($publicManifest.cases | ForEach-Object { $_.requiredKnowledge; $_.optionalKnowledge })
|
||||
foreach ($reference in @($publicReferences | Sort-Object -Unique)) {
|
||||
$destination = Join-Path $publicRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
|
||||
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
|
||||
}
|
||||
$publicPrepared = Join-Path $scratch 'public-prepared'
|
||||
Invoke-EvaluatorTest 'public five-case manifest validation' @('-Root', $publicRoot) $true
|
||||
Invoke-EvaluatorTest 'public five-case manifest preparation' @('-Root', $publicRoot, '-PrepareDirectory', $publicPrepared) $true
|
||||
$initialCase = $publicManifest.cases | Where-Object id -eq 'synthetic-normal-initial-plan'
|
||||
$initialRequest = Read-GuidanceJson (Join-Path $publicPrepared "request-$(Get-GuidanceCaseId $initialCase.id).json")
|
||||
if ($initialRequest.'development-plan' -cne $initialCase.'development-plan') { throw 'Preparation truncated the serialized initial plan.' }
|
||||
$document = $initialRequest.'development-plan' | ConvertFrom-Json -AsHashtable
|
||||
if ($document.metadata.kind -ne 'bug' -or
|
||||
@('Root cause and design', 'Proposed fix', 'Affected files', 'Test strategy', 'Acceptance criteria' |
|
||||
Where-Object { $document.body -notmatch [regex]::Escape($_) }).Count) {
|
||||
throw 'Synthetic consumer boundary lost metadata or markdown plan sections.'
|
||||
}
|
||||
$tests.Add('serialized synthetic initial-plan boundary preserves full metadata and markdown body')
|
||||
|
||||
$scenario = New-TestScenario
|
||||
Invoke-EvaluatorTest 'unchanged target with ignored and untracked files passes' $scenario.scoreArgs $true
|
||||
Invoke-EvaluatorTest 'scoring itself leaves index and content unchanged' $scenario.scoreArgs $true
|
||||
Invoke-EvaluatorTest 'existing baseline cannot silently recapture' $scenario.captureArgs $false 'Baseline already exists'
|
||||
Invoke-EvaluatorTest 'scoring without baseline fails' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results) $false 'require BaselinePath'
|
||||
Invoke-EvaluatorTest 'scoring requires independently retained baseline digest' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath) $false 'runner-retained pre-run BaselineSha256'
|
||||
Invoke-EvaluatorTest 'tampered baseline digest fails' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath, '-BaselineSha256', ('0' * 64)) $false 'digest mismatch'
|
||||
|
||||
$prepare = Join-Path $scenario.directory 'prepared'
|
||||
Invoke-EvaluatorTest 'prepare outside roots with runner workspace binding' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', $prepare, '-WorkspaceMapPath', $scenario.mapPath) $true
|
||||
$prepared = Read-GuidanceJson (Join-Path $prepare "request-$($scenario.result.caseId).json")
|
||||
if ($prepared.repository -cne $scenario.workspace -or $prepared.Contains('expectedOutcome') -or $prepared.Contains('requiredKnowledge')) {
|
||||
throw 'Prepared request lost runner binding or exposed answers.'
|
||||
}
|
||||
Invoke-EvaluatorTest 'preparation never overwrites requests' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', $prepare) $false 'new or empty'
|
||||
foreach ($option in @('PrepareDirectory', 'BaselinePath', 'ResultsDirectory')) {
|
||||
$inside = Join-Path $scenario.workspace 'unsafe-artifact'
|
||||
$arguments = @('-Root', $root, '-ManifestPath', $scenario.manifestPath)
|
||||
if ($option -eq 'PrepareDirectory') { $arguments += @('-PrepareDirectory', $inside, '-WorkspaceMapPath', $scenario.mapPath) }
|
||||
elseif ($option -eq 'BaselinePath') { $arguments += @('-CaptureBaseline', '-BaselinePath', $inside, '-WorkspaceMapPath', $scenario.mapPath) }
|
||||
else { $arguments += @('-BaselinePath', $scenario.baselinePath, '-BaselineSha256', (Get-GuidanceHash $scenario.baselinePath), '-ResultsDirectory', $inside) }
|
||||
Invoke-EvaluatorTest "$option inside target rejected" $arguments $false 'outside target workspaces'
|
||||
}
|
||||
|
||||
foreach ($mutation in @('uncommitted', 'committed', 'empty-commit', 'staged', 'index-only', 'untracked', 'ignored', 'ignored-child', 'added', 'deleted', 'directory', 'ref', 'metadata')) {
|
||||
$scenario = New-TestScenario
|
||||
switch ($mutation) {
|
||||
'uncommitted' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed") }
|
||||
'committed' {
|
||||
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||
Invoke-TestGit $scenario.workspace @('add', 'tracked.al')
|
||||
Invoke-TestGit $scenario.workspace @('commit', '--quiet', '-m', 'Committed forbidden edit')
|
||||
}
|
||||
'staged' {
|
||||
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||
Invoke-TestGit $scenario.workspace @('add', 'tracked.al')
|
||||
}
|
||||
'empty-commit' { Invoke-TestGit $scenario.workspace @('commit', '--quiet', '--allow-empty', '-m', 'Forbidden empty commit') }
|
||||
'index-only' { Invoke-TestGit $scenario.workspace @('update-index', '--assume-unchanged', 'tracked.al') }
|
||||
'untracked' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'untracked.txt'), 'changed') }
|
||||
'ignored' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'ignored.txt'), 'changed') }
|
||||
'ignored-child' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'ignored-directory\child.txt'), 'changed') }
|
||||
'added' { [IO.File]::WriteAllText((Join-Path $scenario.workspace 'new.txt'), 'new ignored/untracked payload') }
|
||||
'deleted' { [IO.File]::Delete((Join-Path $scenario.workspace 'untracked.txt')) }
|
||||
'directory' { [IO.Directory]::CreateDirectory((Join-Path $scenario.workspace 'new-empty-directory')) | Out-Null }
|
||||
'ref' { Invoke-TestGit $scenario.workspace @('branch', 'new-reference') }
|
||||
'metadata' {
|
||||
$path = Join-Path $scenario.workspace 'tracked.al'
|
||||
[IO.File]::SetLastWriteTimeUtc($path, [IO.File]::GetLastWriteTimeUtc($path).AddSeconds(5))
|
||||
}
|
||||
}
|
||||
Invoke-EvaluatorTest "$mutation mutation fails" $scenario.scoreArgs $false 'identity/content changed'
|
||||
}
|
||||
|
||||
$scenario = New-TestScenario -SecondCase
|
||||
Invoke-EvaluatorTest 'two independently bound workspaces pass' $scenario.scoreArgs $true
|
||||
$map = Read-GuidanceJson $scenario.mapPath
|
||||
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||
$scenario.result.workspaceRoot = $map['second-case']
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'self-reported clean workspace cannot hide changed runner target' $scenario.scoreArgs $false 'identity/content changed'
|
||||
$scenario = New-TestScenario -SecondCase
|
||||
$scenario.result.workspaceRoot = (Read-GuidanceJson $scenario.mapPath)['second-case']
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'result workspace swapping rejected even when both are clean' $scenario.scoreArgs $false 'runner binding'
|
||||
$scenario = New-TestScenario -SecondCase
|
||||
$scenario.result.caseId = Get-GuidanceCaseId 'second-case'
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'result case swapping rejected' $scenario.scoreArgs $false 'caseId mismatch'
|
||||
$scenario = New-TestScenario
|
||||
$scenario.manifest.cases[0].expectedKind = 'feature'
|
||||
Set-TestJson $scenario.manifestPath $scenario.manifest
|
||||
Invoke-EvaluatorTest 'changed manifest rejected' $scenario.scoreArgs $false 'root or manifest'
|
||||
$scenario = New-TestScenario
|
||||
$baseline = Read-GuidanceJson $scenario.baselinePath
|
||||
$baseline.workspaces['synthetic-case'].caseId = 'case-00000000'
|
||||
Set-TestJson $scenario.baselinePath $baseline
|
||||
Invoke-EvaluatorTest 'baseline case binding checked even with matching digest' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath, '-BaselineSha256', (Get-GuidanceHash $scenario.baselinePath)) $false 'case identity mismatch'
|
||||
|
||||
foreach ($outcome in @('no-knowledge', 'partial', 'failed', 'not-applicable')) {
|
||||
$scenario = New-TestScenario $outcome
|
||||
Invoke-EvaluatorTest "honest $outcome is distinguishable and passes" $scenario.scoreArgs $true
|
||||
}
|
||||
$scenario = New-TestScenario 'partial' -Unknown
|
||||
Invoke-EvaluatorTest 'material unknown stays partial' $scenario.scoreArgs $true
|
||||
$scenario.result.guidanceReport.outcome = 'completed'
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'material unknown cannot silently complete' $scenario.scoreArgs $false 'expectedOutcome'
|
||||
$scenario = New-TestScenario 'completed' -Unknown
|
||||
Invoke-EvaluatorTest 'nonmaterial unknown can complete with explanation' $scenario.scoreArgs $true
|
||||
$scenario.result.guidanceReport.unresolved = @()
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'unknown cannot disappear from unresolved evidence' $scenario.scoreArgs $false 'unresolved gaps'
|
||||
$scenario = New-TestScenario 'no-knowledge' -Unknown
|
||||
Invoke-EvaluatorTest 'immaterial unknown and no-knowledge are not failures' $scenario.scoreArgs $true
|
||||
foreach ($outcome in @('partial', 'failed')) {
|
||||
$scenario = New-TestScenario $outcome
|
||||
$scenario.result.guidanceReport.Remove('outcome-reason')
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest "$outcome requires outcome-reason" $scenario.scoreArgs $false 'outcome-reason'
|
||||
}
|
||||
$scenario = New-TestScenario 'no-knowledge'
|
||||
$scenario.result.guidanceReport.outcome = 'completed'
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'no-knowledge is not completed-empty' $scenario.scoreArgs $false 'expectedOutcome'
|
||||
$scenario = New-TestScenario 'no-knowledge'
|
||||
$scenario.result.guidanceReport.knowledge = @(@{ path = $article; 'used-for' = 'filler'; constraints = @('filler'); 'sample-paths' = @() })
|
||||
$scenario.result.guidanceReport.summary.candidates = 1
|
||||
$scenario.result.guidanceReport.summary.selected = 1
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'no-knowledge cannot contain filler knowledge' $scenario.scoreArgs $false 'requires empty knowledge'
|
||||
|
||||
Test-ReportMutation 'invalid outcome enum' { param($r) $r.guidanceReport.outcome = 'success' } 'outcome enum'
|
||||
Test-ReportMutation 'missing report fields' { param($r) $r.guidanceReport.Remove('knowledge') } 'missing required field'
|
||||
Test-ReportMutation 'null report object' { param($r) $r.guidanceReport = $null } 'JSON object'
|
||||
Test-ReportMutation 'wrong skill version type' { param($r) $r.guidanceReport.skill.version = '1' } 'identity/version'
|
||||
Test-ReportMutation 'wrong skill id type' { param($r) $r.guidanceReport.skill.id = @('al-development-plan') } 'non-empty string'
|
||||
Test-ReportMutation 'wrong kind type' { param($r) $r.guidanceReport.summary.kind = @('bug') } 'non-empty string'
|
||||
Test-ReportMutation 'wrong summary type' { param($r) $r.guidanceReport.summary = @() } 'JSON object'
|
||||
Test-ReportMutation 'fractional count' { param($r) $r.guidanceReport.summary.candidates = 1.5 } 'non-negative integer'
|
||||
Test-ReportMutation 'negative count' { param($r) $r.guidanceReport.summary.selected = -1 } 'non-negative integer'
|
||||
Test-ReportMutation 'inconsistent selected count' { param($r) $r.guidanceReport.summary.selected = 0 } 'Summary counts'
|
||||
Test-ReportMutation 'candidate count below selected' { param($r) $r.guidanceReport.summary.candidates = 0 } 'Summary counts'
|
||||
Test-ReportMutation 'wrong context list type' { param($r) $r.guidanceReport.context.technologies = 'al' } 'JSON array'
|
||||
Test-ReportMutation 'missing constraints' { param($r) $r.guidanceReport.knowledge[0].constraints = @() } 'must not be empty'
|
||||
Test-ReportMutation 'non-string constraints' { param($r) $r.guidanceReport.knowledge[0].constraints = @(@{ body = 'MODEL_SECRET_SENTINEL' }) } 'non-empty string'
|
||||
Test-ReportMutation 'missing sample array' { param($r) $r.guidanceReport.knowledge[0].Remove('sample-paths') } 'missing required field'
|
||||
Test-ReportMutation 'duplicate knowledge' {
|
||||
param($r)
|
||||
$r.guidanceReport.knowledge += $r.guidanceReport.knowledge[0]
|
||||
$r.guidanceReport.summary.selected = 2
|
||||
$r.guidanceReport.summary.candidates = 2
|
||||
} 'Duplicate knowledge'
|
||||
Test-ReportMutation 'bad validation entry' { param($r) $r.guidanceReport.'validation-considerations'[0].evidence = $false } 'non-empty string'
|
||||
Test-ReportMutation 'invalid suppression shape' { param($r) $r.guidanceReport.suppressed = @(@{ path = $article; reason = 'configuration' }) } 'missing required field'
|
||||
Test-ReportMutation 'invalid unresolved shape' { param($r) $r.guidanceReport.unresolved = @(@{ candidate = $article }) } 'non-empty string'
|
||||
Test-ReportMutation 'invalid SHA provenance' { param($r) $r.guidanceReport.knowledge[0].sha = '0' * 40 } 'recorded live checkout'
|
||||
$scenario = New-TestScenario
|
||||
$scenario.result.guidanceReport.knowledge[0].sha = Invoke-GuidanceGit $root @('rev-parse', 'HEAD')
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'actual pinned knowledge SHA accepted' $scenario.scoreArgs $true
|
||||
|
||||
foreach ($badPath in @('../outside.md', '/absolute.md', 'C:/external.md',
|
||||
'microsoft\knowledge\performance\pair-findset-with-next-loop.md',
|
||||
'microsoft/knowledge/performance/../performance/pair-findset-with-next-loop.md',
|
||||
'https://example.invalid/article.md', 'microsoft/knowledge/performance/missing.md',
|
||||
'microsoft/skills/development/al-development-plan.md')) {
|
||||
$scenario = New-TestScenario
|
||||
$scenario.result.guidanceReport.knowledge[0].path = $badPath
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'unsafe/nonexistent/non-knowledge citation rejected' $scenario.scoreArgs $false
|
||||
}
|
||||
foreach ($badSample in @('../outside.al', $article, $otherArticle.Replace('.md', '.good.al'), 'microsoft/knowledge/performance/other.good.al', 'microsoft\knowledge\performance\pair-findset-with-next-loop.good.al')) {
|
||||
$scenario = New-TestScenario
|
||||
$scenario.result.guidanceReport.knowledge[0].'sample-paths' = @($badSample)
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'unsafe/nonexistent/non-sibling sample rejected' $scenario.scoreArgs $false
|
||||
}
|
||||
foreach ($json in @('{', 'null', '[]', '{"caseId":"MODEL_SECRET_SENTINEL","caseId":"duplicate"}', '{"caseId":true,}', '// comment')) {
|
||||
$scenario = New-TestScenario
|
||||
[IO.File]::WriteAllText($scenario.resultPath, $json)
|
||||
Invoke-EvaluatorTest 'malformed model result fails without runtime crash or content leakage' $scenario.scoreArgs $false
|
||||
}
|
||||
|
||||
$scenario = New-TestScenario
|
||||
$outside = Join-Path $scenario.directory 'outside'
|
||||
[IO.Directory]::CreateDirectory($outside) | Out-Null
|
||||
$link = Join-Path $scenario.workspace 'escape'
|
||||
$linkKind = if ($IsWindows) { 'Junction' } else { 'SymbolicLink' }
|
||||
New-Item -ItemType $linkKind -Path $link -Target $outside | Out-Null
|
||||
try {
|
||||
Invoke-EvaluatorTest 'target junction/symlink rejected instead of followed' $scenario.scoreArgs $false 'Links, junctions'
|
||||
Invoke-EvaluatorTest 'baseline capture rejects junction/symlink target children' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', (Join-Path $scenario.directory 'linked-baseline.json')) $false 'Links, junctions'
|
||||
Invoke-EvaluatorTest 'prepared directory cannot escape through junction' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', (Join-Path $link 'prepared'), '-WorkspaceMapPath', $scenario.mapPath) $false 'Links, junctions'
|
||||
} finally { Remove-Item -LiteralPath $link -Force }
|
||||
$scenario = New-TestScenario
|
||||
$external = Join-Path $scenario.directory 'external.txt'
|
||||
[IO.File]::WriteAllText($external, 'external hard-link target')
|
||||
$hardLink = Join-Path $scenario.workspace 'hard-link.txt'
|
||||
New-Item -ItemType HardLink -Path $hardLink -Target $external | Out-Null
|
||||
try {
|
||||
Invoke-EvaluatorTest 'hard-link escape rejected' $scenario.scoreArgs $false 'Links, junctions'
|
||||
} finally { Remove-Item -LiteralPath $hardLink -Force }
|
||||
$scenario = New-TestScenario
|
||||
$externalArticle = Join-Path $scenario.directory 'outside.md'
|
||||
[IO.File]::Copy((Join-Path $root $article.Replace('/', [IO.Path]::DirectorySeparatorChar)), $externalArticle)
|
||||
$articleLink = Join-Path $root 'custom\knowledge\performance'
|
||||
[IO.Directory]::CreateDirectory((Split-Path $articleLink -Parent)) | Out-Null
|
||||
New-Item -ItemType $linkKind -Path $articleLink -Target $scenario.directory | Out-Null
|
||||
try {
|
||||
$rejected = $false
|
||||
try { $null = Resolve-GuidanceReference $root 'custom/knowledge/performance/outside.md' -Knowledge }
|
||||
catch { $rejected = $_.Exception.Message -match 'Links, junctions' }
|
||||
if (-not $rejected) { throw 'Linked knowledge reference was followed.' }
|
||||
$tests.Add('knowledge junction/symlink reference rejected')
|
||||
$rejected = $false
|
||||
try { $null = Resolve-GuidanceReference $root 'custom/knowledge/performance/pair-findset-with-next-loop.good.al' -Article 'custom/knowledge/performance/pair-findset-with-next-loop.md' }
|
||||
catch { $rejected = $_.Exception.Message -match 'Links, junctions' }
|
||||
if (-not $rejected) { throw 'Linked sample reference was followed.' }
|
||||
$tests.Add('sample junction/symlink reference rejected')
|
||||
} finally { Remove-Item -LiteralPath $articleLink -Force }
|
||||
$normativePath = Join-Path $root 'microsoft\knowledge\performance\single-normative-section.md'
|
||||
foreach ($heading in @('Best Practice', 'Anti Pattern')) {
|
||||
[IO.File]::WriteAllText($normativePath, "---`ndomain: performance`n---`n## Description`nSynthetic contract fixture.`n## $heading`nSynthetic normative constraint.`n")
|
||||
$null = Resolve-GuidanceReference $root 'microsoft/knowledge/performance/single-normative-section.md' -Knowledge
|
||||
$tests.Add("Knowledge article with only $heading accepted")
|
||||
}
|
||||
[IO.File]::Delete($normativePath)
|
||||
$scenario = New-TestScenario
|
||||
$linkedRoot = Join-Path $scratch 'linked-knowledge-checkout'
|
||||
Invoke-TestGit $root @('worktree', 'add', '--quiet', '--detach', $linkedRoot, 'HEAD')
|
||||
$linkedBaseline = Join-Path $scenario.directory 'linked-root-baseline.json'
|
||||
Invoke-EvaluatorTest 'linked knowledge checkout baseline capture' @('-Root', $linkedRoot, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', $linkedBaseline) $true
|
||||
Invoke-EvaluatorTest 'unchanged linked knowledge checkout scoring' @('-Root', $linkedRoot, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $linkedBaseline, '-BaselineSha256', (Get-GuidanceHash $linkedBaseline)) $true
|
||||
$scenario = New-TestScenario
|
||||
$map = Read-GuidanceJson $scenario.mapPath
|
||||
$map['synthetic-case'] = $linkedRoot
|
||||
Set-TestJson $scenario.mapPath $map
|
||||
Invoke-EvaluatorTest 'linked target checkout rejected as external Git storage' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', (Join-Path $scenario.directory 'external-git-baseline.json')) $false 'standalone repositories'
|
||||
$scenario = New-TestScenario
|
||||
[IO.File]::AppendAllText((Join-Path $root $sample.Replace('/', [IO.Path]::DirectorySeparatorChar)), "`n// changed knowledge sample")
|
||||
Invoke-EvaluatorTest 'knowledge checkout changes rejected' $scenario.scoreArgs $false 'Knowledge checkout identity/content changed'
|
||||
Write-Host "Development guidance evaluator regressions PASSED: $($tests.Count) checks."
|
||||
} finally {
|
||||
if (Test-Path -LiteralPath $scratch) {
|
||||
# Only our own uniquely named directory; links made by tests are removed above.
|
||||
Remove-Item -LiteralPath $scratch -Recurse -Force
|
||||
}
|
||||
}
|
||||
|
|
@ -1,352 +1,215 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates, prepares, and scores read-only AL development-guidance fixtures.
|
||||
.DESCRIPTION
|
||||
Capture runner-owned evidence BEFORE invoking an agent:
|
||||
-CaptureBaseline -WorkspaceMapPath <json> -BaselinePath <new-json>
|
||||
The workspace map is {"manifest-case-id":"absolute-standalone-git-root",...}.
|
||||
Score AFTER the agent finishes:
|
||||
-BaselinePath <json> -BaselineSha256 <runner-retained-digest> -ResultsDirectory <directory>
|
||||
Preparation (-PrepareDirectory) is independent; supply -WorkspaceMapPath to
|
||||
include runner-selected target paths. Never derive target paths from results.
|
||||
|
||||
Baseline, map, prepared requests, and results must be outside all targets and
|
||||
the knowledge checkout. Keep the baseline runner-only; retain the printed
|
||||
SHA256 and pass -BaselineSha256 when scoring to detect baseline tampering.
|
||||
Capture never overwrites an existing baseline. The runner must protect this
|
||||
script, the baseline/digest and its invocation from the agent.
|
||||
|
||||
This compares before/after evidence, NOT an OS sandbox or a write monitor.
|
||||
It cannot detect reverted transient writes, prove that articles were opened,
|
||||
or validate semantic faithfulness of prose. Files, directories, hashes,
|
||||
stable metadata, Git HEAD/refs/index and ignored/untracked files are compared.
|
||||
Links/reparse points, hard links, alternate data streams, external Git
|
||||
storage in targets, submodules and sparse checkouts are rejected rather than
|
||||
followed. Run in quiescent repositories. The knowledge checkout may itself
|
||||
be a linked Git worktree; its Git storage identity is recorded explicitly.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
[string] $Root = (Join-Path $PSScriptRoot '..'),
|
||||
[string] $ManifestPath,
|
||||
[string] $PrepareDirectory,
|
||||
[string] $ResultsDirectory
|
||||
[string] $ResultsDirectory,
|
||||
[switch] $CaptureBaseline,
|
||||
[string] $BaselinePath,
|
||||
[string] $BaselineSha256,
|
||||
[string] $WorkspaceMapPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $ManifestPath) {
|
||||
$ManifestPath = Join-Path $Root 'evaluation/development-guidance-fixtures.json'
|
||||
}
|
||||
|
||||
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
|
||||
$problems = [System.Collections.Generic.List[string]]::new()
|
||||
$validKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
||||
|
||||
function Get-ModelCaseId {
|
||||
param([string] $ManifestId)
|
||||
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
|
||||
$hash = $sha.ComputeHash($bytes)
|
||||
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
|
||||
return "case-$token"
|
||||
} finally {
|
||||
$sha.Dispose()
|
||||
try {
|
||||
if ($CaptureBaseline -and ($ResultsDirectory -or $PrepareDirectory)) {
|
||||
throw 'CaptureBaseline is a separate pre-run operation.'
|
||||
}
|
||||
}
|
||||
|
||||
function Get-PlanRequest {
|
||||
param([object] $Plan)
|
||||
|
||||
if ($Plan.PSObject.Properties.Name -contains 'request' -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Plan.request)) {
|
||||
return [string]$Plan.request
|
||||
if (($CaptureBaseline -or $ResultsDirectory) -and -not $BaselinePath) {
|
||||
throw 'Capture and scoring require BaselinePath.'
|
||||
}
|
||||
if ($Plan.PSObject.Properties.Name -contains 'format' -and
|
||||
[string]$Plan.format -eq 'BCFIX-HANDOFF' -and
|
||||
$Plan.PSObject.Properties.Name -contains 'nextStep') {
|
||||
$issue = if ($Plan.PSObject.Properties.Name -contains 'issue') { [string]$Plan.issue } else { 'unknown' }
|
||||
return "Continue BCFIX issue #${issue}: $($Plan.nextStep)"
|
||||
if ($ResultsDirectory -and -not $BaselineSha256) {
|
||||
throw 'Scoring requires the runner-retained pre-run BaselineSha256.'
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
if ($manifest.version -ne 1) {
|
||||
$problems.Add("Unsupported guidance fixture version: $($manifest.version)") | Out-Null
|
||||
}
|
||||
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||
$threshold = [double]$manifest.$thresholdName
|
||||
if ($threshold -lt 0 -or $threshold -gt 1) {
|
||||
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
|
||||
if ($CaptureBaseline -and -not $WorkspaceMapPath) {
|
||||
throw 'Capture requires a runner-owned WorkspaceMapPath.'
|
||||
}
|
||||
}
|
||||
|
||||
$skillPath = [string]$manifest.skill
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
|
||||
$problems.Add("Guidance skill does not exist: $skillPath") | Out-Null
|
||||
}
|
||||
|
||||
$seenIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$id = [string]$case.id
|
||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
|
||||
} elseif (-not $seenIds.Add($id)) {
|
||||
$problems.Add("Duplicate fixture id: $id") | Out-Null
|
||||
if ($ResultsDirectory -and ($WorkspaceMapPath -or $PrepareDirectory)) {
|
||||
throw 'Scoring uses only the recorded workspace map; run preparation separately.'
|
||||
}
|
||||
if ($case.PSObject.Properties.Name -notcontains 'development-plan') {
|
||||
$problems.Add("${id}: development-plan is required.") | Out-Null
|
||||
continue
|
||||
$Root = Get-GuidanceSafePath $Root -Directory
|
||||
if (-not $ManifestPath) { $ManifestPath = Join-Path $Root 'evaluation\development-guidance-fixtures.json' }
|
||||
$ManifestPath = Get-GuidanceSafePath $ManifestPath -File
|
||||
$manifest = Read-GuidanceJson $ManifestPath
|
||||
Assert-GuidanceManifest $manifest $Root
|
||||
$caseIds = @($manifest.cases | ForEach-Object { $_.id })
|
||||
$workspaces = [ordered]@{}
|
||||
$baseline = $null
|
||||
if ($WorkspaceMapPath) {
|
||||
$WorkspaceMapPath = Get-GuidanceSafePath $WorkspaceMapPath -File
|
||||
$map = Read-GuidanceJson $WorkspaceMapPath
|
||||
Assert-GuidanceObject $map 'workspace map'
|
||||
if ($map.Count -ne $caseIds.Count) { throw 'Workspace map must bind exactly every manifest case.' }
|
||||
foreach ($id in $caseIds) {
|
||||
Assert-GuidanceString $map[$id] 'workspace map value'
|
||||
if (-not [IO.Path]::IsPathFullyQualified($map[$id])) { throw 'Workspace roots must be absolute.' }
|
||||
$workspaces[$id] = Get-GuidanceSafePath $map[$id] -Directory
|
||||
}
|
||||
}
|
||||
$plan = $case.'development-plan'
|
||||
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
|
||||
[string]$case.expectedKind
|
||||
} else {
|
||||
''
|
||||
if ($ResultsDirectory) {
|
||||
$BaselinePath = Get-GuidanceSafePath $BaselinePath -File
|
||||
if ($BaselineSha256 -cnotmatch '^[0-9A-Fa-f]{64}$') {
|
||||
throw 'BaselineSha256 must be a SHA256 digest.'
|
||||
}
|
||||
if ((Get-GuidanceHash $BaselinePath) -ne $BaselineSha256) {
|
||||
throw 'Runner baseline digest mismatch.'
|
||||
}
|
||||
$baseline = Read-GuidanceJson $BaselinePath
|
||||
Assert-GuidanceObject $baseline 'baseline' @('version', 'kind', 'root', 'manifestPath', 'manifestSha256', 'workspaces', 'rootSnapshot')
|
||||
if ($baseline.version -ne 1 -or $baseline.kind -cne 'bcquality-guidance-runner-baseline') {
|
||||
throw 'Unsupported runner baseline.'
|
||||
}
|
||||
if ($baseline.root -cne $Root -or $baseline.manifestPath -cne $ManifestPath -or
|
||||
$baseline.manifestSha256 -ne (Get-GuidanceHash $ManifestPath)) {
|
||||
throw 'Runner baseline does not match the knowledge root or manifest.'
|
||||
}
|
||||
Assert-GuidanceObject $baseline.workspaces 'baseline workspaces'
|
||||
if ($baseline.workspaces.Count -ne $caseIds.Count) { throw 'Runner baseline case set mismatch.' }
|
||||
foreach ($id in $caseIds) {
|
||||
$entry = $baseline.workspaces[$id]
|
||||
Assert-GuidanceObject $entry 'baseline workspace entry' @('caseId', 'root', 'snapshot')
|
||||
if ($entry.caseId -cne (Get-GuidanceCaseId $id)) { throw 'Runner baseline case identity mismatch.' }
|
||||
$workspaces[$id] = Get-GuidanceSafePath $entry.root -Directory
|
||||
}
|
||||
}
|
||||
if ($validKinds -notcontains $expectedKind) {
|
||||
$problems.Add("${id}: expectedKind is invalid.") | Out-Null
|
||||
}
|
||||
$isBcfixHandoff = (
|
||||
$plan.PSObject.Properties.Name -contains 'format' -and
|
||||
[string]$plan.format -eq 'BCFIX-HANDOFF'
|
||||
)
|
||||
if ($isBcfixHandoff) {
|
||||
$requiredHandoffFields = @(
|
||||
'version', 'issue', 'phase', 'status', 'baton', 'rootCause',
|
||||
'harnessMap', 'iterationsUsed', 'filesCommitted', 'lastTestResult',
|
||||
'deadEnds', 'nextStep'
|
||||
)
|
||||
foreach ($field in $requiredHandoffFields) {
|
||||
if ($plan.PSObject.Properties.Name -notcontains $field) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF is missing '$field'.") | Out-Null
|
||||
$protectedRoots = @($Root) + @($workspaces.Values)
|
||||
for ($i = 0; $i -lt $protectedRoots.Count; $i++) {
|
||||
for ($j = $i + 1; $j -lt $protectedRoots.Count; $j++) {
|
||||
if ((Test-GuidanceWithin $protectedRoots[$i] $protectedRoots[$j]) -or
|
||||
(Test-GuidanceWithin $protectedRoots[$j] $protectedRoots[$i])) {
|
||||
throw 'Knowledge checkout and case workspaces must be distinct, non-overlapping roots.'
|
||||
}
|
||||
}
|
||||
$handoffVersion = if ($plan.PSObject.Properties.Name -contains 'version') { [int]$plan.version } else { 0 }
|
||||
$handoffPhase = if ($plan.PSObject.Properties.Name -contains 'phase') { [string]$plan.phase } else { '' }
|
||||
$handoffStatus = if ($plan.PSObject.Properties.Name -contains 'status') { [string]$plan.status } else { '' }
|
||||
if ($handoffVersion -ne 1) {
|
||||
$problems.Add("${id}: only BCFIX-HANDOFF version 1 is supported.") | Out-Null
|
||||
}
|
||||
if ($handoffPhase -notin @('plan', 'baseline', 'implement', 'pr')) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF phase is invalid.") | Out-Null
|
||||
}
|
||||
if ($handoffStatus -notin @('in-progress', 'paused', 'done')) {
|
||||
$problems.Add("${id}: BCFIX-HANDOFF status is invalid.") | Out-Null
|
||||
}
|
||||
} elseif (
|
||||
$plan.PSObject.Properties.Name -notcontains 'kind' -or
|
||||
$validKinds -notcontains [string]$plan.kind
|
||||
) {
|
||||
$problems.Add("${id}: development-plan.kind is invalid.") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace((Get-PlanRequest $plan))) {
|
||||
$problems.Add("${id}: development-plan must provide request or BCFIX nextStep.") | Out-Null
|
||||
}
|
||||
|
||||
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
|
||||
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
|
||||
}
|
||||
foreach ($reference in $expectedKnowledge) {
|
||||
$reference = [string]$reference
|
||||
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
|
||||
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
|
||||
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
|
||||
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
|
||||
foreach ($artifact in @($BaselinePath, $WorkspaceMapPath, $PrepareDirectory, $ResultsDirectory)) {
|
||||
if ($artifact) {
|
||||
$safeArtifact = Get-GuidanceSafePath $artifact -AllowMissing
|
||||
foreach ($protectedRoot in $protectedRoots) {
|
||||
if (Test-GuidanceWithin $safeArtifact $protectedRoot) {
|
||||
throw 'Runner artifacts must be outside target workspaces and the knowledge checkout.'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Development guidance fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
if ($CaptureBaseline) {
|
||||
$BaselinePath = Get-GuidanceSafePath $BaselinePath -AllowMissing
|
||||
if (Test-Path -LiteralPath $BaselinePath) { throw 'Baseline already exists; capture never overwrites evidence.' }
|
||||
$snapshots = [ordered]@{}
|
||||
foreach ($id in $caseIds) {
|
||||
$snapshots[$id] = [ordered]@{
|
||||
caseId = Get-GuidanceCaseId $id
|
||||
root = $workspaces[$id]
|
||||
snapshot = Get-GuidanceSnapshot $workspaces[$id] -Target
|
||||
}
|
||||
}
|
||||
$record = [ordered]@{
|
||||
kind = 'bcquality-guidance-runner-baseline'
|
||||
version = 1
|
||||
root = $Root
|
||||
manifestPath = $ManifestPath
|
||||
manifestSha256 = Get-GuidanceHash $ManifestPath
|
||||
rootSnapshot = Get-GuidanceSnapshot $Root
|
||||
workspaces = $snapshots
|
||||
}
|
||||
Write-GuidanceNewJson $BaselinePath $record
|
||||
Write-Host "Guidance baseline captured. Runner SHA256: $(Get-GuidanceHash $BaselinePath)"
|
||||
} elseif ($PrepareDirectory) {
|
||||
$PrepareDirectory = Get-GuidanceSafePath $PrepareDirectory -AllowMissing
|
||||
if ((Test-Path -LiteralPath $PrepareDirectory) -and
|
||||
@(Get-ChildItem -LiteralPath $PrepareDirectory -Force).Count) {
|
||||
throw 'PrepareDirectory must be new or empty; existing requests/evidence are never overwritten.'
|
||||
}
|
||||
[IO.Directory]::CreateDirectory($PrepareDirectory) | Out-Null
|
||||
# The index builder walks recursively; reject linked corpus paths first.
|
||||
Assert-GuidanceTree $Root
|
||||
& (Join-Path $Root 'tools\Build-KnowledgeIndex.ps1') -BCQualityRoot $Root `
|
||||
-IndexPath (Join-Path $PrepareDirectory 'knowledge-index.json') | Out-Null
|
||||
$skillInstructions = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $manifest.skill))
|
||||
foreach ($case in $manifest.cases) {
|
||||
$modelId = Get-GuidanceCaseId $case.id
|
||||
$request = [ordered]@{
|
||||
protocol = 'Run the supplied read-only skill on the runner-assigned repository and existing plan. Return only caseId and guidanceReport. The runner captures evidence before invocation; do not capture or modify it. Do not create artifacts in the target or knowledge checkout.'
|
||||
caseId = $modelId
|
||||
skill = $manifest.skill
|
||||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||
knowledgeRoot = $Root
|
||||
'task-context' = [ordered]@{
|
||||
goal = Get-GuidancePlanRequest $case.'development-plan'
|
||||
'inputs-available' = @('development-plan', 'repository')
|
||||
'bc-version' = $case.context.'bc-version'
|
||||
technologies = $case.context.technologies
|
||||
countries = $case.context.countries
|
||||
'application-area' = $case.context.'application-area'
|
||||
}
|
||||
'development-plan' = $case.'development-plan'
|
||||
resultSchema = Get-GuidanceResultSchema $modelId
|
||||
}
|
||||
if ($workspaces.Count) { $request.repository = $workspaces[$case.id] }
|
||||
Write-GuidanceNewJson (Join-Path $PrepareDirectory "request-$modelId.json") $request
|
||||
}
|
||||
Write-Host "Development guidance preparation PASSED: $($caseIds.Count) case(s)."
|
||||
} elseif ($ResultsDirectory) {
|
||||
$ResultsDirectory = Get-GuidanceSafePath $ResultsDirectory -Directory
|
||||
$failures = [Collections.Generic.List[string]]::new()
|
||||
if (-not (Test-GuidanceSnapshotEqual $baseline.rootSnapshot (Get-GuidanceSnapshot $Root))) {
|
||||
$failures.Add('Knowledge checkout identity/content changed after baseline capture.')
|
||||
}
|
||||
foreach ($case in $manifest.cases) {
|
||||
$id = $case.id
|
||||
try {
|
||||
if (-not (Test-GuidanceSnapshotEqual $baseline.workspaces[$id].snapshot `
|
||||
(Get-GuidanceSnapshot $workspaces[$id] -Target))) {
|
||||
throw 'Target repository identity/content changed after baseline capture.'
|
||||
}
|
||||
$resultPath = Get-GuidanceSafePath (Join-Path $ResultsDirectory "result-$(Get-GuidanceCaseId $id).json") -File
|
||||
$result = Read-GuidanceJson $resultPath
|
||||
Assert-GuidanceResult $result $case $manifest $Root $workspaces[$id]
|
||||
} catch {
|
||||
# Only evaluator-authored diagnostics are printed, never model or file contents.
|
||||
$failures.Add("${id}: $(Get-GuidanceDiagnostic $_)")
|
||||
}
|
||||
}
|
||||
if ($failures.Count) {
|
||||
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):"
|
||||
$failures | ForEach-Object { Write-Host " - $_" }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Development guidance scoring PASSED: $($caseIds.Count) case(s)."
|
||||
} else {
|
||||
Write-Host "Development guidance fixture validation PASSED: $($caseIds.Count) case(s)."
|
||||
}
|
||||
} catch {
|
||||
Write-Host "Development guidance FAILED: $(Get-GuidanceDiagnostic $_)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ($PrepareDirectory) {
|
||||
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-guidance-evaluation'
|
||||
if (Test-Path -LiteralPath $PrepareDirectory) {
|
||||
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
|
||||
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
|
||||
throw "PrepareDirectory is not empty and is not a BCQuality development-guidance evaluation directory: $PrepareDirectory"
|
||||
}
|
||||
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
|
||||
Get-ChildItem -LiteralPath $PrepareDirectory -File |
|
||||
Where-Object {
|
||||
$_.Name -eq 'knowledge-index.json' -or
|
||||
$_.Name -like 'request-*.json' -or
|
||||
$_.Name -like 'result-*.json'
|
||||
} |
|
||||
Remove-Item -Force
|
||||
}
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
|
||||
}
|
||||
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development-guidance evaluation directory' -Encoding UTF8
|
||||
|
||||
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
||||
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
[ordered]@{
|
||||
protocol = 'Run the supplied AL development-plan skill read-only in a clean fixture repository. Return only guidanceReport using the supplied schema and retain the repository for read-only verification.'
|
||||
caseId = $modelId
|
||||
skill = $skillPath
|
||||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = 'knowledge-index.json'
|
||||
'task-context' = [ordered]@{
|
||||
goal = Get-PlanRequest $case.'development-plan'
|
||||
'inputs-available' = @('development-plan', 'repository')
|
||||
technologies = @($case.context.technologies)
|
||||
countries = @($case.context.countries)
|
||||
'application-area' = @($case.context.'application-area')
|
||||
}
|
||||
'development-plan' = $case.'development-plan'
|
||||
resultSchema = [ordered]@{
|
||||
caseId = $modelId
|
||||
workspaceRoot = 'absolute path to the retained clean fixture repository'
|
||||
guidanceReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
||||
'outcome-reason' = 'required for partial or failed'
|
||||
summary = [ordered]@{
|
||||
request = 'planned change'
|
||||
kind = 'feature | bug | refactor | upgrade | maintenance'
|
||||
candidates = 0
|
||||
selected = 0
|
||||
}
|
||||
context = [ordered]@{
|
||||
'bc-version' = 'resolved target or unknown'
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
unknown = @()
|
||||
}
|
||||
knowledge = @([ordered]@{
|
||||
path = 'repo-relative article path'
|
||||
sha = 'optional commit sha'
|
||||
'used-for' = 'plan decision'
|
||||
constraints = @('faithful normative constraint')
|
||||
'sample-paths' = @()
|
||||
})
|
||||
'validation-considerations' = @([ordered]@{
|
||||
id = 'stable id'
|
||||
reason = 'why evidence is needed'
|
||||
evidence = 'evidence implementation should obtain'
|
||||
})
|
||||
suppressed = @()
|
||||
unresolved = @()
|
||||
}
|
||||
}
|
||||
} | ConvertTo-Json -Depth 15 |
|
||||
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
|
||||
}
|
||||
}
|
||||
|
||||
if ($ResultsDirectory) {
|
||||
$failures = [System.Collections.Generic.List[string]]::new()
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
|
||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): missing result file.") | Out-Null
|
||||
continue
|
||||
}
|
||||
try {
|
||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||
} catch {
|
||||
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
|
||||
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'guidanceReport') {
|
||||
$failures.Add("$($case.id): guidanceReport is missing.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$report = $result.guidanceReport
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
|
||||
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
|
||||
$failures.Add("$($case.id): guidance report is missing '$requiredField'.") | Out-Null
|
||||
}
|
||||
}
|
||||
$skillId = if (
|
||||
$report.PSObject.Properties.Name -contains 'skill' -and
|
||||
$report.skill.PSObject.Properties.Name -contains 'id'
|
||||
) { [string]$report.skill.id } else { '' }
|
||||
if ($skillId -ne 'al-development-plan') {
|
||||
$failures.Add("$($case.id): guidance skill is '$skillId'.") | Out-Null
|
||||
}
|
||||
if ($report.PSObject.Properties.Name -notcontains 'outcome' -or [string]$report.outcome -ne 'completed') {
|
||||
$failures.Add("$($case.id): guidance outcome is not completed.") | Out-Null
|
||||
}
|
||||
$kind = if (
|
||||
$report.PSObject.Properties.Name -contains 'summary' -and
|
||||
$report.summary.PSObject.Properties.Name -contains 'kind'
|
||||
) { [string]$report.summary.kind } else { '' }
|
||||
if ($kind -ne [string]$case.expectedKind) {
|
||||
$failures.Add("$($case.id): summary.kind '$kind' does not match the plan.") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$knowledgeEntries = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'knowledge') {
|
||||
$knowledgeEntries = @($report.knowledge)
|
||||
}
|
||||
$usedKnowledge = @()
|
||||
foreach ($entry in $knowledgeEntries) {
|
||||
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
|
||||
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
|
||||
[object[]]$constraints = @()
|
||||
if ($entry.PSObject.Properties.Name -contains 'constraints') {
|
||||
$constraints = @($entry.constraints)
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($path) -or
|
||||
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): invalid knowledge path '$path'.") | Out-Null
|
||||
} else {
|
||||
$usedKnowledge += $path
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($usedFor) -or -not $constraints.Count) {
|
||||
$failures.Add("$($case.id): '$path' lacks used-for or constraints.") | Out-Null
|
||||
}
|
||||
if ($entry.PSObject.Properties.Name -contains 'sample-paths') {
|
||||
foreach ($samplePath in @($entry.'sample-paths')) {
|
||||
$samplePath = [string]$samplePath
|
||||
if ([string]::IsNullOrWhiteSpace($samplePath) -or
|
||||
-not (Test-Path -LiteralPath (Join-Path $Root $samplePath) -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): invalid sample path '$samplePath'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$required = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
|
||||
$matched = @($required | Where-Object { $usedKnowledge -contains $_ }).Count
|
||||
$recall = if ($required.Count) { $matched / $required.Count } else { 1.0 }
|
||||
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
|
||||
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
|
||||
}
|
||||
$accepted = @(
|
||||
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
|
||||
ForEach-Object { [string]$_ } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
$acceptedUsed = @($usedKnowledge | Where-Object { $accepted -contains $_ }).Count
|
||||
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
|
||||
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
|
||||
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
|
||||
}
|
||||
|
||||
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') {
|
||||
[string]$result.workspaceRoot
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
|
||||
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
|
||||
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
|
||||
} else {
|
||||
$status = @(& git -C $workspaceRoot status --porcelain)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
|
||||
} elseif ($status.Count) {
|
||||
$failures.Add("$($case.id): guidance skill changed the target repository.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($failures.Count) {
|
||||
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
|
||||
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Development guidance scoring PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
} else {
|
||||
Write-Host "Development guidance fixture validation PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,240 +0,0 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates Microsoft Learn ingestion coverage and reports progress.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
[string] $CatalogPath,
|
||||
[string] $CoveragePath,
|
||||
[switch] $Json
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $CatalogPath) {
|
||||
$CatalogPath = Join-Path $Root 'coverage/microsoft-learn-developer-catalog.json'
|
||||
}
|
||||
if (-not $CoveragePath) {
|
||||
$CoveragePath = Join-Path $Root 'coverage/learn-coverage.json'
|
||||
}
|
||||
|
||||
foreach ($path in @($CatalogPath, $CoveragePath)) {
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
||||
throw "Coverage input not found: $path"
|
||||
}
|
||||
}
|
||||
|
||||
$catalog = Get-Content -LiteralPath $CatalogPath -Raw | ConvertFrom-Json
|
||||
$coverage = Get-Content -LiteralPath $CoveragePath -Raw | ConvertFrom-Json
|
||||
$problems = [System.Collections.Generic.List[string]]::new()
|
||||
|
||||
if ($catalog.version -ne 1) {
|
||||
$problems.Add("Unsupported catalog version: $($catalog.version)") | Out-Null
|
||||
}
|
||||
if ($coverage.version -ne 1) {
|
||||
$problems.Add("Unsupported coverage version: $($coverage.version)") | Out-Null
|
||||
}
|
||||
if ([string]$coverage.catalog -ne 'coverage/microsoft-learn-developer-catalog.json') {
|
||||
$problems.Add("Coverage catalog path must be coverage/microsoft-learn-developer-catalog.json.") | Out-Null
|
||||
}
|
||||
|
||||
$catalogUnits = @{}
|
||||
foreach ($unit in @($catalog.units)) {
|
||||
$uid = [string]$unit.uid
|
||||
if ($catalogUnits.ContainsKey($uid)) {
|
||||
$problems.Add("Duplicate catalog unit uid: $uid") | Out-Null
|
||||
} else {
|
||||
$catalogUnits[$uid] = $unit
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace([string]$unit.title)) {
|
||||
$problems.Add("${uid}: catalog unit title is empty.") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace([string]$unit.url) -or
|
||||
-not ([string]$unit.url).StartsWith('https://learn.microsoft.com/', [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
$problems.Add("${uid}: catalog unit URL is missing or not a Microsoft Learn URL.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$catalogModules = @{}
|
||||
foreach ($module in @($catalog.modules)) {
|
||||
$uid = [string]$module.uid
|
||||
if ($catalogModules.ContainsKey($uid)) {
|
||||
$problems.Add("Duplicate catalog module uid: $uid") | Out-Null
|
||||
} else {
|
||||
$catalogModules[$uid] = $module
|
||||
}
|
||||
foreach ($unitUid in @($module.unitUids)) {
|
||||
if (-not $catalogUnits.ContainsKey([string]$unitUid)) {
|
||||
$problems.Add("${uid}: references unknown unit '$unitUid'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$catalogPaths = @{}
|
||||
foreach ($path in @($catalog.learningPaths)) {
|
||||
$uid = [string]$path.uid
|
||||
if ($catalogPaths.ContainsKey($uid)) {
|
||||
$problems.Add("Duplicate catalog learning-path uid: $uid") | Out-Null
|
||||
} else {
|
||||
$catalogPaths[$uid] = $path
|
||||
}
|
||||
foreach ($moduleUid in @($path.moduleUids)) {
|
||||
if (-not $catalogModules.ContainsKey([string]$moduleUid)) {
|
||||
$problems.Add("${uid}: references unknown module '$moduleUid'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ([int]$catalog.counts.units -ne $catalogUnits.Count) {
|
||||
$problems.Add("Catalog unit count does not match units array.") | Out-Null
|
||||
}
|
||||
if ([int]$catalog.counts.modules -ne $catalogModules.Count) {
|
||||
$problems.Add("Catalog module count does not match modules array.") | Out-Null
|
||||
}
|
||||
if ([int]$catalog.counts.learningPaths -ne $catalogPaths.Count) {
|
||||
$problems.Add("Catalog learning-path count does not match learningPaths array.") | Out-Null
|
||||
}
|
||||
|
||||
foreach ($unit in @($catalog.units)) {
|
||||
foreach ($moduleUid in @($unit.moduleUids)) {
|
||||
if (-not $catalogModules.ContainsKey([string]$moduleUid)) {
|
||||
$problems.Add("$($unit.uid): references unknown module '$moduleUid'.") | Out-Null
|
||||
} elseif (@($catalogModules[[string]$moduleUid].unitUids) -notcontains [string]$unit.uid) {
|
||||
$problems.Add("$($unit.uid): module '$moduleUid' does not link back to the unit.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$validStatuses = @('in-progress', 'complete')
|
||||
$validDispositions = @('candidate', 'authored', 'covered-existing', 'rejected', 'deferred')
|
||||
$seenUnits = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
$seenOutcomes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
$articlePaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
$dispositionCounts = @{}
|
||||
foreach ($disposition in $validDispositions) {
|
||||
$dispositionCounts[$disposition] = 0
|
||||
}
|
||||
|
||||
foreach ($unitCoverage in @($coverage.units)) {
|
||||
$uid = [string]$unitCoverage.uid
|
||||
if (-not $seenUnits.Add($uid)) {
|
||||
$problems.Add("Duplicate coverage unit uid: $uid") | Out-Null
|
||||
}
|
||||
if (-not $catalogUnits.ContainsKey($uid)) {
|
||||
$problems.Add("Coverage unit is absent from the catalog: $uid") | Out-Null
|
||||
}
|
||||
|
||||
$reviewStatus = [string]$unitCoverage.reviewStatus
|
||||
if ($validStatuses -notcontains $reviewStatus) {
|
||||
$problems.Add("${uid}: invalid reviewStatus '$reviewStatus'.") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$outcomes = @()
|
||||
if ($unitCoverage.PSObject.Properties.Name -contains 'outcomes') {
|
||||
$outcomes = @($unitCoverage.outcomes)
|
||||
}
|
||||
if ($reviewStatus -eq 'in-progress' -and -not $outcomes.Count) {
|
||||
$problems.Add("${uid}: an in-progress unit must contain at least one outcome.") | Out-Null
|
||||
}
|
||||
|
||||
foreach ($outcome in $outcomes) {
|
||||
$id = [string]$outcome.id
|
||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
$problems.Add("${uid}: outcome id must be kebab-case: '$id'.") | Out-Null
|
||||
} elseif (-not $seenOutcomes.Add($id)) {
|
||||
$problems.Add("Duplicate outcome id: $id") | Out-Null
|
||||
}
|
||||
|
||||
$disposition = [string]$outcome.disposition
|
||||
if ($validDispositions -notcontains $disposition) {
|
||||
$problems.Add("${id}: invalid disposition '$disposition'.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$dispositionCounts[$disposition]++
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace([string]$outcome.title)) {
|
||||
$problems.Add("${id}: title is required.") | Out-Null
|
||||
}
|
||||
|
||||
if ($disposition -in @('candidate', 'authored', 'covered-existing')) {
|
||||
if ([string]::IsNullOrWhiteSpace([string]$outcome.domain)) {
|
||||
$problems.Add("${id}: domain is required for disposition '$disposition'.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
[object[]]$paths = @()
|
||||
if ($outcome.PSObject.Properties.Name -contains 'articlePaths') {
|
||||
$paths = @($outcome.articlePaths)
|
||||
}
|
||||
if ($disposition -in @('authored', 'covered-existing')) {
|
||||
if (-not $paths.Count) {
|
||||
$problems.Add("${id}: articlePaths is required for disposition '$disposition'.") | Out-Null
|
||||
}
|
||||
foreach ($relativePath in $paths) {
|
||||
$relativePath = [string]$relativePath
|
||||
if ($relativePath.Contains('\') -or -not $relativePath.EndsWith('.md')) {
|
||||
$problems.Add("${id}: article path must be a forward-slash .md path: $relativePath") | Out-Null
|
||||
continue
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $relativePath) -PathType Leaf)) {
|
||||
$problems.Add("${id}: article does not exist: $relativePath") | Out-Null
|
||||
}
|
||||
$articlePaths.Add($relativePath) | Out-Null
|
||||
}
|
||||
} elseif ($paths.Count) {
|
||||
$problems.Add("${id}: disposition '$disposition' must not set articlePaths.") | Out-Null
|
||||
}
|
||||
|
||||
$rationale = if ($outcome.PSObject.Properties.Name -contains 'rationale') {
|
||||
[string]$outcome.rationale
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($disposition -in @('rejected', 'deferred') -and
|
||||
[string]::IsNullOrWhiteSpace($rationale)) {
|
||||
$problems.Add("${id}: rationale is required for disposition '$disposition'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Learn coverage validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
|
||||
$complete = @($coverage.units | Where-Object reviewStatus -eq 'complete').Count
|
||||
$inProgress = @($coverage.units | Where-Object reviewStatus -eq 'in-progress').Count
|
||||
$summary = [ordered]@{
|
||||
catalogUnits = $catalogUnits.Count
|
||||
completeUnits = $complete
|
||||
inProgressUnits = $inProgress
|
||||
unreviewedUnits = $catalogUnits.Count - $complete - $inProgress
|
||||
candidateOutcomes = $dispositionCounts.candidate
|
||||
authoredOutcomes = $dispositionCounts.authored
|
||||
coveredExistingOutcomes = $dispositionCounts.'covered-existing'
|
||||
rejectedOutcomes = $dispositionCounts.rejected
|
||||
deferredOutcomes = $dispositionCounts.deferred
|
||||
referencedArticles = $articlePaths.Count
|
||||
}
|
||||
|
||||
if ($Json) {
|
||||
$summary | ConvertTo-Json
|
||||
} else {
|
||||
$message = (
|
||||
'Learn coverage: {0} units; {1} complete, {2} in progress, {3} unreviewed; ' +
|
||||
'{4} candidate outcomes, {5} authored outcomes, {6} existing-coverage outcomes.'
|
||||
) -f @(
|
||||
$summary.catalogUnits,
|
||||
$summary.completeUnits,
|
||||
$summary.inProgressUnits,
|
||||
$summary.unreviewedUnits,
|
||||
$summary.candidateOutcomes,
|
||||
$summary.authoredOutcomes,
|
||||
$summary.coveredExistingOutcomes
|
||||
)
|
||||
Write-Host $message
|
||||
}
|
||||
|
|
@ -1,262 +0,0 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Updates the committed Microsoft Learn Business Central developer catalog.
|
||||
|
||||
.DESCRIPTION
|
||||
Filters a Microsoft Learn catalog payload to modules tagged for both
|
||||
dynamics-business-central and developer. The output is intentionally only
|
||||
source metadata; editorial dispositions live in coverage/learn-coverage.json.
|
||||
|
||||
The legacy unauthenticated catalog endpoint remains the default while it is
|
||||
available. Use -CatalogPath with an export from the authenticated Learn
|
||||
Platform API when the legacy endpoint is retired.
|
||||
#>
|
||||
[CmdletBinding(DefaultParameterSetName = 'Remote')]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
|
||||
[Parameter(ParameterSetName = 'Remote')]
|
||||
[uri] $CatalogUri = 'https://learn.microsoft.com/api/catalog/?locale=en-us',
|
||||
|
||||
[Parameter(Mandatory, ParameterSetName = 'File')]
|
||||
[string] $CatalogPath,
|
||||
|
||||
[string] $OutputPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $OutputPath) {
|
||||
$OutputPath = Join-Path $Root 'coverage/microsoft-learn-developer-catalog.json'
|
||||
}
|
||||
|
||||
function Get-CanonicalUrl {
|
||||
param([string] $Url)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Url)) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$parsed = [uri]$Url
|
||||
return "$($parsed.Scheme)://$($parsed.Host)$($parsed.AbsolutePath)".TrimEnd('/')
|
||||
}
|
||||
|
||||
function Get-CatalogProperty {
|
||||
param(
|
||||
[object] $Object,
|
||||
[string] $Name,
|
||||
[object] $Default = $null
|
||||
)
|
||||
|
||||
$property = $Object.PSObject.Properties[$Name]
|
||||
if ($property) {
|
||||
return $property.Value
|
||||
}
|
||||
return $Default
|
||||
}
|
||||
|
||||
function Get-LatestTimestamp {
|
||||
param([object[]] $Values)
|
||||
|
||||
$timestamps = @(
|
||||
$Values |
|
||||
Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } |
|
||||
ForEach-Object { [datetimeoffset]::Parse([string]$_) } |
|
||||
Sort-Object
|
||||
)
|
||||
if (-not $timestamps.Count) {
|
||||
return $null
|
||||
}
|
||||
|
||||
return $timestamps[-1].ToUniversalTime().ToString(
|
||||
"yyyy-MM-dd'T'HH:mm:ss'Z'",
|
||||
[System.Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
}
|
||||
|
||||
function Convert-ToUtcTimestamp {
|
||||
param([object] $Value)
|
||||
|
||||
return ([datetimeoffset]$Value).ToUniversalTime().ToString(
|
||||
"yyyy-MM-dd'T'HH:mm:ss'Z'",
|
||||
[System.Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
}
|
||||
|
||||
$catalog = if ($PSCmdlet.ParameterSetName -eq 'File') {
|
||||
Get-Content -LiteralPath $CatalogPath -Raw | ConvertFrom-Json
|
||||
} else {
|
||||
Invoke-RestMethod -Uri $CatalogUri
|
||||
}
|
||||
|
||||
$product = 'dynamics-business-central'
|
||||
$role = 'developer'
|
||||
$modules = @(
|
||||
$catalog.modules |
|
||||
Where-Object {
|
||||
@(Get-CatalogProperty $_ 'products' @()) -contains $product -and
|
||||
@(Get-CatalogProperty $_ 'roles' @()) -contains $role
|
||||
} |
|
||||
Sort-Object uid
|
||||
)
|
||||
|
||||
$moduleIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($module in $modules) {
|
||||
$moduleIds.Add([string]$module.uid) | Out-Null
|
||||
}
|
||||
|
||||
$learningPaths = @(
|
||||
$catalog.learningPaths |
|
||||
Where-Object {
|
||||
$path = $_
|
||||
$pathModules = @(Get-CatalogProperty $path 'modules' @())
|
||||
@($pathModules | Where-Object { $moduleIds.Contains([string]$_) }).Count -gt 0 -and
|
||||
@(Get-CatalogProperty $path 'products' @()) -contains $product -and
|
||||
@(Get-CatalogProperty $path 'roles' @()) -contains $role
|
||||
} |
|
||||
Sort-Object uid
|
||||
)
|
||||
|
||||
$pathIdsByModule = @{}
|
||||
foreach ($path in $learningPaths) {
|
||||
foreach ($moduleId in @(Get-CatalogProperty $path 'modules' @())) {
|
||||
if (-not $moduleIds.Contains([string]$moduleId)) {
|
||||
continue
|
||||
}
|
||||
if (-not $pathIdsByModule.ContainsKey([string]$moduleId)) {
|
||||
$pathIdsByModule[[string]$moduleId] = [System.Collections.Generic.List[string]]::new()
|
||||
}
|
||||
$pathIdsByModule[[string]$moduleId].Add([string]$path.uid)
|
||||
}
|
||||
}
|
||||
|
||||
$unitById = @{}
|
||||
foreach ($unit in @(Get-CatalogProperty $catalog 'units' @())) {
|
||||
$unitById[[string]$unit.uid] = $unit
|
||||
}
|
||||
|
||||
$moduleIdsByUnit = @{}
|
||||
$moduleById = @{}
|
||||
$unitIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($module in $modules) {
|
||||
$moduleById[[string]$module.uid] = $module
|
||||
foreach ($unitId in @(Get-CatalogProperty $module 'units' @())) {
|
||||
$unitId = [string]$unitId
|
||||
$unitIds.Add($unitId) | Out-Null
|
||||
if (-not $moduleIdsByUnit.ContainsKey($unitId)) {
|
||||
$moduleIdsByUnit[$unitId] = [System.Collections.Generic.List[string]]::new()
|
||||
}
|
||||
$moduleIdsByUnit[$unitId].Add([string]$module.uid)
|
||||
}
|
||||
}
|
||||
|
||||
$pathRecords = @(
|
||||
foreach ($path in $learningPaths) {
|
||||
[ordered]@{
|
||||
uid = [string]$path.uid
|
||||
title = [string]$path.title
|
||||
url = Get-CanonicalUrl ([string]$path.url)
|
||||
lastModified = if (Get-CatalogProperty $path 'last_modified') {
|
||||
Convert-ToUtcTimestamp (Get-CatalogProperty $path 'last_modified')
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
durationInMinutes = [int](Get-CatalogProperty $path 'duration_in_minutes' 0)
|
||||
moduleUids = @(
|
||||
@(Get-CatalogProperty $path 'modules' @()) |
|
||||
Where-Object { $moduleIds.Contains([string]$_) } |
|
||||
ForEach-Object { [string]$_ } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
$moduleRecords = @(
|
||||
foreach ($module in $modules) {
|
||||
[ordered]@{
|
||||
uid = [string]$module.uid
|
||||
title = [string]$module.title
|
||||
summary = [string](Get-CatalogProperty $module 'summary' '')
|
||||
url = Get-CanonicalUrl ([string]$module.url)
|
||||
lastModified = Convert-ToUtcTimestamp (Get-CatalogProperty $module 'last_modified')
|
||||
durationInMinutes = [int](Get-CatalogProperty $module 'duration_in_minutes' 0)
|
||||
levels = @(@(Get-CatalogProperty $module 'levels' @()) | ForEach-Object { [string]$_ } | Sort-Object -Unique)
|
||||
subjects = @(@(Get-CatalogProperty $module 'subjects' @()) | ForEach-Object { [string]$_ } | Sort-Object -Unique)
|
||||
learningPathUids = if ($pathIdsByModule.ContainsKey([string]$module.uid)) {
|
||||
@($pathIdsByModule[[string]$module.uid] | Sort-Object -Unique)
|
||||
} else {
|
||||
@()
|
||||
}
|
||||
unitUids = @(@(Get-CatalogProperty $module 'units' @()) | ForEach-Object { [string]$_ })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
$unitRecords = @(
|
||||
foreach ($unitId in @($unitIds | Sort-Object)) {
|
||||
if (-not $unitById.ContainsKey($unitId)) {
|
||||
throw "Catalog module references missing unit '$unitId'."
|
||||
}
|
||||
$unit = $unitById[$unitId]
|
||||
$unitUrl = [string](Get-CatalogProperty $unit 'url' '')
|
||||
if ([string]::IsNullOrWhiteSpace($unitUrl)) {
|
||||
$moduleId = [string]@($moduleIdsByUnit[$unitId])[0]
|
||||
$moduleUrl = Get-CanonicalUrl ([string]$moduleById[$moduleId].url)
|
||||
$unitSlug = if ($unitId.StartsWith("$moduleId.", [System.StringComparison]::Ordinal)) {
|
||||
$unitId.Substring($moduleId.Length + 1)
|
||||
} else {
|
||||
@($unitId -split '\.')[-1]
|
||||
}
|
||||
$unitUrl = "$moduleUrl/$unitSlug"
|
||||
}
|
||||
[ordered]@{
|
||||
uid = $unitId
|
||||
title = [string]$unit.title
|
||||
url = Get-CanonicalUrl $unitUrl
|
||||
lastModified = if (Get-CatalogProperty $unit 'last_modified') {
|
||||
Convert-ToUtcTimestamp (Get-CatalogProperty $unit 'last_modified')
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
durationInMinutes = [int](Get-CatalogProperty $unit 'duration_in_minutes' 0)
|
||||
moduleUids = @($moduleIdsByUnit[$unitId] | Sort-Object -Unique)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
$snapshot = [ordered]@{
|
||||
version = 1
|
||||
source = [ordered]@{
|
||||
provider = 'Microsoft Learn'
|
||||
catalogUri = if ($PSCmdlet.ParameterSetName -eq 'Remote') { [string]$CatalogUri } else { $null }
|
||||
locale = 'en-us'
|
||||
filters = [ordered]@{
|
||||
product = $product
|
||||
role = $role
|
||||
}
|
||||
catalogLastModified = Get-LatestTimestamp @(
|
||||
@($moduleRecords | ForEach-Object lastModified) +
|
||||
@($unitRecords | ForEach-Object lastModified) +
|
||||
@($pathRecords | ForEach-Object lastModified)
|
||||
)
|
||||
}
|
||||
counts = [ordered]@{
|
||||
learningPaths = $pathRecords.Count
|
||||
modules = $moduleRecords.Count
|
||||
units = $unitRecords.Count
|
||||
}
|
||||
learningPaths = $pathRecords
|
||||
modules = $moduleRecords
|
||||
units = $unitRecords
|
||||
}
|
||||
|
||||
$outputDirectory = Split-Path -Parent $OutputPath
|
||||
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null
|
||||
}
|
||||
$snapshot | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $OutputPath -Encoding UTF8
|
||||
Write-Host "Microsoft Learn developer catalog: $($pathRecords.Count) paths, $($moduleRecords.Count) modules, $($unitRecords.Count) units."
|
||||
Write-Host "Catalog: $OutputPath"
|
||||
Loading…
Add table
Add a link
Reference in a new issue