mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Add knowledge-backed AL development
Add read-only planning and repository-changing development skills so BCQuality knowledge can guide features, bug fixes, refactors, upgrades, and maintenance before the existing AL review gate runs. Track Microsoft Learn ingestion and add development and BCApps-shaped guidance evaluation fixtures. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 638b66d2-9f06-4f60-8781-808709e1485c
This commit is contained in:
Jesper Schulz-Wedde
parent
1a5afdc0eb
commit
56b80e6dcf
50 changed files with 11589 additions and 79 deletions
516
tools/Test-DevelopmentFixtures.ps1
Normal file
516
tools/Test-DevelopmentFixtures.ps1
Normal file
|
|
@ -0,0 +1,516 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates and prepares BCQuality AL development fixtures.
|
||||
|
||||
.DESCRIPTION
|
||||
Static validation checks fixture IDs, capability links, knowledge references,
|
||||
and the development skill. -PrepareDirectory emits opaque requests for model
|
||||
runs. -ResultsDirectory scores implementation reports produced by an
|
||||
external runner that performed the declared compile, test, and review checks.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
[string] $ManifestPath,
|
||||
[string] $CapabilitiesPath,
|
||||
[string] $PrepareDirectory,
|
||||
[string] $ResultsDirectory
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $ManifestPath) {
|
||||
$ManifestPath = Join-Path $Root 'evaluation/development-fixtures.json'
|
||||
}
|
||||
if (-not $CapabilitiesPath) {
|
||||
$CapabilitiesPath = Join-Path $Root 'coverage/development-capabilities.json'
|
||||
}
|
||||
|
||||
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
|
||||
$capabilityManifest = Get-Content -LiteralPath $CapabilitiesPath -Raw | ConvertFrom-Json
|
||||
$problems = [System.Collections.Generic.List[string]]::new()
|
||||
|
||||
function Get-ModelCaseId {
|
||||
param([string] $ManifestId)
|
||||
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
|
||||
$hash = $sha.ComputeHash($bytes)
|
||||
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
|
||||
return "case-$token"
|
||||
} finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
if ($manifest.version -ne 1) {
|
||||
$problems.Add("Unsupported development fixture version: $($manifest.version)") | Out-Null
|
||||
}
|
||||
if ($capabilityManifest.version -ne 1) {
|
||||
$problems.Add("Unsupported capability manifest version: $($capabilityManifest.version)") | Out-Null
|
||||
}
|
||||
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||
$threshold = [double]$manifest.$thresholdName
|
||||
if ($threshold -lt 0 -or $threshold -gt 1) {
|
||||
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$skillPath = [string]$manifest.skill
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
|
||||
$problems.Add("Development skill does not exist: $skillPath") | Out-Null
|
||||
}
|
||||
|
||||
$validChecks = @('compile', 'tests', 'review')
|
||||
$validInputKinds = @('auto', 'feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
||||
$validOutputKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
||||
$caseById = @{}
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$id = [string]$case.id
|
||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
|
||||
} elseif ($caseById.ContainsKey($id)) {
|
||||
$problems.Add("Duplicate fixture id: $id") | Out-Null
|
||||
} else {
|
||||
$caseById[$id] = $case
|
||||
}
|
||||
|
||||
if ($case.PSObject.Properties.Name -notcontains 'development-request') {
|
||||
$problems.Add("${id}: development-request is required.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$request = $case.'development-request'
|
||||
if ($validInputKinds -notcontains [string]$request.kind) {
|
||||
$problems.Add("${id}: development-request.kind must be one of $($validInputKinds -join ', ').") | Out-Null
|
||||
}
|
||||
$description = if ($request.PSObject.Properties.Name -contains 'description') {
|
||||
[string]$request.description
|
||||
} else {
|
||||
''
|
||||
}
|
||||
$planText = if ($request.PSObject.Properties.Name -contains 'plan') {
|
||||
[string]$request.plan
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($description) -and [string]::IsNullOrWhiteSpace($planText)) {
|
||||
$problems.Add("${id}: development-request requires description or plan.") | Out-Null
|
||||
}
|
||||
if ($request.PSObject.Properties.Name -notcontains 'acceptance-criteria' -or
|
||||
-not @($request.'acceptance-criteria').Count) {
|
||||
$problems.Add("${id}: development-request.acceptance-criteria must not be empty.") | Out-Null
|
||||
}
|
||||
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
|
||||
[string]$case.expectedKind
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($validOutputKinds -notcontains $expectedKind) {
|
||||
$problems.Add("${id}: expectedKind must be one of $($validOutputKinds -join ', ').") | Out-Null
|
||||
}
|
||||
if ([string]$request.kind -ne 'auto' -and [string]$request.kind -ne $expectedKind) {
|
||||
$problems.Add("${id}: explicit request kind '$($request.kind)' must equal expectedKind '$expectedKind'.") | Out-Null
|
||||
}
|
||||
|
||||
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
|
||||
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
|
||||
}
|
||||
foreach ($reference in $expectedKnowledge) {
|
||||
$reference = [string]$reference
|
||||
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
|
||||
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
|
||||
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
|
||||
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
|
||||
}
|
||||
}
|
||||
foreach ($check in @($case.requiredChecks)) {
|
||||
if ($validChecks -notcontains [string]$check) {
|
||||
$problems.Add("${id}: unsupported required check '$check'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$validCapabilityStatuses = @('planned', 'fixture', 'validated')
|
||||
$capabilityIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($capability in @($capabilityManifest.capabilities)) {
|
||||
$id = [string]$capability.id
|
||||
if (-not $capabilityIds.Add($id)) {
|
||||
$problems.Add("Duplicate capability id: $id") | Out-Null
|
||||
}
|
||||
if ($validCapabilityStatuses -notcontains [string]$capability.status) {
|
||||
$problems.Add("${id}: invalid capability status '$($capability.status)'.") | Out-Null
|
||||
}
|
||||
$fixtureIds = @($capability.fixtureIds)
|
||||
if ($capability.status -in @('fixture', 'validated') -and -not $fixtureIds.Count) {
|
||||
$problems.Add("${id}: status '$($capability.status)' requires at least one fixture.") | Out-Null
|
||||
}
|
||||
foreach ($fixtureId in $fixtureIds) {
|
||||
if (-not $caseById.ContainsKey([string]$fixtureId)) {
|
||||
$problems.Add("${id}: unknown fixture id '$fixtureId'.") | Out-Null
|
||||
} elseif (@($caseById[[string]$fixtureId].capabilities) -notcontains $id) {
|
||||
$problems.Add("${id}: fixture '$fixtureId' does not link back to the capability.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
foreach ($capabilityId in @($case.capabilities)) {
|
||||
if (-not $capabilityIds.Contains([string]$capabilityId)) {
|
||||
$problems.Add("$($case.id): unknown capability '$capabilityId'.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$capability = @(
|
||||
$capabilityManifest.capabilities |
|
||||
Where-Object id -eq ([string]$capabilityId)
|
||||
)[0]
|
||||
if (@($capability.fixtureIds) -notcontains [string]$case.id) {
|
||||
$problems.Add("$($case.id): capability '$capabilityId' does not link back to the fixture.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Development fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ($PrepareDirectory) {
|
||||
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-evaluation'
|
||||
if (Test-Path -LiteralPath $PrepareDirectory) {
|
||||
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
|
||||
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
|
||||
throw "PrepareDirectory is not empty and is not a BCQuality development evaluation directory: $PrepareDirectory"
|
||||
}
|
||||
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
|
||||
Get-ChildItem -LiteralPath $PrepareDirectory -File |
|
||||
Where-Object {
|
||||
$_.Name -eq 'knowledge-index.json' -or
|
||||
$_.Name -like 'request-*.json' -or
|
||||
$_.Name -like 'result-*.json'
|
||||
} |
|
||||
Remove-Item -Force
|
||||
}
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
|
||||
}
|
||||
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development evaluation directory' -Encoding UTF8
|
||||
|
||||
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
||||
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
$request = [ordered]@{
|
||||
protocol = 'Run the supplied AL development skill in a fresh writable fixture repository. Persist the implementation, run real checks, and return only implementationReport using the supplied schema.'
|
||||
caseId = $modelId
|
||||
skill = $skillPath
|
||||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = 'knowledge-index.json'
|
||||
'task-context' = [ordered]@{
|
||||
goal = if ($case.'development-request'.PSObject.Properties.Name -contains 'description') {
|
||||
[string]$case.'development-request'.description
|
||||
} else {
|
||||
[string]$case.'development-request'.plan
|
||||
}
|
||||
'inputs-available' = @('development-request', 'repository')
|
||||
technologies = @($case.context.technologies)
|
||||
countries = @($case.context.countries)
|
||||
'application-area' = @($case.context.'application-area')
|
||||
}
|
||||
'development-request' = [ordered]@{
|
||||
kind = [string]$case.'development-request'.kind
|
||||
description = if ($case.'development-request'.PSObject.Properties.Name -contains 'description') {
|
||||
[string]$case.'development-request'.description
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
plan = if ($case.'development-request'.PSObject.Properties.Name -contains 'plan') {
|
||||
[string]$case.'development-request'.plan
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
'acceptance-criteria' = @($case.'development-request'.'acceptance-criteria')
|
||||
}
|
||||
resultSchema = [ordered]@{
|
||||
caseId = $modelId
|
||||
workspaceRoot = 'absolute path to the retained fixture repository'
|
||||
implementationReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development'; version = 1 }
|
||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
||||
'outcome-reason' = 'required for partial or failed'
|
||||
summary = [ordered]@{
|
||||
request = 'implemented change'
|
||||
'files-created' = 0
|
||||
'files-modified' = 0
|
||||
'files-deleted' = 0
|
||||
}
|
||||
plan = [ordered]@{
|
||||
kind = 'feature | bug | refactor | upgrade | maintenance'
|
||||
assumptions = @()
|
||||
decisions = @()
|
||||
objects = @()
|
||||
}
|
||||
knowledge = @([ordered]@{ path = 'repo-relative knowledge article path'; sha = 'optional commit sha'; 'used-for' = 'decision' })
|
||||
changes = @([ordered]@{ path = 'repo-relative changed file'; action = 'created | modified | deleted'; purpose = 'reason' })
|
||||
validation = @([ordered]@{ id = 'compile | tests | review'; command = 'command or quality-skill path'; status = 'passed | failed | not-run'; details = 'non-empty evidence' })
|
||||
review = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-code-review'; version = 1 }
|
||||
outcome = 'completed | partial | failed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 0; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 0; 'items-evaluated' = 0 }
|
||||
}
|
||||
findings = @()
|
||||
suppressed = @()
|
||||
}
|
||||
suppressed = @()
|
||||
remaining = @()
|
||||
}
|
||||
}
|
||||
}
|
||||
$request | ConvertTo-Json -Depth 12 |
|
||||
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
|
||||
}
|
||||
}
|
||||
|
||||
if ($ResultsDirectory) {
|
||||
$failures = [System.Collections.Generic.List[string]]::new()
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
|
||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): missing result file.") | Out-Null
|
||||
continue
|
||||
}
|
||||
try {
|
||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||
} catch {
|
||||
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
|
||||
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'implementationReport') {
|
||||
$failures.Add("$($case.id): implementationReport is missing.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$report = $result.implementationReport
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'plan', 'knowledge', 'changes', 'validation', 'review', 'suppressed', 'remaining')) {
|
||||
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
|
||||
$failures.Add("$($case.id): implementation report is missing '$requiredField'.") | Out-Null
|
||||
}
|
||||
}
|
||||
$reportedSkillId = if (
|
||||
$report.PSObject.Properties.Name -contains 'skill' -and
|
||||
$report.skill.PSObject.Properties.Name -contains 'id'
|
||||
) {
|
||||
[string]$report.skill.id
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($reportedSkillId -ne 'al-development') {
|
||||
$failures.Add("$($case.id): implementation report skill is '$reportedSkillId'.") | Out-Null
|
||||
}
|
||||
$summaryRequest = if (
|
||||
$report.PSObject.Properties.Name -contains 'summary' -and
|
||||
$report.summary.PSObject.Properties.Name -contains 'request'
|
||||
) {
|
||||
[string]$report.summary.request
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($summaryRequest)) {
|
||||
$failures.Add("$($case.id): summary.request is missing or empty.") | Out-Null
|
||||
}
|
||||
$reportedKind = if (
|
||||
$report.PSObject.Properties.Name -contains 'plan' -and
|
||||
$report.plan.PSObject.Properties.Name -contains 'kind'
|
||||
) {
|
||||
[string]$report.plan.kind
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($reportedKind -ne [string]$case.expectedKind) {
|
||||
$failures.Add("$($case.id): plan.kind '$reportedKind' does not match expected mode '$($case.expectedKind)'.") | Out-Null
|
||||
}
|
||||
$outcome = if ($report.PSObject.Properties.Name -contains 'outcome') { [string]$report.outcome } else { '' }
|
||||
if ($outcome -ne 'completed') {
|
||||
$failures.Add("$($case.id): implementation outcome is '$outcome'.") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$knowledgeEntries = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'knowledge') {
|
||||
$knowledgeEntries = @($report.knowledge)
|
||||
}
|
||||
if (-not $knowledgeEntries.Count) {
|
||||
$failures.Add("$($case.id): implementation report contains no knowledge entries.") | Out-Null
|
||||
}
|
||||
$usedKnowledge = @(
|
||||
$knowledgeEntries |
|
||||
Where-Object { $_.PSObject.Properties.Name -contains 'path' } |
|
||||
ForEach-Object { [string]$_.path }
|
||||
)
|
||||
if (@($usedKnowledge | Sort-Object -Unique).Count -ne $usedKnowledge.Count) {
|
||||
$failures.Add("$($case.id): knowledge contains duplicate paths.") | Out-Null
|
||||
}
|
||||
foreach ($entry in $knowledgeEntries) {
|
||||
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
|
||||
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
|
||||
if ([string]::IsNullOrWhiteSpace($path) -or
|
||||
[System.IO.Path]::IsPathRooted($path) -or
|
||||
@($path -split '/|\\') -contains '..' -or
|
||||
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): knowledge entry has a missing or invalid path '$path'.") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($usedFor)) {
|
||||
$failures.Add("$($case.id): knowledge entry '$path' has no used-for explanation.") | Out-Null
|
||||
}
|
||||
}
|
||||
$requiredKnowledge = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
|
||||
$matched = @($requiredKnowledge | Where-Object { $usedKnowledge -contains $_ }).Count
|
||||
$recall = if ($requiredKnowledge.Count) { $matched / $requiredKnowledge.Count } else { 1.0 }
|
||||
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
|
||||
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
|
||||
}
|
||||
$acceptedKnowledge = @(
|
||||
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
|
||||
ForEach-Object { [string]$_ } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
$acceptedUsed = @($usedKnowledge | Where-Object { $acceptedKnowledge -contains $_ }).Count
|
||||
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
|
||||
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
|
||||
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$validationEntries = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'validation') {
|
||||
$validationEntries = @($report.validation)
|
||||
}
|
||||
foreach ($requiredCheck in @($case.requiredChecks)) {
|
||||
$check = @($validationEntries | Where-Object {
|
||||
$_.PSObject.Properties.Name -contains 'id' -and [string]$_.id -eq $requiredCheck
|
||||
})
|
||||
$checkStatus = if ($check.Count -eq 1 -and $check[0].PSObject.Properties.Name -contains 'status') {
|
||||
[string]$check[0].status
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($check.Count -ne 1 -or $checkStatus -ne 'passed') {
|
||||
$failures.Add("$($case.id): required check '$requiredCheck' did not pass exactly once.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$command = if ($check[0].PSObject.Properties.Name -contains 'command') { [string]$check[0].command } else { '' }
|
||||
$details = if ($check[0].PSObject.Properties.Name -contains 'details') { [string]$check[0].details } else { '' }
|
||||
if ([string]::IsNullOrWhiteSpace($command) -or [string]::IsNullOrWhiteSpace($details)) {
|
||||
$failures.Add("$($case.id): required check '$requiredCheck' lacks command or evidence details.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
[object[]]$changes = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'changes') {
|
||||
$changes = @($report.changes)
|
||||
}
|
||||
if (-not $changes.Count) {
|
||||
$failures.Add("$($case.id): implementation report contains no changed files.") | Out-Null
|
||||
}
|
||||
|
||||
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') { [string]$result.workspaceRoot } else { '' }
|
||||
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
|
||||
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
|
||||
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
|
||||
} else {
|
||||
& git -C $workspaceRoot rev-parse --is-inside-work-tree 2>$null | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$actualChanges = @(
|
||||
@(& git -C $workspaceRoot diff --name-only HEAD) +
|
||||
@(& git -C $workspaceRoot ls-files --others --exclude-standard) |
|
||||
Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } |
|
||||
ForEach-Object { ([string]$_).Replace('\', '/') } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add("$($case.id): unable to read the workspace diff.") | Out-Null
|
||||
} else {
|
||||
$reportedChanges = @(
|
||||
$changes |
|
||||
Where-Object { $_.PSObject.Properties.Name -contains 'path' } |
|
||||
ForEach-Object {
|
||||
$path = ([string]$_.path).Replace('\', '/')
|
||||
if ([System.IO.Path]::IsPathRooted($path) -or @($path -split '/') -contains '..') {
|
||||
$failures.Add("$($case.id): changed path is not repository-relative: $path") | Out-Null
|
||||
}
|
||||
$path
|
||||
} |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
foreach ($path in @($reportedChanges | Where-Object { $actualChanges -notcontains $_ })) {
|
||||
$failures.Add("$($case.id): reported change is absent from the worktree diff: $path") | Out-Null
|
||||
}
|
||||
foreach ($path in @($actualChanges | Where-Object { $reportedChanges -notcontains $_ })) {
|
||||
$failures.Add("$($case.id): worktree change is absent from the implementation report: $path") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($report.PSObject.Properties.Name -notcontains 'review') {
|
||||
$failures.Add("$($case.id): complete post-implementation review is missing.") | Out-Null
|
||||
} else {
|
||||
$review = $report.review
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'findings', 'suppressed')) {
|
||||
if ($review.PSObject.Properties.Name -notcontains $requiredField) {
|
||||
$failures.Add("$($case.id): review is missing '$requiredField'.") | Out-Null
|
||||
}
|
||||
}
|
||||
[object[]]$reviewFindings = @()
|
||||
if ($review.PSObject.Properties.Name -contains 'findings') {
|
||||
$reviewFindings = @($review.findings)
|
||||
}
|
||||
$reviewOutcome = if ($review.PSObject.Properties.Name -contains 'outcome') { [string]$review.outcome } else { '' }
|
||||
if ($reviewOutcome -ne 'completed') {
|
||||
$failures.Add("$($case.id): post-implementation review outcome is '$reviewOutcome'.") | Out-Null
|
||||
}
|
||||
$gatingFindings = @(
|
||||
$reviewFindings |
|
||||
Where-Object {
|
||||
$_.PSObject.Properties.Name -contains 'severity' -and
|
||||
[string]$_.severity -in @('blocker', 'major')
|
||||
}
|
||||
)
|
||||
if ($gatingFindings.Count) {
|
||||
$failures.Add("$($case.id): post-implementation review has $($gatingFindings.Count) gating finding(s).") | Out-Null
|
||||
}
|
||||
}
|
||||
[object[]]$remaining = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'remaining') {
|
||||
$remaining = @($report.remaining)
|
||||
}
|
||||
if ($remaining.Count) {
|
||||
$failures.Add("$($case.id): completed report still lists remaining work.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
if ($failures.Count) {
|
||||
Write-Host "Development fixture scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
|
||||
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Development fixture scoring PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
} else {
|
||||
$fixtureBackedCapabilities = @(
|
||||
$capabilityManifest.capabilities |
|
||||
Where-Object status -in @('fixture', 'validated')
|
||||
).Count
|
||||
Write-Host "Development fixture validation PASSED: $(@($manifest.cases).Count) cases; $fixtureBackedCapabilities of $($capabilityIds.Count) capabilities have fixtures."
|
||||
}
|
||||
298
tools/Test-DevelopmentGuidanceFixtures.ps1
Normal file
298
tools/Test-DevelopmentGuidanceFixtures.ps1
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates, prepares, and scores read-only AL development-guidance fixtures.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
[string] $ManifestPath,
|
||||
[string] $PrepareDirectory,
|
||||
[string] $ResultsDirectory
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $ManifestPath) {
|
||||
$ManifestPath = Join-Path $Root 'evaluation/development-guidance-fixtures.json'
|
||||
}
|
||||
|
||||
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
|
||||
$problems = [System.Collections.Generic.List[string]]::new()
|
||||
$validKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
||||
|
||||
function Get-ModelCaseId {
|
||||
param([string] $ManifestId)
|
||||
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
|
||||
$hash = $sha.ComputeHash($bytes)
|
||||
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
|
||||
return "case-$token"
|
||||
} finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
if ($manifest.version -ne 1) {
|
||||
$problems.Add("Unsupported guidance fixture version: $($manifest.version)") | Out-Null
|
||||
}
|
||||
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||
$threshold = [double]$manifest.$thresholdName
|
||||
if ($threshold -lt 0 -or $threshold -gt 1) {
|
||||
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$skillPath = [string]$manifest.skill
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
|
||||
$problems.Add("Guidance skill does not exist: $skillPath") | Out-Null
|
||||
}
|
||||
|
||||
$seenIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$id = [string]$case.id
|
||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
|
||||
} elseif (-not $seenIds.Add($id)) {
|
||||
$problems.Add("Duplicate fixture id: $id") | Out-Null
|
||||
}
|
||||
if ($case.PSObject.Properties.Name -notcontains 'development-plan') {
|
||||
$problems.Add("${id}: development-plan is required.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$plan = $case.'development-plan'
|
||||
if ($validKinds -notcontains [string]$plan.kind) {
|
||||
$problems.Add("${id}: development-plan.kind is invalid.") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace([string]$plan.request)) {
|
||||
$problems.Add("${id}: development-plan.request is required.") | Out-Null
|
||||
}
|
||||
|
||||
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
|
||||
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
|
||||
}
|
||||
foreach ($reference in $expectedKnowledge) {
|
||||
$reference = [string]$reference
|
||||
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
|
||||
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
|
||||
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
|
||||
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Development guidance fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ($PrepareDirectory) {
|
||||
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-guidance-evaluation'
|
||||
if (Test-Path -LiteralPath $PrepareDirectory) {
|
||||
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
|
||||
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
|
||||
throw "PrepareDirectory is not empty and is not a BCQuality development-guidance evaluation directory: $PrepareDirectory"
|
||||
}
|
||||
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
|
||||
Get-ChildItem -LiteralPath $PrepareDirectory -File |
|
||||
Where-Object {
|
||||
$_.Name -eq 'knowledge-index.json' -or
|
||||
$_.Name -like 'request-*.json' -or
|
||||
$_.Name -like 'result-*.json'
|
||||
} |
|
||||
Remove-Item -Force
|
||||
}
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
|
||||
}
|
||||
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development-guidance evaluation directory' -Encoding UTF8
|
||||
|
||||
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
||||
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
[ordered]@{
|
||||
protocol = 'Run the supplied AL development-plan skill read-only in a clean fixture repository. Return only guidanceReport using the supplied schema and retain the repository for read-only verification.'
|
||||
caseId = $modelId
|
||||
skill = $skillPath
|
||||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = 'knowledge-index.json'
|
||||
'task-context' = [ordered]@{
|
||||
goal = [string]$case.'development-plan'.request
|
||||
'inputs-available' = @('development-plan', 'repository')
|
||||
technologies = @($case.context.technologies)
|
||||
countries = @($case.context.countries)
|
||||
'application-area' = @($case.context.'application-area')
|
||||
}
|
||||
'development-plan' = $case.'development-plan'
|
||||
resultSchema = [ordered]@{
|
||||
caseId = $modelId
|
||||
workspaceRoot = 'absolute path to the retained clean fixture repository'
|
||||
guidanceReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
||||
'outcome-reason' = 'required for partial or failed'
|
||||
summary = [ordered]@{
|
||||
request = 'planned change'
|
||||
kind = 'feature | bug | refactor | upgrade | maintenance'
|
||||
candidates = 0
|
||||
selected = 0
|
||||
}
|
||||
context = [ordered]@{
|
||||
'bc-version' = 'resolved target or unknown'
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
unknown = @()
|
||||
}
|
||||
knowledge = @([ordered]@{
|
||||
path = 'repo-relative article path'
|
||||
sha = 'optional commit sha'
|
||||
'used-for' = 'plan decision'
|
||||
constraints = @('faithful normative constraint')
|
||||
'sample-paths' = @()
|
||||
})
|
||||
'validation-considerations' = @([ordered]@{
|
||||
id = 'stable id'
|
||||
reason = 'why evidence is needed'
|
||||
evidence = 'evidence implementation should obtain'
|
||||
})
|
||||
suppressed = @()
|
||||
unresolved = @()
|
||||
}
|
||||
}
|
||||
} | ConvertTo-Json -Depth 15 |
|
||||
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
|
||||
}
|
||||
}
|
||||
|
||||
if ($ResultsDirectory) {
|
||||
$failures = [System.Collections.Generic.List[string]]::new()
|
||||
foreach ($case in @($manifest.cases)) {
|
||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
||||
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
|
||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): missing result file.") | Out-Null
|
||||
continue
|
||||
}
|
||||
try {
|
||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
||||
} catch {
|
||||
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
|
||||
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($result.PSObject.Properties.Name -notcontains 'guidanceReport') {
|
||||
$failures.Add("$($case.id): guidanceReport is missing.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$report = $result.guidanceReport
|
||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
|
||||
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
|
||||
$failures.Add("$($case.id): guidance report is missing '$requiredField'.") | Out-Null
|
||||
}
|
||||
}
|
||||
$skillId = if (
|
||||
$report.PSObject.Properties.Name -contains 'skill' -and
|
||||
$report.skill.PSObject.Properties.Name -contains 'id'
|
||||
) { [string]$report.skill.id } else { '' }
|
||||
if ($skillId -ne 'al-development-plan') {
|
||||
$failures.Add("$($case.id): guidance skill is '$skillId'.") | Out-Null
|
||||
}
|
||||
if ($report.PSObject.Properties.Name -notcontains 'outcome' -or [string]$report.outcome -ne 'completed') {
|
||||
$failures.Add("$($case.id): guidance outcome is not completed.") | Out-Null
|
||||
}
|
||||
$kind = if (
|
||||
$report.PSObject.Properties.Name -contains 'summary' -and
|
||||
$report.summary.PSObject.Properties.Name -contains 'kind'
|
||||
) { [string]$report.summary.kind } else { '' }
|
||||
if ($kind -ne [string]$case.'development-plan'.kind) {
|
||||
$failures.Add("$($case.id): summary.kind '$kind' does not match the plan.") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$knowledgeEntries = @()
|
||||
if ($report.PSObject.Properties.Name -contains 'knowledge') {
|
||||
$knowledgeEntries = @($report.knowledge)
|
||||
}
|
||||
$usedKnowledge = @()
|
||||
foreach ($entry in $knowledgeEntries) {
|
||||
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
|
||||
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
|
||||
[object[]]$constraints = @()
|
||||
if ($entry.PSObject.Properties.Name -contains 'constraints') {
|
||||
$constraints = @($entry.constraints)
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($path) -or
|
||||
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): invalid knowledge path '$path'.") | Out-Null
|
||||
} else {
|
||||
$usedKnowledge += $path
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($usedFor) -or -not $constraints.Count) {
|
||||
$failures.Add("$($case.id): '$path' lacks used-for or constraints.") | Out-Null
|
||||
}
|
||||
if ($entry.PSObject.Properties.Name -contains 'sample-paths') {
|
||||
foreach ($samplePath in @($entry.'sample-paths')) {
|
||||
$samplePath = [string]$samplePath
|
||||
if ([string]::IsNullOrWhiteSpace($samplePath) -or
|
||||
-not (Test-Path -LiteralPath (Join-Path $Root $samplePath) -PathType Leaf)) {
|
||||
$failures.Add("$($case.id): invalid sample path '$samplePath'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$required = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
|
||||
$matched = @($required | Where-Object { $usedKnowledge -contains $_ }).Count
|
||||
$recall = if ($required.Count) { $matched / $required.Count } else { 1.0 }
|
||||
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
|
||||
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
|
||||
}
|
||||
$accepted = @(
|
||||
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
|
||||
ForEach-Object { [string]$_ } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
$acceptedUsed = @($usedKnowledge | Where-Object { $accepted -contains $_ }).Count
|
||||
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
|
||||
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
|
||||
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
|
||||
}
|
||||
|
||||
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') {
|
||||
[string]$result.workspaceRoot
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
|
||||
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
|
||||
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
|
||||
} else {
|
||||
$status = @(& git -C $workspaceRoot status --porcelain)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
|
||||
} elseif ($status.Count) {
|
||||
$failures.Add("$($case.id): guidance skill changed the target repository.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($failures.Count) {
|
||||
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
|
||||
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Development guidance scoring PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
} else {
|
||||
Write-Host "Development guidance fixture validation PASSED: $(@($manifest.cases).Count) case(s)."
|
||||
}
|
||||
240
tools/Test-LearnCoverage.ps1
Normal file
240
tools/Test-LearnCoverage.ps1
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates Microsoft Learn ingestion coverage and reports progress.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
[string] $CatalogPath,
|
||||
[string] $CoveragePath,
|
||||
[switch] $Json
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $CatalogPath) {
|
||||
$CatalogPath = Join-Path $Root 'coverage/microsoft-learn-developer-catalog.json'
|
||||
}
|
||||
if (-not $CoveragePath) {
|
||||
$CoveragePath = Join-Path $Root 'coverage/learn-coverage.json'
|
||||
}
|
||||
|
||||
foreach ($path in @($CatalogPath, $CoveragePath)) {
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
||||
throw "Coverage input not found: $path"
|
||||
}
|
||||
}
|
||||
|
||||
$catalog = Get-Content -LiteralPath $CatalogPath -Raw | ConvertFrom-Json
|
||||
$coverage = Get-Content -LiteralPath $CoveragePath -Raw | ConvertFrom-Json
|
||||
$problems = [System.Collections.Generic.List[string]]::new()
|
||||
|
||||
if ($catalog.version -ne 1) {
|
||||
$problems.Add("Unsupported catalog version: $($catalog.version)") | Out-Null
|
||||
}
|
||||
if ($coverage.version -ne 1) {
|
||||
$problems.Add("Unsupported coverage version: $($coverage.version)") | Out-Null
|
||||
}
|
||||
if ([string]$coverage.catalog -ne 'coverage/microsoft-learn-developer-catalog.json') {
|
||||
$problems.Add("Coverage catalog path must be coverage/microsoft-learn-developer-catalog.json.") | Out-Null
|
||||
}
|
||||
|
||||
$catalogUnits = @{}
|
||||
foreach ($unit in @($catalog.units)) {
|
||||
$uid = [string]$unit.uid
|
||||
if ($catalogUnits.ContainsKey($uid)) {
|
||||
$problems.Add("Duplicate catalog unit uid: $uid") | Out-Null
|
||||
} else {
|
||||
$catalogUnits[$uid] = $unit
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace([string]$unit.title)) {
|
||||
$problems.Add("${uid}: catalog unit title is empty.") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace([string]$unit.url) -or
|
||||
-not ([string]$unit.url).StartsWith('https://learn.microsoft.com/', [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
$problems.Add("${uid}: catalog unit URL is missing or not a Microsoft Learn URL.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$catalogModules = @{}
|
||||
foreach ($module in @($catalog.modules)) {
|
||||
$uid = [string]$module.uid
|
||||
if ($catalogModules.ContainsKey($uid)) {
|
||||
$problems.Add("Duplicate catalog module uid: $uid") | Out-Null
|
||||
} else {
|
||||
$catalogModules[$uid] = $module
|
||||
}
|
||||
foreach ($unitUid in @($module.unitUids)) {
|
||||
if (-not $catalogUnits.ContainsKey([string]$unitUid)) {
|
||||
$problems.Add("${uid}: references unknown unit '$unitUid'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$catalogPaths = @{}
|
||||
foreach ($path in @($catalog.learningPaths)) {
|
||||
$uid = [string]$path.uid
|
||||
if ($catalogPaths.ContainsKey($uid)) {
|
||||
$problems.Add("Duplicate catalog learning-path uid: $uid") | Out-Null
|
||||
} else {
|
||||
$catalogPaths[$uid] = $path
|
||||
}
|
||||
foreach ($moduleUid in @($path.moduleUids)) {
|
||||
if (-not $catalogModules.ContainsKey([string]$moduleUid)) {
|
||||
$problems.Add("${uid}: references unknown module '$moduleUid'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ([int]$catalog.counts.units -ne $catalogUnits.Count) {
|
||||
$problems.Add("Catalog unit count does not match units array.") | Out-Null
|
||||
}
|
||||
if ([int]$catalog.counts.modules -ne $catalogModules.Count) {
|
||||
$problems.Add("Catalog module count does not match modules array.") | Out-Null
|
||||
}
|
||||
if ([int]$catalog.counts.learningPaths -ne $catalogPaths.Count) {
|
||||
$problems.Add("Catalog learning-path count does not match learningPaths array.") | Out-Null
|
||||
}
|
||||
|
||||
foreach ($unit in @($catalog.units)) {
|
||||
foreach ($moduleUid in @($unit.moduleUids)) {
|
||||
if (-not $catalogModules.ContainsKey([string]$moduleUid)) {
|
||||
$problems.Add("$($unit.uid): references unknown module '$moduleUid'.") | Out-Null
|
||||
} elseif (@($catalogModules[[string]$moduleUid].unitUids) -notcontains [string]$unit.uid) {
|
||||
$problems.Add("$($unit.uid): module '$moduleUid' does not link back to the unit.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$validStatuses = @('in-progress', 'complete')
|
||||
$validDispositions = @('candidate', 'authored', 'covered-existing', 'rejected', 'deferred')
|
||||
$seenUnits = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
$seenOutcomes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
$articlePaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
$dispositionCounts = @{}
|
||||
foreach ($disposition in $validDispositions) {
|
||||
$dispositionCounts[$disposition] = 0
|
||||
}
|
||||
|
||||
foreach ($unitCoverage in @($coverage.units)) {
|
||||
$uid = [string]$unitCoverage.uid
|
||||
if (-not $seenUnits.Add($uid)) {
|
||||
$problems.Add("Duplicate coverage unit uid: $uid") | Out-Null
|
||||
}
|
||||
if (-not $catalogUnits.ContainsKey($uid)) {
|
||||
$problems.Add("Coverage unit is absent from the catalog: $uid") | Out-Null
|
||||
}
|
||||
|
||||
$reviewStatus = [string]$unitCoverage.reviewStatus
|
||||
if ($validStatuses -notcontains $reviewStatus) {
|
||||
$problems.Add("${uid}: invalid reviewStatus '$reviewStatus'.") | Out-Null
|
||||
}
|
||||
|
||||
[object[]]$outcomes = @()
|
||||
if ($unitCoverage.PSObject.Properties.Name -contains 'outcomes') {
|
||||
$outcomes = @($unitCoverage.outcomes)
|
||||
}
|
||||
if ($reviewStatus -eq 'in-progress' -and -not $outcomes.Count) {
|
||||
$problems.Add("${uid}: an in-progress unit must contain at least one outcome.") | Out-Null
|
||||
}
|
||||
|
||||
foreach ($outcome in $outcomes) {
|
||||
$id = [string]$outcome.id
|
||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
$problems.Add("${uid}: outcome id must be kebab-case: '$id'.") | Out-Null
|
||||
} elseif (-not $seenOutcomes.Add($id)) {
|
||||
$problems.Add("Duplicate outcome id: $id") | Out-Null
|
||||
}
|
||||
|
||||
$disposition = [string]$outcome.disposition
|
||||
if ($validDispositions -notcontains $disposition) {
|
||||
$problems.Add("${id}: invalid disposition '$disposition'.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$dispositionCounts[$disposition]++
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace([string]$outcome.title)) {
|
||||
$problems.Add("${id}: title is required.") | Out-Null
|
||||
}
|
||||
|
||||
if ($disposition -in @('candidate', 'authored', 'covered-existing')) {
|
||||
if ([string]::IsNullOrWhiteSpace([string]$outcome.domain)) {
|
||||
$problems.Add("${id}: domain is required for disposition '$disposition'.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
[object[]]$paths = @()
|
||||
if ($outcome.PSObject.Properties.Name -contains 'articlePaths') {
|
||||
$paths = @($outcome.articlePaths)
|
||||
}
|
||||
if ($disposition -in @('authored', 'covered-existing')) {
|
||||
if (-not $paths.Count) {
|
||||
$problems.Add("${id}: articlePaths is required for disposition '$disposition'.") | Out-Null
|
||||
}
|
||||
foreach ($relativePath in $paths) {
|
||||
$relativePath = [string]$relativePath
|
||||
if ($relativePath.Contains('\') -or -not $relativePath.EndsWith('.md')) {
|
||||
$problems.Add("${id}: article path must be a forward-slash .md path: $relativePath") | Out-Null
|
||||
continue
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $relativePath) -PathType Leaf)) {
|
||||
$problems.Add("${id}: article does not exist: $relativePath") | Out-Null
|
||||
}
|
||||
$articlePaths.Add($relativePath) | Out-Null
|
||||
}
|
||||
} elseif ($paths.Count) {
|
||||
$problems.Add("${id}: disposition '$disposition' must not set articlePaths.") | Out-Null
|
||||
}
|
||||
|
||||
$rationale = if ($outcome.PSObject.Properties.Name -contains 'rationale') {
|
||||
[string]$outcome.rationale
|
||||
} else {
|
||||
''
|
||||
}
|
||||
if ($disposition -in @('rejected', 'deferred') -and
|
||||
[string]::IsNullOrWhiteSpace($rationale)) {
|
||||
$problems.Add("${id}: rationale is required for disposition '$disposition'.") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Learn coverage validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
|
||||
$complete = @($coverage.units | Where-Object reviewStatus -eq 'complete').Count
|
||||
$inProgress = @($coverage.units | Where-Object reviewStatus -eq 'in-progress').Count
|
||||
$summary = [ordered]@{
|
||||
catalogUnits = $catalogUnits.Count
|
||||
completeUnits = $complete
|
||||
inProgressUnits = $inProgress
|
||||
unreviewedUnits = $catalogUnits.Count - $complete - $inProgress
|
||||
candidateOutcomes = $dispositionCounts.candidate
|
||||
authoredOutcomes = $dispositionCounts.authored
|
||||
coveredExistingOutcomes = $dispositionCounts.'covered-existing'
|
||||
rejectedOutcomes = $dispositionCounts.rejected
|
||||
deferredOutcomes = $dispositionCounts.deferred
|
||||
referencedArticles = $articlePaths.Count
|
||||
}
|
||||
|
||||
if ($Json) {
|
||||
$summary | ConvertTo-Json
|
||||
} else {
|
||||
$message = (
|
||||
'Learn coverage: {0} units; {1} complete, {2} in progress, {3} unreviewed; ' +
|
||||
'{4} candidate outcomes, {5} authored outcomes, {6} existing-coverage outcomes.'
|
||||
) -f @(
|
||||
$summary.catalogUnits,
|
||||
$summary.completeUnits,
|
||||
$summary.inProgressUnits,
|
||||
$summary.unreviewedUnits,
|
||||
$summary.candidateOutcomes,
|
||||
$summary.authoredOutcomes,
|
||||
$summary.coveredExistingOutcomes
|
||||
)
|
||||
Write-Host $message
|
||||
}
|
||||
262
tools/Update-LearnCatalog.ps1
Normal file
262
tools/Update-LearnCatalog.ps1
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Updates the committed Microsoft Learn Business Central developer catalog.
|
||||
|
||||
.DESCRIPTION
|
||||
Filters a Microsoft Learn catalog payload to modules tagged for both
|
||||
dynamics-business-central and developer. The output is intentionally only
|
||||
source metadata; editorial dispositions live in coverage/learn-coverage.json.
|
||||
|
||||
The legacy unauthenticated catalog endpoint remains the default while it is
|
||||
available. Use -CatalogPath with an export from the authenticated Learn
|
||||
Platform API when the legacy endpoint is retired.
|
||||
#>
|
||||
[CmdletBinding(DefaultParameterSetName = 'Remote')]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
||||
|
||||
[Parameter(ParameterSetName = 'Remote')]
|
||||
[uri] $CatalogUri = 'https://learn.microsoft.com/api/catalog/?locale=en-us',
|
||||
|
||||
[Parameter(Mandatory, ParameterSetName = 'File')]
|
||||
[string] $CatalogPath,
|
||||
|
||||
[string] $OutputPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
if (-not $OutputPath) {
|
||||
$OutputPath = Join-Path $Root 'coverage/microsoft-learn-developer-catalog.json'
|
||||
}
|
||||
|
||||
function Get-CanonicalUrl {
|
||||
param([string] $Url)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Url)) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$parsed = [uri]$Url
|
||||
return "$($parsed.Scheme)://$($parsed.Host)$($parsed.AbsolutePath)".TrimEnd('/')
|
||||
}
|
||||
|
||||
function Get-CatalogProperty {
|
||||
param(
|
||||
[object] $Object,
|
||||
[string] $Name,
|
||||
[object] $Default = $null
|
||||
)
|
||||
|
||||
$property = $Object.PSObject.Properties[$Name]
|
||||
if ($property) {
|
||||
return $property.Value
|
||||
}
|
||||
return $Default
|
||||
}
|
||||
|
||||
function Get-LatestTimestamp {
|
||||
param([object[]] $Values)
|
||||
|
||||
$timestamps = @(
|
||||
$Values |
|
||||
Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } |
|
||||
ForEach-Object { [datetimeoffset]::Parse([string]$_) } |
|
||||
Sort-Object
|
||||
)
|
||||
if (-not $timestamps.Count) {
|
||||
return $null
|
||||
}
|
||||
|
||||
return $timestamps[-1].ToUniversalTime().ToString(
|
||||
"yyyy-MM-dd'T'HH:mm:ss'Z'",
|
||||
[System.Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
}
|
||||
|
||||
function Convert-ToUtcTimestamp {
|
||||
param([object] $Value)
|
||||
|
||||
return ([datetimeoffset]$Value).ToUniversalTime().ToString(
|
||||
"yyyy-MM-dd'T'HH:mm:ss'Z'",
|
||||
[System.Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
}
|
||||
|
||||
$catalog = if ($PSCmdlet.ParameterSetName -eq 'File') {
|
||||
Get-Content -LiteralPath $CatalogPath -Raw | ConvertFrom-Json
|
||||
} else {
|
||||
Invoke-RestMethod -Uri $CatalogUri
|
||||
}
|
||||
|
||||
$product = 'dynamics-business-central'
|
||||
$role = 'developer'
|
||||
$modules = @(
|
||||
$catalog.modules |
|
||||
Where-Object {
|
||||
@(Get-CatalogProperty $_ 'products' @()) -contains $product -and
|
||||
@(Get-CatalogProperty $_ 'roles' @()) -contains $role
|
||||
} |
|
||||
Sort-Object uid
|
||||
)
|
||||
|
||||
$moduleIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($module in $modules) {
|
||||
$moduleIds.Add([string]$module.uid) | Out-Null
|
||||
}
|
||||
|
||||
$learningPaths = @(
|
||||
$catalog.learningPaths |
|
||||
Where-Object {
|
||||
$path = $_
|
||||
$pathModules = @(Get-CatalogProperty $path 'modules' @())
|
||||
@($pathModules | Where-Object { $moduleIds.Contains([string]$_) }).Count -gt 0 -and
|
||||
@(Get-CatalogProperty $path 'products' @()) -contains $product -and
|
||||
@(Get-CatalogProperty $path 'roles' @()) -contains $role
|
||||
} |
|
||||
Sort-Object uid
|
||||
)
|
||||
|
||||
$pathIdsByModule = @{}
|
||||
foreach ($path in $learningPaths) {
|
||||
foreach ($moduleId in @(Get-CatalogProperty $path 'modules' @())) {
|
||||
if (-not $moduleIds.Contains([string]$moduleId)) {
|
||||
continue
|
||||
}
|
||||
if (-not $pathIdsByModule.ContainsKey([string]$moduleId)) {
|
||||
$pathIdsByModule[[string]$moduleId] = [System.Collections.Generic.List[string]]::new()
|
||||
}
|
||||
$pathIdsByModule[[string]$moduleId].Add([string]$path.uid)
|
||||
}
|
||||
}
|
||||
|
||||
$unitById = @{}
|
||||
foreach ($unit in @(Get-CatalogProperty $catalog 'units' @())) {
|
||||
$unitById[[string]$unit.uid] = $unit
|
||||
}
|
||||
|
||||
$moduleIdsByUnit = @{}
|
||||
$moduleById = @{}
|
||||
$unitIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
||||
foreach ($module in $modules) {
|
||||
$moduleById[[string]$module.uid] = $module
|
||||
foreach ($unitId in @(Get-CatalogProperty $module 'units' @())) {
|
||||
$unitId = [string]$unitId
|
||||
$unitIds.Add($unitId) | Out-Null
|
||||
if (-not $moduleIdsByUnit.ContainsKey($unitId)) {
|
||||
$moduleIdsByUnit[$unitId] = [System.Collections.Generic.List[string]]::new()
|
||||
}
|
||||
$moduleIdsByUnit[$unitId].Add([string]$module.uid)
|
||||
}
|
||||
}
|
||||
|
||||
$pathRecords = @(
|
||||
foreach ($path in $learningPaths) {
|
||||
[ordered]@{
|
||||
uid = [string]$path.uid
|
||||
title = [string]$path.title
|
||||
url = Get-CanonicalUrl ([string]$path.url)
|
||||
lastModified = if (Get-CatalogProperty $path 'last_modified') {
|
||||
Convert-ToUtcTimestamp (Get-CatalogProperty $path 'last_modified')
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
durationInMinutes = [int](Get-CatalogProperty $path 'duration_in_minutes' 0)
|
||||
moduleUids = @(
|
||||
@(Get-CatalogProperty $path 'modules' @()) |
|
||||
Where-Object { $moduleIds.Contains([string]$_) } |
|
||||
ForEach-Object { [string]$_ } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
$moduleRecords = @(
|
||||
foreach ($module in $modules) {
|
||||
[ordered]@{
|
||||
uid = [string]$module.uid
|
||||
title = [string]$module.title
|
||||
summary = [string](Get-CatalogProperty $module 'summary' '')
|
||||
url = Get-CanonicalUrl ([string]$module.url)
|
||||
lastModified = Convert-ToUtcTimestamp (Get-CatalogProperty $module 'last_modified')
|
||||
durationInMinutes = [int](Get-CatalogProperty $module 'duration_in_minutes' 0)
|
||||
levels = @(@(Get-CatalogProperty $module 'levels' @()) | ForEach-Object { [string]$_ } | Sort-Object -Unique)
|
||||
subjects = @(@(Get-CatalogProperty $module 'subjects' @()) | ForEach-Object { [string]$_ } | Sort-Object -Unique)
|
||||
learningPathUids = if ($pathIdsByModule.ContainsKey([string]$module.uid)) {
|
||||
@($pathIdsByModule[[string]$module.uid] | Sort-Object -Unique)
|
||||
} else {
|
||||
@()
|
||||
}
|
||||
unitUids = @(@(Get-CatalogProperty $module 'units' @()) | ForEach-Object { [string]$_ })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
$unitRecords = @(
|
||||
foreach ($unitId in @($unitIds | Sort-Object)) {
|
||||
if (-not $unitById.ContainsKey($unitId)) {
|
||||
throw "Catalog module references missing unit '$unitId'."
|
||||
}
|
||||
$unit = $unitById[$unitId]
|
||||
$unitUrl = [string](Get-CatalogProperty $unit 'url' '')
|
||||
if ([string]::IsNullOrWhiteSpace($unitUrl)) {
|
||||
$moduleId = [string]@($moduleIdsByUnit[$unitId])[0]
|
||||
$moduleUrl = Get-CanonicalUrl ([string]$moduleById[$moduleId].url)
|
||||
$unitSlug = if ($unitId.StartsWith("$moduleId.", [System.StringComparison]::Ordinal)) {
|
||||
$unitId.Substring($moduleId.Length + 1)
|
||||
} else {
|
||||
@($unitId -split '\.')[-1]
|
||||
}
|
||||
$unitUrl = "$moduleUrl/$unitSlug"
|
||||
}
|
||||
[ordered]@{
|
||||
uid = $unitId
|
||||
title = [string]$unit.title
|
||||
url = Get-CanonicalUrl $unitUrl
|
||||
lastModified = if (Get-CatalogProperty $unit 'last_modified') {
|
||||
Convert-ToUtcTimestamp (Get-CatalogProperty $unit 'last_modified')
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
durationInMinutes = [int](Get-CatalogProperty $unit 'duration_in_minutes' 0)
|
||||
moduleUids = @($moduleIdsByUnit[$unitId] | Sort-Object -Unique)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
$snapshot = [ordered]@{
|
||||
version = 1
|
||||
source = [ordered]@{
|
||||
provider = 'Microsoft Learn'
|
||||
catalogUri = if ($PSCmdlet.ParameterSetName -eq 'Remote') { [string]$CatalogUri } else { $null }
|
||||
locale = 'en-us'
|
||||
filters = [ordered]@{
|
||||
product = $product
|
||||
role = $role
|
||||
}
|
||||
catalogLastModified = Get-LatestTimestamp @(
|
||||
@($moduleRecords | ForEach-Object lastModified) +
|
||||
@($unitRecords | ForEach-Object lastModified) +
|
||||
@($pathRecords | ForEach-Object lastModified)
|
||||
)
|
||||
}
|
||||
counts = [ordered]@{
|
||||
learningPaths = $pathRecords.Count
|
||||
modules = $moduleRecords.Count
|
||||
units = $unitRecords.Count
|
||||
}
|
||||
learningPaths = $pathRecords
|
||||
modules = $moduleRecords
|
||||
units = $unitRecords
|
||||
}
|
||||
|
||||
$outputDirectory = Split-Path -Parent $OutputPath
|
||||
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null
|
||||
}
|
||||
$snapshot | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $OutputPath -Encoding UTF8
|
||||
Write-Host "Microsoft Learn developer catalog: $($pathRecords.Count) paths, $($moduleRecords.Count) modules, $($unitRecords.Count) units."
|
||||
Write-Host "Catalog: $OutputPath"
|
||||
Loading…
Add table
Add a link
Reference in a new issue