mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 22:56:55 +01:00
e22352b248
- Resolve conflicts in al-data-modeling-review.md by keeping both sides' additions (folder-path support, InitRecord/Round cues from upstream; the 9 document-distribution/pricing/barcode cues from this branch). - Switch the data-modeling evaluation override from an ad-hoc additionalArticles field to upstream's now-established articles[] convention (used elsewhere for finance/scm/query/reporting/style), removing the redundant parallel code path from Test-ReviewFixtures.ps1. - Fix all 9 new articles' sample references to the markdown-link READ convention required by Knowledge-Retrieval.ps1's Assert-SampleLink (plain backticks satisfy validate_frontmatter.py's regex alone but not this stricter check - both validators must pass). Validators: frontmatter 0/0, review-fixtures 126 cases/20 domains PASSED, knowledge-index 342 articles/575 samples PASSED, skill-index 19 review leaves PASSED. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
595 lines
26 KiB
PowerShell
595 lines
26 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Validates and prepares the BCQuality AL review evaluation corpus.
|
|
|
|
.DESCRIPTION
|
|
CI uses the static validation path to prove every registered AL review leaf
|
|
has one positive and one clean control, every fixture/reference exists, and
|
|
the manifest remains internally consistent.
|
|
|
|
For an actual model run, -PrepareDirectory copies inputs to neutral names and
|
|
emits review-request.json without expected answers. After the model writes a
|
|
result matching evaluation/README.md, -ResultsPath scores exact knowledge-ID
|
|
recall, clean-control rate, and unexpected findings.
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
|
[string] $ManifestPath,
|
|
[string] $PrepareDirectory,
|
|
[string] $ResultsPath,
|
|
[string] $ResultsDirectory
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
$Root = (Resolve-Path -LiteralPath $Root).Path
|
|
if ($ResultsPath -and $ResultsDirectory) {
|
|
throw 'Specify either ResultsPath or ResultsDirectory, not both.'
|
|
}
|
|
if (-not $ManifestPath) {
|
|
$ManifestPath = Join-Path $Root 'evaluation/review-fixtures.json'
|
|
}
|
|
if (-not (Test-Path -LiteralPath $ManifestPath)) {
|
|
throw "Review fixture manifest not found: $ManifestPath"
|
|
}
|
|
|
|
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
|
|
$problems = [System.Collections.Generic.List[string]]::new()
|
|
|
|
function Get-ModelCaseId {
|
|
param([string] $ManifestId)
|
|
|
|
$sha = [System.Security.Cryptography.SHA256]::Create()
|
|
try {
|
|
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
|
|
$hash = $sha.ComputeHash($bytes)
|
|
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
|
|
return "case-$token"
|
|
} finally {
|
|
$sha.Dispose()
|
|
}
|
|
}
|
|
|
|
function Get-RankedArticles {
|
|
param(
|
|
[object[]] $Articles,
|
|
[string] $CaseText,
|
|
[int] $Limit = 10
|
|
)
|
|
|
|
if ($Articles.Count -le $Limit) {
|
|
return @($Articles)
|
|
}
|
|
|
|
$normalized = (($CaseText.ToLowerInvariant() -replace '[^a-z0-9]+', ' ') -replace '\s+', ' ').Trim()
|
|
$compact = $normalized -replace ' ', ''
|
|
$ranked = foreach ($article in $Articles) {
|
|
$score = 0
|
|
foreach ($keyword in @($article.keywords)) {
|
|
$keywordText = ([string]$keyword).ToLowerInvariant()
|
|
$keywordCompact = $keywordText -replace '[^a-z0-9]+', ''
|
|
if ($keywordCompact -and $compact.Contains($keywordCompact)) {
|
|
$score += 8
|
|
}
|
|
foreach ($part in @($keywordText -split '[^a-z0-9]+')) {
|
|
if (($part.Length -ge 4) -and ($normalized -match "(^| )$([regex]::Escape($part))( |$)")) {
|
|
$score += 1
|
|
}
|
|
}
|
|
}
|
|
$topicText = "$($article.title) $($article.description) $($article.path)".ToLowerInvariant()
|
|
foreach ($term in @($normalized -split ' ' | Where-Object Length -ge 5 | Sort-Object -Unique)) {
|
|
if ($topicText.Contains($term)) {
|
|
$score += 0.25
|
|
}
|
|
}
|
|
[pscustomobject]@{ score = $score; path = [string]$article.path; article = $article }
|
|
}
|
|
|
|
return @(
|
|
$ranked |
|
|
Sort-Object @{ Expression = 'score'; Descending = $true }, @{ Expression = 'path'; Descending = $false } |
|
|
Select-Object -First $Limit |
|
|
ForEach-Object article
|
|
)
|
|
}
|
|
|
|
if ($manifest.version -ne 2) {
|
|
$problems.Add("Unsupported manifest version: $($manifest.version)") | Out-Null
|
|
}
|
|
if ($manifest.selection -ne 'first-paired-al-article') {
|
|
$problems.Add("Unsupported selection strategy: $($manifest.selection)") | Out-Null
|
|
}
|
|
if (([double]$manifest.minimumExpectedRecall -lt 0) -or ([double]$manifest.minimumExpectedRecall -gt 1)) {
|
|
$problems.Add('minimumExpectedRecall must be between 0 and 1.') | Out-Null
|
|
}
|
|
if (([double]$manifest.minimumCleanRate -lt 0) -or ([double]$manifest.minimumCleanRate -gt 1)) {
|
|
$problems.Add('minimumCleanRate must be between 0 and 1.') | Out-Null
|
|
}
|
|
|
|
$layers = @(
|
|
[pscustomobject]@{ Name = 'microsoft'; Rank = 1 }
|
|
[pscustomobject]@{ Name = 'community'; Rank = 2 }
|
|
[pscustomobject]@{ Name = 'custom'; Rank = 3 }
|
|
)
|
|
$layerRanks = @{}
|
|
foreach ($layer in $layers) {
|
|
$layerRanks[[string]$layer.Name] = [int]$layer.Rank
|
|
}
|
|
$leafCandidates = @(
|
|
foreach ($layer in $layers) {
|
|
$reviewDirectory = Join-Path $Root "$($layer.Name)/skills/review"
|
|
if (-not (Test-Path -LiteralPath $reviewDirectory -PathType Container)) {
|
|
continue
|
|
}
|
|
Get-ChildItem -LiteralPath $reviewDirectory -File -Filter 'al-*-review.md' |
|
|
Where-Object Name -ne 'al-code-review.md' |
|
|
ForEach-Object {
|
|
[pscustomobject]@{
|
|
Domain = $_.BaseName -replace '^al-', '' -replace '-review$', ''
|
|
Layer = $layer.Name
|
|
Rank = $layer.Rank
|
|
RelativePath = [System.IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
|
|
}
|
|
}
|
|
}
|
|
)
|
|
$leafSkills = @(
|
|
$leafCandidates |
|
|
Group-Object Domain |
|
|
ForEach-Object { $_.Group | Sort-Object Rank -Descending | Select-Object -First 1 } |
|
|
Sort-Object Domain
|
|
)
|
|
$leafDomains = @($leafSkills | ForEach-Object Domain)
|
|
$leafByDomain = @{}
|
|
foreach ($leafSkill in $leafSkills) {
|
|
$leafByDomain[[string]$leafSkill.Domain] = $leafSkill
|
|
}
|
|
|
|
$overrides = @{}
|
|
if ($manifest.PSObject.Properties.Name -contains 'overrides') {
|
|
foreach ($property in $manifest.overrides.PSObject.Properties) {
|
|
$overrides[$property.Name] = $property.Value
|
|
}
|
|
}
|
|
foreach ($overrideDomain in $overrides.Keys) {
|
|
if ($leafDomains -notcontains $overrideDomain) {
|
|
$problems.Add("Override domain '$overrideDomain' has no registered al-$overrideDomain-review leaf.") | Out-Null
|
|
}
|
|
}
|
|
|
|
$caseList = [System.Collections.Generic.List[object]]::new()
|
|
foreach ($domain in $leafDomains) {
|
|
$articleCandidates = @(
|
|
foreach ($layer in $layers) {
|
|
$knowledgeDirectory = Join-Path $Root "$($layer.Name)/knowledge/$domain"
|
|
if (-not (Test-Path -LiteralPath $knowledgeDirectory -PathType Container)) {
|
|
continue
|
|
}
|
|
Get-ChildItem -LiteralPath $knowledgeDirectory -File -Filter '*.md' |
|
|
Where-Object {
|
|
(Test-Path -LiteralPath (Join-Path $knowledgeDirectory "$($_.BaseName).good.al") -PathType Leaf) -and
|
|
(Test-Path -LiteralPath (Join-Path $knowledgeDirectory "$($_.BaseName).bad.al") -PathType Leaf)
|
|
} |
|
|
ForEach-Object {
|
|
[pscustomobject]@{
|
|
BaseName = $_.BaseName
|
|
File = $_
|
|
Rank = $layer.Rank
|
|
ArticlePath = [System.IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
|
|
}
|
|
}
|
|
}
|
|
)
|
|
$articles = @(
|
|
$articleCandidates |
|
|
Group-Object BaseName |
|
|
ForEach-Object { $_.Group | Sort-Object Rank -Descending | Select-Object -First 1 } |
|
|
Sort-Object BaseName
|
|
)
|
|
if (-not $articles.Count) {
|
|
$problems.Add("${domain}: no enabled knowledge layer has an article with both .good.al and .bad.al companion samples.") | Out-Null
|
|
continue
|
|
}
|
|
|
|
$override = if ($overrides.ContainsKey($domain)) { $overrides[$domain] } else { $null }
|
|
$hasArticleOverride = $override -and ($override.PSObject.Properties.Name -contains 'article')
|
|
$hasArticlesOverride = $override -and ($override.PSObject.Properties.Name -contains 'articles')
|
|
if ($hasArticleOverride -and $hasArticlesOverride) {
|
|
$problems.Add("${domain}: override must specify either 'article' or 'articles', not both.") | Out-Null
|
|
continue
|
|
}
|
|
|
|
$articleNames = @()
|
|
if ($hasArticlesOverride) {
|
|
$articleNames = @($override.articles)
|
|
if (-not $articleNames.Count) {
|
|
$problems.Add("${domain}: override 'articles' must contain at least one article.") | Out-Null
|
|
continue
|
|
}
|
|
} elseif ($hasArticleOverride) {
|
|
$articleNames = @($override.article)
|
|
} else {
|
|
$articleNames = @($articles | Select-Object -First 1 | ForEach-Object BaseName)
|
|
}
|
|
|
|
$selectedArticles = [System.Collections.Generic.List[object]]::new()
|
|
$seenArticleNames = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
|
|
foreach ($articleNameValue in $articleNames) {
|
|
if ($articleNameValue -isnot [string] -or [string]::IsNullOrWhiteSpace([string]$articleNameValue)) {
|
|
$problems.Add("${domain}: override article names must be non-empty strings.") | Out-Null
|
|
continue
|
|
}
|
|
$articleName = [string]$articleNameValue
|
|
if ($articleName.EndsWith('.md')) {
|
|
$articleName = [System.IO.Path]::GetFileNameWithoutExtension($articleName)
|
|
}
|
|
if (-not $seenArticleNames.Add($articleName)) {
|
|
$problems.Add("${domain}: override contains duplicate article: $articleName.md") | Out-Null
|
|
continue
|
|
}
|
|
|
|
$selectedArticle = $articles | Where-Object BaseName -eq $articleName | Select-Object -First 1
|
|
if (-not $selectedArticle) {
|
|
$articleExists = @(
|
|
foreach ($layer in $layers) {
|
|
$articleFile = Join-Path $Root "$($layer.Name)/knowledge/$domain/$articleName.md"
|
|
if (Test-Path -LiteralPath $articleFile -PathType Leaf) {
|
|
$articleFile
|
|
}
|
|
}
|
|
).Count -gt 0
|
|
if ($articleExists) {
|
|
$problems.Add("${domain}: override article does not have both .good.al and .bad.al companion samples: $articleName.md") | Out-Null
|
|
} else {
|
|
$problems.Add("${domain}: override article does not exist: $articleName.md") | Out-Null
|
|
}
|
|
continue
|
|
}
|
|
$selectedArticles.Add($selectedArticle) | Out-Null
|
|
}
|
|
if (-not $selectedArticles.Count) {
|
|
if (-not $articleNames.Count) {
|
|
$problems.Add("${domain}: no article has both .good.al and .bad.al companion samples.") | Out-Null
|
|
}
|
|
continue
|
|
}
|
|
|
|
$context = if ($override -and ($override.PSObject.Properties.Name -contains 'context')) {
|
|
[string]$override.context
|
|
} else {
|
|
$null
|
|
}
|
|
for ($articleIndex = 0; $articleIndex -lt $selectedArticles.Count; $articleIndex++) {
|
|
$selectedArticle = $selectedArticles[$articleIndex]
|
|
$articlePath = [string]$selectedArticle.ArticlePath
|
|
$sampleDirectory = (Split-Path -Parent $articlePath).Replace('\', '/')
|
|
foreach ($kind in 'bad', 'good') {
|
|
$caseId = if ($articleIndex -eq 0) {
|
|
"$domain-$kind"
|
|
} else {
|
|
"$domain-$($selectedArticle.BaseName)-$kind"
|
|
}
|
|
$case = [pscustomobject]@{
|
|
id = $caseId
|
|
domain = $domain
|
|
input = "$sampleDirectory/$($selectedArticle.BaseName).$kind.al"
|
|
expected = if ($kind -eq 'bad') { @($articlePath) } else { @() }
|
|
}
|
|
if ($context) {
|
|
$case | Add-Member -NotePropertyName context -NotePropertyValue $context
|
|
}
|
|
$caseList.Add($case) | Out-Null
|
|
}
|
|
}
|
|
|
|
}
|
|
$cases = @($caseList)
|
|
|
|
$seenIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
|
foreach ($case in $cases) {
|
|
$id = [string]$case.id
|
|
$domain = [string]$case.domain
|
|
$inputRelativePath = [string]$case.input
|
|
$expected = @($case.expected)
|
|
|
|
if ([string]::IsNullOrWhiteSpace($id)) {
|
|
$problems.Add('Case with empty id.') | Out-Null
|
|
} elseif (-not $seenIds.Add($id)) {
|
|
$problems.Add("Duplicate case id: $id") | Out-Null
|
|
}
|
|
if ($leafDomains -notcontains $domain) {
|
|
$problems.Add("${id}: domain '$domain' has no registered al-$domain-review leaf.") | Out-Null
|
|
}
|
|
|
|
$inputPath = Join-Path $Root $inputRelativePath
|
|
if (-not (Test-Path -LiteralPath $inputPath -PathType Leaf)) {
|
|
$problems.Add("${id}: input does not exist: $inputRelativePath") | Out-Null
|
|
}
|
|
if ($expected.Count -and $inputRelativePath -notmatch '\.bad\.[^.]+$') {
|
|
$problems.Add("${id}: positive case must use a .bad sample: $inputRelativePath") | Out-Null
|
|
}
|
|
if (-not $expected.Count -and $inputRelativePath -notmatch '\.good\.[^.]+$') {
|
|
$problems.Add("${id}: clean case must use a .good sample: $inputRelativePath") | Out-Null
|
|
}
|
|
|
|
foreach ($reference in $expected) {
|
|
$referencePath = Join-Path $Root ([string]$reference)
|
|
if (-not (Test-Path -LiteralPath $referencePath -PathType Leaf)) {
|
|
$problems.Add("${id}: referenced article does not exist: $reference") | Out-Null
|
|
}
|
|
}
|
|
if ($expected.Count) {
|
|
$sampleSlug = ([System.IO.Path]::GetFileName($inputRelativePath) -replace '\.(?:good|bad)\.[^.]+$', '')
|
|
$primarySlug = [System.IO.Path]::GetFileNameWithoutExtension([string]$expected[0])
|
|
if ($sampleSlug -ne $primarySlug) {
|
|
$problems.Add("${id}: primary expected article '$primarySlug' must match sample slug '$sampleSlug'.") | Out-Null
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach ($domain in $leafDomains) {
|
|
$domainCases = @($cases | Where-Object domain -eq $domain)
|
|
if (-not @($domainCases | Where-Object { @($_.expected).Count -gt 0 }).Count) {
|
|
$problems.Add("${domain}: no positive review fixture.") | Out-Null
|
|
}
|
|
if (-not @($domainCases | Where-Object { @($_.expected).Count -eq 0 }).Count) {
|
|
$problems.Add("${domain}: no clean control fixture.") | Out-Null
|
|
}
|
|
}
|
|
|
|
if ($problems.Count) {
|
|
Write-Host "Review fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
|
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
|
exit 1
|
|
}
|
|
|
|
if ($PrepareDirectory) {
|
|
$markerPath = Join-Path $PrepareDirectory '.bcquality-evaluation'
|
|
if (Test-Path -LiteralPath $PrepareDirectory) {
|
|
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
|
|
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
|
|
throw "PrepareDirectory is not empty and is not a BCQuality evaluation directory: $PrepareDirectory"
|
|
}
|
|
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
|
|
Get-ChildItem -LiteralPath $PrepareDirectory -File |
|
|
Where-Object {
|
|
($_.Name -like 'case*.al') -or
|
|
($_.Name -eq 'review-request.json') -or
|
|
($_.Name -like 'request-*.json') -or
|
|
($_.Name -eq 'knowledge-index.json') -or
|
|
($_.Name -like 'index-*.json') -or
|
|
($_.Name -like 'result-*.json')
|
|
} |
|
|
Remove-Item -Force
|
|
}
|
|
} else {
|
|
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
|
|
}
|
|
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated evaluation directory' -Encoding UTF8
|
|
|
|
$fullIndexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
|
|
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $fullIndexPath | Out-Null
|
|
$fullIndex = Get-Content -LiteralPath $fullIndexPath -Raw | ConvertFrom-Json
|
|
|
|
$requestCases = [System.Collections.Generic.List[object]]::new()
|
|
$requestCasesByDomain = @{}
|
|
$manifestCaseByModelId = @{}
|
|
foreach ($case in $cases) {
|
|
$extension = [System.IO.Path]::GetExtension([string]$case.input)
|
|
$modelId = Get-ModelCaseId -ManifestId ([string]$case.id)
|
|
$neutralName = "$modelId$extension"
|
|
$sourceText = Get-Content -LiteralPath (Join-Path $Root ([string]$case.input)) -Raw
|
|
# Companion samples are human-facing and often label objects/comments as
|
|
# Good, Bad, or Anti-pattern. Strip full-line comments and neutralize those
|
|
# object-name tokens so model-facing fixtures do not reveal the expected
|
|
# outcome while preserving executable AL structure and references.
|
|
$neutralText = [regex]::Replace($sourceText, '(?m)^\s*//.*(?:\r?\n|$)', '')
|
|
$neutralText = [regex]::Replace($neutralText, '\b(?:Good|Bad)\b', 'Eval')
|
|
Set-Content -LiteralPath (Join-Path $PrepareDirectory $neutralName) -Value $neutralText -Encoding UTF8
|
|
$requestCase = [pscustomobject]@{ id = $modelId; file = $neutralName }
|
|
if ($case.PSObject.Properties.Name -contains 'context') {
|
|
$requestCase | Add-Member -NotePropertyName context -NotePropertyValue ([string]$case.context)
|
|
}
|
|
$requestCases.Add($requestCase) | Out-Null
|
|
$manifestCaseByModelId[$modelId] = $case
|
|
$domain = [string]$case.domain
|
|
if (-not $requestCasesByDomain.ContainsKey($domain)) {
|
|
$requestCasesByDomain[$domain] = [System.Collections.Generic.List[object]]::new()
|
|
}
|
|
$requestCasesByDomain[$domain].Add($requestCase) | Out-Null
|
|
}
|
|
$resultSchema = [pscustomobject]@{
|
|
cases = @([pscustomobject]@{
|
|
id = 'case-id'
|
|
findings = @([pscustomobject]@{ id = 'repo-relative knowledge article path' })
|
|
})
|
|
}
|
|
[pscustomobject]@{
|
|
protocol = 'Run BCQuality al-code-review over all files as one PR; return findings per case. Copy every knowledge-backed id from knowledge-index.json.'
|
|
knowledgeIndex = 'knowledge-index.json'
|
|
resultSchema = $resultSchema
|
|
cases = @($requestCases)
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $PrepareDirectory 'review-request.json') -Encoding UTF8
|
|
|
|
foreach ($domain in $leafDomains) {
|
|
$domainArticles = @(
|
|
$fullIndex.articles |
|
|
Where-Object domain -eq $domain |
|
|
Sort-Object @{ Expression = { $layerRanks[[string]$_.layer] }; Descending = $true }, path |
|
|
Group-Object { [System.IO.Path]::GetFileName([string]$_.path) } |
|
|
ForEach-Object { $_.Group | Select-Object -First 1 } |
|
|
Sort-Object path
|
|
)
|
|
$domainIndexName = "index-$domain.json"
|
|
$leafPath = [string]$leafByDomain[$domain].RelativePath
|
|
$leafFullText = Get-Content -LiteralPath (Join-Path $Root $leafPath) -Raw
|
|
$leafInstructions = @($leafFullText -split '(?m)^## Output\s*\r?\n', 2)[0]
|
|
$leafInstructions += "`n## Output`nReturn only the request's resultSchema."
|
|
[pscustomobject]@{
|
|
version = $fullIndex.version
|
|
domain = $domain
|
|
articleCount = $domainArticles.Count
|
|
articles = $domainArticles
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $PrepareDirectory $domainIndexName) -Encoding UTF8
|
|
|
|
[pscustomobject]@{
|
|
protocol = "Run only $leafPath over these files. Follow leafInstructions exactly, use only the supplied candidate article rows, open matching articles in full, and copy every finding id verbatim from candidateArticles[].path."
|
|
skill = $leafPath
|
|
leafInstructions = $leafInstructions
|
|
knowledgeIndex = $domainIndexName
|
|
candidateArticles = $domainArticles
|
|
resultSchema = $resultSchema
|
|
cases = @($requestCasesByDomain[$domain])
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$domain.json") -Encoding UTF8
|
|
|
|
foreach ($requestCase in @($requestCasesByDomain[$domain])) {
|
|
$caseText = Get-Content -LiteralPath (Join-Path $PrepareDirectory ([string]$requestCase.file)) -Raw
|
|
if ($requestCase.PSObject.Properties.Name -contains 'context') {
|
|
$caseText += " $([string]$requestCase.context)"
|
|
}
|
|
$rankedArticles = @(Get-RankedArticles -Articles $domainArticles -CaseText $caseText)
|
|
$manifestCase = $manifestCaseByModelId[[string]$requestCase.id]
|
|
$selectedArticlePath = ([string]$manifestCase.input) -replace '\.(?:good|bad)\.al$', '.md'
|
|
$rankedPaths = @($rankedArticles | ForEach-Object { [string]$_.path })
|
|
if ($rankedPaths -notcontains $selectedArticlePath) {
|
|
throw "$($manifestCase.id): deterministic ranking omitted selected article '$selectedArticlePath'. Improve its retrieval metadata or choose an exceptional override article."
|
|
}
|
|
# Candidate order must not reveal which article owns the fixture.
|
|
$rankedArticles = @($rankedArticles | Sort-Object path)
|
|
[pscustomobject]@{
|
|
protocol = "Run only $leafPath over this case. Follow leafInstructions exactly, evaluate the ranked candidate article rows, open matching articles in full, and copy every finding id verbatim from candidateArticles[].path."
|
|
skill = $leafPath
|
|
leafInstructions = $leafInstructions
|
|
knowledgeIndex = $domainIndexName
|
|
candidateArticles = $rankedArticles
|
|
resultSchema = $resultSchema
|
|
cases = @($requestCase)
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$($requestCase.id).json") -Encoding UTF8
|
|
}
|
|
}
|
|
Write-Host "Prepared $($cases.Count) neutral fixture(s) in $PrepareDirectory." -ForegroundColor Green
|
|
}
|
|
|
|
if (-not $ResultsPath -and -not $ResultsDirectory) {
|
|
& (Join-Path $PSScriptRoot 'Test-ReviewContract.ps1') -Root $Root
|
|
Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $($leafDomains.Count) leaf domains." -ForegroundColor Green
|
|
exit 0
|
|
}
|
|
|
|
$resultCases = [System.Collections.Generic.List[object]]::new()
|
|
if ($ResultsDirectory) {
|
|
if (-not (Test-Path -LiteralPath $ResultsDirectory -PathType Container)) {
|
|
throw "Results directory not found: $ResultsDirectory"
|
|
}
|
|
$resultFiles = @(Get-ChildItem -LiteralPath $ResultsDirectory -File -Filter 'result-case-*.json')
|
|
if (-not $resultFiles.Count) {
|
|
$resultFiles = @(Get-ChildItem -LiteralPath $ResultsDirectory -File -Filter 'result-*.json')
|
|
}
|
|
if (-not $resultFiles.Count) {
|
|
throw "No result-case-*.json or result-*.json files found in: $ResultsDirectory"
|
|
}
|
|
foreach ($resultFile in $resultFiles) {
|
|
try {
|
|
$resultDocument = Get-Content -LiteralPath $resultFile.FullName -Raw | ConvertFrom-Json
|
|
} catch {
|
|
$problems.Add("$($resultFile.Name): invalid JSON: $($_.Exception.Message)") | Out-Null
|
|
continue
|
|
}
|
|
if ($resultDocument.PSObject.Properties.Name -notcontains 'cases') {
|
|
$problems.Add("$($resultFile.Name): result must contain a 'cases' array.") | Out-Null
|
|
continue
|
|
}
|
|
foreach ($resultCase in @($resultDocument.cases)) {
|
|
$resultCases.Add($resultCase) | Out-Null
|
|
}
|
|
}
|
|
} else {
|
|
if (-not (Test-Path -LiteralPath $ResultsPath -PathType Leaf)) {
|
|
throw "Results file not found: $ResultsPath"
|
|
}
|
|
$resultDocument = Get-Content -LiteralPath $ResultsPath -Raw | ConvertFrom-Json
|
|
foreach ($resultCase in @($resultDocument.cases)) {
|
|
$resultCases.Add($resultCase) | Out-Null
|
|
}
|
|
}
|
|
|
|
$resultById = @{}
|
|
$modelToManifestId = @{}
|
|
foreach ($case in $cases) {
|
|
$manifestId = [string]$case.id
|
|
$modelToManifestId[(Get-ModelCaseId -ManifestId $manifestId)] = $manifestId
|
|
# Also accept manifest IDs for maintainers generating local oracle results.
|
|
$modelToManifestId[$manifestId] = $manifestId
|
|
}
|
|
foreach ($resultCase in @($resultCases)) {
|
|
$rawResultId = [string]$resultCase.id
|
|
if (-not $modelToManifestId.ContainsKey($rawResultId)) {
|
|
$problems.Add("Results contain unknown case id: $rawResultId") | Out-Null
|
|
continue
|
|
}
|
|
$resultId = $modelToManifestId[$rawResultId]
|
|
if ($resultById.ContainsKey($resultId)) {
|
|
$problems.Add("Results contain duplicate case id: $rawResultId") | Out-Null
|
|
} else {
|
|
$resultById[$resultId] = $resultCase
|
|
}
|
|
}
|
|
|
|
$positiveTotal = 0
|
|
$positivePassed = 0
|
|
$cleanTotal = 0
|
|
$cleanPassed = 0
|
|
foreach ($case in $cases) {
|
|
$id = [string]$case.id
|
|
if (-not $resultById.ContainsKey($id)) {
|
|
$problems.Add("Results missing case: $id") | Out-Null
|
|
continue
|
|
}
|
|
|
|
$findingIds = @(
|
|
@($resultById[$id].findings) | ForEach-Object {
|
|
if ($_ -is [string]) { [string]$_ } else { [string]$_.id }
|
|
} | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Sort-Object -Unique
|
|
)
|
|
$expected = @($case.expected | ForEach-Object { [string]$_ })
|
|
|
|
if ($expected.Count) {
|
|
$positiveTotal++
|
|
$missing = @($expected | Where-Object { $findingIds -notcontains $_ })
|
|
$unexpected = @($findingIds | Where-Object { $expected -notcontains $_ })
|
|
if (-not $missing.Count -and -not $unexpected.Count) {
|
|
$positivePassed++
|
|
} else {
|
|
if ($missing.Count) { $problems.Add("${id}: missing expected finding(s): $($missing -join ', ')") | Out-Null }
|
|
if ($unexpected.Count) { $problems.Add("${id}: unexpected finding(s): $($unexpected -join ', ')") | Out-Null }
|
|
}
|
|
} else {
|
|
$cleanTotal++
|
|
if (-not $findingIds.Count) {
|
|
$cleanPassed++
|
|
} else {
|
|
$problems.Add("${id}: clean control produced finding(s): $($findingIds -join ', ')") | Out-Null
|
|
}
|
|
}
|
|
}
|
|
|
|
$recall = if ($positiveTotal) { $positivePassed / $positiveTotal } else { 0 }
|
|
$cleanRate = if ($cleanTotal) { $cleanPassed / $cleanTotal } else { 0 }
|
|
if ($recall -lt [double]$manifest.minimumExpectedRecall) {
|
|
$problems.Add("Expected-finding recall $recall is below $($manifest.minimumExpectedRecall).") | Out-Null
|
|
}
|
|
if ($cleanRate -lt [double]$manifest.minimumCleanRate) {
|
|
$problems.Add("Clean-control rate $cleanRate is below $($manifest.minimumCleanRate).") | Out-Null
|
|
}
|
|
|
|
if ($problems.Count) {
|
|
Write-Host "Review evaluation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
|
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
|
exit 1
|
|
}
|
|
|
|
Write-Host "Review evaluation PASSED: recall=$recall ($positivePassed/$positiveTotal), clean-rate=$cleanRate ($cleanPassed/$cleanTotal)." -ForegroundColor Green
|
|
exit 0
|