bcquality/tools/Test-ReviewFixtures.ps1

<#
.SYNOPSIS
    Validates and prepares the BCQuality AL review evaluation corpus.

.DESCRIPTION
    CI uses the static validation path to prove every registered AL review leaf
    has one positive and one clean control, every fixture/reference exists, and
    the manifest remains internally consistent.

    For an actual model run, -PrepareDirectory copies inputs to neutral names and
    emits review-request.json without expected answers. After the model writes a
    result matching evaluation/README.md, -ResultsPath scores exact knowledge-ID
    recall, clean-control rate, and unexpected findings.
#>
[CmdletBinding()]
param(
    [string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
    [string] $ManifestPath,
    [string] $PrepareDirectory,
    [string] $ResultsPath,
    [string] $ResultsDirectory,
    [string] $ChangedPathsFile,
    [string] $CoverageReportPath
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'

$Root = (Resolve-Path -LiteralPath $Root).Path
if ($ResultsPath -and $ResultsDirectory) {
    throw 'Specify either ResultsPath or ResultsDirectory, not both.'
}
if (-not $ManifestPath) {
    $ManifestPath = Join-Path $Root 'evaluation/review-fixtures.json'
}
if (-not (Test-Path -LiteralPath $ManifestPath)) {
    throw "Review fixture manifest not found: $ManifestPath"
}

$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
$problems = [System.Collections.Generic.List[string]]::new()

function Get-ModelCaseId {
    param([string] $ManifestId)

    $sha = [System.Security.Cryptography.SHA256]::Create()
    try {
        $bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
        $hash = $sha.ComputeHash($bytes)
        $token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
        return "case-$token"
    } finally {
        $sha.Dispose()
    }
}

function Get-RankedArticles {
    param(
        [object[]] $Articles,
        [string] $CaseText,
        [int] $Limit = 10
    )

    if ($Articles.Count -le $Limit) {
        return @($Articles)
    }

    $normalized = (($CaseText.ToLowerInvariant() -replace '[^a-z0-9]+', ' ') -replace '\s+', ' ').Trim()
    $compact = $normalized -replace ' ', ''
    $ranked = foreach ($article in $Articles) {
        $score = 0
        foreach ($keyword in @($article.keywords)) {
            $keywordText = ([string]$keyword).ToLowerInvariant()
            $keywordCompact = $keywordText -replace '[^a-z0-9]+', ''
            if ($keywordCompact -and $compact.Contains($keywordCompact)) {
                $score += 8
            }
            foreach ($part in @($keywordText -split '[^a-z0-9]+')) {
                if (($part.Length -ge 4) -and ($normalized -match "(^| )$([regex]::Escape($part))( |$)")) {
                    $score += 1
                }
            }
        }
        $topicText = "$($article.title) $($article.description) $($article.path)".ToLowerInvariant()
        foreach ($term in @($normalized -split ' ' | Where-Object Length -ge 5 | Sort-Object -Unique)) {
            if ($topicText.Contains($term)) {
                $score += 0.25
            }
        }
        [pscustomobject]@{ score = $score; path = [string]$article.path; article = $article }
    }

    return @(
        $ranked |
            Sort-Object @{ Expression = 'score'; Descending = $true }, @{ Expression = 'path'; Descending = $false } |
            Select-Object -First $Limit |
            ForEach-Object article
    )
}

if ($manifest.version -ne 2) {
    $problems.Add("Unsupported manifest version: $($manifest.version)") | Out-Null
}
if ($manifest.selection -ne 'first-paired-al-article') {
    $problems.Add("Unsupported selection strategy: $($manifest.selection)") | Out-Null
}
if (([double]$manifest.minimumExpectedRecall -lt 0) -or ([double]$manifest.minimumExpectedRecall -gt 1)) {
    $problems.Add('minimumExpectedRecall must be between 0 and 1.') | Out-Null
}
if (([double]$manifest.minimumCleanRate -lt 0) -or ([double]$manifest.minimumCleanRate -gt 1)) {
    $problems.Add('minimumCleanRate must be between 0 and 1.') | Out-Null
}

$layers = @(
    [pscustomobject]@{ Name = 'microsoft'; Rank = 1 }
    [pscustomobject]@{ Name = 'community'; Rank = 2 }
    [pscustomobject]@{ Name = 'custom'; Rank = 3 }
)
$layerRanks = @{}
foreach ($layer in $layers) {
    $layerRanks[[string]$layer.Name] = [int]$layer.Rank
}
$leafCandidates = @(
    foreach ($layer in $layers) {
        $reviewDirectory = Join-Path $Root "$($layer.Name)/skills/review"
        if (-not (Test-Path -LiteralPath $reviewDirectory -PathType Container)) {
            continue
        }
        Get-ChildItem -LiteralPath $reviewDirectory -File -Filter 'al-*-review.md' |
            Where-Object Name -ne 'al-code-review.md' |
            ForEach-Object {
                [pscustomobject]@{
                    Domain = $_.BaseName -replace '^al-', '' -replace '-review$', ''
                    Layer = $layer.Name
                    Rank = $layer.Rank
                    RelativePath = [System.IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
                }
            }
    }
)
$leafSkills = @(
    $leafCandidates |
        Group-Object Domain |
        ForEach-Object { $_.Group | Sort-Object Rank -Descending | Select-Object -First 1 } |
        Sort-Object Domain
)
$leafDomains = @($leafSkills | ForEach-Object Domain)
$leafByDomain = @{}
foreach ($leafSkill in $leafSkills) {
    $leafByDomain[[string]$leafSkill.Domain] = $leafSkill
}

$overrides = @{}
if ($manifest.PSObject.Properties.Name -contains 'overrides') {
    foreach ($property in $manifest.overrides.PSObject.Properties) {
        $overrides[$property.Name] = $property.Value
    }
}
foreach ($overrideDomain in $overrides.Keys) {
    if ($leafDomains -notcontains $overrideDomain) {
        $problems.Add("Override domain '$overrideDomain' has no registered al-$overrideDomain-review leaf.") | Out-Null
    }
}

$coverageWaivers = @{}
if ($manifest.PSObject.Properties.Name -contains 'coverageWaivers') {
    foreach ($waiver in @($manifest.coverageWaivers)) {
        if (-not $waiver.path -or -not $waiver.reason) {
            $problems.Add('Each coverage waiver requires non-empty path and reason values.') | Out-Null
            continue
        }
        if ($coverageWaivers.ContainsKey([string]$waiver.path)) {
            $problems.Add("Duplicate coverage waiver: $($waiver.path)") | Out-Null
            continue
        }
        $coverageWaivers[[string]$waiver.path] = [string]$waiver.reason
    }
}

$caseList = [System.Collections.Generic.List[object]]::new()
$pairedArticlesByDomain = @{}
foreach ($domain in $leafDomains) {
    $articleCandidates = @(
        foreach ($layer in $layers) {
            $knowledgeDirectory = Join-Path $Root "$($layer.Name)/knowledge/$domain"
            if (-not (Test-Path -LiteralPath $knowledgeDirectory -PathType Container)) {
                continue
            }
            Get-ChildItem -LiteralPath $knowledgeDirectory -File -Filter '*.md' |
                Where-Object {
                    (Test-Path -LiteralPath (Join-Path $knowledgeDirectory "$($_.BaseName).good.al") -PathType Leaf) -and
                    (Test-Path -LiteralPath (Join-Path $knowledgeDirectory "$($_.BaseName).bad.al") -PathType Leaf)
                } |
                ForEach-Object {
                    [pscustomobject]@{
                        BaseName = $_.BaseName
                        File = $_
                        Rank = $layer.Rank
                        ArticlePath = [System.IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
                    }
                }
        }
    )
    $articles = @(
        $articleCandidates |
            Group-Object BaseName |
            ForEach-Object { $_.Group | Sort-Object Rank -Descending | Select-Object -First 1 } |
            Sort-Object BaseName
    )
            $pairedArticlesByDomain[$domain] = @($articles)
    if (-not $articles.Count) {
        $problems.Add("${domain}: no enabled knowledge layer has an article with both .good.al and .bad.al companion samples.") | Out-Null
        continue
    }

    $override = if ($overrides.ContainsKey($domain)) { $overrides[$domain] } else { $null }
    $hasArticleOverride = $override -and ($override.PSObject.Properties.Name -contains 'article')
    $hasArticlesOverride = $override -and ($override.PSObject.Properties.Name -contains 'articles')
    if ($hasArticleOverride -and $hasArticlesOverride) {
        $problems.Add("${domain}: override must specify either 'article' or 'articles', not both.") | Out-Null
        continue
    }

    $articleNames = @()
    if ($hasArticlesOverride) {
        $articleNames = @($override.articles)
        if (-not $articleNames.Count) {
            $problems.Add("${domain}: override 'articles' must contain at least one article.") | Out-Null
            continue
        }
    } elseif ($hasArticleOverride) {
        $articleNames = @($override.article)
    } else {
        $articleNames = @($articles | Select-Object -First 1 | ForEach-Object BaseName)
    }

    $selectedArticles = [System.Collections.Generic.List[object]]::new()
    $seenArticleNames = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
    foreach ($articleNameValue in $articleNames) {
        if ($articleNameValue -isnot [string] -or [string]::IsNullOrWhiteSpace([string]$articleNameValue)) {
            $problems.Add("${domain}: override article names must be non-empty strings.") | Out-Null
            continue
        }
        $articleName = [string]$articleNameValue
        if ($articleName.EndsWith('.md')) {
            $articleName = [System.IO.Path]::GetFileNameWithoutExtension($articleName)
        }
        if (-not $seenArticleNames.Add($articleName)) {
            $problems.Add("${domain}: override contains duplicate article: $articleName.md") | Out-Null
            continue
        }

        $selectedArticle = $articles | Where-Object BaseName -eq $articleName | Select-Object -First 1
        if (-not $selectedArticle) {
            $articleExists = @(
                foreach ($layer in $layers) {
                    $articleFile = Join-Path $Root "$($layer.Name)/knowledge/$domain/$articleName.md"
                    if (Test-Path -LiteralPath $articleFile -PathType Leaf) {
                        $articleFile
                    }
                }
            ).Count -gt 0
            if ($articleExists) {
                $problems.Add("${domain}: override article does not have both .good.al and .bad.al companion samples: $articleName.md") | Out-Null
            } else {
                $problems.Add("${domain}: override article does not exist: $articleName.md") | Out-Null
            }
            continue
        }
        $selectedArticles.Add($selectedArticle) | Out-Null
    }
    if (-not $selectedArticles.Count) {
        if (-not $articleNames.Count) {
            $problems.Add("${domain}: no article has both .good.al and .bad.al companion samples.") | Out-Null
        }
        continue
    }

    $context = if ($override -and ($override.PSObject.Properties.Name -contains 'context')) {
        [string]$override.context
    } else {
        $null
    }
    for ($articleIndex = 0; $articleIndex -lt $selectedArticles.Count; $articleIndex++) {
        $selectedArticle = $selectedArticles[$articleIndex]
        $articlePath = [string]$selectedArticle.ArticlePath
        $sampleDirectory = (Split-Path -Parent $articlePath).Replace('\', '/')
        foreach ($kind in 'bad', 'good') {
            $caseId = if ($articleIndex -eq 0) {
                "$domain-$kind"
            } else {
                "$domain-$($selectedArticle.BaseName)-$kind"
            }
            $case = [pscustomobject]@{
                id = $caseId
                domain = $domain
                input = "$sampleDirectory/$($selectedArticle.BaseName).$kind.al"
                expected = if ($kind -eq 'bad') { @($articlePath) } else { @() }
            }
            if ($context) {
                $case | Add-Member -NotePropertyName context -NotePropertyValue $context
            }
            $caseList.Add($case) | Out-Null
        }
    }

}
$cases = @($caseList)

$seenIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
foreach ($case in $cases) {
    $id = [string]$case.id
    $domain = [string]$case.domain
    $inputRelativePath = [string]$case.input
    $expected = @($case.expected)

    if ([string]::IsNullOrWhiteSpace($id)) {
        $problems.Add('Case with empty id.') | Out-Null
    } elseif (-not $seenIds.Add($id)) {
        $problems.Add("Duplicate case id: $id") | Out-Null
    }
    if ($leafDomains -notcontains $domain) {
        $problems.Add("${id}: domain '$domain' has no registered al-$domain-review leaf.") | Out-Null
    }

    $inputPath = Join-Path $Root $inputRelativePath
    if (-not (Test-Path -LiteralPath $inputPath -PathType Leaf)) {
        $problems.Add("${id}: input does not exist: $inputRelativePath") | Out-Null
    }
    if ($expected.Count -and $inputRelativePath -notmatch '\.bad\.[^.]+$') {
        $problems.Add("${id}: positive case must use a .bad sample: $inputRelativePath") | Out-Null
    }
    if (-not $expected.Count -and $inputRelativePath -notmatch '\.good\.[^.]+$') {
        $problems.Add("${id}: clean case must use a .good sample: $inputRelativePath") | Out-Null
    }

    foreach ($reference in $expected) {
        $referencePath = Join-Path $Root ([string]$reference)
        if (-not (Test-Path -LiteralPath $referencePath -PathType Leaf)) {
            $problems.Add("${id}: referenced article does not exist: $reference") | Out-Null
        }
    }
    if ($expected.Count) {
        $sampleSlug = ([System.IO.Path]::GetFileName($inputRelativePath) -replace '\.(?:good|bad)\.[^.]+$', '')
        $primarySlug = [System.IO.Path]::GetFileNameWithoutExtension([string]$expected[0])
        if ($sampleSlug -ne $primarySlug) {
            $problems.Add("${id}: primary expected article '$primarySlug' must match sample slug '$sampleSlug'.") | Out-Null
        }
    }
}

foreach ($domain in $leafDomains) {
    $domainCases = @($cases | Where-Object domain -eq $domain)
    if (-not @($domainCases | Where-Object { @($_.expected).Count -gt 0 }).Count) {
        $problems.Add("${domain}: no positive review fixture.") | Out-Null
    }
    if (-not @($domainCases | Where-Object { @($_.expected).Count -eq 0 }).Count) {
        $problems.Add("${domain}: no clean control fixture.") | Out-Null
    }
}

$selectedArticlePaths = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
foreach ($case in $cases) {
    foreach ($reference in @($case.expected)) {
        $selectedArticlePaths.Add([string]$reference) | Out-Null
    }
}
$effectivePairedPaths = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
$coverageDomains = @(
    foreach ($domain in $leafDomains) {
        $paired = @($pairedArticlesByDomain[$domain])
        foreach ($article in $paired) {
            $effectivePairedPaths.Add([string]$article.ArticlePath) | Out-Null
        }
        $selected = @($paired | Where-Object { $selectedArticlePaths.Contains([string]$_.ArticlePath) }).Count
        [pscustomobject][ordered]@{
            domain = $domain
            pairedArticles = $paired.Count
            selectedArticles = $selected
            coverage = if ($paired.Count) { $selected / $paired.Count } else { 0 }
        }
    }
)
$pairedTotal = ($coverageDomains | Measure-Object pairedArticles -Sum).Sum
$selectedTotal = ($coverageDomains | Measure-Object selectedArticles -Sum).Sum
$coverageReport = [pscustomobject][ordered]@{
    pairedArticles = $pairedTotal
    selectedArticles = $selectedTotal
    coverage = if ($pairedTotal) { $selectedTotal / $pairedTotal } else { 0 }
    domains = $coverageDomains
}
if ($CoverageReportPath) {
    $coverageParent = Split-Path -Parent $CoverageReportPath
    if ($coverageParent -and -not (Test-Path -LiteralPath $coverageParent)) {
        New-Item -ItemType Directory -Path $coverageParent -Force | Out-Null
    }
    $coverageReport | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $CoverageReportPath -Encoding utf8NoBOM
}

if ($ChangedPathsFile) {
    if (-not (Test-Path -LiteralPath $ChangedPathsFile -PathType Leaf)) {
        $problems.Add("Changed paths file not found: $ChangedPathsFile") | Out-Null
    }
    else {
        foreach ($changedPathValue in Get-Content -LiteralPath $ChangedPathsFile) {
            $changedPath = ([string]$changedPathValue).Trim().Replace('\', '/')
            if ($changedPath -notmatch '^(microsoft|community|custom)/knowledge/[^/]+/(.+?)(?:\.(?:good|bad)\.al|\.md)$') {
                continue
            }
            $articlePath = "$($Matches[1])/knowledge/$($changedPath.Split('/')[2])/$($Matches[2]).md"
            if (-not $effectivePairedPaths.Contains($articlePath) -or $selectedArticlePaths.Contains($articlePath)) {
                continue
            }
            if (-not $coverageWaivers.ContainsKey($articlePath)) {
                $problems.Add("Changed paired article is not selected for evaluation and has no coverage waiver: $articlePath") | Out-Null
            }
        }
    }
}

if ($problems.Count) {
    Write-Host "Review fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
    $problems | ForEach-Object { Write-Host "  - $_" -ForegroundColor Red }
    exit 1
}

if ($PrepareDirectory) {
    $markerPath = Join-Path $PrepareDirectory '.bcquality-evaluation'
    if (Test-Path -LiteralPath $PrepareDirectory) {
        $existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
        if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
            throw "PrepareDirectory is not empty and is not a BCQuality evaluation directory: $PrepareDirectory"
        }
        if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
            Get-ChildItem -LiteralPath $PrepareDirectory -File |
                Where-Object {
                    ($_.Name -like 'case*.al') -or
                    ($_.Name -eq 'review-request.json') -or
                    ($_.Name -like 'request-*.json') -or
                    ($_.Name -eq 'knowledge-index.json') -or
                    ($_.Name -like 'index-*.json') -or
                    ($_.Name -like 'result-*.json')
                } |
                Remove-Item -Force
        }
    } else {
        New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
    }
    Set-Content -LiteralPath $markerPath -Value 'BCQuality generated evaluation directory' -Encoding UTF8

    $fullIndexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
    & (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $fullIndexPath | Out-Null
    $fullIndex = Get-Content -LiteralPath $fullIndexPath -Raw | ConvertFrom-Json

    $requestCases = [System.Collections.Generic.List[object]]::new()
    $requestCasesByDomain = @{}
    $manifestCaseByModelId = @{}
    foreach ($case in $cases) {
        $extension = [System.IO.Path]::GetExtension([string]$case.input)
        $modelId = Get-ModelCaseId -ManifestId ([string]$case.id)
        $neutralName = "$modelId$extension"
        $sourceText = Get-Content -LiteralPath (Join-Path $Root ([string]$case.input)) -Raw
        # Companion samples are human-facing and often label objects/comments as
        # Good, Bad, or Anti-pattern. Strip full-line comments and neutralize those
        # object-name tokens so model-facing fixtures do not reveal the expected
        # outcome while preserving executable AL structure and references.
        $neutralText = [regex]::Replace($sourceText, '(?m)^\s*//.*(?:\r?\n|$)', '')
        $neutralText = [regex]::Replace($neutralText, '\b(?:Good|Bad)\b', 'Eval')
        Set-Content -LiteralPath (Join-Path $PrepareDirectory $neutralName) -Value $neutralText -Encoding UTF8
        $requestCase = [pscustomobject]@{ id = $modelId; file = $neutralName }
        if ($case.PSObject.Properties.Name -contains 'context') {
            $requestCase | Add-Member -NotePropertyName context -NotePropertyValue ([string]$case.context)
        }
        $requestCases.Add($requestCase) | Out-Null
        $manifestCaseByModelId[$modelId] = $case
        $domain = [string]$case.domain
        if (-not $requestCasesByDomain.ContainsKey($domain)) {
            $requestCasesByDomain[$domain] = [System.Collections.Generic.List[object]]::new()
        }
        $requestCasesByDomain[$domain].Add($requestCase) | Out-Null
    }
    $resultSchema = [pscustomobject]@{
        cases = @([pscustomobject]@{
            id = 'case-id'
            findings = @([pscustomobject]@{ id = 'repo-relative knowledge article path' })
        })
    }
    [pscustomobject]@{
        protocol = 'Run BCQuality al-code-review over all files as one PR; return findings per case. Copy every knowledge-backed id from knowledge-index.json.'
        knowledgeIndex = 'knowledge-index.json'
        resultSchema = $resultSchema
        cases = @($requestCases)
    } | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $PrepareDirectory 'review-request.json') -Encoding UTF8

    foreach ($domain in $leafDomains) {
        $domainArticles = @(
            $fullIndex.articles |
                Where-Object domain -eq $domain |
                Sort-Object @{ Expression = { $layerRanks[[string]$_.layer] }; Descending = $true }, path |
                Group-Object { [System.IO.Path]::GetFileName([string]$_.path) } |
                ForEach-Object { $_.Group | Select-Object -First 1 } |
                Sort-Object path
        )
        $domainIndexName = "index-$domain.json"
        $leafPath = [string]$leafByDomain[$domain].RelativePath
        $leafFullText = Get-Content -LiteralPath (Join-Path $Root $leafPath) -Raw
        $leafInstructions = @($leafFullText -split '(?m)^## Output\s*\r?\n', 2)[0]
        $leafInstructions += "`n## Output`nReturn only the request's resultSchema."
        [pscustomobject]@{
            version = $fullIndex.version
            domain = $domain
            articleCount = $domainArticles.Count
            articles = $domainArticles
        } | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $PrepareDirectory $domainIndexName) -Encoding UTF8

        [pscustomobject]@{
            protocol = "Run only $leafPath over these files. Follow leafInstructions exactly, use only the supplied candidate article rows, open matching articles in full, and copy every finding id verbatim from candidateArticles[].path."
            skill = $leafPath
            leafInstructions = $leafInstructions
            knowledgeIndex = $domainIndexName
            candidateArticles = $domainArticles
            resultSchema = $resultSchema
            cases = @($requestCasesByDomain[$domain])
        } | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$domain.json") -Encoding UTF8

        foreach ($requestCase in @($requestCasesByDomain[$domain])) {
            $caseText = Get-Content -LiteralPath (Join-Path $PrepareDirectory ([string]$requestCase.file)) -Raw
            if ($requestCase.PSObject.Properties.Name -contains 'context') {
                $caseText += " $([string]$requestCase.context)"
            }
            $rankedArticles = @(Get-RankedArticles -Articles $domainArticles -CaseText $caseText)
            $manifestCase = $manifestCaseByModelId[[string]$requestCase.id]
            $selectedArticlePath = ([string]$manifestCase.input) -replace '\.(?:good|bad)\.al$', '.md'
            $rankedPaths = @($rankedArticles | ForEach-Object { [string]$_.path })
            if ($rankedPaths -notcontains $selectedArticlePath) {
                throw "$($manifestCase.id): deterministic ranking omitted selected article '$selectedArticlePath'. Improve its retrieval metadata or choose an exceptional override article."
            }
            # Candidate order must not reveal which article owns the fixture.
            $rankedArticles = @($rankedArticles | Sort-Object path)
            [pscustomobject]@{
                protocol = "Run only $leafPath over this case. Follow leafInstructions exactly, evaluate the ranked candidate article rows, open matching articles in full, and copy every finding id verbatim from candidateArticles[].path."
                skill = $leafPath
                leafInstructions = $leafInstructions
                knowledgeIndex = $domainIndexName
                candidateArticles = $rankedArticles
                resultSchema = $resultSchema
                cases = @($requestCase)
            } | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$($requestCase.id).json") -Encoding UTF8
        }
    }
    Write-Host "Prepared $($cases.Count) neutral fixture(s) in $PrepareDirectory." -ForegroundColor Green
}

if (-not $ResultsPath -and -not $ResultsDirectory) {
    & (Join-Path $PSScriptRoot 'Test-ReviewContract.ps1') -Root $Root
    Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $selectedTotal/$pairedTotal paired articles across $($leafDomains.Count) leaf domains." -ForegroundColor Green
    exit 0
}

$resultCases = [System.Collections.Generic.List[object]]::new()
if ($ResultsDirectory) {
    if (-not (Test-Path -LiteralPath $ResultsDirectory -PathType Container)) {
        throw "Results directory not found: $ResultsDirectory"
    }
    $resultFiles = @(Get-ChildItem -LiteralPath $ResultsDirectory -File -Filter 'result-case-*.json')
    if (-not $resultFiles.Count) {
        $resultFiles = @(Get-ChildItem -LiteralPath $ResultsDirectory -File -Filter 'result-*.json')
    }
    if (-not $resultFiles.Count) {
        throw "No result-case-*.json or result-*.json files found in: $ResultsDirectory"
    }
    foreach ($resultFile in $resultFiles) {
        try {
            $resultDocument = Get-Content -LiteralPath $resultFile.FullName -Raw | ConvertFrom-Json
        } catch {
            $problems.Add("$($resultFile.Name): invalid JSON: $($_.Exception.Message)") | Out-Null
            continue
        }
        if ($resultDocument.PSObject.Properties.Name -notcontains 'cases') {
            $problems.Add("$($resultFile.Name): result must contain a 'cases' array.") | Out-Null
            continue
        }
        foreach ($resultCase in @($resultDocument.cases)) {
            $resultCases.Add($resultCase) | Out-Null
        }
    }
} else {
    if (-not (Test-Path -LiteralPath $ResultsPath -PathType Leaf)) {
        throw "Results file not found: $ResultsPath"
    }
    $resultDocument = Get-Content -LiteralPath $ResultsPath -Raw | ConvertFrom-Json
    foreach ($resultCase in @($resultDocument.cases)) {
        $resultCases.Add($resultCase) | Out-Null
    }
}

$resultById = @{}
$modelToManifestId = @{}
foreach ($case in $cases) {
    $manifestId = [string]$case.id
    $modelToManifestId[(Get-ModelCaseId -ManifestId $manifestId)] = $manifestId
    # Also accept manifest IDs for maintainers generating local oracle results.
    $modelToManifestId[$manifestId] = $manifestId
}
foreach ($resultCase in @($resultCases)) {
    $rawResultId = [string]$resultCase.id
    if (-not $modelToManifestId.ContainsKey($rawResultId)) {
        $problems.Add("Results contain unknown case id: $rawResultId") | Out-Null
        continue
    }
    $resultId = $modelToManifestId[$rawResultId]
    if ($resultById.ContainsKey($resultId)) {
        $problems.Add("Results contain duplicate case id: $rawResultId") | Out-Null
    } else {
        $resultById[$resultId] = $resultCase
    }
}

$positiveTotal = 0
$positivePassed = 0
$cleanTotal = 0
$cleanPassed = 0
foreach ($case in $cases) {
    $id = [string]$case.id
    if (-not $resultById.ContainsKey($id)) {
        $problems.Add("Results missing case: $id") | Out-Null
        continue
    }

    $findingIds = @(
        @($resultById[$id].findings) | ForEach-Object {
            if ($_ -is [string]) { [string]$_ } else { [string]$_.id }
        } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Sort-Object -Unique
    )
    $expected = @($case.expected | ForEach-Object { [string]$_ })

    if ($expected.Count) {
        $positiveTotal++
        $missing = @($expected | Where-Object { $findingIds -notcontains $_ })
        $unexpected = @($findingIds | Where-Object { $expected -notcontains $_ })
        if (-not $missing.Count -and -not $unexpected.Count) {
            $positivePassed++
        } else {
            if ($missing.Count) { $problems.Add("${id}: missing expected finding(s): $($missing -join ', ')") | Out-Null }
            if ($unexpected.Count) { $problems.Add("${id}: unexpected finding(s): $($unexpected -join ', ')") | Out-Null }
        }
    } else {
        $cleanTotal++
        if (-not $findingIds.Count) {
            $cleanPassed++
        } else {
            $problems.Add("${id}: clean control produced finding(s): $($findingIds -join ', ')") | Out-Null
        }
    }
}

$recall = if ($positiveTotal) { $positivePassed / $positiveTotal } else { 0 }
$cleanRate = if ($cleanTotal) { $cleanPassed / $cleanTotal } else { 0 }
if ($recall -lt [double]$manifest.minimumExpectedRecall) {
    $problems.Add("Expected-finding recall $recall is below $($manifest.minimumExpectedRecall).") | Out-Null
}
if ($cleanRate -lt [double]$manifest.minimumCleanRate) {
    $problems.Add("Clean-control rate $cleanRate is below $($manifest.minimumCleanRate).") | Out-Null
}

if ($problems.Count) {
    Write-Host "Review evaluation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
    $problems | ForEach-Object { Write-Host "  - $_" -ForegroundColor Red }
    exit 1
}

Write-Host "Review evaluation PASSED: recall=$recall ($positivePassed/$positiveTotal), clean-rate=$cleanRate ($cleanPassed/$cleanTotal)." -ForegroundColor Green
exit 0