bcquality/tools/Build-KnowledgeIndex.ps1

<#
.SYNOPSIS
    Builds the BCQuality knowledge index — the discovery artifact the review
    skills' §Source step consumes.

.DESCRIPTION
    BCQuality owns the knowledge-index contract: the index schema is part of
    the Source step that the action skills (e.g. al-*-review.md) declare, so
    the generator lives here, next to the skills and knowledge it derives from.
    Consumers (orchestrators such as the BCAppsBCQuality PR-review filter) call
    this script instead of re-implementing the parser, so every consumer gets
    the same faithful index for free and the index stays in lockstep with the
    skill contract.

    The index lets the agent enumerate candidate articles and compute the
    keyword/topic worklist overlap by reading ONE file, instead of opening
    every file under `*/knowledge/<domain>/**` just to read its frontmatter.
    The worklist SELECTION predicate is unchanged: the index carries the same
    inputs the predicate already reads (keywords + frontmatter dimensions +
    domain + path + title), and the agent still opens each worklisted article
    in full for its `## Best Practice` / `## Anti Pattern` rule bodies.

    The index is LEAN by default: the verbatim Description is trimmed to a
    one-line hint and the JSON is emitted compact, so the index prefix the
    agent replays across passes stays small. The selection inputs remain
    lossless. Pass -FullIndex for the verbatim, pretty-printed variant.

    This script does NOT apply layer/allow-deny policy by reading config — it
    indexes whatever knowledge files are present on disk (a consumer is
    expected to prune its clone to policy first). For provenance and to
    reproduce a consumer's exact view, pass -EnabledLayers to restrict the walk
    to those layers and to record the policy in the index header.
    Invalid articles are omitted with a path-specific warning so one bad
    optional layer article cannot block valid siblings.

.PARAMETER BCQualityRoot
    Path to the BCQuality content root to index (typically a filtered clone).
    Defaults to the clone root (the parent of this script's `tools/` folder), so
    the agent can run `pwsh ./tools/Build-KnowledgeIndex.ps1` from the clone root
    with no arguments.

.PARAMETER IndexPath
    Where to write the index JSON. Defaults to `<BCQualityRoot>/knowledge-index.json`.

.PARAMETER EnabledLayers
    Optional layer allowlist (e.g. microsoft, community, custom). When provided,
    only those layers are walked and the value is recorded in the index header.
    Omit to index every layer present on disk.

.PARAMETER KnowledgeAllow
    Optional allow globs to record in the index header (provenance only).

.PARAMETER KnowledgeDeny
    Optional deny globs to record in the index header (provenance only).

.PARAMETER FullIndex
    Emit the verbatim-Description, pretty-printed index instead of the lean one.

.OUTPUTS
    Returns the number of articles written to the index.
#>
[CmdletBinding()]
param(
    [string] $BCQualityRoot,
    [string] $IndexPath,
    [string[]] $EnabledLayers,
    [string[]] $KnowledgeAllow = @(),
    [string[]] $KnowledgeDeny = @(),
    [switch] $FullIndex
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')

if ($PSBoundParameters.ContainsKey('EnabledLayers')) {
    if ($null -eq $EnabledLayers) {
        throw 'EnabledLayers must be an array; omit it to index all layers.'
    }
    if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
        @($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
        throw 'EnabledLayers must contain unique canonical lowercase layer names.'
    }
}

# Default to the clone root (parent of this script's tools/ folder) so the
# agent's Entry preparation step can invoke this with no arguments from the
# checkout root. A consumer/orchestrator may still pass -BCQualityRoot.
if (-not $BCQualityRoot) {
    $BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
}
if (-not (Test-Path $BCQualityRoot)) {
    throw "BCQuality root not found: $BCQualityRoot"
}
# Normalise to an absolute path: Get-ChildItem.FullName below is always
# absolute, so Get-RelativePath's Substring needs an absolute root to strip.
# A relative root (e.g. '.') would otherwise leave the full path intact.
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
if (-not $IndexPath) {
    $IndexPath = Join-Path $BCQualityRoot 'knowledge-index.json'
}

function Get-RelativePath {
    param([string] $Root, [string] $Full)
    $rel = $Full.Substring($Root.Length).TrimStart([char]'/', [char]'\')
    return ($rel -replace '\\', '/')
}

function Get-BytesSha256 {
    param([byte[]] $Bytes)
    $sha = [Security.Cryptography.SHA256]::Create()
    try {
        return ([BitConverter]::ToString($sha.ComputeHash($Bytes)) -replace '-', '').ToLowerInvariant()
    }
    finally {
        $sha.Dispose()
    }
}

function Read-ArticleSource {
    param([string] $Path)
    $bytes = [IO.File]::ReadAllBytes($Path)
    try {
        $text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
    }
    catch [Text.DecoderFallbackException] {
        throw [IO.InvalidDataException]::new('invalid UTF-8', $_.Exception)
    }
    return [pscustomobject]@{
        bytes = $bytes
        text = $text
        sha256 = Get-BytesSha256 -Bytes $bytes
    }
}

function Get-ValueSha256 {
    param([Parameter(Mandatory)] $Value)
    $bytes = [Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject $Value -Depth 8 -Compress))
    $sha = [Security.Cryptography.SHA256]::Create()
    try {
        return ([BitConverter]::ToString($sha.ComputeHash($bytes)) -replace '-', '').ToLowerInvariant()
    }
    finally {
        $sha.Dispose()
    }
}

# Trims a Description to a single short line (<= $Max chars) for the lean
# index. Takes the first sentence; truncates on a word boundary if still long.
function Get-LeanDescription {
    param([string] $Text, [int] $Max = 120)
    if ([string]::IsNullOrWhiteSpace($Text)) { return '' }
    $t = ($Text -replace '\s+', ' ').Trim()
    $m = [regex]::Match($t, '^(.*?[\.!?])(\s|$)')
    if ($m.Success) { $t = $m.Groups[1].Value.Trim() }
    if ($t.Length -le $Max) { return $t }
    $cut = $t.Substring(0, $Max)
    $sp = $cut.LastIndexOf(' ')
    if ($sp -gt 40) { $cut = $cut.Substring(0, $sp) }
    return ($cut.TrimEnd() + '…')
}

function ConvertFrom-ArticleFrontmatter {
    # Parses a knowledge file into the fields the knowledge index needs.
    # Captures the verbatim frontmatter dimensions/keywords, the H1 title,
    # and the full Description section (the article's primary retrieval
    # target per READ). No rule-body content (## Best Practice / ## Anti
    # Pattern) is included; the index is a lossless substitute for the
    # frontmatter + Description the worklist predicate reads, not a
    # substitute for the article's normative guidance.
    param(
        [string] $Path,
        [string] $Text
    )

    $lines = [regex]::Split($Text.TrimStart([char]0xfeff), '\r\n|\n|\r')

    # Frontmatter is the first '---'-delimited block.
    if ($lines.Count -lt 1 -or $lines[0].Trim() -ne '---') { return $null }
    $fmEnd = -1
    for ($i = 1; $i -lt $lines.Count; $i++) {
        if ($lines[$i].Trim() -eq '---') { $fmEnd = $i; break }
    }
    if ($fmEnd -lt 0) { return $null }

    $fm = @{}
    $arrayFields = @('bc-version', 'keywords', 'technologies', 'countries', 'application-area')
    for ($i = 1; $i -lt $fmEnd; $i++) {
        $line = $lines[$i]
        if ($line -match '^\s*([a-zA-Z][\w-]*)\s*:\s*(.*)$') {
            $key = $Matches[1]
            $val = $Matches[2].Trim()
            if ($key -in $arrayFields) {
                if ($val -notmatch '^\[(.*)\]$') {
                    throw [IO.InvalidDataException]::new(
                        "frontmatter field '$key' must use non-empty bracket-array syntax"
                    )
                }
                $inner = $Matches[1].Trim()
                if ($inner -eq '') {
                    throw [IO.InvalidDataException]::new(
                        "frontmatter field '$key' must use non-empty bracket-array syntax"
                    )
                }
                $values = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() })
                if (@($values | Where-Object { [string]::IsNullOrWhiteSpace($_) }).Count) {
                    throw [IO.InvalidDataException]::new(
                        "frontmatter field '$key' must use non-empty bracket-array syntax"
                    )
                }
                $fm[$key] = $values
            }
            elseif ($val -ne '') { $fm[$key] = $val }
        }
    }
    foreach ($field in $arrayFields) {
        if (-not $fm.ContainsKey($field) -or $fm[$field] -isnot [array] -or -not $fm[$field].Count) {
            throw [IO.InvalidDataException]::new(
                "frontmatter field '$field' must use non-empty bracket-array syntax"
            )
        }
    }

    # Body parsing: H1 title and the full Description section. The Description
    # is the article's primary retrieval target per READ and is captured
    # verbatim (it carries no rule-body guidance and no fenced code per the
    # schema), so the index is a lossless substitute for the frontmatter +
    # Description that the worklist predicate reads. Normative rule bodies
    # (## Best Practice / ## Anti Pattern) are deliberately NOT included.
    $title = ''
    $description = ''
    $inDescription = $false
    $descBuffer = [System.Collections.Generic.List[string]]::new()
    for ($i = $fmEnd + 1; $i -lt $lines.Count; $i++) {
        $line = $lines[$i]
        if (-not $title -and $line -match '^\#\s+(.+?)\s*$') { $title = $Matches[1].Trim(); continue }
        if ($line -match '^\#\#\s+Description\s*$') { $inDescription = $true; continue }
        if ($inDescription) {
            if ($line -match '^\#\#\s') { break }   # next section ends Description
            if ($line.Trim() -ne '') { $descBuffer.Add($line.Trim()) | Out-Null }
        }
    }
    if ($descBuffer.Count -gt 0) { $description = ($descBuffer -join ' ').Trim() }

    return [pscustomobject]@{
        domain            = if ($fm.ContainsKey('domain')) { [string]$fm['domain'] } else { '' }
        'bc-version'      = @($fm['bc-version'])
        technologies      = @($fm['technologies'])
        countries         = @($fm['countries'])
        'application-area'= @($fm['application-area'])
        keywords          = @($fm['keywords'])
        title             = $title
        description       = $description
    }
}

# Walk the knowledge files and emit a single compact discovery artifact so
# consumers can enumerate candidate articles and compute keyword/topic worklist
# overlap without opening every file. When -EnabledLayers is supplied the walk
# is restricted to those layers (a consumer reproduces its filtered view); the
# article set otherwise reflects whatever is present on disk.
$indexArticles = [System.Collections.Generic.List[object]]::new()
foreach ($layerDir in @('microsoft', 'community', 'custom')) {
    $kbRoot = Join-Path $BCQualityRoot (Join-Path $layerDir 'knowledge')
    if (-not (Test-Path $kbRoot)) { continue }
    if ($EnabledLayers -and ($EnabledLayers -cnotcontains $layerDir)) { continue }

    $files = @(
        Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
            Sort-Object FullName
    )
    foreach ($file in $files) {
        $rel = Get-RelativePath -Root $BCQualityRoot -Full $file.FullName
        try {
            $source = Read-ArticleSource -Path $file.FullName
            $parsed = ConvertFrom-ArticleFrontmatter -Path $file.FullName -Text $source.text
            if (-not $parsed) {
                throw [IO.InvalidDataException]::new('missing or unterminated frontmatter')
            }
            foreach ($required in @(
                @('domain', $parsed.domain),
                @('H1 title', $parsed.title),
                @('Description', $parsed.description)
            )) {
                if ([string]::IsNullOrWhiteSpace([string]$required[1])) {
                    throw [IO.InvalidDataException]::new("missing $($required[0])")
                }
            }
        }
        catch [IO.InvalidDataException] {
            Write-Warning "Skipping invalid knowledge article '$rel': $($_.Exception.Message)."
            continue
        }

        $article = [ordered]@{
            path               = $rel
            layer              = $layerDir
            domain             = $parsed.domain
            'bc-version'       = @($parsed.'bc-version')
            technologies       = @($parsed.technologies)
            countries          = @($parsed.countries)
            'application-area' = @($parsed.'application-area')
            keywords           = @($parsed.keywords)
            title              = $parsed.title
            description        = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
            parsed             = $true
            sourceSha256       = $source.sha256
        }
        $problem = Get-KnowledgeMetadataProblem -Row $article
        if ($problem) {
            Write-Warning "Skipping invalid knowledge article '$rel': $problem."
            continue
        }
        $indexArticles.Add($article) | Out-Null
    }
}

$articlesByPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
foreach ($article in $indexArticles) {
    if (-not $articlesByPath.TryAdd($article.path, $article)) {
        throw "Duplicate knowledge path while building source snapshot: $($article.path)"
    }
}
$sourcePaths = [string[]]@($articlesByPath.Keys)
[Array]::Sort($sourcePaths, [StringComparer]::Ordinal)
$sourceManifest = @(
    foreach ($path in $sourcePaths) {
        [ordered]@{ path = $path; sha256 = $articlesByPath[$path].sourceSha256 }
    }
)
$index = [pscustomobject]@{
    version       = 1
    generatedAt   = (Get-Date).ToUniversalTime().ToString('o')
    enabledLayers = @($EnabledLayers)
    knowledgeAllow= @($KnowledgeAllow)
    knowledgeDeny = @($KnowledgeDeny)
    articleCount  = $indexArticles.Count
    sourceSnapshot= Get-ValueSha256 -Value $sourceManifest
    articles      = @($indexArticles)
}

$indexDir = Split-Path -Parent $IndexPath
if ($indexDir -and -not (Test-Path $indexDir)) {
    New-Item -ItemType Directory -Force -Path $indexDir | Out-Null
}
if ($FullIndex) {
    $index | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $IndexPath -Encoding UTF8
} else {
    $index | ConvertTo-Json -Depth 8 -Compress | Set-Content -LiteralPath $IndexPath -Encoding UTF8
}

Write-Host "BCQuality index: $($indexArticles.Count) article(s). Index: $IndexPath"
return $indexArticles.Count