From 6ed56b95b7f57d53dfc4e2645c185b32fa69e849 Mon Sep 17 00:00:00 2001 From: dayland Date: Mon, 13 Jul 2026 11:20:30 +0100 Subject: [PATCH] R6 Tier-1: content-owned routing index generator + CI guard Retire the orchestrator's hardcoded ~28-token signal catalog by compiling a routing index from article front-matter. New tools/Build-RoutingIndex.ps1 scans all knowledge layers and emits routing-index.json (signal-token -> domain + backing articles), seeded from the migrated legacy catalog (routing-seed.json) so recall is never below today. Optional per-article 'signals:' front-matter (validator rule R24) is the authored precision path; domain-normalization reconciles front-matter/orchestrator/feedback domain vocabularies. Artifact is a runtime build (gitignored), orchestrator-facing only (not the lean agent-replayed knowledge index). CI guard Test-RoutingIndex.ps1 asserts determinism, seed recall floor, no orphaned signals, and normalization completeness; wired into the index workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/Test-RoutingIndex.ps1 | 123 ++++++++++ .github/scripts/validate_frontmatter.py | 32 ++- .github/workflows/knowledge-index.yml | 20 +- .gitignore | 4 + tools/Build-RoutingIndex.ps1 | 294 ++++++++++++++++++++++++ tools/routing-index.md | 113 +++++++++ tools/routing-index.schema.json | 80 +++++++ tools/routing-seed.json | 63 +++++ 8 files changed, 722 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/Test-RoutingIndex.ps1 create mode 100644 tools/Build-RoutingIndex.ps1 create mode 100644 tools/routing-index.md create mode 100644 tools/routing-index.schema.json create mode 100644 tools/routing-seed.json diff --git a/.github/scripts/Test-RoutingIndex.ps1 b/.github/scripts/Test-RoutingIndex.ps1 new file mode 100644 index 0000000..83ad94c --- /dev/null +++ b/.github/scripts/Test-RoutingIndex.ps1 @@ -0,0 +1,123 @@ +<# +.SYNOPSIS + CI guard for the routing-index generator (tools/Build-RoutingIndex.ps1). + +.DESCRIPTION + BCQuality owns the routing index — the orchestrator-facing companion to the + knowledge index that maps PR-diff signals to review domains and the articles + that back them. Like the knowledge index, no committed artifact is trusted at + runtime (a consumer rebuilds it over its pruned clone); this script proves the + GENERATOR is healthy: + + 1. Determinism — building twice yields byte-identical output once the + volatile `generatedAt` header is normalized. + 2. Recall floor — every seed signal (tools/routing-seed.json) survives into + the compiled index, so routing recall is never below the legacy catalog + it replaced. + 3. No orphaned seed signals — a seed signal whose domain has at least one + indexed article MUST have that article attached (catches domain + normalization drift, e.g. Web Services vs web-services). + 4. Structural integrity — every signal carries token/pattern/domain/source/ + weight/articles; every attached article path exists on disk; every + signal pattern is a valid regex. + 5. Normalization coverage — every front-matter domain present on disk either + normalizes to a canonical orchestrator domain or is a documented + pass-through, so no article silently falls out of routing. + + Exit code 0 = healthy; non-zero = a problem CI must block on. +#> +[CmdletBinding()] +param( + [string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..' '..')) +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$Root = (Resolve-Path -LiteralPath $Root).Path + +$generator = Join-Path $Root 'tools/Build-RoutingIndex.ps1' +$seedPath = Join-Path $Root 'tools/routing-seed.json' +if (-not (Test-Path $generator)) { throw "Generator not found: $generator" } +if (-not (Test-Path $seedPath)) { throw "Seed not found: $seedPath" } + +$tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("routeindex_" + [guid]::NewGuid().ToString('N')) +New-Item -ItemType Directory -Force -Path $tmp | Out-Null +$idxA = Join-Path $tmp 'a.json' +$idxB = Join-Path $tmp 'b.json' + +$problems = [System.Collections.Generic.List[string]]::new() + +& $generator -BCQualityRoot $Root -IndexPath $idxA | Out-Null +& $generator -BCQualityRoot $Root -IndexPath $idxB | Out-Null + +# 1. Determinism (ignoring the volatile generatedAt timestamp). +$norm = { param($p) ((Get-Content -LiteralPath $p -Raw) -replace '"generatedAt":"[^"]*"', '"generatedAt":""') } +if ((& $norm $idxA) -ne (& $norm $idxB)) { + $problems.Add('Non-deterministic: two builds differ beyond generatedAt.') | Out-Null +} + +$index = Get-Content -LiteralPath $idxA -Raw | ConvertFrom-Json +$signals = @($index.signals) +$seed = Get-Content -LiteralPath $seedPath -Raw | ConvertFrom-Json + +# 2. Recall floor: every seed signal token present in the compiled index. +$indexTokens = @($signals | ForEach-Object { $_.token }) +$missingSeed = @($seed.signals | Where-Object { $indexTokens -notcontains $_.token } | ForEach-Object { $_.token }) +if ($missingSeed.Count) { $problems.Add("Seed signals dropped from index: $($missingSeed -join ', ')") | Out-Null } + +# 3. No orphaned seed signals when the domain is populated. +$domainArticleCount = @{} +foreach ($p in $index.domains.PSObject.Properties) { $domainArticleCount[$p.Name] = [int]$p.Value.articleCount } +foreach ($s in $signals) { + if ($s.source -eq 'seed' -and @($s.articles).Count -eq 0) { + $ac = if ($domainArticleCount.ContainsKey($s.domain)) { $domainArticleCount[$s.domain] } else { 0 } + if ($ac -gt 0) { + $problems.Add("Orphaned seed signal '$($s.token)': domain '$($s.domain)' has $ac article(s) but none attached (normalization drift?).") | Out-Null + } + } +} + +# 4. Structural integrity. +foreach ($s in $signals) { + foreach ($f in 'token','pattern','domain','source','weight') { + if ($null -eq $s.$f -or ("$($s.$f)").Trim() -eq '') { $problems.Add("Signal missing '$f': $($s.token)") | Out-Null } + } + try { [void][regex]::new([string]$s.pattern) } catch { $problems.Add("Invalid regex for signal '$($s.token)': $($s.pattern)") | Out-Null } + foreach ($ap in @($s.articles)) { + if (-not (Test-Path (Join-Path $Root $ap))) { $problems.Add("Signal '$($s.token)' attaches missing article: $ap") | Out-Null } + } +} + +# 5. Normalization coverage: every on-disk front-matter domain resolves to a +# canonical orchestrator domain or is a known pass-through. A NEW front-matter +# domain that is not TitleCase and not in the map is flagged so the seed's +# domain-normalization stays complete as content grows. +$norm2 = @{} +foreach ($p in $index.domainNormalization.PSObject.Properties) { $norm2[$p.Name] = $p.Value } +$passThrough = @('appsource') # indexed domains with no leaf skill (route via super-skill) +$diskDomains = @{} +foreach ($layer in 'microsoft','community','custom') { + $kb = Join-Path $Root (Join-Path $layer 'knowledge') + if (-not (Test-Path $kb)) { continue } + Get-ChildItem -LiteralPath $kb -Recurse -File -Filter '*.md' | ForEach-Object { + $l = Get-Content -LiteralPath $_.FullName -TotalCount 12 + $d = ($l | Where-Object { $_ -match '^\s*domain\s*:\s*(.+?)\s*$' } | Select-Object -First 1) + if ($d -and $d -match '^\s*domain\s*:\s*(.+?)\s*$') { $diskDomains[$Matches[1].Trim()] = $true } + } +} +foreach ($d in $diskDomains.Keys) { + $isCanonical = ($d -cmatch '[A-Z]' -or $d -eq 'appsource') # already TitleCase or pass-through + if (-not $norm2.ContainsKey($d) -and -not ($passThrough -contains $d) -and -not $isCanonical) { + $problems.Add("Front-matter domain '$d' is not in domain-normalization and not a pass-through; add it to routing-seed.json.") | Out-Null + } +} + +Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue + +if ($problems.Count) { + Write-Host "Routing-index check FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red + $problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red } + exit 1 +} +Write-Host "Routing-index check PASSED: $($signals.Count) signals, $($index.articleCount) articles, deterministic, seed recall floor held, normalization complete." -ForegroundColor Green +exit 0 diff --git a/.github/scripts/validate_frontmatter.py b/.github/scripts/validate_frontmatter.py index 682a801..1b4de01 100644 --- a/.github/scripts/validate_frontmatter.py +++ b/.github/scripts/validate_frontmatter.py @@ -34,6 +34,11 @@ KNOWLEDGE_REQUIRED_KEYS = { "bc-version", "domain", "keywords", "technologies", "countries", "application-area", } +# Optional knowledge keys (do not trigger the R02 closed-key-set error). +# `signals`: OPTIONAL routing-signal declarations consumed by the routing index +# (tools/Build-RoutingIndex.ps1). Back-compatible — articles that omit it are +# routed from the seed catalog + keyword derivation. See tools/routing-index.md. +KNOWLEDGE_OPTIONAL_KEYS = {"signals"} ACTION_SKILL_REQUIRED_KEYS = { "kind", "id", "version", "title", "description", "inputs", "outputs", } @@ -202,7 +207,7 @@ def validate_knowledge(path: Path, parsed: Parsed, report: Report) -> None: # R02 required keys, no extras, none empty missing = KNOWLEDGE_REQUIRED_KEYS - fm.keys() - extras = fm.keys() - KNOWLEDGE_REQUIRED_KEYS + extras = fm.keys() - KNOWLEDGE_REQUIRED_KEYS - KNOWLEDGE_OPTIONAL_KEYS if missing: report.error(path, "R02", f"missing required frontmatter keys: {sorted(missing)}", 1) if extras: @@ -212,6 +217,31 @@ def validate_knowledge(path: Path, parsed: Parsed, report: Report) -> None: if v is None or v == "" or v == []: report.error(path, "R02", f"frontmatter key '{k}' must not be empty", 1) + # R24 optional routing `signals` block. Each entry is either a bare token + # string or a mapping with a required 'token' and optional 'pattern'/'domain' + # (all non-empty strings). Keeps the routing-index generator's input honest. + if "signals" in fm: + sigs = fm["signals"] + if not isinstance(sigs, list) or not sigs: + report.error(path, "R24", "signals must be a non-empty list", 1) + else: + for entry in sigs: + if isinstance(entry, str): + if not entry.strip(): + report.error(path, "R24", "signals token string must not be empty", 1) + elif isinstance(entry, dict): + tok = entry.get("token") + if not isinstance(tok, str) or not tok.strip(): + report.error(path, "R24", "signals entry must have a non-empty 'token'", 1) + for opt in ("pattern", "domain"): + if opt in entry and (not isinstance(entry[opt], str) or not entry[opt].strip()): + report.error(path, "R24", f"signals '{opt}' must be a non-empty string", 1) + unknown = set(entry.keys()) - {"token", "pattern", "domain"} + if unknown: + report.error(path, "R24", f"signals entry has unknown keys: {sorted(unknown)}", 1) + else: + report.error(path, "R24", "signals entry must be a string or a mapping", 1) + # R03 bc-version if "bc-version" in fm: _, err = expand_bc_version(fm["bc-version"]) diff --git a/.github/workflows/knowledge-index.yml b/.github/workflows/knowledge-index.yml index 71acfcb..7b5b873 100644 --- a/.github/workflows/knowledge-index.yml +++ b/.github/workflows/knowledge-index.yml @@ -1,10 +1,14 @@ -name: Validate knowledge index +name: Validate knowledge & routing indexes -# BCQuality owns the knowledge-index generator (tools/Build-KnowledgeIndex.ps1), -# which the AL review skills' Source step consumes. This guards the generator's -# health so consumers never have to: the index a consumer uses at runtime is -# rebuilt over its own pruned clone, but the generator that builds it lives and -# is validated here. +# BCQuality owns two index generators consumed by the AL review pipeline: +# - tools/Build-KnowledgeIndex.ps1 -> knowledge-index.json (lean, agent-replayed +# in the CLI run via the review skills' Source step) +# - tools/Build-RoutingIndex.ps1 -> routing-index.json (orchestrator-facing; +# maps PR-diff signals to domains + backing articles; consumed by the manifest +# builder behind BCQ_INDEX_V2) +# This workflow guards both generators' health so consumers never have to: each +# index a consumer uses at runtime is rebuilt over its own pruned clone, but the +# generators that build them live and are validated here. on: pull_request: @@ -22,3 +26,7 @@ jobs: - name: Validate knowledge-index generator shell: pwsh run: ./.github/scripts/Test-KnowledgeIndex.ps1 -Root . + + - name: Validate routing-index generator + shell: pwsh + run: ./.github/scripts/Test-RoutingIndex.ps1 -Root . diff --git a/.gitignore b/.gitignore index 1209309..7ff170a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,7 @@ node_modules/ # Runtime artifact: the knowledge index is rebuilt over each consumer's # pruned clone by Entry's preparation step; it is never committed. /knowledge-index.json + +# Runtime artifact: the routing index (orchestrator-facing companion) is rebuilt +# over each consumer's pruned clone by Build-RoutingIndex.ps1; never committed. +/routing-index.json diff --git a/tools/Build-RoutingIndex.ps1 b/tools/Build-RoutingIndex.ps1 new file mode 100644 index 0000000..bfadca3 --- /dev/null +++ b/tools/Build-RoutingIndex.ps1 @@ -0,0 +1,294 @@ +<# +.SYNOPSIS + Builds the BCQuality routing index — the orchestrator-facing companion to + knowledge-index.json that maps PR-diff detection signals to review domains + and the knowledge articles that back them. + +.DESCRIPTION + The PR-review orchestrator used to carry a hand-written ~28-token regex + catalog ($BcqSignalCatalog) INSIDE its code (Invoke-CopilotPRReview.ps1). + That shadow catalog never read the domain/keywords front-matter the articles + already declare, so its blind spots were exactly where findings got missed. + + This generator retires that shadow catalog by compiling a routing index from + CONTENT: it ingests the content-owned seed (tools/routing-seed.json, the + migrated legacy catalog, so recall >= today) and then attaches to every + signal the articles that back it, plus any article-declared `signals:` + front-matter. The result — routing-index.json — is consumed by the + orchestrator's Build-ReviewManifest behind the BCQ_INDEX_V2 flag to score + per-domain suspicion and shortlist candidate articles. + + This is NOT the lean, agent-replayed knowledge-index.json. The routing index + is orchestrator-side only and is never fed into the CLI prompt, so it can be + richer without inflating the token-paid path. It is regenerated + deterministically at authoring/CI time. + + Recall is complete-by-construction: every article whose (normalized) domain + matches a signal's domain is attached to that signal, so a new community + article becomes routable the moment it lands — no orchestrator edit required. + +.PARAMETER BCQualityRoot + Path to the BCQuality content root to index. Defaults to the clone root + (parent of this script's tools/ folder). + +.PARAMETER SeedPath + Path to the routing seed. Defaults to /routing-seed.json. + +.PARAMETER IndexPath + Where to write the routing index JSON. Defaults to + /routing-index.json. + +.PARAMETER EnabledLayers + Optional layer allowlist (microsoft, community, custom). When provided only + those layers are walked and the value is recorded in the index header. + +.PARAMETER IncludeKeywordSignals + ALSO derive soft signals from article keywords (weight 0.5, source + 'keyword'). OFF by default: keywords are lowercase-kebab and match noisily, + and the feedback data warns the agent already over-fires (precision 0.246). + Seed + explicit front-matter `signals:` are the precision path. + +.PARAMETER Pretty + Emit pretty-printed JSON instead of compact. + +.OUTPUTS + Returns the number of signals written to the index. +#> +[CmdletBinding()] +param( + [string] $BCQualityRoot, + [string] $SeedPath, + [string] $IndexPath, + [string[]] $EnabledLayers, + [switch] $IncludeKeywordSignals, + [switch] $Pretty +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not $BCQualityRoot) { + $BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +} +if (-not (Test-Path $BCQualityRoot)) { throw "BCQuality root not found: $BCQualityRoot" } +$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path +if (-not $SeedPath) { $SeedPath = Join-Path $PSScriptRoot 'routing-seed.json' } +if (-not (Test-Path $SeedPath)) { throw "Routing seed not found: $SeedPath" } +if (-not $IndexPath) { $IndexPath = Join-Path $BCQualityRoot 'routing-index.json' } + +$seed = Get-Content -LiteralPath $SeedPath -Raw | ConvertFrom-Json +$domainNorm = @{} +foreach ($p in $seed.'domain-normalization'.PSObject.Properties) { $domainNorm[$p.Name] = $p.Value } + +# Normalize a front-matter domain (lowercase-hyphen) to the orchestrator's +# TitleCase taxonomy. Unknown domains pass through unchanged so nothing is lost. +function ConvertTo-CanonicalDomain { + param([string] $Domain) + if ([string]::IsNullOrWhiteSpace($Domain)) { return '' } + $d = $Domain.Trim() + if ($domainNorm.ContainsKey($d)) { return $domainNorm[$d] } + return $d +} + +function Get-RelativePath { + param([string] $Root, [string] $Full) + return (($Full.Substring($Root.Length).TrimStart([char]'/', [char]'\')) -replace '\\', '/') +} + +# Minimal front-matter reader for routing needs: domain (scalar), keywords +# (inline array), and the OPTIONAL signals block (inline string array, or a +# block list of `- token`/`- token: X` items). Deliberately narrow — the lean +# knowledge index owns the full parse; here we only need routing inputs. +function Read-RoutingFrontmatter { + param([string] $Path) + $lines = Get-Content -LiteralPath $Path -ErrorAction Stop + if ($lines.Count -lt 1 -or $lines[0].Trim() -ne '---') { return $null } + $fmEnd = -1 + for ($i = 1; $i -lt $lines.Count; $i++) { if ($lines[$i].Trim() -eq '---') { $fmEnd = $i; break } } + if ($fmEnd -lt 0) { return $null } + + $domain = '' + $keywords = @() + $signals = [System.Collections.Generic.List[object]]::new() + + for ($i = 1; $i -lt $fmEnd; $i++) { + $line = $lines[$i] + if ($line -match '^\s*domain\s*:\s*(.+?)\s*$') { $domain = $Matches[1].Trim(); continue } + if ($line -match '^\s*keywords\s*:\s*\[(.*)\]\s*$') { + $inner = $Matches[1].Trim() + if ($inner -ne '') { $keywords = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() }) } + continue + } + if ($line -match '^\s*signals\s*:\s*(.*)$') { + $rest = $Matches[1].Trim() + if ($rest -match '^\[(.*)\]$') { + # inline: signals: [tokenA, tokenB] + $inner = $Matches[1].Trim() + if ($inner -ne '') { + foreach ($tok in ($inner -split '\s*,\s*')) { + $t = $tok.Trim().Trim('"',"'") + if ($t) { $signals.Add($t) | Out-Null } + } + } + } else { + # block list following the key: `- token` or `- token: X` / mapping + for ($j = $i + 1; $j -lt $fmEnd; $j++) { + $bl = $lines[$j] + if ($bl -match '^\s*-\s*(.+?)\s*$') { + $item = $Matches[1].Trim() + if ($item -match '^token\s*:\s*(.+)$') { + $signals.Add(@{ token = ($Matches[1].Trim().Trim('"',"'")) }) | Out-Null + } elseif ($item -notmatch ':') { + $signals.Add(($item.Trim('"',"'"))) | Out-Null + } else { + # inline-mapping `- {token: X, pattern: Y}` — best effort + $m = [regex]::Match($item, 'token\s*:\s*([^,}\s]+)') + if ($m.Success) { $signals.Add(@{ token = $m.Groups[1].Value.Trim().Trim('"',"'") }) | Out-Null } + } + } elseif ($bl -match '^\s+\w') { + # continuation of a mapping entry (pattern:/domain:) — attach to last mapping + if ($signals.Count -gt 0 -and ($signals[$signals.Count - 1] -is [hashtable])) { + if ($bl -match '^\s*pattern\s*:\s*(.+)$') { $signals[$signals.Count - 1]['pattern'] = $Matches[1].Trim().Trim('"',"'") } + elseif ($bl -match '^\s*domain\s*:\s*(.+)$') { $signals[$signals.Count - 1]['domain'] = $Matches[1].Trim().Trim('"',"'") } + } + } else { break } + } + } + continue + } + } + return [pscustomobject]@{ domain = $domain; keywords = $keywords; signals = @($signals) } +} + +# ---- Walk articles --------------------------------------------------------- +$articles = [System.Collections.Generic.List[object]]::new() +foreach ($layerDir in @('microsoft', 'community', 'custom')) { + if ($EnabledLayers -and ($EnabledLayers -notcontains $layerDir)) { continue } + $kbRoot = Join-Path $BCQualityRoot (Join-Path $layerDir 'knowledge') + if (-not (Test-Path $kbRoot)) { continue } + Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue | + Sort-Object FullName | + ForEach-Object { + $rel = Get-RelativePath -Root $BCQualityRoot -Full $_.FullName + $fm = $null + try { $fm = Read-RoutingFrontmatter -Path $_.FullName } catch { $fm = $null } + $rawDomain = if ($fm) { $fm.domain } elseif ($rel -match '/knowledge/([^/]+)/') { $Matches[1] } else { '' } + $articles.Add([pscustomobject]@{ + path = $rel + layer = $layerDir + rawDomain = $rawDomain + domain = (ConvertTo-CanonicalDomain $rawDomain) + keywords = if ($fm) { @($fm.keywords) } else { @() } + signals = if ($fm) { @($fm.signals) } else { @() } + }) | Out-Null + } +} + +# ---- Build the domain -> article index (for attaching backing articles) ---- +$articlesByDomain = @{} +foreach ($a in $articles) { + if (-not $a.domain) { continue } + if (-not $articlesByDomain.ContainsKey($a.domain)) { $articlesByDomain[$a.domain] = [System.Collections.Generic.List[string]]::new() } + $articlesByDomain[$a.domain].Add($a.path) | Out-Null +} + +# Signal registry keyed by token so seed + article-declared signals merge and +# never duplicate; articles accumulate onto the matching token. +$signalReg = [ordered]@{} +function Add-Signal { + param([string] $Token, [string] $Pattern, [string] $Domain, [string] $Source, [double] $Weight) + if (-not $signalReg.Contains($Token)) { + $signalReg[$Token] = [ordered]@{ + token = $Token; pattern = $Pattern; domain = $Domain; source = $Source; weight = $Weight + articles = [System.Collections.Generic.List[string]]::new() + } + } + return $signalReg[$Token] +} + +# 1) Seed signals: guarantee recall >= legacy catalog. Attach every article +# whose canonical domain matches the seed signal's domain. +foreach ($s in $seed.signals) { + $sig = Add-Signal -Token $s.token -Pattern $s.pattern -Domain $s.domain -Source 'seed' -Weight 1.0 + if ($articlesByDomain.ContainsKey($s.domain)) { + foreach ($p in $articlesByDomain[$s.domain]) { if (-not $sig.articles.Contains($p)) { $sig.articles.Add($p) | Out-Null } } + } +} + +# 2) Article-declared signals (front-matter `signals:`): highest-precision, +# authored. Default pattern = \b\b; default domain = article domain. +foreach ($a in $articles) { + foreach ($decl in $a.signals) { + $token = $null; $pattern = $null; $domain = $null + if ($decl -is [string]) { $token = $decl } + elseif ($decl -is [hashtable]) { $token = $decl['token']; $pattern = $decl['pattern']; $domain = $decl['domain'] } + if (-not $token) { continue } + if (-not $pattern) { $pattern = '\b' + [regex]::Escape($token) + '\b' } + $domain = if ($domain) { ConvertTo-CanonicalDomain $domain } else { $a.domain } + $sig = Add-Signal -Token $token -Pattern $pattern -Domain $domain -Source 'frontmatter-signal' -Weight 1.0 + # Promote a seed placeholder to authored if the article overrode it. + if ($sig.source -eq 'seed' -and $decl -isnot [string]) { $sig.source = 'frontmatter-signal'; $sig.pattern = $pattern; $sig.domain = $domain } + if (-not $sig.articles.Contains($a.path)) { $sig.articles.Add($a.path) | Out-Null } + } +} + +# 3) OPTIONAL soft keyword signals (off by default). +if ($IncludeKeywordSignals) { + $stop = @{ 'al' = $true; 'bc' = $true; 'all' = $true; 'w1' = $true; 'data' = $true; 'field' = $true; 'table' = $true; 'page' = $true; 'record' = $true; 'value' = $true } + foreach ($a in $articles) { + foreach ($kw in $a.keywords) { + $k = ($kw + '').Trim().ToLowerInvariant() + if (-not $k -or $stop.ContainsKey($k) -or $k.Length -lt 4) { continue } + $token = 'kw:' + $k + $pattern = '(?i)\b' + [regex]::Escape($k) + '\b' + $sig = Add-Signal -Token $token -Pattern $pattern -Domain $a.domain -Source 'keyword' -Weight 0.5 + if (-not $sig.articles.Contains($a.path)) { $sig.articles.Add($a.path) | Out-Null } + } + } +} + +# ---- Assemble output ------------------------------------------------------- +$signalsOut = @($signalReg.Values | ForEach-Object { + [ordered]@{ + token = $_.token; pattern = $_.pattern; domain = $_.domain + source = $_.source; weight = $_.weight; articles = @($_.articles) + } +}) + +$domainsOut = [ordered]@{} +foreach ($d in ($articlesByDomain.Keys | Sort-Object)) { + $domainsOut[$d] = [ordered]@{ + articleCount = $articlesByDomain[$d].Count + signalCount = @($signalsOut | Where-Object { $_.domain -eq $d }).Count + } +} + +$objKind = [ordered]@{} +foreach ($p in $seed.'object-kind-domain'.PSObject.Properties) { $objKind[$p.Name] = $p.Value } +$domNormOut = [ordered]@{} +foreach ($k in ($domainNorm.Keys | Sort-Object)) { $domNormOut[$k] = $domainNorm[$k] } + +$index = [ordered]@{ + '$schema-note' = 'BCQuality routing index. signal-token -> domain + backing articles. Consumed by the PR-review orchestrator manifest (BCQ_INDEX_V2). Regenerated at CI time by tools/Build-RoutingIndex.ps1; NOT the lean agent-replayed knowledge index. Schema: tools/routing-index.schema.json.' + version = 1 + generatedAt = (Get-Date).ToUniversalTime().ToString('o') + generator = 'Build-RoutingIndex.ps1' + enabledLayers = @($EnabledLayers) + articleCount = $articles.Count + signalCount = $signalsOut.Count + domainNormalization = $domNormOut + objectKindDomain = $objKind + signals = $signalsOut + domains = $domainsOut +} + +$indexDir = Split-Path -Parent $IndexPath +if ($indexDir -and -not (Test-Path $indexDir)) { New-Item -ItemType Directory -Force -Path $indexDir | Out-Null } +if ($Pretty) { + $index | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $IndexPath -Encoding UTF8 +} else { + $index | ConvertTo-Json -Depth 12 -Compress | Set-Content -LiteralPath $IndexPath -Encoding UTF8 +} + +Write-Host "BCQuality routing index: $($signalsOut.Count) signal(s), $($articles.Count) article(s). Index: $IndexPath" +return $signalsOut.Count diff --git a/tools/routing-index.md b/tools/routing-index.md new file mode 100644 index 0000000..cf6100a --- /dev/null +++ b/tools/routing-index.md @@ -0,0 +1,113 @@ +# Routing index + +The **routing index** (`routing-index.json`) is the orchestrator-facing companion +to the [knowledge index](../tools/Build-KnowledgeIndex.ps1). Where the knowledge +index lets the *agent* enumerate candidate articles inside the CLI run, the +routing index lets the *orchestrator* decide — deterministically, before any +tokens are spent — which review domains a PR touches and which articles back each +detection signal. + +It exists to retire a shadow catalog. The PR-review orchestrator historically +carried a hand-written ~28-token regex catalog (`$BcqSignalCatalog`) hardcoded in +`Invoke-CopilotPRReview.ps1`. That catalog never read the `domain` / `keywords` +front-matter every article already declares, so its blind spots were exactly +where findings were missed. The routing index moves that knowledge into content: +signals and their domains are compiled *from articles*, so a new article — even a +community one — becomes routable the moment it lands, with no orchestrator edit. + +## What it is (and is not) + +- **It is** an orchestrator-side artifact. It is consumed by `Build-ReviewManifest` + (behind the `BCQ_INDEX_V2` flag) to score per-domain suspicion and shortlist + candidate articles. It is **never** fed into the CLI prompt, so it can be richer + than the lean knowledge index without inflating the token-paid path. +- **It is not** committed. Like `knowledge-index.json`, it is a runtime artifact + rebuilt over each consumer's already-pruned clone (`tools/Build-RoutingIndex.ps1`) + and is `.gitignore`d. +- **It is not** the knowledge index. The two are siblings built from the same + front-matter; the knowledge index is agent-replayed, the routing index is not. + +## How signals are compiled + +`Build-RoutingIndex.ps1` composes signals from three sources, highest precision +first: + +1. **Seed** (`tools/routing-seed.json`) — the migrated legacy catalog. Ingested + first so routing recall is never below what the old hardcoded catalog caught. + Each seed signal is then enriched with **every** article whose (normalized) + domain matches, so recall is complete-by-construction rather than limited by a + hand-maintained token→article map. +2. **Article-declared** (`signals:` front-matter, optional) — the authored, + highest-precision path. Use it when an article catches a specific AL construct + the seed does not name. +3. **Keyword-derived** (`-IncludeKeywordSignals`, off by default) — soft signals + from article keywords at reduced weight. Off by default because keywords are + lowercase-kebab and match noisily, and the online-evals feedback shows the agent + already over-fires. Prefer authored `signals:` for precision. + +## The optional `signals:` front-matter block + +Knowledge articles may declare an **optional** `signals:` block. When omitted, the +article is still routed via the seed + its domain (fully back-compatible — no +existing article must change). Each entry is either a bare token string or a +mapping: + +```yaml +domain: performance +keywords: [lock, readonly, isolation] +signals: + - LockTable # bare token -> pattern \bLockTable\b, domain = article domain + - token: ReadIsolation + pattern: '\bReadIsolation\b' # explicit regex (optional) + domain: performance # override domain (optional; normalized) +``` + +- `token` (required) — stable signal id; need not be an AL identifier verbatim. +- `pattern` (optional) — regex matched against **added** diff lines. Defaults to + `\b\b`. +- `domain` (optional) — defaults to the article's `domain`, then normalized. + +The front-matter validator (`.github/scripts/validate_frontmatter.py`, rule R24) +enforces this shape; `signals` is the only optional key in the otherwise-closed +knowledge key set. + +## Domain normalization + +Three vocabularies describe the same domains and must be reconciled: + +| Front-matter (article) | Orchestrator (`$DomainMap`) | Feedback (`byDomain`) | +|------------------------|-----------------------------|-----------------------| +| `ui` | `Accessibility` | `accessibility` | +| `error-handling` | `Error Handling` | — | +| `web-services` | `Web Services` | — | +| `breaking-changes` | `Breaking Changes` | — | +| `events` / `interfaces`| `Events` / `Interfaces` | — | +| `telemetry` | `Privacy` (folded) | — | + +`routing-seed.json → domain-normalization` maps front-matter domains to the +orchestrator's canonical TitleCase taxonomy so seed signals (authored in that +taxonomy) attach to the right articles. `appsource` is a documented **pass-through** +domain (indexed, but has no review leaf skill, so it routes only via the +`al-code-review` super-skill). The CI guard fails if a new front-matter domain +appears that is neither normalized nor a pass-through, keeping the map complete as +content grows. + +## Layer-replication story + +The whole process is open and per-layer. A partner running the pipeline against +their own `community/` or `custom/` layers gets routing for free: their articles' +front-matter (and any `signals:` they add) compile into the same index by the same +generator. No Microsoft-only data is required to build Tier 1. The later Tier 2 +feedback overlay (precision weights distilled from online evals) is a separate, +aggregate-only artifact that multiplies onto these Tier-1 weights — each layer +carries its own overlay produced from its own feedback. + +## Files + +| File | Role | +|------|------| +| `tools/Build-RoutingIndex.ps1` | Generator (scan layers → `routing-index.json`) | +| `tools/routing-seed.json` | Content-owned seed: migrated catalog + domain-normalization + object-kind hints | +| `tools/routing-index.schema.json` | JSON Schema for the compiled artifact | +| `.github/scripts/Test-RoutingIndex.ps1` | CI guard (determinism, recall floor, no orphans, normalization complete) | +| `.github/workflows/knowledge-index.yml` | Runs both index guards on PR/push to main | diff --git a/tools/routing-index.schema.json b/tools/routing-index.schema.json new file mode 100644 index 0000000..fd8bb6b --- /dev/null +++ b/tools/routing-index.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/microsoft/BCQuality/tools/routing-index.schema.json", + "title": "BCQuality routing index", + "description": "Orchestrator-facing companion to knowledge-index.json. Maps deterministic PR-diff detection signals (AL tokens / regex) to their owning review domain and the knowledge articles that back them. Consumed by the PR-review orchestrator's manifest builder (Build-ReviewManifest, gated behind BCQ_INDEX_V2) to compute per-domain suspicion scores and shortlist candidate articles WITHOUT reading the hardcoded regex catalog that used to live in orchestrator code. Regenerated deterministically at authoring/CI time by tools/Build-RoutingIndex.ps1; never built in the token-paid path. This is NOT the lean agent-replayed knowledge index — it is not fed into the CLI prompt.", + "type": "object", + "required": ["version", "generatedAt", "generator", "articleCount", "signalCount", "domainNormalization", "objectKindDomain", "signals", "domains"], + "additionalProperties": true, + "properties": { + "$schema-note": { "type": "string" }, + "version": { "type": "integer", "const": 1 }, + "generatedAt": { "type": "string", "format": "date-time" }, + "generator": { "type": "string" }, + "enabledLayers": { "type": "array", "items": { "type": "string" } }, + "articleCount": { "type": "integer", "minimum": 0 }, + "signalCount": { "type": "integer", "minimum": 0 }, + "domainNormalization": { + "description": "Front-matter domain (lowercase-hyphen) -> orchestrator domain label (TitleCase). Resolves the vocabulary mismatch between article front-matter (ui, error-handling), orchestrator $DomainMap (Accessibility, Error Handling), and feedback byDomain (lowercase).", + "type": "object", + "additionalProperties": { "type": "string" } + }, + "objectKindDomain": { + "description": "AL object kind -> weak domain hint (applied once per changed object). Content-owned successor to the orchestrator's $BcqObjectKindDomain.", + "type": "object", + "additionalProperties": { "type": "string" } + }, + "signals": { + "description": "Compiled detection signals. Each maps a regex over ADDED diff lines to a domain and the backing articles.", + "type": "array", + "items": { + "type": "object", + "required": ["token", "pattern", "domain", "source", "weight", "articles"], + "additionalProperties": false, + "properties": { + "token": { + "description": "Stable signal identifier (e.g. FindSet). Not necessarily an AL token verbatim.", + "type": "string" + }, + "pattern": { + "description": "Regex matched (case-sensitively, as authored) against each added diff line. Escaped for JSON.", + "type": "string" + }, + "domain": { + "description": "Owning review domain in the orchestrator's TitleCase taxonomy (post-normalization).", + "type": "string" + }, + "source": { + "description": "Where the signal came from: 'seed' (legacy catalog migration), 'frontmatter-signal' (article-declared signals: block), or 'keyword' (soft keyword-derived, weight < 1).", + "type": "string", + "enum": ["seed", "frontmatter-signal", "keyword"] + }, + "weight": { + "description": "Tier-1 confidence multiplier applied to signal hit counts in the domain-suspicion score. 1.0 for authored/seed signals; < 1 for soft keyword-derived signals. Tier-2 feedback weights (later) multiply on top of this.", + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "articles": { + "description": "Repo-relative paths of the knowledge articles this signal routes to. Complete-by-construction: every article whose domain (and, for keyword signals, keyword) matches is attached, so recall is not limited by a hand-maintained catalog.", + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "domains": { + "description": "Per-domain aggregates for quick orchestrator lookups.", + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["articleCount", "signalCount"], + "additionalProperties": true, + "properties": { + "articleCount": { "type": "integer", "minimum": 0 }, + "signalCount": { "type": "integer", "minimum": 0 } + } + } + } + } +} diff --git a/tools/routing-seed.json b/tools/routing-seed.json new file mode 100644 index 0000000..7fce3c6 --- /dev/null +++ b/tools/routing-seed.json @@ -0,0 +1,63 @@ +{ + "$schema-note": "BCQuality routing SEED — content-owned migration of the signal catalog that used to live hardcoded in the PR-review orchestrator ($BcqSignalCatalog / $BcqObjectKindDomain in Invoke-CopilotPRReview.ps1). Build-RoutingIndex.ps1 ingests this seed FIRST so the compiled routing-index.json detects at least everything the legacy code catalog did (recall >= today), then enriches each signal with the articles that back it and adds article-declared signals on top. Over time, entries here migrate into per-article `signals:` front-matter and this seed shrinks. Domains use the orchestrator's TitleCase taxonomy (see domain-normalization).", + "version": 1, + "domain-normalization": { + "error-handling": "Error Handling", + "performance": "Performance", + "privacy": "Privacy", + "security": "Security", + "style": "Style", + "testing": "Testing", + "ui": "Accessibility", + "upgrade": "Upgrade", + "breaking-changes": "Breaking Changes", + "events": "Events", + "interfaces": "Interfaces", + "web-services": "Web Services", + "telemetry": "Privacy" + }, + "object-kind-domain": { + "page": "Accessibility", + "pageextension": "Accessibility", + "pagecustomization": "Accessibility", + "profile": "Accessibility", + "report": "Performance", + "reportextension": "Performance", + "query": "Performance", + "permissionset": "Security", + "permissionsetextension": "Security", + "entitlement": "Security", + "interface": "Interfaces", + "xmlport": "Web Services" + }, + "signals": [ + { "token": "FindSet", "pattern": "\\bFindSet\\b", "domain": "Performance" }, + { "token": "FindFirst", "pattern": "\\bFindFirst\\b", "domain": "Performance" }, + { "token": "FindLast", "pattern": "\\bFindLast\\b", "domain": "Performance" }, + { "token": "SetRange", "pattern": "\\bSetRange\\b", "domain": "Performance" }, + { "token": "SetFilter", "pattern": "\\bSetFilter\\b", "domain": "Performance" }, + { "token": "CalcFields", "pattern": "\\bCalcFields\\b", "domain": "Performance" }, + { "token": "CalcSums", "pattern": "\\bCalcSums\\b", "domain": "Performance" }, + { "token": "SetLoadFields", "pattern": "\\bSetLoadFields\\b", "domain": "Performance" }, + { "token": "SetAutoCalcFields", "pattern": "\\bSetAutoCalcFields\\b", "domain": "Performance" }, + { "token": "loop", "pattern": "\\brepeat\\b", "domain": "Performance" }, + { "token": "HttpClient", "pattern": "\\bHttpClient\\b", "domain": "Security" }, + { "token": "SecretText", "pattern": "\\bSecretText\\b", "domain": "Security" }, + { "token": "IsolatedStorage", "pattern": "\\bIsolatedStorage\\b", "domain": "Security" }, + { "token": "PermissionSet", "pattern": "\\bPermissionSet\\b|\\bPermissions\\s*=", "domain": "Security" }, + { "token": "Entitlement", "pattern": "\\bEntitlement\\b", "domain": "Security" }, + { "token": "DataClassification", "pattern": "\\bDataClassification\\b", "domain": "Privacy" }, + { "token": "Telemetry", "pattern": "\\bLogMessage\\b|\\bFeatureTelemetry\\b", "domain": "Privacy" }, + { "token": "Error", "pattern": "\\bError\\b|\\bFieldError\\b|\\bTestField\\b", "domain": "Error Handling" }, + { "token": "TryFunction", "pattern": "\\[TryFunction\\]", "domain": "Error Handling" }, + { "token": "Commit", "pattern": "\\bCommit\\b", "domain": "Error Handling" }, + { "token": "Caption", "pattern": "\\bCaption\\s*=", "domain": "Accessibility" }, + { "token": "ToolTip", "pattern": "\\bToolTip\\s*=", "domain": "Accessibility" }, + { "token": "ApplicationArea", "pattern": "\\bApplicationArea\\s*=", "domain": "Accessibility" }, + { "token": "Obsolete", "pattern": "\\bObsoleteState\\b|\\bObsoleteReason\\b|\\bObsoleteTag\\b", "domain": "Upgrade" }, + { "token": "UpgradeTag", "pattern": "\\bUpgradeTag\\b", "domain": "Upgrade" }, + { "token": "EventPublisher", "pattern": "\\[IntegrationEvent|\\[BusinessEvent", "domain": "Events" }, + { "token": "EventSubscriber", "pattern": "\\[EventSubscriber", "domain": "Events" }, + { "token": "ApiObject", "pattern": "PageType\\s*=\\s*API|\\bEntityName\\b|\\bEntitySetName\\b|\\bODataKeyFields\\b", "domain": "Web Services" } + ] +}