Microsoft/bcquality
1
0
Fork 0
mirror of https://github.com/microsoft/BCQuality.git synced 2026-08-06 09:26:52 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

R6 Tier-1: content-owned routing index generator + CI guard

Browse source 
Retire the orchestrator's hardcoded ~28-token signal catalog by compiling a
routing index from article front-matter. New tools/Build-RoutingIndex.ps1 scans
all knowledge layers and emits routing-index.json (signal-token -> domain +
backing articles), seeded from the migrated legacy catalog (routing-seed.json)
so recall is never below today. Optional per-article 'signals:' front-matter
(validator rule R24) is the authored precision path; domain-normalization
reconciles front-matter/orchestrator/feedback domain vocabularies.

Artifact is a runtime build (gitignored), orchestrator-facing only (not the
lean agent-replayed knowledge index). CI guard Test-RoutingIndex.ps1 asserts
determinism, seed recall floor, no orphaned signals, and normalization
completeness; wired into the index workflow.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
dayland 2026-07-13 11:20:30 +01:00
parent 4119417ce4
commit 6ed56b95b7
8 changed files with 722 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

123
.github/scripts/Test-RoutingIndex.ps1 vendored Normal file
View file

@ -0,0 +1,123 @@
<#
.SYNOPSIS
CI guard for the routing-index generator (tools/Build-RoutingIndex.ps1).
.DESCRIPTION
BCQuality owns the routing index — the orchestrator-facing companion to the
knowledge index that maps PR-diff signals to review domains and the articles
that back them. Like the knowledge index, no committed artifact is trusted at
runtime (a consumer rebuilds it over its pruned clone); this script proves the
GENERATOR is healthy:
1. Determinism — building twice yields byte-identical output once the
volatile `generatedAt` header is normalized.
2. Recall floor — every seed signal (tools/routing-seed.json) survives into
the compiled index, so routing recall is never below the legacy catalog
it replaced.
3. No orphaned seed signals — a seed signal whose domain has at least one
indexed article MUST have that article attached (catches domain
normalization drift, e.g. Web Services vs web-services).
4. Structural integrity — every signal carries token/pattern/domain/source/
weight/articles; every attached article path exists on disk; every
signal pattern is a valid regex.
5. Normalization coverage — every front-matter domain present on disk either
normalizes to a canonical orchestrator domain or is a documented
pass-through, so no article silently falls out of routing.
Exit code 0 = healthy; non-zero = a problem CI must block on.
#>
[CmdletBinding()]
param(
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..' '..'))
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$Root = (Resolve-Path -LiteralPath $Root).Path
$generator = Join-Path $Root 'tools/Build-RoutingIndex.ps1'
$seedPath = Join-Path $Root 'tools/routing-seed.json'
if (-not (Test-Path $generator)) { throw "Generator not found: $generator" }
if (-not (Test-Path $seedPath)) { throw "Seed not found: $seedPath" }
$tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("routeindex_" + [guid]::NewGuid().ToString('N'))
New-Item -ItemType Directory -Force -Path $tmp | Out-Null
$idxA = Join-Path $tmp 'a.json'
$idxB = Join-Path $tmp 'b.json'
$problems = [System.Collections.Generic.List[string]]::new()
& $generator -BCQualityRoot $Root -IndexPath $idxA | Out-Null
& $generator -BCQualityRoot $Root -IndexPath $idxB | Out-Null
# 1. Determinism (ignoring the volatile generatedAt timestamp).
$norm = { param($p) ((Get-Content -LiteralPath $p -Raw) -replace '"generatedAt":"[^"]*"', '"generatedAt":"<n>"') }
if ((& $norm $idxA) -ne (& $norm $idxB)) {
$problems.Add('Non-deterministic: two builds differ beyond generatedAt.') | Out-Null
}
$index = Get-Content -LiteralPath $idxA -Raw | ConvertFrom-Json
$signals = @($index.signals)
$seed = Get-Content -LiteralPath $seedPath -Raw | ConvertFrom-Json
# 2. Recall floor: every seed signal token present in the compiled index.
$indexTokens = @($signals | ForEach-Object { $_.token })
$missingSeed = @($seed.signals | Where-Object { $indexTokens -notcontains $_.token } | ForEach-Object { $_.token })
if ($missingSeed.Count) { $problems.Add("Seed signals dropped from index: $($missingSeed -join ', ')") | Out-Null }
# 3. No orphaned seed signals when the domain is populated.
$domainArticleCount = @{}
foreach ($p in $index.domains.PSObject.Properties) { $domainArticleCount[$p.Name] = [int]$p.Value.articleCount }
foreach ($s in $signals) {
if ($s.source -eq 'seed' -and @($s.articles).Count -eq 0) {
$ac = if ($domainArticleCount.ContainsKey($s.domain)) { $domainArticleCount[$s.domain] } else { 0 }
if ($ac -gt 0) {
$problems.Add("Orphaned seed signal '$($s.token)': domain '$($s.domain)' has $ac article(s) but none attached (normalization drift?).") | Out-Null
}
}
}
# 4. Structural integrity.
foreach ($s in $signals) {
foreach ($f in 'token','pattern','domain','source','weight') {
if ($null -eq $s.$f -or ("$($s.$f)").Trim() -eq '') { $problems.Add("Signal missing '$f': $($s.token)") | Out-Null }
}
try { [void][regex]::new([string]$s.pattern) } catch { $problems.Add("Invalid regex for signal '$($s.token)': $($s.pattern)") | Out-Null }
foreach ($ap in @($s.articles)) {
if (-not (Test-Path (Join-Path $Root $ap))) { $problems.Add("Signal '$($s.token)' attaches missing article: $ap") | Out-Null }
}
}
# 5. Normalization coverage: every on-disk front-matter domain resolves to a
# canonical orchestrator domain or is a known pass-through. A NEW front-matter
# domain that is not TitleCase and not in the map is flagged so the seed's
# domain-normalization stays complete as content grows.
$norm2 = @{}
foreach ($p in $index.domainNormalization.PSObject.Properties) { $norm2[$p.Name] = $p.Value }
$passThrough = @('appsource') # indexed domains with no leaf skill (route via super-skill)
$diskDomains = @{}
foreach ($layer in 'microsoft','community','custom') {
$kb = Join-Path $Root (Join-Path $layer 'knowledge')
if (-not (Test-Path $kb)) { continue }
Get-ChildItem -LiteralPath $kb -Recurse -File -Filter '*.md' | ForEach-Object {
$l = Get-Content -LiteralPath $_.FullName -TotalCount 12
$d = ($l | Where-Object { $_ -match '^\s*domain\s*:\s*(.+?)\s*$' } | Select-Object -First 1)
if ($d -and $d -match '^\s*domain\s*:\s*(.+?)\s*$') { $diskDomains[$Matches[1].Trim()] = $true }
}
}
foreach ($d in $diskDomains.Keys) {
$isCanonical = ($d -cmatch '[A-Z]' -or $d -eq 'appsource') # already TitleCase or pass-through
if (-not $norm2.ContainsKey($d) -and -not ($passThrough -contains $d) -and -not $isCanonical) {
$problems.Add("Front-matter domain '$d' is not in domain-normalization and not a pass-through; add it to routing-seed.json.") | Out-Null
}
}
Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue
if ($problems.Count) {
Write-Host "Routing-index check FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
exit 1
}
Write-Host "Routing-index check PASSED: $($signals.Count) signals, $($index.articleCount) articles, deterministic, seed recall floor held, normalization complete." -ForegroundColor Green
exit 0

32
.github/scripts/validate_frontmatter.py vendored
View file

@ -34,6 +34,11 @@ KNOWLEDGE_REQUIRED_KEYS = {
"bc-version", "domain", "keywords", "technologies",
"countries", "application-area",
}
# Optional knowledge keys (do not trigger the R02 closed-key-set error).
# `signals`: OPTIONAL routing-signal declarations consumed by the routing index
# (tools/Build-RoutingIndex.ps1). Back-compatible — articles that omit it are
# routed from the seed catalog + keyword derivation. See tools/routing-index.md.
KNOWLEDGE_OPTIONAL_KEYS = {"signals"}
ACTION_SKILL_REQUIRED_KEYS = {
"kind", "id", "version", "title", "description", "inputs", "outputs",
}
@ -202,7 +207,7 @@ def validate_knowledge(path: Path, parsed: Parsed, report: Report) -> None:
# R02 required keys, no extras, none empty
missing = KNOWLEDGE_REQUIRED_KEYS - fm.keys()
extras = fm.keys() - KNOWLEDGE_REQUIRED_KEYS
extras = fm.keys() - KNOWLEDGE_REQUIRED_KEYS - KNOWLEDGE_OPTIONAL_KEYS
if missing:
report.error(path, "R02", f"missing required frontmatter keys: {sorted(missing)}", 1)
if extras:
@ -212,6 +217,31 @@ def validate_knowledge(path: Path, parsed: Parsed, report: Report) -> None:
if v is None or v == "" or v == []:
report.error(path, "R02", f"frontmatter key '{k}' must not be empty", 1)
# R24 optional routing `signals` block. Each entry is either a bare token
# string or a mapping with a required 'token' and optional 'pattern'/'domain'
# (all non-empty strings). Keeps the routing-index generator's input honest.
if "signals" in fm:
sigs = fm["signals"]
if not isinstance(sigs, list) or not sigs:
report.error(path, "R24", "signals must be a non-empty list", 1)
else:
for entry in sigs:
if isinstance(entry, str):
if not entry.strip():
report.error(path, "R24", "signals token string must not be empty", 1)
elif isinstance(entry, dict):
tok = entry.get("token")
if not isinstance(tok, str) or not tok.strip():
report.error(path, "R24", "signals entry must have a non-empty 'token'", 1)
for opt in ("pattern", "domain"):
if opt in entry and (not isinstance(entry[opt], str) or not entry[opt].strip()):
report.error(path, "R24", f"signals '{opt}' must be a non-empty string", 1)
unknown = set(entry.keys()) - {"token", "pattern", "domain"}
if unknown:
report.error(path, "R24", f"signals entry has unknown keys: {sorted(unknown)}", 1)
else:
report.error(path, "R24", "signals entry must be a string or a mapping", 1)
# R03 bc-version
if "bc-version" in fm:
_, err = expand_bc_version(fm["bc-version"])

20
.github/workflows/knowledge-index.yml vendored
View file

@ -1,10 +1,14 @@
name: Validate knowledge index
name: Validate knowledge & routing indexes
# BCQuality owns the knowledge-index generator (tools/Build-KnowledgeIndex.ps1),
# which the AL review skills' Source step consumes. This guards the generator's
# health so consumers never have to: the index a consumer uses at runtime is
# rebuilt over its own pruned clone, but the generator that builds it lives and
# is validated here.
# BCQuality owns two index generators consumed by the AL review pipeline:
# - tools/Build-KnowledgeIndex.ps1 -> knowledge-index.json (lean, agent-replayed
# in the CLI run via the review skills' Source step)
# - tools/Build-RoutingIndex.ps1 -> routing-index.json (orchestrator-facing;
# maps PR-diff signals to domains + backing articles; consumed by the manifest
# builder behind BCQ_INDEX_V2)
# This workflow guards both generators' health so consumers never have to: each
# index a consumer uses at runtime is rebuilt over its own pruned clone, but the
# generators that build them live and are validated here.
on:
pull_request:
@ -22,3 +26,7 @@ jobs:
- name: Validate knowledge-index generator
shell: pwsh
run: ./.github/scripts/Test-KnowledgeIndex.ps1 -Root .
- name: Validate routing-index generator
shell: pwsh
run: ./.github/scripts/Test-RoutingIndex.ps1 -Root .

4
.gitignore vendored
View file

@ -27,3 +27,7 @@ node_modules/
# Runtime artifact: the knowledge index is rebuilt over each consumer's
# pruned clone by Entry's preparation step; it is never committed.
/knowledge-index.json
# Runtime artifact: the routing index (orchestrator-facing companion) is rebuilt
# over each consumer's pruned clone by Build-RoutingIndex.ps1; never committed.
/routing-index.json

294
tools/Build-RoutingIndex.ps1 Normal file
View file

@ -0,0 +1,294 @@
<#
.SYNOPSIS
Builds the BCQuality routing index — the orchestrator-facing companion to
knowledge-index.json that maps PR-diff detection signals to review domains
and the knowledge articles that back them.
.DESCRIPTION
The PR-review orchestrator used to carry a hand-written ~28-token regex
catalog ($BcqSignalCatalog) INSIDE its code (Invoke-CopilotPRReview.ps1).
That shadow catalog never read the domain/keywords front-matter the articles
already declare, so its blind spots were exactly where findings got missed.
This generator retires that shadow catalog by compiling a routing index from
CONTENT: it ingests the content-owned seed (tools/routing-seed.json, the
migrated legacy catalog, so recall >= today) and then attaches to every
signal the articles that back it, plus any article-declared `signals:`
front-matter. The result — routing-index.json — is consumed by the
orchestrator's Build-ReviewManifest behind the BCQ_INDEX_V2 flag to score
per-domain suspicion and shortlist candidate articles.
This is NOT the lean, agent-replayed knowledge-index.json. The routing index
is orchestrator-side only and is never fed into the CLI prompt, so it can be
richer without inflating the token-paid path. It is regenerated
deterministically at authoring/CI time.
Recall is complete-by-construction: every article whose (normalized) domain
matches a signal's domain is attached to that signal, so a new community
article becomes routable the moment it lands — no orchestrator edit required.
.PARAMETER BCQualityRoot
Path to the BCQuality content root to index. Defaults to the clone root
(parent of this script's tools/ folder).
.PARAMETER SeedPath
Path to the routing seed. Defaults to <this script folder>/routing-seed.json.
.PARAMETER IndexPath
Where to write the routing index JSON. Defaults to
<BCQualityRoot>/routing-index.json.
.PARAMETER EnabledLayers
Optional layer allowlist (microsoft, community, custom). When provided only
those layers are walked and the value is recorded in the index header.
.PARAMETER IncludeKeywordSignals
ALSO derive soft signals from article keywords (weight 0.5, source
'keyword'). OFF by default: keywords are lowercase-kebab and match noisily,
and the feedback data warns the agent already over-fires (precision 0.246).
Seed + explicit front-matter `signals:` are the precision path.
.PARAMETER Pretty
Emit pretty-printed JSON instead of compact.
.OUTPUTS
Returns the number of signals written to the index.
#>
[CmdletBinding()]
param(
[string] $BCQualityRoot,
[string] $SeedPath,
[string] $IndexPath,
[string[]] $EnabledLayers,
[switch] $IncludeKeywordSignals,
[switch] $Pretty
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (-not $BCQualityRoot) {
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
}
if (-not (Test-Path $BCQualityRoot)) { throw "BCQuality root not found: $BCQualityRoot" }
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
if (-not $SeedPath) { $SeedPath = Join-Path $PSScriptRoot 'routing-seed.json' }
if (-not (Test-Path $SeedPath)) { throw "Routing seed not found: $SeedPath" }
if (-not $IndexPath) { $IndexPath = Join-Path $BCQualityRoot 'routing-index.json' }
$seed = Get-Content -LiteralPath $SeedPath -Raw | ConvertFrom-Json
$domainNorm = @{}
foreach ($p in $seed.'domain-normalization'.PSObject.Properties) { $domainNorm[$p.Name] = $p.Value }
# Normalize a front-matter domain (lowercase-hyphen) to the orchestrator's
# TitleCase taxonomy. Unknown domains pass through unchanged so nothing is lost.
function ConvertTo-CanonicalDomain {
param([string] $Domain)
if ([string]::IsNullOrWhiteSpace($Domain)) { return '' }
$d = $Domain.Trim()
if ($domainNorm.ContainsKey($d)) { return $domainNorm[$d] }
return $d
}
function Get-RelativePath {
param([string] $Root, [string] $Full)
return (($Full.Substring($Root.Length).TrimStart([char]'/', [char]'\')) -replace '\\', '/')
}
# Minimal front-matter reader for routing needs: domain (scalar), keywords
# (inline array), and the OPTIONAL signals block (inline string array, or a
# block list of `- token`/`- token: X` items). Deliberately narrow — the lean
# knowledge index owns the full parse; here we only need routing inputs.
function Read-RoutingFrontmatter {
param([string] $Path)
$lines = Get-Content -LiteralPath $Path -ErrorAction Stop
if ($lines.Count -lt 1 -or $lines[0].Trim() -ne '---') { return $null }
$fmEnd = -1
for ($i = 1; $i -lt $lines.Count; $i++) { if ($lines[$i].Trim() -eq '---') { $fmEnd = $i; break } }
if ($fmEnd -lt 0) { return $null }
$domain = ''
$keywords = @()
$signals = [System.Collections.Generic.List[object]]::new()
for ($i = 1; $i -lt $fmEnd; $i++) {
$line = $lines[$i]
if ($line -match '^\s*domain\s*:\s*(.+?)\s*$') { $domain = $Matches[1].Trim(); continue }
if ($line -match '^\s*keywords\s*:\s*\[(.*)\]\s*$') {
$inner = $Matches[1].Trim()
if ($inner -ne '') { $keywords = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() }) }
continue
}
if ($line -match '^\s*signals\s*:\s*(.*)$') {
$rest = $Matches[1].Trim()
if ($rest -match '^\[(.*)\]$') {
# inline: signals: [tokenA, tokenB]
$inner = $Matches[1].Trim()
if ($inner -ne '') {
foreach ($tok in ($inner -split '\s*,\s*')) {
$t = $tok.Trim().Trim('"',"'")
if ($t) { $signals.Add($t) | Out-Null }
}
}
} else {
# block list following the key: `- token` or `- token: X` / mapping
for ($j = $i + 1; $j -lt $fmEnd; $j++) {
$bl = $lines[$j]
if ($bl -match '^\s*-\s*(.+?)\s*$') {
$item = $Matches[1].Trim()
if ($item -match '^token\s*:\s*(.+)$') {
$signals.Add(@{ token = ($Matches[1].Trim().Trim('"',"'")) }) | Out-Null
} elseif ($item -notmatch ':') {
$signals.Add(($item.Trim('"',"'"))) | Out-Null
} else {
# inline-mapping `- {token: X, pattern: Y}` — best effort
$m = [regex]::Match($item, 'token\s*:\s*([^,}\s]+)')
if ($m.Success) { $signals.Add(@{ token = $m.Groups[1].Value.Trim().Trim('"',"'") }) | Out-Null }
}
} elseif ($bl -match '^\s+\w') {
# continuation of a mapping entry (pattern:/domain:) — attach to last mapping
if ($signals.Count -gt 0 -and ($signals[$signals.Count - 1] -is [hashtable])) {
if ($bl -match '^\s*pattern\s*:\s*(.+)$') { $signals[$signals.Count - 1]['pattern'] = $Matches[1].Trim().Trim('"',"'") }
elseif ($bl -match '^\s*domain\s*:\s*(.+)$') { $signals[$signals.Count - 1]['domain'] = $Matches[1].Trim().Trim('"',"'") }
}
} else { break }
}
}
continue
}
}
return [pscustomobject]@{ domain = $domain; keywords = $keywords; signals = @($signals) }
}
# ---- Walk articles ---------------------------------------------------------
$articles = [System.Collections.Generic.List[object]]::new()
foreach ($layerDir in @('microsoft', 'community', 'custom')) {
if ($EnabledLayers -and ($EnabledLayers -notcontains $layerDir)) { continue }
$kbRoot = Join-Path $BCQualityRoot (Join-Path $layerDir 'knowledge')
if (-not (Test-Path $kbRoot)) { continue }
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
Sort-Object FullName |
ForEach-Object {
$rel = Get-RelativePath -Root $BCQualityRoot -Full $_.FullName
$fm = $null
try { $fm = Read-RoutingFrontmatter -Path $_.FullName } catch { $fm = $null }
$rawDomain = if ($fm) { $fm.domain } elseif ($rel -match '/knowledge/([^/]+)/') { $Matches[1] } else { '' }
$articles.Add([pscustomobject]@{
path = $rel
layer = $layerDir
rawDomain = $rawDomain
domain = (ConvertTo-CanonicalDomain $rawDomain)
keywords = if ($fm) { @($fm.keywords) } else { @() }
signals = if ($fm) { @($fm.signals) } else { @() }
}) | Out-Null
}
}
# ---- Build the domain -> article index (for attaching backing articles) ----
$articlesByDomain = @{}
foreach ($a in $articles) {
if (-not $a.domain) { continue }
if (-not $articlesByDomain.ContainsKey($a.domain)) { $articlesByDomain[$a.domain] = [System.Collections.Generic.List[string]]::new() }
$articlesByDomain[$a.domain].Add($a.path) | Out-Null
}
# Signal registry keyed by token so seed + article-declared signals merge and
# never duplicate; articles accumulate onto the matching token.
$signalReg = [ordered]@{}
function Add-Signal {
param([string] $Token, [string] $Pattern, [string] $Domain, [string] $Source, [double] $Weight)
if (-not $signalReg.Contains($Token)) {
$signalReg[$Token] = [ordered]@{
token = $Token; pattern = $Pattern; domain = $Domain; source = $Source; weight = $Weight
articles = [System.Collections.Generic.List[string]]::new()
}
}
return $signalReg[$Token]
}
# 1) Seed signals: guarantee recall >= legacy catalog. Attach every article
# whose canonical domain matches the seed signal's domain.
foreach ($s in $seed.signals) {
$sig = Add-Signal -Token $s.token -Pattern $s.pattern -Domain $s.domain -Source 'seed' -Weight 1.0
if ($articlesByDomain.ContainsKey($s.domain)) {
foreach ($p in $articlesByDomain[$s.domain]) { if (-not $sig.articles.Contains($p)) { $sig.articles.Add($p) | Out-Null } }
}
}
# 2) Article-declared signals (front-matter `signals:`): highest-precision,
# authored. Default pattern = \b<token>\b; default domain = article domain.
foreach ($a in $articles) {
foreach ($decl in $a.signals) {
$token = $null; $pattern = $null; $domain = $null
if ($decl -is [string]) { $token = $decl }
elseif ($decl -is [hashtable]) { $token = $decl['token']; $pattern = $decl['pattern']; $domain = $decl['domain'] }
if (-not $token) { continue }
if (-not $pattern) { $pattern = '\b' + [regex]::Escape($token) + '\b' }
$domain = if ($domain) { ConvertTo-CanonicalDomain $domain } else { $a.domain }
$sig = Add-Signal -Token $token -Pattern $pattern -Domain $domain -Source 'frontmatter-signal' -Weight 1.0
# Promote a seed placeholder to authored if the article overrode it.
if ($sig.source -eq 'seed' -and $decl -isnot [string]) { $sig.source = 'frontmatter-signal'; $sig.pattern = $pattern; $sig.domain = $domain }
if (-not $sig.articles.Contains($a.path)) { $sig.articles.Add($a.path) | Out-Null }
}
}
# 3) OPTIONAL soft keyword signals (off by default).
if ($IncludeKeywordSignals) {
$stop = @{ 'al' = $true; 'bc' = $true; 'all' = $true; 'w1' = $true; 'data' = $true; 'field' = $true; 'table' = $true; 'page' = $true; 'record' = $true; 'value' = $true }
foreach ($a in $articles) {
foreach ($kw in $a.keywords) {
$k = ($kw + '').Trim().ToLowerInvariant()
if (-not $k -or $stop.ContainsKey($k) -or $k.Length -lt 4) { continue }
$token = 'kw:' + $k
$pattern = '(?i)\b' + [regex]::Escape($k) + '\b'
$sig = Add-Signal -Token $token -Pattern $pattern -Domain $a.domain -Source 'keyword' -Weight 0.5
if (-not $sig.articles.Contains($a.path)) { $sig.articles.Add($a.path) | Out-Null }
}
}
}
# ---- Assemble output -------------------------------------------------------
$signalsOut = @($signalReg.Values | ForEach-Object {
[ordered]@{
token = $_.token; pattern = $_.pattern; domain = $_.domain
source = $_.source; weight = $_.weight; articles = @($_.articles)
}
})
$domainsOut = [ordered]@{}
foreach ($d in ($articlesByDomain.Keys | Sort-Object)) {
$domainsOut[$d] = [ordered]@{
articleCount = $articlesByDomain[$d].Count
signalCount = @($signalsOut | Where-Object { $_.domain -eq $d }).Count
}
}
$objKind = [ordered]@{}
foreach ($p in $seed.'object-kind-domain'.PSObject.Properties) { $objKind[$p.Name] = $p.Value }
$domNormOut = [ordered]@{}
foreach ($k in ($domainNorm.Keys | Sort-Object)) { $domNormOut[$k] = $domainNorm[$k] }
$index = [ordered]@{
'$schema-note' = 'BCQuality routing index. signal-token -> domain + backing articles. Consumed by the PR-review orchestrator manifest (BCQ_INDEX_V2). Regenerated at CI time by tools/Build-RoutingIndex.ps1; NOT the lean agent-replayed knowledge index. Schema: tools/routing-index.schema.json.'
version = 1
generatedAt = (Get-Date).ToUniversalTime().ToString('o')
generator = 'Build-RoutingIndex.ps1'
enabledLayers = @($EnabledLayers)
articleCount = $articles.Count
signalCount = $signalsOut.Count
domainNormalization = $domNormOut
objectKindDomain = $objKind
signals = $signalsOut
domains = $domainsOut
}
$indexDir = Split-Path -Parent $IndexPath
if ($indexDir -and -not (Test-Path $indexDir)) { New-Item -ItemType Directory -Force -Path $indexDir | Out-Null }
if ($Pretty) {
$index | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $IndexPath -Encoding UTF8
} else {
$index | ConvertTo-Json -Depth 12 -Compress | Set-Content -LiteralPath $IndexPath -Encoding UTF8
}
Write-Host "BCQuality routing index: $($signalsOut.Count) signal(s), $($articles.Count) article(s). Index: $IndexPath"
return $signalsOut.Count

113
tools/routing-index.md Normal file
View file

@ -0,0 +1,113 @@
# Routing index
The **routing index** (`routing-index.json`) is the orchestrator-facing companion
to the [knowledge index](../tools/Build-KnowledgeIndex.ps1). Where the knowledge
index lets the *agent* enumerate candidate articles inside the CLI run, the
routing index lets the *orchestrator* decide — deterministically, before any
tokens are spent — which review domains a PR touches and which articles back each
detection signal.
It exists to retire a shadow catalog. The PR-review orchestrator historically
carried a hand-written ~28-token regex catalog (`$BcqSignalCatalog`) hardcoded in
`Invoke-CopilotPRReview.ps1`. That catalog never read the `domain` / `keywords`
front-matter every article already declares, so its blind spots were exactly
where findings were missed. The routing index moves that knowledge into content:
signals and their domains are compiled *from articles*, so a new article — even a
community one — becomes routable the moment it lands, with no orchestrator edit.
## What it is (and is not)
- **It is** an orchestrator-side artifact. It is consumed by `Build-ReviewManifest`
(behind the `BCQ_INDEX_V2` flag) to score per-domain suspicion and shortlist
candidate articles. It is **never** fed into the CLI prompt, so it can be richer
than the lean knowledge index without inflating the token-paid path.
- **It is not** committed. Like `knowledge-index.json`, it is a runtime artifact
rebuilt over each consumer's already-pruned clone (`tools/Build-RoutingIndex.ps1`)
and is `.gitignore`d.
- **It is not** the knowledge index. The two are siblings built from the same
front-matter; the knowledge index is agent-replayed, the routing index is not.
## How signals are compiled
`Build-RoutingIndex.ps1` composes signals from three sources, highest precision
first:
1. **Seed** (`tools/routing-seed.json`) — the migrated legacy catalog. Ingested
first so routing recall is never below what the old hardcoded catalog caught.
Each seed signal is then enriched with **every** article whose (normalized)
domain matches, so recall is complete-by-construction rather than limited by a
hand-maintained token→article map.
2. **Article-declared** (`signals:` front-matter, optional) — the authored,
highest-precision path. Use it when an article catches a specific AL construct
the seed does not name.
3. **Keyword-derived** (`-IncludeKeywordSignals`, off by default) — soft signals
from article keywords at reduced weight. Off by default because keywords are
lowercase-kebab and match noisily, and the online-evals feedback shows the agent
already over-fires. Prefer authored `signals:` for precision.
## The optional `signals:` front-matter block
Knowledge articles may declare an **optional** `signals:` block. When omitted, the
article is still routed via the seed + its domain (fully back-compatible — no
existing article must change). Each entry is either a bare token string or a
mapping:
```yaml
domain: performance
keywords: [lock, readonly, isolation]
signals:
- LockTable # bare token -> pattern \bLockTable\b, domain = article domain
- token: ReadIsolation
pattern: '\bReadIsolation\b' # explicit regex (optional)
domain: performance # override domain (optional; normalized)
```
- `token` (required) — stable signal id; need not be an AL identifier verbatim.
- `pattern` (optional) — regex matched against **added** diff lines. Defaults to
`\b<token>\b`.
- `domain` (optional) — defaults to the article's `domain`, then normalized.
The front-matter validator (`.github/scripts/validate_frontmatter.py`, rule R24)
enforces this shape; `signals` is the only optional key in the otherwise-closed
knowledge key set.
## Domain normalization
Three vocabularies describe the same domains and must be reconciled:
| Front-matter (article) | Orchestrator (`$DomainMap`) | Feedback (`byDomain`) |
|------------------------|-----------------------------|-----------------------|
| `ui` | `Accessibility` | `accessibility` |
| `error-handling` | `Error Handling` | — |
| `web-services` | `Web Services` | — |
| `breaking-changes` | `Breaking Changes` | — |
| `events` / `interfaces`| `Events` / `Interfaces` | — |
| `telemetry` | `Privacy` (folded) | — |
`routing-seed.json → domain-normalization` maps front-matter domains to the
orchestrator's canonical TitleCase taxonomy so seed signals (authored in that
taxonomy) attach to the right articles. `appsource` is a documented **pass-through**
domain (indexed, but has no review leaf skill, so it routes only via the
`al-code-review` super-skill). The CI guard fails if a new front-matter domain
appears that is neither normalized nor a pass-through, keeping the map complete as
content grows.
## Layer-replication story
The whole process is open and per-layer. A partner running the pipeline against
their own `community/` or `custom/` layers gets routing for free: their articles'
front-matter (and any `signals:` they add) compile into the same index by the same
generator. No Microsoft-only data is required to build Tier 1. The later Tier 2
feedback overlay (precision weights distilled from online evals) is a separate,
aggregate-only artifact that multiplies onto these Tier-1 weights — each layer
carries its own overlay produced from its own feedback.
## Files
| File | Role |
|------|------|
| `tools/Build-RoutingIndex.ps1` | Generator (scan layers → `routing-index.json`) |
| `tools/routing-seed.json` | Content-owned seed: migrated catalog + domain-normalization + object-kind hints |
| `tools/routing-index.schema.json` | JSON Schema for the compiled artifact |
| `.github/scripts/Test-RoutingIndex.ps1` | CI guard (determinism, recall floor, no orphans, normalization complete) |
| `.github/workflows/knowledge-index.yml` | Runs both index guards on PR/push to main |

80
tools/routing-index.schema.json Normal file
View file

@ -0,0 +1,80 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/microsoft/BCQuality/tools/routing-index.schema.json",
"title": "BCQuality routing index",
"description": "Orchestrator-facing companion to knowledge-index.json. Maps deterministic PR-diff detection signals (AL tokens / regex) to their owning review domain and the knowledge articles that back them. Consumed by the PR-review orchestrator's manifest builder (Build-ReviewManifest, gated behind BCQ_INDEX_V2) to compute per-domain suspicion scores and shortlist candidate articles WITHOUT reading the hardcoded regex catalog that used to live in orchestrator code. Regenerated deterministically at authoring/CI time by tools/Build-RoutingIndex.ps1; never built in the token-paid path. This is NOT the lean agent-replayed knowledge index — it is not fed into the CLI prompt.",
"type": "object",
"required": ["version", "generatedAt", "generator", "articleCount", "signalCount", "domainNormalization", "objectKindDomain", "signals", "domains"],
"additionalProperties": true,
"properties": {
"$schema-note": { "type": "string" },
"version": { "type": "integer", "const": 1 },
"generatedAt": { "type": "string", "format": "date-time" },
"generator": { "type": "string" },
"enabledLayers": { "type": "array", "items": { "type": "string" } },
"articleCount": { "type": "integer", "minimum": 0 },
"signalCount": { "type": "integer", "minimum": 0 },
"domainNormalization": {
"description": "Front-matter domain (lowercase-hyphen) -> orchestrator domain label (TitleCase). Resolves the vocabulary mismatch between article front-matter (ui, error-handling), orchestrator $DomainMap (Accessibility, Error Handling), and feedback byDomain (lowercase).",
"type": "object",
"additionalProperties": { "type": "string" }
},
"objectKindDomain": {
"description": "AL object kind -> weak domain hint (applied once per changed object). Content-owned successor to the orchestrator's $BcqObjectKindDomain.",
"type": "object",
"additionalProperties": { "type": "string" }
},
"signals": {
"description": "Compiled detection signals. Each maps a regex over ADDED diff lines to a domain and the backing articles.",
"type": "array",
"items": {
"type": "object",
"required": ["token", "pattern", "domain", "source", "weight", "articles"],
"additionalProperties": false,
"properties": {
"token": {
"description": "Stable signal identifier (e.g. FindSet). Not necessarily an AL token verbatim.",
"type": "string"
},
"pattern": {
"description": "Regex matched (case-sensitively, as authored) against each added diff line. Escaped for JSON.",
"type": "string"
},
"domain": {
"description": "Owning review domain in the orchestrator's TitleCase taxonomy (post-normalization).",
"type": "string"
},
"source": {
"description": "Where the signal came from: 'seed' (legacy catalog migration), 'frontmatter-signal' (article-declared signals: block), or 'keyword' (soft keyword-derived, weight < 1).",
"type": "string",
"enum": ["seed", "frontmatter-signal", "keyword"]
},
"weight": {
"description": "Tier-1 confidence multiplier applied to signal hit counts in the domain-suspicion score. 1.0 for authored/seed signals; < 1 for soft keyword-derived signals. Tier-2 feedback weights (later) multiply on top of this.",
"type": "number",
"minimum": 0,
"maximum": 1
},
"articles": {
"description": "Repo-relative paths of the knowledge articles this signal routes to. Complete-by-construction: every article whose domain (and, for keyword signals, keyword) matches is attached, so recall is not limited by a hand-maintained catalog.",
"type": "array",
"items": { "type": "string" }
}
}
}
},
"domains": {
"description": "Per-domain aggregates for quick orchestrator lookups.",
"type": "object",
"additionalProperties": {
"type": "object",
"required": ["articleCount", "signalCount"],
"additionalProperties": true,
"properties": {
"articleCount": { "type": "integer", "minimum": 0 },
"signalCount": { "type": "integer", "minimum": 0 }
}
}
}
}
}

63
tools/routing-seed.json Normal file
View file

@ -0,0 +1,63 @@
{
"$schema-note": "BCQuality routing SEED — content-owned migration of the signal catalog that used to live hardcoded in the PR-review orchestrator ($BcqSignalCatalog / $BcqObjectKindDomain in Invoke-CopilotPRReview.ps1). Build-RoutingIndex.ps1 ingests this seed FIRST so the compiled routing-index.json detects at least everything the legacy code catalog did (recall >= today), then enriches each signal with the articles that back it and adds article-declared signals on top. Over time, entries here migrate into per-article `signals:` front-matter and this seed shrinks. Domains use the orchestrator's TitleCase taxonomy (see domain-normalization).",
"version": 1,
"domain-normalization": {
"error-handling": "Error Handling",
"performance": "Performance",
"privacy": "Privacy",
"security": "Security",
"style": "Style",
"testing": "Testing",
"ui": "Accessibility",
"upgrade": "Upgrade",
"breaking-changes": "Breaking Changes",
"events": "Events",
"interfaces": "Interfaces",
"web-services": "Web Services",
"telemetry": "Privacy"
},
"object-kind-domain": {
"page": "Accessibility",
"pageextension": "Accessibility",
"pagecustomization": "Accessibility",
"profile": "Accessibility",
"report": "Performance",
"reportextension": "Performance",
"query": "Performance",
"permissionset": "Security",
"permissionsetextension": "Security",
"entitlement": "Security",
"interface": "Interfaces",
"xmlport": "Web Services"
},
"signals": [
{ "token": "FindSet", "pattern": "\\bFindSet\\b", "domain": "Performance" },
{ "token": "FindFirst", "pattern": "\\bFindFirst\\b", "domain": "Performance" },
{ "token": "FindLast", "pattern": "\\bFindLast\\b", "domain": "Performance" },
{ "token": "SetRange", "pattern": "\\bSetRange\\b", "domain": "Performance" },
{ "token": "SetFilter", "pattern": "\\bSetFilter\\b", "domain": "Performance" },
{ "token": "CalcFields", "pattern": "\\bCalcFields\\b", "domain": "Performance" },
{ "token": "CalcSums", "pattern": "\\bCalcSums\\b", "domain": "Performance" },
{ "token": "SetLoadFields", "pattern": "\\bSetLoadFields\\b", "domain": "Performance" },
{ "token": "SetAutoCalcFields", "pattern": "\\bSetAutoCalcFields\\b", "domain": "Performance" },
{ "token": "loop", "pattern": "\\brepeat\\b", "domain": "Performance" },
{ "token": "HttpClient", "pattern": "\\bHttpClient\\b", "domain": "Security" },
{ "token": "SecretText", "pattern": "\\bSecretText\\b", "domain": "Security" },
{ "token": "IsolatedStorage", "pattern": "\\bIsolatedStorage\\b", "domain": "Security" },
{ "token": "PermissionSet", "pattern": "\\bPermissionSet\\b|\\bPermissions\\s*=", "domain": "Security" },
{ "token": "Entitlement", "pattern": "\\bEntitlement\\b", "domain": "Security" },
{ "token": "DataClassification", "pattern": "\\bDataClassification\\b", "domain": "Privacy" },
{ "token": "Telemetry", "pattern": "\\bLogMessage\\b|\\bFeatureTelemetry\\b", "domain": "Privacy" },
{ "token": "Error", "pattern": "\\bError\\b|\\bFieldError\\b|\\bTestField\\b", "domain": "Error Handling" },
{ "token": "TryFunction", "pattern": "\\[TryFunction\\]", "domain": "Error Handling" },
{ "token": "Commit", "pattern": "\\bCommit\\b", "domain": "Error Handling" },
{ "token": "Caption", "pattern": "\\bCaption\\s*=", "domain": "Accessibility" },
{ "token": "ToolTip", "pattern": "\\bToolTip\\s*=", "domain": "Accessibility" },
{ "token": "ApplicationArea", "pattern": "\\bApplicationArea\\s*=", "domain": "Accessibility" },
{ "token": "Obsolete", "pattern": "\\bObsoleteState\\b|\\bObsoleteReason\\b|\\bObsoleteTag\\b", "domain": "Upgrade" },
{ "token": "UpgradeTag", "pattern": "\\bUpgradeTag\\b", "domain": "Upgrade" },
{ "token": "EventPublisher", "pattern": "\\[IntegrationEvent|\\[BusinessEvent", "domain": "Events" },
{ "token": "EventSubscriber", "pattern": "\\[EventSubscriber", "domain": "Events" },
{ "token": "ApiObject", "pattern": "PageType\\s*=\\s*API|\\bEntityName\\b|\\bEntitySetName\\b|\\bODataKeyFields\\b", "domain": "Web Services" }
]
}