mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-08-06 17:36:53 +01:00
R6 Tier-1: content-owned routing index generator + CI guard
Retire the orchestrator's hardcoded ~28-token signal catalog by compiling a routing index from article front-matter. New tools/Build-RoutingIndex.ps1 scans all knowledge layers and emits routing-index.json (signal-token -> domain + backing articles), seeded from the migrated legacy catalog (routing-seed.json) so recall is never below today. Optional per-article 'signals:' front-matter (validator rule R24) is the authored precision path; domain-normalization reconciles front-matter/orchestrator/feedback domain vocabularies. Artifact is a runtime build (gitignored), orchestrator-facing only (not the lean agent-replayed knowledge index). CI guard Test-RoutingIndex.ps1 asserts determinism, seed recall floor, no orphaned signals, and normalization completeness; wired into the index workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
parent
4119417ce4
commit
6ed56b95b7
8 changed files with 722 additions and 7 deletions
123
.github/scripts/Test-RoutingIndex.ps1
vendored
Normal file
123
.github/scripts/Test-RoutingIndex.ps1
vendored
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
CI guard for the routing-index generator (tools/Build-RoutingIndex.ps1).
|
||||
|
||||
.DESCRIPTION
|
||||
BCQuality owns the routing index — the orchestrator-facing companion to the
|
||||
knowledge index that maps PR-diff signals to review domains and the articles
|
||||
that back them. Like the knowledge index, no committed artifact is trusted at
|
||||
runtime (a consumer rebuilds it over its pruned clone); this script proves the
|
||||
GENERATOR is healthy:
|
||||
|
||||
1. Determinism — building twice yields byte-identical output once the
|
||||
volatile `generatedAt` header is normalized.
|
||||
2. Recall floor — every seed signal (tools/routing-seed.json) survives into
|
||||
the compiled index, so routing recall is never below the legacy catalog
|
||||
it replaced.
|
||||
3. No orphaned seed signals — a seed signal whose domain has at least one
|
||||
indexed article MUST have that article attached (catches domain
|
||||
normalization drift, e.g. Web Services vs web-services).
|
||||
4. Structural integrity — every signal carries token/pattern/domain/source/
|
||||
weight/articles; every attached article path exists on disk; every
|
||||
signal pattern is a valid regex.
|
||||
5. Normalization coverage — every front-matter domain present on disk either
|
||||
normalizes to a canonical orchestrator domain or is a documented
|
||||
pass-through, so no article silently falls out of routing.
|
||||
|
||||
Exit code 0 = healthy; non-zero = a problem CI must block on.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..' '..'))
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
|
||||
$generator = Join-Path $Root 'tools/Build-RoutingIndex.ps1'
|
||||
$seedPath = Join-Path $Root 'tools/routing-seed.json'
|
||||
if (-not (Test-Path $generator)) { throw "Generator not found: $generator" }
|
||||
if (-not (Test-Path $seedPath)) { throw "Seed not found: $seedPath" }
|
||||
|
||||
$tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("routeindex_" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Force -Path $tmp | Out-Null
|
||||
$idxA = Join-Path $tmp 'a.json'
|
||||
$idxB = Join-Path $tmp 'b.json'
|
||||
|
||||
$problems = [System.Collections.Generic.List[string]]::new()
|
||||
|
||||
& $generator -BCQualityRoot $Root -IndexPath $idxA | Out-Null
|
||||
& $generator -BCQualityRoot $Root -IndexPath $idxB | Out-Null
|
||||
|
||||
# 1. Determinism (ignoring the volatile generatedAt timestamp).
|
||||
$norm = { param($p) ((Get-Content -LiteralPath $p -Raw) -replace '"generatedAt":"[^"]*"', '"generatedAt":"<n>"') }
|
||||
if ((& $norm $idxA) -ne (& $norm $idxB)) {
|
||||
$problems.Add('Non-deterministic: two builds differ beyond generatedAt.') | Out-Null
|
||||
}
|
||||
|
||||
$index = Get-Content -LiteralPath $idxA -Raw | ConvertFrom-Json
|
||||
$signals = @($index.signals)
|
||||
$seed = Get-Content -LiteralPath $seedPath -Raw | ConvertFrom-Json
|
||||
|
||||
# 2. Recall floor: every seed signal token present in the compiled index.
|
||||
$indexTokens = @($signals | ForEach-Object { $_.token })
|
||||
$missingSeed = @($seed.signals | Where-Object { $indexTokens -notcontains $_.token } | ForEach-Object { $_.token })
|
||||
if ($missingSeed.Count) { $problems.Add("Seed signals dropped from index: $($missingSeed -join ', ')") | Out-Null }
|
||||
|
||||
# 3. No orphaned seed signals when the domain is populated.
|
||||
$domainArticleCount = @{}
|
||||
foreach ($p in $index.domains.PSObject.Properties) { $domainArticleCount[$p.Name] = [int]$p.Value.articleCount }
|
||||
foreach ($s in $signals) {
|
||||
if ($s.source -eq 'seed' -and @($s.articles).Count -eq 0) {
|
||||
$ac = if ($domainArticleCount.ContainsKey($s.domain)) { $domainArticleCount[$s.domain] } else { 0 }
|
||||
if ($ac -gt 0) {
|
||||
$problems.Add("Orphaned seed signal '$($s.token)': domain '$($s.domain)' has $ac article(s) but none attached (normalization drift?).") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# 4. Structural integrity.
|
||||
foreach ($s in $signals) {
|
||||
foreach ($f in 'token','pattern','domain','source','weight') {
|
||||
if ($null -eq $s.$f -or ("$($s.$f)").Trim() -eq '') { $problems.Add("Signal missing '$f': $($s.token)") | Out-Null }
|
||||
}
|
||||
try { [void][regex]::new([string]$s.pattern) } catch { $problems.Add("Invalid regex for signal '$($s.token)': $($s.pattern)") | Out-Null }
|
||||
foreach ($ap in @($s.articles)) {
|
||||
if (-not (Test-Path (Join-Path $Root $ap))) { $problems.Add("Signal '$($s.token)' attaches missing article: $ap") | Out-Null }
|
||||
}
|
||||
}
|
||||
|
||||
# 5. Normalization coverage: every on-disk front-matter domain resolves to a
|
||||
# canonical orchestrator domain or is a known pass-through. A NEW front-matter
|
||||
# domain that is not TitleCase and not in the map is flagged so the seed's
|
||||
# domain-normalization stays complete as content grows.
|
||||
$norm2 = @{}
|
||||
foreach ($p in $index.domainNormalization.PSObject.Properties) { $norm2[$p.Name] = $p.Value }
|
||||
$passThrough = @('appsource') # indexed domains with no leaf skill (route via super-skill)
|
||||
$diskDomains = @{}
|
||||
foreach ($layer in 'microsoft','community','custom') {
|
||||
$kb = Join-Path $Root (Join-Path $layer 'knowledge')
|
||||
if (-not (Test-Path $kb)) { continue }
|
||||
Get-ChildItem -LiteralPath $kb -Recurse -File -Filter '*.md' | ForEach-Object {
|
||||
$l = Get-Content -LiteralPath $_.FullName -TotalCount 12
|
||||
$d = ($l | Where-Object { $_ -match '^\s*domain\s*:\s*(.+?)\s*$' } | Select-Object -First 1)
|
||||
if ($d -and $d -match '^\s*domain\s*:\s*(.+?)\s*$') { $diskDomains[$Matches[1].Trim()] = $true }
|
||||
}
|
||||
}
|
||||
foreach ($d in $diskDomains.Keys) {
|
||||
$isCanonical = ($d -cmatch '[A-Z]' -or $d -eq 'appsource') # already TitleCase or pass-through
|
||||
if (-not $norm2.ContainsKey($d) -and -not ($passThrough -contains $d) -and -not $isCanonical) {
|
||||
$problems.Add("Front-matter domain '$d' is not in domain-normalization and not a pass-through; add it to routing-seed.json.") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Routing-index check FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Routing-index check PASSED: $($signals.Count) signals, $($index.articleCount) articles, deterministic, seed recall floor held, normalization complete." -ForegroundColor Green
|
||||
exit 0
|
||||
32
.github/scripts/validate_frontmatter.py
vendored
32
.github/scripts/validate_frontmatter.py
vendored
|
|
@ -34,6 +34,11 @@ KNOWLEDGE_REQUIRED_KEYS = {
|
|||
"bc-version", "domain", "keywords", "technologies",
|
||||
"countries", "application-area",
|
||||
}
|
||||
# Optional knowledge keys (do not trigger the R02 closed-key-set error).
|
||||
# `signals`: OPTIONAL routing-signal declarations consumed by the routing index
|
||||
# (tools/Build-RoutingIndex.ps1). Back-compatible — articles that omit it are
|
||||
# routed from the seed catalog + keyword derivation. See tools/routing-index.md.
|
||||
KNOWLEDGE_OPTIONAL_KEYS = {"signals"}
|
||||
ACTION_SKILL_REQUIRED_KEYS = {
|
||||
"kind", "id", "version", "title", "description", "inputs", "outputs",
|
||||
}
|
||||
|
|
@ -202,7 +207,7 @@ def validate_knowledge(path: Path, parsed: Parsed, report: Report) -> None:
|
|||
|
||||
# R02 required keys, no extras, none empty
|
||||
missing = KNOWLEDGE_REQUIRED_KEYS - fm.keys()
|
||||
extras = fm.keys() - KNOWLEDGE_REQUIRED_KEYS
|
||||
extras = fm.keys() - KNOWLEDGE_REQUIRED_KEYS - KNOWLEDGE_OPTIONAL_KEYS
|
||||
if missing:
|
||||
report.error(path, "R02", f"missing required frontmatter keys: {sorted(missing)}", 1)
|
||||
if extras:
|
||||
|
|
@ -212,6 +217,31 @@ def validate_knowledge(path: Path, parsed: Parsed, report: Report) -> None:
|
|||
if v is None or v == "" or v == []:
|
||||
report.error(path, "R02", f"frontmatter key '{k}' must not be empty", 1)
|
||||
|
||||
# R24 optional routing `signals` block. Each entry is either a bare token
|
||||
# string or a mapping with a required 'token' and optional 'pattern'/'domain'
|
||||
# (all non-empty strings). Keeps the routing-index generator's input honest.
|
||||
if "signals" in fm:
|
||||
sigs = fm["signals"]
|
||||
if not isinstance(sigs, list) or not sigs:
|
||||
report.error(path, "R24", "signals must be a non-empty list", 1)
|
||||
else:
|
||||
for entry in sigs:
|
||||
if isinstance(entry, str):
|
||||
if not entry.strip():
|
||||
report.error(path, "R24", "signals token string must not be empty", 1)
|
||||
elif isinstance(entry, dict):
|
||||
tok = entry.get("token")
|
||||
if not isinstance(tok, str) or not tok.strip():
|
||||
report.error(path, "R24", "signals entry must have a non-empty 'token'", 1)
|
||||
for opt in ("pattern", "domain"):
|
||||
if opt in entry and (not isinstance(entry[opt], str) or not entry[opt].strip()):
|
||||
report.error(path, "R24", f"signals '{opt}' must be a non-empty string", 1)
|
||||
unknown = set(entry.keys()) - {"token", "pattern", "domain"}
|
||||
if unknown:
|
||||
report.error(path, "R24", f"signals entry has unknown keys: {sorted(unknown)}", 1)
|
||||
else:
|
||||
report.error(path, "R24", "signals entry must be a string or a mapping", 1)
|
||||
|
||||
# R03 bc-version
|
||||
if "bc-version" in fm:
|
||||
_, err = expand_bc_version(fm["bc-version"])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue