mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Add bounded knowledge retrieval (#179)
* Add bounded knowledge retrieval Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d8b7764-f15a-49ea-8d50-d9147334f8af * Fix bc-version overflow and pathless-row identity in bounded retrieval Catalog matching compared an Int32 -BCVersion against a bigint range bound. PowerShell coerces the right operand to the left operand's type, so a bound wider than Int32 threw a conversion error and failed the whole domain catalog rather than the single row. Metadata validation already accepts such bounds, so compare as bigint on both sides. The shared pager built its oversized-row message with $row.path, which throws under Set-StrictMode -Version Latest when a row carries no path, replacing the explicit bound failure with a property-lookup error. Resolve the path defensively for dictionary and object rows so the offset-based fallback is reachable. Both paths gain regression coverage that fails without these fixes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Jesper Schulz-Wedde <jesper.schulzwedde@microsoft.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d8b7764-f15a-49ea-8d50-d9147334f8af
This commit is contained in:
Jesper Schulz-Wedde
GitHub
parent
c12b2f0a88
commit
51597068b1
26 changed files with 1891 additions and 55 deletions
|
|
@ -30,6 +30,8 @@
|
|||
expected to prune its clone to policy first). For provenance and to
|
||||
reproduce a consumer's exact view, pass -EnabledLayers to restrict the walk
|
||||
to those layers and to record the policy in the index header.
|
||||
Invalid articles are omitted with a path-specific warning so one bad
|
||||
optional layer article cannot block valid siblings.
|
||||
|
||||
.PARAMETER BCQualityRoot
|
||||
Path to the BCQuality content root to index (typically a filtered clone).
|
||||
|
|
@ -69,6 +71,17 @@ param(
|
|||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
|
||||
if ($PSBoundParameters.ContainsKey('EnabledLayers')) {
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array; omit it to index all layers.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
}
|
||||
|
||||
# Default to the clone root (parent of this script's tools/ folder) so the
|
||||
# agent's Entry preparation step can invoke this with no arguments from the
|
||||
|
|
@ -93,6 +106,45 @@ function Get-RelativePath {
|
|||
return ($rel -replace '\\', '/')
|
||||
}
|
||||
|
||||
function Get-BytesSha256 {
|
||||
param([byte[]] $Bytes)
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($Bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Read-ArticleSource {
|
||||
param([string] $Path)
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
try {
|
||||
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||
}
|
||||
catch [Text.DecoderFallbackException] {
|
||||
throw [IO.InvalidDataException]::new('invalid UTF-8', $_.Exception)
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
bytes = $bytes
|
||||
text = $text
|
||||
sha256 = Get-BytesSha256 -Bytes $bytes
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ValueSha256 {
|
||||
param([Parameter(Mandatory)] $Value)
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject $Value -Depth 8 -Compress))
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
# Trims a Description to a single short line (<= $Max chars) for the lean
|
||||
# index. Takes the first sentence; truncates on a word boundary if still long.
|
||||
function Get-LeanDescription {
|
||||
|
|
@ -116,9 +168,12 @@ function ConvertFrom-ArticleFrontmatter {
|
|||
# Pattern) is included; the index is a lossless substitute for the
|
||||
# frontmatter + Description the worklist predicate reads, not a
|
||||
# substitute for the article's normative guidance.
|
||||
param([string] $Path)
|
||||
param(
|
||||
[string] $Path,
|
||||
[string] $Text
|
||||
)
|
||||
|
||||
$lines = Get-Content -LiteralPath $Path -ErrorAction Stop
|
||||
$lines = [regex]::Split($Text.TrimStart([char]0xfeff), '\r\n|\n|\r')
|
||||
|
||||
# Frontmatter is the first '---'-delimited block.
|
||||
if ($lines.Count -lt 1 -or $lines[0].Trim() -ne '---') { return $null }
|
||||
|
|
@ -129,19 +184,42 @@ function ConvertFrom-ArticleFrontmatter {
|
|||
if ($fmEnd -lt 0) { return $null }
|
||||
|
||||
$fm = @{}
|
||||
$arrayFields = @('bc-version', 'keywords', 'technologies', 'countries', 'application-area')
|
||||
for ($i = 1; $i -lt $fmEnd; $i++) {
|
||||
$line = $lines[$i]
|
||||
if ($line -match '^\s*([a-zA-Z][\w-]*)\s*:\s*(.*)$') {
|
||||
$key = $Matches[1]
|
||||
$val = $Matches[2].Trim()
|
||||
if ($val -match '^\[(.*)\]$') {
|
||||
if ($key -in $arrayFields) {
|
||||
if ($val -notmatch '^\[(.*)\]$') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$inner = $Matches[1].Trim()
|
||||
if ($inner -eq '') { $fm[$key] = @() }
|
||||
else { $fm[$key] = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() }) }
|
||||
if ($inner -eq '') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$values = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() })
|
||||
if (@($values | Where-Object { [string]::IsNullOrWhiteSpace($_) }).Count) {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$fm[$key] = $values
|
||||
}
|
||||
elseif ($val -ne '') { $fm[$key] = $val }
|
||||
}
|
||||
}
|
||||
foreach ($field in $arrayFields) {
|
||||
if (-not $fm.ContainsKey($field) -or $fm[$field] -isnot [array] -or -not $fm[$field].Count) {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$field' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
# Body parsing: H1 title and the full Description section. The Description
|
||||
# is the article's primary retrieval target per READ and is captured
|
||||
|
|
@ -185,42 +263,71 @@ $indexArticles = [System.Collections.Generic.List[object]]::new()
|
|||
foreach ($layerDir in @('microsoft', 'community', 'custom')) {
|
||||
$kbRoot = Join-Path $BCQualityRoot (Join-Path $layerDir 'knowledge')
|
||||
if (-not (Test-Path $kbRoot)) { continue }
|
||||
if ($EnabledLayers -and ($EnabledLayers -notcontains $layerDir)) { continue }
|
||||
if ($EnabledLayers -and ($EnabledLayers -cnotcontains $layerDir)) { continue }
|
||||
|
||||
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName |
|
||||
ForEach-Object {
|
||||
$rel = Get-RelativePath -Root $BCQualityRoot -Full $_.FullName
|
||||
$parsed = $null
|
||||
try { $parsed = ConvertFrom-ArticleFrontmatter -Path $_.FullName } catch { $parsed = $null }
|
||||
$files = @(
|
||||
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName
|
||||
)
|
||||
foreach ($file in $files) {
|
||||
$rel = Get-RelativePath -Root $BCQualityRoot -Full $file.FullName
|
||||
try {
|
||||
$source = Read-ArticleSource -Path $file.FullName
|
||||
$parsed = ConvertFrom-ArticleFrontmatter -Path $file.FullName -Text $source.text
|
||||
if (-not $parsed) {
|
||||
# Invalid/unparseable file: list path + domain-from-path so it
|
||||
# is never silently dropped from discovery. Consumers fall back
|
||||
# to reading it in full.
|
||||
$domainFromPath = if ($rel -match '/knowledge/([^/]+)/') { $Matches[1] } else { '' }
|
||||
$indexArticles.Add([pscustomobject]@{
|
||||
path = $rel; layer = $layerDir; domain = $domainFromPath
|
||||
'bc-version' = @(); technologies = @(); countries = @(); 'application-area' = @()
|
||||
keywords = @(); title = ''; description = ''; parsed = $false
|
||||
}) | Out-Null
|
||||
return
|
||||
throw [IO.InvalidDataException]::new('missing or unterminated frontmatter')
|
||||
}
|
||||
foreach ($required in @(
|
||||
@('domain', $parsed.domain),
|
||||
@('H1 title', $parsed.title),
|
||||
@('Description', $parsed.description)
|
||||
)) {
|
||||
if ([string]::IsNullOrWhiteSpace([string]$required[1])) {
|
||||
throw [IO.InvalidDataException]::new("missing $($required[0])")
|
||||
}
|
||||
}
|
||||
$indexArticles.Add([pscustomobject]@{
|
||||
path = $rel
|
||||
layer = $layerDir
|
||||
domain = $parsed.domain
|
||||
'bc-version' = @($parsed.'bc-version')
|
||||
technologies = @($parsed.technologies)
|
||||
countries = @($parsed.countries)
|
||||
'application-area' = @($parsed.'application-area')
|
||||
keywords = @($parsed.keywords)
|
||||
title = $parsed.title
|
||||
description = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
|
||||
parsed = $true
|
||||
}) | Out-Null
|
||||
}
|
||||
catch [IO.InvalidDataException] {
|
||||
Write-Warning "Skipping invalid knowledge article '$rel': $($_.Exception.Message)."
|
||||
continue
|
||||
}
|
||||
|
||||
$article = [ordered]@{
|
||||
path = $rel
|
||||
layer = $layerDir
|
||||
domain = $parsed.domain
|
||||
'bc-version' = @($parsed.'bc-version')
|
||||
technologies = @($parsed.technologies)
|
||||
countries = @($parsed.countries)
|
||||
'application-area' = @($parsed.'application-area')
|
||||
keywords = @($parsed.keywords)
|
||||
title = $parsed.title
|
||||
description = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
|
||||
parsed = $true
|
||||
sourceSha256 = $source.sha256
|
||||
}
|
||||
$problem = Get-KnowledgeMetadataProblem -Row $article
|
||||
if ($problem) {
|
||||
Write-Warning "Skipping invalid knowledge article '$rel': $problem."
|
||||
continue
|
||||
}
|
||||
$indexArticles.Add($article) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$articlesByPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($article in $indexArticles) {
|
||||
if (-not $articlesByPath.TryAdd($article.path, $article)) {
|
||||
throw "Duplicate knowledge path while building source snapshot: $($article.path)"
|
||||
}
|
||||
}
|
||||
$sourcePaths = [string[]]@($articlesByPath.Keys)
|
||||
[Array]::Sort($sourcePaths, [StringComparer]::Ordinal)
|
||||
$sourceManifest = @(
|
||||
foreach ($path in $sourcePaths) {
|
||||
[ordered]@{ path = $path; sha256 = $articlesByPath[$path].sourceSha256 }
|
||||
}
|
||||
)
|
||||
$index = [pscustomobject]@{
|
||||
version = 1
|
||||
generatedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
|
@ -228,6 +335,7 @@ $index = [pscustomobject]@{
|
|||
knowledgeAllow= @($KnowledgeAllow)
|
||||
knowledgeDeny = @($KnowledgeDeny)
|
||||
articleCount = $indexArticles.Count
|
||||
sourceSnapshot= Get-ValueSha256 -Value $sourceManifest
|
||||
articles = @($indexArticles)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue