mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 06:36:55 +01:00
Add bounded knowledge retrieval (#179)
* Add bounded knowledge retrieval Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d8b7764-f15a-49ea-8d50-d9147334f8af * Fix bc-version overflow and pathless-row identity in bounded retrieval Catalog matching compared an Int32 -BCVersion against a bigint range bound. PowerShell coerces the right operand to the left operand's type, so a bound wider than Int32 threw a conversion error and failed the whole domain catalog rather than the single row. Metadata validation already accepts such bounds, so compare as bigint on both sides. The shared pager built its oversized-row message with $row.path, which throws under Set-StrictMode -Version Latest when a row carries no path, replacing the explicit bound failure with a property-lookup error. Resolve the path defensively for dictionary and object rows so the offset-based fallback is reachable. Both paths gain regression coverage that fails without these fixes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Jesper Schulz-Wedde <jesper.schulzwedde@microsoft.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d8b7764-f15a-49ea-8d50-d9147334f8af
This commit is contained in:
Jesper Schulz-Wedde
GitHub
parent
c12b2f0a88
commit
51597068b1
26 changed files with 1891 additions and 55 deletions
117
tools/Bounded-Results.ps1
Normal file
117
tools/Bounded-Results.ps1
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
# Shared deterministic paging. Callers build the complete immutable result first.
|
||||
#requires -Version 7.2
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Get-ResultSnapshot {
|
||||
param([Parameter(Mandatory)] $Value)
|
||||
|
||||
$json = ConvertTo-Json -InputObject $Value -Depth 30 -Compress
|
||||
return [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($json))
|
||||
).ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Get-SerializedByteCount {
|
||||
param([Parameter(Mandatory)] [string] $Json)
|
||||
|
||||
# PowerShell writes one platform newline after the returned JSON string.
|
||||
return [Text.Encoding]::UTF8.GetByteCount($Json) +
|
||||
[Text.Encoding]::UTF8.GetByteCount([Environment]::NewLine)
|
||||
}
|
||||
|
||||
function ConvertTo-BoundedPage {
|
||||
param(
|
||||
[Parameter(Mandatory)] [Collections.IDictionary] $Header,
|
||||
[Parameter(Mandatory)] [Collections.IDictionary] $Groups,
|
||||
[ValidateRange(0, 2147483647)] [int] $Offset = 0,
|
||||
[string] $Snapshot,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000
|
||||
)
|
||||
|
||||
$total = 0
|
||||
foreach ($name in $Groups.Keys) {
|
||||
$total += $Groups[$name].Count
|
||||
}
|
||||
if (($total -eq 0 -and $Offset -ne 0) -or ($total -gt 0 -and $Offset -ge $total)) {
|
||||
throw "Invalid Offset=$Offset for totalCount=$total; no rows were returned."
|
||||
}
|
||||
if ($Offset -gt 0 -and -not $Snapshot) {
|
||||
throw 'Continuation requires Snapshot from the preceding page.'
|
||||
}
|
||||
if ($Snapshot -and $Snapshot -cne $Header.snapshot) {
|
||||
throw 'Snapshot changed or continuation belongs to another request. Discard partial results and restart at Offset=0.'
|
||||
}
|
||||
|
||||
$page = [ordered]@{}
|
||||
foreach ($key in $Header.Keys) {
|
||||
$page[$key] = $Header[$key]
|
||||
}
|
||||
$page.offset = $Offset
|
||||
$page.returnedCount = 0
|
||||
$page.totalCount = $total
|
||||
$page.remainingCount = $total - $Offset
|
||||
$page.complete = ($total -eq 0)
|
||||
$page.continuation = if ($total) {
|
||||
[ordered]@{ offset = $Offset; snapshot = $Header.snapshot }
|
||||
}
|
||||
else {
|
||||
$null
|
||||
}
|
||||
foreach ($name in $Groups.Keys) {
|
||||
$page[$name] = [Collections.Generic.List[object]]::new()
|
||||
}
|
||||
|
||||
$json = ConvertTo-Json -InputObject $page -Depth 30 -Compress
|
||||
if ((Get-SerializedByteCount -Json $json) -gt $MaxBytes) {
|
||||
throw "Page envelope exceeds MaxBytes=$MaxBytes. Use READ's path-discovery fallback; never truncate."
|
||||
}
|
||||
|
||||
$position = 0
|
||||
foreach ($name in $Groups.Keys) {
|
||||
foreach ($row in $Groups[$name]) {
|
||||
if ($position++ -lt $Offset) {
|
||||
continue
|
||||
}
|
||||
|
||||
$page[$name].Add($row)
|
||||
$page.returnedCount++
|
||||
$page.remainingCount--
|
||||
$page.complete = ($page.remainingCount -eq 0)
|
||||
$page.continuation = if ($page.complete) {
|
||||
$null
|
||||
}
|
||||
else {
|
||||
[ordered]@{
|
||||
offset = $Offset + $page.returnedCount
|
||||
snapshot = $Header.snapshot
|
||||
}
|
||||
}
|
||||
|
||||
$next = ConvertTo-Json -InputObject $page -Depth 30 -Compress
|
||||
if ((Get-SerializedByteCount -Json $next) -gt $MaxBytes) {
|
||||
$page[$name].RemoveAt($page[$name].Count - 1)
|
||||
$page.returnedCount--
|
||||
$page.remainingCount++
|
||||
$page.complete = $false
|
||||
$page.continuation = [ordered]@{
|
||||
offset = $Offset + $page.returnedCount
|
||||
snapshot = $Header.snapshot
|
||||
}
|
||||
if ($page.returnedCount -eq 0) {
|
||||
$rowPath = $null
|
||||
if ($row -is [Collections.IDictionary]) {
|
||||
if ($row.Contains('path')) { $rowPath = $row['path'] }
|
||||
}
|
||||
elseif ($null -ne $row -and $row.PSObject.Properties['path']) {
|
||||
$rowPath = $row.PSObject.Properties['path'].Value
|
||||
}
|
||||
$identity = if ($rowPath) { " at $rowPath" } else { " at Offset=$Offset" }
|
||||
throw "One complete $name row plus envelope exceeds MaxBytes=$MaxBytes$identity. No row was clipped."
|
||||
}
|
||||
return $json
|
||||
}
|
||||
$json = $next
|
||||
}
|
||||
}
|
||||
return $json
|
||||
}
|
||||
|
|
@ -30,6 +30,8 @@
|
|||
expected to prune its clone to policy first). For provenance and to
|
||||
reproduce a consumer's exact view, pass -EnabledLayers to restrict the walk
|
||||
to those layers and to record the policy in the index header.
|
||||
Invalid articles are omitted with a path-specific warning so one bad
|
||||
optional layer article cannot block valid siblings.
|
||||
|
||||
.PARAMETER BCQualityRoot
|
||||
Path to the BCQuality content root to index (typically a filtered clone).
|
||||
|
|
@ -69,6 +71,17 @@ param(
|
|||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
|
||||
if ($PSBoundParameters.ContainsKey('EnabledLayers')) {
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array; omit it to index all layers.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
}
|
||||
|
||||
# Default to the clone root (parent of this script's tools/ folder) so the
|
||||
# agent's Entry preparation step can invoke this with no arguments from the
|
||||
|
|
@ -93,6 +106,45 @@ function Get-RelativePath {
|
|||
return ($rel -replace '\\', '/')
|
||||
}
|
||||
|
||||
function Get-BytesSha256 {
|
||||
param([byte[]] $Bytes)
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($Bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Read-ArticleSource {
|
||||
param([string] $Path)
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
try {
|
||||
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||
}
|
||||
catch [Text.DecoderFallbackException] {
|
||||
throw [IO.InvalidDataException]::new('invalid UTF-8', $_.Exception)
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
bytes = $bytes
|
||||
text = $text
|
||||
sha256 = Get-BytesSha256 -Bytes $bytes
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ValueSha256 {
|
||||
param([Parameter(Mandatory)] $Value)
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject $Value -Depth 8 -Compress))
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
# Trims a Description to a single short line (<= $Max chars) for the lean
|
||||
# index. Takes the first sentence; truncates on a word boundary if still long.
|
||||
function Get-LeanDescription {
|
||||
|
|
@ -116,9 +168,12 @@ function ConvertFrom-ArticleFrontmatter {
|
|||
# Pattern) is included; the index is a lossless substitute for the
|
||||
# frontmatter + Description the worklist predicate reads, not a
|
||||
# substitute for the article's normative guidance.
|
||||
param([string] $Path)
|
||||
param(
|
||||
[string] $Path,
|
||||
[string] $Text
|
||||
)
|
||||
|
||||
$lines = Get-Content -LiteralPath $Path -ErrorAction Stop
|
||||
$lines = [regex]::Split($Text.TrimStart([char]0xfeff), '\r\n|\n|\r')
|
||||
|
||||
# Frontmatter is the first '---'-delimited block.
|
||||
if ($lines.Count -lt 1 -or $lines[0].Trim() -ne '---') { return $null }
|
||||
|
|
@ -129,19 +184,42 @@ function ConvertFrom-ArticleFrontmatter {
|
|||
if ($fmEnd -lt 0) { return $null }
|
||||
|
||||
$fm = @{}
|
||||
$arrayFields = @('bc-version', 'keywords', 'technologies', 'countries', 'application-area')
|
||||
for ($i = 1; $i -lt $fmEnd; $i++) {
|
||||
$line = $lines[$i]
|
||||
if ($line -match '^\s*([a-zA-Z][\w-]*)\s*:\s*(.*)$') {
|
||||
$key = $Matches[1]
|
||||
$val = $Matches[2].Trim()
|
||||
if ($val -match '^\[(.*)\]$') {
|
||||
if ($key -in $arrayFields) {
|
||||
if ($val -notmatch '^\[(.*)\]$') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$inner = $Matches[1].Trim()
|
||||
if ($inner -eq '') { $fm[$key] = @() }
|
||||
else { $fm[$key] = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() }) }
|
||||
if ($inner -eq '') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$values = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() })
|
||||
if (@($values | Where-Object { [string]::IsNullOrWhiteSpace($_) }).Count) {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$fm[$key] = $values
|
||||
}
|
||||
elseif ($val -ne '') { $fm[$key] = $val }
|
||||
}
|
||||
}
|
||||
foreach ($field in $arrayFields) {
|
||||
if (-not $fm.ContainsKey($field) -or $fm[$field] -isnot [array] -or -not $fm[$field].Count) {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$field' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
# Body parsing: H1 title and the full Description section. The Description
|
||||
# is the article's primary retrieval target per READ and is captured
|
||||
|
|
@ -185,42 +263,71 @@ $indexArticles = [System.Collections.Generic.List[object]]::new()
|
|||
foreach ($layerDir in @('microsoft', 'community', 'custom')) {
|
||||
$kbRoot = Join-Path $BCQualityRoot (Join-Path $layerDir 'knowledge')
|
||||
if (-not (Test-Path $kbRoot)) { continue }
|
||||
if ($EnabledLayers -and ($EnabledLayers -notcontains $layerDir)) { continue }
|
||||
if ($EnabledLayers -and ($EnabledLayers -cnotcontains $layerDir)) { continue }
|
||||
|
||||
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName |
|
||||
ForEach-Object {
|
||||
$rel = Get-RelativePath -Root $BCQualityRoot -Full $_.FullName
|
||||
$parsed = $null
|
||||
try { $parsed = ConvertFrom-ArticleFrontmatter -Path $_.FullName } catch { $parsed = $null }
|
||||
$files = @(
|
||||
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName
|
||||
)
|
||||
foreach ($file in $files) {
|
||||
$rel = Get-RelativePath -Root $BCQualityRoot -Full $file.FullName
|
||||
try {
|
||||
$source = Read-ArticleSource -Path $file.FullName
|
||||
$parsed = ConvertFrom-ArticleFrontmatter -Path $file.FullName -Text $source.text
|
||||
if (-not $parsed) {
|
||||
# Invalid/unparseable file: list path + domain-from-path so it
|
||||
# is never silently dropped from discovery. Consumers fall back
|
||||
# to reading it in full.
|
||||
$domainFromPath = if ($rel -match '/knowledge/([^/]+)/') { $Matches[1] } else { '' }
|
||||
$indexArticles.Add([pscustomobject]@{
|
||||
path = $rel; layer = $layerDir; domain = $domainFromPath
|
||||
'bc-version' = @(); technologies = @(); countries = @(); 'application-area' = @()
|
||||
keywords = @(); title = ''; description = ''; parsed = $false
|
||||
}) | Out-Null
|
||||
return
|
||||
throw [IO.InvalidDataException]::new('missing or unterminated frontmatter')
|
||||
}
|
||||
foreach ($required in @(
|
||||
@('domain', $parsed.domain),
|
||||
@('H1 title', $parsed.title),
|
||||
@('Description', $parsed.description)
|
||||
)) {
|
||||
if ([string]::IsNullOrWhiteSpace([string]$required[1])) {
|
||||
throw [IO.InvalidDataException]::new("missing $($required[0])")
|
||||
}
|
||||
}
|
||||
$indexArticles.Add([pscustomobject]@{
|
||||
path = $rel
|
||||
layer = $layerDir
|
||||
domain = $parsed.domain
|
||||
'bc-version' = @($parsed.'bc-version')
|
||||
technologies = @($parsed.technologies)
|
||||
countries = @($parsed.countries)
|
||||
'application-area' = @($parsed.'application-area')
|
||||
keywords = @($parsed.keywords)
|
||||
title = $parsed.title
|
||||
description = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
|
||||
parsed = $true
|
||||
}) | Out-Null
|
||||
}
|
||||
catch [IO.InvalidDataException] {
|
||||
Write-Warning "Skipping invalid knowledge article '$rel': $($_.Exception.Message)."
|
||||
continue
|
||||
}
|
||||
|
||||
$article = [ordered]@{
|
||||
path = $rel
|
||||
layer = $layerDir
|
||||
domain = $parsed.domain
|
||||
'bc-version' = @($parsed.'bc-version')
|
||||
technologies = @($parsed.technologies)
|
||||
countries = @($parsed.countries)
|
||||
'application-area' = @($parsed.'application-area')
|
||||
keywords = @($parsed.keywords)
|
||||
title = $parsed.title
|
||||
description = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
|
||||
parsed = $true
|
||||
sourceSha256 = $source.sha256
|
||||
}
|
||||
$problem = Get-KnowledgeMetadataProblem -Row $article
|
||||
if ($problem) {
|
||||
Write-Warning "Skipping invalid knowledge article '$rel': $problem."
|
||||
continue
|
||||
}
|
||||
$indexArticles.Add($article) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$articlesByPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($article in $indexArticles) {
|
||||
if (-not $articlesByPath.TryAdd($article.path, $article)) {
|
||||
throw "Duplicate knowledge path while building source snapshot: $($article.path)"
|
||||
}
|
||||
}
|
||||
$sourcePaths = [string[]]@($articlesByPath.Keys)
|
||||
[Array]::Sort($sourcePaths, [StringComparer]::Ordinal)
|
||||
$sourceManifest = @(
|
||||
foreach ($path in $sourcePaths) {
|
||||
[ordered]@{ path = $path; sha256 = $articlesByPath[$path].sourceSha256 }
|
||||
}
|
||||
)
|
||||
$index = [pscustomobject]@{
|
||||
version = 1
|
||||
generatedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
|
@ -228,6 +335,7 @@ $index = [pscustomobject]@{
|
|||
knowledgeAllow= @($KnowledgeAllow)
|
||||
knowledgeDeny = @($KnowledgeDeny)
|
||||
articleCount = $indexArticles.Count
|
||||
sourceSnapshot= Get-ValueSha256 -Value $sourceManifest
|
||||
articles = @($indexArticles)
|
||||
}
|
||||
|
||||
|
|
|
|||
187
tools/Get-KnowledgeArticles.ps1
Normal file
187
tools/Get-KnowledgeArticles.ps1
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Reads a bounded prefix of exact article or sample paths without altering bodies.
|
||||
.DESCRIPTION
|
||||
The UTF-8 byte size bound covers the complete serialized JSON plus its output
|
||||
newline. A body that cannot fit fails explicitly; it is never summarized or
|
||||
truncated. Samples are loaded only with -Samples and must be linked by their
|
||||
sibling article using READ's exact link convention.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateNotNullOrEmpty()] [string] $BCQualityRoot = (Split-Path $PSScriptRoot -Parent),
|
||||
[Parameter(Mandatory)] [ValidateNotNullOrEmpty()] [string[]] $Paths,
|
||||
[ValidateRange(1, 8)] [int] $MaxArticles = 8,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000,
|
||||
[ValidateSet('microsoft', 'community', 'custom')]
|
||||
[AllowEmptyCollection()] [string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
|
||||
[string] $IndexPath,
|
||||
[ValidatePattern('^[a-f0-9]{64}$')] [string] $Snapshot,
|
||||
[switch] $Samples
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
. (Join-Path $PSScriptRoot 'Bounded-Results.ps1')
|
||||
|
||||
$BCQualityRoot = Resolve-KnowledgeRoot $BCQualityRoot
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'knowledge-index.json'
|
||||
}
|
||||
if ($Paths.Count -gt $MaxArticles) {
|
||||
throw "Paths count $($Paths.Count) exceeds MaxArticles=$MaxArticles. Split the worklist into stable chunks of at most $MaxArticles exact paths."
|
||||
}
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
|
||||
$recovery = "Run Entry preparation once before dispatch, or use READ's bounded native-file fallback. Do not rebuild in a leaf."
|
||||
$preparedIndex = Read-PreparedKnowledgeIndex -IndexPath $IndexPath -Recovery $recovery
|
||||
$index = $preparedIndex.index
|
||||
$byPath = $preparedIndex.byPath
|
||||
$unrestricted = $index.enabledLayers.Count -eq 0 -or
|
||||
($index.enabledLayers.Count -eq 1 -and $null -eq $index.enabledLayers[0])
|
||||
$indexedLayers = @(
|
||||
if ($unrestricted) { 'microsoft', 'community', 'custom' } else { $index.enabledLayers }
|
||||
)
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin $indexedLayers }).Count) {
|
||||
throw "Index layer coverage does not cover EnabledLayers. $recovery"
|
||||
}
|
||||
|
||||
$resolved = [Collections.Generic.List[string]]::new()
|
||||
$records = [Collections.Generic.List[object]]::new()
|
||||
$seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
$articleTexts = [Collections.Generic.Dictionary[string, string]]::new([StringComparer]::Ordinal)
|
||||
$sampleContents = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($path in $Paths) {
|
||||
$kind = if ($Samples) { 'sample' } else { 'article' }
|
||||
$fullPath = Resolve-KnowledgePath -Root $BCQualityRoot -Path $path -Kind $kind
|
||||
if ($path.Split('/')[0] -cnotin $EnabledLayers) {
|
||||
throw "Layer disabled for path: $path"
|
||||
}
|
||||
if (-not $seen.Add($path)) {
|
||||
throw "Duplicate requested path: $path"
|
||||
}
|
||||
if ($Samples) {
|
||||
$articlePath = $path -replace '\.(good|bad)\.[a-z0-9]+$', '.md'
|
||||
if (-not $byPath.ContainsKey($articlePath)) {
|
||||
throw "Sample article is absent from the prepared index: $articlePath"
|
||||
}
|
||||
$fullArticlePath = Resolve-KnowledgePath -Root $BCQualityRoot -Path $articlePath
|
||||
if (-not $articleTexts.ContainsKey($articlePath)) {
|
||||
$articleContent = Read-KnowledgeText -Path $fullArticlePath
|
||||
if ($articleContent.sha256 -cne $byPath[$articlePath].sourceSha256) {
|
||||
throw "Selected article hash does not match the prepared index: $articlePath"
|
||||
}
|
||||
$articleTexts.Add($articlePath, $articleContent.text)
|
||||
}
|
||||
Assert-SampleLink -ArticleText $articleTexts[$articlePath] -SamplePath $fullPath
|
||||
$sampleContent = Read-KnowledgeText -Path $fullPath
|
||||
$sampleContents.Add($path, $sampleContent)
|
||||
$records.Add([ordered]@{
|
||||
path = $path
|
||||
articlePath = $articlePath
|
||||
articleSha256 = $byPath[$articlePath].sourceSha256
|
||||
sampleSha256 = $sampleContent.sha256
|
||||
sampleBytes = $sampleContent.bytes
|
||||
})
|
||||
}
|
||||
else {
|
||||
if (-not $byPath.ContainsKey($path)) {
|
||||
throw "Selected article is absent from the prepared index: $path"
|
||||
}
|
||||
$records.Add([ordered]@{
|
||||
path = $path
|
||||
expectedSha256 = $byPath[$path].sourceSha256
|
||||
})
|
||||
}
|
||||
$resolved.Add($fullPath)
|
||||
}
|
||||
|
||||
$requestSnapshot = Get-ResultSnapshot -Value ([ordered]@{
|
||||
root = $BCQualityRoot
|
||||
preparedIndexSha256 = $preparedIndex.content.sha256
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
enabledLayers = @($EnabledLayers)
|
||||
files = @($records)
|
||||
})
|
||||
if ($Snapshot -and $Snapshot -cne $requestSnapshot) {
|
||||
throw 'Article snapshot changed or continuation belongs to another exact path batch. Discard partial results and restart.'
|
||||
}
|
||||
|
||||
$articles = [Collections.Generic.List[object]]::new()
|
||||
function ConvertTo-BatchJson {
|
||||
param([int] $ReadCount)
|
||||
|
||||
$remaining = @(
|
||||
if ($ReadCount -lt $Paths.Count) {
|
||||
$Paths[$ReadCount..($Paths.Count - 1)]
|
||||
}
|
||||
)
|
||||
$remainingRecords = @(
|
||||
if ($ReadCount -lt $records.Count) {
|
||||
$records[$ReadCount..($records.Count - 1)]
|
||||
}
|
||||
)
|
||||
$continuation = if ($remaining.Count) {
|
||||
[ordered]@{
|
||||
snapshot = Get-ResultSnapshot -Value ([ordered]@{
|
||||
root = $BCQualityRoot
|
||||
preparedIndexSha256 = $preparedIndex.content.sha256
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
enabledLayers = @($EnabledLayers)
|
||||
files = $remainingRecords
|
||||
})
|
||||
}
|
||||
}
|
||||
else {
|
||||
$null
|
||||
}
|
||||
return [ordered]@{
|
||||
version = 1
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
snapshot = $requestSnapshot
|
||||
requestedCount = $Paths.Count
|
||||
returnedCount = $ReadCount
|
||||
complete = ($ReadCount -eq $Paths.Count)
|
||||
articles = @($articles)
|
||||
remainingPaths = $remaining
|
||||
continuation = $continuation
|
||||
} | ConvertTo-Json -Depth 8 -Compress
|
||||
}
|
||||
|
||||
$json = ''
|
||||
for ($i = 0; $i -lt [Math]::Min($MaxArticles, $Paths.Count); $i++) {
|
||||
$content = if ($Samples) {
|
||||
$sampleContents[$Paths[$i]]
|
||||
}
|
||||
else {
|
||||
Read-KnowledgeText -Path $resolved[$i]
|
||||
}
|
||||
if (-not $Samples -and $content.sha256 -cne $records[$i].expectedSha256) {
|
||||
throw "Selected article hash does not match the prepared index: $($Paths[$i])"
|
||||
}
|
||||
$articles.Add([ordered]@{
|
||||
path = $Paths[$i]
|
||||
bytes = $content.bytes
|
||||
sha256 = $content.sha256
|
||||
body = $content.text
|
||||
})
|
||||
$next = ConvertTo-BatchJson -ReadCount ($i + 1)
|
||||
if ((Get-SerializedByteCount -Json $next) -gt $MaxBytes) {
|
||||
$articles.RemoveAt($articles.Count - 1)
|
||||
if ($i -eq 0) {
|
||||
throw "No complete body plus continuation fits MaxBytes=$MaxBytes at $($Paths[$i]). Use a smaller exact path batch or READ's bounded native-file fallback; never truncate."
|
||||
}
|
||||
break
|
||||
}
|
||||
$json = $next
|
||||
}
|
||||
|
||||
$json
|
||||
298
tools/Knowledge-Retrieval.ps1
Normal file
298
tools/Knowledge-Retrieval.ps1
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
# Shared filesystem guards for catalog and exact article retrieval.
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Resolve-KnowledgeRoot {
|
||||
param([string] $Root)
|
||||
|
||||
$item = Get-Item -LiteralPath $Root -Force -ErrorAction Stop
|
||||
if ($item.PSProvider.Name -ne 'FileSystem' -or -not $item.PSIsContainer) {
|
||||
throw "BCQuality root must be a filesystem directory: $Root"
|
||||
}
|
||||
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "Linked BCQuality roots are not supported: $Root"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Assert-KnowledgePath {
|
||||
param(
|
||||
[string] $Path,
|
||||
[ValidateSet('article', 'sample')] [string] $Kind = 'article'
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Path) -or
|
||||
$Path -cnotmatch '^(microsoft|community|custom)/knowledge/[^/]+/.+' -or
|
||||
$Path -match '[\\:*?"<>|\x00-\x1f]' -or
|
||||
@($Path.Split('/') | Where-Object { $_ -in '', '.', '..' -or $_ -match '[. ]$' }).Count) {
|
||||
throw "Invalid knowledge path: $Path"
|
||||
}
|
||||
if (($Kind -eq 'article' -and -not $Path.EndsWith('.md', [StringComparison]::Ordinal)) -or
|
||||
($Kind -eq 'sample' -and $Path -cnotmatch '\.(good|bad)\.[a-z0-9]+$')) {
|
||||
throw "Expected an exact $Kind path: $Path"
|
||||
}
|
||||
}
|
||||
|
||||
function Resolve-KnowledgePath {
|
||||
param(
|
||||
[string] $Root,
|
||||
[string] $Path,
|
||||
[ValidateSet('article', 'sample')] [string] $Kind = 'article'
|
||||
)
|
||||
|
||||
Assert-KnowledgePath -Path $Path -Kind $Kind
|
||||
$current = $Root
|
||||
foreach ($part in $Path.Split('/')) {
|
||||
$items = @(
|
||||
Get-ChildItem -LiteralPath $current -Filter $part -Force -ErrorAction Stop |
|
||||
Where-Object Name -CEQ $part
|
||||
)
|
||||
if ($items.Count -ne 1) {
|
||||
throw "Knowledge path does not exist with exact casing: $Path"
|
||||
}
|
||||
$item = $items[0]
|
||||
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "Linked knowledge paths are not supported: $Path"
|
||||
}
|
||||
$current = $item.FullName
|
||||
}
|
||||
if ($item.PSIsContainer) {
|
||||
throw "Knowledge path is not a file: $Path"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Read-KnowledgeText {
|
||||
param([string] $Path)
|
||||
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
try {
|
||||
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||
}
|
||||
catch {
|
||||
throw "Knowledge file is not valid strict UTF-8: $Path"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
text = $text
|
||||
bytes = $bytes.Length
|
||||
sha256 = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($bytes)
|
||||
).ToLowerInvariant()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NormalizedKnowledgeVersions {
|
||||
param([string[]] $Values)
|
||||
|
||||
foreach ($value in $Values) {
|
||||
if ($value -match '^"([^"]*)"$' -or $value -match "^'([^']*)'$") {
|
||||
$Matches[1]
|
||||
}
|
||||
else {
|
||||
$value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-KnowledgeMetadataProblem {
|
||||
param([Collections.IDictionary] $Row)
|
||||
|
||||
if ($Row['parsed'] -isnot [bool] -or -not $Row['parsed']) {
|
||||
return 'unparsed frontmatter'
|
||||
}
|
||||
if ($Row['domain'] -isnot [string] -or
|
||||
$Row['domain'] -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$') {
|
||||
return 'missing/invalid domain'
|
||||
}
|
||||
foreach ($field in @('bc-version', 'technologies', 'countries', 'application-area', 'keywords')) {
|
||||
if ($Row[$field] -isnot [array] -or -not $Row[$field].Count) {
|
||||
return "missing/invalid $field"
|
||||
}
|
||||
foreach ($value in $Row[$field]) {
|
||||
if ($value -isnot [string] -or [string]::IsNullOrWhiteSpace($value)) {
|
||||
return "invalid $field value"
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach ($field in @('title', 'description')) {
|
||||
if ($Row[$field] -isnot [string] -or
|
||||
[string]::IsNullOrWhiteSpace($Row[$field]) -or
|
||||
$Row[$field] -match '[\r\n]') {
|
||||
return "missing/invalid $field"
|
||||
}
|
||||
}
|
||||
|
||||
$versions = @(Get-NormalizedKnowledgeVersions -Values $Row['bc-version'])
|
||||
if ($versions -ccontains 'all') {
|
||||
if ($versions.Count -ne 1) {
|
||||
return 'mixed bc-version sentinel'
|
||||
}
|
||||
}
|
||||
elseif ($versions.Count -eq 1 -and $versions[0] -match '^(\d+)\.\.(\d+)?$') {
|
||||
$start = [bigint]::Parse($Matches[1])
|
||||
if ($start -le 0 -or ($Matches[2] -and [bigint]::Parse($Matches[2]) -le 0)) {
|
||||
return 'invalid bc-version range bound'
|
||||
}
|
||||
if ($Matches[2] -and $start -gt [bigint]::Parse($Matches[2])) {
|
||||
return 'descending bc-version range'
|
||||
}
|
||||
}
|
||||
else {
|
||||
foreach ($version in $versions) {
|
||||
if ($version -notmatch '^\d+$' -or [bigint]::Parse($version) -le 0) {
|
||||
return 'invalid bc-version'
|
||||
}
|
||||
}
|
||||
}
|
||||
if (@($Row.technologies | Where-Object { $_ -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$' }).Count) {
|
||||
return 'invalid technologies'
|
||||
}
|
||||
if ($Row.technologies -ccontains 'all') {
|
||||
return 'invalid technologies sentinel'
|
||||
}
|
||||
if ($Row.countries -ccontains 'w1') {
|
||||
if ($Row.countries.Count -ne 1) {
|
||||
return 'mixed countries sentinel'
|
||||
}
|
||||
}
|
||||
elseif (@($Row.countries | Where-Object { $_ -cnotmatch '^[a-z]{2}$' }).Count) {
|
||||
return 'invalid countries'
|
||||
}
|
||||
if (@($Row['application-area'] | Where-Object { $_ -cnotmatch '^(all|[a-z0-9]+(-[a-z0-9]+)*)$' }).Count) {
|
||||
return 'invalid application-area'
|
||||
}
|
||||
if ($Row['application-area'] -ccontains 'all' -and $Row['application-area'].Count -ne 1) {
|
||||
return 'mixed application-area sentinel'
|
||||
}
|
||||
if (@($Row.keywords | Where-Object { $_ -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$' }).Count) {
|
||||
return 'invalid keywords'
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function Get-PreparedManifestSha256 {
|
||||
param(
|
||||
[string[]] $Paths,
|
||||
[Collections.Generic.Dictionary[string, object]] $ByPath
|
||||
)
|
||||
|
||||
$hash = [Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
$hash.AppendData([byte[]][char]'[')
|
||||
for ($i = 0; $i -lt $Paths.Count; $i++) {
|
||||
if ($i) {
|
||||
$hash.AppendData([byte[]][char]',')
|
||||
}
|
||||
$row = [ordered]@{
|
||||
path = $Paths[$i]
|
||||
sha256 = $ByPath[$Paths[$i]].sourceSha256
|
||||
}
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $row -Depth 8 -Compress)
|
||||
))
|
||||
}
|
||||
$hash.AppendData([byte[]][char]']')
|
||||
return [Convert]::ToHexString($hash.GetHashAndReset()).ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$hash.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Read-PreparedKnowledgeIndex {
|
||||
param(
|
||||
[string] $IndexPath,
|
||||
[string] $Recovery
|
||||
)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $IndexPath -PathType Leaf)) {
|
||||
throw "Knowledge index missing: $IndexPath. $Recovery"
|
||||
}
|
||||
$indexItem = Get-Item -LiteralPath $IndexPath -Force -ErrorAction Stop
|
||||
if ($indexItem.PSProvider.Name -ne 'FileSystem' -or
|
||||
($indexItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "Knowledge index must be an unlinked filesystem file: $IndexPath. $Recovery"
|
||||
}
|
||||
|
||||
$content = Read-KnowledgeText -Path $indexItem.FullName
|
||||
try {
|
||||
$index = $content.text.TrimStart([char]0xfeff) |
|
||||
ConvertFrom-Json -AsHashtable -ErrorAction Stop
|
||||
}
|
||||
catch {
|
||||
throw "Malformed knowledge index JSON: $($_.Exception.Message). $Recovery"
|
||||
}
|
||||
if ($index -isnot [Collections.IDictionary] -or
|
||||
$index.version -ne 1 -or
|
||||
$index.articles -isnot [array] -or
|
||||
$index.articleCount -ne $index.articles.Count -or
|
||||
$index.enabledLayers -isnot [array] -or
|
||||
$index.knowledgeAllow -isnot [array] -or
|
||||
$index.knowledgeDeny -isnot [array] -or
|
||||
$index.sourceSnapshot -isnot [string] -or
|
||||
$index.sourceSnapshot -cnotmatch '^[a-f0-9]{64}$') {
|
||||
throw "Invalid knowledge index envelope. $Recovery"
|
||||
}
|
||||
|
||||
$generatedAt = [DateTimeOffset]::MinValue
|
||||
if ($index.generatedAt -is [DateTime]) {
|
||||
$generatedAt = [DateTimeOffset]$index.generatedAt
|
||||
}
|
||||
elseif (-not [DateTimeOffset]::TryParse(
|
||||
[string]$index.generatedAt,
|
||||
[Globalization.CultureInfo]::InvariantCulture,
|
||||
[Globalization.DateTimeStyles]::RoundtripKind,
|
||||
[ref]$generatedAt
|
||||
)) {
|
||||
throw "Invalid knowledge index generatedAt. $Recovery"
|
||||
}
|
||||
if ($generatedAt -gt [DateTimeOffset]::UtcNow.AddMinutes(1)) {
|
||||
throw "Invalid knowledge index generatedAt. $Recovery"
|
||||
}
|
||||
|
||||
$byPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($row in $index.articles) {
|
||||
if ($row -isnot [Collections.IDictionary] -or $row.path -isnot [string]) {
|
||||
throw "Index row has no exact path. $Recovery"
|
||||
}
|
||||
Assert-KnowledgePath -Path $row.path
|
||||
if ($row.layer -cne $row.path.Split('/')[0] -or
|
||||
$row.layer -cnotin @('microsoft', 'community', 'custom')) {
|
||||
throw "Invalid index layer: $($row.path). $Recovery"
|
||||
}
|
||||
if ($row.sourceSha256 -isnot [string] -or
|
||||
$row.sourceSha256 -cnotmatch '^[a-f0-9]{64}$') {
|
||||
throw "Invalid source hash in knowledge index: $($row.path). $Recovery"
|
||||
}
|
||||
if (-not $byPath.TryAdd($row.path, $row)) {
|
||||
throw "Duplicate index path: $($row.path). $Recovery"
|
||||
}
|
||||
}
|
||||
|
||||
$paths = [string[]]@($byPath.Keys)
|
||||
[Array]::Sort($paths, [StringComparer]::Ordinal)
|
||||
if ((Get-PreparedManifestSha256 -Paths $paths -ByPath $byPath) -cne $index.sourceSnapshot) {
|
||||
throw "Stale or internally inconsistent prepared index snapshot. $Recovery"
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
index = $index
|
||||
content = $content
|
||||
byPath = $byPath
|
||||
paths = $paths
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-SampleLink {
|
||||
param(
|
||||
[string] $ArticleText,
|
||||
[string] $SamplePath
|
||||
)
|
||||
|
||||
$sampleName = [IO.Path]::GetFileName($SamplePath)
|
||||
$expected = '[`' + $sampleName + '`](' + $sampleName + ')'
|
||||
if (-not $ArticleText.Contains($expected, [StringComparison]::Ordinal)) {
|
||||
throw "Sample is not linked by its article using the READ convention: $sampleName"
|
||||
}
|
||||
}
|
||||
244
tools/Search-Knowledge.ps1
Normal file
244
tools/Search-Knowledge.ps1
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Returns bounded pages of every domain/layer/READ-applicable catalog row.
|
||||
.DESCRIPTION
|
||||
Consumes Entry's prepared index read-only. Results are never ranked, sampled,
|
||||
top-k limited, deduplicated by basename, or narrowed by query text. Omit an
|
||||
unknown task dimension; an explicit empty array is a known empty set.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateNotNullOrEmpty()] [string] $BCQualityRoot = (Split-Path $PSScriptRoot -Parent),
|
||||
[Parameter(Mandatory)] [ValidateNotNullOrEmpty()]
|
||||
[ValidateScript({ -not [string]::IsNullOrWhiteSpace($_) })] [string] $Domain,
|
||||
[ValidateSet('microsoft', 'community', 'custom')]
|
||||
[AllowEmptyCollection()] [string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
|
||||
[ValidateRange(1, 2147483647)] [int] $BCVersion,
|
||||
[AllowEmptyCollection()] [string[]] $Technologies,
|
||||
[AllowEmptyCollection()] [string[]] $Countries,
|
||||
[AllowEmptyCollection()] [string[]] $ApplicationAreas,
|
||||
[switch] $ExcludeConditional,
|
||||
[string] $IndexPath,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000,
|
||||
[ValidateRange(0, 2147483647)] [int] $Offset = 0,
|
||||
[ValidatePattern('^[a-f0-9]{64}$')] [string] $Snapshot
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
. (Join-Path $PSScriptRoot 'Bounded-Results.ps1')
|
||||
|
||||
$BCQualityRoot = Resolve-KnowledgeRoot $BCQualityRoot
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'knowledge-index.json'
|
||||
}
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array; use an empty array to disable all layers.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
|
||||
$context = [ordered]@{}
|
||||
foreach ($pair in @(
|
||||
@('BCVersion', 'bc-version'),
|
||||
@('Technologies', 'technologies'),
|
||||
@('Countries', 'countries'),
|
||||
@('ApplicationAreas', 'application-area')
|
||||
)) {
|
||||
if (-not $PSBoundParameters.ContainsKey($pair[0])) {
|
||||
continue
|
||||
}
|
||||
$value = $PSBoundParameters[$pair[0]]
|
||||
if ($null -eq $value) {
|
||||
throw "Omit unknown context; do not pass null for $($pair[0])."
|
||||
}
|
||||
if ($pair[0] -ne 'BCVersion') {
|
||||
foreach ($entry in $value) {
|
||||
if ([string]::IsNullOrWhiteSpace($entry) -or $entry -cne $entry.Trim()) {
|
||||
throw "Invalid context value for $($pair[0]): '$entry'"
|
||||
}
|
||||
}
|
||||
}
|
||||
$context[$pair[1]] = $value
|
||||
}
|
||||
if ($context.Contains('technologies') -and $context['technologies'] -ccontains 'all') {
|
||||
throw "Technologies has no 'all' sentinel. Omit unknown context."
|
||||
}
|
||||
|
||||
$recovery = "Run Entry preparation once before dispatch, or use READ's path-discovery fallback. Do not rebuild in a leaf."
|
||||
$preparedIndex = Read-PreparedKnowledgeIndex -IndexPath $IndexPath -Recovery $recovery
|
||||
$index = $preparedIndex.index
|
||||
$indexContent = $preparedIndex.content
|
||||
$byPath = $preparedIndex.byPath
|
||||
$paths = $preparedIndex.paths
|
||||
|
||||
# The v1 generator historically serialized an omitted EnabledLayers parameter as [null].
|
||||
$unrestricted = $index.enabledLayers.Count -eq 0 -or
|
||||
($index.enabledLayers.Count -eq 1 -and $null -eq $index.enabledLayers[0])
|
||||
$indexedLayers = @(
|
||||
if ($unrestricted) {
|
||||
'microsoft', 'community', 'custom'
|
||||
}
|
||||
else {
|
||||
$index.enabledLayers
|
||||
}
|
||||
)
|
||||
if (@($indexedLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($indexedLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count -or
|
||||
@($EnabledLayers | Where-Object { $_ -cnotin $indexedLayers }).Count) {
|
||||
throw "Index layer coverage does not cover EnabledLayers. $recovery"
|
||||
}
|
||||
|
||||
foreach ($path in $paths) {
|
||||
$row = $byPath[$path]
|
||||
if ($row.layer -cnotin $indexedLayers) {
|
||||
throw "Index row layer is outside index coverage: $path. $recovery"
|
||||
}
|
||||
$problem = Get-KnowledgeMetadataProblem -Row $row
|
||||
if ($problem) {
|
||||
throw "Malformed knowledge index row at ${path}: $problem. $recovery"
|
||||
}
|
||||
}
|
||||
|
||||
$defaults = [ordered]@{
|
||||
'bc-version' = @('all')
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
}
|
||||
$candidates = [Collections.Generic.List[object]]::new()
|
||||
$excluded = [Collections.Generic.List[object]]::new()
|
||||
foreach ($path in $paths) {
|
||||
$row = $byPath[$path]
|
||||
if ($row.domain -cne $Domain) {
|
||||
continue
|
||||
}
|
||||
|
||||
$unknown = [Collections.Generic.List[string]]::new()
|
||||
$matchesContext = $true
|
||||
foreach ($field in $defaults.Keys) {
|
||||
$values = $row[$field]
|
||||
if ($field -eq 'bc-version') {
|
||||
$values = @(Get-NormalizedKnowledgeVersions -Values $values)
|
||||
}
|
||||
$sentinel = switch ($field) {
|
||||
'bc-version' { 'all' }
|
||||
'countries' { 'w1' }
|
||||
'application-area' { 'all' }
|
||||
default { '' }
|
||||
}
|
||||
if ($sentinel -and $values -ccontains $sentinel) {
|
||||
continue
|
||||
}
|
||||
if (-not $context.Contains($field)) {
|
||||
$unknown.Add($field)
|
||||
continue
|
||||
}
|
||||
|
||||
$target = $context[$field]
|
||||
$matched = $false
|
||||
if ($field -eq 'bc-version') {
|
||||
# Compare as bigint on both sides: metadata validation accepts bounds
|
||||
# wider than Int32, and an int left operand would coerce them down.
|
||||
$targetVersion = [bigint]$target
|
||||
if ($values.Count -eq 1 -and $values[0] -match '^(\d+)\.\.(\d+)?$') {
|
||||
$matched = $targetVersion -ge [bigint]::Parse($Matches[1]) -and
|
||||
(-not $Matches[2] -or $targetVersion -le [bigint]::Parse($Matches[2]))
|
||||
}
|
||||
else {
|
||||
$matched = @($values | Where-Object { [bigint]::Parse($_) -eq $targetVersion }).Count -gt 0
|
||||
}
|
||||
}
|
||||
else {
|
||||
$matched = @($values | Where-Object { $target -ccontains $_ }).Count -gt 0
|
||||
}
|
||||
if (-not $matched) {
|
||||
$matchesContext = $false
|
||||
break
|
||||
}
|
||||
}
|
||||
if (-not $matchesContext -or ($ExcludeConditional -and $unknown.Count)) {
|
||||
continue
|
||||
}
|
||||
$null = Resolve-KnowledgePath -Root $BCQualityRoot -Path $path
|
||||
|
||||
$candidate = [ordered]@{
|
||||
path = $path
|
||||
layer = $row.layer
|
||||
keywords = $row.keywords
|
||||
title = $row.title
|
||||
description = $row.description
|
||||
}
|
||||
foreach ($field in $defaults.Keys) {
|
||||
if (($row[$field] -join "`0") -cne ($defaults[$field] -join "`0")) {
|
||||
$candidate[$field] = $row[$field]
|
||||
}
|
||||
}
|
||||
$candidate.applicability = if ($unknown.Count) { 'conditional' } else { 'applicable' }
|
||||
$candidate.unknownDimensions = @($unknown)
|
||||
if ($row.layer -cin $EnabledLayers) {
|
||||
$candidates.Add($candidate)
|
||||
}
|
||||
else {
|
||||
$excluded.Add($candidate)
|
||||
}
|
||||
}
|
||||
|
||||
$header = [ordered]@{
|
||||
version = 2
|
||||
domain = $Domain
|
||||
context = $context
|
||||
enabledLayers = @($EnabledLayers)
|
||||
indexedLayers = @($indexedLayers)
|
||||
excludeConditional = [bool]$ExcludeConditional
|
||||
defaults = $defaults
|
||||
candidateCount = $candidates.Count
|
||||
excludedByConfigurationCount = $excluded.Count
|
||||
}
|
||||
|
||||
function Get-CatalogSnapshot {
|
||||
param(
|
||||
[string] $PreparedIndexSha256,
|
||||
[Collections.IDictionary] $Request,
|
||||
[Collections.IDictionary] $Groups
|
||||
)
|
||||
|
||||
$hash = [Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
foreach ($value in @(
|
||||
$PreparedIndexSha256,
|
||||
(ConvertTo-Json -InputObject $Request -Depth 8 -Compress)
|
||||
)) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes($value))
|
||||
$hash.AppendData([byte[]](10))
|
||||
}
|
||||
foreach ($groupName in $Groups.Keys) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes("[$groupName]"))
|
||||
$hash.AppendData([byte[]](10))
|
||||
foreach ($row in $Groups[$groupName]) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $row -Depth 8 -Compress)
|
||||
))
|
||||
$hash.AppendData([byte[]](10))
|
||||
}
|
||||
}
|
||||
return [Convert]::ToHexString($hash.GetHashAndReset()).ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$hash.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
$groups = [ordered]@{
|
||||
candidates = $candidates
|
||||
excludedByConfiguration = $excluded
|
||||
}
|
||||
$header.snapshot = Get-CatalogSnapshot -PreparedIndexSha256 $indexContent.sha256 -Request $header -Groups $groups
|
||||
|
||||
ConvertTo-BoundedPage -Header $header -Groups $groups -Offset $Offset -Snapshot $Snapshot -MaxBytes $MaxBytes
|
||||
788
tools/Test-KnowledgeRetrieval.ps1
Normal file
788
tools/Test-KnowledgeRetrieval.ps1
Normal file
|
|
@ -0,0 +1,788 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates lossless bounded catalog and exact-body retrieval.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..'))
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
|
||||
$generator = Join-Path $Root 'tools/Build-KnowledgeIndex.ps1'
|
||||
$search = Join-Path $Root 'tools/Search-Knowledge.ps1'
|
||||
$getArticles = Join-Path $Root 'tools/Get-KnowledgeArticles.ps1'
|
||||
$utf8 = [Text.UTF8Encoding]::new($false, $true)
|
||||
|
||||
function Assert-True {
|
||||
param([bool] $Condition, [string] $Message)
|
||||
if (-not $Condition) {
|
||||
throw "Assertion failed: $Message"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Equal {
|
||||
param($Actual, $Expected, [string] $Message)
|
||||
if ($Actual -cne $Expected) {
|
||||
throw "Assertion failed: $Message. Expected '$Expected', got '$Actual'."
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Sequence {
|
||||
param($Actual, $Expected, [string] $Message)
|
||||
$actualJson = ConvertTo-Json -InputObject @($Actual) -Compress
|
||||
$expectedJson = ConvertTo-Json -InputObject @($Expected) -Compress
|
||||
if ($actualJson -cne $expectedJson) {
|
||||
throw "Assertion failed: $Message. Expected $expectedJson, got $actualJson."
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Throws {
|
||||
param([scriptblock] $Action, [string] $Pattern, [string] $Message)
|
||||
try {
|
||||
& $Action
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Message -notmatch $Pattern) {
|
||||
throw "Assertion failed: $Message. Wrong error: $($_.Exception.Message)"
|
||||
}
|
||||
return
|
||||
}
|
||||
throw "Assertion failed: $Message. No error was thrown."
|
||||
}
|
||||
|
||||
function Get-OutputByteCount {
|
||||
param([string] $Text)
|
||||
return [Text.Encoding]::UTF8.GetByteCount($Text) +
|
||||
[Text.Encoding]::UTF8.GetByteCount([Environment]::NewLine)
|
||||
}
|
||||
|
||||
function Invoke-CatalogPages {
|
||||
param(
|
||||
[hashtable] $Arguments,
|
||||
[int] $MaxBytes = 4096
|
||||
)
|
||||
|
||||
$allCandidates = [Collections.Generic.List[object]]::new()
|
||||
$allExcluded = [Collections.Generic.List[object]]::new()
|
||||
$offset = 0
|
||||
$snapshot = ''
|
||||
$shared = ''
|
||||
$pageCount = 0
|
||||
$lastPage = $null
|
||||
do {
|
||||
$pageArguments = @{} + $Arguments
|
||||
$pageArguments.MaxBytes = $MaxBytes
|
||||
$pageArguments.Offset = $offset
|
||||
if ($snapshot) {
|
||||
$pageArguments.Snapshot = $snapshot
|
||||
}
|
||||
$raw = & $search @pageArguments
|
||||
Assert-True ($raw -is [string]) 'catalog helper emitted exactly one JSON string'
|
||||
Assert-True ((Get-OutputByteCount -Text $raw) -le $MaxBytes) 'catalog page includes its newline in MaxBytes'
|
||||
$page = $raw | ConvertFrom-Json
|
||||
$pageCount++
|
||||
Assert-True ($pageCount -le 1000) 'catalog continuation terminates'
|
||||
Assert-Equal $page.offset $offset 'catalog offset is exact'
|
||||
Assert-Equal $page.returnedCount (@($page.candidates).Count + @($page.excludedByConfiguration).Count) 'page returnedCount matches rows'
|
||||
Assert-Equal $page.remainingCount ($page.totalCount - $offset - $page.returnedCount) 'page remainingCount is exact'
|
||||
|
||||
$currentShared = [ordered]@{
|
||||
version = $page.version
|
||||
domain = $page.domain
|
||||
context = $page.context
|
||||
enabledLayers = $page.enabledLayers
|
||||
indexedLayers = $page.indexedLayers
|
||||
excludeConditional = $page.excludeConditional
|
||||
defaults = $page.defaults
|
||||
candidateCount = $page.candidateCount
|
||||
excludedByConfigurationCount = $page.excludedByConfigurationCount
|
||||
snapshot = $page.snapshot
|
||||
totalCount = $page.totalCount
|
||||
} | ConvertTo-Json -Depth 8 -Compress
|
||||
if (-not $shared) {
|
||||
$shared = $currentShared
|
||||
$snapshot = $page.snapshot
|
||||
}
|
||||
else {
|
||||
Assert-Equal $currentShared $shared 'catalog pages repeat shared context, defaults, totals, and snapshot'
|
||||
}
|
||||
|
||||
foreach ($row in @($page.candidates)) {
|
||||
$allCandidates.Add($row)
|
||||
}
|
||||
foreach ($row in @($page.excludedByConfiguration)) {
|
||||
$allExcluded.Add($row)
|
||||
}
|
||||
if (-not $page.complete) {
|
||||
Assert-True ($null -ne $page.continuation) 'incomplete page has continuation'
|
||||
Assert-Equal $page.continuation.snapshot $snapshot 'continuation is snapshot-bound'
|
||||
Assert-True ($page.continuation.offset -gt $offset) 'continuation makes progress'
|
||||
$offset = $page.continuation.offset
|
||||
}
|
||||
$lastPage = $page
|
||||
} while (-not $page.complete)
|
||||
|
||||
Assert-True ($null -eq $lastPage.continuation) 'final page has no continuation'
|
||||
Assert-Equal $allCandidates.Count $lastPage.candidateCount 'candidate total survives paging'
|
||||
Assert-Equal $allExcluded.Count $lastPage.excludedByConfigurationCount 'excluded total survives paging'
|
||||
return [pscustomobject]@{
|
||||
candidates = @($allCandidates)
|
||||
excluded = @($allExcluded)
|
||||
pages = $pageCount
|
||||
snapshot = $snapshot
|
||||
lastPage = $lastPage
|
||||
}
|
||||
}
|
||||
|
||||
function Test-BodyRoundTrip {
|
||||
param(
|
||||
[string[]] $Paths,
|
||||
[string] $IndexPath,
|
||||
[switch] $Samples
|
||||
)
|
||||
|
||||
$seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
for ($start = 0; $start -lt $Paths.Count; $start += 8) {
|
||||
$end = [Math]::Min($start + 7, $Paths.Count - 1)
|
||||
$remaining = @($Paths[$start..$end])
|
||||
$snapshot = ''
|
||||
do {
|
||||
$arguments = @{
|
||||
BCQualityRoot = $Root
|
||||
IndexPath = $IndexPath
|
||||
Paths = $remaining
|
||||
MaxArticles = 8
|
||||
MaxBytes = 16000
|
||||
}
|
||||
if ($Samples) {
|
||||
$arguments.Samples = $true
|
||||
}
|
||||
if ($snapshot) {
|
||||
$arguments.Snapshot = $snapshot
|
||||
}
|
||||
$raw = & $getArticles @arguments
|
||||
Assert-True ($raw -is [string]) 'article helper emitted exactly one JSON string'
|
||||
Assert-True ((Get-OutputByteCount -Text $raw) -le 16000) 'article batch includes its newline in MaxBytes'
|
||||
$batch = $raw | ConvertFrom-Json
|
||||
Assert-True ($batch.returnedCount -gt 0) 'article batching makes progress'
|
||||
Assert-Equal $batch.returnedCount @($batch.articles).Count 'article returnedCount matches rows'
|
||||
Assert-Equal $batch.complete (@($batch.remainingPaths).Count -eq 0) 'article completion matches remaining paths'
|
||||
if ($batch.complete) {
|
||||
Assert-True ($null -eq $batch.continuation) 'complete article batch has no continuation'
|
||||
}
|
||||
else {
|
||||
Assert-True ($batch.continuation.snapshot -match '^[a-f0-9]{64}$') 'article continuation is snapshot-bound'
|
||||
}
|
||||
|
||||
foreach ($article in @($batch.articles)) {
|
||||
Assert-True ($seen.Add($article.path)) "body returned once: $($article.path)"
|
||||
$fullPath = Join-Path $Root ($article.path.Replace('/', [IO.Path]::DirectorySeparatorChar))
|
||||
$bytes = [IO.File]::ReadAllBytes($fullPath)
|
||||
$text = $utf8.GetString($bytes)
|
||||
$hash = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($bytes)
|
||||
).ToLowerInvariant()
|
||||
Assert-Equal $article.bytes $bytes.Length "byte count round-trips: $($article.path)"
|
||||
Assert-Equal $article.sha256 $hash "SHA-256 round-trips: $($article.path)"
|
||||
Assert-Equal $article.body $text "body round-trips: $($article.path)"
|
||||
}
|
||||
$remaining = @($batch.remainingPaths)
|
||||
$snapshot = if ($batch.complete) { '' } else { $batch.continuation.snapshot }
|
||||
} while ($remaining.Count)
|
||||
}
|
||||
Assert-Equal $seen.Count $Paths.Count 'every requested body round-trips without loss'
|
||||
}
|
||||
|
||||
function New-NeutralArticle {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $Layer,
|
||||
[string] $Slug,
|
||||
[string] $Version = 'all',
|
||||
[string] $Technology = 'al',
|
||||
[string] $Country = 'w1',
|
||||
[string] $Area = 'all',
|
||||
[string] $Title = 'Neutral retrieval example',
|
||||
[string] $Description = 'Neutral retrieval metadata for deterministic tests.'
|
||||
)
|
||||
|
||||
$directory = Join-Path $FixtureRoot "$Layer\knowledge\neutral"
|
||||
New-Item -ItemType Directory -Force -Path $directory | Out-Null
|
||||
$content = @"
|
||||
---
|
||||
bc-version: [$Version]
|
||||
domain: neutral
|
||||
keywords: [neutral, retrieval, deterministic]
|
||||
technologies: [$Technology]
|
||||
countries: [$Country]
|
||||
application-area: [$Area]
|
||||
---
|
||||
|
||||
# $Title
|
||||
|
||||
## Description
|
||||
|
||||
$Description
|
||||
"@
|
||||
Set-Content -LiteralPath (Join-Path $directory "$Slug.md") -Value $content -Encoding utf8NoBOM
|
||||
}
|
||||
|
||||
function Test-InvalidSourceIndexing {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $Field,
|
||||
[string] $ValidValue,
|
||||
[string] $InvalidValue
|
||||
)
|
||||
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer microsoft -Slug valid-source
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer community -Slug invalid-source
|
||||
$articlePath = Join-Path $FixtureRoot 'community\knowledge\neutral\invalid-source.md'
|
||||
$text = [IO.File]::ReadAllText($articlePath, $utf8)
|
||||
$text = $text.Replace("$Field`: $ValidValue", "$Field`: $InvalidValue")
|
||||
[IO.File]::WriteAllText($articlePath, $text, $utf8)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("$Field-index.json")
|
||||
$generation = @(& $generator -BCQualityRoot $FixtureRoot -IndexPath $indexPath 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 "scalar $Field source emits one omission warning"
|
||||
Assert-True (
|
||||
$warnings[0].Message -match
|
||||
"Skipping invalid knowledge article 'community/knowledge/neutral/invalid-source\.md': frontmatter field '$([regex]::Escape($Field))' must use non-empty bracket-array syntax\."
|
||||
) "scalar $Field warning identifies the exact path and reason"
|
||||
$prepared = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
Assert-Equal $prepared.articleCount 1 "scalar $Field source is omitted while its valid sibling is indexed"
|
||||
Assert-Sequence $prepared.articles.path @('microsoft/knowledge/neutral/valid-source.md') "scalar $Field index contains only the valid sibling"
|
||||
$catalog = & $search -BCQualityRoot $FixtureRoot -IndexPath $indexPath -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $catalog.candidates.path @('microsoft/knowledge/neutral/valid-source.md') "scalar $Field catalog retrieves the valid sibling"
|
||||
$valid = & $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-source.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $valid.complete "scalar $Field valid sibling body retrieves completely"
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'community/knowledge/neutral/invalid-source.md'
|
||||
} 'Selected article is absent from the prepared index' "scalar $Field omitted source cannot be retrieved"
|
||||
}
|
||||
|
||||
function Test-InvalidSemanticIndexing {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $CaseName,
|
||||
[string] $Field,
|
||||
[string] $ValidValue,
|
||||
[string] $InvalidValue,
|
||||
[string] $ExpectedReason
|
||||
)
|
||||
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer microsoft -Slug valid-source
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer community -Slug invalid-source
|
||||
$articlePath = Join-Path $FixtureRoot 'community\knowledge\neutral\invalid-source.md'
|
||||
$text = [IO.File]::ReadAllText($articlePath, $utf8)
|
||||
$text = $text.Replace("$Field`: $ValidValue", "$Field`: $InvalidValue")
|
||||
[IO.File]::WriteAllText($articlePath, $text, $utf8)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("$CaseName-index.json")
|
||||
$generation = @(& $generator -BCQualityRoot $FixtureRoot -IndexPath $indexPath 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 "$CaseName emits one omission warning"
|
||||
Assert-Equal $warnings[0].Message "Skipping invalid knowledge article 'community/knowledge/neutral/invalid-source.md': $ExpectedReason." "$CaseName warning identifies exact path and reason"
|
||||
|
||||
$prepared = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
Assert-Equal $prepared.articleCount 1 "$CaseName omits invalid source and retains valid sibling"
|
||||
Assert-Sequence $prepared.articles.path @('microsoft/knowledge/neutral/valid-source.md') "$CaseName index contains only valid sibling"
|
||||
Assert-True ($prepared.sourceSnapshot -match '^[a-f0-9]{64}$') "$CaseName source snapshot remains valid"
|
||||
$validRow = $prepared.articles[0]
|
||||
$manifest = @(
|
||||
[ordered]@{ path = $validRow.path; sha256 = $validRow.sourceSha256 }
|
||||
)
|
||||
$manifestBytes = [Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $manifest -Depth 8 -Compress)
|
||||
)
|
||||
$expectedSnapshot = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($manifestBytes)
|
||||
).ToLowerInvariant()
|
||||
Assert-Equal $prepared.sourceSnapshot $expectedSnapshot "$CaseName source snapshot covers only retained rows"
|
||||
|
||||
$catalog = & $search -BCQualityRoot $FixtureRoot -IndexPath $indexPath -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $catalog.candidates.path @('microsoft/knowledge/neutral/valid-source.md') "$CaseName catalog retains valid sibling"
|
||||
$valid = & $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-source.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $valid.complete "$CaseName valid sibling body retrieves"
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'community/knowledge/neutral/invalid-source.md'
|
||||
} 'Selected article is absent from the prepared index' "$CaseName invalid source cannot be retrieved"
|
||||
}
|
||||
|
||||
function Test-InvalidEnabledLayers {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $CaseName,
|
||||
$Layers,
|
||||
[string] $ExpectedPattern
|
||||
)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("layers-$CaseName.json")
|
||||
$arguments = @{
|
||||
BCQualityRoot = $FixtureRoot
|
||||
IndexPath = $indexPath
|
||||
EnabledLayers = $Layers
|
||||
}
|
||||
Assert-Throws {
|
||||
& $generator @arguments
|
||||
} $ExpectedPattern "$CaseName EnabledLayers fails"
|
||||
Assert-True (-not (Test-Path -LiteralPath $indexPath)) "$CaseName fails before index creation"
|
||||
}
|
||||
|
||||
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("bcquality_retrieval_" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Force -Path $tmp | Out-Null
|
||||
try {
|
||||
$indexPath = Join-Path $tmp 'knowledge-index.json'
|
||||
& $generator -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$index = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$diskArticlePaths = @(
|
||||
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||
$knowledge = Join-Path $Root "$layer\knowledge"
|
||||
if (Test-Path -LiteralPath $knowledge) {
|
||||
Get-ChildItem -LiteralPath $knowledge -Recurse -File -Filter '*.md' |
|
||||
ForEach-Object {
|
||||
[IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
|
||||
}
|
||||
}
|
||||
}
|
||||
) | Sort-Object
|
||||
Assert-Equal $index.articleCount $diskArticlePaths.Count 'index covers every current article'
|
||||
Assert-True ($index.sourceSnapshot -match '^[a-f0-9]{64}$') 'index carries an exact source snapshot'
|
||||
|
||||
$allCatalogRows = [Collections.Generic.List[object]]::new()
|
||||
$domains = @($index.articles.domain | Sort-Object -Unique)
|
||||
foreach ($domain in $domains) {
|
||||
$catalog = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $Root
|
||||
IndexPath = $indexPath
|
||||
Domain = $domain
|
||||
}
|
||||
Assert-Equal $catalog.excluded.Count 0 "all layers enabled for $domain"
|
||||
foreach ($row in $catalog.candidates) {
|
||||
$allCatalogRows.Add($row)
|
||||
}
|
||||
}
|
||||
|
||||
$expectedRows = @($index.articles | Sort-Object path)
|
||||
$actualRows = @($allCatalogRows | Sort-Object path)
|
||||
Assert-Equal $actualRows.Count $expectedRows.Count 'paged union has no top-k or query-based loss'
|
||||
Assert-Sequence ($actualRows.path) ($expectedRows.path) 'paged union equals all READ-filtered candidates'
|
||||
Assert-Equal @($actualRows.path | Sort-Object -Unique).Count $actualRows.Count 'catalog does not deduplicate distinct paths'
|
||||
|
||||
$defaults = [ordered]@{
|
||||
'bc-version' = @('all')
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
}
|
||||
for ($i = 0; $i -lt $actualRows.Count; $i++) {
|
||||
$actual = $actualRows[$i]
|
||||
$expected = $expectedRows[$i]
|
||||
Assert-Equal $actual.path $expected.path 'catalog preserves exact path'
|
||||
Assert-Equal $actual.layer $expected.layer 'catalog preserves layer'
|
||||
Assert-Sequence $actual.keywords $expected.keywords 'catalog preserves full keywords'
|
||||
Assert-Equal $actual.title $expected.title 'catalog preserves title'
|
||||
Assert-Equal $actual.description $expected.description 'catalog preserves one-line description'
|
||||
|
||||
$unknown = [Collections.Generic.List[string]]::new()
|
||||
foreach ($field in $defaults.Keys) {
|
||||
$expectedValues = @($expected.$field)
|
||||
$sentinel = switch ($field) {
|
||||
'bc-version' { 'all' }
|
||||
'countries' { 'w1' }
|
||||
'application-area' { 'all' }
|
||||
default { '' }
|
||||
}
|
||||
if (-not $sentinel -or $expectedValues -notcontains $sentinel) {
|
||||
$unknown.Add($field)
|
||||
}
|
||||
$hasField = $actual.PSObject.Properties.Name -ccontains $field
|
||||
if (($expectedValues -join "`0") -ceq (@($defaults[$field]) -join "`0")) {
|
||||
Assert-True (-not $hasField) "default field is inherited from page: $field"
|
||||
}
|
||||
else {
|
||||
Assert-True $hasField "non-default field survives paging: $field"
|
||||
Assert-Sequence $actual.$field $expectedValues "non-default field is exact: $field"
|
||||
}
|
||||
}
|
||||
Assert-Equal $actual.applicability ($(if ($unknown.Count) { 'conditional' } else { 'applicable' })) 'applicability verdict is explicit'
|
||||
Assert-Sequence $actual.unknownDimensions @($unknown) 'unknown dimensions are explicit'
|
||||
}
|
||||
|
||||
$performanceFirst = & $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -MaxBytes 4096 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $performanceFirst.complete) 'large domain produces deterministic continuation'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -MaxBytes 4096 -Offset $performanceFirst.continuation.offset
|
||||
} 'Continuation requires Snapshot' 'continuation without snapshot fails'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -Offset $performanceFirst.totalCount
|
||||
} 'Invalid Offset' 'offset at total fails'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -Offset 1 -Snapshot ('0' * 64)
|
||||
} 'Snapshot changed' 'wrong snapshot fails'
|
||||
|
||||
$changedRawIndex = Join-Path $tmp 'changed-raw-index.json'
|
||||
$changedRaw = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$changedRaw.generatedAt = [DateTimeOffset]::UtcNow.ToString('O')
|
||||
$changedRaw | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $changedRawIndex -Encoding utf8NoBOM -NoNewline
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $changedRawIndex -Domain performance `
|
||||
-MaxBytes 4096 -Offset $performanceFirst.continuation.offset `
|
||||
-Snapshot $performanceFirst.continuation.snapshot
|
||||
} 'Snapshot changed' 'continuation is bound to the exact prepared index bytes'
|
||||
|
||||
$malformedIndex = Join-Path $tmp 'malformed.json'
|
||||
Set-Content -LiteralPath $malformedIndex -Value '{not-json' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $malformedIndex -Domain performance
|
||||
} 'Malformed knowledge index JSON' 'malformed JSON fails'
|
||||
|
||||
$unsafeIndex = Join-Path $tmp 'unsafe.json'
|
||||
$unsafe = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$unsafe.articles[0].path = '../outside.md'
|
||||
$unsafe | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $unsafeIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $unsafeIndex -Domain performance
|
||||
} 'Invalid knowledge path' 'unsafe indexed path fails'
|
||||
|
||||
$invalidRowIndex = Join-Path $tmp 'invalid-row.json'
|
||||
$invalidRow = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$invalidRow.articles[0].keywords = @()
|
||||
$invalidRow | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $invalidRowIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $invalidRowIndex -Domain performance
|
||||
} 'Malformed knowledge index row' 'malformed index row fails'
|
||||
|
||||
$semanticCorruptIndex = Join-Path $tmp 'semantic-corrupt-row.json'
|
||||
$semanticCorrupt = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$semanticCorrupt.articles[0].countries = @('usa')
|
||||
$semanticCorrupt | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $semanticCorruptIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $semanticCorruptIndex -Domain performance
|
||||
} 'Malformed knowledge index row.*invalid countries' 'search rejects semantically invalid external index rows'
|
||||
|
||||
$corruptSemanticCases = @(
|
||||
@{ name = 'uppercase-all'; field = 'bc-version'; value = @('ALL'); reason = 'invalid bc-version' },
|
||||
@{ name = 'uppercase-w1'; field = 'countries'; value = @('W1'); reason = 'invalid countries' },
|
||||
@{ name = 'zero-open-range'; field = 'bc-version'; value = @('"0.."'); reason = 'invalid bc-version range bound' },
|
||||
@{ name = 'zero-closed-range'; field = 'bc-version'; value = @('"0..0"'); reason = 'invalid bc-version range bound' }
|
||||
)
|
||||
foreach ($case in $corruptSemanticCases) {
|
||||
$corruptPath = Join-Path $tmp ("corrupt-$($case.name).json")
|
||||
$corrupt = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$corrupt.articles[0].PSObject.Properties[$case.field].Value = $case.value
|
||||
$corrupt | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $corruptPath -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $corruptPath -Domain performance
|
||||
} "Malformed knowledge index row.*$([regex]::Escape($case.reason))" "search rejects $($case.name) in an external index"
|
||||
}
|
||||
|
||||
$invalidUtf8Root = Join-Path $tmp 'invalid-utf8-source'
|
||||
New-NeutralArticle -FixtureRoot $invalidUtf8Root -Layer microsoft -Slug valid-catalog
|
||||
New-NeutralArticle -FixtureRoot $invalidUtf8Root -Layer community -Slug invalid-utf8-source
|
||||
$invalidUtf8Article = Join-Path $invalidUtf8Root 'community\knowledge\neutral\invalid-utf8-source.md'
|
||||
$validBytes = [IO.File]::ReadAllBytes($invalidUtf8Article)
|
||||
[IO.File]::WriteAllBytes($invalidUtf8Article, [byte[]]@($validBytes + @(0xc3, 0x28)))
|
||||
$invalidUtf8Index = Join-Path $tmp 'invalid-utf8-index.json'
|
||||
$generation = @(& $generator -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 'malformed UTF-8 source emits one omission warning'
|
||||
Assert-Equal $warnings[0].Message "Skipping invalid knowledge article 'community/knowledge/neutral/invalid-utf8-source.md': invalid UTF-8." 'malformed UTF-8 warning identifies the exact path and reason'
|
||||
$invalidUtf8Prepared = Get-Content -LiteralPath $invalidUtf8Index -Raw -Encoding utf8 |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $invalidUtf8Prepared.articleCount 1 'malformed UTF-8 source is omitted while its valid sibling is indexed'
|
||||
Assert-Sequence $invalidUtf8Prepared.articles.path @('microsoft/knowledge/neutral/valid-catalog.md') 'malformed UTF-8 index contains only the valid sibling'
|
||||
$validCatalog = & $search -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $validCatalog.candidates.path @('microsoft/knowledge/neutral/valid-catalog.md') 'catalog retrieves the valid sibling after malformed UTF-8 omission'
|
||||
$validBody = & $getArticles -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-catalog.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $validBody.complete 'valid sibling body retrieves after malformed UTF-8 omission'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index `
|
||||
-Paths 'community/knowledge/neutral/invalid-utf8-source.md'
|
||||
} 'Selected article is absent from the prepared index' 'omitted malformed UTF-8 source cannot be retrieved'
|
||||
|
||||
$scalarCases = @(
|
||||
@{ field = 'bc-version'; valid = '[all]'; invalid = 'all' },
|
||||
@{ field = 'keywords'; valid = '[neutral, retrieval, deterministic]'; invalid = 'neutral' },
|
||||
@{ field = 'technologies'; valid = '[al]'; invalid = 'al' },
|
||||
@{ field = 'countries'; valid = '[w1]'; invalid = 'w1' },
|
||||
@{ field = 'application-area'; valid = '[all]'; invalid = 'all' }
|
||||
)
|
||||
foreach ($case in $scalarCases) {
|
||||
Test-InvalidSourceIndexing -FixtureRoot (Join-Path $tmp "scalar-$($case.field)") `
|
||||
-Field $case.field -ValidValue $case.valid -InvalidValue $case.invalid
|
||||
}
|
||||
|
||||
$semanticCases = @(
|
||||
@{ name = 'mixed-version-sentinel'; field = 'bc-version'; valid = '[all]'; invalid = '[all, 27]'; reason = 'mixed bc-version sentinel' },
|
||||
@{ name = 'invalid-country'; field = 'countries'; valid = '[w1]'; invalid = '[usa]'; reason = 'invalid countries' },
|
||||
@{ name = 'descending-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["28..27"]'; reason = 'descending bc-version range' },
|
||||
@{ name = 'malformed-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '[twenty-seven]'; reason = 'invalid bc-version' },
|
||||
@{ name = 'malformed-keyword'; field = 'keywords'; valid = '[neutral, retrieval, deterministic]'; invalid = '[neutral, Bad_Token, deterministic]'; reason = 'invalid keywords' },
|
||||
@{ name = 'malformed-technology'; field = 'technologies'; valid = '[al]'; invalid = '[AL]'; reason = 'invalid technologies' },
|
||||
@{ name = 'malformed-application-area'; field = 'application-area'; valid = '[all]'; invalid = '[finance_]'; reason = 'invalid application-area' },
|
||||
@{ name = 'uppercase-version-sentinel'; field = 'bc-version'; valid = '[all]'; invalid = '[ALL]'; reason = 'invalid bc-version' },
|
||||
@{ name = 'uppercase-country-sentinel'; field = 'countries'; valid = '[w1]'; invalid = '[W1]'; reason = 'invalid countries' },
|
||||
@{ name = 'zero-open-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["0.."]'; reason = 'invalid bc-version range bound' },
|
||||
@{ name = 'zero-closed-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["0..0"]'; reason = 'invalid bc-version range bound' }
|
||||
)
|
||||
foreach ($case in $semanticCases) {
|
||||
Test-InvalidSemanticIndexing -FixtureRoot (Join-Path $tmp "semantic-$($case.name)") `
|
||||
-CaseName $case.name -Field $case.field -ValidValue $case.valid `
|
||||
-InvalidValue $case.invalid -ExpectedReason $case.reason
|
||||
}
|
||||
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain ('x' * 2000) -MaxBytes 1024
|
||||
} 'Page envelope exceeds' 'oversized page envelope fails'
|
||||
|
||||
$articlePaths = @($index.articles.path | Sort-Object)
|
||||
Assert-Sequence $articlePaths $diskArticlePaths 'exact article path union matches disk'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $Root -IndexPath $indexPath -Paths @($articlePaths[0..8])
|
||||
} 'exceeds MaxArticles=8' 'exact retrieval rejects path batches larger than eight'
|
||||
$samplePaths = @(
|
||||
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||
$knowledge = Join-Path $Root "$layer\knowledge"
|
||||
if (Test-Path -LiteralPath $knowledge) {
|
||||
Get-ChildItem -LiteralPath $knowledge -Recurse -File |
|
||||
Where-Object Name -Match '\.(good|bad)\.[a-z0-9]+$' |
|
||||
ForEach-Object {
|
||||
[IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
|
||||
}
|
||||
}
|
||||
}
|
||||
) | Sort-Object
|
||||
Test-BodyRoundTrip -Paths $articlePaths -IndexPath $indexPath
|
||||
Test-BodyRoundTrip -Paths $samplePaths -IndexPath $indexPath -Samples
|
||||
|
||||
$fixtureRoot = Join-Path $tmp 'neutral'
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer microsoft -Slug default
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer community -Slug versioned -Version '"27.."' -Technology javascript -Country dk -Area finance -Title 'Versioned neutral example'
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer custom -Slug localized -Version 28 -Technology al -Country de -Area service -Title 'Localized neutral example'
|
||||
$fixtureIndex = Join-Path $tmp 'neutral-index.json'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex | Out-Null
|
||||
|
||||
foreach ($case in @(
|
||||
@{ name = 'uppercase'; layers = @('Microsoft'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'duplicate'; layers = @('microsoft', 'microsoft'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'unknown'; layers = @('partner'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'null'; layers = $null; pattern = 'must be an array' }
|
||||
)) {
|
||||
Test-InvalidEnabledLayers -FixtureRoot $fixtureRoot -CaseName $case.name `
|
||||
-Layers $case.layers -ExpectedPattern $case.pattern
|
||||
}
|
||||
$subsetIndex = Join-Path $tmp 'community-only-index.json'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $subsetIndex `
|
||||
-EnabledLayers @('community') | Out-Null
|
||||
$subset = & $search -BCQualityRoot $fixtureRoot -IndexPath $subsetIndex `
|
||||
-Domain neutral -EnabledLayers @('community') |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $subset.candidateCount 1 'valid EnabledLayers subset builds and is consumable'
|
||||
Assert-Sequence $subset.candidates.path @('community/knowledge/neutral/versioned.md') 'valid subset contains only its exact layer'
|
||||
|
||||
$applicable = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
BCVersion = 28
|
||||
Technologies = @('al', 'javascript')
|
||||
Countries = @('dk', 'de')
|
||||
ApplicationAreas = @('finance', 'service')
|
||||
} -MaxBytes 16000
|
||||
Assert-Equal $applicable.candidates.Count 3 'neutral layer/version rows all survive matching context'
|
||||
Assert-True (@($applicable.candidates | Where-Object applicability -CEQ applicable).Count -eq 3) 'matching rows are applicable'
|
||||
$versioned = $applicable.candidates | Where-Object path -CEQ 'community/knowledge/neutral/versioned.md'
|
||||
Assert-Equal $versioned.layer community 'non-default layer survives'
|
||||
Assert-Sequence $versioned.'bc-version' @('"27.."') 'original version metadata survives'
|
||||
Assert-Equal $versioned.applicability applicable 'lowercase sentinels and positive open range remain applicable'
|
||||
Assert-Sequence $versioned.technologies @('javascript') 'non-default technology survives'
|
||||
Assert-Sequence $versioned.countries @('dk') 'non-default country survives'
|
||||
Assert-Sequence $versioned.'application-area' @('finance') 'non-default application area survives'
|
||||
|
||||
# Metadata validation accepts range bounds wider than Int32, so version
|
||||
# matching must compare as bigint rather than coercing the bound down.
|
||||
$wideRoot = Join-Path $tmp 'wide-version'
|
||||
New-NeutralArticle -FixtureRoot $wideRoot -Layer microsoft -Slug wide-closed -Version '"1..99999999999"'
|
||||
New-NeutralArticle -FixtureRoot $wideRoot -Layer microsoft -Slug wide-open -Version '"99999999999.."'
|
||||
$wideIndex = Join-Path $tmp 'wide-version-index.json'
|
||||
& $generator -BCQualityRoot $wideRoot -IndexPath $wideIndex | Out-Null
|
||||
$wide = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $wideRoot
|
||||
IndexPath = $wideIndex
|
||||
Domain = 'neutral'
|
||||
BCVersion = 28
|
||||
} -MaxBytes 16000
|
||||
Assert-Sequence $wide.candidates.path @('microsoft/knowledge/neutral/wide-closed.md') 'bc-version bounds beyond Int32 compare without overflow'
|
||||
|
||||
$conditional = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
BCVersion = 28
|
||||
Technologies = @('al', 'javascript')
|
||||
} -MaxBytes 16000
|
||||
$conditionalVersioned = $conditional.candidates |
|
||||
Where-Object path -CEQ 'community/knowledge/neutral/versioned.md'
|
||||
Assert-Equal $conditionalVersioned.applicability conditional 'unknown context produces conditional verdict'
|
||||
Assert-Sequence $conditionalVersioned.unknownDimensions @('countries', 'application-area') 'unknown dimensions survive'
|
||||
|
||||
$layerFiltered = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
EnabledLayers = @('microsoft')
|
||||
} -MaxBytes 16000
|
||||
Assert-Equal $layerFiltered.candidates.Count 1 'enabled layer remains a candidate'
|
||||
Assert-Equal $layerFiltered.excluded.Count 2 'disabled layers remain explicit'
|
||||
Assert-Sequence ($layerFiltered.excluded.layer | Sort-Object) @('community', 'custom') 'excluded rows preserve layer'
|
||||
|
||||
$oldSnapshot = $conditional.snapshot
|
||||
Add-Content -LiteralPath (Join-Path $fixtureRoot 'community\knowledge\neutral\versioned.md') -Value ' ' -Encoding utf8NoBOM
|
||||
$preparedCatalog = & $search -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $preparedCatalog.candidateCount 3 'catalog uses the prepared index without rehashing article bodies'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex `
|
||||
-Paths 'community/knowledge/neutral/versioned.md'
|
||||
} 'Selected article hash does not match the prepared index' 'exact retrieval detects selected article changes'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex -Domain neutral -Offset 1 -Snapshot $oldSnapshot
|
||||
} 'Snapshot changed' 'continuation cannot cross rebuilt snapshots'
|
||||
|
||||
$largeRoot = Join-Path $tmp 'large-catalog'
|
||||
New-NeutralArticle -FixtureRoot $largeRoot -Layer microsoft -Slug huge-title -Title ('T' * 3000)
|
||||
$largeIndex = Join-Path $tmp 'large-index.json'
|
||||
& $generator -BCQualityRoot $largeRoot -IndexPath $largeIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $largeRoot -IndexPath $largeIndex -Domain neutral -MaxBytes 1024
|
||||
} 'One complete candidates row|Page envelope exceeds' 'oversized catalog row fails without clipping'
|
||||
|
||||
# The shared pager reports the oversized row's identity for any row shape;
|
||||
# a row without a path must still reach its explicit offset-based failure.
|
||||
. (Join-Path $Root 'tools/Bounded-Results.ps1')
|
||||
$pagerHeader = [ordered]@{ version = 2; snapshot = ('0' * 64) }
|
||||
foreach ($shape in @(
|
||||
@{ name = 'dictionary'; row = [ordered]@{ blob = ('x' * 3000) } },
|
||||
@{ name = 'object'; row = [pscustomobject]@{ blob = ('x' * 3000) } }
|
||||
)) {
|
||||
Assert-Throws {
|
||||
ConvertTo-BoundedPage -Header $pagerHeader `
|
||||
-Groups ([ordered]@{ rows = @($shape.row) }) -MaxBytes 1024
|
||||
} 'One complete rows row plus envelope exceeds MaxBytes=1024 at Offset=0' "oversized pathless $($shape.name) row fails with its offset identity"
|
||||
}
|
||||
|
||||
$bodyRoot = Join-Path $tmp 'body-failures'
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug huge-body -Description ('x' * 3000)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug broken-link
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug continuation-one -Description ('a' * 300)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug continuation-two -Description ('b' * 300)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug invalid-utf8
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug sample-one
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug sample-two
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-one.md') `
|
||||
-Value '[`sample-one.good.al`](sample-one.good.al)' -Encoding utf8NoBOM
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.md') `
|
||||
-Value '[`sample-two.good.al`](sample-two.good.al)' -Encoding utf8NoBOM
|
||||
Set-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-one.good.al') `
|
||||
-Value ('a' * 900) -Encoding utf8NoBOM
|
||||
Set-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.good.al') `
|
||||
-Value ('b' * 900) -Encoding utf8NoBOM
|
||||
$bodyIndex = Join-Path $tmp 'body-index.json'
|
||||
& $generator -BCQualityRoot $bodyRoot -IndexPath $bodyIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/huge-body.md' -MaxBytes 1024
|
||||
} 'No complete body plus continuation fits' 'oversized body fails without truncation'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex -Paths '../outside.md'
|
||||
} 'Invalid knowledge path' 'unsafe requested path fails'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/huge-body.md' -EnabledLayers community
|
||||
} 'Layer disabled' 'disabled article layer fails'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex -Paths @(
|
||||
'microsoft/knowledge/neutral/huge-body.md',
|
||||
'microsoft/knowledge/neutral/huge-body.md'
|
||||
)
|
||||
} 'Duplicate requested path' 'duplicate exact paths fail'
|
||||
|
||||
$brokenSample = Join-Path $bodyRoot 'microsoft\knowledge\neutral\broken-link.good.al'
|
||||
Set-Content -LiteralPath $brokenSample -Value 'codeunit 1 Neutral { }' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/broken-link.good.al' -Samples
|
||||
} 'Sample is not linked' 'unlinked sample fails'
|
||||
|
||||
$sampleContinuationPaths = @(
|
||||
'microsoft/knowledge/neutral/sample-one.good.al',
|
||||
'microsoft/knowledge/neutral/sample-two.good.al'
|
||||
)
|
||||
$firstSamplePage = & $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths $sampleContinuationPaths -Samples -MaxBytes 1600 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $firstSamplePage.complete) 'bounded sample batch produces continuation'
|
||||
Assert-Sequence $firstSamplePage.remainingPaths @('microsoft/knowledge/neutral/sample-two.good.al') 'sample continuation preserves pending path'
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.good.al') `
|
||||
-Value 'changed' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstSamplePage.remainingPaths) -Samples `
|
||||
-Snapshot $firstSamplePage.continuation.snapshot
|
||||
} 'Article snapshot changed' 'sample continuation rejects a changed pending sample'
|
||||
|
||||
$continuationPaths = @(
|
||||
'microsoft/knowledge/neutral/continuation-one.md',
|
||||
'microsoft/knowledge/neutral/continuation-two.md'
|
||||
)
|
||||
$firstBodyPage = & $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths $continuationPaths -MaxBytes 1300 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $firstBodyPage.complete) 'bounded article batch produces continuation'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstBodyPage.remainingPaths) -Snapshot ('0' * 64)
|
||||
} 'Article snapshot changed' 'wrong article continuation snapshot fails'
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\continuation-two.md') -Value 'changed' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstBodyPage.remainingPaths) -Snapshot $firstBodyPage.continuation.snapshot
|
||||
} 'Selected article hash does not match the prepared index' 'article continuation rejects a changed remaining body'
|
||||
|
||||
$invalidUtf8 = Join-Path $bodyRoot 'microsoft\knowledge\neutral\invalid-utf8.md'
|
||||
$indexedBytes = [IO.File]::ReadAllBytes($invalidUtf8)
|
||||
[IO.File]::WriteAllBytes($invalidUtf8, [byte[]]@($indexedBytes + @(0xc3, 0x28)))
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/invalid-utf8.md'
|
||||
} 'Knowledge file is not valid strict UTF-8' 'invalid UTF-8 fails'
|
||||
|
||||
Write-Host "Knowledge retrieval check PASSED: $($articlePaths.Count) articles and $($samplePaths.Count) samples round-tripped; catalog union was lossless and bounded." -ForegroundColor Green
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue