mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-06 23:26:55 +01:00
Add bounded knowledge retrieval
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d8b7764-f15a-49ea-8d50-d9147334f8af
This commit is contained in:
Jesper Schulz-Wedde
parent
c12b2f0a88
commit
6a89831c5c
26 changed files with 1852 additions and 55 deletions
110
tools/Bounded-Results.ps1
Normal file
110
tools/Bounded-Results.ps1
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
# Shared deterministic paging. Callers build the complete immutable result first.
|
||||
#requires -Version 7.2
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Get-ResultSnapshot {
|
||||
param([Parameter(Mandatory)] $Value)
|
||||
|
||||
$json = ConvertTo-Json -InputObject $Value -Depth 30 -Compress
|
||||
return [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($json))
|
||||
).ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Get-SerializedByteCount {
|
||||
param([Parameter(Mandatory)] [string] $Json)
|
||||
|
||||
# PowerShell writes one platform newline after the returned JSON string.
|
||||
return [Text.Encoding]::UTF8.GetByteCount($Json) +
|
||||
[Text.Encoding]::UTF8.GetByteCount([Environment]::NewLine)
|
||||
}
|
||||
|
||||
function ConvertTo-BoundedPage {
|
||||
param(
|
||||
[Parameter(Mandatory)] [Collections.IDictionary] $Header,
|
||||
[Parameter(Mandatory)] [Collections.IDictionary] $Groups,
|
||||
[ValidateRange(0, 2147483647)] [int] $Offset = 0,
|
||||
[string] $Snapshot,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000
|
||||
)
|
||||
|
||||
$total = 0
|
||||
foreach ($name in $Groups.Keys) {
|
||||
$total += $Groups[$name].Count
|
||||
}
|
||||
if (($total -eq 0 -and $Offset -ne 0) -or ($total -gt 0 -and $Offset -ge $total)) {
|
||||
throw "Invalid Offset=$Offset for totalCount=$total; no rows were returned."
|
||||
}
|
||||
if ($Offset -gt 0 -and -not $Snapshot) {
|
||||
throw 'Continuation requires Snapshot from the preceding page.'
|
||||
}
|
||||
if ($Snapshot -and $Snapshot -cne $Header.snapshot) {
|
||||
throw 'Snapshot changed or continuation belongs to another request. Discard partial results and restart at Offset=0.'
|
||||
}
|
||||
|
||||
$page = [ordered]@{}
|
||||
foreach ($key in $Header.Keys) {
|
||||
$page[$key] = $Header[$key]
|
||||
}
|
||||
$page.offset = $Offset
|
||||
$page.returnedCount = 0
|
||||
$page.totalCount = $total
|
||||
$page.remainingCount = $total - $Offset
|
||||
$page.complete = ($total -eq 0)
|
||||
$page.continuation = if ($total) {
|
||||
[ordered]@{ offset = $Offset; snapshot = $Header.snapshot }
|
||||
}
|
||||
else {
|
||||
$null
|
||||
}
|
||||
foreach ($name in $Groups.Keys) {
|
||||
$page[$name] = [Collections.Generic.List[object]]::new()
|
||||
}
|
||||
|
||||
$json = ConvertTo-Json -InputObject $page -Depth 30 -Compress
|
||||
if ((Get-SerializedByteCount -Json $json) -gt $MaxBytes) {
|
||||
throw "Page envelope exceeds MaxBytes=$MaxBytes. Use READ's path-discovery fallback; never truncate."
|
||||
}
|
||||
|
||||
$position = 0
|
||||
foreach ($name in $Groups.Keys) {
|
||||
foreach ($row in $Groups[$name]) {
|
||||
if ($position++ -lt $Offset) {
|
||||
continue
|
||||
}
|
||||
|
||||
$page[$name].Add($row)
|
||||
$page.returnedCount++
|
||||
$page.remainingCount--
|
||||
$page.complete = ($page.remainingCount -eq 0)
|
||||
$page.continuation = if ($page.complete) {
|
||||
$null
|
||||
}
|
||||
else {
|
||||
[ordered]@{
|
||||
offset = $Offset + $page.returnedCount
|
||||
snapshot = $Header.snapshot
|
||||
}
|
||||
}
|
||||
|
||||
$next = ConvertTo-Json -InputObject $page -Depth 30 -Compress
|
||||
if ((Get-SerializedByteCount -Json $next) -gt $MaxBytes) {
|
||||
$page[$name].RemoveAt($page[$name].Count - 1)
|
||||
$page.returnedCount--
|
||||
$page.remainingCount++
|
||||
$page.complete = $false
|
||||
$page.continuation = [ordered]@{
|
||||
offset = $Offset + $page.returnedCount
|
||||
snapshot = $Header.snapshot
|
||||
}
|
||||
if ($page.returnedCount -eq 0) {
|
||||
$identity = if ($row.path) { " at $($row.path)" } else { " at Offset=$Offset" }
|
||||
throw "One complete $name row plus envelope exceeds MaxBytes=$MaxBytes$identity. No row was clipped."
|
||||
}
|
||||
return $json
|
||||
}
|
||||
$json = $next
|
||||
}
|
||||
}
|
||||
return $json
|
||||
}
|
||||
|
|
@ -30,6 +30,8 @@
|
|||
expected to prune its clone to policy first). For provenance and to
|
||||
reproduce a consumer's exact view, pass -EnabledLayers to restrict the walk
|
||||
to those layers and to record the policy in the index header.
|
||||
Invalid articles are omitted with a path-specific warning so one bad
|
||||
optional layer article cannot block valid siblings.
|
||||
|
||||
.PARAMETER BCQualityRoot
|
||||
Path to the BCQuality content root to index (typically a filtered clone).
|
||||
|
|
@ -69,6 +71,17 @@ param(
|
|||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
|
||||
if ($PSBoundParameters.ContainsKey('EnabledLayers')) {
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array; omit it to index all layers.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
}
|
||||
|
||||
# Default to the clone root (parent of this script's tools/ folder) so the
|
||||
# agent's Entry preparation step can invoke this with no arguments from the
|
||||
|
|
@ -93,6 +106,45 @@ function Get-RelativePath {
|
|||
return ($rel -replace '\\', '/')
|
||||
}
|
||||
|
||||
function Get-BytesSha256 {
|
||||
param([byte[]] $Bytes)
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($Bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Read-ArticleSource {
|
||||
param([string] $Path)
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
try {
|
||||
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||
}
|
||||
catch [Text.DecoderFallbackException] {
|
||||
throw [IO.InvalidDataException]::new('invalid UTF-8', $_.Exception)
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
bytes = $bytes
|
||||
text = $text
|
||||
sha256 = Get-BytesSha256 -Bytes $bytes
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ValueSha256 {
|
||||
param([Parameter(Mandatory)] $Value)
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject $Value -Depth 8 -Compress))
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
# Trims a Description to a single short line (<= $Max chars) for the lean
|
||||
# index. Takes the first sentence; truncates on a word boundary if still long.
|
||||
function Get-LeanDescription {
|
||||
|
|
@ -116,9 +168,12 @@ function ConvertFrom-ArticleFrontmatter {
|
|||
# Pattern) is included; the index is a lossless substitute for the
|
||||
# frontmatter + Description the worklist predicate reads, not a
|
||||
# substitute for the article's normative guidance.
|
||||
param([string] $Path)
|
||||
param(
|
||||
[string] $Path,
|
||||
[string] $Text
|
||||
)
|
||||
|
||||
$lines = Get-Content -LiteralPath $Path -ErrorAction Stop
|
||||
$lines = [regex]::Split($Text.TrimStart([char]0xfeff), '\r\n|\n|\r')
|
||||
|
||||
# Frontmatter is the first '---'-delimited block.
|
||||
if ($lines.Count -lt 1 -or $lines[0].Trim() -ne '---') { return $null }
|
||||
|
|
@ -129,19 +184,42 @@ function ConvertFrom-ArticleFrontmatter {
|
|||
if ($fmEnd -lt 0) { return $null }
|
||||
|
||||
$fm = @{}
|
||||
$arrayFields = @('bc-version', 'keywords', 'technologies', 'countries', 'application-area')
|
||||
for ($i = 1; $i -lt $fmEnd; $i++) {
|
||||
$line = $lines[$i]
|
||||
if ($line -match '^\s*([a-zA-Z][\w-]*)\s*:\s*(.*)$') {
|
||||
$key = $Matches[1]
|
||||
$val = $Matches[2].Trim()
|
||||
if ($val -match '^\[(.*)\]$') {
|
||||
if ($key -in $arrayFields) {
|
||||
if ($val -notmatch '^\[(.*)\]$') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$inner = $Matches[1].Trim()
|
||||
if ($inner -eq '') { $fm[$key] = @() }
|
||||
else { $fm[$key] = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() }) }
|
||||
if ($inner -eq '') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$values = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() })
|
||||
if (@($values | Where-Object { [string]::IsNullOrWhiteSpace($_) }).Count) {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$fm[$key] = $values
|
||||
}
|
||||
elseif ($val -ne '') { $fm[$key] = $val }
|
||||
}
|
||||
}
|
||||
foreach ($field in $arrayFields) {
|
||||
if (-not $fm.ContainsKey($field) -or $fm[$field] -isnot [array] -or -not $fm[$field].Count) {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$field' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
# Body parsing: H1 title and the full Description section. The Description
|
||||
# is the article's primary retrieval target per READ and is captured
|
||||
|
|
@ -185,42 +263,71 @@ $indexArticles = [System.Collections.Generic.List[object]]::new()
|
|||
foreach ($layerDir in @('microsoft', 'community', 'custom')) {
|
||||
$kbRoot = Join-Path $BCQualityRoot (Join-Path $layerDir 'knowledge')
|
||||
if (-not (Test-Path $kbRoot)) { continue }
|
||||
if ($EnabledLayers -and ($EnabledLayers -notcontains $layerDir)) { continue }
|
||||
if ($EnabledLayers -and ($EnabledLayers -cnotcontains $layerDir)) { continue }
|
||||
|
||||
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName |
|
||||
ForEach-Object {
|
||||
$rel = Get-RelativePath -Root $BCQualityRoot -Full $_.FullName
|
||||
$parsed = $null
|
||||
try { $parsed = ConvertFrom-ArticleFrontmatter -Path $_.FullName } catch { $parsed = $null }
|
||||
$files = @(
|
||||
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName
|
||||
)
|
||||
foreach ($file in $files) {
|
||||
$rel = Get-RelativePath -Root $BCQualityRoot -Full $file.FullName
|
||||
try {
|
||||
$source = Read-ArticleSource -Path $file.FullName
|
||||
$parsed = ConvertFrom-ArticleFrontmatter -Path $file.FullName -Text $source.text
|
||||
if (-not $parsed) {
|
||||
# Invalid/unparseable file: list path + domain-from-path so it
|
||||
# is never silently dropped from discovery. Consumers fall back
|
||||
# to reading it in full.
|
||||
$domainFromPath = if ($rel -match '/knowledge/([^/]+)/') { $Matches[1] } else { '' }
|
||||
$indexArticles.Add([pscustomobject]@{
|
||||
path = $rel; layer = $layerDir; domain = $domainFromPath
|
||||
'bc-version' = @(); technologies = @(); countries = @(); 'application-area' = @()
|
||||
keywords = @(); title = ''; description = ''; parsed = $false
|
||||
}) | Out-Null
|
||||
return
|
||||
throw [IO.InvalidDataException]::new('missing or unterminated frontmatter')
|
||||
}
|
||||
foreach ($required in @(
|
||||
@('domain', $parsed.domain),
|
||||
@('H1 title', $parsed.title),
|
||||
@('Description', $parsed.description)
|
||||
)) {
|
||||
if ([string]::IsNullOrWhiteSpace([string]$required[1])) {
|
||||
throw [IO.InvalidDataException]::new("missing $($required[0])")
|
||||
}
|
||||
}
|
||||
$indexArticles.Add([pscustomobject]@{
|
||||
path = $rel
|
||||
layer = $layerDir
|
||||
domain = $parsed.domain
|
||||
'bc-version' = @($parsed.'bc-version')
|
||||
technologies = @($parsed.technologies)
|
||||
countries = @($parsed.countries)
|
||||
'application-area' = @($parsed.'application-area')
|
||||
keywords = @($parsed.keywords)
|
||||
title = $parsed.title
|
||||
description = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
|
||||
parsed = $true
|
||||
}) | Out-Null
|
||||
}
|
||||
catch [IO.InvalidDataException] {
|
||||
Write-Warning "Skipping invalid knowledge article '$rel': $($_.Exception.Message)."
|
||||
continue
|
||||
}
|
||||
|
||||
$article = [ordered]@{
|
||||
path = $rel
|
||||
layer = $layerDir
|
||||
domain = $parsed.domain
|
||||
'bc-version' = @($parsed.'bc-version')
|
||||
technologies = @($parsed.technologies)
|
||||
countries = @($parsed.countries)
|
||||
'application-area' = @($parsed.'application-area')
|
||||
keywords = @($parsed.keywords)
|
||||
title = $parsed.title
|
||||
description = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
|
||||
parsed = $true
|
||||
sourceSha256 = $source.sha256
|
||||
}
|
||||
$problem = Get-KnowledgeMetadataProblem -Row $article
|
||||
if ($problem) {
|
||||
Write-Warning "Skipping invalid knowledge article '$rel': $problem."
|
||||
continue
|
||||
}
|
||||
$indexArticles.Add($article) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$articlesByPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($article in $indexArticles) {
|
||||
if (-not $articlesByPath.TryAdd($article.path, $article)) {
|
||||
throw "Duplicate knowledge path while building source snapshot: $($article.path)"
|
||||
}
|
||||
}
|
||||
$sourcePaths = [string[]]@($articlesByPath.Keys)
|
||||
[Array]::Sort($sourcePaths, [StringComparer]::Ordinal)
|
||||
$sourceManifest = @(
|
||||
foreach ($path in $sourcePaths) {
|
||||
[ordered]@{ path = $path; sha256 = $articlesByPath[$path].sourceSha256 }
|
||||
}
|
||||
)
|
||||
$index = [pscustomobject]@{
|
||||
version = 1
|
||||
generatedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
|
@ -228,6 +335,7 @@ $index = [pscustomobject]@{
|
|||
knowledgeAllow= @($KnowledgeAllow)
|
||||
knowledgeDeny = @($KnowledgeDeny)
|
||||
articleCount = $indexArticles.Count
|
||||
sourceSnapshot= Get-ValueSha256 -Value $sourceManifest
|
||||
articles = @($indexArticles)
|
||||
}
|
||||
|
||||
|
|
|
|||
187
tools/Get-KnowledgeArticles.ps1
Normal file
187
tools/Get-KnowledgeArticles.ps1
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Reads a bounded prefix of exact article or sample paths without altering bodies.
|
||||
.DESCRIPTION
|
||||
The UTF-8 byte size bound covers the complete serialized JSON plus its output
|
||||
newline. A body that cannot fit fails explicitly; it is never summarized or
|
||||
truncated. Samples are loaded only with -Samples and must be linked by their
|
||||
sibling article using READ's exact link convention.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateNotNullOrEmpty()] [string] $BCQualityRoot = (Split-Path $PSScriptRoot -Parent),
|
||||
[Parameter(Mandatory)] [ValidateNotNullOrEmpty()] [string[]] $Paths,
|
||||
[ValidateRange(1, 8)] [int] $MaxArticles = 8,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000,
|
||||
[ValidateSet('microsoft', 'community', 'custom')]
|
||||
[AllowEmptyCollection()] [string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
|
||||
[string] $IndexPath,
|
||||
[ValidatePattern('^[a-f0-9]{64}$')] [string] $Snapshot,
|
||||
[switch] $Samples
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
. (Join-Path $PSScriptRoot 'Bounded-Results.ps1')
|
||||
|
||||
$BCQualityRoot = Resolve-KnowledgeRoot $BCQualityRoot
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'knowledge-index.json'
|
||||
}
|
||||
if ($Paths.Count -gt $MaxArticles) {
|
||||
throw "Paths count $($Paths.Count) exceeds MaxArticles=$MaxArticles. Split the worklist into stable chunks of at most $MaxArticles exact paths."
|
||||
}
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
|
||||
$recovery = "Run Entry preparation once before dispatch, or use READ's bounded native-file fallback. Do not rebuild in a leaf."
|
||||
$preparedIndex = Read-PreparedKnowledgeIndex -IndexPath $IndexPath -Recovery $recovery
|
||||
$index = $preparedIndex.index
|
||||
$byPath = $preparedIndex.byPath
|
||||
$unrestricted = $index.enabledLayers.Count -eq 0 -or
|
||||
($index.enabledLayers.Count -eq 1 -and $null -eq $index.enabledLayers[0])
|
||||
$indexedLayers = @(
|
||||
if ($unrestricted) { 'microsoft', 'community', 'custom' } else { $index.enabledLayers }
|
||||
)
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin $indexedLayers }).Count) {
|
||||
throw "Index layer coverage does not cover EnabledLayers. $recovery"
|
||||
}
|
||||
|
||||
$resolved = [Collections.Generic.List[string]]::new()
|
||||
$records = [Collections.Generic.List[object]]::new()
|
||||
$seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
$articleTexts = [Collections.Generic.Dictionary[string, string]]::new([StringComparer]::Ordinal)
|
||||
$sampleContents = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($path in $Paths) {
|
||||
$kind = if ($Samples) { 'sample' } else { 'article' }
|
||||
$fullPath = Resolve-KnowledgePath -Root $BCQualityRoot -Path $path -Kind $kind
|
||||
if ($path.Split('/')[0] -cnotin $EnabledLayers) {
|
||||
throw "Layer disabled for path: $path"
|
||||
}
|
||||
if (-not $seen.Add($path)) {
|
||||
throw "Duplicate requested path: $path"
|
||||
}
|
||||
if ($Samples) {
|
||||
$articlePath = $path -replace '\.(good|bad)\.[a-z0-9]+$', '.md'
|
||||
if (-not $byPath.ContainsKey($articlePath)) {
|
||||
throw "Sample article is absent from the prepared index: $articlePath"
|
||||
}
|
||||
$fullArticlePath = Resolve-KnowledgePath -Root $BCQualityRoot -Path $articlePath
|
||||
if (-not $articleTexts.ContainsKey($articlePath)) {
|
||||
$articleContent = Read-KnowledgeText -Path $fullArticlePath
|
||||
if ($articleContent.sha256 -cne $byPath[$articlePath].sourceSha256) {
|
||||
throw "Selected article hash does not match the prepared index: $articlePath"
|
||||
}
|
||||
$articleTexts.Add($articlePath, $articleContent.text)
|
||||
}
|
||||
Assert-SampleLink -ArticleText $articleTexts[$articlePath] -SamplePath $fullPath
|
||||
$sampleContent = Read-KnowledgeText -Path $fullPath
|
||||
$sampleContents.Add($path, $sampleContent)
|
||||
$records.Add([ordered]@{
|
||||
path = $path
|
||||
articlePath = $articlePath
|
||||
articleSha256 = $byPath[$articlePath].sourceSha256
|
||||
sampleSha256 = $sampleContent.sha256
|
||||
sampleBytes = $sampleContent.bytes
|
||||
})
|
||||
}
|
||||
else {
|
||||
if (-not $byPath.ContainsKey($path)) {
|
||||
throw "Selected article is absent from the prepared index: $path"
|
||||
}
|
||||
$records.Add([ordered]@{
|
||||
path = $path
|
||||
expectedSha256 = $byPath[$path].sourceSha256
|
||||
})
|
||||
}
|
||||
$resolved.Add($fullPath)
|
||||
}
|
||||
|
||||
$requestSnapshot = Get-ResultSnapshot -Value ([ordered]@{
|
||||
root = $BCQualityRoot
|
||||
preparedIndexSha256 = $preparedIndex.content.sha256
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
enabledLayers = @($EnabledLayers)
|
||||
files = @($records)
|
||||
})
|
||||
if ($Snapshot -and $Snapshot -cne $requestSnapshot) {
|
||||
throw 'Article snapshot changed or continuation belongs to another exact path batch. Discard partial results and restart.'
|
||||
}
|
||||
|
||||
$articles = [Collections.Generic.List[object]]::new()
|
||||
function ConvertTo-BatchJson {
|
||||
param([int] $ReadCount)
|
||||
|
||||
$remaining = @(
|
||||
if ($ReadCount -lt $Paths.Count) {
|
||||
$Paths[$ReadCount..($Paths.Count - 1)]
|
||||
}
|
||||
)
|
||||
$remainingRecords = @(
|
||||
if ($ReadCount -lt $records.Count) {
|
||||
$records[$ReadCount..($records.Count - 1)]
|
||||
}
|
||||
)
|
||||
$continuation = if ($remaining.Count) {
|
||||
[ordered]@{
|
||||
snapshot = Get-ResultSnapshot -Value ([ordered]@{
|
||||
root = $BCQualityRoot
|
||||
preparedIndexSha256 = $preparedIndex.content.sha256
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
enabledLayers = @($EnabledLayers)
|
||||
files = $remainingRecords
|
||||
})
|
||||
}
|
||||
}
|
||||
else {
|
||||
$null
|
||||
}
|
||||
return [ordered]@{
|
||||
version = 1
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
snapshot = $requestSnapshot
|
||||
requestedCount = $Paths.Count
|
||||
returnedCount = $ReadCount
|
||||
complete = ($ReadCount -eq $Paths.Count)
|
||||
articles = @($articles)
|
||||
remainingPaths = $remaining
|
||||
continuation = $continuation
|
||||
} | ConvertTo-Json -Depth 8 -Compress
|
||||
}
|
||||
|
||||
$json = ''
|
||||
for ($i = 0; $i -lt [Math]::Min($MaxArticles, $Paths.Count); $i++) {
|
||||
$content = if ($Samples) {
|
||||
$sampleContents[$Paths[$i]]
|
||||
}
|
||||
else {
|
||||
Read-KnowledgeText -Path $resolved[$i]
|
||||
}
|
||||
if (-not $Samples -and $content.sha256 -cne $records[$i].expectedSha256) {
|
||||
throw "Selected article hash does not match the prepared index: $($Paths[$i])"
|
||||
}
|
||||
$articles.Add([ordered]@{
|
||||
path = $Paths[$i]
|
||||
bytes = $content.bytes
|
||||
sha256 = $content.sha256
|
||||
body = $content.text
|
||||
})
|
||||
$next = ConvertTo-BatchJson -ReadCount ($i + 1)
|
||||
if ((Get-SerializedByteCount -Json $next) -gt $MaxBytes) {
|
||||
$articles.RemoveAt($articles.Count - 1)
|
||||
if ($i -eq 0) {
|
||||
throw "No complete body plus continuation fits MaxBytes=$MaxBytes at $($Paths[$i]). Use a smaller exact path batch or READ's bounded native-file fallback; never truncate."
|
||||
}
|
||||
break
|
||||
}
|
||||
$json = $next
|
||||
}
|
||||
|
||||
$json
|
||||
298
tools/Knowledge-Retrieval.ps1
Normal file
298
tools/Knowledge-Retrieval.ps1
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
# Shared filesystem guards for catalog and exact article retrieval.
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Resolve-KnowledgeRoot {
|
||||
param([string] $Root)
|
||||
|
||||
$item = Get-Item -LiteralPath $Root -Force -ErrorAction Stop
|
||||
if ($item.PSProvider.Name -ne 'FileSystem' -or -not $item.PSIsContainer) {
|
||||
throw "BCQuality root must be a filesystem directory: $Root"
|
||||
}
|
||||
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "Linked BCQuality roots are not supported: $Root"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Assert-KnowledgePath {
|
||||
param(
|
||||
[string] $Path,
|
||||
[ValidateSet('article', 'sample')] [string] $Kind = 'article'
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Path) -or
|
||||
$Path -cnotmatch '^(microsoft|community|custom)/knowledge/[^/]+/.+' -or
|
||||
$Path -match '[\\:*?"<>|\x00-\x1f]' -or
|
||||
@($Path.Split('/') | Where-Object { $_ -in '', '.', '..' -or $_ -match '[. ]$' }).Count) {
|
||||
throw "Invalid knowledge path: $Path"
|
||||
}
|
||||
if (($Kind -eq 'article' -and -not $Path.EndsWith('.md', [StringComparison]::Ordinal)) -or
|
||||
($Kind -eq 'sample' -and $Path -cnotmatch '\.(good|bad)\.[a-z0-9]+$')) {
|
||||
throw "Expected an exact $Kind path: $Path"
|
||||
}
|
||||
}
|
||||
|
||||
function Resolve-KnowledgePath {
|
||||
param(
|
||||
[string] $Root,
|
||||
[string] $Path,
|
||||
[ValidateSet('article', 'sample')] [string] $Kind = 'article'
|
||||
)
|
||||
|
||||
Assert-KnowledgePath -Path $Path -Kind $Kind
|
||||
$current = $Root
|
||||
foreach ($part in $Path.Split('/')) {
|
||||
$items = @(
|
||||
Get-ChildItem -LiteralPath $current -Filter $part -Force -ErrorAction Stop |
|
||||
Where-Object Name -CEQ $part
|
||||
)
|
||||
if ($items.Count -ne 1) {
|
||||
throw "Knowledge path does not exist with exact casing: $Path"
|
||||
}
|
||||
$item = $items[0]
|
||||
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "Linked knowledge paths are not supported: $Path"
|
||||
}
|
||||
$current = $item.FullName
|
||||
}
|
||||
if ($item.PSIsContainer) {
|
||||
throw "Knowledge path is not a file: $Path"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Read-KnowledgeText {
|
||||
param([string] $Path)
|
||||
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
try {
|
||||
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||
}
|
||||
catch {
|
||||
throw "Knowledge file is not valid strict UTF-8: $Path"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
text = $text
|
||||
bytes = $bytes.Length
|
||||
sha256 = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($bytes)
|
||||
).ToLowerInvariant()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NormalizedKnowledgeVersions {
|
||||
param([string[]] $Values)
|
||||
|
||||
foreach ($value in $Values) {
|
||||
if ($value -match '^"([^"]*)"$' -or $value -match "^'([^']*)'$") {
|
||||
$Matches[1]
|
||||
}
|
||||
else {
|
||||
$value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-KnowledgeMetadataProblem {
|
||||
param([Collections.IDictionary] $Row)
|
||||
|
||||
if ($Row['parsed'] -isnot [bool] -or -not $Row['parsed']) {
|
||||
return 'unparsed frontmatter'
|
||||
}
|
||||
if ($Row['domain'] -isnot [string] -or
|
||||
$Row['domain'] -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$') {
|
||||
return 'missing/invalid domain'
|
||||
}
|
||||
foreach ($field in @('bc-version', 'technologies', 'countries', 'application-area', 'keywords')) {
|
||||
if ($Row[$field] -isnot [array] -or -not $Row[$field].Count) {
|
||||
return "missing/invalid $field"
|
||||
}
|
||||
foreach ($value in $Row[$field]) {
|
||||
if ($value -isnot [string] -or [string]::IsNullOrWhiteSpace($value)) {
|
||||
return "invalid $field value"
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach ($field in @('title', 'description')) {
|
||||
if ($Row[$field] -isnot [string] -or
|
||||
[string]::IsNullOrWhiteSpace($Row[$field]) -or
|
||||
$Row[$field] -match '[\r\n]') {
|
||||
return "missing/invalid $field"
|
||||
}
|
||||
}
|
||||
|
||||
$versions = @(Get-NormalizedKnowledgeVersions -Values $Row['bc-version'])
|
||||
if ($versions -ccontains 'all') {
|
||||
if ($versions.Count -ne 1) {
|
||||
return 'mixed bc-version sentinel'
|
||||
}
|
||||
}
|
||||
elseif ($versions.Count -eq 1 -and $versions[0] -match '^(\d+)\.\.(\d+)?$') {
|
||||
$start = [bigint]::Parse($Matches[1])
|
||||
if ($start -le 0 -or ($Matches[2] -and [bigint]::Parse($Matches[2]) -le 0)) {
|
||||
return 'invalid bc-version range bound'
|
||||
}
|
||||
if ($Matches[2] -and $start -gt [bigint]::Parse($Matches[2])) {
|
||||
return 'descending bc-version range'
|
||||
}
|
||||
}
|
||||
else {
|
||||
foreach ($version in $versions) {
|
||||
if ($version -notmatch '^\d+$' -or [bigint]::Parse($version) -le 0) {
|
||||
return 'invalid bc-version'
|
||||
}
|
||||
}
|
||||
}
|
||||
if (@($Row.technologies | Where-Object { $_ -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$' }).Count) {
|
||||
return 'invalid technologies'
|
||||
}
|
||||
if ($Row.technologies -ccontains 'all') {
|
||||
return 'invalid technologies sentinel'
|
||||
}
|
||||
if ($Row.countries -ccontains 'w1') {
|
||||
if ($Row.countries.Count -ne 1) {
|
||||
return 'mixed countries sentinel'
|
||||
}
|
||||
}
|
||||
elseif (@($Row.countries | Where-Object { $_ -cnotmatch '^[a-z]{2}$' }).Count) {
|
||||
return 'invalid countries'
|
||||
}
|
||||
if (@($Row['application-area'] | Where-Object { $_ -cnotmatch '^(all|[a-z0-9]+(-[a-z0-9]+)*)$' }).Count) {
|
||||
return 'invalid application-area'
|
||||
}
|
||||
if ($Row['application-area'] -ccontains 'all' -and $Row['application-area'].Count -ne 1) {
|
||||
return 'mixed application-area sentinel'
|
||||
}
|
||||
if (@($Row.keywords | Where-Object { $_ -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$' }).Count) {
|
||||
return 'invalid keywords'
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function Get-PreparedManifestSha256 {
|
||||
param(
|
||||
[string[]] $Paths,
|
||||
[Collections.Generic.Dictionary[string, object]] $ByPath
|
||||
)
|
||||
|
||||
$hash = [Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
$hash.AppendData([byte[]][char]'[')
|
||||
for ($i = 0; $i -lt $Paths.Count; $i++) {
|
||||
if ($i) {
|
||||
$hash.AppendData([byte[]][char]',')
|
||||
}
|
||||
$row = [ordered]@{
|
||||
path = $Paths[$i]
|
||||
sha256 = $ByPath[$Paths[$i]].sourceSha256
|
||||
}
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $row -Depth 8 -Compress)
|
||||
))
|
||||
}
|
||||
$hash.AppendData([byte[]][char]']')
|
||||
return [Convert]::ToHexString($hash.GetHashAndReset()).ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$hash.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Read-PreparedKnowledgeIndex {
|
||||
param(
|
||||
[string] $IndexPath,
|
||||
[string] $Recovery
|
||||
)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $IndexPath -PathType Leaf)) {
|
||||
throw "Knowledge index missing: $IndexPath. $Recovery"
|
||||
}
|
||||
$indexItem = Get-Item -LiteralPath $IndexPath -Force -ErrorAction Stop
|
||||
if ($indexItem.PSProvider.Name -ne 'FileSystem' -or
|
||||
($indexItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "Knowledge index must be an unlinked filesystem file: $IndexPath. $Recovery"
|
||||
}
|
||||
|
||||
$content = Read-KnowledgeText -Path $indexItem.FullName
|
||||
try {
|
||||
$index = $content.text.TrimStart([char]0xfeff) |
|
||||
ConvertFrom-Json -AsHashtable -ErrorAction Stop
|
||||
}
|
||||
catch {
|
||||
throw "Malformed knowledge index JSON: $($_.Exception.Message). $Recovery"
|
||||
}
|
||||
if ($index -isnot [Collections.IDictionary] -or
|
||||
$index.version -ne 1 -or
|
||||
$index.articles -isnot [array] -or
|
||||
$index.articleCount -ne $index.articles.Count -or
|
||||
$index.enabledLayers -isnot [array] -or
|
||||
$index.knowledgeAllow -isnot [array] -or
|
||||
$index.knowledgeDeny -isnot [array] -or
|
||||
$index.sourceSnapshot -isnot [string] -or
|
||||
$index.sourceSnapshot -cnotmatch '^[a-f0-9]{64}$') {
|
||||
throw "Invalid knowledge index envelope. $Recovery"
|
||||
}
|
||||
|
||||
$generatedAt = [DateTimeOffset]::MinValue
|
||||
if ($index.generatedAt -is [DateTime]) {
|
||||
$generatedAt = [DateTimeOffset]$index.generatedAt
|
||||
}
|
||||
elseif (-not [DateTimeOffset]::TryParse(
|
||||
[string]$index.generatedAt,
|
||||
[Globalization.CultureInfo]::InvariantCulture,
|
||||
[Globalization.DateTimeStyles]::RoundtripKind,
|
||||
[ref]$generatedAt
|
||||
)) {
|
||||
throw "Invalid knowledge index generatedAt. $Recovery"
|
||||
}
|
||||
if ($generatedAt -gt [DateTimeOffset]::UtcNow.AddMinutes(1)) {
|
||||
throw "Invalid knowledge index generatedAt. $Recovery"
|
||||
}
|
||||
|
||||
$byPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($row in $index.articles) {
|
||||
if ($row -isnot [Collections.IDictionary] -or $row.path -isnot [string]) {
|
||||
throw "Index row has no exact path. $Recovery"
|
||||
}
|
||||
Assert-KnowledgePath -Path $row.path
|
||||
if ($row.layer -cne $row.path.Split('/')[0] -or
|
||||
$row.layer -cnotin @('microsoft', 'community', 'custom')) {
|
||||
throw "Invalid index layer: $($row.path). $Recovery"
|
||||
}
|
||||
if ($row.sourceSha256 -isnot [string] -or
|
||||
$row.sourceSha256 -cnotmatch '^[a-f0-9]{64}$') {
|
||||
throw "Invalid source hash in knowledge index: $($row.path). $Recovery"
|
||||
}
|
||||
if (-not $byPath.TryAdd($row.path, $row)) {
|
||||
throw "Duplicate index path: $($row.path). $Recovery"
|
||||
}
|
||||
}
|
||||
|
||||
$paths = [string[]]@($byPath.Keys)
|
||||
[Array]::Sort($paths, [StringComparer]::Ordinal)
|
||||
if ((Get-PreparedManifestSha256 -Paths $paths -ByPath $byPath) -cne $index.sourceSnapshot) {
|
||||
throw "Stale or internally inconsistent prepared index snapshot. $Recovery"
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
index = $index
|
||||
content = $content
|
||||
byPath = $byPath
|
||||
paths = $paths
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-SampleLink {
|
||||
param(
|
||||
[string] $ArticleText,
|
||||
[string] $SamplePath
|
||||
)
|
||||
|
||||
$sampleName = [IO.Path]::GetFileName($SamplePath)
|
||||
$expected = '[`' + $sampleName + '`](' + $sampleName + ')'
|
||||
if (-not $ArticleText.Contains($expected, [StringComparison]::Ordinal)) {
|
||||
throw "Sample is not linked by its article using the READ convention: $sampleName"
|
||||
}
|
||||
}
|
||||
241
tools/Search-Knowledge.ps1
Normal file
241
tools/Search-Knowledge.ps1
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Returns bounded pages of every domain/layer/READ-applicable catalog row.
|
||||
.DESCRIPTION
|
||||
Consumes Entry's prepared index read-only. Results are never ranked, sampled,
|
||||
top-k limited, deduplicated by basename, or narrowed by query text. Omit an
|
||||
unknown task dimension; an explicit empty array is a known empty set.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateNotNullOrEmpty()] [string] $BCQualityRoot = (Split-Path $PSScriptRoot -Parent),
|
||||
[Parameter(Mandatory)] [ValidateNotNullOrEmpty()]
|
||||
[ValidateScript({ -not [string]::IsNullOrWhiteSpace($_) })] [string] $Domain,
|
||||
[ValidateSet('microsoft', 'community', 'custom')]
|
||||
[AllowEmptyCollection()] [string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
|
||||
[ValidateRange(1, 2147483647)] [int] $BCVersion,
|
||||
[AllowEmptyCollection()] [string[]] $Technologies,
|
||||
[AllowEmptyCollection()] [string[]] $Countries,
|
||||
[AllowEmptyCollection()] [string[]] $ApplicationAreas,
|
||||
[switch] $ExcludeConditional,
|
||||
[string] $IndexPath,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000,
|
||||
[ValidateRange(0, 2147483647)] [int] $Offset = 0,
|
||||
[ValidatePattern('^[a-f0-9]{64}$')] [string] $Snapshot
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
. (Join-Path $PSScriptRoot 'Bounded-Results.ps1')
|
||||
|
||||
$BCQualityRoot = Resolve-KnowledgeRoot $BCQualityRoot
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'knowledge-index.json'
|
||||
}
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array; use an empty array to disable all layers.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
|
||||
$context = [ordered]@{}
|
||||
foreach ($pair in @(
|
||||
@('BCVersion', 'bc-version'),
|
||||
@('Technologies', 'technologies'),
|
||||
@('Countries', 'countries'),
|
||||
@('ApplicationAreas', 'application-area')
|
||||
)) {
|
||||
if (-not $PSBoundParameters.ContainsKey($pair[0])) {
|
||||
continue
|
||||
}
|
||||
$value = $PSBoundParameters[$pair[0]]
|
||||
if ($null -eq $value) {
|
||||
throw "Omit unknown context; do not pass null for $($pair[0])."
|
||||
}
|
||||
if ($pair[0] -ne 'BCVersion') {
|
||||
foreach ($entry in $value) {
|
||||
if ([string]::IsNullOrWhiteSpace($entry) -or $entry -cne $entry.Trim()) {
|
||||
throw "Invalid context value for $($pair[0]): '$entry'"
|
||||
}
|
||||
}
|
||||
}
|
||||
$context[$pair[1]] = $value
|
||||
}
|
||||
if ($context.Contains('technologies') -and $context['technologies'] -ccontains 'all') {
|
||||
throw "Technologies has no 'all' sentinel. Omit unknown context."
|
||||
}
|
||||
|
||||
$recovery = "Run Entry preparation once before dispatch, or use READ's path-discovery fallback. Do not rebuild in a leaf."
|
||||
$preparedIndex = Read-PreparedKnowledgeIndex -IndexPath $IndexPath -Recovery $recovery
|
||||
$index = $preparedIndex.index
|
||||
$indexContent = $preparedIndex.content
|
||||
$byPath = $preparedIndex.byPath
|
||||
$paths = $preparedIndex.paths
|
||||
|
||||
# The v1 generator historically serialized an omitted EnabledLayers parameter as [null].
|
||||
$unrestricted = $index.enabledLayers.Count -eq 0 -or
|
||||
($index.enabledLayers.Count -eq 1 -and $null -eq $index.enabledLayers[0])
|
||||
$indexedLayers = @(
|
||||
if ($unrestricted) {
|
||||
'microsoft', 'community', 'custom'
|
||||
}
|
||||
else {
|
||||
$index.enabledLayers
|
||||
}
|
||||
)
|
||||
if (@($indexedLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($indexedLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count -or
|
||||
@($EnabledLayers | Where-Object { $_ -cnotin $indexedLayers }).Count) {
|
||||
throw "Index layer coverage does not cover EnabledLayers. $recovery"
|
||||
}
|
||||
|
||||
foreach ($path in $paths) {
|
||||
$row = $byPath[$path]
|
||||
if ($row.layer -cnotin $indexedLayers) {
|
||||
throw "Index row layer is outside index coverage: $path. $recovery"
|
||||
}
|
||||
$problem = Get-KnowledgeMetadataProblem -Row $row
|
||||
if ($problem) {
|
||||
throw "Malformed knowledge index row at ${path}: $problem. $recovery"
|
||||
}
|
||||
}
|
||||
|
||||
$defaults = [ordered]@{
|
||||
'bc-version' = @('all')
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
}
|
||||
$candidates = [Collections.Generic.List[object]]::new()
|
||||
$excluded = [Collections.Generic.List[object]]::new()
|
||||
foreach ($path in $paths) {
|
||||
$row = $byPath[$path]
|
||||
if ($row.domain -cne $Domain) {
|
||||
continue
|
||||
}
|
||||
|
||||
$unknown = [Collections.Generic.List[string]]::new()
|
||||
$matchesContext = $true
|
||||
foreach ($field in $defaults.Keys) {
|
||||
$values = $row[$field]
|
||||
if ($field -eq 'bc-version') {
|
||||
$values = @(Get-NormalizedKnowledgeVersions -Values $values)
|
||||
}
|
||||
$sentinel = switch ($field) {
|
||||
'bc-version' { 'all' }
|
||||
'countries' { 'w1' }
|
||||
'application-area' { 'all' }
|
||||
default { '' }
|
||||
}
|
||||
if ($sentinel -and $values -ccontains $sentinel) {
|
||||
continue
|
||||
}
|
||||
if (-not $context.Contains($field)) {
|
||||
$unknown.Add($field)
|
||||
continue
|
||||
}
|
||||
|
||||
$target = $context[$field]
|
||||
$matched = $false
|
||||
if ($field -eq 'bc-version') {
|
||||
if ($values.Count -eq 1 -and $values[0] -match '^(\d+)\.\.(\d+)?$') {
|
||||
$matched = $target -ge [bigint]::Parse($Matches[1]) -and
|
||||
(-not $Matches[2] -or $target -le [bigint]::Parse($Matches[2]))
|
||||
}
|
||||
else {
|
||||
$matched = @($values | Where-Object { [bigint]::Parse($_) -eq $target }).Count -gt 0
|
||||
}
|
||||
}
|
||||
else {
|
||||
$matched = @($values | Where-Object { $target -ccontains $_ }).Count -gt 0
|
||||
}
|
||||
if (-not $matched) {
|
||||
$matchesContext = $false
|
||||
break
|
||||
}
|
||||
}
|
||||
if (-not $matchesContext -or ($ExcludeConditional -and $unknown.Count)) {
|
||||
continue
|
||||
}
|
||||
$null = Resolve-KnowledgePath -Root $BCQualityRoot -Path $path
|
||||
|
||||
$candidate = [ordered]@{
|
||||
path = $path
|
||||
layer = $row.layer
|
||||
keywords = $row.keywords
|
||||
title = $row.title
|
||||
description = $row.description
|
||||
}
|
||||
foreach ($field in $defaults.Keys) {
|
||||
if (($row[$field] -join "`0") -cne ($defaults[$field] -join "`0")) {
|
||||
$candidate[$field] = $row[$field]
|
||||
}
|
||||
}
|
||||
$candidate.applicability = if ($unknown.Count) { 'conditional' } else { 'applicable' }
|
||||
$candidate.unknownDimensions = @($unknown)
|
||||
if ($row.layer -cin $EnabledLayers) {
|
||||
$candidates.Add($candidate)
|
||||
}
|
||||
else {
|
||||
$excluded.Add($candidate)
|
||||
}
|
||||
}
|
||||
|
||||
$header = [ordered]@{
|
||||
version = 2
|
||||
domain = $Domain
|
||||
context = $context
|
||||
enabledLayers = @($EnabledLayers)
|
||||
indexedLayers = @($indexedLayers)
|
||||
excludeConditional = [bool]$ExcludeConditional
|
||||
defaults = $defaults
|
||||
candidateCount = $candidates.Count
|
||||
excludedByConfigurationCount = $excluded.Count
|
||||
}
|
||||
|
||||
function Get-CatalogSnapshot {
|
||||
param(
|
||||
[string] $PreparedIndexSha256,
|
||||
[Collections.IDictionary] $Request,
|
||||
[Collections.IDictionary] $Groups
|
||||
)
|
||||
|
||||
$hash = [Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
foreach ($value in @(
|
||||
$PreparedIndexSha256,
|
||||
(ConvertTo-Json -InputObject $Request -Depth 8 -Compress)
|
||||
)) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes($value))
|
||||
$hash.AppendData([byte[]](10))
|
||||
}
|
||||
foreach ($groupName in $Groups.Keys) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes("[$groupName]"))
|
||||
$hash.AppendData([byte[]](10))
|
||||
foreach ($row in $Groups[$groupName]) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $row -Depth 8 -Compress)
|
||||
))
|
||||
$hash.AppendData([byte[]](10))
|
||||
}
|
||||
}
|
||||
return [Convert]::ToHexString($hash.GetHashAndReset()).ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$hash.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
$groups = [ordered]@{
|
||||
candidates = $candidates
|
||||
excludedByConfiguration = $excluded
|
||||
}
|
||||
$header.snapshot = Get-CatalogSnapshot -PreparedIndexSha256 $indexContent.sha256 -Request $header -Groups $groups
|
||||
|
||||
ConvertTo-BoundedPage -Header $header -Groups $groups -Offset $Offset -Snapshot $Snapshot -MaxBytes $MaxBytes
|
||||
759
tools/Test-KnowledgeRetrieval.ps1
Normal file
759
tools/Test-KnowledgeRetrieval.ps1
Normal file
|
|
@ -0,0 +1,759 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates lossless bounded catalog and exact-body retrieval.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..'))
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
|
||||
$generator = Join-Path $Root 'tools/Build-KnowledgeIndex.ps1'
|
||||
$search = Join-Path $Root 'tools/Search-Knowledge.ps1'
|
||||
$getArticles = Join-Path $Root 'tools/Get-KnowledgeArticles.ps1'
|
||||
$utf8 = [Text.UTF8Encoding]::new($false, $true)
|
||||
|
||||
function Assert-True {
|
||||
param([bool] $Condition, [string] $Message)
|
||||
if (-not $Condition) {
|
||||
throw "Assertion failed: $Message"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Equal {
|
||||
param($Actual, $Expected, [string] $Message)
|
||||
if ($Actual -cne $Expected) {
|
||||
throw "Assertion failed: $Message. Expected '$Expected', got '$Actual'."
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Sequence {
|
||||
param($Actual, $Expected, [string] $Message)
|
||||
$actualJson = ConvertTo-Json -InputObject @($Actual) -Compress
|
||||
$expectedJson = ConvertTo-Json -InputObject @($Expected) -Compress
|
||||
if ($actualJson -cne $expectedJson) {
|
||||
throw "Assertion failed: $Message. Expected $expectedJson, got $actualJson."
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Throws {
|
||||
param([scriptblock] $Action, [string] $Pattern, [string] $Message)
|
||||
try {
|
||||
& $Action
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Message -notmatch $Pattern) {
|
||||
throw "Assertion failed: $Message. Wrong error: $($_.Exception.Message)"
|
||||
}
|
||||
return
|
||||
}
|
||||
throw "Assertion failed: $Message. No error was thrown."
|
||||
}
|
||||
|
||||
function Get-OutputByteCount {
|
||||
param([string] $Text)
|
||||
return [Text.Encoding]::UTF8.GetByteCount($Text) +
|
||||
[Text.Encoding]::UTF8.GetByteCount([Environment]::NewLine)
|
||||
}
|
||||
|
||||
function Invoke-CatalogPages {
|
||||
param(
|
||||
[hashtable] $Arguments,
|
||||
[int] $MaxBytes = 4096
|
||||
)
|
||||
|
||||
$allCandidates = [Collections.Generic.List[object]]::new()
|
||||
$allExcluded = [Collections.Generic.List[object]]::new()
|
||||
$offset = 0
|
||||
$snapshot = ''
|
||||
$shared = ''
|
||||
$pageCount = 0
|
||||
$lastPage = $null
|
||||
do {
|
||||
$pageArguments = @{} + $Arguments
|
||||
$pageArguments.MaxBytes = $MaxBytes
|
||||
$pageArguments.Offset = $offset
|
||||
if ($snapshot) {
|
||||
$pageArguments.Snapshot = $snapshot
|
||||
}
|
||||
$raw = & $search @pageArguments
|
||||
Assert-True ($raw -is [string]) 'catalog helper emitted exactly one JSON string'
|
||||
Assert-True ((Get-OutputByteCount -Text $raw) -le $MaxBytes) 'catalog page includes its newline in MaxBytes'
|
||||
$page = $raw | ConvertFrom-Json
|
||||
$pageCount++
|
||||
Assert-True ($pageCount -le 1000) 'catalog continuation terminates'
|
||||
Assert-Equal $page.offset $offset 'catalog offset is exact'
|
||||
Assert-Equal $page.returnedCount (@($page.candidates).Count + @($page.excludedByConfiguration).Count) 'page returnedCount matches rows'
|
||||
Assert-Equal $page.remainingCount ($page.totalCount - $offset - $page.returnedCount) 'page remainingCount is exact'
|
||||
|
||||
$currentShared = [ordered]@{
|
||||
version = $page.version
|
||||
domain = $page.domain
|
||||
context = $page.context
|
||||
enabledLayers = $page.enabledLayers
|
||||
indexedLayers = $page.indexedLayers
|
||||
excludeConditional = $page.excludeConditional
|
||||
defaults = $page.defaults
|
||||
candidateCount = $page.candidateCount
|
||||
excludedByConfigurationCount = $page.excludedByConfigurationCount
|
||||
snapshot = $page.snapshot
|
||||
totalCount = $page.totalCount
|
||||
} | ConvertTo-Json -Depth 8 -Compress
|
||||
if (-not $shared) {
|
||||
$shared = $currentShared
|
||||
$snapshot = $page.snapshot
|
||||
}
|
||||
else {
|
||||
Assert-Equal $currentShared $shared 'catalog pages repeat shared context, defaults, totals, and snapshot'
|
||||
}
|
||||
|
||||
foreach ($row in @($page.candidates)) {
|
||||
$allCandidates.Add($row)
|
||||
}
|
||||
foreach ($row in @($page.excludedByConfiguration)) {
|
||||
$allExcluded.Add($row)
|
||||
}
|
||||
if (-not $page.complete) {
|
||||
Assert-True ($null -ne $page.continuation) 'incomplete page has continuation'
|
||||
Assert-Equal $page.continuation.snapshot $snapshot 'continuation is snapshot-bound'
|
||||
Assert-True ($page.continuation.offset -gt $offset) 'continuation makes progress'
|
||||
$offset = $page.continuation.offset
|
||||
}
|
||||
$lastPage = $page
|
||||
} while (-not $page.complete)
|
||||
|
||||
Assert-True ($null -eq $lastPage.continuation) 'final page has no continuation'
|
||||
Assert-Equal $allCandidates.Count $lastPage.candidateCount 'candidate total survives paging'
|
||||
Assert-Equal $allExcluded.Count $lastPage.excludedByConfigurationCount 'excluded total survives paging'
|
||||
return [pscustomobject]@{
|
||||
candidates = @($allCandidates)
|
||||
excluded = @($allExcluded)
|
||||
pages = $pageCount
|
||||
snapshot = $snapshot
|
||||
lastPage = $lastPage
|
||||
}
|
||||
}
|
||||
|
||||
function Test-BodyRoundTrip {
|
||||
param(
|
||||
[string[]] $Paths,
|
||||
[string] $IndexPath,
|
||||
[switch] $Samples
|
||||
)
|
||||
|
||||
$seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
for ($start = 0; $start -lt $Paths.Count; $start += 8) {
|
||||
$end = [Math]::Min($start + 7, $Paths.Count - 1)
|
||||
$remaining = @($Paths[$start..$end])
|
||||
$snapshot = ''
|
||||
do {
|
||||
$arguments = @{
|
||||
BCQualityRoot = $Root
|
||||
IndexPath = $IndexPath
|
||||
Paths = $remaining
|
||||
MaxArticles = 8
|
||||
MaxBytes = 16000
|
||||
}
|
||||
if ($Samples) {
|
||||
$arguments.Samples = $true
|
||||
}
|
||||
if ($snapshot) {
|
||||
$arguments.Snapshot = $snapshot
|
||||
}
|
||||
$raw = & $getArticles @arguments
|
||||
Assert-True ($raw -is [string]) 'article helper emitted exactly one JSON string'
|
||||
Assert-True ((Get-OutputByteCount -Text $raw) -le 16000) 'article batch includes its newline in MaxBytes'
|
||||
$batch = $raw | ConvertFrom-Json
|
||||
Assert-True ($batch.returnedCount -gt 0) 'article batching makes progress'
|
||||
Assert-Equal $batch.returnedCount @($batch.articles).Count 'article returnedCount matches rows'
|
||||
Assert-Equal $batch.complete (@($batch.remainingPaths).Count -eq 0) 'article completion matches remaining paths'
|
||||
if ($batch.complete) {
|
||||
Assert-True ($null -eq $batch.continuation) 'complete article batch has no continuation'
|
||||
}
|
||||
else {
|
||||
Assert-True ($batch.continuation.snapshot -match '^[a-f0-9]{64}$') 'article continuation is snapshot-bound'
|
||||
}
|
||||
|
||||
foreach ($article in @($batch.articles)) {
|
||||
Assert-True ($seen.Add($article.path)) "body returned once: $($article.path)"
|
||||
$fullPath = Join-Path $Root ($article.path.Replace('/', [IO.Path]::DirectorySeparatorChar))
|
||||
$bytes = [IO.File]::ReadAllBytes($fullPath)
|
||||
$text = $utf8.GetString($bytes)
|
||||
$hash = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($bytes)
|
||||
).ToLowerInvariant()
|
||||
Assert-Equal $article.bytes $bytes.Length "byte count round-trips: $($article.path)"
|
||||
Assert-Equal $article.sha256 $hash "SHA-256 round-trips: $($article.path)"
|
||||
Assert-Equal $article.body $text "body round-trips: $($article.path)"
|
||||
}
|
||||
$remaining = @($batch.remainingPaths)
|
||||
$snapshot = if ($batch.complete) { '' } else { $batch.continuation.snapshot }
|
||||
} while ($remaining.Count)
|
||||
}
|
||||
Assert-Equal $seen.Count $Paths.Count 'every requested body round-trips without loss'
|
||||
}
|
||||
|
||||
function New-NeutralArticle {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $Layer,
|
||||
[string] $Slug,
|
||||
[string] $Version = 'all',
|
||||
[string] $Technology = 'al',
|
||||
[string] $Country = 'w1',
|
||||
[string] $Area = 'all',
|
||||
[string] $Title = 'Neutral retrieval example',
|
||||
[string] $Description = 'Neutral retrieval metadata for deterministic tests.'
|
||||
)
|
||||
|
||||
$directory = Join-Path $FixtureRoot "$Layer\knowledge\neutral"
|
||||
New-Item -ItemType Directory -Force -Path $directory | Out-Null
|
||||
$content = @"
|
||||
---
|
||||
bc-version: [$Version]
|
||||
domain: neutral
|
||||
keywords: [neutral, retrieval, deterministic]
|
||||
technologies: [$Technology]
|
||||
countries: [$Country]
|
||||
application-area: [$Area]
|
||||
---
|
||||
|
||||
# $Title
|
||||
|
||||
## Description
|
||||
|
||||
$Description
|
||||
"@
|
||||
Set-Content -LiteralPath (Join-Path $directory "$Slug.md") -Value $content -Encoding utf8NoBOM
|
||||
}
|
||||
|
||||
function Test-InvalidSourceIndexing {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $Field,
|
||||
[string] $ValidValue,
|
||||
[string] $InvalidValue
|
||||
)
|
||||
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer microsoft -Slug valid-source
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer community -Slug invalid-source
|
||||
$articlePath = Join-Path $FixtureRoot 'community\knowledge\neutral\invalid-source.md'
|
||||
$text = [IO.File]::ReadAllText($articlePath, $utf8)
|
||||
$text = $text.Replace("$Field`: $ValidValue", "$Field`: $InvalidValue")
|
||||
[IO.File]::WriteAllText($articlePath, $text, $utf8)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("$Field-index.json")
|
||||
$generation = @(& $generator -BCQualityRoot $FixtureRoot -IndexPath $indexPath 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 "scalar $Field source emits one omission warning"
|
||||
Assert-True (
|
||||
$warnings[0].Message -match
|
||||
"Skipping invalid knowledge article 'community/knowledge/neutral/invalid-source\.md': frontmatter field '$([regex]::Escape($Field))' must use non-empty bracket-array syntax\."
|
||||
) "scalar $Field warning identifies the exact path and reason"
|
||||
$prepared = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
Assert-Equal $prepared.articleCount 1 "scalar $Field source is omitted while its valid sibling is indexed"
|
||||
Assert-Sequence $prepared.articles.path @('microsoft/knowledge/neutral/valid-source.md') "scalar $Field index contains only the valid sibling"
|
||||
$catalog = & $search -BCQualityRoot $FixtureRoot -IndexPath $indexPath -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $catalog.candidates.path @('microsoft/knowledge/neutral/valid-source.md') "scalar $Field catalog retrieves the valid sibling"
|
||||
$valid = & $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-source.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $valid.complete "scalar $Field valid sibling body retrieves completely"
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'community/knowledge/neutral/invalid-source.md'
|
||||
} 'Selected article is absent from the prepared index' "scalar $Field omitted source cannot be retrieved"
|
||||
}
|
||||
|
||||
function Test-InvalidSemanticIndexing {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $CaseName,
|
||||
[string] $Field,
|
||||
[string] $ValidValue,
|
||||
[string] $InvalidValue,
|
||||
[string] $ExpectedReason
|
||||
)
|
||||
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer microsoft -Slug valid-source
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer community -Slug invalid-source
|
||||
$articlePath = Join-Path $FixtureRoot 'community\knowledge\neutral\invalid-source.md'
|
||||
$text = [IO.File]::ReadAllText($articlePath, $utf8)
|
||||
$text = $text.Replace("$Field`: $ValidValue", "$Field`: $InvalidValue")
|
||||
[IO.File]::WriteAllText($articlePath, $text, $utf8)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("$CaseName-index.json")
|
||||
$generation = @(& $generator -BCQualityRoot $FixtureRoot -IndexPath $indexPath 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 "$CaseName emits one omission warning"
|
||||
Assert-Equal $warnings[0].Message "Skipping invalid knowledge article 'community/knowledge/neutral/invalid-source.md': $ExpectedReason." "$CaseName warning identifies exact path and reason"
|
||||
|
||||
$prepared = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
Assert-Equal $prepared.articleCount 1 "$CaseName omits invalid source and retains valid sibling"
|
||||
Assert-Sequence $prepared.articles.path @('microsoft/knowledge/neutral/valid-source.md') "$CaseName index contains only valid sibling"
|
||||
Assert-True ($prepared.sourceSnapshot -match '^[a-f0-9]{64}$') "$CaseName source snapshot remains valid"
|
||||
$validRow = $prepared.articles[0]
|
||||
$manifest = @(
|
||||
[ordered]@{ path = $validRow.path; sha256 = $validRow.sourceSha256 }
|
||||
)
|
||||
$manifestBytes = [Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $manifest -Depth 8 -Compress)
|
||||
)
|
||||
$expectedSnapshot = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($manifestBytes)
|
||||
).ToLowerInvariant()
|
||||
Assert-Equal $prepared.sourceSnapshot $expectedSnapshot "$CaseName source snapshot covers only retained rows"
|
||||
|
||||
$catalog = & $search -BCQualityRoot $FixtureRoot -IndexPath $indexPath -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $catalog.candidates.path @('microsoft/knowledge/neutral/valid-source.md') "$CaseName catalog retains valid sibling"
|
||||
$valid = & $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-source.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $valid.complete "$CaseName valid sibling body retrieves"
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'community/knowledge/neutral/invalid-source.md'
|
||||
} 'Selected article is absent from the prepared index' "$CaseName invalid source cannot be retrieved"
|
||||
}
|
||||
|
||||
function Test-InvalidEnabledLayers {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $CaseName,
|
||||
$Layers,
|
||||
[string] $ExpectedPattern
|
||||
)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("layers-$CaseName.json")
|
||||
$arguments = @{
|
||||
BCQualityRoot = $FixtureRoot
|
||||
IndexPath = $indexPath
|
||||
EnabledLayers = $Layers
|
||||
}
|
||||
Assert-Throws {
|
||||
& $generator @arguments
|
||||
} $ExpectedPattern "$CaseName EnabledLayers fails"
|
||||
Assert-True (-not (Test-Path -LiteralPath $indexPath)) "$CaseName fails before index creation"
|
||||
}
|
||||
|
||||
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("bcquality_retrieval_" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Force -Path $tmp | Out-Null
|
||||
try {
|
||||
$indexPath = Join-Path $tmp 'knowledge-index.json'
|
||||
& $generator -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$index = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$diskArticlePaths = @(
|
||||
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||
$knowledge = Join-Path $Root "$layer\knowledge"
|
||||
if (Test-Path -LiteralPath $knowledge) {
|
||||
Get-ChildItem -LiteralPath $knowledge -Recurse -File -Filter '*.md' |
|
||||
ForEach-Object {
|
||||
[IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
|
||||
}
|
||||
}
|
||||
}
|
||||
) | Sort-Object
|
||||
Assert-Equal $index.articleCount $diskArticlePaths.Count 'index covers every current article'
|
||||
Assert-True ($index.sourceSnapshot -match '^[a-f0-9]{64}$') 'index carries an exact source snapshot'
|
||||
|
||||
$allCatalogRows = [Collections.Generic.List[object]]::new()
|
||||
$domains = @($index.articles.domain | Sort-Object -Unique)
|
||||
foreach ($domain in $domains) {
|
||||
$catalog = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $Root
|
||||
IndexPath = $indexPath
|
||||
Domain = $domain
|
||||
}
|
||||
Assert-Equal $catalog.excluded.Count 0 "all layers enabled for $domain"
|
||||
foreach ($row in $catalog.candidates) {
|
||||
$allCatalogRows.Add($row)
|
||||
}
|
||||
}
|
||||
|
||||
$expectedRows = @($index.articles | Sort-Object path)
|
||||
$actualRows = @($allCatalogRows | Sort-Object path)
|
||||
Assert-Equal $actualRows.Count $expectedRows.Count 'paged union has no top-k or query-based loss'
|
||||
Assert-Sequence ($actualRows.path) ($expectedRows.path) 'paged union equals all READ-filtered candidates'
|
||||
Assert-Equal @($actualRows.path | Sort-Object -Unique).Count $actualRows.Count 'catalog does not deduplicate distinct paths'
|
||||
|
||||
$defaults = [ordered]@{
|
||||
'bc-version' = @('all')
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
}
|
||||
for ($i = 0; $i -lt $actualRows.Count; $i++) {
|
||||
$actual = $actualRows[$i]
|
||||
$expected = $expectedRows[$i]
|
||||
Assert-Equal $actual.path $expected.path 'catalog preserves exact path'
|
||||
Assert-Equal $actual.layer $expected.layer 'catalog preserves layer'
|
||||
Assert-Sequence $actual.keywords $expected.keywords 'catalog preserves full keywords'
|
||||
Assert-Equal $actual.title $expected.title 'catalog preserves title'
|
||||
Assert-Equal $actual.description $expected.description 'catalog preserves one-line description'
|
||||
|
||||
$unknown = [Collections.Generic.List[string]]::new()
|
||||
foreach ($field in $defaults.Keys) {
|
||||
$expectedValues = @($expected.$field)
|
||||
$sentinel = switch ($field) {
|
||||
'bc-version' { 'all' }
|
||||
'countries' { 'w1' }
|
||||
'application-area' { 'all' }
|
||||
default { '' }
|
||||
}
|
||||
if (-not $sentinel -or $expectedValues -notcontains $sentinel) {
|
||||
$unknown.Add($field)
|
||||
}
|
||||
$hasField = $actual.PSObject.Properties.Name -ccontains $field
|
||||
if (($expectedValues -join "`0") -ceq (@($defaults[$field]) -join "`0")) {
|
||||
Assert-True (-not $hasField) "default field is inherited from page: $field"
|
||||
}
|
||||
else {
|
||||
Assert-True $hasField "non-default field survives paging: $field"
|
||||
Assert-Sequence $actual.$field $expectedValues "non-default field is exact: $field"
|
||||
}
|
||||
}
|
||||
Assert-Equal $actual.applicability ($(if ($unknown.Count) { 'conditional' } else { 'applicable' })) 'applicability verdict is explicit'
|
||||
Assert-Sequence $actual.unknownDimensions @($unknown) 'unknown dimensions are explicit'
|
||||
}
|
||||
|
||||
$performanceFirst = & $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -MaxBytes 4096 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $performanceFirst.complete) 'large domain produces deterministic continuation'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -MaxBytes 4096 -Offset $performanceFirst.continuation.offset
|
||||
} 'Continuation requires Snapshot' 'continuation without snapshot fails'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -Offset $performanceFirst.totalCount
|
||||
} 'Invalid Offset' 'offset at total fails'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -Offset 1 -Snapshot ('0' * 64)
|
||||
} 'Snapshot changed' 'wrong snapshot fails'
|
||||
|
||||
$changedRawIndex = Join-Path $tmp 'changed-raw-index.json'
|
||||
$changedRaw = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$changedRaw.generatedAt = [DateTimeOffset]::UtcNow.ToString('O')
|
||||
$changedRaw | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $changedRawIndex -Encoding utf8NoBOM -NoNewline
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $changedRawIndex -Domain performance `
|
||||
-MaxBytes 4096 -Offset $performanceFirst.continuation.offset `
|
||||
-Snapshot $performanceFirst.continuation.snapshot
|
||||
} 'Snapshot changed' 'continuation is bound to the exact prepared index bytes'
|
||||
|
||||
$malformedIndex = Join-Path $tmp 'malformed.json'
|
||||
Set-Content -LiteralPath $malformedIndex -Value '{not-json' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $malformedIndex -Domain performance
|
||||
} 'Malformed knowledge index JSON' 'malformed JSON fails'
|
||||
|
||||
$unsafeIndex = Join-Path $tmp 'unsafe.json'
|
||||
$unsafe = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$unsafe.articles[0].path = '../outside.md'
|
||||
$unsafe | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $unsafeIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $unsafeIndex -Domain performance
|
||||
} 'Invalid knowledge path' 'unsafe indexed path fails'
|
||||
|
||||
$invalidRowIndex = Join-Path $tmp 'invalid-row.json'
|
||||
$invalidRow = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$invalidRow.articles[0].keywords = @()
|
||||
$invalidRow | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $invalidRowIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $invalidRowIndex -Domain performance
|
||||
} 'Malformed knowledge index row' 'malformed index row fails'
|
||||
|
||||
$semanticCorruptIndex = Join-Path $tmp 'semantic-corrupt-row.json'
|
||||
$semanticCorrupt = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$semanticCorrupt.articles[0].countries = @('usa')
|
||||
$semanticCorrupt | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $semanticCorruptIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $semanticCorruptIndex -Domain performance
|
||||
} 'Malformed knowledge index row.*invalid countries' 'search rejects semantically invalid external index rows'
|
||||
|
||||
$corruptSemanticCases = @(
|
||||
@{ name = 'uppercase-all'; field = 'bc-version'; value = @('ALL'); reason = 'invalid bc-version' },
|
||||
@{ name = 'uppercase-w1'; field = 'countries'; value = @('W1'); reason = 'invalid countries' },
|
||||
@{ name = 'zero-open-range'; field = 'bc-version'; value = @('"0.."'); reason = 'invalid bc-version range bound' },
|
||||
@{ name = 'zero-closed-range'; field = 'bc-version'; value = @('"0..0"'); reason = 'invalid bc-version range bound' }
|
||||
)
|
||||
foreach ($case in $corruptSemanticCases) {
|
||||
$corruptPath = Join-Path $tmp ("corrupt-$($case.name).json")
|
||||
$corrupt = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$corrupt.articles[0].PSObject.Properties[$case.field].Value = $case.value
|
||||
$corrupt | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $corruptPath -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $corruptPath -Domain performance
|
||||
} "Malformed knowledge index row.*$([regex]::Escape($case.reason))" "search rejects $($case.name) in an external index"
|
||||
}
|
||||
|
||||
$invalidUtf8Root = Join-Path $tmp 'invalid-utf8-source'
|
||||
New-NeutralArticle -FixtureRoot $invalidUtf8Root -Layer microsoft -Slug valid-catalog
|
||||
New-NeutralArticle -FixtureRoot $invalidUtf8Root -Layer community -Slug invalid-utf8-source
|
||||
$invalidUtf8Article = Join-Path $invalidUtf8Root 'community\knowledge\neutral\invalid-utf8-source.md'
|
||||
$validBytes = [IO.File]::ReadAllBytes($invalidUtf8Article)
|
||||
[IO.File]::WriteAllBytes($invalidUtf8Article, [byte[]]@($validBytes + @(0xc3, 0x28)))
|
||||
$invalidUtf8Index = Join-Path $tmp 'invalid-utf8-index.json'
|
||||
$generation = @(& $generator -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 'malformed UTF-8 source emits one omission warning'
|
||||
Assert-Equal $warnings[0].Message "Skipping invalid knowledge article 'community/knowledge/neutral/invalid-utf8-source.md': invalid UTF-8." 'malformed UTF-8 warning identifies the exact path and reason'
|
||||
$invalidUtf8Prepared = Get-Content -LiteralPath $invalidUtf8Index -Raw -Encoding utf8 |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $invalidUtf8Prepared.articleCount 1 'malformed UTF-8 source is omitted while its valid sibling is indexed'
|
||||
Assert-Sequence $invalidUtf8Prepared.articles.path @('microsoft/knowledge/neutral/valid-catalog.md') 'malformed UTF-8 index contains only the valid sibling'
|
||||
$validCatalog = & $search -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $validCatalog.candidates.path @('microsoft/knowledge/neutral/valid-catalog.md') 'catalog retrieves the valid sibling after malformed UTF-8 omission'
|
||||
$validBody = & $getArticles -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-catalog.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $validBody.complete 'valid sibling body retrieves after malformed UTF-8 omission'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index `
|
||||
-Paths 'community/knowledge/neutral/invalid-utf8-source.md'
|
||||
} 'Selected article is absent from the prepared index' 'omitted malformed UTF-8 source cannot be retrieved'
|
||||
|
||||
$scalarCases = @(
|
||||
@{ field = 'bc-version'; valid = '[all]'; invalid = 'all' },
|
||||
@{ field = 'keywords'; valid = '[neutral, retrieval, deterministic]'; invalid = 'neutral' },
|
||||
@{ field = 'technologies'; valid = '[al]'; invalid = 'al' },
|
||||
@{ field = 'countries'; valid = '[w1]'; invalid = 'w1' },
|
||||
@{ field = 'application-area'; valid = '[all]'; invalid = 'all' }
|
||||
)
|
||||
foreach ($case in $scalarCases) {
|
||||
Test-InvalidSourceIndexing -FixtureRoot (Join-Path $tmp "scalar-$($case.field)") `
|
||||
-Field $case.field -ValidValue $case.valid -InvalidValue $case.invalid
|
||||
}
|
||||
|
||||
$semanticCases = @(
|
||||
@{ name = 'mixed-version-sentinel'; field = 'bc-version'; valid = '[all]'; invalid = '[all, 27]'; reason = 'mixed bc-version sentinel' },
|
||||
@{ name = 'invalid-country'; field = 'countries'; valid = '[w1]'; invalid = '[usa]'; reason = 'invalid countries' },
|
||||
@{ name = 'descending-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["28..27"]'; reason = 'descending bc-version range' },
|
||||
@{ name = 'malformed-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '[twenty-seven]'; reason = 'invalid bc-version' },
|
||||
@{ name = 'malformed-keyword'; field = 'keywords'; valid = '[neutral, retrieval, deterministic]'; invalid = '[neutral, Bad_Token, deterministic]'; reason = 'invalid keywords' },
|
||||
@{ name = 'malformed-technology'; field = 'technologies'; valid = '[al]'; invalid = '[AL]'; reason = 'invalid technologies' },
|
||||
@{ name = 'malformed-application-area'; field = 'application-area'; valid = '[all]'; invalid = '[finance_]'; reason = 'invalid application-area' },
|
||||
@{ name = 'uppercase-version-sentinel'; field = 'bc-version'; valid = '[all]'; invalid = '[ALL]'; reason = 'invalid bc-version' },
|
||||
@{ name = 'uppercase-country-sentinel'; field = 'countries'; valid = '[w1]'; invalid = '[W1]'; reason = 'invalid countries' },
|
||||
@{ name = 'zero-open-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["0.."]'; reason = 'invalid bc-version range bound' },
|
||||
@{ name = 'zero-closed-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["0..0"]'; reason = 'invalid bc-version range bound' }
|
||||
)
|
||||
foreach ($case in $semanticCases) {
|
||||
Test-InvalidSemanticIndexing -FixtureRoot (Join-Path $tmp "semantic-$($case.name)") `
|
||||
-CaseName $case.name -Field $case.field -ValidValue $case.valid `
|
||||
-InvalidValue $case.invalid -ExpectedReason $case.reason
|
||||
}
|
||||
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain ('x' * 2000) -MaxBytes 1024
|
||||
} 'Page envelope exceeds' 'oversized page envelope fails'
|
||||
|
||||
$articlePaths = @($index.articles.path | Sort-Object)
|
||||
Assert-Sequence $articlePaths $diskArticlePaths 'exact article path union matches disk'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $Root -IndexPath $indexPath -Paths @($articlePaths[0..8])
|
||||
} 'exceeds MaxArticles=8' 'exact retrieval rejects path batches larger than eight'
|
||||
$samplePaths = @(
|
||||
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||
$knowledge = Join-Path $Root "$layer\knowledge"
|
||||
if (Test-Path -LiteralPath $knowledge) {
|
||||
Get-ChildItem -LiteralPath $knowledge -Recurse -File |
|
||||
Where-Object Name -Match '\.(good|bad)\.[a-z0-9]+$' |
|
||||
ForEach-Object {
|
||||
[IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
|
||||
}
|
||||
}
|
||||
}
|
||||
) | Sort-Object
|
||||
Test-BodyRoundTrip -Paths $articlePaths -IndexPath $indexPath
|
||||
Test-BodyRoundTrip -Paths $samplePaths -IndexPath $indexPath -Samples
|
||||
|
||||
$fixtureRoot = Join-Path $tmp 'neutral'
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer microsoft -Slug default
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer community -Slug versioned -Version '"27.."' -Technology javascript -Country dk -Area finance -Title 'Versioned neutral example'
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer custom -Slug localized -Version 28 -Technology al -Country de -Area service -Title 'Localized neutral example'
|
||||
$fixtureIndex = Join-Path $tmp 'neutral-index.json'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex | Out-Null
|
||||
|
||||
foreach ($case in @(
|
||||
@{ name = 'uppercase'; layers = @('Microsoft'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'duplicate'; layers = @('microsoft', 'microsoft'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'unknown'; layers = @('partner'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'null'; layers = $null; pattern = 'must be an array' }
|
||||
)) {
|
||||
Test-InvalidEnabledLayers -FixtureRoot $fixtureRoot -CaseName $case.name `
|
||||
-Layers $case.layers -ExpectedPattern $case.pattern
|
||||
}
|
||||
$subsetIndex = Join-Path $tmp 'community-only-index.json'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $subsetIndex `
|
||||
-EnabledLayers @('community') | Out-Null
|
||||
$subset = & $search -BCQualityRoot $fixtureRoot -IndexPath $subsetIndex `
|
||||
-Domain neutral -EnabledLayers @('community') |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $subset.candidateCount 1 'valid EnabledLayers subset builds and is consumable'
|
||||
Assert-Sequence $subset.candidates.path @('community/knowledge/neutral/versioned.md') 'valid subset contains only its exact layer'
|
||||
|
||||
$applicable = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
BCVersion = 28
|
||||
Technologies = @('al', 'javascript')
|
||||
Countries = @('dk', 'de')
|
||||
ApplicationAreas = @('finance', 'service')
|
||||
} -MaxBytes 16000
|
||||
Assert-Equal $applicable.candidates.Count 3 'neutral layer/version rows all survive matching context'
|
||||
Assert-True (@($applicable.candidates | Where-Object applicability -CEQ applicable).Count -eq 3) 'matching rows are applicable'
|
||||
$versioned = $applicable.candidates | Where-Object path -CEQ 'community/knowledge/neutral/versioned.md'
|
||||
Assert-Equal $versioned.layer community 'non-default layer survives'
|
||||
Assert-Sequence $versioned.'bc-version' @('"27.."') 'original version metadata survives'
|
||||
Assert-Equal $versioned.applicability applicable 'lowercase sentinels and positive open range remain applicable'
|
||||
Assert-Sequence $versioned.technologies @('javascript') 'non-default technology survives'
|
||||
Assert-Sequence $versioned.countries @('dk') 'non-default country survives'
|
||||
Assert-Sequence $versioned.'application-area' @('finance') 'non-default application area survives'
|
||||
|
||||
$conditional = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
BCVersion = 28
|
||||
Technologies = @('al', 'javascript')
|
||||
} -MaxBytes 16000
|
||||
$conditionalVersioned = $conditional.candidates |
|
||||
Where-Object path -CEQ 'community/knowledge/neutral/versioned.md'
|
||||
Assert-Equal $conditionalVersioned.applicability conditional 'unknown context produces conditional verdict'
|
||||
Assert-Sequence $conditionalVersioned.unknownDimensions @('countries', 'application-area') 'unknown dimensions survive'
|
||||
|
||||
$layerFiltered = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
EnabledLayers = @('microsoft')
|
||||
} -MaxBytes 16000
|
||||
Assert-Equal $layerFiltered.candidates.Count 1 'enabled layer remains a candidate'
|
||||
Assert-Equal $layerFiltered.excluded.Count 2 'disabled layers remain explicit'
|
||||
Assert-Sequence ($layerFiltered.excluded.layer | Sort-Object) @('community', 'custom') 'excluded rows preserve layer'
|
||||
|
||||
$oldSnapshot = $conditional.snapshot
|
||||
Add-Content -LiteralPath (Join-Path $fixtureRoot 'community\knowledge\neutral\versioned.md') -Value ' ' -Encoding utf8NoBOM
|
||||
$preparedCatalog = & $search -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $preparedCatalog.candidateCount 3 'catalog uses the prepared index without rehashing article bodies'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex `
|
||||
-Paths 'community/knowledge/neutral/versioned.md'
|
||||
} 'Selected article hash does not match the prepared index' 'exact retrieval detects selected article changes'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex -Domain neutral -Offset 1 -Snapshot $oldSnapshot
|
||||
} 'Snapshot changed' 'continuation cannot cross rebuilt snapshots'
|
||||
|
||||
$largeRoot = Join-Path $tmp 'large-catalog'
|
||||
New-NeutralArticle -FixtureRoot $largeRoot -Layer microsoft -Slug huge-title -Title ('T' * 3000)
|
||||
$largeIndex = Join-Path $tmp 'large-index.json'
|
||||
& $generator -BCQualityRoot $largeRoot -IndexPath $largeIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $largeRoot -IndexPath $largeIndex -Domain neutral -MaxBytes 1024
|
||||
} 'One complete candidates row|Page envelope exceeds' 'oversized catalog row fails without clipping'
|
||||
|
||||
$bodyRoot = Join-Path $tmp 'body-failures'
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug huge-body -Description ('x' * 3000)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug broken-link
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug continuation-one -Description ('a' * 300)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug continuation-two -Description ('b' * 300)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug invalid-utf8
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug sample-one
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug sample-two
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-one.md') `
|
||||
-Value '[`sample-one.good.al`](sample-one.good.al)' -Encoding utf8NoBOM
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.md') `
|
||||
-Value '[`sample-two.good.al`](sample-two.good.al)' -Encoding utf8NoBOM
|
||||
Set-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-one.good.al') `
|
||||
-Value ('a' * 900) -Encoding utf8NoBOM
|
||||
Set-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.good.al') `
|
||||
-Value ('b' * 900) -Encoding utf8NoBOM
|
||||
$bodyIndex = Join-Path $tmp 'body-index.json'
|
||||
& $generator -BCQualityRoot $bodyRoot -IndexPath $bodyIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/huge-body.md' -MaxBytes 1024
|
||||
} 'No complete body plus continuation fits' 'oversized body fails without truncation'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex -Paths '../outside.md'
|
||||
} 'Invalid knowledge path' 'unsafe requested path fails'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/huge-body.md' -EnabledLayers community
|
||||
} 'Layer disabled' 'disabled article layer fails'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex -Paths @(
|
||||
'microsoft/knowledge/neutral/huge-body.md',
|
||||
'microsoft/knowledge/neutral/huge-body.md'
|
||||
)
|
||||
} 'Duplicate requested path' 'duplicate exact paths fail'
|
||||
|
||||
$brokenSample = Join-Path $bodyRoot 'microsoft\knowledge\neutral\broken-link.good.al'
|
||||
Set-Content -LiteralPath $brokenSample -Value 'codeunit 1 Neutral { }' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/broken-link.good.al' -Samples
|
||||
} 'Sample is not linked' 'unlinked sample fails'
|
||||
|
||||
$sampleContinuationPaths = @(
|
||||
'microsoft/knowledge/neutral/sample-one.good.al',
|
||||
'microsoft/knowledge/neutral/sample-two.good.al'
|
||||
)
|
||||
$firstSamplePage = & $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths $sampleContinuationPaths -Samples -MaxBytes 1600 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $firstSamplePage.complete) 'bounded sample batch produces continuation'
|
||||
Assert-Sequence $firstSamplePage.remainingPaths @('microsoft/knowledge/neutral/sample-two.good.al') 'sample continuation preserves pending path'
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.good.al') `
|
||||
-Value 'changed' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstSamplePage.remainingPaths) -Samples `
|
||||
-Snapshot $firstSamplePage.continuation.snapshot
|
||||
} 'Article snapshot changed' 'sample continuation rejects a changed pending sample'
|
||||
|
||||
$continuationPaths = @(
|
||||
'microsoft/knowledge/neutral/continuation-one.md',
|
||||
'microsoft/knowledge/neutral/continuation-two.md'
|
||||
)
|
||||
$firstBodyPage = & $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths $continuationPaths -MaxBytes 1300 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $firstBodyPage.complete) 'bounded article batch produces continuation'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstBodyPage.remainingPaths) -Snapshot ('0' * 64)
|
||||
} 'Article snapshot changed' 'wrong article continuation snapshot fails'
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\continuation-two.md') -Value 'changed' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstBodyPage.remainingPaths) -Snapshot $firstBodyPage.continuation.snapshot
|
||||
} 'Selected article hash does not match the prepared index' 'article continuation rejects a changed remaining body'
|
||||
|
||||
$invalidUtf8 = Join-Path $bodyRoot 'microsoft\knowledge\neutral\invalid-utf8.md'
|
||||
$indexedBytes = [IO.File]::ReadAllBytes($invalidUtf8)
|
||||
[IO.File]::WriteAllBytes($invalidUtf8, [byte[]]@($indexedBytes + @(0xc3, 0x28)))
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/invalid-utf8.md'
|
||||
} 'Knowledge file is not valid strict UTF-8' 'invalid UTF-8 fails'
|
||||
|
||||
Write-Host "Knowledge retrieval check PASSED: $($articlePaths.Count) articles and $($samplePaths.Count) samples round-tripped; catalog union was lossless and bounded." -ForegroundColor Green
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue