mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Merge current main into development guidance
Reconcile the read-only guidance output with the machine-readable skill index, adopt linked sample references required by bounded retrieval, and update the guidance regression fixture for the retrieval helper dependency. Permit only the known endpoint-DLP metadata stream during read-only evidence capture. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 638b66d2-9f06-4f60-8781-808709e1485c
This commit is contained in:
Jesper Schulz-Wedde
commit
8f025ac679
127 changed files with 5251 additions and 136 deletions
117
tools/Bounded-Results.ps1
Normal file
117
tools/Bounded-Results.ps1
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
# Shared deterministic paging. Callers build the complete immutable result first.
|
||||
#requires -Version 7.2
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Get-ResultSnapshot {
|
||||
param([Parameter(Mandatory)] $Value)
|
||||
|
||||
$json = ConvertTo-Json -InputObject $Value -Depth 30 -Compress
|
||||
return [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($json))
|
||||
).ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Get-SerializedByteCount {
|
||||
param([Parameter(Mandatory)] [string] $Json)
|
||||
|
||||
# PowerShell writes one platform newline after the returned JSON string.
|
||||
return [Text.Encoding]::UTF8.GetByteCount($Json) +
|
||||
[Text.Encoding]::UTF8.GetByteCount([Environment]::NewLine)
|
||||
}
|
||||
|
||||
function ConvertTo-BoundedPage {
|
||||
param(
|
||||
[Parameter(Mandatory)] [Collections.IDictionary] $Header,
|
||||
[Parameter(Mandatory)] [Collections.IDictionary] $Groups,
|
||||
[ValidateRange(0, 2147483647)] [int] $Offset = 0,
|
||||
[string] $Snapshot,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000
|
||||
)
|
||||
|
||||
$total = 0
|
||||
foreach ($name in $Groups.Keys) {
|
||||
$total += $Groups[$name].Count
|
||||
}
|
||||
if (($total -eq 0 -and $Offset -ne 0) -or ($total -gt 0 -and $Offset -ge $total)) {
|
||||
throw "Invalid Offset=$Offset for totalCount=$total; no rows were returned."
|
||||
}
|
||||
if ($Offset -gt 0 -and -not $Snapshot) {
|
||||
throw 'Continuation requires Snapshot from the preceding page.'
|
||||
}
|
||||
if ($Snapshot -and $Snapshot -cne $Header.snapshot) {
|
||||
throw 'Snapshot changed or continuation belongs to another request. Discard partial results and restart at Offset=0.'
|
||||
}
|
||||
|
||||
$page = [ordered]@{}
|
||||
foreach ($key in $Header.Keys) {
|
||||
$page[$key] = $Header[$key]
|
||||
}
|
||||
$page.offset = $Offset
|
||||
$page.returnedCount = 0
|
||||
$page.totalCount = $total
|
||||
$page.remainingCount = $total - $Offset
|
||||
$page.complete = ($total -eq 0)
|
||||
$page.continuation = if ($total) {
|
||||
[ordered]@{ offset = $Offset; snapshot = $Header.snapshot }
|
||||
}
|
||||
else {
|
||||
$null
|
||||
}
|
||||
foreach ($name in $Groups.Keys) {
|
||||
$page[$name] = [Collections.Generic.List[object]]::new()
|
||||
}
|
||||
|
||||
$json = ConvertTo-Json -InputObject $page -Depth 30 -Compress
|
||||
if ((Get-SerializedByteCount -Json $json) -gt $MaxBytes) {
|
||||
throw "Page envelope exceeds MaxBytes=$MaxBytes. Use READ's path-discovery fallback; never truncate."
|
||||
}
|
||||
|
||||
$position = 0
|
||||
foreach ($name in $Groups.Keys) {
|
||||
foreach ($row in $Groups[$name]) {
|
||||
if ($position++ -lt $Offset) {
|
||||
continue
|
||||
}
|
||||
|
||||
$page[$name].Add($row)
|
||||
$page.returnedCount++
|
||||
$page.remainingCount--
|
||||
$page.complete = ($page.remainingCount -eq 0)
|
||||
$page.continuation = if ($page.complete) {
|
||||
$null
|
||||
}
|
||||
else {
|
||||
[ordered]@{
|
||||
offset = $Offset + $page.returnedCount
|
||||
snapshot = $Header.snapshot
|
||||
}
|
||||
}
|
||||
|
||||
$next = ConvertTo-Json -InputObject $page -Depth 30 -Compress
|
||||
if ((Get-SerializedByteCount -Json $next) -gt $MaxBytes) {
|
||||
$page[$name].RemoveAt($page[$name].Count - 1)
|
||||
$page.returnedCount--
|
||||
$page.remainingCount++
|
||||
$page.complete = $false
|
||||
$page.continuation = [ordered]@{
|
||||
offset = $Offset + $page.returnedCount
|
||||
snapshot = $Header.snapshot
|
||||
}
|
||||
if ($page.returnedCount -eq 0) {
|
||||
$rowPath = $null
|
||||
if ($row -is [Collections.IDictionary]) {
|
||||
if ($row.Contains('path')) { $rowPath = $row['path'] }
|
||||
}
|
||||
elseif ($null -ne $row -and $row.PSObject.Properties['path']) {
|
||||
$rowPath = $row.PSObject.Properties['path'].Value
|
||||
}
|
||||
$identity = if ($rowPath) { " at $rowPath" } else { " at Offset=$Offset" }
|
||||
throw "One complete $name row plus envelope exceeds MaxBytes=$MaxBytes$identity. No row was clipped."
|
||||
}
|
||||
return $json
|
||||
}
|
||||
$json = $next
|
||||
}
|
||||
}
|
||||
return $json
|
||||
}
|
||||
|
|
@ -30,6 +30,8 @@
|
|||
expected to prune its clone to policy first). For provenance and to
|
||||
reproduce a consumer's exact view, pass -EnabledLayers to restrict the walk
|
||||
to those layers and to record the policy in the index header.
|
||||
Invalid articles are omitted with a path-specific warning so one bad
|
||||
optional layer article cannot block valid siblings.
|
||||
|
||||
.PARAMETER BCQualityRoot
|
||||
Path to the BCQuality content root to index (typically a filtered clone).
|
||||
|
|
@ -69,6 +71,17 @@ param(
|
|||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
|
||||
if ($PSBoundParameters.ContainsKey('EnabledLayers')) {
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array; omit it to index all layers.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
}
|
||||
|
||||
# Default to the clone root (parent of this script's tools/ folder) so the
|
||||
# agent's Entry preparation step can invoke this with no arguments from the
|
||||
|
|
@ -93,6 +106,45 @@ function Get-RelativePath {
|
|||
return ($rel -replace '\\', '/')
|
||||
}
|
||||
|
||||
function Get-BytesSha256 {
|
||||
param([byte[]] $Bytes)
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($Bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Read-ArticleSource {
|
||||
param([string] $Path)
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
try {
|
||||
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||
}
|
||||
catch [Text.DecoderFallbackException] {
|
||||
throw [IO.InvalidDataException]::new('invalid UTF-8', $_.Exception)
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
bytes = $bytes
|
||||
text = $text
|
||||
sha256 = Get-BytesSha256 -Bytes $bytes
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ValueSha256 {
|
||||
param([Parameter(Mandatory)] $Value)
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject $Value -Depth 8 -Compress))
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
# Trims a Description to a single short line (<= $Max chars) for the lean
|
||||
# index. Takes the first sentence; truncates on a word boundary if still long.
|
||||
function Get-LeanDescription {
|
||||
|
|
@ -116,9 +168,12 @@ function ConvertFrom-ArticleFrontmatter {
|
|||
# Pattern) is included; the index is a lossless substitute for the
|
||||
# frontmatter + Description the worklist predicate reads, not a
|
||||
# substitute for the article's normative guidance.
|
||||
param([string] $Path)
|
||||
param(
|
||||
[string] $Path,
|
||||
[string] $Text
|
||||
)
|
||||
|
||||
$lines = Get-Content -LiteralPath $Path -ErrorAction Stop
|
||||
$lines = [regex]::Split($Text.TrimStart([char]0xfeff), '\r\n|\n|\r')
|
||||
|
||||
# Frontmatter is the first '---'-delimited block.
|
||||
if ($lines.Count -lt 1 -or $lines[0].Trim() -ne '---') { return $null }
|
||||
|
|
@ -129,19 +184,42 @@ function ConvertFrom-ArticleFrontmatter {
|
|||
if ($fmEnd -lt 0) { return $null }
|
||||
|
||||
$fm = @{}
|
||||
$arrayFields = @('bc-version', 'keywords', 'technologies', 'countries', 'application-area')
|
||||
for ($i = 1; $i -lt $fmEnd; $i++) {
|
||||
$line = $lines[$i]
|
||||
if ($line -match '^\s*([a-zA-Z][\w-]*)\s*:\s*(.*)$') {
|
||||
$key = $Matches[1]
|
||||
$val = $Matches[2].Trim()
|
||||
if ($val -match '^\[(.*)\]$') {
|
||||
if ($key -in $arrayFields) {
|
||||
if ($val -notmatch '^\[(.*)\]$') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$inner = $Matches[1].Trim()
|
||||
if ($inner -eq '') { $fm[$key] = @() }
|
||||
else { $fm[$key] = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() }) }
|
||||
if ($inner -eq '') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$values = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() })
|
||||
if (@($values | Where-Object { [string]::IsNullOrWhiteSpace($_) }).Count) {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$key' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
$fm[$key] = $values
|
||||
}
|
||||
elseif ($val -ne '') { $fm[$key] = $val }
|
||||
}
|
||||
}
|
||||
foreach ($field in $arrayFields) {
|
||||
if (-not $fm.ContainsKey($field) -or $fm[$field] -isnot [array] -or -not $fm[$field].Count) {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"frontmatter field '$field' must use non-empty bracket-array syntax"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
# Body parsing: H1 title and the full Description section. The Description
|
||||
# is the article's primary retrieval target per READ and is captured
|
||||
|
|
@ -185,42 +263,71 @@ $indexArticles = [System.Collections.Generic.List[object]]::new()
|
|||
foreach ($layerDir in @('microsoft', 'community', 'custom')) {
|
||||
$kbRoot = Join-Path $BCQualityRoot (Join-Path $layerDir 'knowledge')
|
||||
if (-not (Test-Path $kbRoot)) { continue }
|
||||
if ($EnabledLayers -and ($EnabledLayers -notcontains $layerDir)) { continue }
|
||||
if ($EnabledLayers -and ($EnabledLayers -cnotcontains $layerDir)) { continue }
|
||||
|
||||
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName |
|
||||
ForEach-Object {
|
||||
$rel = Get-RelativePath -Root $BCQualityRoot -Full $_.FullName
|
||||
$parsed = $null
|
||||
try { $parsed = ConvertFrom-ArticleFrontmatter -Path $_.FullName } catch { $parsed = $null }
|
||||
$files = @(
|
||||
Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName
|
||||
)
|
||||
foreach ($file in $files) {
|
||||
$rel = Get-RelativePath -Root $BCQualityRoot -Full $file.FullName
|
||||
try {
|
||||
$source = Read-ArticleSource -Path $file.FullName
|
||||
$parsed = ConvertFrom-ArticleFrontmatter -Path $file.FullName -Text $source.text
|
||||
if (-not $parsed) {
|
||||
# Invalid/unparseable file: list path + domain-from-path so it
|
||||
# is never silently dropped from discovery. Consumers fall back
|
||||
# to reading it in full.
|
||||
$domainFromPath = if ($rel -match '/knowledge/([^/]+)/') { $Matches[1] } else { '' }
|
||||
$indexArticles.Add([pscustomobject]@{
|
||||
path = $rel; layer = $layerDir; domain = $domainFromPath
|
||||
'bc-version' = @(); technologies = @(); countries = @(); 'application-area' = @()
|
||||
keywords = @(); title = ''; description = ''; parsed = $false
|
||||
}) | Out-Null
|
||||
return
|
||||
throw [IO.InvalidDataException]::new('missing or unterminated frontmatter')
|
||||
}
|
||||
foreach ($required in @(
|
||||
@('domain', $parsed.domain),
|
||||
@('H1 title', $parsed.title),
|
||||
@('Description', $parsed.description)
|
||||
)) {
|
||||
if ([string]::IsNullOrWhiteSpace([string]$required[1])) {
|
||||
throw [IO.InvalidDataException]::new("missing $($required[0])")
|
||||
}
|
||||
}
|
||||
$indexArticles.Add([pscustomobject]@{
|
||||
path = $rel
|
||||
layer = $layerDir
|
||||
domain = $parsed.domain
|
||||
'bc-version' = @($parsed.'bc-version')
|
||||
technologies = @($parsed.technologies)
|
||||
countries = @($parsed.countries)
|
||||
'application-area' = @($parsed.'application-area')
|
||||
keywords = @($parsed.keywords)
|
||||
title = $parsed.title
|
||||
description = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
|
||||
parsed = $true
|
||||
}) | Out-Null
|
||||
}
|
||||
catch [IO.InvalidDataException] {
|
||||
Write-Warning "Skipping invalid knowledge article '$rel': $($_.Exception.Message)."
|
||||
continue
|
||||
}
|
||||
|
||||
$article = [ordered]@{
|
||||
path = $rel
|
||||
layer = $layerDir
|
||||
domain = $parsed.domain
|
||||
'bc-version' = @($parsed.'bc-version')
|
||||
technologies = @($parsed.technologies)
|
||||
countries = @($parsed.countries)
|
||||
'application-area' = @($parsed.'application-area')
|
||||
keywords = @($parsed.keywords)
|
||||
title = $parsed.title
|
||||
description = if ($FullIndex) { $parsed.description } else { Get-LeanDescription -Text $parsed.description }
|
||||
parsed = $true
|
||||
sourceSha256 = $source.sha256
|
||||
}
|
||||
$problem = Get-KnowledgeMetadataProblem -Row $article
|
||||
if ($problem) {
|
||||
Write-Warning "Skipping invalid knowledge article '$rel': $problem."
|
||||
continue
|
||||
}
|
||||
$indexArticles.Add($article) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$articlesByPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($article in $indexArticles) {
|
||||
if (-not $articlesByPath.TryAdd($article.path, $article)) {
|
||||
throw "Duplicate knowledge path while building source snapshot: $($article.path)"
|
||||
}
|
||||
}
|
||||
$sourcePaths = [string[]]@($articlesByPath.Keys)
|
||||
[Array]::Sort($sourcePaths, [StringComparer]::Ordinal)
|
||||
$sourceManifest = @(
|
||||
foreach ($path in $sourcePaths) {
|
||||
[ordered]@{ path = $path; sha256 = $articlesByPath[$path].sourceSha256 }
|
||||
}
|
||||
)
|
||||
$index = [pscustomobject]@{
|
||||
version = 1
|
||||
generatedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
|
@ -228,6 +335,7 @@ $index = [pscustomobject]@{
|
|||
knowledgeAllow= @($KnowledgeAllow)
|
||||
knowledgeDeny = @($KnowledgeDeny)
|
||||
articleCount = $indexArticles.Count
|
||||
sourceSnapshot= Get-ValueSha256 -Value $sourceManifest
|
||||
articles = @($indexArticles)
|
||||
}
|
||||
|
||||
|
|
|
|||
247
tools/Build-SkillIndex.ps1
Normal file
247
tools/Build-SkillIndex.ps1
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Builds the machine-readable BCQuality action-skill index.
|
||||
|
||||
.DESCRIPTION
|
||||
Action-skill frontmatter remains the source of truth. This script emits the
|
||||
versioned JSON contract orchestrators consume so they do not need to parse
|
||||
Markdown or duplicate composition rules.
|
||||
|
||||
.PARAMETER BCQualityRoot
|
||||
BCQuality repository or filtered content root.
|
||||
|
||||
.PARAMETER IndexPath
|
||||
Output path. Defaults to <BCQualityRoot>/skill-index.json.
|
||||
|
||||
.OUTPUTS
|
||||
Returns the number of indexed action skills.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $BCQualityRoot,
|
||||
[string] $IndexPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not $BCQualityRoot) {
|
||||
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $BCQualityRoot -PathType Container)) {
|
||||
throw "BCQuality root not found: $BCQualityRoot"
|
||||
}
|
||||
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'skill-index.json'
|
||||
}
|
||||
|
||||
function Get-RelativePath {
|
||||
param([string] $Root, [string] $Full)
|
||||
|
||||
return ($Full.Substring($Root.Length).TrimStart([char]'/', [char]'\') -replace '\\', '/')
|
||||
}
|
||||
|
||||
function Get-Sha256 {
|
||||
param([byte[]] $Bytes)
|
||||
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return ([BitConverter]::ToString($sha.ComputeHash($Bytes)) -replace '-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ValueSha256 {
|
||||
param([Parameter(Mandatory)] $Value)
|
||||
|
||||
return Get-Sha256 -Bytes ([Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $Value -Depth 12 -Compress)
|
||||
))
|
||||
}
|
||||
|
||||
function ConvertFrom-SkillFrontmatter {
|
||||
param(
|
||||
[string] $Path,
|
||||
[string] $Text
|
||||
)
|
||||
|
||||
$lines = [regex]::Split($Text.TrimStart([char]0xfeff), '\r\n|\n|\r')
|
||||
if ($lines.Count -lt 3 -or $lines[0].Trim() -ne '---') {
|
||||
throw [IO.InvalidDataException]::new("Missing frontmatter in '$Path'.")
|
||||
}
|
||||
|
||||
$end = -1
|
||||
for ($i = 1; $i -lt $lines.Count; $i++) {
|
||||
if ($lines[$i].Trim() -eq '---') {
|
||||
$end = $i
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($end -lt 0) {
|
||||
throw [IO.InvalidDataException]::new("Unterminated frontmatter in '$Path'.")
|
||||
}
|
||||
|
||||
$frontmatter = [ordered]@{}
|
||||
for ($i = 1; $i -lt $end; $i++) {
|
||||
$line = $lines[$i]
|
||||
if ($line -notmatch '^([a-zA-Z][\w-]*)\s*:\s*(.*)$') {
|
||||
continue
|
||||
}
|
||||
|
||||
$key = $Matches[1]
|
||||
$value = $Matches[2].Trim()
|
||||
if ($value -eq '') {
|
||||
$items = [System.Collections.Generic.List[string]]::new()
|
||||
while ($i + 1 -lt $end -and $lines[$i + 1] -match '^\s+-\s+(.+?)\s*$') {
|
||||
$i++
|
||||
$items.Add($Matches[1].Trim().Trim('"', "'")) | Out-Null
|
||||
}
|
||||
$frontmatter[$key] = @($items)
|
||||
continue
|
||||
}
|
||||
|
||||
if ($value -match '^\[(.*)\]$') {
|
||||
$inner = $Matches[1].Trim()
|
||||
$values = [System.Collections.Generic.List[object]]::new()
|
||||
if ($inner) {
|
||||
foreach ($item in $inner -split '\s*,\s*') {
|
||||
$normalized = $item.Trim().Trim('"', "'")
|
||||
$number = 0
|
||||
if ($key -eq 'bc-version' -and [int]::TryParse($normalized, [ref]$number)) {
|
||||
$values.Add($number) | Out-Null
|
||||
}
|
||||
else {
|
||||
$values.Add($normalized) | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
$frontmatter[$key] = [object[]]@($values)
|
||||
continue
|
||||
}
|
||||
|
||||
$frontmatter[$key] = $value.Trim('"', "'")
|
||||
}
|
||||
|
||||
return $frontmatter
|
||||
}
|
||||
|
||||
$records = [System.Collections.Generic.List[object]]::new()
|
||||
$recordsByPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
$sourceManifest = [System.Collections.Generic.List[object]]::new()
|
||||
|
||||
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||
$skillsRoot = Join-Path $BCQualityRoot (Join-Path $layer 'skills')
|
||||
if (-not (Test-Path -LiteralPath $skillsRoot -PathType Container)) {
|
||||
continue
|
||||
}
|
||||
|
||||
foreach ($file in Get-ChildItem -LiteralPath $skillsRoot -Recurse -File -Filter '*.md' | Sort-Object FullName) {
|
||||
$bytes = [IO.File]::ReadAllBytes($file.FullName)
|
||||
try {
|
||||
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||
}
|
||||
catch [Text.DecoderFallbackException] {
|
||||
throw [IO.InvalidDataException]::new("Invalid UTF-8 in '$($file.FullName)'.", $_.Exception)
|
||||
}
|
||||
|
||||
$frontmatter = ConvertFrom-SkillFrontmatter -Path $file.FullName -Text $text
|
||||
if ($frontmatter['kind'] -ne 'action-skill') {
|
||||
continue
|
||||
}
|
||||
|
||||
foreach ($required in 'id', 'version', 'title', 'description', 'inputs', 'outputs') {
|
||||
if (-not $frontmatter.Contains($required) -or $null -eq $frontmatter[$required] -or
|
||||
([string]$frontmatter[$required]).Trim() -eq '') {
|
||||
throw [IO.InvalidDataException]::new(
|
||||
"Action skill '$($file.FullName)' is missing required frontmatter '$required'."
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
$path = Get-RelativePath -Root $BCQualityRoot -Full $file.FullName
|
||||
$sourceSha256 = Get-Sha256 -Bytes $bytes
|
||||
$version = 0
|
||||
if (-not [int]::TryParse([string]$frontmatter['version'], [ref]$version) -or $version -le 0) {
|
||||
throw [IO.InvalidDataException]::new("Action skill '$path' has an invalid version.")
|
||||
}
|
||||
|
||||
$subSkills = @()
|
||||
if ($frontmatter.Contains('sub-skills')) {
|
||||
$subSkills = @($frontmatter['sub-skills'])
|
||||
if (-not $subSkills.Count) {
|
||||
throw [IO.InvalidDataException]::new("Super-skill '$path' has an empty sub-skills list.")
|
||||
}
|
||||
}
|
||||
|
||||
$record = [pscustomobject][ordered]@{
|
||||
path = $path
|
||||
layer = $layer
|
||||
id = [string]$frontmatter['id']
|
||||
version = $version
|
||||
title = [string]$frontmatter['title']
|
||||
description = [string]$frontmatter['description']
|
||||
inputs = [string[]]@($frontmatter['inputs'])
|
||||
outputs = [string[]]@($frontmatter['outputs'])
|
||||
filters = [ordered]@{
|
||||
'bc-version' = [object[]]$(if ($frontmatter.Contains('bc-version')) { $frontmatter['bc-version'] })
|
||||
technologies = [string[]]$(if ($frontmatter.Contains('technologies')) { $frontmatter['technologies'] })
|
||||
countries = [string[]]$(if ($frontmatter.Contains('countries')) { $frontmatter['countries'] })
|
||||
'application-area' = [string[]]$(if ($frontmatter.Contains('application-area')) { $frontmatter['application-area'] })
|
||||
}
|
||||
subSkills = [string[]]$subSkills
|
||||
sourceSha256 = $sourceSha256
|
||||
}
|
||||
|
||||
if (-not $recordsByPath.TryAdd($path, $record)) {
|
||||
throw "Duplicate action-skill path: $path"
|
||||
}
|
||||
$records.Add($record) | Out-Null
|
||||
$sourceManifest.Add([ordered]@{ path = $path; sha256 = $sourceSha256 }) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$ids = @($records | Group-Object id | Where-Object Count -gt 1)
|
||||
if ($ids.Count) {
|
||||
throw "Duplicate action-skill IDs: $($ids.Name -join ', ')"
|
||||
}
|
||||
|
||||
foreach ($record in $records) {
|
||||
$seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($subSkillPath in @($record.subSkills)) {
|
||||
if (-not $seen.Add($subSkillPath)) {
|
||||
throw "Super-skill '$($record.path)' declares duplicate sub-skill '$subSkillPath'."
|
||||
}
|
||||
if (-not $recordsByPath.ContainsKey($subSkillPath)) {
|
||||
throw "Super-skill '$($record.path)' references missing action skill '$subSkillPath'."
|
||||
}
|
||||
|
||||
$leaf = $recordsByPath[$subSkillPath]
|
||||
if (@($leaf.subSkills).Count) {
|
||||
throw "Nested super-skills are not supported: '$($record.path)' references '$subSkillPath'."
|
||||
}
|
||||
if (@($leaf.outputs).Count -ne 1 -or $leaf.outputs[0] -ne 'findings-report') {
|
||||
throw "Sub-skill '$subSkillPath' must produce findings-report."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$index = [ordered]@{
|
||||
version = 1
|
||||
generatedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
skillCount = $records.Count
|
||||
sourceSnapshot = Get-ValueSha256 -Value @($sourceManifest)
|
||||
skills = @($records)
|
||||
}
|
||||
|
||||
$parent = Split-Path -Parent $IndexPath
|
||||
if ($parent -and -not (Test-Path -LiteralPath $parent)) {
|
||||
New-Item -ItemType Directory -Path $parent -Force | Out-Null
|
||||
}
|
||||
Set-Content -LiteralPath $IndexPath -Value (
|
||||
ConvertTo-Json -InputObject $index -Depth 12 -Compress
|
||||
) -Encoding utf8NoBOM
|
||||
|
||||
return $records.Count
|
||||
|
|
@ -80,24 +80,15 @@ function Assert-GuidanceItem {
|
|||
if (($Item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $Item.LinkType -or $Item.LinkTarget) {
|
||||
throw 'Links, junctions, hard links and reparse points are not supported.'
|
||||
}
|
||||
}
|
||||
|
||||
function Get-GuidanceStreams {
|
||||
param([string] $Path)
|
||||
if (-not $IsWindows) { return @() }
|
||||
return @(
|
||||
Get-Item -LiteralPath $Path -Stream '*' -Force -ErrorAction Stop |
|
||||
Where-Object Stream -ne ':$DATA' |
|
||||
Sort-Object Stream -CaseSensitive |
|
||||
ForEach-Object {
|
||||
$streamPath = "$($_.FileName):$($_.Stream)"
|
||||
[ordered]@{
|
||||
name = $_.Stream
|
||||
length = $_.Length
|
||||
sha256 = Get-GuidanceHash $streamPath
|
||||
}
|
||||
}
|
||||
)
|
||||
if (-not $Item.PSIsContainer -and $IsWindows) {
|
||||
$unsupportedStreams = @(
|
||||
Get-Item -LiteralPath $Item.FullName -Stream '*' -Force -ErrorAction Stop |
|
||||
Where-Object Stream -notin @(':$DATA', 'sec.endpointdlp')
|
||||
)
|
||||
if ($unsupportedStreams.Count) {
|
||||
throw 'Alternate data streams are not supported.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-GuidanceSafePath {
|
||||
|
|
@ -210,7 +201,6 @@ function Get-GuidanceSnapshot {
|
|||
$entry.length = $item.Length
|
||||
$entry.lastWriteUtcTicks = $item.LastWriteTimeUtc.Ticks
|
||||
$entry.sha256 = Get-GuidanceHash $item.FullName
|
||||
$entry.streams = @(Get-GuidanceStreams $item.FullName)
|
||||
}
|
||||
$files.Add($entry)
|
||||
}
|
||||
|
|
|
|||
187
tools/Get-KnowledgeArticles.ps1
Normal file
187
tools/Get-KnowledgeArticles.ps1
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Reads a bounded prefix of exact article or sample paths without altering bodies.
|
||||
.DESCRIPTION
|
||||
The UTF-8 byte size bound covers the complete serialized JSON plus its output
|
||||
newline. A body that cannot fit fails explicitly; it is never summarized or
|
||||
truncated. Samples are loaded only with -Samples and must be linked by their
|
||||
sibling article using READ's exact link convention.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateNotNullOrEmpty()] [string] $BCQualityRoot = (Split-Path $PSScriptRoot -Parent),
|
||||
[Parameter(Mandatory)] [ValidateNotNullOrEmpty()] [string[]] $Paths,
|
||||
[ValidateRange(1, 8)] [int] $MaxArticles = 8,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000,
|
||||
[ValidateSet('microsoft', 'community', 'custom')]
|
||||
[AllowEmptyCollection()] [string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
|
||||
[string] $IndexPath,
|
||||
[ValidatePattern('^[a-f0-9]{64}$')] [string] $Snapshot,
|
||||
[switch] $Samples
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
. (Join-Path $PSScriptRoot 'Bounded-Results.ps1')
|
||||
|
||||
$BCQualityRoot = Resolve-KnowledgeRoot $BCQualityRoot
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'knowledge-index.json'
|
||||
}
|
||||
if ($Paths.Count -gt $MaxArticles) {
|
||||
throw "Paths count $($Paths.Count) exceeds MaxArticles=$MaxArticles. Split the worklist into stable chunks of at most $MaxArticles exact paths."
|
||||
}
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
|
||||
$recovery = "Run Entry preparation once before dispatch, or use READ's bounded native-file fallback. Do not rebuild in a leaf."
|
||||
$preparedIndex = Read-PreparedKnowledgeIndex -IndexPath $IndexPath -Recovery $recovery
|
||||
$index = $preparedIndex.index
|
||||
$byPath = $preparedIndex.byPath
|
||||
$unrestricted = $index.enabledLayers.Count -eq 0 -or
|
||||
($index.enabledLayers.Count -eq 1 -and $null -eq $index.enabledLayers[0])
|
||||
$indexedLayers = @(
|
||||
if ($unrestricted) { 'microsoft', 'community', 'custom' } else { $index.enabledLayers }
|
||||
)
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin $indexedLayers }).Count) {
|
||||
throw "Index layer coverage does not cover EnabledLayers. $recovery"
|
||||
}
|
||||
|
||||
$resolved = [Collections.Generic.List[string]]::new()
|
||||
$records = [Collections.Generic.List[object]]::new()
|
||||
$seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
$articleTexts = [Collections.Generic.Dictionary[string, string]]::new([StringComparer]::Ordinal)
|
||||
$sampleContents = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($path in $Paths) {
|
||||
$kind = if ($Samples) { 'sample' } else { 'article' }
|
||||
$fullPath = Resolve-KnowledgePath -Root $BCQualityRoot -Path $path -Kind $kind
|
||||
if ($path.Split('/')[0] -cnotin $EnabledLayers) {
|
||||
throw "Layer disabled for path: $path"
|
||||
}
|
||||
if (-not $seen.Add($path)) {
|
||||
throw "Duplicate requested path: $path"
|
||||
}
|
||||
if ($Samples) {
|
||||
$articlePath = $path -replace '\.(good|bad)\.[a-z0-9]+$', '.md'
|
||||
if (-not $byPath.ContainsKey($articlePath)) {
|
||||
throw "Sample article is absent from the prepared index: $articlePath"
|
||||
}
|
||||
$fullArticlePath = Resolve-KnowledgePath -Root $BCQualityRoot -Path $articlePath
|
||||
if (-not $articleTexts.ContainsKey($articlePath)) {
|
||||
$articleContent = Read-KnowledgeText -Path $fullArticlePath
|
||||
if ($articleContent.sha256 -cne $byPath[$articlePath].sourceSha256) {
|
||||
throw "Selected article hash does not match the prepared index: $articlePath"
|
||||
}
|
||||
$articleTexts.Add($articlePath, $articleContent.text)
|
||||
}
|
||||
Assert-SampleLink -ArticleText $articleTexts[$articlePath] -SamplePath $fullPath
|
||||
$sampleContent = Read-KnowledgeText -Path $fullPath
|
||||
$sampleContents.Add($path, $sampleContent)
|
||||
$records.Add([ordered]@{
|
||||
path = $path
|
||||
articlePath = $articlePath
|
||||
articleSha256 = $byPath[$articlePath].sourceSha256
|
||||
sampleSha256 = $sampleContent.sha256
|
||||
sampleBytes = $sampleContent.bytes
|
||||
})
|
||||
}
|
||||
else {
|
||||
if (-not $byPath.ContainsKey($path)) {
|
||||
throw "Selected article is absent from the prepared index: $path"
|
||||
}
|
||||
$records.Add([ordered]@{
|
||||
path = $path
|
||||
expectedSha256 = $byPath[$path].sourceSha256
|
||||
})
|
||||
}
|
||||
$resolved.Add($fullPath)
|
||||
}
|
||||
|
||||
$requestSnapshot = Get-ResultSnapshot -Value ([ordered]@{
|
||||
root = $BCQualityRoot
|
||||
preparedIndexSha256 = $preparedIndex.content.sha256
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
enabledLayers = @($EnabledLayers)
|
||||
files = @($records)
|
||||
})
|
||||
if ($Snapshot -and $Snapshot -cne $requestSnapshot) {
|
||||
throw 'Article snapshot changed or continuation belongs to another exact path batch. Discard partial results and restart.'
|
||||
}
|
||||
|
||||
$articles = [Collections.Generic.List[object]]::new()
|
||||
function ConvertTo-BatchJson {
|
||||
param([int] $ReadCount)
|
||||
|
||||
$remaining = @(
|
||||
if ($ReadCount -lt $Paths.Count) {
|
||||
$Paths[$ReadCount..($Paths.Count - 1)]
|
||||
}
|
||||
)
|
||||
$remainingRecords = @(
|
||||
if ($ReadCount -lt $records.Count) {
|
||||
$records[$ReadCount..($records.Count - 1)]
|
||||
}
|
||||
)
|
||||
$continuation = if ($remaining.Count) {
|
||||
[ordered]@{
|
||||
snapshot = Get-ResultSnapshot -Value ([ordered]@{
|
||||
root = $BCQualityRoot
|
||||
preparedIndexSha256 = $preparedIndex.content.sha256
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
enabledLayers = @($EnabledLayers)
|
||||
files = $remainingRecords
|
||||
})
|
||||
}
|
||||
}
|
||||
else {
|
||||
$null
|
||||
}
|
||||
return [ordered]@{
|
||||
version = 1
|
||||
kind = if ($Samples) { 'samples' } else { 'articles' }
|
||||
snapshot = $requestSnapshot
|
||||
requestedCount = $Paths.Count
|
||||
returnedCount = $ReadCount
|
||||
complete = ($ReadCount -eq $Paths.Count)
|
||||
articles = @($articles)
|
||||
remainingPaths = $remaining
|
||||
continuation = $continuation
|
||||
} | ConvertTo-Json -Depth 8 -Compress
|
||||
}
|
||||
|
||||
$json = ''
|
||||
for ($i = 0; $i -lt [Math]::Min($MaxArticles, $Paths.Count); $i++) {
|
||||
$content = if ($Samples) {
|
||||
$sampleContents[$Paths[$i]]
|
||||
}
|
||||
else {
|
||||
Read-KnowledgeText -Path $resolved[$i]
|
||||
}
|
||||
if (-not $Samples -and $content.sha256 -cne $records[$i].expectedSha256) {
|
||||
throw "Selected article hash does not match the prepared index: $($Paths[$i])"
|
||||
}
|
||||
$articles.Add([ordered]@{
|
||||
path = $Paths[$i]
|
||||
bytes = $content.bytes
|
||||
sha256 = $content.sha256
|
||||
body = $content.text
|
||||
})
|
||||
$next = ConvertTo-BatchJson -ReadCount ($i + 1)
|
||||
if ((Get-SerializedByteCount -Json $next) -gt $MaxBytes) {
|
||||
$articles.RemoveAt($articles.Count - 1)
|
||||
if ($i -eq 0) {
|
||||
throw "No complete body plus continuation fits MaxBytes=$MaxBytes at $($Paths[$i]). Use a smaller exact path batch or READ's bounded native-file fallback; never truncate."
|
||||
}
|
||||
break
|
||||
}
|
||||
$json = $next
|
||||
}
|
||||
|
||||
$json
|
||||
298
tools/Knowledge-Retrieval.ps1
Normal file
298
tools/Knowledge-Retrieval.ps1
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
# Shared filesystem guards for catalog and exact article retrieval.
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Resolve-KnowledgeRoot {
|
||||
param([string] $Root)
|
||||
|
||||
$item = Get-Item -LiteralPath $Root -Force -ErrorAction Stop
|
||||
if ($item.PSProvider.Name -ne 'FileSystem' -or -not $item.PSIsContainer) {
|
||||
throw "BCQuality root must be a filesystem directory: $Root"
|
||||
}
|
||||
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "Linked BCQuality roots are not supported: $Root"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Assert-KnowledgePath {
|
||||
param(
|
||||
[string] $Path,
|
||||
[ValidateSet('article', 'sample')] [string] $Kind = 'article'
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Path) -or
|
||||
$Path -cnotmatch '^(microsoft|community|custom)/knowledge/[^/]+/.+' -or
|
||||
$Path -match '[\\:*?"<>|\x00-\x1f]' -or
|
||||
@($Path.Split('/') | Where-Object { $_ -in '', '.', '..' -or $_ -match '[. ]$' }).Count) {
|
||||
throw "Invalid knowledge path: $Path"
|
||||
}
|
||||
if (($Kind -eq 'article' -and -not $Path.EndsWith('.md', [StringComparison]::Ordinal)) -or
|
||||
($Kind -eq 'sample' -and $Path -cnotmatch '\.(good|bad)\.[a-z0-9]+$')) {
|
||||
throw "Expected an exact $Kind path: $Path"
|
||||
}
|
||||
}
|
||||
|
||||
function Resolve-KnowledgePath {
|
||||
param(
|
||||
[string] $Root,
|
||||
[string] $Path,
|
||||
[ValidateSet('article', 'sample')] [string] $Kind = 'article'
|
||||
)
|
||||
|
||||
Assert-KnowledgePath -Path $Path -Kind $Kind
|
||||
$current = $Root
|
||||
foreach ($part in $Path.Split('/')) {
|
||||
$items = @(
|
||||
Get-ChildItem -LiteralPath $current -Filter $part -Force -ErrorAction Stop |
|
||||
Where-Object Name -CEQ $part
|
||||
)
|
||||
if ($items.Count -ne 1) {
|
||||
throw "Knowledge path does not exist with exact casing: $Path"
|
||||
}
|
||||
$item = $items[0]
|
||||
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "Linked knowledge paths are not supported: $Path"
|
||||
}
|
||||
$current = $item.FullName
|
||||
}
|
||||
if ($item.PSIsContainer) {
|
||||
throw "Knowledge path is not a file: $Path"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Read-KnowledgeText {
|
||||
param([string] $Path)
|
||||
|
||||
$bytes = [IO.File]::ReadAllBytes($Path)
|
||||
try {
|
||||
$text = [Text.UTF8Encoding]::new($false, $true).GetString($bytes)
|
||||
}
|
||||
catch {
|
||||
throw "Knowledge file is not valid strict UTF-8: $Path"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
text = $text
|
||||
bytes = $bytes.Length
|
||||
sha256 = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($bytes)
|
||||
).ToLowerInvariant()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NormalizedKnowledgeVersions {
|
||||
param([string[]] $Values)
|
||||
|
||||
foreach ($value in $Values) {
|
||||
if ($value -match '^"([^"]*)"$' -or $value -match "^'([^']*)'$") {
|
||||
$Matches[1]
|
||||
}
|
||||
else {
|
||||
$value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-KnowledgeMetadataProblem {
|
||||
param([Collections.IDictionary] $Row)
|
||||
|
||||
if ($Row['parsed'] -isnot [bool] -or -not $Row['parsed']) {
|
||||
return 'unparsed frontmatter'
|
||||
}
|
||||
if ($Row['domain'] -isnot [string] -or
|
||||
$Row['domain'] -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$') {
|
||||
return 'missing/invalid domain'
|
||||
}
|
||||
foreach ($field in @('bc-version', 'technologies', 'countries', 'application-area', 'keywords')) {
|
||||
if ($Row[$field] -isnot [array] -or -not $Row[$field].Count) {
|
||||
return "missing/invalid $field"
|
||||
}
|
||||
foreach ($value in $Row[$field]) {
|
||||
if ($value -isnot [string] -or [string]::IsNullOrWhiteSpace($value)) {
|
||||
return "invalid $field value"
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach ($field in @('title', 'description')) {
|
||||
if ($Row[$field] -isnot [string] -or
|
||||
[string]::IsNullOrWhiteSpace($Row[$field]) -or
|
||||
$Row[$field] -match '[\r\n]') {
|
||||
return "missing/invalid $field"
|
||||
}
|
||||
}
|
||||
|
||||
$versions = @(Get-NormalizedKnowledgeVersions -Values $Row['bc-version'])
|
||||
if ($versions -ccontains 'all') {
|
||||
if ($versions.Count -ne 1) {
|
||||
return 'mixed bc-version sentinel'
|
||||
}
|
||||
}
|
||||
elseif ($versions.Count -eq 1 -and $versions[0] -match '^(\d+)\.\.(\d+)?$') {
|
||||
$start = [bigint]::Parse($Matches[1])
|
||||
if ($start -le 0 -or ($Matches[2] -and [bigint]::Parse($Matches[2]) -le 0)) {
|
||||
return 'invalid bc-version range bound'
|
||||
}
|
||||
if ($Matches[2] -and $start -gt [bigint]::Parse($Matches[2])) {
|
||||
return 'descending bc-version range'
|
||||
}
|
||||
}
|
||||
else {
|
||||
foreach ($version in $versions) {
|
||||
if ($version -notmatch '^\d+$' -or [bigint]::Parse($version) -le 0) {
|
||||
return 'invalid bc-version'
|
||||
}
|
||||
}
|
||||
}
|
||||
if (@($Row.technologies | Where-Object { $_ -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$' }).Count) {
|
||||
return 'invalid technologies'
|
||||
}
|
||||
if ($Row.technologies -ccontains 'all') {
|
||||
return 'invalid technologies sentinel'
|
||||
}
|
||||
if ($Row.countries -ccontains 'w1') {
|
||||
if ($Row.countries.Count -ne 1) {
|
||||
return 'mixed countries sentinel'
|
||||
}
|
||||
}
|
||||
elseif (@($Row.countries | Where-Object { $_ -cnotmatch '^[a-z]{2}$' }).Count) {
|
||||
return 'invalid countries'
|
||||
}
|
||||
if (@($Row['application-area'] | Where-Object { $_ -cnotmatch '^(all|[a-z0-9]+(-[a-z0-9]+)*)$' }).Count) {
|
||||
return 'invalid application-area'
|
||||
}
|
||||
if ($Row['application-area'] -ccontains 'all' -and $Row['application-area'].Count -ne 1) {
|
||||
return 'mixed application-area sentinel'
|
||||
}
|
||||
if (@($Row.keywords | Where-Object { $_ -cnotmatch '^[a-z0-9]+(-[a-z0-9]+)*$' }).Count) {
|
||||
return 'invalid keywords'
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function Get-PreparedManifestSha256 {
|
||||
param(
|
||||
[string[]] $Paths,
|
||||
[Collections.Generic.Dictionary[string, object]] $ByPath
|
||||
)
|
||||
|
||||
$hash = [Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
$hash.AppendData([byte[]][char]'[')
|
||||
for ($i = 0; $i -lt $Paths.Count; $i++) {
|
||||
if ($i) {
|
||||
$hash.AppendData([byte[]][char]',')
|
||||
}
|
||||
$row = [ordered]@{
|
||||
path = $Paths[$i]
|
||||
sha256 = $ByPath[$Paths[$i]].sourceSha256
|
||||
}
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $row -Depth 8 -Compress)
|
||||
))
|
||||
}
|
||||
$hash.AppendData([byte[]][char]']')
|
||||
return [Convert]::ToHexString($hash.GetHashAndReset()).ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$hash.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Read-PreparedKnowledgeIndex {
|
||||
param(
|
||||
[string] $IndexPath,
|
||||
[string] $Recovery
|
||||
)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $IndexPath -PathType Leaf)) {
|
||||
throw "Knowledge index missing: $IndexPath. $Recovery"
|
||||
}
|
||||
$indexItem = Get-Item -LiteralPath $IndexPath -Force -ErrorAction Stop
|
||||
if ($indexItem.PSProvider.Name -ne 'FileSystem' -or
|
||||
($indexItem.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "Knowledge index must be an unlinked filesystem file: $IndexPath. $Recovery"
|
||||
}
|
||||
|
||||
$content = Read-KnowledgeText -Path $indexItem.FullName
|
||||
try {
|
||||
$index = $content.text.TrimStart([char]0xfeff) |
|
||||
ConvertFrom-Json -AsHashtable -ErrorAction Stop
|
||||
}
|
||||
catch {
|
||||
throw "Malformed knowledge index JSON: $($_.Exception.Message). $Recovery"
|
||||
}
|
||||
if ($index -isnot [Collections.IDictionary] -or
|
||||
$index.version -ne 1 -or
|
||||
$index.articles -isnot [array] -or
|
||||
$index.articleCount -ne $index.articles.Count -or
|
||||
$index.enabledLayers -isnot [array] -or
|
||||
$index.knowledgeAllow -isnot [array] -or
|
||||
$index.knowledgeDeny -isnot [array] -or
|
||||
$index.sourceSnapshot -isnot [string] -or
|
||||
$index.sourceSnapshot -cnotmatch '^[a-f0-9]{64}$') {
|
||||
throw "Invalid knowledge index envelope. $Recovery"
|
||||
}
|
||||
|
||||
$generatedAt = [DateTimeOffset]::MinValue
|
||||
if ($index.generatedAt -is [DateTime]) {
|
||||
$generatedAt = [DateTimeOffset]$index.generatedAt
|
||||
}
|
||||
elseif (-not [DateTimeOffset]::TryParse(
|
||||
[string]$index.generatedAt,
|
||||
[Globalization.CultureInfo]::InvariantCulture,
|
||||
[Globalization.DateTimeStyles]::RoundtripKind,
|
||||
[ref]$generatedAt
|
||||
)) {
|
||||
throw "Invalid knowledge index generatedAt. $Recovery"
|
||||
}
|
||||
if ($generatedAt -gt [DateTimeOffset]::UtcNow.AddMinutes(1)) {
|
||||
throw "Invalid knowledge index generatedAt. $Recovery"
|
||||
}
|
||||
|
||||
$byPath = [Collections.Generic.Dictionary[string, object]]::new([StringComparer]::Ordinal)
|
||||
foreach ($row in $index.articles) {
|
||||
if ($row -isnot [Collections.IDictionary] -or $row.path -isnot [string]) {
|
||||
throw "Index row has no exact path. $Recovery"
|
||||
}
|
||||
Assert-KnowledgePath -Path $row.path
|
||||
if ($row.layer -cne $row.path.Split('/')[0] -or
|
||||
$row.layer -cnotin @('microsoft', 'community', 'custom')) {
|
||||
throw "Invalid index layer: $($row.path). $Recovery"
|
||||
}
|
||||
if ($row.sourceSha256 -isnot [string] -or
|
||||
$row.sourceSha256 -cnotmatch '^[a-f0-9]{64}$') {
|
||||
throw "Invalid source hash in knowledge index: $($row.path). $Recovery"
|
||||
}
|
||||
if (-not $byPath.TryAdd($row.path, $row)) {
|
||||
throw "Duplicate index path: $($row.path). $Recovery"
|
||||
}
|
||||
}
|
||||
|
||||
$paths = [string[]]@($byPath.Keys)
|
||||
[Array]::Sort($paths, [StringComparer]::Ordinal)
|
||||
if ((Get-PreparedManifestSha256 -Paths $paths -ByPath $byPath) -cne $index.sourceSnapshot) {
|
||||
throw "Stale or internally inconsistent prepared index snapshot. $Recovery"
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
index = $index
|
||||
content = $content
|
||||
byPath = $byPath
|
||||
paths = $paths
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-SampleLink {
|
||||
param(
|
||||
[string] $ArticleText,
|
||||
[string] $SamplePath
|
||||
)
|
||||
|
||||
$sampleName = [IO.Path]::GetFileName($SamplePath)
|
||||
$expected = '[`' + $sampleName + '`](' + $sampleName + ')'
|
||||
if (-not $ArticleText.Contains($expected, [StringComparison]::Ordinal)) {
|
||||
throw "Sample is not linked by its article using the READ convention: $sampleName"
|
||||
}
|
||||
}
|
||||
244
tools/Search-Knowledge.ps1
Normal file
244
tools/Search-Knowledge.ps1
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Returns bounded pages of every domain/layer/READ-applicable catalog row.
|
||||
.DESCRIPTION
|
||||
Consumes Entry's prepared index read-only. Results are never ranked, sampled,
|
||||
top-k limited, deduplicated by basename, or narrowed by query text. Omit an
|
||||
unknown task dimension; an explicit empty array is a known empty set.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateNotNullOrEmpty()] [string] $BCQualityRoot = (Split-Path $PSScriptRoot -Parent),
|
||||
[Parameter(Mandatory)] [ValidateNotNullOrEmpty()]
|
||||
[ValidateScript({ -not [string]::IsNullOrWhiteSpace($_) })] [string] $Domain,
|
||||
[ValidateSet('microsoft', 'community', 'custom')]
|
||||
[AllowEmptyCollection()] [string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
|
||||
[ValidateRange(1, 2147483647)] [int] $BCVersion,
|
||||
[AllowEmptyCollection()] [string[]] $Technologies,
|
||||
[AllowEmptyCollection()] [string[]] $Countries,
|
||||
[AllowEmptyCollection()] [string[]] $ApplicationAreas,
|
||||
[switch] $ExcludeConditional,
|
||||
[string] $IndexPath,
|
||||
[ValidateRange(1024, 16000)] [int] $MaxBytes = 16000,
|
||||
[ValidateRange(0, 2147483647)] [int] $Offset = 0,
|
||||
[ValidatePattern('^[a-f0-9]{64}$')] [string] $Snapshot
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'Knowledge-Retrieval.ps1')
|
||||
. (Join-Path $PSScriptRoot 'Bounded-Results.ps1')
|
||||
|
||||
$BCQualityRoot = Resolve-KnowledgeRoot $BCQualityRoot
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'knowledge-index.json'
|
||||
}
|
||||
if ($null -eq $EnabledLayers) {
|
||||
throw 'EnabledLayers must be an array; use an empty array to disable all layers.'
|
||||
}
|
||||
if (@($EnabledLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($EnabledLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count) {
|
||||
throw 'EnabledLayers must contain unique canonical lowercase layer names.'
|
||||
}
|
||||
|
||||
$context = [ordered]@{}
|
||||
foreach ($pair in @(
|
||||
@('BCVersion', 'bc-version'),
|
||||
@('Technologies', 'technologies'),
|
||||
@('Countries', 'countries'),
|
||||
@('ApplicationAreas', 'application-area')
|
||||
)) {
|
||||
if (-not $PSBoundParameters.ContainsKey($pair[0])) {
|
||||
continue
|
||||
}
|
||||
$value = $PSBoundParameters[$pair[0]]
|
||||
if ($null -eq $value) {
|
||||
throw "Omit unknown context; do not pass null for $($pair[0])."
|
||||
}
|
||||
if ($pair[0] -ne 'BCVersion') {
|
||||
foreach ($entry in $value) {
|
||||
if ([string]::IsNullOrWhiteSpace($entry) -or $entry -cne $entry.Trim()) {
|
||||
throw "Invalid context value for $($pair[0]): '$entry'"
|
||||
}
|
||||
}
|
||||
}
|
||||
$context[$pair[1]] = $value
|
||||
}
|
||||
if ($context.Contains('technologies') -and $context['technologies'] -ccontains 'all') {
|
||||
throw "Technologies has no 'all' sentinel. Omit unknown context."
|
||||
}
|
||||
|
||||
$recovery = "Run Entry preparation once before dispatch, or use READ's path-discovery fallback. Do not rebuild in a leaf."
|
||||
$preparedIndex = Read-PreparedKnowledgeIndex -IndexPath $IndexPath -Recovery $recovery
|
||||
$index = $preparedIndex.index
|
||||
$indexContent = $preparedIndex.content
|
||||
$byPath = $preparedIndex.byPath
|
||||
$paths = $preparedIndex.paths
|
||||
|
||||
# The v1 generator historically serialized an omitted EnabledLayers parameter as [null].
|
||||
$unrestricted = $index.enabledLayers.Count -eq 0 -or
|
||||
($index.enabledLayers.Count -eq 1 -and $null -eq $index.enabledLayers[0])
|
||||
$indexedLayers = @(
|
||||
if ($unrestricted) {
|
||||
'microsoft', 'community', 'custom'
|
||||
}
|
||||
else {
|
||||
$index.enabledLayers
|
||||
}
|
||||
)
|
||||
if (@($indexedLayers | Where-Object { $_ -cnotin @('microsoft', 'community', 'custom') }).Count -or
|
||||
@($indexedLayers | Group-Object -CaseSensitive | Where-Object Count -gt 1).Count -or
|
||||
@($EnabledLayers | Where-Object { $_ -cnotin $indexedLayers }).Count) {
|
||||
throw "Index layer coverage does not cover EnabledLayers. $recovery"
|
||||
}
|
||||
|
||||
foreach ($path in $paths) {
|
||||
$row = $byPath[$path]
|
||||
if ($row.layer -cnotin $indexedLayers) {
|
||||
throw "Index row layer is outside index coverage: $path. $recovery"
|
||||
}
|
||||
$problem = Get-KnowledgeMetadataProblem -Row $row
|
||||
if ($problem) {
|
||||
throw "Malformed knowledge index row at ${path}: $problem. $recovery"
|
||||
}
|
||||
}
|
||||
|
||||
$defaults = [ordered]@{
|
||||
'bc-version' = @('all')
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
}
|
||||
$candidates = [Collections.Generic.List[object]]::new()
|
||||
$excluded = [Collections.Generic.List[object]]::new()
|
||||
foreach ($path in $paths) {
|
||||
$row = $byPath[$path]
|
||||
if ($row.domain -cne $Domain) {
|
||||
continue
|
||||
}
|
||||
|
||||
$unknown = [Collections.Generic.List[string]]::new()
|
||||
$matchesContext = $true
|
||||
foreach ($field in $defaults.Keys) {
|
||||
$values = $row[$field]
|
||||
if ($field -eq 'bc-version') {
|
||||
$values = @(Get-NormalizedKnowledgeVersions -Values $values)
|
||||
}
|
||||
$sentinel = switch ($field) {
|
||||
'bc-version' { 'all' }
|
||||
'countries' { 'w1' }
|
||||
'application-area' { 'all' }
|
||||
default { '' }
|
||||
}
|
||||
if ($sentinel -and $values -ccontains $sentinel) {
|
||||
continue
|
||||
}
|
||||
if (-not $context.Contains($field)) {
|
||||
$unknown.Add($field)
|
||||
continue
|
||||
}
|
||||
|
||||
$target = $context[$field]
|
||||
$matched = $false
|
||||
if ($field -eq 'bc-version') {
|
||||
# Compare as bigint on both sides: metadata validation accepts bounds
|
||||
# wider than Int32, and an int left operand would coerce them down.
|
||||
$targetVersion = [bigint]$target
|
||||
if ($values.Count -eq 1 -and $values[0] -match '^(\d+)\.\.(\d+)?$') {
|
||||
$matched = $targetVersion -ge [bigint]::Parse($Matches[1]) -and
|
||||
(-not $Matches[2] -or $targetVersion -le [bigint]::Parse($Matches[2]))
|
||||
}
|
||||
else {
|
||||
$matched = @($values | Where-Object { [bigint]::Parse($_) -eq $targetVersion }).Count -gt 0
|
||||
}
|
||||
}
|
||||
else {
|
||||
$matched = @($values | Where-Object { $target -ccontains $_ }).Count -gt 0
|
||||
}
|
||||
if (-not $matched) {
|
||||
$matchesContext = $false
|
||||
break
|
||||
}
|
||||
}
|
||||
if (-not $matchesContext -or ($ExcludeConditional -and $unknown.Count)) {
|
||||
continue
|
||||
}
|
||||
$null = Resolve-KnowledgePath -Root $BCQualityRoot -Path $path
|
||||
|
||||
$candidate = [ordered]@{
|
||||
path = $path
|
||||
layer = $row.layer
|
||||
keywords = $row.keywords
|
||||
title = $row.title
|
||||
description = $row.description
|
||||
}
|
||||
foreach ($field in $defaults.Keys) {
|
||||
if (($row[$field] -join "`0") -cne ($defaults[$field] -join "`0")) {
|
||||
$candidate[$field] = $row[$field]
|
||||
}
|
||||
}
|
||||
$candidate.applicability = if ($unknown.Count) { 'conditional' } else { 'applicable' }
|
||||
$candidate.unknownDimensions = @($unknown)
|
||||
if ($row.layer -cin $EnabledLayers) {
|
||||
$candidates.Add($candidate)
|
||||
}
|
||||
else {
|
||||
$excluded.Add($candidate)
|
||||
}
|
||||
}
|
||||
|
||||
$header = [ordered]@{
|
||||
version = 2
|
||||
domain = $Domain
|
||||
context = $context
|
||||
enabledLayers = @($EnabledLayers)
|
||||
indexedLayers = @($indexedLayers)
|
||||
excludeConditional = [bool]$ExcludeConditional
|
||||
defaults = $defaults
|
||||
candidateCount = $candidates.Count
|
||||
excludedByConfigurationCount = $excluded.Count
|
||||
}
|
||||
|
||||
function Get-CatalogSnapshot {
|
||||
param(
|
||||
[string] $PreparedIndexSha256,
|
||||
[Collections.IDictionary] $Request,
|
||||
[Collections.IDictionary] $Groups
|
||||
)
|
||||
|
||||
$hash = [Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
foreach ($value in @(
|
||||
$PreparedIndexSha256,
|
||||
(ConvertTo-Json -InputObject $Request -Depth 8 -Compress)
|
||||
)) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes($value))
|
||||
$hash.AppendData([byte[]](10))
|
||||
}
|
||||
foreach ($groupName in $Groups.Keys) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes("[$groupName]"))
|
||||
$hash.AppendData([byte[]](10))
|
||||
foreach ($row in $Groups[$groupName]) {
|
||||
$hash.AppendData([Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $row -Depth 8 -Compress)
|
||||
))
|
||||
$hash.AppendData([byte[]](10))
|
||||
}
|
||||
}
|
||||
return [Convert]::ToHexString($hash.GetHashAndReset()).ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$hash.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
$groups = [ordered]@{
|
||||
candidates = $candidates
|
||||
excludedByConfiguration = $excluded
|
||||
}
|
||||
$header.snapshot = Get-CatalogSnapshot -PreparedIndexSha256 $indexContent.sha256 -Request $header -Groups $groups
|
||||
|
||||
ConvertTo-BoundedPage -Header $header -Groups $groups -Offset $Offset -Snapshot $Snapshot -MaxBytes $MaxBytes
|
||||
|
|
@ -159,7 +159,8 @@ function Test-ReportMutation([string] $Name, [scriptblock] $Change, [string] $Di
|
|||
try {
|
||||
[IO.Directory]::CreateDirectory($root) | Out-Null
|
||||
foreach ($reference in @($article, $otherArticle, $sample, $otherArticle.Replace('.md', '.good.al'),
|
||||
'microsoft/skills/development/al-development-plan.md', 'tools/Build-KnowledgeIndex.ps1')) {
|
||||
'microsoft/skills/development/al-development-plan.md', 'tools/Build-KnowledgeIndex.ps1',
|
||||
'tools/Knowledge-Retrieval.ps1')) {
|
||||
$destination = Join-Path $root $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
|
||||
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
|
||||
|
|
@ -169,8 +170,35 @@ try {
|
|||
$publicManifestPath = Join-Path $sourceRoot 'evaluation\development-guidance-fixtures.json'
|
||||
$publicManifest = Read-GuidanceJson $publicManifestPath
|
||||
$publicRoot = Join-Path $scratch 'public-fixture-checkout'
|
||||
$publicReferences = @($publicManifest.skill, 'tools/Build-KnowledgeIndex.ps1', 'evaluation/development-guidance-fixtures.json') +
|
||||
@($publicManifest.cases | ForEach-Object { $_.requiredKnowledge; $_.optionalKnowledge })
|
||||
$publicKnowledgeReferences = @(
|
||||
$publicManifest.cases |
|
||||
ForEach-Object { $_.requiredKnowledge; $_.optionalKnowledge } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
$publicSampleReferences = @(
|
||||
foreach ($articleReference in $publicKnowledgeReferences) {
|
||||
$articlePath = Join-Path $sourceRoot $articleReference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||
$articleDirectory = Split-Path $articleReference -Parent
|
||||
$articleText = [IO.File]::ReadAllText($articlePath)
|
||||
foreach ($match in [regex]::Matches(
|
||||
$articleText,
|
||||
'\[[^\]]+\]\((?<sample>[a-z0-9-]+\.(?:good|bad)\.[a-zA-Z0-9]+)\)'
|
||||
)) {
|
||||
"$($articleDirectory.Replace('\', '/'))/$($match.Groups['sample'].Value)"
|
||||
}
|
||||
}
|
||||
)
|
||||
$publicReferences = @(
|
||||
@(
|
||||
$publicManifest.skill,
|
||||
'tools/Build-KnowledgeIndex.ps1',
|
||||
'tools/Knowledge-Retrieval.ps1',
|
||||
'evaluation/development-guidance-fixtures.json'
|
||||
) +
|
||||
$publicKnowledgeReferences +
|
||||
$publicSampleReferences |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
foreach ($reference in @($publicReferences | Sort-Object -Unique)) {
|
||||
$destination = Join-Path $publicRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
|
||||
|
|
|
|||
|
|
@ -22,8 +22,10 @@
|
|||
stable metadata, Git HEAD/refs/index and ignored/untracked files are compared.
|
||||
Links/reparse points, hard links, external Git storage in targets,
|
||||
submodules and sparse checkouts are rejected rather than followed. Windows
|
||||
alternate data streams are included in the evidence by name, length, and
|
||||
hash; direct stream paths remain rejected. Run in quiescent repositories.
|
||||
alternate data streams are rejected except for the known endpoint-DLP
|
||||
metadata stream `sec.endpointdlp`, which is ignored because endpoint
|
||||
protection may add or refresh it asynchronously without changing file
|
||||
content. Direct stream paths remain rejected. Run in quiescent repositories.
|
||||
The knowledge checkout may itself be a linked Git worktree; its Git storage
|
||||
identity is recorded explicitly.
|
||||
#>
|
||||
|
|
|
|||
788
tools/Test-KnowledgeRetrieval.ps1
Normal file
788
tools/Test-KnowledgeRetrieval.ps1
Normal file
|
|
@ -0,0 +1,788 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates lossless bounded catalog and exact-body retrieval.
|
||||
#>
|
||||
#requires -Version 7.2
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..'))
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
|
||||
$generator = Join-Path $Root 'tools/Build-KnowledgeIndex.ps1'
|
||||
$search = Join-Path $Root 'tools/Search-Knowledge.ps1'
|
||||
$getArticles = Join-Path $Root 'tools/Get-KnowledgeArticles.ps1'
|
||||
$utf8 = [Text.UTF8Encoding]::new($false, $true)
|
||||
|
||||
function Assert-True {
|
||||
param([bool] $Condition, [string] $Message)
|
||||
if (-not $Condition) {
|
||||
throw "Assertion failed: $Message"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Equal {
|
||||
param($Actual, $Expected, [string] $Message)
|
||||
if ($Actual -cne $Expected) {
|
||||
throw "Assertion failed: $Message. Expected '$Expected', got '$Actual'."
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Sequence {
|
||||
param($Actual, $Expected, [string] $Message)
|
||||
$actualJson = ConvertTo-Json -InputObject @($Actual) -Compress
|
||||
$expectedJson = ConvertTo-Json -InputObject @($Expected) -Compress
|
||||
if ($actualJson -cne $expectedJson) {
|
||||
throw "Assertion failed: $Message. Expected $expectedJson, got $actualJson."
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Throws {
|
||||
param([scriptblock] $Action, [string] $Pattern, [string] $Message)
|
||||
try {
|
||||
& $Action
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Message -notmatch $Pattern) {
|
||||
throw "Assertion failed: $Message. Wrong error: $($_.Exception.Message)"
|
||||
}
|
||||
return
|
||||
}
|
||||
throw "Assertion failed: $Message. No error was thrown."
|
||||
}
|
||||
|
||||
function Get-OutputByteCount {
|
||||
param([string] $Text)
|
||||
return [Text.Encoding]::UTF8.GetByteCount($Text) +
|
||||
[Text.Encoding]::UTF8.GetByteCount([Environment]::NewLine)
|
||||
}
|
||||
|
||||
function Invoke-CatalogPages {
|
||||
param(
|
||||
[hashtable] $Arguments,
|
||||
[int] $MaxBytes = 4096
|
||||
)
|
||||
|
||||
$allCandidates = [Collections.Generic.List[object]]::new()
|
||||
$allExcluded = [Collections.Generic.List[object]]::new()
|
||||
$offset = 0
|
||||
$snapshot = ''
|
||||
$shared = ''
|
||||
$pageCount = 0
|
||||
$lastPage = $null
|
||||
do {
|
||||
$pageArguments = @{} + $Arguments
|
||||
$pageArguments.MaxBytes = $MaxBytes
|
||||
$pageArguments.Offset = $offset
|
||||
if ($snapshot) {
|
||||
$pageArguments.Snapshot = $snapshot
|
||||
}
|
||||
$raw = & $search @pageArguments
|
||||
Assert-True ($raw -is [string]) 'catalog helper emitted exactly one JSON string'
|
||||
Assert-True ((Get-OutputByteCount -Text $raw) -le $MaxBytes) 'catalog page includes its newline in MaxBytes'
|
||||
$page = $raw | ConvertFrom-Json
|
||||
$pageCount++
|
||||
Assert-True ($pageCount -le 1000) 'catalog continuation terminates'
|
||||
Assert-Equal $page.offset $offset 'catalog offset is exact'
|
||||
Assert-Equal $page.returnedCount (@($page.candidates).Count + @($page.excludedByConfiguration).Count) 'page returnedCount matches rows'
|
||||
Assert-Equal $page.remainingCount ($page.totalCount - $offset - $page.returnedCount) 'page remainingCount is exact'
|
||||
|
||||
$currentShared = [ordered]@{
|
||||
version = $page.version
|
||||
domain = $page.domain
|
||||
context = $page.context
|
||||
enabledLayers = $page.enabledLayers
|
||||
indexedLayers = $page.indexedLayers
|
||||
excludeConditional = $page.excludeConditional
|
||||
defaults = $page.defaults
|
||||
candidateCount = $page.candidateCount
|
||||
excludedByConfigurationCount = $page.excludedByConfigurationCount
|
||||
snapshot = $page.snapshot
|
||||
totalCount = $page.totalCount
|
||||
} | ConvertTo-Json -Depth 8 -Compress
|
||||
if (-not $shared) {
|
||||
$shared = $currentShared
|
||||
$snapshot = $page.snapshot
|
||||
}
|
||||
else {
|
||||
Assert-Equal $currentShared $shared 'catalog pages repeat shared context, defaults, totals, and snapshot'
|
||||
}
|
||||
|
||||
foreach ($row in @($page.candidates)) {
|
||||
$allCandidates.Add($row)
|
||||
}
|
||||
foreach ($row in @($page.excludedByConfiguration)) {
|
||||
$allExcluded.Add($row)
|
||||
}
|
||||
if (-not $page.complete) {
|
||||
Assert-True ($null -ne $page.continuation) 'incomplete page has continuation'
|
||||
Assert-Equal $page.continuation.snapshot $snapshot 'continuation is snapshot-bound'
|
||||
Assert-True ($page.continuation.offset -gt $offset) 'continuation makes progress'
|
||||
$offset = $page.continuation.offset
|
||||
}
|
||||
$lastPage = $page
|
||||
} while (-not $page.complete)
|
||||
|
||||
Assert-True ($null -eq $lastPage.continuation) 'final page has no continuation'
|
||||
Assert-Equal $allCandidates.Count $lastPage.candidateCount 'candidate total survives paging'
|
||||
Assert-Equal $allExcluded.Count $lastPage.excludedByConfigurationCount 'excluded total survives paging'
|
||||
return [pscustomobject]@{
|
||||
candidates = @($allCandidates)
|
||||
excluded = @($allExcluded)
|
||||
pages = $pageCount
|
||||
snapshot = $snapshot
|
||||
lastPage = $lastPage
|
||||
}
|
||||
}
|
||||
|
||||
function Test-BodyRoundTrip {
|
||||
param(
|
||||
[string[]] $Paths,
|
||||
[string] $IndexPath,
|
||||
[switch] $Samples
|
||||
)
|
||||
|
||||
$seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
for ($start = 0; $start -lt $Paths.Count; $start += 8) {
|
||||
$end = [Math]::Min($start + 7, $Paths.Count - 1)
|
||||
$remaining = @($Paths[$start..$end])
|
||||
$snapshot = ''
|
||||
do {
|
||||
$arguments = @{
|
||||
BCQualityRoot = $Root
|
||||
IndexPath = $IndexPath
|
||||
Paths = $remaining
|
||||
MaxArticles = 8
|
||||
MaxBytes = 16000
|
||||
}
|
||||
if ($Samples) {
|
||||
$arguments.Samples = $true
|
||||
}
|
||||
if ($snapshot) {
|
||||
$arguments.Snapshot = $snapshot
|
||||
}
|
||||
$raw = & $getArticles @arguments
|
||||
Assert-True ($raw -is [string]) 'article helper emitted exactly one JSON string'
|
||||
Assert-True ((Get-OutputByteCount -Text $raw) -le 16000) 'article batch includes its newline in MaxBytes'
|
||||
$batch = $raw | ConvertFrom-Json
|
||||
Assert-True ($batch.returnedCount -gt 0) 'article batching makes progress'
|
||||
Assert-Equal $batch.returnedCount @($batch.articles).Count 'article returnedCount matches rows'
|
||||
Assert-Equal $batch.complete (@($batch.remainingPaths).Count -eq 0) 'article completion matches remaining paths'
|
||||
if ($batch.complete) {
|
||||
Assert-True ($null -eq $batch.continuation) 'complete article batch has no continuation'
|
||||
}
|
||||
else {
|
||||
Assert-True ($batch.continuation.snapshot -match '^[a-f0-9]{64}$') 'article continuation is snapshot-bound'
|
||||
}
|
||||
|
||||
foreach ($article in @($batch.articles)) {
|
||||
Assert-True ($seen.Add($article.path)) "body returned once: $($article.path)"
|
||||
$fullPath = Join-Path $Root ($article.path.Replace('/', [IO.Path]::DirectorySeparatorChar))
|
||||
$bytes = [IO.File]::ReadAllBytes($fullPath)
|
||||
$text = $utf8.GetString($bytes)
|
||||
$hash = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($bytes)
|
||||
).ToLowerInvariant()
|
||||
Assert-Equal $article.bytes $bytes.Length "byte count round-trips: $($article.path)"
|
||||
Assert-Equal $article.sha256 $hash "SHA-256 round-trips: $($article.path)"
|
||||
Assert-Equal $article.body $text "body round-trips: $($article.path)"
|
||||
}
|
||||
$remaining = @($batch.remainingPaths)
|
||||
$snapshot = if ($batch.complete) { '' } else { $batch.continuation.snapshot }
|
||||
} while ($remaining.Count)
|
||||
}
|
||||
Assert-Equal $seen.Count $Paths.Count 'every requested body round-trips without loss'
|
||||
}
|
||||
|
||||
function New-NeutralArticle {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $Layer,
|
||||
[string] $Slug,
|
||||
[string] $Version = 'all',
|
||||
[string] $Technology = 'al',
|
||||
[string] $Country = 'w1',
|
||||
[string] $Area = 'all',
|
||||
[string] $Title = 'Neutral retrieval example',
|
||||
[string] $Description = 'Neutral retrieval metadata for deterministic tests.'
|
||||
)
|
||||
|
||||
$directory = Join-Path $FixtureRoot "$Layer\knowledge\neutral"
|
||||
New-Item -ItemType Directory -Force -Path $directory | Out-Null
|
||||
$content = @"
|
||||
---
|
||||
bc-version: [$Version]
|
||||
domain: neutral
|
||||
keywords: [neutral, retrieval, deterministic]
|
||||
technologies: [$Technology]
|
||||
countries: [$Country]
|
||||
application-area: [$Area]
|
||||
---
|
||||
|
||||
# $Title
|
||||
|
||||
## Description
|
||||
|
||||
$Description
|
||||
"@
|
||||
Set-Content -LiteralPath (Join-Path $directory "$Slug.md") -Value $content -Encoding utf8NoBOM
|
||||
}
|
||||
|
||||
function Test-InvalidSourceIndexing {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $Field,
|
||||
[string] $ValidValue,
|
||||
[string] $InvalidValue
|
||||
)
|
||||
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer microsoft -Slug valid-source
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer community -Slug invalid-source
|
||||
$articlePath = Join-Path $FixtureRoot 'community\knowledge\neutral\invalid-source.md'
|
||||
$text = [IO.File]::ReadAllText($articlePath, $utf8)
|
||||
$text = $text.Replace("$Field`: $ValidValue", "$Field`: $InvalidValue")
|
||||
[IO.File]::WriteAllText($articlePath, $text, $utf8)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("$Field-index.json")
|
||||
$generation = @(& $generator -BCQualityRoot $FixtureRoot -IndexPath $indexPath 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 "scalar $Field source emits one omission warning"
|
||||
Assert-True (
|
||||
$warnings[0].Message -match
|
||||
"Skipping invalid knowledge article 'community/knowledge/neutral/invalid-source\.md': frontmatter field '$([regex]::Escape($Field))' must use non-empty bracket-array syntax\."
|
||||
) "scalar $Field warning identifies the exact path and reason"
|
||||
$prepared = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
Assert-Equal $prepared.articleCount 1 "scalar $Field source is omitted while its valid sibling is indexed"
|
||||
Assert-Sequence $prepared.articles.path @('microsoft/knowledge/neutral/valid-source.md') "scalar $Field index contains only the valid sibling"
|
||||
$catalog = & $search -BCQualityRoot $FixtureRoot -IndexPath $indexPath -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $catalog.candidates.path @('microsoft/knowledge/neutral/valid-source.md') "scalar $Field catalog retrieves the valid sibling"
|
||||
$valid = & $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-source.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $valid.complete "scalar $Field valid sibling body retrieves completely"
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'community/knowledge/neutral/invalid-source.md'
|
||||
} 'Selected article is absent from the prepared index' "scalar $Field omitted source cannot be retrieved"
|
||||
}
|
||||
|
||||
function Test-InvalidSemanticIndexing {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $CaseName,
|
||||
[string] $Field,
|
||||
[string] $ValidValue,
|
||||
[string] $InvalidValue,
|
||||
[string] $ExpectedReason
|
||||
)
|
||||
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer microsoft -Slug valid-source
|
||||
New-NeutralArticle -FixtureRoot $FixtureRoot -Layer community -Slug invalid-source
|
||||
$articlePath = Join-Path $FixtureRoot 'community\knowledge\neutral\invalid-source.md'
|
||||
$text = [IO.File]::ReadAllText($articlePath, $utf8)
|
||||
$text = $text.Replace("$Field`: $ValidValue", "$Field`: $InvalidValue")
|
||||
[IO.File]::WriteAllText($articlePath, $text, $utf8)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("$CaseName-index.json")
|
||||
$generation = @(& $generator -BCQualityRoot $FixtureRoot -IndexPath $indexPath 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 "$CaseName emits one omission warning"
|
||||
Assert-Equal $warnings[0].Message "Skipping invalid knowledge article 'community/knowledge/neutral/invalid-source.md': $ExpectedReason." "$CaseName warning identifies exact path and reason"
|
||||
|
||||
$prepared = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
Assert-Equal $prepared.articleCount 1 "$CaseName omits invalid source and retains valid sibling"
|
||||
Assert-Sequence $prepared.articles.path @('microsoft/knowledge/neutral/valid-source.md') "$CaseName index contains only valid sibling"
|
||||
Assert-True ($prepared.sourceSnapshot -match '^[a-f0-9]{64}$') "$CaseName source snapshot remains valid"
|
||||
$validRow = $prepared.articles[0]
|
||||
$manifest = @(
|
||||
[ordered]@{ path = $validRow.path; sha256 = $validRow.sourceSha256 }
|
||||
)
|
||||
$manifestBytes = [Text.Encoding]::UTF8.GetBytes(
|
||||
(ConvertTo-Json -InputObject $manifest -Depth 8 -Compress)
|
||||
)
|
||||
$expectedSnapshot = [Convert]::ToHexString(
|
||||
[Security.Cryptography.SHA256]::HashData($manifestBytes)
|
||||
).ToLowerInvariant()
|
||||
Assert-Equal $prepared.sourceSnapshot $expectedSnapshot "$CaseName source snapshot covers only retained rows"
|
||||
|
||||
$catalog = & $search -BCQualityRoot $FixtureRoot -IndexPath $indexPath -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $catalog.candidates.path @('microsoft/knowledge/neutral/valid-source.md') "$CaseName catalog retains valid sibling"
|
||||
$valid = & $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-source.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $valid.complete "$CaseName valid sibling body retrieves"
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $FixtureRoot -IndexPath $indexPath `
|
||||
-Paths 'community/knowledge/neutral/invalid-source.md'
|
||||
} 'Selected article is absent from the prepared index' "$CaseName invalid source cannot be retrieved"
|
||||
}
|
||||
|
||||
function Test-InvalidEnabledLayers {
|
||||
param(
|
||||
[string] $FixtureRoot,
|
||||
[string] $CaseName,
|
||||
$Layers,
|
||||
[string] $ExpectedPattern
|
||||
)
|
||||
|
||||
$indexPath = Join-Path (Split-Path $FixtureRoot -Parent) ("layers-$CaseName.json")
|
||||
$arguments = @{
|
||||
BCQualityRoot = $FixtureRoot
|
||||
IndexPath = $indexPath
|
||||
EnabledLayers = $Layers
|
||||
}
|
||||
Assert-Throws {
|
||||
& $generator @arguments
|
||||
} $ExpectedPattern "$CaseName EnabledLayers fails"
|
||||
Assert-True (-not (Test-Path -LiteralPath $indexPath)) "$CaseName fails before index creation"
|
||||
}
|
||||
|
||||
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("bcquality_retrieval_" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Force -Path $tmp | Out-Null
|
||||
try {
|
||||
$indexPath = Join-Path $tmp 'knowledge-index.json'
|
||||
& $generator -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$index = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$diskArticlePaths = @(
|
||||
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||
$knowledge = Join-Path $Root "$layer\knowledge"
|
||||
if (Test-Path -LiteralPath $knowledge) {
|
||||
Get-ChildItem -LiteralPath $knowledge -Recurse -File -Filter '*.md' |
|
||||
ForEach-Object {
|
||||
[IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
|
||||
}
|
||||
}
|
||||
}
|
||||
) | Sort-Object
|
||||
Assert-Equal $index.articleCount $diskArticlePaths.Count 'index covers every current article'
|
||||
Assert-True ($index.sourceSnapshot -match '^[a-f0-9]{64}$') 'index carries an exact source snapshot'
|
||||
|
||||
$allCatalogRows = [Collections.Generic.List[object]]::new()
|
||||
$domains = @($index.articles.domain | Sort-Object -Unique)
|
||||
foreach ($domain in $domains) {
|
||||
$catalog = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $Root
|
||||
IndexPath = $indexPath
|
||||
Domain = $domain
|
||||
}
|
||||
Assert-Equal $catalog.excluded.Count 0 "all layers enabled for $domain"
|
||||
foreach ($row in $catalog.candidates) {
|
||||
$allCatalogRows.Add($row)
|
||||
}
|
||||
}
|
||||
|
||||
$expectedRows = @($index.articles | Sort-Object path)
|
||||
$actualRows = @($allCatalogRows | Sort-Object path)
|
||||
Assert-Equal $actualRows.Count $expectedRows.Count 'paged union has no top-k or query-based loss'
|
||||
Assert-Sequence ($actualRows.path) ($expectedRows.path) 'paged union equals all READ-filtered candidates'
|
||||
Assert-Equal @($actualRows.path | Sort-Object -Unique).Count $actualRows.Count 'catalog does not deduplicate distinct paths'
|
||||
|
||||
$defaults = [ordered]@{
|
||||
'bc-version' = @('all')
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
}
|
||||
for ($i = 0; $i -lt $actualRows.Count; $i++) {
|
||||
$actual = $actualRows[$i]
|
||||
$expected = $expectedRows[$i]
|
||||
Assert-Equal $actual.path $expected.path 'catalog preserves exact path'
|
||||
Assert-Equal $actual.layer $expected.layer 'catalog preserves layer'
|
||||
Assert-Sequence $actual.keywords $expected.keywords 'catalog preserves full keywords'
|
||||
Assert-Equal $actual.title $expected.title 'catalog preserves title'
|
||||
Assert-Equal $actual.description $expected.description 'catalog preserves one-line description'
|
||||
|
||||
$unknown = [Collections.Generic.List[string]]::new()
|
||||
foreach ($field in $defaults.Keys) {
|
||||
$expectedValues = @($expected.$field)
|
||||
$sentinel = switch ($field) {
|
||||
'bc-version' { 'all' }
|
||||
'countries' { 'w1' }
|
||||
'application-area' { 'all' }
|
||||
default { '' }
|
||||
}
|
||||
if (-not $sentinel -or $expectedValues -notcontains $sentinel) {
|
||||
$unknown.Add($field)
|
||||
}
|
||||
$hasField = $actual.PSObject.Properties.Name -ccontains $field
|
||||
if (($expectedValues -join "`0") -ceq (@($defaults[$field]) -join "`0")) {
|
||||
Assert-True (-not $hasField) "default field is inherited from page: $field"
|
||||
}
|
||||
else {
|
||||
Assert-True $hasField "non-default field survives paging: $field"
|
||||
Assert-Sequence $actual.$field $expectedValues "non-default field is exact: $field"
|
||||
}
|
||||
}
|
||||
Assert-Equal $actual.applicability ($(if ($unknown.Count) { 'conditional' } else { 'applicable' })) 'applicability verdict is explicit'
|
||||
Assert-Sequence $actual.unknownDimensions @($unknown) 'unknown dimensions are explicit'
|
||||
}
|
||||
|
||||
$performanceFirst = & $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -MaxBytes 4096 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $performanceFirst.complete) 'large domain produces deterministic continuation'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -MaxBytes 4096 -Offset $performanceFirst.continuation.offset
|
||||
} 'Continuation requires Snapshot' 'continuation without snapshot fails'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -Offset $performanceFirst.totalCount
|
||||
} 'Invalid Offset' 'offset at total fails'
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain performance -Offset 1 -Snapshot ('0' * 64)
|
||||
} 'Snapshot changed' 'wrong snapshot fails'
|
||||
|
||||
$changedRawIndex = Join-Path $tmp 'changed-raw-index.json'
|
||||
$changedRaw = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$changedRaw.generatedAt = [DateTimeOffset]::UtcNow.ToString('O')
|
||||
$changedRaw | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $changedRawIndex -Encoding utf8NoBOM -NoNewline
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $changedRawIndex -Domain performance `
|
||||
-MaxBytes 4096 -Offset $performanceFirst.continuation.offset `
|
||||
-Snapshot $performanceFirst.continuation.snapshot
|
||||
} 'Snapshot changed' 'continuation is bound to the exact prepared index bytes'
|
||||
|
||||
$malformedIndex = Join-Path $tmp 'malformed.json'
|
||||
Set-Content -LiteralPath $malformedIndex -Value '{not-json' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $malformedIndex -Domain performance
|
||||
} 'Malformed knowledge index JSON' 'malformed JSON fails'
|
||||
|
||||
$unsafeIndex = Join-Path $tmp 'unsafe.json'
|
||||
$unsafe = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$unsafe.articles[0].path = '../outside.md'
|
||||
$unsafe | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $unsafeIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $unsafeIndex -Domain performance
|
||||
} 'Invalid knowledge path' 'unsafe indexed path fails'
|
||||
|
||||
$invalidRowIndex = Join-Path $tmp 'invalid-row.json'
|
||||
$invalidRow = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$invalidRow.articles[0].keywords = @()
|
||||
$invalidRow | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $invalidRowIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $invalidRowIndex -Domain performance
|
||||
} 'Malformed knowledge index row' 'malformed index row fails'
|
||||
|
||||
$semanticCorruptIndex = Join-Path $tmp 'semantic-corrupt-row.json'
|
||||
$semanticCorrupt = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$semanticCorrupt.articles[0].countries = @('usa')
|
||||
$semanticCorrupt | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $semanticCorruptIndex -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $semanticCorruptIndex -Domain performance
|
||||
} 'Malformed knowledge index row.*invalid countries' 'search rejects semantically invalid external index rows'
|
||||
|
||||
$corruptSemanticCases = @(
|
||||
@{ name = 'uppercase-all'; field = 'bc-version'; value = @('ALL'); reason = 'invalid bc-version' },
|
||||
@{ name = 'uppercase-w1'; field = 'countries'; value = @('W1'); reason = 'invalid countries' },
|
||||
@{ name = 'zero-open-range'; field = 'bc-version'; value = @('"0.."'); reason = 'invalid bc-version range bound' },
|
||||
@{ name = 'zero-closed-range'; field = 'bc-version'; value = @('"0..0"'); reason = 'invalid bc-version range bound' }
|
||||
)
|
||||
foreach ($case in $corruptSemanticCases) {
|
||||
$corruptPath = Join-Path $tmp ("corrupt-$($case.name).json")
|
||||
$corrupt = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
$corrupt.articles[0].PSObject.Properties[$case.field].Value = $case.value
|
||||
$corrupt | ConvertTo-Json -Depth 8 -Compress |
|
||||
Set-Content -LiteralPath $corruptPath -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $corruptPath -Domain performance
|
||||
} "Malformed knowledge index row.*$([regex]::Escape($case.reason))" "search rejects $($case.name) in an external index"
|
||||
}
|
||||
|
||||
$invalidUtf8Root = Join-Path $tmp 'invalid-utf8-source'
|
||||
New-NeutralArticle -FixtureRoot $invalidUtf8Root -Layer microsoft -Slug valid-catalog
|
||||
New-NeutralArticle -FixtureRoot $invalidUtf8Root -Layer community -Slug invalid-utf8-source
|
||||
$invalidUtf8Article = Join-Path $invalidUtf8Root 'community\knowledge\neutral\invalid-utf8-source.md'
|
||||
$validBytes = [IO.File]::ReadAllBytes($invalidUtf8Article)
|
||||
[IO.File]::WriteAllBytes($invalidUtf8Article, [byte[]]@($validBytes + @(0xc3, 0x28)))
|
||||
$invalidUtf8Index = Join-Path $tmp 'invalid-utf8-index.json'
|
||||
$generation = @(& $generator -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index 3>&1)
|
||||
$warnings = @($generation | Where-Object { $_ -is [Management.Automation.WarningRecord] })
|
||||
Assert-Equal $warnings.Count 1 'malformed UTF-8 source emits one omission warning'
|
||||
Assert-Equal $warnings[0].Message "Skipping invalid knowledge article 'community/knowledge/neutral/invalid-utf8-source.md': invalid UTF-8." 'malformed UTF-8 warning identifies the exact path and reason'
|
||||
$invalidUtf8Prepared = Get-Content -LiteralPath $invalidUtf8Index -Raw -Encoding utf8 |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $invalidUtf8Prepared.articleCount 1 'malformed UTF-8 source is omitted while its valid sibling is indexed'
|
||||
Assert-Sequence $invalidUtf8Prepared.articles.path @('microsoft/knowledge/neutral/valid-catalog.md') 'malformed UTF-8 index contains only the valid sibling'
|
||||
$validCatalog = & $search -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Sequence $validCatalog.candidates.path @('microsoft/knowledge/neutral/valid-catalog.md') 'catalog retrieves the valid sibling after malformed UTF-8 omission'
|
||||
$validBody = & $getArticles -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index `
|
||||
-Paths 'microsoft/knowledge/neutral/valid-catalog.md' |
|
||||
ConvertFrom-Json
|
||||
Assert-True $validBody.complete 'valid sibling body retrieves after malformed UTF-8 omission'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $invalidUtf8Root -IndexPath $invalidUtf8Index `
|
||||
-Paths 'community/knowledge/neutral/invalid-utf8-source.md'
|
||||
} 'Selected article is absent from the prepared index' 'omitted malformed UTF-8 source cannot be retrieved'
|
||||
|
||||
$scalarCases = @(
|
||||
@{ field = 'bc-version'; valid = '[all]'; invalid = 'all' },
|
||||
@{ field = 'keywords'; valid = '[neutral, retrieval, deterministic]'; invalid = 'neutral' },
|
||||
@{ field = 'technologies'; valid = '[al]'; invalid = 'al' },
|
||||
@{ field = 'countries'; valid = '[w1]'; invalid = 'w1' },
|
||||
@{ field = 'application-area'; valid = '[all]'; invalid = 'all' }
|
||||
)
|
||||
foreach ($case in $scalarCases) {
|
||||
Test-InvalidSourceIndexing -FixtureRoot (Join-Path $tmp "scalar-$($case.field)") `
|
||||
-Field $case.field -ValidValue $case.valid -InvalidValue $case.invalid
|
||||
}
|
||||
|
||||
$semanticCases = @(
|
||||
@{ name = 'mixed-version-sentinel'; field = 'bc-version'; valid = '[all]'; invalid = '[all, 27]'; reason = 'mixed bc-version sentinel' },
|
||||
@{ name = 'invalid-country'; field = 'countries'; valid = '[w1]'; invalid = '[usa]'; reason = 'invalid countries' },
|
||||
@{ name = 'descending-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["28..27"]'; reason = 'descending bc-version range' },
|
||||
@{ name = 'malformed-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '[twenty-seven]'; reason = 'invalid bc-version' },
|
||||
@{ name = 'malformed-keyword'; field = 'keywords'; valid = '[neutral, retrieval, deterministic]'; invalid = '[neutral, Bad_Token, deterministic]'; reason = 'invalid keywords' },
|
||||
@{ name = 'malformed-technology'; field = 'technologies'; valid = '[al]'; invalid = '[AL]'; reason = 'invalid technologies' },
|
||||
@{ name = 'malformed-application-area'; field = 'application-area'; valid = '[all]'; invalid = '[finance_]'; reason = 'invalid application-area' },
|
||||
@{ name = 'uppercase-version-sentinel'; field = 'bc-version'; valid = '[all]'; invalid = '[ALL]'; reason = 'invalid bc-version' },
|
||||
@{ name = 'uppercase-country-sentinel'; field = 'countries'; valid = '[w1]'; invalid = '[W1]'; reason = 'invalid countries' },
|
||||
@{ name = 'zero-open-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["0.."]'; reason = 'invalid bc-version range bound' },
|
||||
@{ name = 'zero-closed-version-range'; field = 'bc-version'; valid = '[all]'; invalid = '["0..0"]'; reason = 'invalid bc-version range bound' }
|
||||
)
|
||||
foreach ($case in $semanticCases) {
|
||||
Test-InvalidSemanticIndexing -FixtureRoot (Join-Path $tmp "semantic-$($case.name)") `
|
||||
-CaseName $case.name -Field $case.field -ValidValue $case.valid `
|
||||
-InvalidValue $case.invalid -ExpectedReason $case.reason
|
||||
}
|
||||
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $Root -IndexPath $indexPath -Domain ('x' * 2000) -MaxBytes 1024
|
||||
} 'Page envelope exceeds' 'oversized page envelope fails'
|
||||
|
||||
$articlePaths = @($index.articles.path | Sort-Object)
|
||||
Assert-Sequence $articlePaths $diskArticlePaths 'exact article path union matches disk'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $Root -IndexPath $indexPath -Paths @($articlePaths[0..8])
|
||||
} 'exceeds MaxArticles=8' 'exact retrieval rejects path batches larger than eight'
|
||||
$samplePaths = @(
|
||||
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||
$knowledge = Join-Path $Root "$layer\knowledge"
|
||||
if (Test-Path -LiteralPath $knowledge) {
|
||||
Get-ChildItem -LiteralPath $knowledge -Recurse -File |
|
||||
Where-Object Name -Match '\.(good|bad)\.[a-z0-9]+$' |
|
||||
ForEach-Object {
|
||||
[IO.Path]::GetRelativePath($Root, $_.FullName).Replace('\', '/')
|
||||
}
|
||||
}
|
||||
}
|
||||
) | Sort-Object
|
||||
Test-BodyRoundTrip -Paths $articlePaths -IndexPath $indexPath
|
||||
Test-BodyRoundTrip -Paths $samplePaths -IndexPath $indexPath -Samples
|
||||
|
||||
$fixtureRoot = Join-Path $tmp 'neutral'
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer microsoft -Slug default
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer community -Slug versioned -Version '"27.."' -Technology javascript -Country dk -Area finance -Title 'Versioned neutral example'
|
||||
New-NeutralArticle -FixtureRoot $fixtureRoot -Layer custom -Slug localized -Version 28 -Technology al -Country de -Area service -Title 'Localized neutral example'
|
||||
$fixtureIndex = Join-Path $tmp 'neutral-index.json'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex | Out-Null
|
||||
|
||||
foreach ($case in @(
|
||||
@{ name = 'uppercase'; layers = @('Microsoft'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'duplicate'; layers = @('microsoft', 'microsoft'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'unknown'; layers = @('partner'); pattern = 'unique canonical lowercase layer names' },
|
||||
@{ name = 'null'; layers = $null; pattern = 'must be an array' }
|
||||
)) {
|
||||
Test-InvalidEnabledLayers -FixtureRoot $fixtureRoot -CaseName $case.name `
|
||||
-Layers $case.layers -ExpectedPattern $case.pattern
|
||||
}
|
||||
$subsetIndex = Join-Path $tmp 'community-only-index.json'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $subsetIndex `
|
||||
-EnabledLayers @('community') | Out-Null
|
||||
$subset = & $search -BCQualityRoot $fixtureRoot -IndexPath $subsetIndex `
|
||||
-Domain neutral -EnabledLayers @('community') |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $subset.candidateCount 1 'valid EnabledLayers subset builds and is consumable'
|
||||
Assert-Sequence $subset.candidates.path @('community/knowledge/neutral/versioned.md') 'valid subset contains only its exact layer'
|
||||
|
||||
$applicable = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
BCVersion = 28
|
||||
Technologies = @('al', 'javascript')
|
||||
Countries = @('dk', 'de')
|
||||
ApplicationAreas = @('finance', 'service')
|
||||
} -MaxBytes 16000
|
||||
Assert-Equal $applicable.candidates.Count 3 'neutral layer/version rows all survive matching context'
|
||||
Assert-True (@($applicable.candidates | Where-Object applicability -CEQ applicable).Count -eq 3) 'matching rows are applicable'
|
||||
$versioned = $applicable.candidates | Where-Object path -CEQ 'community/knowledge/neutral/versioned.md'
|
||||
Assert-Equal $versioned.layer community 'non-default layer survives'
|
||||
Assert-Sequence $versioned.'bc-version' @('"27.."') 'original version metadata survives'
|
||||
Assert-Equal $versioned.applicability applicable 'lowercase sentinels and positive open range remain applicable'
|
||||
Assert-Sequence $versioned.technologies @('javascript') 'non-default technology survives'
|
||||
Assert-Sequence $versioned.countries @('dk') 'non-default country survives'
|
||||
Assert-Sequence $versioned.'application-area' @('finance') 'non-default application area survives'
|
||||
|
||||
# Metadata validation accepts range bounds wider than Int32, so version
|
||||
# matching must compare as bigint rather than coercing the bound down.
|
||||
$wideRoot = Join-Path $tmp 'wide-version'
|
||||
New-NeutralArticle -FixtureRoot $wideRoot -Layer microsoft -Slug wide-closed -Version '"1..99999999999"'
|
||||
New-NeutralArticle -FixtureRoot $wideRoot -Layer microsoft -Slug wide-open -Version '"99999999999.."'
|
||||
$wideIndex = Join-Path $tmp 'wide-version-index.json'
|
||||
& $generator -BCQualityRoot $wideRoot -IndexPath $wideIndex | Out-Null
|
||||
$wide = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $wideRoot
|
||||
IndexPath = $wideIndex
|
||||
Domain = 'neutral'
|
||||
BCVersion = 28
|
||||
} -MaxBytes 16000
|
||||
Assert-Sequence $wide.candidates.path @('microsoft/knowledge/neutral/wide-closed.md') 'bc-version bounds beyond Int32 compare without overflow'
|
||||
|
||||
$conditional = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
BCVersion = 28
|
||||
Technologies = @('al', 'javascript')
|
||||
} -MaxBytes 16000
|
||||
$conditionalVersioned = $conditional.candidates |
|
||||
Where-Object path -CEQ 'community/knowledge/neutral/versioned.md'
|
||||
Assert-Equal $conditionalVersioned.applicability conditional 'unknown context produces conditional verdict'
|
||||
Assert-Sequence $conditionalVersioned.unknownDimensions @('countries', 'application-area') 'unknown dimensions survive'
|
||||
|
||||
$layerFiltered = Invoke-CatalogPages -Arguments @{
|
||||
BCQualityRoot = $fixtureRoot
|
||||
IndexPath = $fixtureIndex
|
||||
Domain = 'neutral'
|
||||
EnabledLayers = @('microsoft')
|
||||
} -MaxBytes 16000
|
||||
Assert-Equal $layerFiltered.candidates.Count 1 'enabled layer remains a candidate'
|
||||
Assert-Equal $layerFiltered.excluded.Count 2 'disabled layers remain explicit'
|
||||
Assert-Sequence ($layerFiltered.excluded.layer | Sort-Object) @('community', 'custom') 'excluded rows preserve layer'
|
||||
|
||||
$oldSnapshot = $conditional.snapshot
|
||||
Add-Content -LiteralPath (Join-Path $fixtureRoot 'community\knowledge\neutral\versioned.md') -Value ' ' -Encoding utf8NoBOM
|
||||
$preparedCatalog = & $search -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex -Domain neutral |
|
||||
ConvertFrom-Json
|
||||
Assert-Equal $preparedCatalog.candidateCount 3 'catalog uses the prepared index without rehashing article bodies'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex `
|
||||
-Paths 'community/knowledge/neutral/versioned.md'
|
||||
} 'Selected article hash does not match the prepared index' 'exact retrieval detects selected article changes'
|
||||
& $generator -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $fixtureRoot -IndexPath $fixtureIndex -Domain neutral -Offset 1 -Snapshot $oldSnapshot
|
||||
} 'Snapshot changed' 'continuation cannot cross rebuilt snapshots'
|
||||
|
||||
$largeRoot = Join-Path $tmp 'large-catalog'
|
||||
New-NeutralArticle -FixtureRoot $largeRoot -Layer microsoft -Slug huge-title -Title ('T' * 3000)
|
||||
$largeIndex = Join-Path $tmp 'large-index.json'
|
||||
& $generator -BCQualityRoot $largeRoot -IndexPath $largeIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $search -BCQualityRoot $largeRoot -IndexPath $largeIndex -Domain neutral -MaxBytes 1024
|
||||
} 'One complete candidates row|Page envelope exceeds' 'oversized catalog row fails without clipping'
|
||||
|
||||
# The shared pager reports the oversized row's identity for any row shape;
|
||||
# a row without a path must still reach its explicit offset-based failure.
|
||||
. (Join-Path $Root 'tools/Bounded-Results.ps1')
|
||||
$pagerHeader = [ordered]@{ version = 2; snapshot = ('0' * 64) }
|
||||
foreach ($shape in @(
|
||||
@{ name = 'dictionary'; row = [ordered]@{ blob = ('x' * 3000) } },
|
||||
@{ name = 'object'; row = [pscustomobject]@{ blob = ('x' * 3000) } }
|
||||
)) {
|
||||
Assert-Throws {
|
||||
ConvertTo-BoundedPage -Header $pagerHeader `
|
||||
-Groups ([ordered]@{ rows = @($shape.row) }) -MaxBytes 1024
|
||||
} 'One complete rows row plus envelope exceeds MaxBytes=1024 at Offset=0' "oversized pathless $($shape.name) row fails with its offset identity"
|
||||
}
|
||||
|
||||
$bodyRoot = Join-Path $tmp 'body-failures'
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug huge-body -Description ('x' * 3000)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug broken-link
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug continuation-one -Description ('a' * 300)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug continuation-two -Description ('b' * 300)
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug invalid-utf8
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug sample-one
|
||||
New-NeutralArticle -FixtureRoot $bodyRoot -Layer microsoft -Slug sample-two
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-one.md') `
|
||||
-Value '[`sample-one.good.al`](sample-one.good.al)' -Encoding utf8NoBOM
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.md') `
|
||||
-Value '[`sample-two.good.al`](sample-two.good.al)' -Encoding utf8NoBOM
|
||||
Set-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-one.good.al') `
|
||||
-Value ('a' * 900) -Encoding utf8NoBOM
|
||||
Set-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.good.al') `
|
||||
-Value ('b' * 900) -Encoding utf8NoBOM
|
||||
$bodyIndex = Join-Path $tmp 'body-index.json'
|
||||
& $generator -BCQualityRoot $bodyRoot -IndexPath $bodyIndex | Out-Null
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/huge-body.md' -MaxBytes 1024
|
||||
} 'No complete body plus continuation fits' 'oversized body fails without truncation'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex -Paths '../outside.md'
|
||||
} 'Invalid knowledge path' 'unsafe requested path fails'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/huge-body.md' -EnabledLayers community
|
||||
} 'Layer disabled' 'disabled article layer fails'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex -Paths @(
|
||||
'microsoft/knowledge/neutral/huge-body.md',
|
||||
'microsoft/knowledge/neutral/huge-body.md'
|
||||
)
|
||||
} 'Duplicate requested path' 'duplicate exact paths fail'
|
||||
|
||||
$brokenSample = Join-Path $bodyRoot 'microsoft\knowledge\neutral\broken-link.good.al'
|
||||
Set-Content -LiteralPath $brokenSample -Value 'codeunit 1 Neutral { }' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/broken-link.good.al' -Samples
|
||||
} 'Sample is not linked' 'unlinked sample fails'
|
||||
|
||||
$sampleContinuationPaths = @(
|
||||
'microsoft/knowledge/neutral/sample-one.good.al',
|
||||
'microsoft/knowledge/neutral/sample-two.good.al'
|
||||
)
|
||||
$firstSamplePage = & $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths $sampleContinuationPaths -Samples -MaxBytes 1600 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $firstSamplePage.complete) 'bounded sample batch produces continuation'
|
||||
Assert-Sequence $firstSamplePage.remainingPaths @('microsoft/knowledge/neutral/sample-two.good.al') 'sample continuation preserves pending path'
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\sample-two.good.al') `
|
||||
-Value 'changed' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstSamplePage.remainingPaths) -Samples `
|
||||
-Snapshot $firstSamplePage.continuation.snapshot
|
||||
} 'Article snapshot changed' 'sample continuation rejects a changed pending sample'
|
||||
|
||||
$continuationPaths = @(
|
||||
'microsoft/knowledge/neutral/continuation-one.md',
|
||||
'microsoft/knowledge/neutral/continuation-two.md'
|
||||
)
|
||||
$firstBodyPage = & $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths $continuationPaths -MaxBytes 1300 |
|
||||
ConvertFrom-Json
|
||||
Assert-True (-not $firstBodyPage.complete) 'bounded article batch produces continuation'
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstBodyPage.remainingPaths) -Snapshot ('0' * 64)
|
||||
} 'Article snapshot changed' 'wrong article continuation snapshot fails'
|
||||
Add-Content -LiteralPath (Join-Path $bodyRoot 'microsoft\knowledge\neutral\continuation-two.md') -Value 'changed' -Encoding utf8NoBOM
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths @($firstBodyPage.remainingPaths) -Snapshot $firstBodyPage.continuation.snapshot
|
||||
} 'Selected article hash does not match the prepared index' 'article continuation rejects a changed remaining body'
|
||||
|
||||
$invalidUtf8 = Join-Path $bodyRoot 'microsoft\knowledge\neutral\invalid-utf8.md'
|
||||
$indexedBytes = [IO.File]::ReadAllBytes($invalidUtf8)
|
||||
[IO.File]::WriteAllBytes($invalidUtf8, [byte[]]@($indexedBytes + @(0xc3, 0x28)))
|
||||
Assert-Throws {
|
||||
& $getArticles -BCQualityRoot $bodyRoot -IndexPath $bodyIndex `
|
||||
-Paths 'microsoft/knowledge/neutral/invalid-utf8.md'
|
||||
} 'Knowledge file is not valid strict UTF-8' 'invalid UTF-8 fails'
|
||||
|
||||
Write-Host "Knowledge retrieval check PASSED: $($articlePaths.Count) articles and $($samplePaths.Count) samples round-tripped; catalog union was lossless and bounded." -ForegroundColor Green
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
171
tools/Test-ReviewContract.ps1
Normal file
171
tools/Test-ReviewContract.ps1
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates the bounded leaf-range normalization contract.
|
||||
|
||||
.DESCRIPTION
|
||||
BCQuality has no executable findings-report consumer. These assertions keep
|
||||
the normative DO contract, AL coordinator, and standalone runner aligned
|
||||
while exercising the exact normalization predicate against representative
|
||||
safe and ambiguous inputs.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..'))
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
||||
|
||||
function Assert-True {
|
||||
param(
|
||||
[bool] $Condition,
|
||||
[string] $Message
|
||||
)
|
||||
|
||||
if (-not $Condition) {
|
||||
throw "Assertion failed: $Message"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Contains {
|
||||
param(
|
||||
[string] $Text,
|
||||
[string] $Expected,
|
||||
[string] $Message
|
||||
)
|
||||
|
||||
Assert-True $Text.Contains($Expected) $Message
|
||||
}
|
||||
|
||||
function Test-PositiveInteger {
|
||||
param([object] $Value)
|
||||
|
||||
if (($null -eq $Value) -or ($Value -is [bool]) -or ($Value -isnot [ValueType])) {
|
||||
return $false
|
||||
}
|
||||
|
||||
$number = [double]$Value
|
||||
return [double]::IsFinite($number) -and ($number -gt 0) -and ([math]::Truncate($number) -eq $number)
|
||||
}
|
||||
|
||||
function Test-RangeNormalizationEligibility {
|
||||
param([pscustomobject] $Finding)
|
||||
|
||||
if ($Finding.PSObject.Properties.Name -contains 'suggested-code') {
|
||||
return $false
|
||||
}
|
||||
if (-not ($Finding.PSObject.Properties.Name -contains 'location')) {
|
||||
return $false
|
||||
}
|
||||
if (-not ($Finding.location.PSObject.Properties.Name -contains 'line')) {
|
||||
return $false
|
||||
}
|
||||
if (-not ($Finding.location.PSObject.Properties.Name -contains 'range')) {
|
||||
return $false
|
||||
}
|
||||
|
||||
$range = $Finding.location.range
|
||||
if (-not ($range.PSObject.Properties.Name -contains 'start-line') -or
|
||||
-not ($range.PSObject.Properties.Name -contains 'end-line')) {
|
||||
return $false
|
||||
}
|
||||
|
||||
$line = $Finding.location.line
|
||||
$startLine = $range.'start-line'
|
||||
$endLine = $range.'end-line'
|
||||
if (-not (Test-PositiveInteger $line) -or
|
||||
-not (Test-PositiveInteger $startLine) -or
|
||||
-not (Test-PositiveInteger $endLine)) {
|
||||
return $false
|
||||
}
|
||||
|
||||
return ($startLine -le $line) -and ($line -le $endLine) -and ($startLine -ne $line)
|
||||
}
|
||||
|
||||
$transportSentence = 'Capture the exact Task return as the immutable raw audit payload and primary transport.'
|
||||
$doContract = Get-Content -LiteralPath (Join-Path $Root 'skills/do.md') -Raw
|
||||
$coordinatorContract = Get-Content -LiteralPath (Join-Path $Root 'microsoft/skills/review/al-code-review.md') -Raw
|
||||
$runnerContract = Get-Content -LiteralPath (Join-Path $Root 'docs/standalone-runner.md') -Raw
|
||||
|
||||
foreach ($surface in @(
|
||||
[pscustomobject]@{ Name = 'DO'; Text = ($doContract -replace '\s+', ' ') }
|
||||
[pscustomobject]@{ Name = 'AL coordinator'; Text = ($coordinatorContract -replace '\s+', ' ') }
|
||||
[pscustomobject]@{ Name = 'standalone runner'; Text = ($runnerContract -replace '\s+', ' ') }
|
||||
)) {
|
||||
Assert-Contains $surface.Text $transportSentence "$($surface.Name) preserves exact Task transport wording"
|
||||
}
|
||||
|
||||
$normalizedDoContract = $doContract -replace '\s+', ' '
|
||||
foreach ($expected in @(
|
||||
'positive integers',
|
||||
'start-line <= line <= end-line',
|
||||
'does not contain the `suggested-code` field',
|
||||
'remove only',
|
||||
'private run telemetry or artifacts',
|
||||
'Validate the entire normalized candidate',
|
||||
'If any other validation defect exists',
|
||||
'salvage arbitrary individual findings'
|
||||
)) {
|
||||
Assert-Contains $normalizedDoContract $expected "DO documents '$expected'"
|
||||
}
|
||||
|
||||
$cases = @(
|
||||
[pscustomobject]@{
|
||||
Name = 'contained mismatched range without suggested code'
|
||||
Expected = $true
|
||||
Finding = '{"message":"keep me","location":{"file":"src/codeunit.al","line":37,"range":{"start-line":36,"end-line":38}}}' | ConvertFrom-Json
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Name = 'aligned range'
|
||||
Expected = $false
|
||||
Finding = '{"location":{"line":37,"range":{"start-line":37,"end-line":38}}}' | ConvertFrom-Json
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Name = 'suggested code present'
|
||||
Expected = $false
|
||||
Finding = '{"location":{"line":37,"range":{"start-line":36,"end-line":38}},"suggested-code":""}' | ConvertFrom-Json
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Name = 'line outside range'
|
||||
Expected = $false
|
||||
Finding = '{"location":{"line":39,"range":{"start-line":36,"end-line":38}}}' | ConvertFrom-Json
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Name = 'reversed range'
|
||||
Expected = $false
|
||||
Finding = '{"location":{"line":37,"range":{"start-line":38,"end-line":36}}}' | ConvertFrom-Json
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Name = 'zero bound'
|
||||
Expected = $false
|
||||
Finding = '{"location":{"line":1,"range":{"start-line":0,"end-line":2}}}' | ConvertFrom-Json
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Name = 'fractional primary line'
|
||||
Expected = $false
|
||||
Finding = '{"location":{"line":37.5,"range":{"start-line":36,"end-line":38}}}' | ConvertFrom-Json
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Name = 'missing end line'
|
||||
Expected = $false
|
||||
Finding = '{"location":{"line":37,"range":{"start-line":36}}}' | ConvertFrom-Json
|
||||
}
|
||||
)
|
||||
|
||||
foreach ($case in $cases) {
|
||||
$actual = Test-RangeNormalizationEligibility $case.Finding
|
||||
Assert-True ($actual -eq $case.Expected) "$($case.Name) eligibility is $($case.Expected)"
|
||||
}
|
||||
|
||||
$rawFinding = $cases[0].Finding
|
||||
$candidateFinding = $rawFinding | ConvertTo-Json -Depth 10 | ConvertFrom-Json
|
||||
$candidateFinding.location.PSObject.Properties.Remove('range')
|
||||
|
||||
Assert-True ($rawFinding.location.PSObject.Properties.Name -contains 'range') 'raw finding remains unchanged'
|
||||
Assert-True (-not ($candidateFinding.location.PSObject.Properties.Name -contains 'range')) 'candidate removes only the optional range'
|
||||
Assert-True ($candidateFinding.location.line -eq $rawFinding.location.line) 'candidate preserves the primary line'
|
||||
Assert-True ($candidateFinding.message -ceq $rawFinding.message) 'candidate preserves all other finding content'
|
||||
|
||||
Write-Output "Review contract validation passed ($($cases.Count) normalization cases)."
|
||||
|
|
@ -195,12 +195,42 @@ foreach ($domain in $leafDomains) {
|
|||
}
|
||||
|
||||
$override = if ($overrides.ContainsKey($domain)) { $overrides[$domain] } else { $null }
|
||||
$selectedArticle = $null
|
||||
if ($override -and ($override.PSObject.Properties.Name -contains 'article')) {
|
||||
$articleName = [string]$override.article
|
||||
$hasArticleOverride = $override -and ($override.PSObject.Properties.Name -contains 'article')
|
||||
$hasArticlesOverride = $override -and ($override.PSObject.Properties.Name -contains 'articles')
|
||||
if ($hasArticleOverride -and $hasArticlesOverride) {
|
||||
$problems.Add("${domain}: override must specify either 'article' or 'articles', not both.") | Out-Null
|
||||
continue
|
||||
}
|
||||
|
||||
$articleNames = @()
|
||||
if ($hasArticlesOverride) {
|
||||
$articleNames = @($override.articles)
|
||||
if (-not $articleNames.Count) {
|
||||
$problems.Add("${domain}: override 'articles' must contain at least one article.") | Out-Null
|
||||
continue
|
||||
}
|
||||
} elseif ($hasArticleOverride) {
|
||||
$articleNames = @($override.article)
|
||||
} else {
|
||||
$articleNames = @($articles | Select-Object -First 1 | ForEach-Object BaseName)
|
||||
}
|
||||
|
||||
$selectedArticles = [System.Collections.Generic.List[object]]::new()
|
||||
$seenArticleNames = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($articleNameValue in $articleNames) {
|
||||
if ($articleNameValue -isnot [string] -or [string]::IsNullOrWhiteSpace([string]$articleNameValue)) {
|
||||
$problems.Add("${domain}: override article names must be non-empty strings.") | Out-Null
|
||||
continue
|
||||
}
|
||||
$articleName = [string]$articleNameValue
|
||||
if ($articleName.EndsWith('.md')) {
|
||||
$articleName = [System.IO.Path]::GetFileNameWithoutExtension($articleName)
|
||||
}
|
||||
if (-not $seenArticleNames.Add($articleName)) {
|
||||
$problems.Add("${domain}: override contains duplicate article: $articleName.md") | Out-Null
|
||||
continue
|
||||
}
|
||||
|
||||
$selectedArticle = $articles | Where-Object BaseName -eq $articleName | Select-Object -First 1
|
||||
if (-not $selectedArticle) {
|
||||
$articleExists = @(
|
||||
|
|
@ -218,32 +248,41 @@ foreach ($domain in $leafDomains) {
|
|||
}
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
$selectedArticle = $articles | Select-Object -First 1
|
||||
$selectedArticles.Add($selectedArticle) | Out-Null
|
||||
}
|
||||
if (-not $selectedArticle) {
|
||||
$problems.Add("${domain}: no article has both .good.al and .bad.al companion samples.") | Out-Null
|
||||
if (-not $selectedArticles.Count) {
|
||||
if (-not $articleNames.Count) {
|
||||
$problems.Add("${domain}: no article has both .good.al and .bad.al companion samples.") | Out-Null
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
$articlePath = [string]$selectedArticle.ArticlePath
|
||||
$sampleDirectory = (Split-Path -Parent $articlePath).Replace('\', '/')
|
||||
$context = if ($override -and ($override.PSObject.Properties.Name -contains 'context')) {
|
||||
[string]$override.context
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
foreach ($kind in 'bad', 'good') {
|
||||
$case = [pscustomobject]@{
|
||||
id = "$domain-$kind"
|
||||
domain = $domain
|
||||
input = "$sampleDirectory/$($selectedArticle.BaseName).$kind.al"
|
||||
expected = if ($kind -eq 'bad') { @($articlePath) } else { @() }
|
||||
for ($articleIndex = 0; $articleIndex -lt $selectedArticles.Count; $articleIndex++) {
|
||||
$selectedArticle = $selectedArticles[$articleIndex]
|
||||
$articlePath = [string]$selectedArticle.ArticlePath
|
||||
$sampleDirectory = (Split-Path -Parent $articlePath).Replace('\', '/')
|
||||
foreach ($kind in 'bad', 'good') {
|
||||
$caseId = if ($articleIndex -eq 0) {
|
||||
"$domain-$kind"
|
||||
} else {
|
||||
"$domain-$($selectedArticle.BaseName)-$kind"
|
||||
}
|
||||
$case = [pscustomobject]@{
|
||||
id = $caseId
|
||||
domain = $domain
|
||||
input = "$sampleDirectory/$($selectedArticle.BaseName).$kind.al"
|
||||
expected = if ($kind -eq 'bad') { @($articlePath) } else { @() }
|
||||
}
|
||||
if ($context) {
|
||||
$case | Add-Member -NotePropertyName context -NotePropertyValue $context
|
||||
}
|
||||
$caseList.Add($case) | Out-Null
|
||||
}
|
||||
if ($context) {
|
||||
$case | Add-Member -NotePropertyName context -NotePropertyValue $context
|
||||
}
|
||||
$caseList.Add($case) | Out-Null
|
||||
}
|
||||
}
|
||||
$cases = @($caseList)
|
||||
|
|
@ -434,6 +473,7 @@ if ($PrepareDirectory) {
|
|||
}
|
||||
|
||||
if (-not $ResultsPath -and -not $ResultsDirectory) {
|
||||
& (Join-Path $PSScriptRoot 'Test-ReviewContract.ps1') -Root $Root
|
||||
Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $($leafDomains.Count) leaf domains." -ForegroundColor Green
|
||||
exit 0
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue