mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Strengthen review contracts and HTTP guidance
- add outbound HttpClient transport and HTTP status review rules with paired fixtures`n- resolve layered action-skill overrides deterministically across enabled layers`n- validate findings reports and enforce measurable changed-fixture coverage
This commit is contained in:
demiliani
parent
07e324ddbc
commit
45ca57a23e
21 changed files with 830 additions and 34 deletions
|
|
@ -203,9 +203,19 @@ foreach ($layer in 'microsoft', 'community', 'custom') {
|
|||
}
|
||||
}
|
||||
|
||||
$ids = @($records | Group-Object id | Where-Object Count -gt 1)
|
||||
if ($ids.Count) {
|
||||
throw "Duplicate action-skill IDs: $($ids.Name -join ', ')"
|
||||
$idsWithinLayer = @(
|
||||
$records |
|
||||
Group-Object { "$($_.layer)`0$($_.id)" } |
|
||||
Where-Object Count -gt 1
|
||||
)
|
||||
if ($idsWithinLayer.Count) {
|
||||
$duplicates = @(
|
||||
$idsWithinLayer | ForEach-Object {
|
||||
$parts = $_.Name -split "`0", 2
|
||||
"$($parts[0]):$($parts[1])"
|
||||
}
|
||||
)
|
||||
throw "Duplicate action-skill IDs within a layer: $($duplicates -join ', ')"
|
||||
}
|
||||
|
||||
foreach ($record in $records) {
|
||||
|
|
|
|||
92
tools/Resolve-SkillWorklist.ps1
Normal file
92
tools/Resolve-SkillWorklist.ps1
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Resolves a super-skill's ordered leaf worklist across enabled layers.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $BCQualityRoot,
|
||||
[string] $IndexPath,
|
||||
[Parameter(Mandatory)]
|
||||
[string] $SuperSkillPath,
|
||||
[string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
|
||||
[string[]] $DisabledSkills = @()
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not $BCQualityRoot) {
|
||||
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
|
||||
}
|
||||
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'skill-index.json'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $IndexPath -PathType Leaf)) {
|
||||
& (Join-Path $PSScriptRoot 'Build-SkillIndex.ps1') -BCQualityRoot $BCQualityRoot -IndexPath $IndexPath | Out-Null
|
||||
}
|
||||
|
||||
$knownLayers = @('microsoft', 'community', 'custom')
|
||||
$unknownLayers = @($EnabledLayers | Where-Object { $_ -cnotin $knownLayers })
|
||||
if ($unknownLayers.Count) {
|
||||
throw "Unknown enabled layers: $($unknownLayers -join ', ')"
|
||||
}
|
||||
|
||||
$index = Get-Content -LiteralPath $IndexPath -Raw | ConvertFrom-Json
|
||||
$skills = @($index.skills)
|
||||
$superSkills = @($skills | Where-Object path -CEQ $SuperSkillPath)
|
||||
if ($superSkills.Count -ne 1) {
|
||||
throw "Expected one indexed super-skill at '$SuperSkillPath', found $($superSkills.Count)."
|
||||
}
|
||||
$superSkill = $superSkills[0]
|
||||
if (-not @($superSkill.subSkills).Count) {
|
||||
throw "Action skill '$SuperSkillPath' is not a super-skill."
|
||||
}
|
||||
|
||||
$precedence = @{ microsoft = 0; community = 1; custom = 2 }
|
||||
$resolved = [Collections.Generic.List[object]]::new()
|
||||
$skipped = [Collections.Generic.List[object]]::new()
|
||||
foreach ($declaredPath in @($superSkill.subSkills)) {
|
||||
$declared = @($skills | Where-Object path -CEQ $declaredPath)
|
||||
if ($declared.Count -ne 1) {
|
||||
throw "Declared sub-skill '$declaredPath' is not uniquely indexed."
|
||||
}
|
||||
|
||||
$candidates = @(
|
||||
$skills |
|
||||
Where-Object {
|
||||
$_.id -CEQ $declared[0].id -and
|
||||
$_.layer -cin $EnabledLayers -and
|
||||
$_.path -cnotin $DisabledSkills -and
|
||||
-not @($_.subSkills).Count
|
||||
} |
|
||||
Sort-Object @{ Expression = { $precedence[$_.layer] }; Descending = $true }, path
|
||||
)
|
||||
if (-not $candidates.Count) {
|
||||
$skipped.Add([pscustomobject][ordered]@{
|
||||
id = $declared[0].id
|
||||
declaredPath = $declaredPath
|
||||
reason = 'configuration'
|
||||
}) | Out-Null
|
||||
continue
|
||||
}
|
||||
|
||||
$winner = $candidates[0]
|
||||
$resolved.Add([pscustomobject][ordered]@{
|
||||
id = $winner.id
|
||||
path = $winner.path
|
||||
version = $winner.version
|
||||
layer = $winner.layer
|
||||
declaredPath = $declaredPath
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
return [pscustomobject][ordered]@{
|
||||
superSkill = [pscustomobject][ordered]@{
|
||||
id = $superSkill.id
|
||||
path = $superSkill.path
|
||||
version = $superSkill.version
|
||||
}
|
||||
subSkills = @($resolved)
|
||||
skipped = @($skipped)
|
||||
}
|
||||
|
|
@ -1,12 +1,11 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates the bounded leaf-range normalization contract.
|
||||
Validates executable findings-report acceptance and bounded normalization.
|
||||
|
||||
.DESCRIPTION
|
||||
BCQuality has no executable findings-report consumer. These assertions keep
|
||||
the normative DO contract, AL coordinator, and standalone runner aligned
|
||||
while exercising the exact normalization predicate against representative
|
||||
safe and ambiguous inputs.
|
||||
These assertions keep the normative DO contract, executable validator, AL
|
||||
coordinator, and standalone runner aligned while exercising semantic report
|
||||
validation and the exact normalization predicate.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
|
|
@ -39,6 +38,24 @@ function Assert-Contains {
|
|||
Assert-True $Text.Contains($Expected) $Message
|
||||
}
|
||||
|
||||
function Assert-ThrowsLike {
|
||||
param(
|
||||
[scriptblock] $Action,
|
||||
[string] $Pattern
|
||||
)
|
||||
|
||||
try {
|
||||
& $Action
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Message -like $Pattern) {
|
||||
return
|
||||
}
|
||||
throw "Expected error like '$Pattern', received: $($_.Exception.Message)"
|
||||
}
|
||||
throw "Expected error like '$Pattern', but no error was thrown."
|
||||
}
|
||||
|
||||
function Test-PositiveInteger {
|
||||
param([object] $Value)
|
||||
|
||||
|
|
@ -168,4 +185,85 @@ Assert-True (-not ($candidateFinding.location.PSObject.Properties.Name -contains
|
|||
Assert-True ($candidateFinding.location.line -eq $rawFinding.location.line) 'candidate preserves the primary line'
|
||||
Assert-True ($candidateFinding.message -ceq $rawFinding.message) 'candidate preserves all other finding content'
|
||||
|
||||
Write-Output "Review contract validation passed ($($cases.Count) normalization cases)."
|
||||
$validator = Join-Path $Root 'tools/Validate-FindingsReport.ps1'
|
||||
Assert-True (Test-Path -LiteralPath $validator -PathType Leaf) 'executable report validator exists'
|
||||
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("reviewcontract_" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Path $tmp -Force | Out-Null
|
||||
try {
|
||||
$sourcePath = 'src/codeunit.al'
|
||||
$sourceFile = Join-Path $tmp 'src/codeunit.al'
|
||||
New-Item -ItemType Directory -Path (Split-Path -Parent $sourceFile) -Force | Out-Null
|
||||
Set-Content -LiteralPath $sourceFile -Value @('line one', 'line two', 'line three') -Encoding utf8NoBOM
|
||||
$articlePath = 'microsoft/knowledge/style/caption-required-on-page-fields.md'
|
||||
$reportPath = Join-Path $tmp 'report.json'
|
||||
|
||||
$validReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-style-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 1; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 1; 'items-evaluated' = 1 }
|
||||
}
|
||||
findings = @(
|
||||
[ordered]@{
|
||||
id = $articlePath
|
||||
severity = 'minor'
|
||||
message = 'A concrete style defect.'
|
||||
location = [ordered]@{
|
||||
file = $sourcePath
|
||||
line = 2
|
||||
range = [ordered]@{ 'start-line' = 2; 'end-line' = 3 }
|
||||
}
|
||||
references = @([ordered]@{ path = $articlePath })
|
||||
confidence = 'high'
|
||||
domain = 'Style'
|
||||
}
|
||||
)
|
||||
suppressed = @()
|
||||
}
|
||||
Set-Content -LiteralPath $reportPath -Value ($validReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$accepted = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
Assert-True (-not $accepted.normalized) 'valid report is accepted without normalization'
|
||||
|
||||
$invalidCounts = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$invalidCounts.summary.counts.minor = 0
|
||||
Set-Content -LiteralPath $reportPath -Value ($invalidCounts | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*COUNT_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
Set-Content -LiteralPath $reportPath -Value ($validReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*REFERENCE_NOT_RETRIEVED*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp -SourcePaths $sourcePath
|
||||
}
|
||||
|
||||
$invalidAgent = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$invalidAgent.findings[0].id = 'agent:uncited-defect'
|
||||
$invalidAgent.findings[0].references = @()
|
||||
$invalidAgent.findings[0].confidence = 'high'
|
||||
Set-Content -LiteralPath $reportPath -Value ($invalidAgent | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*AGENT_CONFIDENCE_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp -SourcePaths $sourcePath
|
||||
}
|
||||
|
||||
$normalizable = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$normalizable.findings[0].location.range.'start-line' = 1
|
||||
Set-Content -LiteralPath $reportPath -Value ($normalizable | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*RANGE_START_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
$normalized = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath -AllowBoundedNormalization
|
||||
Assert-True $normalized.normalized 'eligible range mismatch is normalized'
|
||||
Assert-True ($normalized.removedRanges.Count -eq 1) 'normalization records one removed range'
|
||||
Assert-True (-not ($normalized.report.findings[0].location.PSObject.Properties.Name -contains 'range')) `
|
||||
'accepted normalized report removes only the optional range'
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
Write-Output "Review contract validation passed ($($cases.Count) predicate cases plus executable acceptance cases)."
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@ param(
|
|||
[string] $ManifestPath,
|
||||
[string] $PrepareDirectory,
|
||||
[string] $ResultsPath,
|
||||
[string] $ResultsDirectory
|
||||
[string] $ResultsDirectory,
|
||||
[string] $ChangedPathsFile,
|
||||
[string] $CoverageReportPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
|
|
@ -160,7 +162,23 @@ foreach ($overrideDomain in $overrides.Keys) {
|
|||
}
|
||||
}
|
||||
|
||||
$coverageWaivers = @{}
|
||||
if ($manifest.PSObject.Properties.Name -contains 'coverageWaivers') {
|
||||
foreach ($waiver in @($manifest.coverageWaivers)) {
|
||||
if (-not $waiver.path -or -not $waiver.reason) {
|
||||
$problems.Add('Each coverage waiver requires non-empty path and reason values.') | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($coverageWaivers.ContainsKey([string]$waiver.path)) {
|
||||
$problems.Add("Duplicate coverage waiver: $($waiver.path)") | Out-Null
|
||||
continue
|
||||
}
|
||||
$coverageWaivers[[string]$waiver.path] = [string]$waiver.reason
|
||||
}
|
||||
}
|
||||
|
||||
$caseList = [System.Collections.Generic.List[object]]::new()
|
||||
$pairedArticlesByDomain = @{}
|
||||
foreach ($domain in $leafDomains) {
|
||||
$articleCandidates = @(
|
||||
foreach ($layer in $layers) {
|
||||
|
|
@ -189,6 +207,7 @@ foreach ($domain in $leafDomains) {
|
|||
ForEach-Object { $_.Group | Sort-Object Rank -Descending | Select-Object -First 1 } |
|
||||
Sort-Object BaseName
|
||||
)
|
||||
$pairedArticlesByDomain[$domain] = @($articles)
|
||||
if (-not $articles.Count) {
|
||||
$problems.Add("${domain}: no enabled knowledge layer has an article with both .good.al and .bad.al companion samples.") | Out-Null
|
||||
continue
|
||||
|
|
@ -339,6 +358,65 @@ foreach ($domain in $leafDomains) {
|
|||
}
|
||||
}
|
||||
|
||||
$selectedArticlePaths = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($case in $cases) {
|
||||
foreach ($reference in @($case.expected)) {
|
||||
$selectedArticlePaths.Add([string]$reference) | Out-Null
|
||||
}
|
||||
}
|
||||
$effectivePairedPaths = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
$coverageDomains = @(
|
||||
foreach ($domain in $leafDomains) {
|
||||
$paired = @($pairedArticlesByDomain[$domain])
|
||||
foreach ($article in $paired) {
|
||||
$effectivePairedPaths.Add([string]$article.ArticlePath) | Out-Null
|
||||
}
|
||||
$selected = @($paired | Where-Object { $selectedArticlePaths.Contains([string]$_.ArticlePath) }).Count
|
||||
[pscustomobject][ordered]@{
|
||||
domain = $domain
|
||||
pairedArticles = $paired.Count
|
||||
selectedArticles = $selected
|
||||
coverage = if ($paired.Count) { $selected / $paired.Count } else { 0 }
|
||||
}
|
||||
}
|
||||
)
|
||||
$pairedTotal = ($coverageDomains | Measure-Object pairedArticles -Sum).Sum
|
||||
$selectedTotal = ($coverageDomains | Measure-Object selectedArticles -Sum).Sum
|
||||
$coverageReport = [pscustomobject][ordered]@{
|
||||
pairedArticles = $pairedTotal
|
||||
selectedArticles = $selectedTotal
|
||||
coverage = if ($pairedTotal) { $selectedTotal / $pairedTotal } else { 0 }
|
||||
domains = $coverageDomains
|
||||
}
|
||||
if ($CoverageReportPath) {
|
||||
$coverageParent = Split-Path -Parent $CoverageReportPath
|
||||
if ($coverageParent -and -not (Test-Path -LiteralPath $coverageParent)) {
|
||||
New-Item -ItemType Directory -Path $coverageParent -Force | Out-Null
|
||||
}
|
||||
$coverageReport | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $CoverageReportPath -Encoding utf8NoBOM
|
||||
}
|
||||
|
||||
if ($ChangedPathsFile) {
|
||||
if (-not (Test-Path -LiteralPath $ChangedPathsFile -PathType Leaf)) {
|
||||
$problems.Add("Changed paths file not found: $ChangedPathsFile") | Out-Null
|
||||
}
|
||||
else {
|
||||
foreach ($changedPathValue in Get-Content -LiteralPath $ChangedPathsFile) {
|
||||
$changedPath = ([string]$changedPathValue).Trim().Replace('\', '/')
|
||||
if ($changedPath -notmatch '^(microsoft|community|custom)/knowledge/[^/]+/(.+?)(?:\.(?:good|bad)\.al|\.md)$') {
|
||||
continue
|
||||
}
|
||||
$articlePath = "$($Matches[1])/knowledge/$($changedPath.Split('/')[2])/$($Matches[2]).md"
|
||||
if (-not $effectivePairedPaths.Contains($articlePath) -or $selectedArticlePaths.Contains($articlePath)) {
|
||||
continue
|
||||
}
|
||||
if (-not $coverageWaivers.ContainsKey($articlePath)) {
|
||||
$problems.Add("Changed paired article is not selected for evaluation and has no coverage waiver: $articlePath") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Review fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
|
|
@ -474,7 +552,7 @@ if ($PrepareDirectory) {
|
|||
|
||||
if (-not $ResultsPath -and -not $ResultsDirectory) {
|
||||
& (Join-Path $PSScriptRoot 'Test-ReviewContract.ps1') -Root $Root
|
||||
Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $($leafDomains.Count) leaf domains." -ForegroundColor Green
|
||||
Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $selectedTotal/$pairedTotal paired articles across $($leafDomains.Count) leaf domains." -ForegroundColor Green
|
||||
exit 0
|
||||
}
|
||||
|
||||
|
|
|
|||
215
tools/Validate-FindingsReport.ps1
Normal file
215
tools/Validate-FindingsReport.ps1
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates a BCQuality findings-report against its structural and semantic contract.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $ReportPath,
|
||||
[string] $BCQualityRoot,
|
||||
[string] $SourceRoot,
|
||||
[string[]] $SourcePaths = @(),
|
||||
[string[]] $RetrievedArticlePaths = @(),
|
||||
[switch] $AllowBoundedNormalization
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not $BCQualityRoot) {
|
||||
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
|
||||
}
|
||||
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
|
||||
$schemaPath = Join-Path $BCQualityRoot 'schemas/findings-report.schema.json'
|
||||
$raw = Get-Content -LiteralPath $ReportPath -Raw
|
||||
try {
|
||||
if (-not ($raw | Test-Json -SchemaFile $schemaPath -ErrorAction Stop)) {
|
||||
throw 'Report does not satisfy schemas/findings-report.schema.json.'
|
||||
}
|
||||
$report = $raw | ConvertFrom-Json -Depth 100
|
||||
}
|
||||
catch {
|
||||
throw "Invalid findings-report JSON or schema: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
$retrieved = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($path in $RetrievedArticlePaths) {
|
||||
$retrieved.Add($path) | Out-Null
|
||||
}
|
||||
$sources = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($path in $SourcePaths) {
|
||||
$sources.Add($path) | Out-Null
|
||||
}
|
||||
$lineCounts = @{}
|
||||
|
||||
function Test-HasProperty {
|
||||
param([object] $Object, [string] $Name)
|
||||
return $null -ne $Object -and $Object.PSObject.Properties.Name -ccontains $Name
|
||||
}
|
||||
|
||||
function Get-SourceLineCount {
|
||||
param([string] $Path)
|
||||
|
||||
if ($lineCounts.ContainsKey($Path)) {
|
||||
return $lineCounts[$Path]
|
||||
}
|
||||
if (-not $SourceRoot) {
|
||||
return -1
|
||||
}
|
||||
$fullPath = Join-Path $SourceRoot ($Path -replace '/', [IO.Path]::DirectorySeparatorChar)
|
||||
if (-not (Test-Path -LiteralPath $fullPath -PathType Leaf)) {
|
||||
return -1
|
||||
}
|
||||
$lineCounts[$Path] = [IO.File]::ReadAllLines($fullPath).Count
|
||||
return $lineCounts[$Path]
|
||||
}
|
||||
|
||||
function Get-SemanticErrors {
|
||||
param(
|
||||
[object] $Candidate,
|
||||
[switch] $PermitRangeStartMismatch
|
||||
)
|
||||
|
||||
$errors = [Collections.Generic.List[object]]::new()
|
||||
function Add-Error {
|
||||
param([string] $Code, [string] $Path, [string] $Message)
|
||||
$errors.Add([pscustomobject]@{ Code = $Code; Path = $Path; Message = $Message }) | Out-Null
|
||||
}
|
||||
|
||||
function Test-Report {
|
||||
param([object] $Current, [string] $ReportPathPrefix)
|
||||
|
||||
$findings = @($Current.findings)
|
||||
foreach ($severity in 'blocker', 'major', 'minor', 'info') {
|
||||
$actual = @($findings | Where-Object severity -CEQ $severity).Count
|
||||
if ($Current.summary.counts.$severity -ne $actual) {
|
||||
Add-Error 'COUNT_MISMATCH' "$ReportPathPrefix.summary.counts.$severity" "Expected $actual."
|
||||
}
|
||||
}
|
||||
if ($Current.summary.coverage.'items-evaluated' -gt $Current.summary.coverage.'worklist-size') {
|
||||
Add-Error 'COVERAGE_INVALID' "$ReportPathPrefix.summary.coverage" 'items-evaluated exceeds worklist-size.'
|
||||
}
|
||||
|
||||
for ($index = 0; $index -lt $findings.Count; $index++) {
|
||||
$finding = $findings[$index]
|
||||
$findingPath = "$ReportPathPrefix.findings[$index]"
|
||||
$references = @($finding.references)
|
||||
if (-not $references.Count) {
|
||||
if ($finding.id -cnotmatch '(^|:)agent:[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
Add-Error 'AGENT_ID_INVALID' "$findingPath.id" 'An agent finding id must contain an agent: slug marker.'
|
||||
}
|
||||
if ($finding.confidence -ceq 'high') {
|
||||
Add-Error 'AGENT_CONFIDENCE_INVALID' "$findingPath.confidence" 'Agent confidence cannot be high.'
|
||||
}
|
||||
if ($finding.severity -cin @('blocker', 'major')) {
|
||||
Add-Error 'AGENT_SEVERITY_INVALID' "$findingPath.severity" 'Agent severity cannot exceed minor.'
|
||||
}
|
||||
}
|
||||
else {
|
||||
if ($finding.id -cne $references[0].path) {
|
||||
Add-Error 'PRIMARY_REFERENCE_MISMATCH' "$findingPath.id" 'Finding id must equal the primary reference path.'
|
||||
}
|
||||
foreach ($reference in $references) {
|
||||
if ($reference.path -cnotmatch '^(microsoft|community|custom)/knowledge/.+\.md$') {
|
||||
Add-Error 'REFERENCE_PATH_INVALID' "$findingPath.references" "Invalid knowledge path '$($reference.path)'."
|
||||
continue
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $BCQualityRoot $reference.path) -PathType Leaf)) {
|
||||
Add-Error 'REFERENCE_MISSING' "$findingPath.references" "Knowledge path '$($reference.path)' does not exist."
|
||||
}
|
||||
if (-not $retrieved.Contains($reference.path)) {
|
||||
Add-Error 'REFERENCE_NOT_RETRIEVED' "$findingPath.references" "Knowledge path '$($reference.path)' was not retrieved in full."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Test-HasProperty $finding 'location') {
|
||||
$location = $finding.location
|
||||
if (-not $sources.Contains($location.file)) {
|
||||
Add-Error 'SOURCE_OUT_OF_SCOPE' "$findingPath.location.file" "Source path '$($location.file)' is outside the supplied scope."
|
||||
}
|
||||
$lineCount = Get-SourceLineCount $location.file
|
||||
if ($lineCount -lt 0) {
|
||||
Add-Error 'SOURCE_MISSING' "$findingPath.location.file" "Source path '$($location.file)' does not exist."
|
||||
}
|
||||
elseif ($location.line -gt $lineCount) {
|
||||
Add-Error 'SOURCE_LINE_INVALID' "$findingPath.location.line" "Line exceeds the file's $lineCount lines."
|
||||
}
|
||||
|
||||
if (Test-HasProperty $location 'range') {
|
||||
$range = $location.range
|
||||
if ($range.'start-line' -ne $location.line) {
|
||||
Add-Error 'RANGE_START_MISMATCH' "$findingPath.location.range.start-line" 'start-line must equal line.'
|
||||
}
|
||||
if ($range.'end-line' -lt $range.'start-line' -or
|
||||
($lineCount -ge 0 -and $range.'end-line' -gt $lineCount)) {
|
||||
Add-Error 'SOURCE_RANGE_INVALID' "$findingPath.location.range" 'Range is reversed or exceeds the source file.'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Test-HasProperty $Current 'sub-results') {
|
||||
$subResults = @($Current.'sub-results')
|
||||
for ($index = 0; $index -lt $subResults.Count; $index++) {
|
||||
Test-Report $subResults[$index] "$ReportPathPrefix.sub-results[$index]"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Test-Report $Candidate '$'
|
||||
if ($PermitRangeStartMismatch) {
|
||||
return @($errors | Where-Object Code -CNE 'RANGE_START_MISMATCH')
|
||||
}
|
||||
return @($errors)
|
||||
}
|
||||
|
||||
$errors = @(Get-SemanticErrors $report)
|
||||
$normalized = $false
|
||||
$removedRanges = [Collections.Generic.List[object]]::new()
|
||||
if ($errors.Count -and $AllowBoundedNormalization) {
|
||||
$otherErrors = @($errors | Where-Object Code -CNE 'RANGE_START_MISMATCH')
|
||||
$rangeErrors = @($errors | Where-Object Code -CEQ 'RANGE_START_MISMATCH')
|
||||
if (-not $otherErrors.Count -and $rangeErrors.Count) {
|
||||
$candidate = $report | ConvertTo-Json -Depth 100 | ConvertFrom-Json -Depth 100
|
||||
$eligible = $true
|
||||
foreach ($finding in @($candidate.findings)) {
|
||||
if (-not (Test-HasProperty $finding 'location') -or
|
||||
-not (Test-HasProperty $finding.location 'range') -or
|
||||
$finding.location.range.'start-line' -eq $finding.location.line) {
|
||||
continue
|
||||
}
|
||||
$range = $finding.location.range
|
||||
if ($range.'start-line' -gt $finding.location.line -or
|
||||
$finding.location.line -gt $range.'end-line' -or
|
||||
(Test-HasProperty $finding 'suggested-code')) {
|
||||
$eligible = $false
|
||||
break
|
||||
}
|
||||
$removedRanges.Add([pscustomobject]@{
|
||||
findingId = $finding.id
|
||||
file = $finding.location.file
|
||||
line = $finding.location.line
|
||||
startLine = $range.'start-line'
|
||||
endLine = $range.'end-line'
|
||||
}) | Out-Null
|
||||
$finding.location.PSObject.Properties.Remove('range')
|
||||
}
|
||||
if ($eligible -and -not @(Get-SemanticErrors $candidate).Count) {
|
||||
$report = $candidate
|
||||
$normalized = $true
|
||||
$errors = @()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($errors.Count) {
|
||||
$details = @($errors | ForEach-Object { "$($_.Code) at $($_.Path): $($_.Message)" }) -join '; '
|
||||
throw "Findings-report acceptance failed: $details"
|
||||
}
|
||||
|
||||
return [pscustomobject][ordered]@{
|
||||
normalized = $normalized
|
||||
report = $report
|
||||
removedRanges = @($removedRanges)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue