mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 06:36:55 +01:00
Strengthen review contracts and add AL reliability guidance (#196)
* Strengthen review contracts and HTTP guidance - add outbound HttpClient transport and HTTP status review rules with paired fixtures`n- resolve layered action-skill overrides deterministically across enabled layers`n- validate findings reports and enforce measurable changed-fixture coverage * Add data handling and test isolation guidance - add SCM guidance for deriving base quantities through line unit-of-measure validation`n- add security guidance for parameterizing SetFilter with external text`n- add test isolation guidance for resetting per-test state before initialization guards`n- add web-service guidance for JSON null handling and invariant standard format 9`n- route and cover all five rules with paired evaluation fixtures * Fix findings report rollup validation * Validate findings report rollups * Enforce merged finding identity * Fix locationless finding deduplication * Reject conflicting merged corrections * Detect conflicting leaf corrections * Route HTTP error checks to canonical web-services knowledge Let the Error Handling leaf conditionally retrieve the existing HTTP owner articles, preserving applicability and exact-path provenance. Add deterministic source-contract and retrieval regressions without duplicating knowledge rules. Copilot-Session-Id: a92a7788-103e-4651-9b84-19e34caffb94 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: wenjiefan <wenjiefan@microsoft.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Jesper Schulz-Wedde <jesper.schulzwedde@microsoft.com>
This commit is contained in:
Stefano Demiliani
GitHub
parent
130d5de6c4
commit
4287233f80
41 changed files with 1974 additions and 38 deletions
|
|
@ -203,9 +203,19 @@ foreach ($layer in 'microsoft', 'community', 'custom') {
|
|||
}
|
||||
}
|
||||
|
||||
$ids = @($records | Group-Object id | Where-Object Count -gt 1)
|
||||
if ($ids.Count) {
|
||||
throw "Duplicate action-skill IDs: $($ids.Name -join ', ')"
|
||||
$idsWithinLayer = @(
|
||||
$records |
|
||||
Group-Object { "$($_.layer)`0$($_.id)" } |
|
||||
Where-Object Count -gt 1
|
||||
)
|
||||
if ($idsWithinLayer.Count) {
|
||||
$duplicates = @(
|
||||
$idsWithinLayer | ForEach-Object {
|
||||
$parts = $_.Name -split "`0", 2
|
||||
"$($parts[0]):$($parts[1])"
|
||||
}
|
||||
)
|
||||
throw "Duplicate action-skill IDs within a layer: $($duplicates -join ', ')"
|
||||
}
|
||||
|
||||
foreach ($record in $records) {
|
||||
|
|
|
|||
92
tools/Resolve-SkillWorklist.ps1
Normal file
92
tools/Resolve-SkillWorklist.ps1
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Resolves a super-skill's ordered leaf worklist across enabled layers.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $BCQualityRoot,
|
||||
[string] $IndexPath,
|
||||
[Parameter(Mandatory)]
|
||||
[string] $SuperSkillPath,
|
||||
[string[]] $EnabledLayers = @('microsoft', 'community', 'custom'),
|
||||
[string[]] $DisabledSkills = @()
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not $BCQualityRoot) {
|
||||
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
|
||||
}
|
||||
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
|
||||
if (-not $IndexPath) {
|
||||
$IndexPath = Join-Path $BCQualityRoot 'skill-index.json'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $IndexPath -PathType Leaf)) {
|
||||
& (Join-Path $PSScriptRoot 'Build-SkillIndex.ps1') -BCQualityRoot $BCQualityRoot -IndexPath $IndexPath | Out-Null
|
||||
}
|
||||
|
||||
$knownLayers = @('microsoft', 'community', 'custom')
|
||||
$unknownLayers = @($EnabledLayers | Where-Object { $_ -cnotin $knownLayers })
|
||||
if ($unknownLayers.Count) {
|
||||
throw "Unknown enabled layers: $($unknownLayers -join ', ')"
|
||||
}
|
||||
|
||||
$index = Get-Content -LiteralPath $IndexPath -Raw | ConvertFrom-Json
|
||||
$skills = @($index.skills)
|
||||
$superSkills = @($skills | Where-Object path -CEQ $SuperSkillPath)
|
||||
if ($superSkills.Count -ne 1) {
|
||||
throw "Expected one indexed super-skill at '$SuperSkillPath', found $($superSkills.Count)."
|
||||
}
|
||||
$superSkill = $superSkills[0]
|
||||
if (-not @($superSkill.subSkills).Count) {
|
||||
throw "Action skill '$SuperSkillPath' is not a super-skill."
|
||||
}
|
||||
|
||||
$precedence = @{ microsoft = 0; community = 1; custom = 2 }
|
||||
$resolved = [Collections.Generic.List[object]]::new()
|
||||
$skipped = [Collections.Generic.List[object]]::new()
|
||||
foreach ($declaredPath in @($superSkill.subSkills)) {
|
||||
$declared = @($skills | Where-Object path -CEQ $declaredPath)
|
||||
if ($declared.Count -ne 1) {
|
||||
throw "Declared sub-skill '$declaredPath' is not uniquely indexed."
|
||||
}
|
||||
|
||||
$candidates = @(
|
||||
$skills |
|
||||
Where-Object {
|
||||
$_.id -CEQ $declared[0].id -and
|
||||
$_.layer -cin $EnabledLayers -and
|
||||
$_.path -cnotin $DisabledSkills -and
|
||||
-not @($_.subSkills).Count
|
||||
} |
|
||||
Sort-Object @{ Expression = { $precedence[$_.layer] }; Descending = $true }, path
|
||||
)
|
||||
if (-not $candidates.Count) {
|
||||
$skipped.Add([pscustomobject][ordered]@{
|
||||
id = $declared[0].id
|
||||
declaredPath = $declaredPath
|
||||
reason = 'configuration'
|
||||
}) | Out-Null
|
||||
continue
|
||||
}
|
||||
|
||||
$winner = $candidates[0]
|
||||
$resolved.Add([pscustomobject][ordered]@{
|
||||
id = $winner.id
|
||||
path = $winner.path
|
||||
version = $winner.version
|
||||
layer = $winner.layer
|
||||
declaredPath = $declaredPath
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
return [pscustomobject][ordered]@{
|
||||
superSkill = [pscustomobject][ordered]@{
|
||||
id = $superSkill.id
|
||||
path = $superSkill.path
|
||||
version = $superSkill.version
|
||||
}
|
||||
subSkills = @($resolved)
|
||||
skipped = @($skipped)
|
||||
}
|
||||
|
|
@ -348,6 +348,60 @@ try {
|
|||
$indexPath = Join-Path $tmp 'knowledge-index.json'
|
||||
& $generator -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
||||
$index = Get-Content -LiteralPath $indexPath -Raw -Encoding utf8 | ConvertFrom-Json
|
||||
|
||||
# Check the declared finder route and real tool reachability, not model compliance.
|
||||
$errorSkill = Get-Content -LiteralPath (Join-Path $Root 'microsoft/skills/review/al-error-handling-review.md') -Raw
|
||||
$errorSource = [regex]::Match($errorSkill, '(?ms)^## Source\r?\n(.*?)(?=^## )').Groups[1].Value
|
||||
$sourceDomains = @([regex]::Matches($errorSource, '-Domain ([a-z-]+)') | ForEach-Object { $_.Groups[1].Value })
|
||||
Assert-Sequence $sourceDomains @('error-handling', 'web-services') 'error-handling declares its own and supplementary HTTP catalogs'
|
||||
foreach ($cue in @('HttpClient.Get', 'HttpClient.Post', 'resolved call paths', 'same known task dimensions and enabled layers')) {
|
||||
Assert-True ($errorSource.Contains($cue)) "supplementary source retains '$cue'"
|
||||
}
|
||||
$httpArticleNames = @(
|
||||
[regex]::Matches($errorSource, '\]\(\.\./\.\./knowledge/web-services/([a-z-]+\.md)\)') |
|
||||
ForEach-Object { $_.Groups[1].Value }
|
||||
)
|
||||
Assert-Sequence $httpArticleNames @(
|
||||
'handle-httpclient-platform-failure-before-response-access.md'
|
||||
'check-http-status-before-consuming-response-body.md'
|
||||
) 'supplementary source names only the two canonical HTTP articles'
|
||||
$httpArguments = @{
|
||||
BCQualityRoot = $Root
|
||||
IndexPath = $indexPath
|
||||
EnabledLayers = @('microsoft', 'community', 'custom')
|
||||
Technologies = @('al')
|
||||
BCVersion = 28
|
||||
Countries = @('w1')
|
||||
}
|
||||
$ownCatalog = Invoke-CatalogPages -Arguments ($httpArguments + @{ Domain = $sourceDomains[0] })
|
||||
Assert-True (-not @($ownCatalog.candidates | Where-Object { $_.path -like '*/knowledge/web-services/*' }).Count) 'the primary catalog does not silently expand domains'
|
||||
$httpCatalog = Invoke-CatalogPages -Arguments ($httpArguments + @{ Domain = $sourceDomains[1] })
|
||||
$httpRows = @($httpCatalog.candidates | Where-Object { $httpArticleNames -ccontains ($_.path -split '/')[-1] })
|
||||
$expectedHttpPaths = @(
|
||||
$index.articles |
|
||||
Where-Object { $_.domain -ceq 'web-services' -and $httpArticleNames -ccontains ($_.path -split '/')[-1] } |
|
||||
ForEach-Object path |
|
||||
Sort-Object
|
||||
)
|
||||
foreach ($name in $httpArticleNames) {
|
||||
Assert-True ($expectedHttpPaths -ccontains "microsoft/knowledge/web-services/$name") "canonical owner exists for $name"
|
||||
}
|
||||
Assert-Sequence @($httpRows.path | Sort-Object) $expectedHttpPaths 'supplementary selection preserves exact paths across layers'
|
||||
Test-BodyRoundTrip -Paths $httpRows.path -IndexPath $indexPath
|
||||
foreach ($excludedContext in @(
|
||||
@{ EnabledLayers = @() }
|
||||
@{ Technologies = @('javascript') }
|
||||
)) {
|
||||
$arguments = $httpArguments + @{ Domain = $sourceDomains[1] }
|
||||
foreach ($key in $excludedContext.Keys) {
|
||||
$arguments[$key] = $excludedContext[$key]
|
||||
}
|
||||
$catalog = Invoke-CatalogPages -Arguments $arguments
|
||||
$selected = @($catalog.candidates | Where-Object { $httpArticleNames -ccontains ($_.path -split '/')[-1] })
|
||||
Assert-Equal $selected.Count 0 'supplementary selection respects disabled layers and nonmatching technology'
|
||||
}
|
||||
Write-Host 'HTTP source contract and exact-body reachability passed; model routing was not evaluated.'
|
||||
|
||||
$diskArticlePaths = @(
|
||||
foreach ($layer in 'microsoft', 'community', 'custom') {
|
||||
$knowledge = Join-Path $Root "$layer\knowledge"
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates the bounded leaf-range normalization contract.
|
||||
Validates executable findings-report acceptance and bounded normalization.
|
||||
|
||||
.DESCRIPTION
|
||||
BCQuality has no executable findings-report consumer. These assertions keep
|
||||
the normative DO contract, AL coordinator, and standalone runner aligned
|
||||
while exercising the exact normalization predicate against representative
|
||||
safe and ambiguous inputs.
|
||||
These assertions keep the normative DO contract, executable validator, AL
|
||||
coordinator, and standalone runner aligned while exercising semantic report
|
||||
validation and the exact normalization predicate.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
|
|
@ -39,6 +38,24 @@ function Assert-Contains {
|
|||
Assert-True $Text.Contains($Expected) $Message
|
||||
}
|
||||
|
||||
function Assert-ThrowsLike {
|
||||
param(
|
||||
[scriptblock] $Action,
|
||||
[string] $Pattern
|
||||
)
|
||||
|
||||
try {
|
||||
& $Action
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Message -like $Pattern) {
|
||||
return
|
||||
}
|
||||
throw "Expected error like '$Pattern', received: $($_.Exception.Message)"
|
||||
}
|
||||
throw "Expected error like '$Pattern', but no error was thrown."
|
||||
}
|
||||
|
||||
function Test-PositiveInteger {
|
||||
param([object] $Value)
|
||||
|
||||
|
|
@ -168,4 +185,334 @@ Assert-True (-not ($candidateFinding.location.PSObject.Properties.Name -contains
|
|||
Assert-True ($candidateFinding.location.line -eq $rawFinding.location.line) 'candidate preserves the primary line'
|
||||
Assert-True ($candidateFinding.message -ceq $rawFinding.message) 'candidate preserves all other finding content'
|
||||
|
||||
Write-Output "Review contract validation passed ($($cases.Count) normalization cases)."
|
||||
$validator = Join-Path $Root 'tools/Validate-FindingsReport.ps1'
|
||||
Assert-True (Test-Path -LiteralPath $validator -PathType Leaf) 'executable report validator exists'
|
||||
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("reviewcontract_" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Path $tmp -Force | Out-Null
|
||||
try {
|
||||
$sourcePath = 'src/codeunit.al'
|
||||
$sourceFile = Join-Path $tmp 'src/codeunit.al'
|
||||
New-Item -ItemType Directory -Path (Split-Path -Parent $sourceFile) -Force | Out-Null
|
||||
Set-Content -LiteralPath $sourceFile -Value @('line one', 'line two', 'line three') -Encoding utf8NoBOM
|
||||
$articlePath = 'microsoft/knowledge/style/caption-required-on-page-fields.md'
|
||||
$supportingArticlePath = 'microsoft/knowledge/style/tooltip-required-on-page-fields.md'
|
||||
$reportPath = Join-Path $tmp 'report.json'
|
||||
|
||||
$validReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-style-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 1; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 1; 'items-evaluated' = 1 }
|
||||
}
|
||||
findings = @(
|
||||
[ordered]@{
|
||||
id = $articlePath
|
||||
severity = 'minor'
|
||||
message = 'A concrete style defect.'
|
||||
location = [ordered]@{
|
||||
file = $sourcePath
|
||||
line = 2
|
||||
range = [ordered]@{ 'start-line' = 2; 'end-line' = 3 }
|
||||
}
|
||||
references = @([ordered]@{ path = $articlePath })
|
||||
confidence = 'high'
|
||||
domain = 'Style'
|
||||
}
|
||||
)
|
||||
suppressed = @()
|
||||
}
|
||||
Set-Content -LiteralPath $reportPath -Value ($validReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$accepted = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
Assert-True (-not $accepted.normalized) 'valid report is accepted without normalization'
|
||||
|
||||
$invalidCounts = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$invalidCounts.summary.counts.minor = 0
|
||||
Set-Content -LiteralPath $reportPath -Value ($invalidCounts | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*COUNT_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$completedUndercoverage = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$completedUndercoverage.summary.coverage.'items-evaluated' = 0
|
||||
Set-Content -LiteralPath $reportPath -Value ($completedUndercoverage | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*COMPLETED_COVERAGE_INCOMPLETE*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$partialFullCoverage = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$partialFullCoverage.outcome = 'partial'
|
||||
$partialFullCoverage | Add-Member -NotePropertyName 'outcome-reason' -NotePropertyValue 'Stopped early.'
|
||||
Set-Content -LiteralPath $reportPath -Value ($partialFullCoverage | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*PARTIAL_COVERAGE_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$completedLeaf = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-style-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 0; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 1; 'items-evaluated' = 1 }
|
||||
}
|
||||
findings = @()
|
||||
suppressed = @()
|
||||
}
|
||||
$leafWithSubResults = $completedLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$leafWithSubResults | Add-Member -NotePropertyName 'sub-results' -NotePropertyValue @($completedLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($leafWithSubResults | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*LEAF_COMPOSITION_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root
|
||||
}
|
||||
|
||||
$validSuperReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-code-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 0; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 2; 'items-evaluated' = 2 }
|
||||
}
|
||||
findings = @()
|
||||
suppressed = @()
|
||||
'sub-results' = @($completedLeaf, $completedLeaf)
|
||||
}
|
||||
Set-Content -LiteralPath $reportPath -Value ($validSuperReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedSuper = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super
|
||||
Assert-True (-not $acceptedSuper.normalized) 'valid super-skill report is accepted'
|
||||
|
||||
$styleFindingLeaf = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$securityFindingLeaf = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$securityFindingLeaf.skill.id = 'al-security-review'
|
||||
$rolledFinding = $validReport.findings[0] | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$rolledFinding | Add-Member -NotePropertyName 'from-sub-skill' -NotePropertyValue 'al-style-review'
|
||||
$deduplicatedSuperReport = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$deduplicatedSuperReport.summary.counts.minor = 1
|
||||
$deduplicatedSuperReport.findings = @($rolledFinding)
|
||||
$deduplicatedSuperReport.'sub-results' = @($styleFindingLeaf, $securityFindingLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($deduplicatedSuperReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedDeduplicatedSuper = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
Assert-True (-not $acceptedDeduplicatedSuper.normalized) 'one top-level finding may deduplicate the same citation from two leaves'
|
||||
|
||||
$twoOccurrenceLeaf = $styleFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$secondOccurrence = $twoOccurrenceLeaf.findings[0] | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$secondOccurrence.location.line = 1
|
||||
$secondOccurrence.location.range.'start-line' = 1
|
||||
$secondOccurrence.location.range.'end-line' = 1
|
||||
$twoOccurrenceLeaf.findings = @($twoOccurrenceLeaf.findings[0], $secondOccurrence)
|
||||
$twoOccurrenceLeaf.summary.counts.minor = 2
|
||||
$emptySecurityLeaf = $completedLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$emptySecurityLeaf.skill.id = 'al-security-review'
|
||||
$sameIdOccurrenceOmitted = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$sameIdOccurrenceOmitted.'sub-results' = @($twoOccurrenceLeaf, $emptySecurityLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($sameIdOccurrenceOmitted | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISSING*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$mergeOwnerLeaf = $styleFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$mergeOwnerLeaf.findings[0] | Add-Member -NotePropertyName 'suggested-code' -NotePropertyValue 'Caption = ''Customer name'';'
|
||||
$supportingFindingLeaf = $securityFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$supportingFindingLeaf.findings[0].id = $supportingArticlePath
|
||||
$supportingFindingLeaf.findings[0].references[0].path = $supportingArticlePath
|
||||
$supportingFindingLeaf.findings[0].confidence = 'medium'
|
||||
$supportingFindingLeaf.findings[0].message = 'The field needs the same mechanical correction for a supporting rule.'
|
||||
$supportingFindingLeaf.findings[0] | Add-Member -NotePropertyName 'suggested-code' -NotePropertyValue 'Caption = ''Customer name'';'
|
||||
$mergedFinding = $rolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$mergedFinding | Add-Member -NotePropertyName 'suggested-code' -NotePropertyValue 'Caption = ''Customer name'';'
|
||||
$mergedFinding.references = @(
|
||||
[pscustomobject]@{ path = $articlePath }
|
||||
[pscustomobject]@{ path = $supportingArticlePath }
|
||||
)
|
||||
$mergedSuperReport = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$mergedSuperReport.findings = @($mergedFinding)
|
||||
$mergedSuperReport.'sub-results' = @($mergeOwnerLeaf, $supportingFindingLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($mergedSuperReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedMergedSuper = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
Assert-True (-not $acceptedMergedSuper.normalized) 'overlapping A and B findings may merge into A with B as a supporting reference'
|
||||
|
||||
$textOnlyOwnerLeaf = $mergeOwnerLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$textOnlyOwnerLeaf.findings[0].PSObject.Properties.Remove('suggested-code')
|
||||
$textOnlySupportingLeaf = $supportingFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$textOnlySupportingLeaf.findings[0].PSObject.Properties.Remove('suggested-code')
|
||||
$textOnlyMergedFinding = $mergedFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$textOnlyMergedFinding.PSObject.Properties.Remove('suggested-code')
|
||||
$textOnlyMergedSuper = $mergedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$textOnlyMergedSuper.findings = @($textOnlyMergedFinding)
|
||||
$textOnlyMergedSuper.'sub-results' = @($textOnlyOwnerLeaf, $textOnlySupportingLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($textOnlyMergedSuper | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedTextOnlyMerge = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
Assert-True (-not $acceptedTextOnlyMerge.normalized) 'supporting references permit an overlapping A and B merge with different messages and no suggested code'
|
||||
|
||||
$conflictingSupportingLeaf = $supportingFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$conflictingSupportingLeaf.findings[0].'suggested-code' = 'ToolTip = ''Customer name'';'
|
||||
$conflictingCorrectionMerge = $mergedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$conflictingCorrectionMerge.'sub-results' = @($mergeOwnerLeaf, $conflictingSupportingLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($conflictingCorrectionMerge | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISSING*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
}
|
||||
|
||||
$omittedConflictingCorrectionMerge = $conflictingCorrectionMerge | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$omittedConflictingCorrectionMerge.findings[0].PSObject.Properties.Remove('suggested-code')
|
||||
Set-Content -LiteralPath $reportPath -Value ($omittedConflictingCorrectionMerge | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
}
|
||||
|
||||
$unmergedSupportingFinding = $supportingFindingLeaf.findings[0] | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$unmergedSupportingFinding | Add-Member -NotePropertyName 'from-sub-skill' -NotePropertyValue 'al-security-review'
|
||||
$unmergedDuplicates = $mergedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$unmergedDuplicates.summary.counts.minor = 2
|
||||
$unmergedDuplicates.findings = @($mergedFinding, $unmergedSupportingFinding)
|
||||
Set-Content -LiteralPath $reportPath -Value ($unmergedDuplicates | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_DUPLICATE_FINDINGS*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
}
|
||||
|
||||
$omittedLeafFinding = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$omittedLeafFinding.summary.counts.minor = 0
|
||||
$omittedLeafFinding.findings = @()
|
||||
Set-Content -LiteralPath $reportPath -Value ($omittedLeafFinding | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISSING*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$locationlessLeaf = $styleFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$locationlessLeaf.findings[0].PSObject.Properties.Remove('location')
|
||||
$secondLocationlessFinding = $locationlessLeaf.findings[0] | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$locationlessLeaf.findings = @($locationlessLeaf.findings[0], $secondLocationlessFinding)
|
||||
$locationlessLeaf.summary.counts.minor = 2
|
||||
$locationlessRolledFinding = $rolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$locationlessRolledFinding.PSObject.Properties.Remove('location')
|
||||
$locationlessOmission = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$locationlessOmission.findings = @($locationlessRolledFinding)
|
||||
$locationlessOmission.'sub-results' = @($locationlessLeaf, $emptySecurityLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($locationlessOmission | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISSING*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$completeLocationlessRollup = $locationlessOmission | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$completeLocationlessRollup.summary.counts.minor = 2
|
||||
$completeLocationlessRollup.findings = @(
|
||||
$locationlessRolledFinding
|
||||
($locationlessRolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json)
|
||||
)
|
||||
Set-Content -LiteralPath $reportPath -Value ($completeLocationlessRollup | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedLocationlessRollup = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
Assert-True (-not $acceptedLocationlessRollup.normalized) 'two locationless leaf occurrences require and accept two distinct rolled findings'
|
||||
|
||||
$nonexistentProducer = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$nonexistentProducer.findings[0].'from-sub-skill' = 'al-missing-review'
|
||||
Set-Content -LiteralPath $reportPath -Value ($nonexistentProducer | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_PRODUCER_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$rewrittenLeafFinding = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$rewrittenLeafFinding.findings[0].message = 'A rewritten rollup message.'
|
||||
Set-Content -LiteralPath $reportPath -Value ($rewrittenLeafFinding | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$failedLeaf = $completedLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$failedLeaf.skill.id = 'al-security-review'
|
||||
$failedLeaf.outcome = 'failed'
|
||||
$failedLeaf | Add-Member -NotePropertyName 'outcome-reason' -NotePropertyValue 'Validation failed.'
|
||||
$failedLeaf.summary.coverage.'items-evaluated' = 0
|
||||
$partialSuperReport = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$partialSuperReport.outcome = 'partial'
|
||||
$partialSuperReport | Add-Member -NotePropertyName 'outcome-reason' -NotePropertyValue 'One sub-skill failed.'
|
||||
$partialSuperReport.summary.coverage.'worklist-size' = 1
|
||||
$partialSuperReport.summary.coverage.'items-evaluated' = 1
|
||||
$partialSuperReport.'sub-results' = @($completedLeaf, $failedLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($partialSuperReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedPartialSuper = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super
|
||||
Assert-True (-not $acceptedPartialSuper.normalized) 'partial super-skill excludes failed coverage from its rollup'
|
||||
|
||||
$failedLeafLeakage = $partialSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$failedLeafLeakage.summary.counts.minor = 1
|
||||
$failedLeafLeakage.findings = @($rolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json)
|
||||
$failedLeafLeakage.findings[0].'from-sub-skill' = 'al-security-review'
|
||||
Set-Content -LiteralPath $reportPath -Value ($failedLeafLeakage | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FAILED_FINDING_LEAKAGE*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$failedLeafRelabeledAsAgent = $partialSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$failedLeafRelabeledAsAgent.summary.counts.minor = 1
|
||||
$failedLeafRelabeledAsAgent.findings = @($rolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json)
|
||||
$failedLeafRelabeledAsAgent.findings[0].'from-sub-skill' = 'agent'
|
||||
$failedLeafRelabeledAsAgent.findings[0].domain = 'Agent'
|
||||
Set-Content -LiteralPath $reportPath -Value ($failedLeafRelabeledAsAgent | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_AGENT_FINDING_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$incorrectOutcome = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$incorrectOutcome.outcome = 'not-applicable'
|
||||
Set-Content -LiteralPath $reportPath -Value ($incorrectOutcome | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_OUTCOME_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super
|
||||
}
|
||||
|
||||
$incorrectRollup = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$incorrectRollup.summary.coverage.'worklist-size' = 1
|
||||
$incorrectRollup.summary.coverage.'items-evaluated' = 1
|
||||
Set-Content -LiteralPath $reportPath -Value ($incorrectRollup | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_COVERAGE_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super
|
||||
}
|
||||
|
||||
Set-Content -LiteralPath $reportPath -Value ($validReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*REFERENCE_NOT_RETRIEVED*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp -SourcePaths $sourcePath
|
||||
}
|
||||
|
||||
$invalidAgent = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$invalidAgent.findings[0].id = 'agent:uncited-defect'
|
||||
$invalidAgent.findings[0].references = @()
|
||||
$invalidAgent.findings[0].confidence = 'high'
|
||||
Set-Content -LiteralPath $reportPath -Value ($invalidAgent | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*AGENT_CONFIDENCE_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp -SourcePaths $sourcePath
|
||||
}
|
||||
|
||||
$normalizable = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$normalizable.findings[0].location.range.'start-line' = 1
|
||||
Set-Content -LiteralPath $reportPath -Value ($normalizable | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*RANGE_START_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
$normalized = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath -AllowBoundedNormalization
|
||||
Assert-True $normalized.normalized 'eligible range mismatch is normalized'
|
||||
Assert-True ($normalized.removedRanges.Count -eq 1) 'normalization records one removed range'
|
||||
Assert-True (-not ($normalized.report.findings[0].location.PSObject.Properties.Name -contains 'range')) `
|
||||
'accepted normalized report removes only the optional range'
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
Write-Output "Review contract validation passed ($($cases.Count) predicate cases plus executable acceptance cases)."
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@ param(
|
|||
[string] $ManifestPath,
|
||||
[string] $PrepareDirectory,
|
||||
[string] $ResultsPath,
|
||||
[string] $ResultsDirectory
|
||||
[string] $ResultsDirectory,
|
||||
[string] $ChangedPathsFile,
|
||||
[string] $CoverageReportPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
|
|
@ -160,7 +162,23 @@ foreach ($overrideDomain in $overrides.Keys) {
|
|||
}
|
||||
}
|
||||
|
||||
$coverageWaivers = @{}
|
||||
if ($manifest.PSObject.Properties.Name -contains 'coverageWaivers') {
|
||||
foreach ($waiver in @($manifest.coverageWaivers)) {
|
||||
if (-not $waiver.path -or -not $waiver.reason) {
|
||||
$problems.Add('Each coverage waiver requires non-empty path and reason values.') | Out-Null
|
||||
continue
|
||||
}
|
||||
if ($coverageWaivers.ContainsKey([string]$waiver.path)) {
|
||||
$problems.Add("Duplicate coverage waiver: $($waiver.path)") | Out-Null
|
||||
continue
|
||||
}
|
||||
$coverageWaivers[[string]$waiver.path] = [string]$waiver.reason
|
||||
}
|
||||
}
|
||||
|
||||
$caseList = [System.Collections.Generic.List[object]]::new()
|
||||
$pairedArticlesByDomain = @{}
|
||||
foreach ($domain in $leafDomains) {
|
||||
$articleCandidates = @(
|
||||
foreach ($layer in $layers) {
|
||||
|
|
@ -189,6 +207,7 @@ foreach ($domain in $leafDomains) {
|
|||
ForEach-Object { $_.Group | Sort-Object Rank -Descending | Select-Object -First 1 } |
|
||||
Sort-Object BaseName
|
||||
)
|
||||
$pairedArticlesByDomain[$domain] = @($articles)
|
||||
if (-not $articles.Count) {
|
||||
$problems.Add("${domain}: no enabled knowledge layer has an article with both .good.al and .bad.al companion samples.") | Out-Null
|
||||
continue
|
||||
|
|
@ -339,6 +358,65 @@ foreach ($domain in $leafDomains) {
|
|||
}
|
||||
}
|
||||
|
||||
$selectedArticlePaths = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($case in $cases) {
|
||||
foreach ($reference in @($case.expected)) {
|
||||
$selectedArticlePaths.Add([string]$reference) | Out-Null
|
||||
}
|
||||
}
|
||||
$effectivePairedPaths = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
$coverageDomains = @(
|
||||
foreach ($domain in $leafDomains) {
|
||||
$paired = @($pairedArticlesByDomain[$domain])
|
||||
foreach ($article in $paired) {
|
||||
$effectivePairedPaths.Add([string]$article.ArticlePath) | Out-Null
|
||||
}
|
||||
$selected = @($paired | Where-Object { $selectedArticlePaths.Contains([string]$_.ArticlePath) }).Count
|
||||
[pscustomobject][ordered]@{
|
||||
domain = $domain
|
||||
pairedArticles = $paired.Count
|
||||
selectedArticles = $selected
|
||||
coverage = if ($paired.Count) { $selected / $paired.Count } else { 0 }
|
||||
}
|
||||
}
|
||||
)
|
||||
$pairedTotal = ($coverageDomains | Measure-Object pairedArticles -Sum).Sum
|
||||
$selectedTotal = ($coverageDomains | Measure-Object selectedArticles -Sum).Sum
|
||||
$coverageReport = [pscustomobject][ordered]@{
|
||||
pairedArticles = $pairedTotal
|
||||
selectedArticles = $selectedTotal
|
||||
coverage = if ($pairedTotal) { $selectedTotal / $pairedTotal } else { 0 }
|
||||
domains = $coverageDomains
|
||||
}
|
||||
if ($CoverageReportPath) {
|
||||
$coverageParent = Split-Path -Parent $CoverageReportPath
|
||||
if ($coverageParent -and -not (Test-Path -LiteralPath $coverageParent)) {
|
||||
New-Item -ItemType Directory -Path $coverageParent -Force | Out-Null
|
||||
}
|
||||
$coverageReport | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $CoverageReportPath -Encoding utf8NoBOM
|
||||
}
|
||||
|
||||
if ($ChangedPathsFile) {
|
||||
if (-not (Test-Path -LiteralPath $ChangedPathsFile -PathType Leaf)) {
|
||||
$problems.Add("Changed paths file not found: $ChangedPathsFile") | Out-Null
|
||||
}
|
||||
else {
|
||||
foreach ($changedPathValue in Get-Content -LiteralPath $ChangedPathsFile) {
|
||||
$changedPath = ([string]$changedPathValue).Trim().Replace('\', '/')
|
||||
if ($changedPath -notmatch '^(microsoft|community|custom)/knowledge/[^/]+/(.+?)(?:\.(?:good|bad)\.al|\.md)$') {
|
||||
continue
|
||||
}
|
||||
$articlePath = "$($Matches[1])/knowledge/$($changedPath.Split('/')[2])/$($Matches[2]).md"
|
||||
if (-not $effectivePairedPaths.Contains($articlePath) -or $selectedArticlePaths.Contains($articlePath)) {
|
||||
continue
|
||||
}
|
||||
if (-not $coverageWaivers.ContainsKey($articlePath)) {
|
||||
$problems.Add("Changed paired article is not selected for evaluation and has no coverage waiver: $articlePath") | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($problems.Count) {
|
||||
Write-Host "Review fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
||||
|
|
@ -474,7 +552,7 @@ if ($PrepareDirectory) {
|
|||
|
||||
if (-not $ResultsPath -and -not $ResultsDirectory) {
|
||||
& (Join-Path $PSScriptRoot 'Test-ReviewContract.ps1') -Root $Root
|
||||
Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $($leafDomains.Count) leaf domains." -ForegroundColor Green
|
||||
Write-Host "Review fixture validation PASSED: $($cases.Count) cases cover $selectedTotal/$pairedTotal paired articles across $($leafDomains.Count) leaf domains." -ForegroundColor Green
|
||||
exit 0
|
||||
}
|
||||
|
||||
|
|
|
|||
540
tools/Validate-FindingsReport.ps1
Normal file
540
tools/Validate-FindingsReport.ps1
Normal file
|
|
@ -0,0 +1,540 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates a BCQuality findings-report against its structural and semantic contract.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $ReportPath,
|
||||
[string] $BCQualityRoot,
|
||||
[string] $SourceRoot,
|
||||
[string[]] $SourcePaths = @(),
|
||||
[string[]] $RetrievedArticlePaths = @(),
|
||||
[ValidateSet('leaf', 'super')]
|
||||
[string] $SkillKind = 'leaf',
|
||||
[switch] $AllowBoundedNormalization
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not $BCQualityRoot) {
|
||||
$BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
|
||||
}
|
||||
$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path
|
||||
$schemaPath = Join-Path $BCQualityRoot 'schemas/findings-report.schema.json'
|
||||
$raw = Get-Content -LiteralPath $ReportPath -Raw
|
||||
try {
|
||||
if (-not ($raw | Test-Json -SchemaFile $schemaPath -ErrorAction Stop)) {
|
||||
throw 'Report does not satisfy schemas/findings-report.schema.json.'
|
||||
}
|
||||
$report = $raw | ConvertFrom-Json -Depth 100
|
||||
}
|
||||
catch {
|
||||
throw "Invalid findings-report JSON or schema: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
$retrieved = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($path in $RetrievedArticlePaths) {
|
||||
$retrieved.Add($path) | Out-Null
|
||||
}
|
||||
$sources = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($path in $SourcePaths) {
|
||||
$sources.Add($path) | Out-Null
|
||||
}
|
||||
$lineCounts = @{}
|
||||
|
||||
function Test-HasProperty {
|
||||
param([object] $Object, [string] $Name)
|
||||
return $null -ne $Object -and $Object.PSObject.Properties.Name -ccontains $Name
|
||||
}
|
||||
|
||||
function Get-SourceLineCount {
|
||||
param([string] $Path)
|
||||
|
||||
if ($lineCounts.ContainsKey($Path)) {
|
||||
return $lineCounts[$Path]
|
||||
}
|
||||
if (-not $SourceRoot) {
|
||||
return -1
|
||||
}
|
||||
$fullPath = Join-Path $SourceRoot ($Path -replace '/', [IO.Path]::DirectorySeparatorChar)
|
||||
if (-not (Test-Path -LiteralPath $fullPath -PathType Leaf)) {
|
||||
return -1
|
||||
}
|
||||
$lineCounts[$Path] = [IO.File]::ReadAllLines($fullPath).Count
|
||||
return $lineCounts[$Path]
|
||||
}
|
||||
|
||||
function Get-SemanticErrors {
|
||||
param(
|
||||
[object] $Candidate,
|
||||
[switch] $PermitRangeStartMismatch
|
||||
)
|
||||
|
||||
$errors = [Collections.Generic.List[object]]::new()
|
||||
function Add-Error {
|
||||
param([string] $Code, [string] $Path, [string] $Message)
|
||||
$errors.Add([pscustomobject]@{ Code = $Code; Path = $Path; Message = $Message }) | Out-Null
|
||||
}
|
||||
|
||||
function Get-DerivedSuperOutcome {
|
||||
param([object[]] $SubResults)
|
||||
|
||||
if (-not $SubResults.Count) {
|
||||
return 'not-applicable'
|
||||
}
|
||||
|
||||
$outcomes = @($SubResults | ForEach-Object { $_.outcome })
|
||||
if (-not @($outcomes | Where-Object { $_ -cne 'failed' }).Count) {
|
||||
return 'failed'
|
||||
}
|
||||
if (($outcomes -ccontains 'partial') -or
|
||||
(($outcomes -ccontains 'failed') -and @($outcomes | Where-Object { $_ -cne 'failed' }).Count)) {
|
||||
return 'partial'
|
||||
}
|
||||
if (-not @($outcomes | Where-Object { $_ -cne 'not-applicable' }).Count) {
|
||||
return 'not-applicable'
|
||||
}
|
||||
if (($outcomes -ccontains 'no-knowledge') -and
|
||||
-not @($outcomes | Where-Object { $_ -cnotin @('no-knowledge', 'not-applicable') }).Count) {
|
||||
return 'no-knowledge'
|
||||
}
|
||||
return 'completed'
|
||||
}
|
||||
|
||||
function Get-SeverityRank {
|
||||
param([string] $Severity)
|
||||
return @{'info' = 0; 'minor' = 1; 'major' = 2; 'blocker' = 3}[$Severity]
|
||||
}
|
||||
|
||||
function Get-ConfidenceRank {
|
||||
param([string] $Confidence)
|
||||
return @{'low' = 0; 'medium' = 1; 'high' = 2}[$Confidence]
|
||||
}
|
||||
|
||||
function Test-LocationsOverlap {
|
||||
param([object] $First, [object] $Second)
|
||||
|
||||
$firstHasLocation = Test-HasProperty $First 'location'
|
||||
$secondHasLocation = Test-HasProperty $Second 'location'
|
||||
if ($firstHasLocation -ne $secondHasLocation) {
|
||||
return $false
|
||||
}
|
||||
if (-not $firstHasLocation) {
|
||||
return $false
|
||||
}
|
||||
if ($First.location.file -cne $Second.location.file) {
|
||||
return $false
|
||||
}
|
||||
$firstEnd = if (Test-HasProperty $First.location 'range') { $First.location.range.'end-line' } else { $First.location.line }
|
||||
$secondEnd = if (Test-HasProperty $Second.location 'range') { $Second.location.range.'end-line' } else { $Second.location.line }
|
||||
return $First.location.line -le $secondEnd -and $Second.location.line -le $firstEnd
|
||||
}
|
||||
|
||||
function Test-SameCorrection {
|
||||
param([object] $First, [object] $Second)
|
||||
|
||||
$firstHasCode = Test-HasProperty $First 'suggested-code'
|
||||
$secondHasCode = Test-HasProperty $Second 'suggested-code'
|
||||
if ($firstHasCode -or $secondHasCode) {
|
||||
return $firstHasCode -and $secondHasCode -and $First.'suggested-code' -ceq $Second.'suggested-code'
|
||||
}
|
||||
return $First.message -ceq $Second.message
|
||||
}
|
||||
|
||||
function Test-ReferencesInclude {
|
||||
param([object[]] $RolledReferences, [object[]] $LeafReferences)
|
||||
|
||||
foreach ($leafReference in $LeafReferences) {
|
||||
$matched = @($RolledReferences | Where-Object {
|
||||
if ($_.path -cne $leafReference.path) {
|
||||
return $false
|
||||
}
|
||||
$rolledHasSha = Test-HasProperty $_ 'sha'
|
||||
$leafHasSha = Test-HasProperty $leafReference 'sha'
|
||||
return $rolledHasSha -eq $leafHasSha -and
|
||||
(-not $rolledHasSha -or $_.sha -ceq $leafReference.sha)
|
||||
}).Count
|
||||
if (-not $matched) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-RolledFindingRepresents {
|
||||
param(
|
||||
[object] $RolledFinding,
|
||||
[object] $LeafFinding,
|
||||
[string] $LeafProducerId,
|
||||
[switch] $RequirePrimaryOwner
|
||||
)
|
||||
|
||||
$rolledHasLocation = Test-HasProperty $RolledFinding 'location'
|
||||
$leafHasLocation = Test-HasProperty $LeafFinding 'location'
|
||||
$leafReferences = @($LeafFinding.references)
|
||||
$rolledReferences = @($RolledFinding.references)
|
||||
if (-not $rolledHasLocation -and -not $leafHasLocation) {
|
||||
$expectedId = if ($leafReferences.Count) { $LeafFinding.id } else { "${LeafProducerId}:$($LeafFinding.id)" }
|
||||
if ($RolledFinding.'from-sub-skill' -cne $LeafProducerId -or
|
||||
$RolledFinding.id -cne $expectedId -or
|
||||
$RolledFinding.severity -cne $LeafFinding.severity -or
|
||||
$RolledFinding.confidence -cne $LeafFinding.confidence -or
|
||||
$RolledFinding.message -cne $LeafFinding.message -or
|
||||
$rolledReferences.Count -ne $leafReferences.Count -or
|
||||
-not (Test-ReferencesInclude $rolledReferences $leafReferences)) {
|
||||
return $false
|
||||
}
|
||||
foreach ($name in 'domain', 'suggested-code', 'suggested-code-omission-reason') {
|
||||
$rolledHasProperty = Test-HasProperty $RolledFinding $name
|
||||
$leafHasProperty = Test-HasProperty $LeafFinding $name
|
||||
if ($rolledHasProperty -ne $leafHasProperty -or
|
||||
($rolledHasProperty -and $RolledFinding.$name -cne $LeafFinding.$name)) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
if (-not (Test-LocationsOverlap $RolledFinding $LeafFinding) -or
|
||||
(Get-SeverityRank $RolledFinding.severity) -lt (Get-SeverityRank $LeafFinding.severity) -or
|
||||
(Get-ConfidenceRank $RolledFinding.confidence) -lt (Get-ConfidenceRank $LeafFinding.confidence)) {
|
||||
return $false
|
||||
}
|
||||
|
||||
$sameCorrection = Test-SameCorrection $RolledFinding $LeafFinding
|
||||
$correctionsConflict = (Test-HasProperty $RolledFinding 'suggested-code') -and
|
||||
(Test-HasProperty $LeafFinding 'suggested-code') -and
|
||||
$RolledFinding.'suggested-code' -cne $LeafFinding.'suggested-code'
|
||||
$explicitCrossRuleMerge = $leafReferences.Count -and
|
||||
$rolledReferences.Count -gt $leafReferences.Count -and
|
||||
-not $correctionsConflict -and
|
||||
(Test-ReferencesInclude $rolledReferences $leafReferences)
|
||||
if (-not $sameCorrection -and -not $explicitCrossRuleMerge) {
|
||||
return $false
|
||||
}
|
||||
|
||||
if (-not $leafReferences.Count) {
|
||||
return $RolledFinding.'from-sub-skill' -ceq $LeafProducerId -and
|
||||
$RolledFinding.id -ceq "${LeafProducerId}:$($LeafFinding.id)" -and
|
||||
-not $rolledReferences.Count
|
||||
}
|
||||
if (-not (Test-ReferencesInclude $rolledReferences $leafReferences)) {
|
||||
return $false
|
||||
}
|
||||
if ($RequirePrimaryOwner) {
|
||||
$rolledHasDomain = Test-HasProperty $RolledFinding 'domain'
|
||||
$leafHasDomain = Test-HasProperty $LeafFinding 'domain'
|
||||
return $RolledFinding.'from-sub-skill' -ceq $LeafProducerId -and
|
||||
$RolledFinding.id -ceq $LeafFinding.id -and
|
||||
@($RolledFinding.references)[0].path -ceq $leafReferences[0].path -and
|
||||
$rolledHasDomain -eq $leafHasDomain -and
|
||||
(-not $rolledHasDomain -or $RolledFinding.domain -ceq $LeafFinding.domain)
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-Report {
|
||||
param(
|
||||
[object] $Current,
|
||||
[string] $ReportPathPrefix,
|
||||
[ValidateSet('leaf', 'super')]
|
||||
[string] $CurrentSkillKind
|
||||
)
|
||||
|
||||
$findings = @($Current.findings)
|
||||
foreach ($severity in 'blocker', 'major', 'minor', 'info') {
|
||||
$actual = @($findings | Where-Object severity -CEQ $severity).Count
|
||||
if ($Current.summary.counts.$severity -ne $actual) {
|
||||
Add-Error 'COUNT_MISMATCH' "$ReportPathPrefix.summary.counts.$severity" "Expected $actual."
|
||||
}
|
||||
}
|
||||
$worklistSize = $Current.summary.coverage.'worklist-size'
|
||||
$itemsEvaluated = $Current.summary.coverage.'items-evaluated'
|
||||
if ($itemsEvaluated -gt $worklistSize) {
|
||||
Add-Error 'COVERAGE_INVALID' "$ReportPathPrefix.summary.coverage" 'items-evaluated exceeds worklist-size.'
|
||||
}
|
||||
elseif ($Current.outcome -ceq 'completed' -and $itemsEvaluated -ne $worklistSize) {
|
||||
Add-Error 'COMPLETED_COVERAGE_INCOMPLETE' "$ReportPathPrefix.summary.coverage" 'A completed report must evaluate its full worklist.'
|
||||
}
|
||||
elseif ($CurrentSkillKind -ceq 'leaf' -and $Current.outcome -ceq 'partial' -and
|
||||
($itemsEvaluated -le 0 -or $itemsEvaluated -ge $worklistSize)) {
|
||||
Add-Error 'PARTIAL_COVERAGE_INVALID' "$ReportPathPrefix.summary.coverage" 'A partial report must evaluate a non-zero proper subset of its worklist.'
|
||||
}
|
||||
|
||||
$hasSubResults = Test-HasProperty $Current 'sub-results'
|
||||
$hasSkippedSubSkills = Test-HasProperty $Current 'skipped-sub-skills'
|
||||
if ($CurrentSkillKind -ceq 'leaf') {
|
||||
if ($hasSubResults -or $hasSkippedSubSkills) {
|
||||
Add-Error 'LEAF_COMPOSITION_INVALID' $ReportPathPrefix 'A leaf report must not contain sub-results or skipped-sub-skills.'
|
||||
}
|
||||
}
|
||||
elseif (-not $hasSubResults) {
|
||||
Add-Error 'SUPER_SUB_RESULTS_REQUIRED' $ReportPathPrefix 'A super-skill report must contain sub-results.'
|
||||
}
|
||||
|
||||
for ($index = 0; $index -lt $findings.Count; $index++) {
|
||||
$finding = $findings[$index]
|
||||
$findingPath = "$ReportPathPrefix.findings[$index]"
|
||||
$references = @($finding.references)
|
||||
$hasProducer = Test-HasProperty $finding 'from-sub-skill'
|
||||
if ($CurrentSkillKind -ceq 'leaf' -and $hasProducer) {
|
||||
Add-Error 'LEAF_PRODUCER_INVALID' "$findingPath.from-sub-skill" 'A leaf finding must not contain from-sub-skill.'
|
||||
}
|
||||
elseif ($CurrentSkillKind -ceq 'super' -and -not $hasProducer) {
|
||||
Add-Error 'SUPER_PRODUCER_REQUIRED' $findingPath 'A super-skill finding must identify its producer in from-sub-skill.'
|
||||
}
|
||||
if (-not $references.Count) {
|
||||
if ($finding.id -cnotmatch '(^|:)agent:[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
||||
Add-Error 'AGENT_ID_INVALID' "$findingPath.id" 'An agent finding id must contain an agent: slug marker.'
|
||||
}
|
||||
if ($finding.confidence -ceq 'high') {
|
||||
Add-Error 'AGENT_CONFIDENCE_INVALID' "$findingPath.confidence" 'Agent confidence cannot be high.'
|
||||
}
|
||||
if ($finding.severity -cin @('blocker', 'major')) {
|
||||
Add-Error 'AGENT_SEVERITY_INVALID' "$findingPath.severity" 'Agent severity cannot exceed minor.'
|
||||
}
|
||||
}
|
||||
else {
|
||||
if ($finding.id -cne $references[0].path) {
|
||||
Add-Error 'PRIMARY_REFERENCE_MISMATCH' "$findingPath.id" 'Finding id must equal the primary reference path.'
|
||||
}
|
||||
foreach ($reference in $references) {
|
||||
if ($reference.path -cnotmatch '^(microsoft|community|custom)/knowledge/.+\.md$') {
|
||||
Add-Error 'REFERENCE_PATH_INVALID' "$findingPath.references" "Invalid knowledge path '$($reference.path)'."
|
||||
continue
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $BCQualityRoot $reference.path) -PathType Leaf)) {
|
||||
Add-Error 'REFERENCE_MISSING' "$findingPath.references" "Knowledge path '$($reference.path)' does not exist."
|
||||
}
|
||||
if (-not $retrieved.Contains($reference.path)) {
|
||||
Add-Error 'REFERENCE_NOT_RETRIEVED' "$findingPath.references" "Knowledge path '$($reference.path)' was not retrieved in full."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Test-HasProperty $finding 'location') {
|
||||
$location = $finding.location
|
||||
if (-not $sources.Contains($location.file)) {
|
||||
Add-Error 'SOURCE_OUT_OF_SCOPE' "$findingPath.location.file" "Source path '$($location.file)' is outside the supplied scope."
|
||||
}
|
||||
$lineCount = Get-SourceLineCount $location.file
|
||||
if ($lineCount -lt 0) {
|
||||
Add-Error 'SOURCE_MISSING' "$findingPath.location.file" "Source path '$($location.file)' does not exist."
|
||||
}
|
||||
elseif ($location.line -gt $lineCount) {
|
||||
Add-Error 'SOURCE_LINE_INVALID' "$findingPath.location.line" "Line exceeds the file's $lineCount lines."
|
||||
}
|
||||
|
||||
if (Test-HasProperty $location 'range') {
|
||||
$range = $location.range
|
||||
if ($range.'start-line' -ne $location.line) {
|
||||
Add-Error 'RANGE_START_MISMATCH' "$findingPath.location.range.start-line" 'start-line must equal line.'
|
||||
}
|
||||
if ($range.'end-line' -lt $range.'start-line' -or
|
||||
($lineCount -ge 0 -and $range.'end-line' -gt $lineCount)) {
|
||||
Add-Error 'SOURCE_RANGE_INVALID' "$findingPath.location.range" 'Range is reversed or exceeds the source file.'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($CurrentSkillKind -ceq 'super' -and $hasSubResults) {
|
||||
$subResults = @($Current.'sub-results')
|
||||
for ($index = 0; $index -lt $subResults.Count; $index++) {
|
||||
Test-Report $subResults[$index] "$ReportPathPrefix.sub-results[$index]" 'leaf'
|
||||
}
|
||||
|
||||
$expectedOutcome = Get-DerivedSuperOutcome $subResults
|
||||
if ($Current.outcome -cne $expectedOutcome) {
|
||||
Add-Error 'SUPER_OUTCOME_MISMATCH' "$ReportPathPrefix.outcome" "Expected '$expectedOutcome' from sub-results."
|
||||
}
|
||||
|
||||
$includedSubResults = @($subResults | Where-Object outcome -CNE 'failed')
|
||||
$expectedWorklistSize = ($includedSubResults | Measure-Object -Property { $_.summary.coverage.'worklist-size' } -Sum).Sum
|
||||
$expectedItemsEvaluated = ($includedSubResults | Measure-Object -Property { $_.summary.coverage.'items-evaluated' } -Sum).Sum
|
||||
if ($null -eq $expectedWorklistSize) { $expectedWorklistSize = 0 }
|
||||
if ($null -eq $expectedItemsEvaluated) { $expectedItemsEvaluated = 0 }
|
||||
if ($worklistSize -ne $expectedWorklistSize -or $itemsEvaluated -ne $expectedItemsEvaluated) {
|
||||
Add-Error 'SUPER_COVERAGE_MISMATCH' "$ReportPathPrefix.summary.coverage" `
|
||||
"Expected worklist-size $expectedWorklistSize and items-evaluated $expectedItemsEvaluated from non-failed sub-results."
|
||||
}
|
||||
|
||||
$failedProducerIds = @($subResults | Where-Object outcome -CEQ 'failed' | ForEach-Object { $_.skill.id })
|
||||
$includedProducerIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
$eligibleFindings = [Collections.Generic.List[object]]::new()
|
||||
foreach ($subResult in $includedSubResults) {
|
||||
$includedProducerIds.Add([string]$subResult.skill.id) | Out-Null
|
||||
foreach ($finding in @($subResult.findings)) {
|
||||
$eligibleFindings.Add([pscustomobject]@{
|
||||
ProducerId = [string]$subResult.skill.id
|
||||
Finding = $finding
|
||||
}) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
for ($index = 0; $index -lt $findings.Count; $index++) {
|
||||
$finding = $findings[$index]
|
||||
$producerId = [string]$finding.'from-sub-skill'
|
||||
if ($producerId -ceq 'agent') {
|
||||
if (@($finding.references).Count -or
|
||||
$finding.id -cnotmatch '^agent:[a-z0-9]+(?:-[a-z0-9]+)*$' -or
|
||||
-not (Test-HasProperty $finding 'domain') -or
|
||||
$finding.domain -cne 'Agent') {
|
||||
Add-Error 'SUPER_AGENT_FINDING_INVALID' "$ReportPathPrefix.findings[$index]" `
|
||||
'A super-skill agent finding must use an agent: id, Agent domain, and no references.'
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ($producerId -cin $failedProducerIds) {
|
||||
Add-Error 'SUPER_FAILED_FINDING_LEAKAGE' "$ReportPathPrefix.findings[$index].from-sub-skill" `
|
||||
"Finding is attributed to failed sub-skill '$producerId'."
|
||||
continue
|
||||
}
|
||||
if (-not $includedProducerIds.Contains($producerId)) {
|
||||
Add-Error 'SUPER_PRODUCER_INVALID' "$ReportPathPrefix.findings[$index].from-sub-skill" `
|
||||
"Sub-skill '$producerId' has no non-failed result."
|
||||
continue
|
||||
}
|
||||
$ownedLeafFindings = @($eligibleFindings | Where-Object {
|
||||
$_.ProducerId -ceq $producerId -and
|
||||
(Test-RolledFindingRepresents $finding $_.Finding $_.ProducerId -RequirePrimaryOwner)
|
||||
})
|
||||
if (-not $ownedLeafFindings.Count) {
|
||||
Add-Error 'SUPER_FINDING_MISMATCH' "$ReportPathPrefix.findings[$index]" `
|
||||
"Rolled-up finding '$($finding.id)' is not owned by a matching finding from '$producerId'."
|
||||
continue
|
||||
}
|
||||
$representedLeafFindings = @($eligibleFindings | Where-Object {
|
||||
Test-RolledFindingRepresents $finding $_.Finding $_.ProducerId
|
||||
})
|
||||
$representedCorrections = @(
|
||||
$representedLeafFindings |
|
||||
Where-Object { Test-HasProperty $_.Finding 'suggested-code' } |
|
||||
ForEach-Object { $_.Finding.'suggested-code' } |
|
||||
Sort-Object -CaseSensitive -Unique
|
||||
)
|
||||
if ($representedCorrections.Count -gt 1) {
|
||||
Add-Error 'SUPER_FINDING_MISMATCH' "$ReportPathPrefix.findings[$index]" `
|
||||
"Rolled-up finding '$($finding.id)' represents leaf findings with conflicting suggested-code replacements."
|
||||
continue
|
||||
}
|
||||
$expectedSeverityRank = ($representedLeafFindings | ForEach-Object { Get-SeverityRank $_.Finding.severity } | Measure-Object -Maximum).Maximum
|
||||
$expectedConfidenceRank = ($representedLeafFindings | ForEach-Object { Get-ConfidenceRank $_.Finding.confidence } | Measure-Object -Maximum).Maximum
|
||||
if ((Get-SeverityRank $finding.severity) -ne $expectedSeverityRank -or
|
||||
(Get-ConfidenceRank $finding.confidence) -ne $expectedConfidenceRank) {
|
||||
Add-Error 'SUPER_FINDING_MISMATCH' "$ReportPathPrefix.findings[$index]" `
|
||||
"Rolled-up finding '$($finding.id)' must retain the highest severity and confidence justified by its represented leaf findings."
|
||||
}
|
||||
}
|
||||
|
||||
for ($firstIndex = 0; $firstIndex -lt $findings.Count; $firstIndex++) {
|
||||
for ($secondIndex = $firstIndex + 1; $secondIndex -lt $findings.Count; $secondIndex++) {
|
||||
if ((Test-HasProperty $findings[$firstIndex] 'location') -and
|
||||
(Test-HasProperty $findings[$secondIndex] 'location') -and
|
||||
(Test-LocationsOverlap $findings[$firstIndex] $findings[$secondIndex]) -and
|
||||
(Test-SameCorrection $findings[$firstIndex] $findings[$secondIndex])) {
|
||||
Add-Error 'SUPER_DUPLICATE_FINDINGS' "$ReportPathPrefix.findings[$secondIndex]" `
|
||||
"Top-level findings $firstIndex and $secondIndex overlap and prescribe the same correction; they must be merged."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$usedLocationlessFindings = [Collections.Generic.HashSet[int]]::new()
|
||||
foreach ($eligibleFinding in @($eligibleFindings | Where-Object { -not (Test-HasProperty $_.Finding 'location') })) {
|
||||
$matchingIndex = -1
|
||||
for ($index = 0; $index -lt $findings.Count; $index++) {
|
||||
if (-not $usedLocationlessFindings.Contains($index) -and
|
||||
-not (Test-HasProperty $findings[$index] 'location') -and
|
||||
(Test-RolledFindingRepresents $findings[$index] $eligibleFinding.Finding $eligibleFinding.ProducerId)) {
|
||||
$matchingIndex = $index
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($matchingIndex -lt 0) {
|
||||
Add-Error 'SUPER_FINDING_MISSING' "$ReportPathPrefix.findings" `
|
||||
"No distinct rolled-up finding represents a locationless finding from '$($eligibleFinding.ProducerId)'."
|
||||
}
|
||||
else {
|
||||
$usedLocationlessFindings.Add($matchingIndex) | Out-Null
|
||||
}
|
||||
}
|
||||
for ($index = 0; $index -lt $findings.Count; $index++) {
|
||||
if ($findings[$index].'from-sub-skill' -cne 'agent' -and
|
||||
-not (Test-HasProperty $findings[$index] 'location') -and
|
||||
-not $usedLocationlessFindings.Contains($index)) {
|
||||
Add-Error 'SUPER_FINDING_MISMATCH' "$ReportPathPrefix.findings[$index]" `
|
||||
'No distinct locationless leaf finding corresponds to this rolled-up finding.'
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($eligibleFinding in @($eligibleFindings | Where-Object { Test-HasProperty $_.Finding 'location' })) {
|
||||
$represented = @($findings | Where-Object {
|
||||
$_.'from-sub-skill' -cne 'agent' -and
|
||||
(Test-RolledFindingRepresents $_ $eligibleFinding.Finding $eligibleFinding.ProducerId)
|
||||
}).Count
|
||||
if (-not $represented) {
|
||||
Add-Error 'SUPER_FINDING_MISSING' "$ReportPathPrefix.findings" `
|
||||
"No valid rolled-up finding represents a finding from '$($eligibleFinding.ProducerId)' at its source location."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Test-Report $Candidate '$' $SkillKind
|
||||
if ($PermitRangeStartMismatch) {
|
||||
return @($errors | Where-Object Code -CNE 'RANGE_START_MISMATCH')
|
||||
}
|
||||
return @($errors)
|
||||
}
|
||||
|
||||
$errors = @(Get-SemanticErrors $report)
|
||||
$normalized = $false
|
||||
$removedRanges = [Collections.Generic.List[object]]::new()
|
||||
if ($errors.Count -and $AllowBoundedNormalization) {
|
||||
$otherErrors = @($errors | Where-Object Code -CNE 'RANGE_START_MISMATCH')
|
||||
$rangeErrors = @($errors | Where-Object Code -CEQ 'RANGE_START_MISMATCH')
|
||||
if (-not $otherErrors.Count -and $rangeErrors.Count) {
|
||||
$candidate = $report | ConvertTo-Json -Depth 100 | ConvertFrom-Json -Depth 100
|
||||
$eligible = $true
|
||||
foreach ($finding in @($candidate.findings)) {
|
||||
if (-not (Test-HasProperty $finding 'location') -or
|
||||
-not (Test-HasProperty $finding.location 'range') -or
|
||||
$finding.location.range.'start-line' -eq $finding.location.line) {
|
||||
continue
|
||||
}
|
||||
$range = $finding.location.range
|
||||
if ($range.'start-line' -gt $finding.location.line -or
|
||||
$finding.location.line -gt $range.'end-line' -or
|
||||
(Test-HasProperty $finding 'suggested-code')) {
|
||||
$eligible = $false
|
||||
break
|
||||
}
|
||||
$removedRanges.Add([pscustomobject]@{
|
||||
findingId = $finding.id
|
||||
file = $finding.location.file
|
||||
line = $finding.location.line
|
||||
startLine = $range.'start-line'
|
||||
endLine = $range.'end-line'
|
||||
}) | Out-Null
|
||||
$finding.location.PSObject.Properties.Remove('range')
|
||||
}
|
||||
if ($eligible -and -not @(Get-SemanticErrors $candidate).Count) {
|
||||
$report = $candidate
|
||||
$normalized = $true
|
||||
$errors = @()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($errors.Count) {
|
||||
$details = @($errors | ForEach-Object { "$($_.Code) at $($_.Path): $($_.Message)" }) -join '; '
|
||||
throw "Findings-report acceptance failed: $details"
|
||||
}
|
||||
|
||||
return [pscustomobject][ordered]@{
|
||||
normalized = $normalized
|
||||
report = $report
|
||||
removedRanges = @($removedRanges)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue