mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-07 07:36:54 +01:00
Strengthen review contracts and add AL reliability guidance (#196)
* Strengthen review contracts and HTTP guidance - add outbound HttpClient transport and HTTP status review rules with paired fixtures`n- resolve layered action-skill overrides deterministically across enabled layers`n- validate findings reports and enforce measurable changed-fixture coverage * Add data handling and test isolation guidance - add SCM guidance for deriving base quantities through line unit-of-measure validation`n- add security guidance for parameterizing SetFilter with external text`n- add test isolation guidance for resetting per-test state before initialization guards`n- add web-service guidance for JSON null handling and invariant standard format 9`n- route and cover all five rules with paired evaluation fixtures * Fix findings report rollup validation * Validate findings report rollups * Enforce merged finding identity * Fix locationless finding deduplication * Reject conflicting merged corrections * Detect conflicting leaf corrections * Route HTTP error checks to canonical web-services knowledge Let the Error Handling leaf conditionally retrieve the existing HTTP owner articles, preserving applicability and exact-path provenance. Add deterministic source-contract and retrieval regressions without duplicating knowledge rules. Copilot-Session-Id: a92a7788-103e-4651-9b84-19e34caffb94 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: wenjiefan <wenjiefan@microsoft.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Jesper Schulz-Wedde <jesper.schulzwedde@microsoft.com>
This commit is contained in:
Stefano Demiliani
GitHub
parent
130d5de6c4
commit
4287233f80
41 changed files with 1974 additions and 38 deletions
|
|
@ -1,12 +1,11 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates the bounded leaf-range normalization contract.
|
||||
Validates executable findings-report acceptance and bounded normalization.
|
||||
|
||||
.DESCRIPTION
|
||||
BCQuality has no executable findings-report consumer. These assertions keep
|
||||
the normative DO contract, AL coordinator, and standalone runner aligned
|
||||
while exercising the exact normalization predicate against representative
|
||||
safe and ambiguous inputs.
|
||||
These assertions keep the normative DO contract, executable validator, AL
|
||||
coordinator, and standalone runner aligned while exercising semantic report
|
||||
validation and the exact normalization predicate.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
|
|
@ -39,6 +38,24 @@ function Assert-Contains {
|
|||
Assert-True $Text.Contains($Expected) $Message
|
||||
}
|
||||
|
||||
function Assert-ThrowsLike {
|
||||
param(
|
||||
[scriptblock] $Action,
|
||||
[string] $Pattern
|
||||
)
|
||||
|
||||
try {
|
||||
& $Action
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Message -like $Pattern) {
|
||||
return
|
||||
}
|
||||
throw "Expected error like '$Pattern', received: $($_.Exception.Message)"
|
||||
}
|
||||
throw "Expected error like '$Pattern', but no error was thrown."
|
||||
}
|
||||
|
||||
function Test-PositiveInteger {
|
||||
param([object] $Value)
|
||||
|
||||
|
|
@ -168,4 +185,334 @@ Assert-True (-not ($candidateFinding.location.PSObject.Properties.Name -contains
|
|||
Assert-True ($candidateFinding.location.line -eq $rawFinding.location.line) 'candidate preserves the primary line'
|
||||
Assert-True ($candidateFinding.message -ceq $rawFinding.message) 'candidate preserves all other finding content'
|
||||
|
||||
Write-Output "Review contract validation passed ($($cases.Count) normalization cases)."
|
||||
$validator = Join-Path $Root 'tools/Validate-FindingsReport.ps1'
|
||||
Assert-True (Test-Path -LiteralPath $validator -PathType Leaf) 'executable report validator exists'
|
||||
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("reviewcontract_" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Path $tmp -Force | Out-Null
|
||||
try {
|
||||
$sourcePath = 'src/codeunit.al'
|
||||
$sourceFile = Join-Path $tmp 'src/codeunit.al'
|
||||
New-Item -ItemType Directory -Path (Split-Path -Parent $sourceFile) -Force | Out-Null
|
||||
Set-Content -LiteralPath $sourceFile -Value @('line one', 'line two', 'line three') -Encoding utf8NoBOM
|
||||
$articlePath = 'microsoft/knowledge/style/caption-required-on-page-fields.md'
|
||||
$supportingArticlePath = 'microsoft/knowledge/style/tooltip-required-on-page-fields.md'
|
||||
$reportPath = Join-Path $tmp 'report.json'
|
||||
|
||||
$validReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-style-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 1; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 1; 'items-evaluated' = 1 }
|
||||
}
|
||||
findings = @(
|
||||
[ordered]@{
|
||||
id = $articlePath
|
||||
severity = 'minor'
|
||||
message = 'A concrete style defect.'
|
||||
location = [ordered]@{
|
||||
file = $sourcePath
|
||||
line = 2
|
||||
range = [ordered]@{ 'start-line' = 2; 'end-line' = 3 }
|
||||
}
|
||||
references = @([ordered]@{ path = $articlePath })
|
||||
confidence = 'high'
|
||||
domain = 'Style'
|
||||
}
|
||||
)
|
||||
suppressed = @()
|
||||
}
|
||||
Set-Content -LiteralPath $reportPath -Value ($validReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$accepted = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
Assert-True (-not $accepted.normalized) 'valid report is accepted without normalization'
|
||||
|
||||
$invalidCounts = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$invalidCounts.summary.counts.minor = 0
|
||||
Set-Content -LiteralPath $reportPath -Value ($invalidCounts | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*COUNT_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$completedUndercoverage = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$completedUndercoverage.summary.coverage.'items-evaluated' = 0
|
||||
Set-Content -LiteralPath $reportPath -Value ($completedUndercoverage | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*COMPLETED_COVERAGE_INCOMPLETE*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$partialFullCoverage = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$partialFullCoverage.outcome = 'partial'
|
||||
$partialFullCoverage | Add-Member -NotePropertyName 'outcome-reason' -NotePropertyValue 'Stopped early.'
|
||||
Set-Content -LiteralPath $reportPath -Value ($partialFullCoverage | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*PARTIAL_COVERAGE_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$completedLeaf = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-style-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 0; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 1; 'items-evaluated' = 1 }
|
||||
}
|
||||
findings = @()
|
||||
suppressed = @()
|
||||
}
|
||||
$leafWithSubResults = $completedLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$leafWithSubResults | Add-Member -NotePropertyName 'sub-results' -NotePropertyValue @($completedLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($leafWithSubResults | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*LEAF_COMPOSITION_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root
|
||||
}
|
||||
|
||||
$validSuperReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-code-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
summary = [ordered]@{
|
||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 0; info = 0 }
|
||||
coverage = [ordered]@{ 'worklist-size' = 2; 'items-evaluated' = 2 }
|
||||
}
|
||||
findings = @()
|
||||
suppressed = @()
|
||||
'sub-results' = @($completedLeaf, $completedLeaf)
|
||||
}
|
||||
Set-Content -LiteralPath $reportPath -Value ($validSuperReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedSuper = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super
|
||||
Assert-True (-not $acceptedSuper.normalized) 'valid super-skill report is accepted'
|
||||
|
||||
$styleFindingLeaf = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$securityFindingLeaf = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$securityFindingLeaf.skill.id = 'al-security-review'
|
||||
$rolledFinding = $validReport.findings[0] | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$rolledFinding | Add-Member -NotePropertyName 'from-sub-skill' -NotePropertyValue 'al-style-review'
|
||||
$deduplicatedSuperReport = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$deduplicatedSuperReport.summary.counts.minor = 1
|
||||
$deduplicatedSuperReport.findings = @($rolledFinding)
|
||||
$deduplicatedSuperReport.'sub-results' = @($styleFindingLeaf, $securityFindingLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($deduplicatedSuperReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedDeduplicatedSuper = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
Assert-True (-not $acceptedDeduplicatedSuper.normalized) 'one top-level finding may deduplicate the same citation from two leaves'
|
||||
|
||||
$twoOccurrenceLeaf = $styleFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$secondOccurrence = $twoOccurrenceLeaf.findings[0] | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$secondOccurrence.location.line = 1
|
||||
$secondOccurrence.location.range.'start-line' = 1
|
||||
$secondOccurrence.location.range.'end-line' = 1
|
||||
$twoOccurrenceLeaf.findings = @($twoOccurrenceLeaf.findings[0], $secondOccurrence)
|
||||
$twoOccurrenceLeaf.summary.counts.minor = 2
|
||||
$emptySecurityLeaf = $completedLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$emptySecurityLeaf.skill.id = 'al-security-review'
|
||||
$sameIdOccurrenceOmitted = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$sameIdOccurrenceOmitted.'sub-results' = @($twoOccurrenceLeaf, $emptySecurityLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($sameIdOccurrenceOmitted | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISSING*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$mergeOwnerLeaf = $styleFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$mergeOwnerLeaf.findings[0] | Add-Member -NotePropertyName 'suggested-code' -NotePropertyValue 'Caption = ''Customer name'';'
|
||||
$supportingFindingLeaf = $securityFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$supportingFindingLeaf.findings[0].id = $supportingArticlePath
|
||||
$supportingFindingLeaf.findings[0].references[0].path = $supportingArticlePath
|
||||
$supportingFindingLeaf.findings[0].confidence = 'medium'
|
||||
$supportingFindingLeaf.findings[0].message = 'The field needs the same mechanical correction for a supporting rule.'
|
||||
$supportingFindingLeaf.findings[0] | Add-Member -NotePropertyName 'suggested-code' -NotePropertyValue 'Caption = ''Customer name'';'
|
||||
$mergedFinding = $rolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$mergedFinding | Add-Member -NotePropertyName 'suggested-code' -NotePropertyValue 'Caption = ''Customer name'';'
|
||||
$mergedFinding.references = @(
|
||||
[pscustomobject]@{ path = $articlePath }
|
||||
[pscustomobject]@{ path = $supportingArticlePath }
|
||||
)
|
||||
$mergedSuperReport = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$mergedSuperReport.findings = @($mergedFinding)
|
||||
$mergedSuperReport.'sub-results' = @($mergeOwnerLeaf, $supportingFindingLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($mergedSuperReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedMergedSuper = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
Assert-True (-not $acceptedMergedSuper.normalized) 'overlapping A and B findings may merge into A with B as a supporting reference'
|
||||
|
||||
$textOnlyOwnerLeaf = $mergeOwnerLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$textOnlyOwnerLeaf.findings[0].PSObject.Properties.Remove('suggested-code')
|
||||
$textOnlySupportingLeaf = $supportingFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$textOnlySupportingLeaf.findings[0].PSObject.Properties.Remove('suggested-code')
|
||||
$textOnlyMergedFinding = $mergedFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$textOnlyMergedFinding.PSObject.Properties.Remove('suggested-code')
|
||||
$textOnlyMergedSuper = $mergedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$textOnlyMergedSuper.findings = @($textOnlyMergedFinding)
|
||||
$textOnlyMergedSuper.'sub-results' = @($textOnlyOwnerLeaf, $textOnlySupportingLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($textOnlyMergedSuper | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedTextOnlyMerge = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
Assert-True (-not $acceptedTextOnlyMerge.normalized) 'supporting references permit an overlapping A and B merge with different messages and no suggested code'
|
||||
|
||||
$conflictingSupportingLeaf = $supportingFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$conflictingSupportingLeaf.findings[0].'suggested-code' = 'ToolTip = ''Customer name'';'
|
||||
$conflictingCorrectionMerge = $mergedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$conflictingCorrectionMerge.'sub-results' = @($mergeOwnerLeaf, $conflictingSupportingLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($conflictingCorrectionMerge | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISSING*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
}
|
||||
|
||||
$omittedConflictingCorrectionMerge = $conflictingCorrectionMerge | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$omittedConflictingCorrectionMerge.findings[0].PSObject.Properties.Remove('suggested-code')
|
||||
Set-Content -LiteralPath $reportPath -Value ($omittedConflictingCorrectionMerge | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
}
|
||||
|
||||
$unmergedSupportingFinding = $supportingFindingLeaf.findings[0] | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$unmergedSupportingFinding | Add-Member -NotePropertyName 'from-sub-skill' -NotePropertyValue 'al-security-review'
|
||||
$unmergedDuplicates = $mergedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$unmergedDuplicates.summary.counts.minor = 2
|
||||
$unmergedDuplicates.findings = @($mergedFinding, $unmergedSupportingFinding)
|
||||
Set-Content -LiteralPath $reportPath -Value ($unmergedDuplicates | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_DUPLICATE_FINDINGS*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath, $supportingArticlePath
|
||||
}
|
||||
|
||||
$omittedLeafFinding = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$omittedLeafFinding.summary.counts.minor = 0
|
||||
$omittedLeafFinding.findings = @()
|
||||
Set-Content -LiteralPath $reportPath -Value ($omittedLeafFinding | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISSING*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$locationlessLeaf = $styleFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$locationlessLeaf.findings[0].PSObject.Properties.Remove('location')
|
||||
$secondLocationlessFinding = $locationlessLeaf.findings[0] | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$locationlessLeaf.findings = @($locationlessLeaf.findings[0], $secondLocationlessFinding)
|
||||
$locationlessLeaf.summary.counts.minor = 2
|
||||
$locationlessRolledFinding = $rolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$locationlessRolledFinding.PSObject.Properties.Remove('location')
|
||||
$locationlessOmission = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$locationlessOmission.findings = @($locationlessRolledFinding)
|
||||
$locationlessOmission.'sub-results' = @($locationlessLeaf, $emptySecurityLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($locationlessOmission | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISSING*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$completeLocationlessRollup = $locationlessOmission | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$completeLocationlessRollup.summary.counts.minor = 2
|
||||
$completeLocationlessRollup.findings = @(
|
||||
$locationlessRolledFinding
|
||||
($locationlessRolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json)
|
||||
)
|
||||
Set-Content -LiteralPath $reportPath -Value ($completeLocationlessRollup | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedLocationlessRollup = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
Assert-True (-not $acceptedLocationlessRollup.normalized) 'two locationless leaf occurrences require and accept two distinct rolled findings'
|
||||
|
||||
$nonexistentProducer = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$nonexistentProducer.findings[0].'from-sub-skill' = 'al-missing-review'
|
||||
Set-Content -LiteralPath $reportPath -Value ($nonexistentProducer | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_PRODUCER_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$rewrittenLeafFinding = $deduplicatedSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$rewrittenLeafFinding.findings[0].message = 'A rewritten rollup message.'
|
||||
Set-Content -LiteralPath $reportPath -Value ($rewrittenLeafFinding | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FINDING_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$failedLeaf = $completedLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$failedLeaf.skill.id = 'al-security-review'
|
||||
$failedLeaf.outcome = 'failed'
|
||||
$failedLeaf | Add-Member -NotePropertyName 'outcome-reason' -NotePropertyValue 'Validation failed.'
|
||||
$failedLeaf.summary.coverage.'items-evaluated' = 0
|
||||
$partialSuperReport = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$partialSuperReport.outcome = 'partial'
|
||||
$partialSuperReport | Add-Member -NotePropertyName 'outcome-reason' -NotePropertyValue 'One sub-skill failed.'
|
||||
$partialSuperReport.summary.coverage.'worklist-size' = 1
|
||||
$partialSuperReport.summary.coverage.'items-evaluated' = 1
|
||||
$partialSuperReport.'sub-results' = @($completedLeaf, $failedLeaf)
|
||||
Set-Content -LiteralPath $reportPath -Value ($partialSuperReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
$acceptedPartialSuper = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super
|
||||
Assert-True (-not $acceptedPartialSuper.normalized) 'partial super-skill excludes failed coverage from its rollup'
|
||||
|
||||
$failedLeafLeakage = $partialSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$failedLeafLeakage.summary.counts.minor = 1
|
||||
$failedLeafLeakage.findings = @($rolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json)
|
||||
$failedLeafLeakage.findings[0].'from-sub-skill' = 'al-security-review'
|
||||
Set-Content -LiteralPath $reportPath -Value ($failedLeafLeakage | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_FAILED_FINDING_LEAKAGE*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$failedLeafRelabeledAsAgent = $partialSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$failedLeafRelabeledAsAgent.summary.counts.minor = 1
|
||||
$failedLeafRelabeledAsAgent.findings = @($rolledFinding | ConvertTo-Json -Depth 20 | ConvertFrom-Json)
|
||||
$failedLeafRelabeledAsAgent.findings[0].'from-sub-skill' = 'agent'
|
||||
$failedLeafRelabeledAsAgent.findings[0].domain = 'Agent'
|
||||
Set-Content -LiteralPath $reportPath -Value ($failedLeafRelabeledAsAgent | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_AGENT_FINDING_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||
-SourceRoot $tmp -SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
|
||||
$incorrectOutcome = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$incorrectOutcome.outcome = 'not-applicable'
|
||||
Set-Content -LiteralPath $reportPath -Value ($incorrectOutcome | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_OUTCOME_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super
|
||||
}
|
||||
|
||||
$incorrectRollup = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$incorrectRollup.summary.coverage.'worklist-size' = 1
|
||||
$incorrectRollup.summary.coverage.'items-evaluated' = 1
|
||||
Set-Content -LiteralPath $reportPath -Value ($incorrectRollup | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*SUPER_COVERAGE_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super
|
||||
}
|
||||
|
||||
Set-Content -LiteralPath $reportPath -Value ($validReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*REFERENCE_NOT_RETRIEVED*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp -SourcePaths $sourcePath
|
||||
}
|
||||
|
||||
$invalidAgent = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$invalidAgent.findings[0].id = 'agent:uncited-defect'
|
||||
$invalidAgent.findings[0].references = @()
|
||||
$invalidAgent.findings[0].confidence = 'high'
|
||||
Set-Content -LiteralPath $reportPath -Value ($invalidAgent | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*AGENT_CONFIDENCE_INVALID*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp -SourcePaths $sourcePath
|
||||
}
|
||||
|
||||
$normalizable = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||
$normalizable.findings[0].location.range.'start-line' = 1
|
||||
Set-Content -LiteralPath $reportPath -Value ($normalizable | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||
Assert-ThrowsLike -Pattern '*RANGE_START_MISMATCH*' -Action {
|
||||
& $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath
|
||||
}
|
||||
$normalized = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath -AllowBoundedNormalization
|
||||
Assert-True $normalized.normalized 'eligible range mismatch is normalized'
|
||||
Assert-True ($normalized.removedRanges.Count -eq 1) 'normalization records one removed range'
|
||||
Assert-True (-not ($normalized.report.findings[0].location.PSObject.Properties.Name -contains 'range')) `
|
||||
'accepted normalized report removes only the optional range'
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
Write-Output "Review contract validation passed ($($cases.Count) predicate cases plus executable acceptance cases)."
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue