mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Preserve literal strings in composed report acceptance
This commit is contained in:
parent
aecd081778
commit
5d1399f9fb
4 changed files with 73 additions and 10 deletions
|
|
@ -154,7 +154,8 @@ not a deployable or compiled application.
|
||||||
## Before opening a PR
|
## Before opening a PR
|
||||||
|
|
||||||
From your BCQuality checkout, use the existing validators. The Python
|
From your BCQuality checkout, use the existing validators. The Python
|
||||||
validator needs Python and PyYAML; the fixture harness needs PowerShell 7.
|
validator needs Python and PyYAML; the fixture harness needs PowerShell 7.5 or later
|
||||||
|
so findings-report parsing preserves timestamp-shaped JSON strings verbatim.
|
||||||
If PyYAML is not installed in your development environment, install it with
|
If PyYAML is not installed in your development environment, install it with
|
||||||
`python -m pip install pyyaml`.
|
`python -m pip install pyyaml`.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -78,6 +78,9 @@ its own task-level applicability and reports `not-applicable` or
|
||||||
|
|
||||||
## Composition acceptance
|
## Composition acceptance
|
||||||
|
|
||||||
|
The findings-report validator requires PowerShell 7.5 or later to preserve
|
||||||
|
literal JSON strings with `ConvertFrom-Json -DateKind String`.
|
||||||
|
|
||||||
A report can be internally consistent while omitting a selected review. Bind
|
A report can be internally consistent while omitting a selected review. Bind
|
||||||
the final acceptance gate to the host's selection, not just the returned
|
the final acceptance gate to the host's selection, not just the returned
|
||||||
reports. The private JSON input to `-ExpectedCompositionPath` follows the
|
reports. The private JSON input to `-ExpectedCompositionPath` follows the
|
||||||
|
|
|
||||||
|
|
@ -451,6 +451,59 @@ try {
|
||||||
$removedFindingReport.findings = @()
|
$removedFindingReport.findings = @()
|
||||||
$removedFindingReport.summary.counts.minor = 0
|
$removedFindingReport.summary.counts.minor = 0
|
||||||
Assert-CompositionReport $removedFindingReport '*SUPER_LEAF_CONTENT_MISMATCH*'
|
Assert-CompositionReport $removedFindingReport '*SUPER_LEAF_CONTENT_MISMATCH*'
|
||||||
|
|
||||||
|
$capturedCorrectionLeaf = $capturedFindingLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$capturedCorrectionLeaf.findings[0] | Add-Member -NotePropertyName 'suggested-code' -NotePropertyValue 'exit(1);'
|
||||||
|
$correctionComposition = $expectedComposition | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$correctionComposition.acceptedResults = @(Save-AcceptedLeafReports @($capturedCorrectionLeaf, $completedSecurityLeaf))
|
||||||
|
Set-Content -LiteralPath $compositionPath -Value ($correctionComposition | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||||
|
$capturedCorrectionReport = $capturedFindingReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$capturedCorrectionReport.'sub-results'[0] = $capturedCorrectionLeaf
|
||||||
|
$capturedCorrectionReport.findings[0] | Add-Member -NotePropertyName 'suggested-code' -NotePropertyValue 'exit(1);'
|
||||||
|
Assert-CompositionReport $capturedCorrectionReport
|
||||||
|
$correctionCapturePath = $correctionComposition.acceptedResults[0].reportPath
|
||||||
|
$immutableCorrectionCapture = [IO.File]::ReadAllText($correctionCapturePath)
|
||||||
|
foreach ($codePoint in @(0x0000, 0x00AD, 0x200B, 0xFEFF)) {
|
||||||
|
$alteredCorrectionReport = $capturedCorrectionReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$alteredCorrection = 'ex' + [char]$codePoint + 'it(1);'
|
||||||
|
$alteredCorrectionReport.'sub-results'[0].findings[0].'suggested-code' = $alteredCorrection
|
||||||
|
$alteredCorrectionReport.findings[0].'suggested-code' = $alteredCorrection
|
||||||
|
Assert-CompositionReport $alteredCorrectionReport '*SUPER_LEAF_CONTENT_MISMATCH*'
|
||||||
|
$rolledOnlyCorrectionReport = $capturedCorrectionReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$rolledOnlyCorrectionReport.findings[0].'suggested-code' = $alteredCorrection
|
||||||
|
Assert-CompositionReport $rolledOnlyCorrectionReport '*SUPER_FINDING_MISMATCH*'
|
||||||
|
Assert-True ([string]::Equals([IO.File]::ReadAllText($correctionCapturePath), $immutableCorrectionCapture, [StringComparison]::Ordinal)) `
|
||||||
|
'rejecting altered corrections leaves the immutable host capture unchanged'
|
||||||
|
}
|
||||||
|
$timestampReason = '2026-10-02T09:00:00Z'
|
||||||
|
$timestampLeaf = $completedLeaf | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$timestampLeaf | Add-Member -NotePropertyName 'outcome-reason' -NotePropertyValue $timestampReason
|
||||||
|
$timestampComposition = $expectedComposition | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$timestampComposition.acceptedResults = @(Save-AcceptedLeafReports @($timestampLeaf, $completedSecurityLeaf))
|
||||||
|
Set-Content -LiteralPath $compositionPath -Value ($timestampComposition | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||||
|
$timestampReport = $validSuperReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$timestampReport.'sub-results'[0] = $timestampLeaf
|
||||||
|
foreach ($alteredTimestamp in @('2026-10-02T09:00:00.000Z', '2026-10-02T09:00:00+00:00')) {
|
||||||
|
$timestampLeaf.'outcome-reason' = $alteredTimestamp
|
||||||
|
Assert-CompositionReport $timestampReport '*SUPER_LEAF_CONTENT_MISMATCH*'
|
||||||
|
}
|
||||||
|
$timestampLeaf.'outcome-reason' = $timestampReason
|
||||||
|
Set-Content -LiteralPath $reportPath -Value ($timestampReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||||
|
$acceptedTimestampReport = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SkillKind super `
|
||||||
|
-ExpectedCompositionPath $compositionPath
|
||||||
|
$acceptedReason = $acceptedTimestampReport.report.'sub-results'[0].'outcome-reason'
|
||||||
|
Assert-True ($acceptedReason -is [string] -and [string]::Equals($acceptedReason, $timestampReason, [StringComparison]::Ordinal)) `
|
||||||
|
'accepted timestamp-shaped JSON text remains the original literal string'
|
||||||
|
$normalizedTimestampReport = $validReport | ConvertTo-Json -Depth 20 | ConvertFrom-Json
|
||||||
|
$normalizedTimestampReport | Add-Member -NotePropertyName 'outcome-reason' -NotePropertyValue '2026-10-02T09:00:00.000Z'
|
||||||
|
$normalizedTimestampReport.findings[0].location.range.'start-line' = 1
|
||||||
|
Set-Content -LiteralPath $reportPath -Value ($normalizedTimestampReport | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||||
|
$acceptedNormalizedTimestamp = & $validator -ReportPath $reportPath -BCQualityRoot $Root -SourceRoot $tmp `
|
||||||
|
-SourcePaths $sourcePath -RetrievedArticlePaths $articlePath -AllowBoundedNormalization
|
||||||
|
Assert-True $acceptedNormalizedTimestamp.normalized 'bounded normalization still applies to an eligible range'
|
||||||
|
Assert-True ($acceptedNormalizedTimestamp.report.'outcome-reason' -is [string] -and
|
||||||
|
[string]::Equals($acceptedNormalizedTimestamp.report.'outcome-reason', $normalizedTimestampReport.'outcome-reason', [StringComparison]::Ordinal)) `
|
||||||
|
'bounded normalization preserves unrelated timestamp-shaped text exactly'
|
||||||
Set-Content -LiteralPath $compositionPath -Value ($expectedComposition | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
Set-Content -LiteralPath $compositionPath -Value ($expectedComposition | ConvertTo-Json -Depth 20) -Encoding utf8NoBOM
|
||||||
|
|
||||||
foreach ($case in @(
|
foreach ($case in @(
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
#Requires -Version 7.5
|
||||||
<#
|
<#
|
||||||
.SYNOPSIS
|
.SYNOPSIS
|
||||||
Validates a BCQuality findings-report against its structural and semantic contract.
|
Validates a BCQuality findings-report against its structural and semantic contract.
|
||||||
|
|
@ -29,7 +30,7 @@ try {
|
||||||
if (-not ($raw | Test-Json -SchemaFile $schemaPath -ErrorAction Stop)) {
|
if (-not ($raw | Test-Json -SchemaFile $schemaPath -ErrorAction Stop)) {
|
||||||
throw 'Report does not satisfy schemas/findings-report.schema.json.'
|
throw 'Report does not satisfy schemas/findings-report.schema.json.'
|
||||||
}
|
}
|
||||||
$report = $raw | ConvertFrom-Json -Depth 100
|
$report = $raw | ConvertFrom-Json -Depth 100 -DateKind String
|
||||||
}
|
}
|
||||||
catch {
|
catch {
|
||||||
throw "Invalid findings-report JSON or schema: $($_.Exception.Message)"
|
throw "Invalid findings-report JSON or schema: $($_.Exception.Message)"
|
||||||
|
|
@ -83,7 +84,7 @@ if ($ExpectedCompositionPath) {
|
||||||
if (-not ($compositionRaw | Test-Json -Schema $compositionSchema -ErrorAction Stop)) {
|
if (-not ($compositionRaw | Test-Json -Schema $compositionSchema -ErrorAction Stop)) {
|
||||||
throw 'Expected composition does not satisfy its input contract.'
|
throw 'Expected composition does not satisfy its input contract.'
|
||||||
}
|
}
|
||||||
$expectedComposition = $compositionRaw | ConvertFrom-Json -Depth 100
|
$expectedComposition = $compositionRaw | ConvertFrom-Json -Depth 100 -DateKind String
|
||||||
$expectedIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
$expectedIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||||
foreach ($leaf in @($expectedComposition.subSkills)) {
|
foreach ($leaf in @($expectedComposition.subSkills)) {
|
||||||
if (-not $expectedIds.Add([string]$leaf.id)) {
|
if (-not $expectedIds.Add([string]$leaf.id)) {
|
||||||
|
|
@ -114,7 +115,7 @@ if ($ExpectedCompositionPath) {
|
||||||
if (-not ($acceptedRaw | Test-Json -SchemaFile $schemaPath -ErrorAction Stop)) {
|
if (-not ($acceptedRaw | Test-Json -SchemaFile $schemaPath -ErrorAction Stop)) {
|
||||||
throw "Accepted result '$($accepted.id)' does not satisfy the report schema."
|
throw "Accepted result '$($accepted.id)' does not satisfy the report schema."
|
||||||
}
|
}
|
||||||
$acceptedReport = $acceptedRaw | ConvertFrom-Json -Depth 100
|
$acceptedReport = $acceptedRaw | ConvertFrom-Json -Depth 100 -DateKind String
|
||||||
if ($acceptedReport.skill.id -cne $accepted.id -or $acceptedReport.skill.version -ne $accepted.version -or
|
if ($acceptedReport.skill.id -cne $accepted.id -or $acceptedReport.skill.version -ne $accepted.version -or
|
||||||
$acceptedReport.PSObject.Properties.Name -ccontains 'sub-results' -or
|
$acceptedReport.PSObject.Properties.Name -ccontains 'sub-results' -or
|
||||||
$acceptedReport.PSObject.Properties.Name -ccontains 'skipped-sub-skills') {
|
$acceptedReport.PSObject.Properties.Name -ccontains 'skipped-sub-skills') {
|
||||||
|
|
@ -173,6 +174,10 @@ function Test-JsonContentEqual {
|
||||||
}
|
}
|
||||||
return $true
|
return $true
|
||||||
}
|
}
|
||||||
|
if ($First -is [string] -or $Second -is [string]) {
|
||||||
|
return $First -is [string] -and $Second -is [string] -and
|
||||||
|
[string]::Equals($First, $Second, [StringComparison]::Ordinal)
|
||||||
|
}
|
||||||
return $First.GetType() -eq $Second.GetType() -and $First -ceq $Second
|
return $First.GetType() -eq $Second.GetType() -and $First -ceq $Second
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -271,9 +276,10 @@ function Get-SemanticErrors {
|
||||||
$firstHasCode = Test-HasProperty $First 'suggested-code'
|
$firstHasCode = Test-HasProperty $First 'suggested-code'
|
||||||
$secondHasCode = Test-HasProperty $Second 'suggested-code'
|
$secondHasCode = Test-HasProperty $Second 'suggested-code'
|
||||||
if ($firstHasCode -or $secondHasCode) {
|
if ($firstHasCode -or $secondHasCode) {
|
||||||
return $firstHasCode -and $secondHasCode -and $First.'suggested-code' -ceq $Second.'suggested-code'
|
return $firstHasCode -and $secondHasCode -and
|
||||||
|
[string]::Equals($First.'suggested-code', $Second.'suggested-code', [StringComparison]::Ordinal)
|
||||||
}
|
}
|
||||||
return $First.message -ceq $Second.message
|
return [string]::Equals($First.message, $Second.message, [StringComparison]::Ordinal)
|
||||||
}
|
}
|
||||||
|
|
||||||
function Test-ReferencesInclude {
|
function Test-ReferencesInclude {
|
||||||
|
|
@ -314,7 +320,7 @@ function Get-SemanticErrors {
|
||||||
$RolledFinding.id -cne $expectedId -or
|
$RolledFinding.id -cne $expectedId -or
|
||||||
$RolledFinding.severity -cne $LeafFinding.severity -or
|
$RolledFinding.severity -cne $LeafFinding.severity -or
|
||||||
$RolledFinding.confidence -cne $LeafFinding.confidence -or
|
$RolledFinding.confidence -cne $LeafFinding.confidence -or
|
||||||
$RolledFinding.message -cne $LeafFinding.message -or
|
-not [string]::Equals($RolledFinding.message, $LeafFinding.message, [StringComparison]::Ordinal) -or
|
||||||
$rolledReferences.Count -ne $leafReferences.Count -or
|
$rolledReferences.Count -ne $leafReferences.Count -or
|
||||||
-not (Test-ReferencesInclude $rolledReferences $leafReferences)) {
|
-not (Test-ReferencesInclude $rolledReferences $leafReferences)) {
|
||||||
return $false
|
return $false
|
||||||
|
|
@ -323,7 +329,7 @@ function Get-SemanticErrors {
|
||||||
$rolledHasProperty = Test-HasProperty $RolledFinding $name
|
$rolledHasProperty = Test-HasProperty $RolledFinding $name
|
||||||
$leafHasProperty = Test-HasProperty $LeafFinding $name
|
$leafHasProperty = Test-HasProperty $LeafFinding $name
|
||||||
if ($rolledHasProperty -ne $leafHasProperty -or
|
if ($rolledHasProperty -ne $leafHasProperty -or
|
||||||
($rolledHasProperty -and $RolledFinding.$name -cne $LeafFinding.$name)) {
|
($rolledHasProperty -and -not [string]::Equals($RolledFinding.$name, $LeafFinding.$name, [StringComparison]::Ordinal))) {
|
||||||
return $false
|
return $false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -339,7 +345,7 @@ function Get-SemanticErrors {
|
||||||
$sameCorrection = Test-SameCorrection $RolledFinding $LeafFinding
|
$sameCorrection = Test-SameCorrection $RolledFinding $LeafFinding
|
||||||
$correctionsConflict = (Test-HasProperty $RolledFinding 'suggested-code') -and
|
$correctionsConflict = (Test-HasProperty $RolledFinding 'suggested-code') -and
|
||||||
(Test-HasProperty $LeafFinding 'suggested-code') -and
|
(Test-HasProperty $LeafFinding 'suggested-code') -and
|
||||||
$RolledFinding.'suggested-code' -cne $LeafFinding.'suggested-code'
|
-not [string]::Equals($RolledFinding.'suggested-code', $LeafFinding.'suggested-code', [StringComparison]::Ordinal)
|
||||||
$explicitCrossRuleMerge = $leafReferences.Count -and
|
$explicitCrossRuleMerge = $leafReferences.Count -and
|
||||||
$rolledReferences.Count -gt $leafReferences.Count -and
|
$rolledReferences.Count -gt $leafReferences.Count -and
|
||||||
-not $correctionsConflict -and
|
-not $correctionsConflict -and
|
||||||
|
|
@ -718,7 +724,7 @@ if ($errors.Count -and $AllowBoundedNormalization) {
|
||||||
$otherErrors = @($errors | Where-Object Code -CNE 'RANGE_START_MISMATCH')
|
$otherErrors = @($errors | Where-Object Code -CNE 'RANGE_START_MISMATCH')
|
||||||
$rangeErrors = @($errors | Where-Object Code -CEQ 'RANGE_START_MISMATCH')
|
$rangeErrors = @($errors | Where-Object Code -CEQ 'RANGE_START_MISMATCH')
|
||||||
if (-not $otherErrors.Count -and $rangeErrors.Count) {
|
if (-not $otherErrors.Count -and $rangeErrors.Count) {
|
||||||
$candidate = $report | ConvertTo-Json -Depth 100 | ConvertFrom-Json -Depth 100
|
$candidate = $report | ConvertTo-Json -Depth 100 | ConvertFrom-Json -Depth 100 -DateKind String
|
||||||
$eligible = $true
|
$eligible = $true
|
||||||
foreach ($finding in @($candidate.findings)) {
|
foreach ($finding in @($candidate.findings)) {
|
||||||
if (-not (Test-HasProperty $finding 'location') -or
|
if (-not (Test-HasProperty $finding 'location') -or
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue