From 6ccd60311261b10c46b8584333f801296d49aeff Mon Sep 17 00:00:00 2001 From: Michael Dieringer <65093775+MichaelDieringer@users.noreply.github.com> Date: Sat, 20 Jun 2026 08:54:21 +0200 Subject: [PATCH] Add CURABIS shared eval + evidence scripts to custom/scripts - Invoke-CurabisEval.ps1: general compile + analyzers quality eval (hill-climbing score, reads each project's own al.codeAnalyzers + ruleset, logs .eval/history.jsonl). - Invoke-CurabisEvidence.ps1: citation evidence validator that fails on hallucinated knowledge-file or CURABIS rule-code references (cite-or-flag enforcement). - README documenting both. Fetched by Setup-CurabisAppSource.ps1 into each project's scripts folder. Co-Authored-By: Claude Opus 4.8 --- custom/scripts/Invoke-CurabisEval.ps1 | 249 ++++++++++++++++++++++ custom/scripts/Invoke-CurabisEvidence.ps1 | 129 +++++++++++ custom/scripts/README.md | 15 ++ 3 files changed, 393 insertions(+) create mode 100644 custom/scripts/Invoke-CurabisEval.ps1 create mode 100644 custom/scripts/Invoke-CurabisEvidence.ps1 create mode 100644 custom/scripts/README.md diff --git a/custom/scripts/Invoke-CurabisEval.ps1 b/custom/scripts/Invoke-CurabisEval.ps1 new file mode 100644 index 0000000..572a9c4 --- /dev/null +++ b/custom/scripts/Invoke-CurabisEval.ps1 @@ -0,0 +1,249 @@ +# Invoke-CurabisEval.ps1 +# +# Generel "hill climbing"-eval for ETHVERT CURABIS AL-projekt. +# Maaler det objektive signal paa succesfuld udfoersel: kompilerer koden, og er +# cop-analyzerne rene -> en score 0..1, logget over tid. +# +# Ingen projektspecifik logik: +# - app-projekter auto-opdages via app.json +# - analyzere + ruleset laeses fra projektets EGEN .vscode\settings.json +# (saa scoren maales mod projektets bar, ikke harness'ens mening) +# +# Score: +# - kompilerer ikke (errors > 0) -> 0.0 (du kan ikke klatre foer den bygger) +# - ellers: 1 / (1 + WarnWeight * warnings) (falder bloedt, floorer ikke) +# Koer den, aendr EN ting, koer igen, og se trenden i .eval\history.jsonl. +# +# Brug: +# pwsh -File scripts\Invoke-CurabisEval.ps1 +# pwsh -File scripts\Invoke-CurabisEval.ps1 -FailUnder 0.5 # CI-gate +# pwsh -File scripts\Invoke-CurabisEval.ps1 -AppPath ".apps\summatim" + +[CmdletBinding()] +param( + # Repo-rod. Default: foraelder til scripts-mappen (altsaa projektroden). + [string]$ProjectRoot, + + # Et eller flere app-projekter (mappe med app.json). Default: auto-opdag. + [string[]]$AppPath, + + # Override af analyzere. Default: laes projektets egen al.codeAnalyzers. + [ValidateSet('AppSourceCop', 'CodeCop', 'UICop', 'PerTenantExtensionCop')] + [string[]]$Analyzers, + + # Vaegt pr. warning i scoren. errors er altid hard-fail -> 0. + [double]$WarnWeight = 0.01, + + # Hvis sat: exit 1 naar samlet score < dette tal (til CI). + [Nullable[double]]$FailUnder, + + [switch]$Quiet +) + +$ErrorActionPreference = 'Stop' + +function Write-Line([string]$msg, [string]$color = 'Gray') { + if (-not $Quiet) { Write-Host $msg -ForegroundColor $color } +} + +# --- Find projektrod --- +if (-not $ProjectRoot) { $ProjectRoot = Split-Path -Parent $PSScriptRoot } +$ProjectRoot = (Resolve-Path $ProjectRoot).Path + +# --- Find AL-compiler + analyzere i nyeste AL Language extension --- +$ext = Get-ChildItem "$env:USERPROFILE\.vscode\extensions" -Filter 'ms-dynamics-smb.al-*' -ErrorAction SilentlyContinue | + Sort-Object Name -Descending | Select-Object -First 1 +if (-not $ext) { throw 'AL Language extension ikke fundet. Installer ms-dynamics-smb.al.' } + +$alc = Join-Path $ext.FullName 'bin\win32\alc.exe' +if (-not (Test-Path $alc)) { throw "alc.exe ikke fundet i $($ext.FullName)" } +$analyzerDir = Join-Path $ext.FullName 'bin\Analyzers' + +# Map fra token/navn -> analyzer-DLL. Tager baade '${CodeCop}' og 'CodeCop'. +$analyzerDll = @{ + appsourcecop = 'Microsoft.Dynamics.Nav.AppSourceCop.dll' + codecop = 'Microsoft.Dynamics.Nav.CodeCop.dll' + uicop = 'Microsoft.Dynamics.Nav.UICop.dll' + pertenantextensioncop = 'Microsoft.Dynamics.Nav.PerTenantExtensionCop.dll' +} + +function Read-JsonC([string]$path) { + # Laes JSON med // linje-kommentarer (VS Code settings er JSONC). + $lines = Get-Content $path | Where-Object { $_.TrimStart() -notlike '//*' } + ($lines -join "`n") | ConvertFrom-Json +} + +function Resolve-AnalyzerEntry([string]$entry, [string]$appDir) { + # Oversaetter en al.codeAnalyzers-entry til en DLL-sti. Haandterer: + # - kendte tokens: ${CodeCop} / CodeCop / ${AppSourceCop} osv. + # - custom DLL'er: ${analyzerFolder}BusinessCentral.LinterCop.dll + # - relative/absolutte stier til en .dll + $key = ($entry -replace '[${}]', '').ToLower() + if ($analyzerDll.ContainsKey($key)) { return (Join-Path $analyzerDir $analyzerDll[$key]) } + $p = $entry -replace '\$\{analyzerFolder\}', ($analyzerDir + '\') + if ($p -match '\$\{') { Write-Line " !! kan ikke resolve analyzer: $entry" 'Yellow'; return $null } + if (-not [System.IO.Path]::IsPathRooted($p)) { $p = Join-Path $appDir $p } + return $p +} + +function Resolve-Analyzers([string]$appDir) { + # Praeferer projektets egen .vscode\settings.json; ellers -Analyzers/default. + $entries = @() + $settings = Join-Path $appDir '.vscode\settings.json' + if (-not $Analyzers -and (Test-Path $settings)) { + $s = Read-JsonC $settings + $entries = @($s.'al.codeAnalyzers') + } + if (-not $entries -or $entries.Count -eq 0) { + $entries = if ($Analyzers) { $Analyzers } else { @('${AppSourceCop}', '${CodeCop}', '${UICop}') } + } + $dlls = @() + foreach ($e in $entries) { + if (-not $e) { continue } + $dll = Resolve-AnalyzerEntry $e $appDir + if ($dll) { + if (Test-Path $dll) { $dlls += $dll } else { Write-Line " !! analyzer-DLL findes ikke: $dll" 'Yellow' } + } + } + return ($dlls | Select-Object -Unique) +} + +function Resolve-Ruleset([string]$appDir) { + $settings = Join-Path $appDir '.vscode\settings.json' + if (Test-Path $settings) { + $s = Read-JsonC $settings + $rs = $s.'al.ruleSetPath' + if ($rs) { + $p = if ([System.IO.Path]::IsPathRooted($rs)) { $rs } else { Join-Path $appDir $rs } + if (Test-Path $p) { return (Resolve-Path $p).Path } + } + } + return $null +} + +# --- Find app-projekter --- +if (-not $AppPath) { + $AppPath = Get-ChildItem -Path $ProjectRoot -Recurse -Filter 'app.json' -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -notmatch '\\\.alpackages\\' } | + ForEach-Object { Split-Path $_.FullName -Parent } +} +else { + $AppPath = $AppPath | ForEach-Object { + if ([System.IO.Path]::IsPathRooted($_)) { $_ } else { Join-Path $ProjectRoot $_ } + } +} +if (-not $AppPath) { throw "Ingen app.json fundet under $ProjectRoot" } + +Write-Line "AL compiler : $alc" 'DarkGray' +Write-Line '' + +$tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("curabis-eval-" + [guid]::NewGuid().ToString('N')) +New-Item -ItemType Directory -Path $tmp -Force | Out-Null + +$appResults = @() + +foreach ($app in $AppPath) { + $app = (Resolve-Path $app).Path + $manifest = Join-Path $app 'app.json' + if (-not (Test-Path $manifest)) { Write-Line " Springer over (ingen app.json): $app" 'Yellow'; continue } + $appJson = Get-Content $manifest -Raw | ConvertFrom-Json + $name = $appJson.name + + $analyzerDlls = Resolve-Analyzers $app + $ruleset = Resolve-Ruleset $app + $analyzerNames = $analyzerDlls | ForEach-Object { [System.IO.Path]::GetFileNameWithoutExtension($_) -replace '^Microsoft\.Dynamics\.Nav\.', '' } + + Write-Line "-> $name" 'Cyan' + Write-Line (" analyzere: {0}{1}" -f ($analyzerNames -join ', '), $(if ($ruleset) { " | ruleset: $(Split-Path $ruleset -Leaf)" } else { '' })) 'DarkGray' + + $errorLog = Join-Path $tmp ((Split-Path $app -Leaf) + '.json') + $pkgCache = Join-Path $app '.alpackages' + + $alcArgs = @( + "/project:$app", + "/packagecachepath:$pkgCache", + "/outfolder:$tmp", + "/errorlog:$errorLog", + '/loglevel:Warning' + ) + foreach ($d in $analyzerDlls) { $alcArgs += "/analyzer:$d" } + if ($ruleset) { $alcArgs += "/ruleset:$ruleset" } + + & $alc @alcArgs 2>&1 | Out-Null + $alcExit = $LASTEXITCODE + + # --- Parse diagnostik --- + # alc /errorlog skriver legacy-format (version 0.2): { issues: [ { ruleId, + # properties.severity } ] }. Nyere compilere kan skrive SARIF 2.x + # (runs[].results[].level). Vi haandterer begge. + $errors = 0; $warnings = 0; $byRule = @{} + if (Test-Path $errorLog) { + $log = Get-Content $errorLog -Raw | ConvertFrom-Json + $diags = @() + if ($log.PSObject.Properties.Name -contains 'issues') { + foreach ($i in $log.issues) { $diags += [PSCustomObject]@{ rule = "$($i.ruleId)"; sev = "$($i.properties.severity)" } } + } + elseif ($log.PSObject.Properties.Name -contains 'runs') { + foreach ($run in $log.runs) { foreach ($r in $run.results) { $diags += [PSCustomObject]@{ rule = "$($r.ruleId)"; sev = "$($r.level)" } } } + } + foreach ($d in $diags) { + $sev = $d.sev.ToLower() + if ($sev -ne 'error' -and $sev -ne 'warning') { continue } # spring Info over + if ($sev -eq 'error') { $errors++ } else { $warnings++ } + if ($d.rule) { if ($byRule.ContainsKey($d.rule)) { $byRule[$d.rule]++ } else { $byRule[$d.rule] = 1 } } + } + } + elseif ($alcExit -ne 0) { $errors = 1 } + + # --- Score: errors = hard fail (0). Ellers bloed warning-kurve. --- + if ($errors -gt 0) { $score = 0.0 } + else { $score = [Math]::Round(1.0 / (1.0 + $WarnWeight * $warnings), 3) } + + $color = if ($errors -gt 0) { 'Red' } elseif ($warnings -gt 0) { 'Yellow' } else { 'Green' } + Write-Line (" errors={0} warnings={1} score={2}" -f $errors, $warnings, $score) $color + + # Top-overtraedelser (hjaelper med at vide hvad man skal fixe foerst) + if (-not $Quiet -and $byRule.Count -gt 0) { + $top = $byRule.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First 5 + Write-Line (" top: " + (($top | ForEach-Object { "$($_.Key)x$($_.Value)" }) -join ' ')) 'DarkGray' + } + + $appResults += [PSCustomObject]@{ + app = $name + path = $app + analyzers = $analyzerNames + ruleset = $(if ($ruleset) { Split-Path $ruleset -Leaf } else { $null }) + errors = $errors + warnings = $warnings + byRule = $byRule + score = $score + } +} + +# --- Samlet score = gennemsnit over apps --- +$overall = if ($appResults.Count -gt 0) { [Math]::Round(($appResults | Measure-Object -Property score -Average).Average, 3) } else { 0.0 } + +$run = [PSCustomObject]@{ + timestamp = (Get-Date).ToString('o') + overall = $overall + apps = $appResults +} + +# --- Skriv resultat + historik --- +$evalDir = Join-Path $ProjectRoot '.eval' +New-Item -ItemType Directory -Path $evalDir -Force | Out-Null +$run | ConvertTo-Json -Depth 10 | Set-Content (Join-Path $evalDir 'last-run.json') -Encoding UTF8 +($run | ConvertTo-Json -Depth 10 -Compress) | Add-Content (Join-Path $evalDir 'history.jsonl') -Encoding UTF8 + +Remove-Item $tmp -Recurse -Force -ErrorAction SilentlyContinue + +Write-Line '' +Write-Line ("=== SAMLET SCORE: {0} ===" -f $overall) $(if ($overall -ge 0.9) { 'Green' } elseif ($overall -ge 0.5) { 'Yellow' } else { 'Red' }) +Write-Line "Historik: $($evalDir)\history.jsonl" 'DarkGray' + +# --- CI-gate --- +if ($null -ne $FailUnder -and $overall -lt $FailUnder) { + Write-Line ("FAIL: score {0} < taerskel {1}" -f $overall, $FailUnder) 'Red' + exit 1 +} +exit 0 diff --git a/custom/scripts/Invoke-CurabisEvidence.ps1 b/custom/scripts/Invoke-CurabisEvidence.ps1 new file mode 100644 index 0000000..725315a --- /dev/null +++ b/custom/scripts/Invoke-CurabisEvidence.ps1 @@ -0,0 +1,129 @@ +# Invoke-CurabisEvidence.ps1 +# +# Validerer at hver citation i en gemt review/audit/triage-rapport peger paa noget +# der FAKTISK findes - hegnet mod hallucinerede henvisninger (jf. CURABIS-TRIAGE-005 +# "cite or flag" og ALDC's validate_evidence.py). +# +# Tjekker to slags citationer: +# 1. Knowledge-filer - BCQuality-URL'er eller relative stier (custom/.., microsoft/..) +# -> skal resolve mod en lokal BCQuality-klon eller via HTTP. +# 2. CURABIS-regelkoder - CURABIS-ARCH/TRIAGE/COMPLEXITY-NNN +# -> skal vaere defineret i .github\.agents\*.agent.md. +# (AL-diagnostikkoder som AS0084/AA0218 er Microsofts og valideres ikke her.) +# +# Exit 1 hvis en eneste citation ikke kan resolves (egnet til CI / PR-gate). +# +# Brug: +# pwsh -File scripts\Invoke-CurabisEvidence.ps1 -ReportPath review.md +# "AL Triage cited CURABIS-ARCH-002" | pwsh -File scripts\Invoke-CurabisEvidence.ps1 + +[CmdletBinding()] +param( + # Rapportfil der skal valideres. Kan ogsaa pipes ind paa stdin. + [Parameter(ValueFromPipeline = $true)] + [string]$ReportPath, + + [string]$ProjectRoot, + + # Lokal BCQuality-klon. Default: proev ..\bcquality og .\.bcquality. + [string]$BCQualityHome, + + # Bruges naar der ikke er en lokal klon: knowledge-filer HTTP-tjekkes herfra. + [string]$RawBase = 'https://raw.githubusercontent.com/Curabis/BCQuality/main', + + [switch]$Quiet +) + +$ErrorActionPreference = 'Stop' + +function Write-Line([string]$msg, [string]$color = 'Gray') { + if (-not $Quiet) { Write-Host $msg -ForegroundColor $color } +} + +# --- Projektrod --- +if (-not $ProjectRoot) { $ProjectRoot = Split-Path -Parent $PSScriptRoot } +$ProjectRoot = (Resolve-Path $ProjectRoot).Path + +# --- Hent rapport-tekst (fil eller stdin) --- +$report = $null +if ($ReportPath -and (Test-Path $ReportPath)) { + $report = Get-Content $ReportPath -Raw +} +elseif ($ReportPath) { + # Ikke en sti -> behandl som raa tekst (fx pipet ind) + $report = $ReportPath +} +if (-not $report) { throw "Ingen rapport. Angiv -ReportPath eller pipe tekst ind." } + +# --- Find lokal BCQuality-klon --- +if (-not $BCQualityHome) { + foreach ($c in @((Join-Path $ProjectRoot '..\bcquality'), (Join-Path $ProjectRoot '.bcquality'))) { + if (Test-Path $c) { $BCQualityHome = (Resolve-Path $c).Path; break } + } +} +$useLocal = [bool]$BCQualityHome -and (Test-Path $BCQualityHome) +Write-Line ("Validering: {0}" -f $(if ($useLocal) { "lokal klon $BCQualityHome" } else { "HTTP mod $RawBase" })) 'DarkGray' + +# --- Udtraek citationer --- +# Knowledge-stier: fra URL'er (efter .../main/) og relative custom|microsoft|community-stier. +$paths = New-Object System.Collections.Generic.HashSet[string] +foreach ($m in [regex]::Matches($report, '(?<=BCQuality/(?:main|master)/)[^\s)\"''<>]+\.md')) { [void]$paths.Add($m.Value) } +foreach ($m in [regex]::Matches($report, '(?]+\.md)')) { [void]$paths.Add($m.Groups[1].Value) } + +# CURABIS-regelkoder +$codes = New-Object System.Collections.Generic.HashSet[string] +foreach ($m in [regex]::Matches($report, 'CURABIS-[A-Z]+-\d+')) { [void]$codes.Add($m.Value) } + +# --- Byg saet af gyldige regelkoder fra agent-filerne --- +$validCodes = New-Object System.Collections.Generic.HashSet[string] +$agentDir = Join-Path $ProjectRoot '.github\.agents' +if (Test-Path $agentDir) { + foreach ($f in Get-ChildItem $agentDir -Filter '*.agent.md') { + $txt = Get-Content $f.FullName -Raw + foreach ($m in [regex]::Matches($txt, 'CURABIS-[A-Z]+-\d+')) { [void]$validCodes.Add($m.Value) } + } +} + +$results = @() + +# --- Valider knowledge-filer --- +foreach ($p in $paths) { + $ok = $false + if ($useLocal) { + $ok = Test-Path (Join-Path $BCQualityHome ($p -replace '/', '\')) + } + else { + try { + $resp = Invoke-WebRequest -Uri "$RawBase/$p" -Method Head -UseBasicParsing -TimeoutSec 15 + $ok = ($resp.StatusCode -eq 200) + } catch { $ok = $false } + } + $results += [PSCustomObject]@{ kind = 'file'; citation = $p; resolved = $ok } +} + +# --- Valider regelkoder --- +foreach ($c in $codes) { + $results += [PSCustomObject]@{ kind = 'rule'; citation = $c; resolved = $validCodes.Contains($c) } +} + +# --- Rapport --- +Write-Line '' +if ($results.Count -eq 0) { + Write-Line 'Ingen citationer fundet i rapporten.' 'Yellow' + exit 0 +} + +$missing = 0 +foreach ($r in ($results | Sort-Object kind, citation)) { + if ($r.resolved) { Write-Line (" OK [{0}] {1}" -f $r.kind, $r.citation) 'Green' } + else { Write-Line (" MISSING [{0}] {1}" -f $r.kind, $r.citation) 'Red'; $missing++ } +} + +Write-Line '' +$total = $results.Count +if ($missing -gt 0) { + Write-Line ("FAIL: {0}/{1} citationer kunne ikke resolves (hallucineret?)." -f $missing, $total) 'Red' + exit 1 +} +Write-Line ("OK: alle {0} citationer resolver." -f $total) 'Green' +exit 0 diff --git a/custom/scripts/README.md b/custom/scripts/README.md new file mode 100644 index 0000000..22c8a58 --- /dev/null +++ b/custom/scripts/README.md @@ -0,0 +1,15 @@ +# custom/scripts + +CURABIS shared PowerShell tooling, fetched by `Setup-CurabisAppSource.ps1` into each +project's `scripts\` (or `Scripts\`) folder. + +- **Invoke-CurabisEval.ps1** — general "hill climbing" quality eval. Compiles every app + with the project's own analyzers and emits a score (errors = hard fail; warnings lower it + on a soft curve), logged to `.eval\history.jsonl`. Run it, change one thing, run it again. + - `pwsh -File scripts\Invoke-CurabisEval.ps1` + - `pwsh -File scripts\Invoke-CurabisEval.ps1 -FailUnder 0.5` (CI gate) + +- **Invoke-CurabisEvidence.ps1** — enforces "cite or flag". Validates that every citation + in a saved review/triage report (knowledge files + `CURABIS-*` rule codes) actually + exists. Fails on hallucinated citations. + - `pwsh -File scripts\Invoke-CurabisEvidence.ps1 -ReportPath review.md`