mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 06:36:55 +01:00
Merge 8f025ac679 into 38ad6b8810
This commit is contained in:
Jesper Schulz-Wedde
GitHub
commit
ab06998818
41 changed files with 2385 additions and 76 deletions
|
|
@ -8,8 +8,8 @@
|
|||
{
|
||||
"name": "bcquality",
|
||||
"source": "./",
|
||||
"description": "Business Central AL quality knowledge base and review skills, packaged as an installable plugin. Exposes an AL review adapter while preserving BCQuality's internal Entry and action-skill protocols.",
|
||||
"version": "0.2.0",
|
||||
"description": "Business Central AL quality knowledge base and skills, packaged as an installable plugin. Exposes read-only plan enrichment and review adapters through BCQuality's Entry protocol.",
|
||||
"version": "0.3.0",
|
||||
"skills": [
|
||||
"./skills/"
|
||||
]
|
||||
|
|
|
|||
14
.github/scripts/Test-SkillIndex.ps1
vendored
14
.github/scripts/Test-SkillIndex.ps1
vendored
|
|
@ -109,6 +109,20 @@ try {
|
|||
}
|
||||
}
|
||||
|
||||
$guidance = @($skills | Where-Object id -eq 'al-development-plan')
|
||||
if ($guidance.Count -ne 1) {
|
||||
throw "Expected exactly one al-development-plan record, found $($guidance.Count)."
|
||||
}
|
||||
if ((@($guidance[0].inputs) -join "`n") -cne ("development-plan`nrepository")) {
|
||||
throw 'al-development-plan inputs were not indexed in declared order.'
|
||||
}
|
||||
if ((@($guidance[0].outputs) -join "`n") -cne 'development-guidance-report') {
|
||||
throw 'al-development-plan output kind was not preserved in the skill index.'
|
||||
}
|
||||
if (@($guidance[0].subSkills).Count) {
|
||||
throw 'al-development-plan must remain a leaf action skill.'
|
||||
}
|
||||
|
||||
$minimalReport = @{
|
||||
skill = @{ id = 'al-style-review'; version = 1 }
|
||||
outcome = 'completed'
|
||||
|
|
|
|||
52
.github/scripts/validate_frontmatter.py
vendored
52
.github/scripts/validate_frontmatter.py
vendored
|
|
@ -18,7 +18,7 @@ import os
|
|||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Iterable
|
||||
|
||||
try:
|
||||
|
|
@ -46,8 +46,9 @@ HOST_SKILL_REQUIRED_KEYS = {"name", "description"}
|
|||
|
||||
STANDARD_INPUTS = {
|
||||
"pr-diff", "object-list", "file-path", "folder-path", "repository", "telemetry-query",
|
||||
"development-plan",
|
||||
}
|
||||
ALLOWED_OUTPUTS = {"findings-report"}
|
||||
ALLOWED_OUTPUTS = {"findings-report", "development-guidance-report"}
|
||||
VALID_SAMPLE_KINDS = {"good", "bad"}
|
||||
|
||||
ACTION_SKILL_SECTIONS = ["Source", "Relevance", "Worklist", "Action", "Output"]
|
||||
|
|
@ -147,6 +148,28 @@ def is_non_empty_list_of_str(value: Any) -> bool:
|
|||
return isinstance(value, list) and len(value) > 0 and all(isinstance(v, str) and v for v in value)
|
||||
|
||||
|
||||
def normalize_repo_md_path(value: Any) -> tuple[str | None, str | None]:
|
||||
"""Validate a canonical repo-relative Markdown path."""
|
||||
if not isinstance(value, str) or not value:
|
||||
return None, "must be a non-empty string"
|
||||
if "\\" in value:
|
||||
return None, "must use forward slashes"
|
||||
|
||||
normalized = value
|
||||
segments = value.split("/")
|
||||
if (
|
||||
not normalized
|
||||
or normalized.startswith("/")
|
||||
or re.match(r"^[A-Za-z]:", normalized)
|
||||
or any(segment in ("", ".", "..") for segment in segments)
|
||||
or PurePosixPath(normalized).is_absolute()
|
||||
):
|
||||
return None, "must be a repository-relative path that does not escape the repository"
|
||||
if not normalized.endswith(".md"):
|
||||
return None, "must end in '.md'"
|
||||
return normalized, None
|
||||
|
||||
|
||||
def expand_bc_version(value: Any) -> tuple[list[int] | str | None, str | None]:
|
||||
"""Return (expanded, error-message). One of the two is None.
|
||||
|
||||
|
|
@ -340,9 +363,11 @@ def validate_action_skill(path: Path, parsed: Parsed, report: Report) -> None:
|
|||
if not is_non_empty_list_of_str(out):
|
||||
report.error(path, "R18", "outputs must be a non-empty list of strings", 1)
|
||||
else:
|
||||
if len(out) != 1:
|
||||
report.error(path, "R18", "outputs must contain exactly one output kind", 1)
|
||||
bad = [x for x in out if x not in ALLOWED_OUTPUTS]
|
||||
if bad:
|
||||
report.error(path, "R18", f"outputs contains non-allowed values {bad}; currently only {sorted(ALLOWED_OUTPUTS)} is defined", 1)
|
||||
report.error(path, "R18", f"outputs contains non-allowed values {bad}; allowed values are {sorted(ALLOWED_OUTPUTS)}", 1)
|
||||
|
||||
# R19 optional filter dimensions, if present
|
||||
if "bc-version" in fm:
|
||||
|
|
@ -378,20 +403,9 @@ def validate_action_skill(path: Path, parsed: Parsed, report: Report) -> None:
|
|||
if not is_non_empty_list_of_str(ss):
|
||||
report.error(path, "R20", "sub-skills must be a non-empty list of repo-relative paths", 1)
|
||||
else:
|
||||
bad = [x for x in ss if not x.endswith(".md")]
|
||||
bad = [f"{x}: {err}" for x in ss if (err := normalize_repo_md_path(x)[1])]
|
||||
if bad:
|
||||
report.error(path, "R20", f"sub-skills entries must end in '.md': {bad}", 1)
|
||||
non_canonical = [
|
||||
x for x in ss
|
||||
if "\\" in x or x.startswith("/") or ".." in Path(x).parts or x.startswith("./")
|
||||
]
|
||||
if non_canonical:
|
||||
report.error(
|
||||
path,
|
||||
"R20",
|
||||
f"sub-skills entries must be canonical repo-relative paths: {non_canonical}",
|
||||
1,
|
||||
)
|
||||
report.error(path, "R20", f"invalid sub-skills paths: {bad}", 1)
|
||||
duplicates = sorted({x for x in ss if ss.count(x) > 1})
|
||||
if duplicates:
|
||||
report.error(path, "R20", f"sub-skills contains duplicate paths: {duplicates}", 1)
|
||||
|
|
@ -598,7 +612,11 @@ def validate_sub_skills_registry(
|
|||
if not is_non_empty_list_of_str(ss):
|
||||
return
|
||||
|
||||
declared = {s.lstrip("./") for s in ss}
|
||||
declared = {
|
||||
normalized
|
||||
for s in ss
|
||||
if (normalized := normalize_repo_md_path(s)[0]) is not None
|
||||
}
|
||||
|
||||
# Sibling leaves on disk, excluding the super-skill file itself.
|
||||
leaves = {
|
||||
|
|
|
|||
25
.github/workflows/development-guidance.yml
vendored
Normal file
25
.github/workflows/development-guidance.yml
vendored
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
name: Validate read-only development guidance
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
validate-development-guidance:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Validate and prepare development-guidance fixtures
|
||||
shell: pwsh
|
||||
run: ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory "$env:RUNNER_TEMP/bcquality-development-guidance-fixtures"
|
||||
|
||||
- name: Run credential-free guidance evaluator regressions
|
||||
shell: pwsh
|
||||
run: ./tools/Test-DevelopmentGuidanceEvaluator.ps1
|
||||
66
README.md
66
README.md
|
|
@ -26,9 +26,34 @@ copilot plugin install microsoft/BCQuality
|
|||
copilot plugin list
|
||||
```
|
||||
|
||||
The list should include `bcquality`. The plugin currently exposes the
|
||||
[`al-code-review`](skills/al-code-review/SKILL.md) skill. Installation and skill
|
||||
discovery are the general pattern; reviewing an app is one example of using it.
|
||||
The list should include `bcquality`. The plugin exposes
|
||||
[`al-code-review`](skills/al-code-review/SKILL.md) and the read-only
|
||||
[`al-development-plan`](skills/al-development-plan/SKILL.md) plan-enrichment
|
||||
skill. Installation and skill discovery are the general pattern; reviewing an
|
||||
app is one example of using it.
|
||||
|
||||
Plugin version `0.3.0` adds `al-development-plan`, a read-only adapter for
|
||||
enriching an **existing** plan. It does not generate a plan or implement code.
|
||||
|
||||
The adapters are intentionally not second implementations:
|
||||
|
||||
```text
|
||||
standalone host skill: skills/al-code-review/SKILL.md
|
||||
-> routing contract: skills/entry.md
|
||||
-> review coordinator: microsoft/skills/review/al-code-review.md
|
||||
-> domain review leaves
|
||||
|
||||
standalone host skill: skills/al-development-plan/SKILL.md
|
||||
-> routing contract: skills/entry.md
|
||||
-> enrichment skill: microsoft/skills/development/al-development-plan.md
|
||||
-> referenced constraints for the consumer's existing workflow (read-only)
|
||||
```
|
||||
|
||||
Only the files under `skills/*/SKILL.md` follow the host's packaging format.
|
||||
The remaining files are BCQuality's internal protocol and layered action
|
||||
skills. Entry remains the single owner of routing and index preparation. This
|
||||
separation keeps standalone installation available without duplicating policy
|
||||
in either adapter.
|
||||
|
||||
### Example: Review a complete app folder
|
||||
|
||||
|
|
@ -48,6 +73,10 @@ Approve access only to a project you trust, then ask:
|
|||
The folder should contain `app.json` and your AL source; it does **not** need
|
||||
to be a Git repository. On macOS or Linux, use your app's local path instead.
|
||||
|
||||
Each host adapter and internal action skill intentionally share a name: they
|
||||
expose the same operation in two different skill formats. Their paths make the
|
||||
boundary explicit.
|
||||
|
||||
Expect a report for each selected review, with findings, source locations,
|
||||
severity, confidence, and references to the relevant guidance. Some hosts show
|
||||
the structured JSON directly. `completed` with no findings means nothing was
|
||||
|
|
@ -82,10 +111,35 @@ available domains and the difference between a folder review and a comparison.
|
|||
Mechanical issues already enforced by the AL compiler or standard analyzers are
|
||||
intentionally left to those deterministic tools rather than duplicated here.
|
||||
|
||||
The read-only `al-development-plan` interface selects relevant constraints
|
||||
before the consumer implements its own existing plan.
|
||||
|
||||
Repository-specific orchestrators retain planning, implementation, approvals,
|
||||
tests, environment, propagation, and delivery ownership. The intended flow is
|
||||
consumer analysis and normalized plan -> read-only BCQuality guidance ->
|
||||
existing implementation phases -> independent final BCQuality review ->
|
||||
delivery. Consumer uptake and a real runtime pilot are follow-up work, not
|
||||
implemented integrations or demonstrated authoring improvements.
|
||||
|
||||
`no-knowledge` means no additional applicable BCQuality constraints, with empty
|
||||
`knowledge`; it does not make a plan unsafe or prevent the consumer from using
|
||||
its ordinary gates. Retrieval failures and materially unresolved conditional
|
||||
guidance are distinct outcomes, not empty knowledge. Do not add generic advice
|
||||
just to avoid a `no-knowledge` result.
|
||||
|
||||
Functional areas such as Finance, Supply Chain Management, Manufacturing, Jobs,
|
||||
Warehousing, and Service, and technologies such as PowerShell, pipelines, and
|
||||
Power Platform, remain valid future scope, **not current coverage claims**.
|
||||
|
||||
## Evidence and follow-up scope
|
||||
|
||||
The [guidance evaluation](evaluation/README.md#read-only-plan-guidance) separates
|
||||
credential-free contract/scorer regressions from external agent and runtime
|
||||
evidence. Prepared requests and fixture counts do not establish compilation,
|
||||
test execution, better repairs, or a capability percentage. Consumer adoption,
|
||||
a pinned baseline comparison and runtime pilot, standalone authoring, and
|
||||
source-ingestion catalog work remain separate follow-ups.
|
||||
|
||||
## What's in this repo
|
||||
|
||||
Knowledge articles cover one concern each. Skills tell an agent how to find
|
||||
|
|
@ -97,6 +151,12 @@ and apply the relevant knowledge. Both live in three layers:
|
|||
| [Community](community/) | Community-owned skills and their knowledge. |
|
||||
| [Custom](custom/) | Organization-specific additions and overrides in your own fork. |
|
||||
|
||||
Review skills emit a `findings-report`; plan enrichment emits a read-only
|
||||
`development-guidance-report`. Both contracts are defined in
|
||||
[`skills/do.md`](skills/do.md). See
|
||||
[how agents consume BCQuality](docs/agent-consumption.md) for the integration
|
||||
flow.
|
||||
|
||||
All three are enabled by default; Custom is empty upstream. You do not need
|
||||
to configure layers to get started.
|
||||
|
||||
|
|
|
|||
|
|
@ -78,8 +78,10 @@ Add scheduling, retries, and rendering only when needed, using the
|
|||
- **Layer content** in `/microsoft/`, `/community/`, and `/custom/` — knowledge files and action skills grouped by authority.
|
||||
|
||||
When BCQuality is installed as a standalone plugin, it additionally exposes
|
||||
`skills/al-code-review/SKILL.md`. This is a host-format adapter, not another
|
||||
action skill: it creates the task context and enters the same flow at Entry.
|
||||
`skills/al-code-review/SKILL.md` and
|
||||
`skills/al-development-plan/SKILL.md`. These are host-format adapters, not
|
||||
additional action skills: each creates the task context and enters the same
|
||||
flow at Entry.
|
||||
|
||||
## Repository structure
|
||||
|
||||
|
|
@ -109,7 +111,7 @@ flowchart LR
|
|||
E -->|3 dispatch record| A
|
||||
A -->|4 invoke dispatched skill| S[Action skill<br/>e.g. al-code-review]
|
||||
S -->|5 execute| P[Source → Relevance<br/>→ Worklist → Action<br/>reading READ · DO on demand]
|
||||
P -->|6 emit| R[Findings · Domain labels<br/>· References · Confidence]
|
||||
P -->|6 emit| R[Findings report<br/>or read-only guidance report]
|
||||
R -->|7 integrate| O
|
||||
```
|
||||
|
||||
|
|
@ -119,14 +121,18 @@ The orchestrator has a URL setting that points at BCQuality (default: `github.co
|
|||
### 2. Agent invokes Entry
|
||||
The agent reads `/skills/entry.md` and runs it against the task context. Entry applies its Source → Relevance → Worklist → Action steps over the action skills under `*/skills/**/*.md` and returns a **dispatch record**: the set of action skills to invoke, plus a list of candidates it skipped (with reasons). Routing is a skill, not orchestrator logic.
|
||||
|
||||
For a standalone plugin installation, the host activates the
|
||||
`skills/al-code-review/SKILL.md` adapter first. That adapter preserves the
|
||||
caller's actual goal, constructs the task context, and invokes Entry. It does
|
||||
not select the internal `microsoft/skills/review/al-code-review.md` action skill
|
||||
itself or duplicate Entry's preparation, routing, and failure semantics.
|
||||
For a standalone plugin installation, the host activates the matching adapter
|
||||
first. The adapter preserves the caller's actual goal, constructs the task
|
||||
context, and invokes Entry. It does not select the internal review or
|
||||
plan-enrichment action skill itself or duplicate Entry's preparation, routing, and
|
||||
failure semantics.
|
||||
|
||||
### 3. Agent consumes the dispatch record
|
||||
The dispatch record names one or more action skills and the subset of inputs each should receive. If the outcome is `no-match` or `failed`, the agent returns the record to the orchestrator unchanged.
|
||||
The dispatch record names one or more action skills, the subset of inputs each
|
||||
should receive, and each skill's output kind. The output kind distinguishes
|
||||
findings from read-only plan guidance before invocation; it is not proof of
|
||||
runtime side effects. If the outcome is `no-match` or `failed`, the agent returns
|
||||
the record to the orchestrator unchanged.
|
||||
|
||||
### 4. Agent invokes each dispatched action skill
|
||||
Action skills live inside the layers — `/microsoft/skills/`, `/community/skills/`, `/custom/skills/` — so their authority is carried by their location. For a PR review, Entry typically dispatches `microsoft/skills/review/al-code-review.md`. The agent reads the file and executes it.
|
||||
|
|
@ -165,19 +171,82 @@ security boundary. See [layer selection](customizing-bcquality.md#select-layers-
|
|||
The index changes only *how candidates are discovered*, never *which are selected*. The Worklist predicate is unchanged — `keywords` still drive selection — and the agent still opens each worklisted article **in full** to read its `## Best Practice` / `## Anti Pattern` rule bodies; the index is discovery metadata only and never substitutes for the article body. When no index is present, skills fall back to path-based discovery (collect by domain folder), so review still works.
|
||||
|
||||
### 6. Agent emits structured output
|
||||
The output contract is defined in the DO meta-skill so that every action skill — today's and next year's — produces the same shape:
|
||||
The output contracts are defined in the DO meta-skill:
|
||||
|
||||
- **Outcome** — `completed`, `not-applicable`, `no-knowledge`, `partial`, or `failed`. An orchestrator can distinguish a clean run from a no-op from a failure without guessing.
|
||||
- **Findings** — what the skill observed (severity, message, optional location).
|
||||
- **Domain** — the producer-owned, human-readable display label on each review finding.
|
||||
- **References** — structured objects (`path` plus optional commit `sha`) pointing to the knowledge files that informed each finding.
|
||||
- **Confidence** — per-finding evidence strength.
|
||||
- **Suppressed** — knowledge files that were discarded by layer precedence or configuration, so reviewers can see what was overridden.
|
||||
- A **findings report** carries review findings, domain labels, references, confidence, and suppressions.
|
||||
- A **development guidance report** carries read-only knowledge constraints and validation considerations for an existing plan.
|
||||
|
||||
The orchestrator parses this **without skill-specific logic**. This is the point of the contract: orchestrators and action skills evolve independently.
|
||||
|
||||
For plan enrichment, the skill reads the existing plan and target repository,
|
||||
selects applicable knowledge, and returns constraints without changing the
|
||||
target. It does not generate a replacement plan, run tests, implement code, or
|
||||
drive a review/fix loop. Implementation stays in the consuming workflow.
|
||||
|
||||
### 7. Orchestrator integrates
|
||||
The orchestrator turns findings into PR comments, build gates, or IDE diagnostics, and links the references back to the knowledge files so the PR author — human or agent — can read the guidance.
|
||||
The orchestrator turns findings into PR comments, build gates, or IDE diagnostics. It can feed read-only guidance into its own implementation phases, preserving all existing approvals and delivery gates.
|
||||
|
||||
## Repository-specific development orchestrators
|
||||
|
||||
A repository-specific workflow can consume this read-only foundation before
|
||||
authoring while retaining its independent final review. This is the intended
|
||||
integration boundary, not a shipped consumer integration:
|
||||
|
||||
1. Investigate and produce the consumer's normal initial plan. Normalize any
|
||||
consumer-specific format outside BCQuality. A full serialized plan document
|
||||
containing metadata plus a markdown body (root cause or design intent,
|
||||
proposed changes, affected files, test strategy, acceptance criteria) is a
|
||||
valid boundary. A continuation/checkpoint payload is not a substitute for
|
||||
initial-plan coverage; workflow identifiers and state stay with the consumer.
|
||||
2. Resolve and record an immutable BCQuality checkout and filtering policy.
|
||||
Invoke Entry with a read-only enrichment goal, the existing
|
||||
`development-plan`, `repository`, and established applicability dimensions.
|
||||
Keep index, guidance, and runner artifacts outside the target repository.
|
||||
3. Execute the dispatched `al-development-plan` skill. Persist the unchanged
|
||||
report and provenance **after** any consumer state initialization or cleanup
|
||||
that could erase them. BCQuality does not own the state directory or lifecycle.
|
||||
4. Inject relevant constraints and validation considerations into the existing
|
||||
Baseline, Implement, propagation (such as MiApp), and Critique phases, or
|
||||
equivalents. Re-enrich on material plan or applicability changes; preserve
|
||||
the relationship between plan version, guidance, and implementation attempt.
|
||||
5. Run an independent final BCQuality review against the completed diff using
|
||||
the **same recorded immutable checkout** used for enrichment. Review the
|
||||
actual changes, not the guidance report as proof of correctness, then apply
|
||||
the consumer's ordinary delivery gates.
|
||||
|
||||
The consumer owns analysis, normalization, persistence, per-phase injection,
|
||||
approvals, TDD and runtime execution, propagation, retries, commits, and PR
|
||||
delivery. BCQuality supplies additional referenced product knowledge, not a
|
||||
replacement orchestrator.
|
||||
|
||||
### Outcomes are additive, not a universal coding gate
|
||||
|
||||
`no-knowledge` with empty `knowledge` means no additional applicable BCQuality
|
||||
constraints. The consumer may proceed under its ordinary gates. It must not be
|
||||
conflated with failed retrieval/reference integrity (`failed`), incomplete
|
||||
evaluation or materially unresolved conditional guidance (`partial`), or absent
|
||||
required inputs (`not-applicable`). Consumers own the policy for handling those
|
||||
outcomes and recorded unknowns: seek missing context, re-enrich, escalate, or
|
||||
apply their existing risk controls without relabeling the report as successful.
|
||||
Do not fill gaps with generic articles simply to unlock implementation.
|
||||
|
||||
### Pinning and pilot evidence
|
||||
|
||||
A configured tag or ref alone does not establish runtime pinning. Record the
|
||||
resolved commit and actual checkout/content identity used at invocation, along
|
||||
with enabled layers, pruning policy, index identity, plan version, and runner
|
||||
provenance. Verify that identity at both enrichment and final review; fetching
|
||||
default HEAD into an explicitly supplied checkout can bypass a configured ref.
|
||||
Use an isolated checkout that cannot drift during the run.
|
||||
|
||||
Consumer rollout and an external pilot remain follow-up work. A pilot must
|
||||
compare a pinned independent baseline run of the existing workflow without
|
||||
enrichment against a matched enriched run, keeping starting code, task, model,
|
||||
tools, runtime, and gates controlled and recording the actual BCQuality
|
||||
checkout. Retain external logs, diffs, test/compile outcomes, and independent
|
||||
final reviews, including failures and unresolved results. Credential-free
|
||||
fixture preparation and scorer regressions do not demonstrate improved repair
|
||||
quality, compilation, runtime success, or production integration.
|
||||
|
||||
## Knowledge-backed and agent findings
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# AL review evaluation
|
||||
# AL review and guidance evaluation
|
||||
|
||||
The evaluation is convention-driven. The harness discovers every `<layer>/skills/review/al-<domain>-review.md` leaf across the enabled `microsoft`, `community`, and `custom` layers. Duplicate domains resolve with `custom > community > microsoft` precedence. For each selected leaf, the harness finds paired knowledge across the same layers, applies the same precedence to duplicate article slugs, selects the first article (by filename) with both `.bad.al` and `.good.al` companions, and derives the expected positive and clean control automatically. Adding a conforming leaf requires no scoring-contract edit.
|
||||
|
||||
|
|
@ -54,3 +54,117 @@ This credential-free check proves every selected leaf maps to a same-named knowl
|
|||
For a single combined stress-test result, use `-ResultsPath` instead.
|
||||
|
||||
The committed gate requires full expected recall, the exact convention-derived article ID, and no findings on clean controls.
|
||||
|
||||
## Read-only plan guidance
|
||||
|
||||
`development-guidance-fixtures.json` evaluates the planning interface used by
|
||||
existing workflows. It supplies an existing plan and expects referenced
|
||||
constraints without target-repository changes. The initial-plan fixture is
|
||||
anonymized and synthetic: a full document with metadata and a markdown body
|
||||
covering root cause, proposed fix, affected files, tests, and acceptance
|
||||
criteria. It is integration-shaped input, not private consumer content, a
|
||||
continuation checkpoint, or proof that any production consumer is integrated.
|
||||
|
||||
### Credential-free contract and scorer coverage
|
||||
|
||||
CI validates the manifest, prepares opaque model requests, and runs
|
||||
deterministic scorer regressions with controlled reports and temporary Git
|
||||
repositories. These checks cover report shape, outcomes, reference paths, and
|
||||
the evaluator's pre/post read-only comparison. They do **not** run an agent,
|
||||
compile AL, run Business Central tests, or establish better code authoring.
|
||||
|
||||
Prepare requests in a runner-owned artifact directory outside every target
|
||||
workspace:
|
||||
|
||||
```powershell
|
||||
$run = Join-Path ([IO.Path]::GetTempPath()) 'bcquality-guidance-run'
|
||||
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory $run
|
||||
```
|
||||
|
||||
Preparation is not a model run. A scorer can validate a citation's path and
|
||||
required fields, but only external agent traces and expert evaluation can
|
||||
establish that the article was opened and its normative constraints faithfully
|
||||
applied. Expected knowledge recall/precision is fixture-specific, not a corpus
|
||||
coverage or authoring capability percentage.
|
||||
|
||||
`no-knowledge` means no additional applicable BCQuality constraints, not unsafe
|
||||
work. It requires empty `knowledge`. Partial evaluation, failed retrieval,
|
||||
unknown context, and materially unresolved applicability must remain visible
|
||||
and distinct; they cannot be counted as successful enrichment merely because
|
||||
the JSON is parseable.
|
||||
|
||||
Run the deterministic regression suite without an agent or AL environment:
|
||||
|
||||
```powershell
|
||||
pwsh ./tools/Test-DevelopmentGuidanceEvaluator.ps1
|
||||
```
|
||||
|
||||
### Runner-owned read-only evidence
|
||||
|
||||
For an external guidance run, first provision a representative, standalone Git
|
||||
repository for each manifest case. The runner supplies a JSON workspace map
|
||||
whose keys are the manifest IDs (not the hashed model IDs) and whose values are
|
||||
absolute workspace roots. It may pass `-WorkspaceMapPath` during preparation
|
||||
to bind the generated requests to those roots. The model must not select its
|
||||
own workspace for scoring.
|
||||
|
||||
Capture evidence **before** invoking the agent, with the manifest, workspace
|
||||
map, and source checkout already finalized:
|
||||
|
||||
```powershell
|
||||
# Runner-selected paths, all outside the targets and BCQuality checkout.
|
||||
$map = Join-Path $evidenceDirectory 'workspace-map.json'
|
||||
$baseline = Join-Path $evidenceDirectory 'baseline.json'
|
||||
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . `
|
||||
-CaptureBaseline -WorkspaceMapPath $map -BaselinePath $baseline
|
||||
|
||||
# Retain the printed SHA256 in runner-only state BEFORE agent invocation.
|
||||
# After the external agent writes result-case-<hash>.json files:
|
||||
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . `
|
||||
-ResultsDirectory $resultsDirectory -BaselinePath $baseline `
|
||||
-BaselineSha256 $preRunDigest
|
||||
```
|
||||
|
||||
`$evidenceDirectory`, `$resultsDirectory`, and `$preRunDigest` are supplied by
|
||||
the runner; the digest must not be recomputed from potentially modified evidence
|
||||
after the agent runs. Protect the baseline, digest, evaluator, and invocation
|
||||
from agent changes. Capture refuses to overwrite an existing baseline. Results
|
||||
contain only `caseId` and `guidanceReport`; a legacy `workspaceRoot`, if present,
|
||||
must agree with the independently captured binding and never overrides it.
|
||||
Missing baselines or digests, malformed reports, and escaped reference paths
|
||||
fail scoring.
|
||||
|
||||
The comparison checks target identity, Git HEAD, refs and index, filesystem
|
||||
content and stable metadata, including tracked, untracked, ignored files and
|
||||
empty directories. Committing edits or making an empty commit does not evade
|
||||
the check. It also compares the actual knowledge checkout and manifest identity.
|
||||
Targets must have internal Git storage; linked target worktrees, submodules,
|
||||
sparse checkouts, links/junctions/reparse points, hard links, and alternate data
|
||||
streams are unsupported and rejected rather than silently excluded. A linked
|
||||
**knowledge** checkout is supported with its Git storage identity recorded.
|
||||
Use quiescent, isolated repositories; concurrent changes also fail the gate.
|
||||
|
||||
This is before/after evidence, not an OS sandbox or a complete write monitor.
|
||||
It cannot prove that no transient write was reverted, that articles were opened,
|
||||
or that constraints are semantically faithful. Reports and generated artifacts
|
||||
must stay outside all target workspaces and the knowledge checkout. The
|
||||
regression suite creates and removes its own uniquely named fixture directory;
|
||||
it does not run against or clean a caller's target.
|
||||
|
||||
### External agent/runtime pilot (follow-up)
|
||||
|
||||
Consumer uptake and a real before-authoring pilot are not implemented by these
|
||||
fixtures. The consumer must normalize its normal initial plan, persist guidance
|
||||
after state initialization, inject it into existing phases, re-enrich on
|
||||
material changes, and run an independent final review. See
|
||||
[the integration boundary](../agent-consumption.md#repository-specific-development-orchestrators).
|
||||
|
||||
Before claiming improved repairs, run an independent pinned baseline without
|
||||
enrichment and a matched enriched run. Hold starting code, task, model, tools,
|
||||
runtime, and gates constant; record actual immutable BCQuality checkout and
|
||||
policy identities rather than trusting a configured ref. Use that same
|
||||
recorded checkout for enrichment and final review. Retain external logs,
|
||||
article-read traces, resulting diffs, compile/test outcomes, and independent
|
||||
review evidence, including failures, no-knowledge, partial, and unresolved
|
||||
results. No compile/run or authoring-quality claim follows from the
|
||||
credential-free checks above.
|
||||
|
|
|
|||
175
evaluation/development-guidance-fixtures.json
Normal file
175
evaluation/development-guidance-fixtures.json
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
{
|
||||
"version": 1,
|
||||
"skill": "microsoft/skills/development/al-development-plan.md",
|
||||
"minimumKnowledgeRecall": 1.0,
|
||||
"minimumKnowledgePrecision": 0.67,
|
||||
"cases": [
|
||||
{
|
||||
"id": "synthetic-normal-initial-plan",
|
||||
"title": "Anonymized normal initial-plan consumer boundary",
|
||||
"evidenceType": "integration-shaped-synthetic",
|
||||
"boundary": "A synthetic consumer produces metadata plus a markdown plan body, serializes the full document, and passes that document as the generic existing development-plan. This is not an external real pilot and contains no consumer workflow-state schema.",
|
||||
"expectedKind": "bug",
|
||||
"expectedOutcome": "completed",
|
||||
"expectedUnknown": [],
|
||||
"requiresUnresolved": false,
|
||||
"requiresMaterialUnresolved": false,
|
||||
"development-plan": "{\"metadata\":{\"kind\":\"bug\",\"request\":\"Update every entry in the supplied filtered record set while preserving the caller's selection.\",\"origin\":\"anonymized synthetic initial plan\"},\"body\":\"## Root cause and design\\nThe routine reads and updates only the first record rather than iterating the supplied filtered set. Preserve the supplied filters and update each selected row.\\n\\n## Proposed fix\\nUse an update-safe FindSet/Next loop with an explicit update on each selected entry.\\n\\n## Affected files\\n- src/Batch/UpdateSelectedEntries.Codeunit.al\\n- test/Batch/UpdateSelectedEntriesTests.Codeunit.al\\n\\n## Test strategy\\nUse existing AL test library codeunits to arrange three selected rows and an excluded row. Assert all selected rows are updated and the excluded row is unchanged. This is a proposed test, not a reported result.\\n\\n## Acceptance criteria\\n- Every selected entry is updated exactly once.\\n- The supplied filters remain effective.\\n- No excluded entry changes.\\n- Empty selections cause no changes.\\n\"}",
|
||||
"context": {
|
||||
"bc-version": "28",
|
||||
"technologies": [
|
||||
"al"
|
||||
],
|
||||
"countries": [
|
||||
"w1"
|
||||
],
|
||||
"application-area": [
|
||||
"all"
|
||||
],
|
||||
"unknown": []
|
||||
},
|
||||
"requiredKnowledge": [
|
||||
"microsoft/knowledge/performance/pair-findset-with-next-loop.md",
|
||||
"microsoft/knowledge/performance/findset-true-applies-updlock-on-read.md",
|
||||
"microsoft/knowledge/testing/use-library-codeunits-for-test-fixtures.md"
|
||||
],
|
||||
"optionalKnowledge": [
|
||||
"microsoft/knowledge/performance/pass-var-record-to-preserve-partial-load-enumerator.md"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "versioned-upgrade-plan-guidance",
|
||||
"title": "Select guidance for a versioned data upgrade",
|
||||
"expectedKind": "upgrade",
|
||||
"expectedOutcome": "completed",
|
||||
"expectedUnknown": [],
|
||||
"requiresUnresolved": false,
|
||||
"requiresMaterialUnresolved": false,
|
||||
"development-plan": {
|
||||
"kind": "upgrade",
|
||||
"request": "Migrate existing customer tier text values to a new enum field in an app upgrade.",
|
||||
"root-cause": "The new schema needs an explicit, rerunnable migration for existing tenant data.",
|
||||
"affected-files": [
|
||||
"src/Upgrade/CustomerTierUpgrade.Codeunit.al",
|
||||
"test/Upgrade/CustomerTierUpgradeTests.Codeunit.al"
|
||||
],
|
||||
"proposed-changes": [
|
||||
"Add a tagged upgrade step that copies existing values without validation triggers.",
|
||||
"Add upgrade tests from multiple historical data versions."
|
||||
],
|
||||
"test-strategy": "Run upgrade tests from two prior data versions and verify a second invocation makes no further changes.",
|
||||
"acceptance-criteria": [
|
||||
"Existing values are preserved.",
|
||||
"The migration is rerunnable.",
|
||||
"Fresh installation does not execute upgrade migration."
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"bc-version": "28",
|
||||
"technologies": [
|
||||
"al"
|
||||
],
|
||||
"countries": [
|
||||
"w1"
|
||||
],
|
||||
"application-area": [
|
||||
"all"
|
||||
],
|
||||
"unknown": []
|
||||
},
|
||||
"requiredKnowledge": [
|
||||
"microsoft/knowledge/upgrade/use-upgrade-tags-not-version-checks.md",
|
||||
"microsoft/knowledge/upgrade/check-only-triggers-do-not-migrate-data.md",
|
||||
"microsoft/knowledge/upgrade/install-code-does-not-run-on-version-upgrade.md"
|
||||
],
|
||||
"optionalKnowledge": [
|
||||
"microsoft/knowledge/upgrade/datatransfer-skips-triggers-and-subscribers.md",
|
||||
"microsoft/knowledge/upgrade/appversion-meaning-depends-on-execution-context.md"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "no-additional-knowledge",
|
||||
"title": "Honest empty enrichment does not prohibit ordinary work",
|
||||
"expectedKind": "maintenance",
|
||||
"expectedOutcome": "no-knowledge",
|
||||
"expectedUnknown": [],
|
||||
"requiresUnresolved": false,
|
||||
"requiresMaterialUnresolved": false,
|
||||
"development-plan": {
|
||||
"kind": "maintenance",
|
||||
"request": "Correct a spelling error in an existing internal explanatory comment in an AL procedure. Do not change the explanation, executable code, UI captions, schema, diagnostic text, configuration or behavior.",
|
||||
"affected-files": ["src/Batch/EntryProcessor.Codeunit.al"],
|
||||
"proposed-changes": ["Replace the misspelled word in the existing comment; add no new advice."],
|
||||
"test-strategy": "Inspect the diff to confirm that only the comment spelling changes.",
|
||||
"acceptance-criteria": ["Only the intended comment spelling changes; executable AL remains identical."]
|
||||
},
|
||||
"context": {
|
||||
"bc-version": "28",
|
||||
"technologies": ["al"],
|
||||
"countries": ["w1"],
|
||||
"application-area": ["all"],
|
||||
"unknown": []
|
||||
},
|
||||
"requiredKnowledge": [],
|
||||
"optionalKnowledge": []
|
||||
},
|
||||
{
|
||||
"id": "unknown-material-version",
|
||||
"title": "Materially unresolved version-sensitive guidance stays partial",
|
||||
"expectedKind": "feature",
|
||||
"expectedOutcome": "partial",
|
||||
"expectedUnknown": ["bc-version"],
|
||||
"requiresUnresolved": true,
|
||||
"requiresMaterialUnresolved": true,
|
||||
"development-plan": {
|
||||
"kind": "feature",
|
||||
"request": "Add an expensive Sum FlowField as the source of a usually-hidden page control. The proposed design relies on visibility suppressing calculation.",
|
||||
"affected-files": ["src/Pages/EntryOverview.Page.al"],
|
||||
"proposed-changes": ["Bind the page control directly to the FlowField and set Visible to a conditional expression."],
|
||||
"test-strategy": "Verify aggregate queries are not executed while the control is hidden.",
|
||||
"acceptance-criteria": ["Hidden controls do not cause expensive aggregate queries."],
|
||||
"unknown": ["The deployment BC version and visible-only calculation feature state cannot be established from this fixture. Do not invent either."]
|
||||
},
|
||||
"context": {
|
||||
"bc-version": "unknown",
|
||||
"technologies": ["al"],
|
||||
"countries": ["w1"],
|
||||
"application-area": ["all"],
|
||||
"unknown": ["bc-version"]
|
||||
},
|
||||
"requiredKnowledge": [
|
||||
"microsoft/knowledge/performance/hidden-flowfields-still-calculate-before-bc26-opt-in.md"
|
||||
],
|
||||
"optionalKnowledge": []
|
||||
},
|
||||
{
|
||||
"id": "partial-plan-decision",
|
||||
"title": "Known platform context does not resolve an incomplete plan decision",
|
||||
"expectedKind": "refactor",
|
||||
"expectedOutcome": "partial",
|
||||
"expectedUnknown": [],
|
||||
"requiresUnresolved": true,
|
||||
"requiresMaterialUnresolved": true,
|
||||
"development-plan": {
|
||||
"kind": "refactor",
|
||||
"request": "Refactor a record read helper currently using FindFirst followed by Next. The caller contract does not establish whether to return one row or enumerate the entire filtered set.",
|
||||
"affected-files": ["src/Queries/EntryReader.Codeunit.al"],
|
||||
"proposed-changes": ["Choose the read method consistent with the intended cardinality after that decision is clarified."],
|
||||
"test-strategy": "Add cardinality assertions after the caller contract is decided.",
|
||||
"acceptance-criteria": ["The method and enumeration agree with the clarified caller contract."],
|
||||
"unknown": ["Single-record versus multi-record caller intent remains materially unresolved."]
|
||||
},
|
||||
"context": {
|
||||
"bc-version": "28",
|
||||
"technologies": ["al"],
|
||||
"countries": ["w1"],
|
||||
"application-area": ["all"],
|
||||
"unknown": []
|
||||
},
|
||||
"requiredKnowledge": [
|
||||
"microsoft/knowledge/performance/pair-findset-with-next-loop.md"
|
||||
],
|
||||
"optionalKnowledge": []
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
table 50603 "Sample Order Header Bad"
|
||||
{
|
||||
fields
|
||||
{
|
||||
field(1; "No."; Code[20])
|
||||
{
|
||||
DataClassification = CustomerContent;
|
||||
}
|
||||
field(2; "Document Date"; Date)
|
||||
{
|
||||
DataClassification = CustomerContent;
|
||||
}
|
||||
}
|
||||
|
||||
trigger OnInsert()
|
||||
var
|
||||
SalesSetup: Record "Sales & Receivables Setup";
|
||||
NoSeries: Codeunit "No. Series";
|
||||
begin
|
||||
"Document Date" := WorkDate();
|
||||
|
||||
if "No." = '' then begin
|
||||
SalesSetup.Get();
|
||||
SalesSetup.TestField("Order Nos.");
|
||||
"No." := NoSeries.GetNextNo(SalesSetup."Order Nos.");
|
||||
end;
|
||||
end;
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
table 50602 "Sample Order Header Good"
|
||||
{
|
||||
fields
|
||||
{
|
||||
field(1; "No."; Code[20])
|
||||
{
|
||||
DataClassification = CustomerContent;
|
||||
}
|
||||
field(2; "Document Date"; Date)
|
||||
{
|
||||
DataClassification = CustomerContent;
|
||||
}
|
||||
}
|
||||
|
||||
trigger OnInsert()
|
||||
var
|
||||
SalesSetup: Record "Sales & Receivables Setup";
|
||||
NoSeries: Codeunit "No. Series";
|
||||
begin
|
||||
if "No." = '' then begin
|
||||
SalesSetup.Get();
|
||||
SalesSetup.TestField("Order Nos.");
|
||||
"No." := NoSeries.GetNextNo(SalesSetup."Order Nos.");
|
||||
end;
|
||||
|
||||
InitRecord();
|
||||
end;
|
||||
|
||||
procedure InitRecord()
|
||||
begin
|
||||
OnBeforeInitRecord(Rec);
|
||||
"Document Date" := WorkDate();
|
||||
OnAfterInitRecord(Rec);
|
||||
end;
|
||||
|
||||
[IntegrationEvent(false, false)]
|
||||
local procedure OnBeforeInitRecord(var SampleOrderHeader: Record "Sample Order Header Good")
|
||||
begin
|
||||
end;
|
||||
|
||||
[IntegrationEvent(false, false)]
|
||||
local procedure OnAfterInitRecord(var SampleOrderHeader: Record "Sample Order Header Good")
|
||||
begin
|
||||
end;
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
bc-version: [all]
|
||||
domain: data-modeling
|
||||
keywords: [document-header, initrecord, number-series, default-values, oninsert, initialization]
|
||||
technologies: [al]
|
||||
countries: [w1]
|
||||
application-area: [all]
|
||||
---
|
||||
|
||||
# Initialize document defaults in `InitRecord` after assigning the number
|
||||
|
||||
## Description
|
||||
|
||||
Business Central document headers assign their number series first and then call an `InitRecord` procedure that owns the remaining business defaults, such as posting and document dates. Keeping that sequence and extensibility point makes initialization consistent for every creation path and lets extensions subscribe around one documented operation. Defaults scattered across page triggers or unrelated helpers can differ between UI, API, test, and background creation.
|
||||
|
||||
## Best Practice
|
||||
|
||||
In the document table's insert path, assign the document number and then call `InitRecord`. Keep the default assignments in that procedure and expose narrow before/after events when other extensions must participate.
|
||||
|
||||
See sample: [`initialize-document-defaults-in-initrecord.good.al`](initialize-document-defaults-in-initrecord.good.al).
|
||||
|
||||
## Anti Pattern
|
||||
|
||||
Assigning document defaults in a page trigger, or scattering them directly through `OnInsert` with no `InitRecord` boundary. Non-page creation paths can then miss the defaults, and extensions have no stable initialization hook.
|
||||
|
||||
See sample: [`initialize-document-defaults-in-initrecord.bad.al`](initialize-document-defaults-in-initrecord.bad.al).
|
||||
|
||||
## Reference
|
||||
|
||||
[Use the InitRecord function](https://learn.microsoft.com/en-us/training/modules/use-document-standards-business-central/3-use-initrecord-function)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
codeunit 50601 "Directed Rounding Bad"
|
||||
{
|
||||
procedure FloorAmount(Value: Decimal; Precision: Decimal): Decimal
|
||||
begin
|
||||
// For negative values, '<' rounds toward zero rather than toward negative infinity.
|
||||
exit(Round(Value, Precision, '<'));
|
||||
end;
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
codeunit 50600 "Directed Rounding Good"
|
||||
{
|
||||
procedure RoundAmount(Value: Decimal; Precision: Decimal; IncreaseMagnitude: Boolean): Decimal
|
||||
begin
|
||||
if IncreaseMagnitude then
|
||||
exit(Round(Value, Precision, '>'));
|
||||
|
||||
exit(Round(Value, Precision, '<'));
|
||||
end;
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
bc-version: [all]
|
||||
domain: data-modeling
|
||||
keywords: [round, rounding, direction, precision, negative-decimal, amount]
|
||||
technologies: [al]
|
||||
countries: [w1]
|
||||
application-area: [all]
|
||||
---
|
||||
|
||||
# `Round` direction symbols follow magnitude, not mathematical ordering
|
||||
|
||||
## Description
|
||||
|
||||
AL's `Round(Number, Precision, Direction)` uses `'>'` to round away from zero and `'<'` to round toward zero. For a negative value this reverses mathematical ordering: `Round(-1234.56789, 0.001, '<')` returns `-1234.567`, while direction `'>'` returns `-1234.568`. Code that treats the symbols as mathematical ceiling and floor produces sign-dependent amount errors, commonly on credit documents and negative adjustments.
|
||||
|
||||
## Best Practice
|
||||
|
||||
Choose the direction from the business meaning: `'>'` increases absolute magnitude and `'<'` decreases absolute magnitude for both positive and negative values. Include positive and negative cases whenever a directed rounding rule is tested.
|
||||
|
||||
See sample: [`round-direction-symbols-use-magnitude.good.al`](round-direction-symbols-use-magnitude.good.al).
|
||||
|
||||
## Anti Pattern
|
||||
|
||||
Using `'<'` as a mathematical floor or `'>'` as a mathematical ceiling. The result looks correct for positive amounts but moves in the opposite mathematical direction for negative amounts.
|
||||
|
||||
See sample: [`round-direction-symbols-use-magnitude.bad.al`](round-direction-symbols-use-magnitude.bad.al).
|
||||
|
||||
## Reference
|
||||
|
||||
[Use the Round function](https://learn.microsoft.com/en-us/training/modules/use-document-standards-business-central/4a-use-round-function)
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
interface "I Quote Amount Bad"
|
||||
{
|
||||
procedure GetAmount(): Decimal;
|
||||
}
|
||||
|
||||
interface "I Quote Date Bad"
|
||||
{
|
||||
procedure GetDate(): Date;
|
||||
}
|
||||
|
||||
codeunit 50611 "Quote Reader Bad"
|
||||
{
|
||||
procedure GetDate(Quote: Interface "I Quote Amount Bad"): Date
|
||||
var
|
||||
DatedQuote: Interface "I Quote Date Bad";
|
||||
begin
|
||||
DatedQuote := Quote as "I Quote Date Bad";
|
||||
exit(DatedQuote.GetDate());
|
||||
end;
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
interface "I Quote Amount Good"
|
||||
{
|
||||
procedure GetAmount(): Decimal;
|
||||
}
|
||||
|
||||
interface "I Quote Date Good"
|
||||
{
|
||||
procedure GetDate(): Date;
|
||||
}
|
||||
|
||||
codeunit 50610 "Quote Reader Good"
|
||||
{
|
||||
procedure TryGetDate(Quote: Interface "I Quote Amount Good"; var QuoteDate: Date): Boolean
|
||||
var
|
||||
DatedQuote: Interface "I Quote Date Good";
|
||||
begin
|
||||
if not (Quote is "I Quote Date Good") then
|
||||
exit(false);
|
||||
|
||||
DatedQuote := Quote as "I Quote Date Good";
|
||||
QuoteDate := DatedQuote.GetDate();
|
||||
exit(true);
|
||||
end;
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
bc-version: [25..]
|
||||
domain: interfaces
|
||||
keywords: [interface, is-operator, as-operator, type-test, cast, variant, runtime-error]
|
||||
technologies: [al]
|
||||
countries: [w1]
|
||||
application-area: [all]
|
||||
---
|
||||
|
||||
# Guard optional interface casts with `is`
|
||||
|
||||
## Description
|
||||
|
||||
From runtime 14.0, AL can type-test an interface or `Variant` with `is` and cast it to another interface with `as`. The test is non-throwing, but `as` raises a runtime error when the underlying codeunit does not implement the target interface. This matters when an extended capability is optional or implementations can come from other extensions.
|
||||
|
||||
## Best Practice
|
||||
|
||||
Use `is` to establish that the value supports the target interface before using `as`. Cast directly only where the target implementation is an invariant guaranteed by the surrounding contract.
|
||||
|
||||
See sample: [`guard-interface-casts-with-is.good.al`](guard-interface-casts-with-is.good.al).
|
||||
|
||||
## Anti Pattern
|
||||
|
||||
Using `as` unconditionally for an optional extended interface. An otherwise valid implementation of the base interface then fails at runtime merely because it does not implement the additional contract.
|
||||
|
||||
See sample: [`guard-interface-casts-with-is.bad.al`](guard-interface-casts-with-is.bad.al).
|
||||
|
||||
## Reference
|
||||
|
||||
[Understand type testing and casting operators for interfaces](https://learn.microsoft.com/en-us/training/modules/business-central-interfaces/type-testing)
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
tableextension 50622 "Ship-to Dropdown Bad" extends "Ship-to Address"
|
||||
{
|
||||
fieldgroups
|
||||
{
|
||||
addlast(DropDown; "Address 2")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
tableextension 50620 "Ship-to Dropdown Good" extends "Ship-to Address"
|
||||
{
|
||||
fieldgroups
|
||||
{
|
||||
addlast(DropDown; "Address 2")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pageextension 50621 "Ship-to Lookup Good" extends "Ship-to Address List"
|
||||
{
|
||||
layout
|
||||
{
|
||||
modify("Address 2")
|
||||
{
|
||||
Visible = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
bc-version: [all]
|
||||
domain: ui
|
||||
keywords: [fieldgroup, dropdown, addlast, lookup-page, visible, tableextension, pageextension]
|
||||
technologies: [al]
|
||||
countries: [w1]
|
||||
application-area: [all]
|
||||
---
|
||||
|
||||
# A `DropDown` field remains hidden when its lookup-page control is hidden
|
||||
|
||||
## Description
|
||||
|
||||
A tableextension can append a field to the `DropDown` field group with `addlast`, but the client still omits that field when its control on the underlying lookup page has `Visible = false`. Changing only the table field group therefore compiles while producing no visible UI change. The field-group name is case-sensitive and must be written as `DropDown`.
|
||||
|
||||
## Best Practice
|
||||
|
||||
When adding a hidden field to a `DropDown` field group, also extend the page used for the lookup and make that field control visible. Verify the actual lookup page rather than assuming the table definition alone controls the drop-down.
|
||||
|
||||
See sample: [`dropdown-fieldgroup-respects-lookup-page-visibility.good.al`](dropdown-fieldgroup-respects-lookup-page-visibility.good.al).
|
||||
|
||||
## Anti Pattern
|
||||
|
||||
Adding the field with `addlast(DropDown; ...)` while leaving its lookup-page control hidden, then expecting the field to appear in the drop-down.
|
||||
|
||||
See sample: [`dropdown-fieldgroup-respects-lookup-page-visibility.bad.al`](dropdown-fieldgroup-respects-lookup-page-visibility.bad.al).
|
||||
|
||||
## Reference
|
||||
|
||||
[Add a new FieldGroup to an existing table](https://learn.microsoft.com/en-us/training/modules/extend-modify-existing-table/add-field-group)
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
page 50631 "Sample Order Bad"
|
||||
{
|
||||
PageType = Document;
|
||||
SourceTable = "Sales Header";
|
||||
|
||||
layout
|
||||
{
|
||||
area(Content)
|
||||
{
|
||||
group(General)
|
||||
{
|
||||
field(Amount; Rec.Amount)
|
||||
{
|
||||
ApplicationArea = All;
|
||||
ToolTip = 'Specifies the total amount of the order.';
|
||||
}
|
||||
}
|
||||
part(Lines; "Sales Order Subform")
|
||||
{
|
||||
ApplicationArea = All;
|
||||
SubPageLink = "Document Type" = field("Document Type"),
|
||||
"Document No." = field("No.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
page 50630 "Sample Order Good"
|
||||
{
|
||||
PageType = Document;
|
||||
SourceTable = "Sales Header";
|
||||
|
||||
layout
|
||||
{
|
||||
area(Content)
|
||||
{
|
||||
group(General)
|
||||
{
|
||||
field(Amount; Rec.Amount)
|
||||
{
|
||||
ApplicationArea = All;
|
||||
ToolTip = 'Specifies the total amount of the order.';
|
||||
}
|
||||
}
|
||||
part(Lines; "Sales Order Subform")
|
||||
{
|
||||
ApplicationArea = All;
|
||||
SubPageLink = "Document Type" = field("Document Type"),
|
||||
"Document No." = field("No.");
|
||||
UpdatePropagation = Both;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
bc-version: [all]
|
||||
domain: ui
|
||||
keywords: [updatepropagation, page-part, subpage, main-page, refresh, flowfield, document-lines]
|
||||
technologies: [al]
|
||||
countries: [w1]
|
||||
application-area: [all]
|
||||
---
|
||||
|
||||
# Use `UpdatePropagation = Both` when line edits must refresh the main page
|
||||
|
||||
## Description
|
||||
|
||||
A page part does not automatically refresh its parent page when the subpage changes. `UpdatePropagation = Subpage` updates only the part; `Both` also refreshes the main page. Without `Both`, header totals, FlowFields, and FactBoxes that depend on edited lines can remain stale until another user action refreshes the page.
|
||||
|
||||
## Best Practice
|
||||
|
||||
Set `UpdatePropagation = Both` on a part when edits in that subpage must immediately update values rendered by the main page. Leave propagation at `Subpage` when the parent has no dependent presentation to avoid unnecessary refreshes.
|
||||
|
||||
See sample: [`updatepropagation-both-refreshes-main-page.good.al`](updatepropagation-both-refreshes-main-page.good.al).
|
||||
|
||||
## Anti Pattern
|
||||
|
||||
Displaying a line-dependent total on the main page while the editable lines part updates only itself. The persisted values can be correct while the parent page continues to show an old total.
|
||||
|
||||
See sample: [`updatepropagation-both-refreshes-main-page.bad.al`](updatepropagation-both-refreshes-main-page.bad.al).
|
||||
|
||||
## Reference
|
||||
|
||||
[Set different control properties](https://learn.microsoft.com/en-us/training/modules/work-with-pages/8-controls)
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
---
|
||||
bc-version: [all]
|
||||
domain: upgrade
|
||||
keywords: [appversion, dataversion, moduleinfo, install-codeunit, upgrade-codeunit, version-context]
|
||||
technologies: [al]
|
||||
countries: [w1]
|
||||
application-area: [all]
|
||||
---
|
||||
|
||||
# `ModuleInfo.AppVersion` changes meaning with execution context
|
||||
|
||||
## Description
|
||||
|
||||
`ModuleInfo.AppVersion()` is the installed version during normal operation, the version being installed inside install code, and the target version inside upgrade code. It is therefore not the source data version during an upgrade. In upgrade code, `DataVersion()` describes the version of the existing data, whether from the currently installed app or the version most recently uninstalled.
|
||||
|
||||
## Best Practice
|
||||
|
||||
Interpret `AppVersion()` as the code package entering the context and `DataVersion()` as the existing data state. Prefer upgrade tags for controlling individual migration steps; when version information is needed for diagnostics or preconditions, name variables so target app version and source data version cannot be confused.
|
||||
|
||||
## Anti Pattern
|
||||
|
||||
Reading `AppVersion()` from an upgrade codeunit and treating it as the version being upgraded from. The comparison actually observes the target package and can skip or misroute migration logic.
|
||||
|
||||
## Reference
|
||||
|
||||
[Create proper installation and upgrade codeunits](https://learn.microsoft.com/en-us/training/modules/easy-application-upgrade/3-installation-upgrade-codeunits)
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
codeunit 50641 "Sample Upgrade Part One"
|
||||
{
|
||||
Subtype = Upgrade;
|
||||
|
||||
trigger OnUpgradePerCompany()
|
||||
begin
|
||||
CreateUpgradeState();
|
||||
end;
|
||||
|
||||
local procedure CreateUpgradeState()
|
||||
begin
|
||||
end;
|
||||
}
|
||||
|
||||
codeunit 50642 "Sample Upgrade Part Two"
|
||||
{
|
||||
Subtype = Upgrade;
|
||||
|
||||
trigger OnUpgradePerCompany()
|
||||
begin
|
||||
// This can run before Part One; object IDs do not sequence upgrade codeunits.
|
||||
MigrateDataThatRequiresUpgradeState();
|
||||
end;
|
||||
|
||||
local procedure MigrateDataThatRequiresUpgradeState()
|
||||
begin
|
||||
end;
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
codeunit 50640 "Sample Upgrade Good"
|
||||
{
|
||||
Subtype = Upgrade;
|
||||
|
||||
trigger OnUpgradePerCompany()
|
||||
begin
|
||||
CreateUpgradeState();
|
||||
MigrateDependentData();
|
||||
end;
|
||||
|
||||
local procedure CreateUpgradeState()
|
||||
begin
|
||||
end;
|
||||
|
||||
local procedure MigrateDependentData()
|
||||
begin
|
||||
end;
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
bc-version: [all]
|
||||
domain: upgrade
|
||||
keywords: [install-codeunit, upgrade-codeunit, execution-order, subtype-install, subtype-upgrade, sequencing]
|
||||
technologies: [al]
|
||||
countries: [w1]
|
||||
application-area: [all]
|
||||
---
|
||||
|
||||
# Separate install or upgrade codeunits have no execution order
|
||||
|
||||
## Description
|
||||
|
||||
An extension can contain multiple `Install` or `Upgrade` codeunits, but Business Central does not guarantee the order in which codeunits of the same subtype execute. Upgrade trigger phases are ordered globally, yet one codeunit's `OnUpgradePerCompany` must not assume another codeunit's same-phase trigger already ran. Object ID and source-file order do not provide sequencing.
|
||||
|
||||
## Best Practice
|
||||
|
||||
Keep separate install or upgrade codeunits independent. When two steps have a real dependency, coordinate them from one owning trigger in the required order; use upgrade tags to make each completed step idempotent.
|
||||
|
||||
See sample: [`install-and-upgrade-codeunits-have-no-order.good.al`](install-and-upgrade-codeunits-have-no-order.good.al).
|
||||
|
||||
## Anti Pattern
|
||||
|
||||
Splitting dependent steps into separate codeunits and relying on names, object IDs, or declaration order. The dependent codeunit can run first and fail or observe partially migrated data.
|
||||
|
||||
See sample: [`install-and-upgrade-codeunits-have-no-order.bad.al`](install-and-upgrade-codeunits-have-no-order.bad.al).
|
||||
|
||||
## Reference
|
||||
|
||||
[Create proper installation and upgrade codeunits](https://learn.microsoft.com/en-us/training/modules/easy-application-upgrade/3-installation-upgrade-codeunits)
|
||||
72
microsoft/skills/development/al-development-plan.md
Normal file
72
microsoft/skills/development/al-development-plan.md
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
---
|
||||
kind: action-skill
|
||||
id: al-development-plan
|
||||
version: 1
|
||||
title: AL development plan guidance
|
||||
description: Produces a read-only BCQuality knowledge bundle for an existing Business Central AL development plan.
|
||||
inputs: [development-plan, repository]
|
||||
outputs: [development-guidance-report]
|
||||
bc-version: [all]
|
||||
technologies: [al]
|
||||
countries: [w1]
|
||||
application-area: [all]
|
||||
---
|
||||
|
||||
# AL development plan guidance
|
||||
|
||||
Selects the BCQuality knowledge that should constrain an existing AL development plan. It does not implement, edit, stage, commit, or publish anything in the target repository. Repository-specific orchestrators can consume this skill before their own test and implementation phases while retaining ownership of workflow, tooling, and delivery.
|
||||
|
||||
Both a readable `repository` and a non-empty `development-plan` are required. The plan may be structured data or text, but it must identify the intended change. Return `not-applicable` without changing files when either input is absent or the repository is not an AL project.
|
||||
|
||||
The caller supplies its existing plan, not a request to generate one. Consumer-specific formats must be normalized by the consumer before invocation. This skill does not interpret issue records, continuation markers, batons, retries, or workflow state. A serialized document containing plan metadata and a markdown body is acceptable when it states the intended change, affected surfaces, proposed approach, test strategy, and acceptance criteria. Missing details remain unknown; do not invent them.
|
||||
|
||||
## Source
|
||||
|
||||
Read the BCQuality knowledge index once, using the external path supplied by Entry when present. If no index is available, use READ's path-based discovery across enabled layers; inability to read the corpus is `failed`, not `no-knowledge`. Use entries from every enabled layer and domain. The index supplies candidate paths, applicability dimensions, keywords, titles, and descriptions; it never substitutes for opening selected articles in full.
|
||||
|
||||
Inspect the target repository read-only for `app.json`, affected files and symbols named by the plan, relevant tests, permission sets, dependencies, target/runtime versions, countries, application areas, and repository conventions. Do not create scratch or generated files inside the target repository.
|
||||
|
||||
## Relevance
|
||||
|
||||
Apply READ's matching semantics using:
|
||||
|
||||
- `bc-version` from the plan, target application, or supplied context; for upgrades, distinguish source and target versions.
|
||||
- `technologies` from the affected files, beginning with `[al]`.
|
||||
- `countries` from the plan, `app.json`, or workspace configuration.
|
||||
- `application-area` from the plan and affected objects.
|
||||
|
||||
When a dimension cannot be resolved, retain conditionally applicable candidates only when they can materially constrain the plan. Record the dimension in `context.unknown` and explain it in `unresolved`; do not silently treat it as a match.
|
||||
|
||||
## Worklist
|
||||
|
||||
1. Read the supplied plan for its request summary, development kind, assumptions, root cause or design intent, affected files and symbols, proposed changes, test strategy, and acceptance criteria. Preserve its intent; do not generate a replacement plan. If kind is not explicit, classify the stated intent: new or expanded behavior is `feature`, a defect correction is `bug`, behavior-preserving restructuring is `refactor`, migration is `upgrade`, and other bounded work is `maintenance`. Do not redesign the consumer's workflow.
|
||||
2. Build retrieval vocabulary from the plan and confirmed repository symbols. Give exact object types, properties, methods, analyzers, errors, and affected domains more weight than broad business nouns.
|
||||
3. Search the index in separate passes:
|
||||
- data ownership, keys, setup, numbering, validation, transactions, and upgrade;
|
||||
- behavior, events, interfaces, errors, permissions, privacy, and telemetry;
|
||||
- pages, reports, APIs, integrations, localization, and accessibility;
|
||||
- tests, analyzers, packaging, and deployment constraints.
|
||||
4. Add an article when its keywords or indexed topic match a concrete planned change, affected symbol, acceptance criterion, or validation obligation. Applicability alone is not enough.
|
||||
5. Open every selected article in full. Read any referenced `.good.*` and `.bad.*` sibling needed to make the constraint concrete. Never cite an index row that was not opened.
|
||||
6. Resolve contradictory normative guidance with READ's layer precedence and record losing candidates in `suppressed`.
|
||||
7. Check the resulting worklist across the whole plan. A bug fix may require testing, data, performance, and upgrade guidance at once; a feature plan may require security and lifecycle constraints that are not named in its title.
|
||||
|
||||
Keep the worklist focused. Do not include generic engineering advice, an entire domain, or an article that would not change implementation or validation.
|
||||
|
||||
## Action
|
||||
|
||||
For each worklist article:
|
||||
|
||||
1. Copy its exact path and optional commit SHA.
|
||||
2. State `used-for` as the concrete plan decision or affected surface.
|
||||
3. Translate its normative Best Practice and Anti Pattern into short implementation constraints without adding facts or weakening conditions.
|
||||
4. Include only opened, existing sibling samples in `sample-paths`.
|
||||
5. Derive validation considerations only where the plan or selected knowledge requires observable evidence. Describe the evidence to obtain; do not claim it already exists or passed.
|
||||
|
||||
Do not change the target repository. Before emitting, verify every knowledge and sample path exists in the live BCQuality checkout and was opened during this run. If reference integrity cannot be established, return `failed` rather than fabricating guidance.
|
||||
|
||||
## Output
|
||||
|
||||
Return one `development-guidance-report` conforming to DO. `completed` requires that every selected article was opened and faithfully converted into constraints, with no material unresolved applicability. `no-knowledge` means there are no additional applicable BCQuality constraints for this plan; emit empty `knowledge`. It does not mean the work is unsafe or unimplementable, and the consumer can proceed under its ordinary gates. Never add generic or filler guidance to avoid this outcome.
|
||||
|
||||
Return `partial` for incomplete evaluation or materially unresolved conditional guidance, naming every gap. Failed retrieval or reference-integrity checks are `failed`, never `no-knowledge`. Consumers own handling of partial, failed, and unresolved results, including clarification and re-enrichment; this read-only interface does not define a universal implementation gate.
|
||||
|
|
@ -39,7 +39,7 @@ Narrow the relevant files to the subset that applies to the changes under review
|
|||
|
||||
- The changed AL object names and types — especially `* Setup` singleton tables and Card pages, custom master tables, tableextensions that add master-data fields, and document or journal lines that reference a master.
|
||||
- The changed fields, keys, triggers, and procedures, weighted toward `Primary Key`, `No.`, `No. Series`, `Blocked`, `Last Date Modified`, `OnInsert`, `OnModify`, `OnRename`, reference-field `OnValidate`, and posting validation.
|
||||
- Tokens extracted from the diff that relate to data modeling (`setup`, `master`, `Primary Key`, `Code[10]`, `Code[20]`, `AutoIncrement`, `SystemId`, `No.`, `No. Series`, `NoSeriesManagement`, `Codeunit "No. Series"`, `GetNextNo`, `IsManual`, `TestManual`, `Blocked`, `TestField`, `Last Date Modified`, `Today`, `WorkDate`, `InsertAllowed`, `DeleteAllowed`, `PageType = Card`, `OnOpenPage`, `GetRecordOnce`, `OnInsert`, `OnModify`, `OnRename`, `TableRelation`, `tableextension`, `enumextension`, `Media`, `MediaSet`, `Item`, `Count`).
|
||||
- Tokens extracted from the diff that relate to data modeling (`setup`, `master`, `Primary Key`, `Code[10]`, `Code[20]`, `AutoIncrement`, `SystemId`, `No.`, `No. Series`, `NoSeriesManagement`, `Codeunit "No. Series"`, `GetNextNo`, `IsManual`, `TestManual`, `Blocked`, `TestField`, `Last Date Modified`, `Today`, `WorkDate`, `InsertAllowed`, `DeleteAllowed`, `PageType = Card`, `OnOpenPage`, `GetRecordOnce`, `OnInsert`, `OnModify`, `OnRename`, `InitRecord`, `Round`, `Precision`, `Direction`, `TableRelation`, `tableextension`, `enumextension`, `Media`, `MediaSet`, `Item`, `Count`).
|
||||
|
||||
A file enters the candidate worklist when its `keywords` intersect the extracted tokens or its topic (derived from the index entry's `path`, `title`, and `description`) matches a changed object type. Read an article's full file — its `## Best Practice` / `## Anti Pattern` bodies — only after it makes the worklist; candidate selection uses the index alone. When the diff contains no data-modeling changes by any of the above signals, return `outcome: "not-applicable"` without evaluating files.
|
||||
|
||||
|
|
@ -52,6 +52,8 @@ The following targeted checks cover every current `data-modeling` article. Treat
|
|||
- A master table adds or changes `Last Date Modified`, `OnModify`, or `OnRename`, but the non-editable field is not assigned `Today()` in both triggers — `set-last-date-modified-in-onmodify-and-onrename`.
|
||||
- A `tableextension` appends a conditional `TableRelation` as if it overrides an earlier unconditional relation, or relation branches are otherwise designed without accounting for additive top-down evaluation — `table-relation-extensions-are-additive-and-top-down`.
|
||||
- A `Media` or `MediaSet` field is assigned directly between different table types or different field IDs instead of registering each shared item with `MediaSet.Insert` — `share-mediaset-items-with-insert-not-field-assignment`.
|
||||
- A custom document header assigns defaults outside an `InitRecord` boundary, calls `InitRecord` before assigning its number, or places UI-independent defaults only in a page trigger — `initialize-document-defaults-in-initrecord`.
|
||||
- Directed `Round` calls use `'<'` as mathematical floor or `'>'` as mathematical ceiling, especially where negative amounts are possible — `round-direction-symbols-use-magnitude`.
|
||||
|
||||
Once the candidate worklist is known, resolve layer-precedence conflicts per READ. Drop lower-precedence files whose normative guidance (`## Best Practice` or `## Anti Pattern`) directly contradicts a higher-precedence candidate, and record each dropped file in `suppressed` with `reason: "layer-precedence"`. Files that would have been candidates but are hidden because their layer is disabled in consumer configuration are recorded with `reason: "configuration"`. Files that never became candidates are NOT recorded in `suppressed`.
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ Narrow the relevant files to the subset that applies to the changes under review
|
|||
|
||||
- The changed AL object names and types — especially `interface` objects, codeunits and enums declared with the `implements` keyword, and consumers that declare or assign an `Interface` variable.
|
||||
- The changed procedures and triggers, weighted toward factory or dispatch routines that resolve a variant to behaviour, setter-injection procedures that take an `Interface` parameter, and `case`-over-enum blocks that select between strategies.
|
||||
- Tokens extracted from the diff that relate to interfaces and enum-backed implementation (`interface`, `extends`, `implements`, `Implementation`, `DefaultImplementation`, `UnknownValueImplementation`, `enum`, `Extensible`, `Interface`, `case`, and the `case <enum> of` anti-pattern signal — a `case` over an enum value whose branches choose between variant computations).
|
||||
- Tokens extracted from the diff that relate to interfaces and enum-backed implementation (`interface`, `extends`, `implements`, `Implementation`, `DefaultImplementation`, `UnknownValueImplementation`, `enum`, `Extensible`, `Interface`, `Variant`, `is`, `as`, `case`, and the `case <enum> of` anti-pattern signal — a `case` over an enum value whose branches choose between variant computations).
|
||||
|
||||
A file enters the candidate worklist when its `keywords` intersect the extracted tokens or its topic (derived from the index entry's `path`, `title`, and `description`) matches a changed object type. Read an article's full file — its `## Best Practice` / `## Anti Pattern` bodies — only after it makes the worklist; candidate selection uses the index alone.
|
||||
|
||||
|
|
@ -54,6 +54,7 @@ The following targeted checks map diff signals to specific `interfaces` articles
|
|||
- `DefaultImplementation` used as the only fallback where a persisted ordinal may no longer match any declared enum value, or a persisted enum lacks `UnknownValueImplementation` on BC18 or later — `handle-unknown-enum-ordinals-with-unknownvalueimplementation`.
|
||||
- A method added directly to an interface that exists in the baseline, instead of adding a BC25+ interface that `extends` it or a versioned sibling for older targets — `extend-published-interfaces-dont-edit-them`.
|
||||
- A declared enum value with no `Implementation` and no enum-level `DefaultImplementation` — `set-defaultimplementation-on-enum`.
|
||||
- An `Interface` or `Variant` is cast with `as` to an optional extended interface without first establishing support with `is` — `guard-interface-casts-with-is`.
|
||||
|
||||
For `set-defaultimplementation-on-enum`, inspect the complete containing enum before emitting. An enum-level `DefaultImplementation = <Interface> = <Codeunit>;` conclusively covers every declared value that omits its own `Implementation`; do not flag such a value and do not replace the intentional fallback with a per-value mapping.
|
||||
|
||||
|
|
|
|||
|
|
@ -41,10 +41,15 @@ Narrow the relevant files to the subset that applies to the changes under review
|
|||
|
||||
- **UI-file filter.** UI review applies to files declaring `page`, `pageextension`, or `pagecustomization`, and to JavaScript/CSS/HTML that implements a control add-in's rendering or Business Central communication. When the diff contains no such files, return `outcome: "not-applicable"` without evaluating knowledge files.
|
||||
- For each relevant knowledge file, compute overlap against changed page declarations and control add-in files, weighted toward `Caption`, `ToolTip`, `AboutTitle`, `AboutText`, `OptionCaption`, `ShowCaption`, `InstructionalText`, `GridLayout`, `Style`, `StyleExpr`, promoted action definitions, field importance, page background tasks, DOM creation, ARIA attributes, keyboard/focus handlers, packaged-resource AJAX, and calls from JavaScript into AL.
|
||||
- Tokens extracted from the diff (`Caption`, `ToolTip`, `AboutTitle`, `AboutText`, `PageType`, `ShowCaption`, `InstructionalText`, `grid`, `fixed`, `GridLayout`, `Style`, `StyleExpr`, `Importance`, `Promoted`, `Additional`, `area(Promoted)`, `actionref`, `PromotedCategory`, `PromotedOnly`, `PromotedIsBig`, `ShowAs`, `SplitButton`, `EnqueueBackgroundTask`, `OnAfterGetCurrRecord`, `OnAfterGetRecord`, `OnPageBackgroundTaskCompleted`, `OnPageBackgroundTaskError`, `RunPageBackgroundTask`, `Favorable`, `Unfavorable`, `Ambiguous`, `cuegroup`, `controladdin`, `control-add-in`, `usercontrol`, `aria-`, `tabindex`, `keydown`, `focus`, `innerHTML`, `createElement`, `packaged-resource`, `ajax`, `$.get`, `$.ajax`, `XMLHttpRequest`, `xhrFields`, `withCredentials`, `withcredentials`, `InvokeExtensibilityMethod`, `invokeextensibilitymethod`, `skipIfBusy`, `successCallback`, `success-callback`, `errorCallback`, `setInterval`, `JSON.stringify`, `payload`, `throttling`, `reduced-functionality`, `ClientServicesMaxUploadSize`, `&`, `Specifies`, `Message(`, `Confirm(`, `Error(` in a page context, `Disabled`, `Invalid`, `Whitelist`, `Blacklist`, trailing punctuation patterns on captions).
|
||||
- Tokens extracted from the diff (`Caption`, `ToolTip`, `AboutTitle`, `AboutText`, `PageType`, `ShowCaption`, `InstructionalText`, `grid`, `fixed`, `GridLayout`, `Style`, `StyleExpr`, `Importance`, `Promoted`, `Additional`, `area(Promoted)`, `actionref`, `PromotedCategory`, `PromotedOnly`, `PromotedIsBig`, `ShowAs`, `SplitButton`, `fieldgroups`, `DropDown`, `UpdatePropagation`, `EnqueueBackgroundTask`, `OnAfterGetCurrRecord`, `OnAfterGetRecord`, `OnPageBackgroundTaskCompleted`, `OnPageBackgroundTaskError`, `RunPageBackgroundTask`, `Favorable`, `Unfavorable`, `Ambiguous`, `cuegroup`, `controladdin`, `control-add-in`, `usercontrol`, `aria-`, `tabindex`, `keydown`, `focus`, `innerHTML`, `createElement`, `packaged-resource`, `ajax`, `$.get`, `$.ajax`, `XMLHttpRequest`, `xhrFields`, `withCredentials`, `withcredentials`, `InvokeExtensibilityMethod`, `invokeextensibilitymethod`, `skipIfBusy`, `successCallback`, `success-callback`, `errorCallback`, `setInterval`, `JSON.stringify`, `payload`, `throttling`, `reduced-functionality`, `ClientServicesMaxUploadSize`, `&`, `Specifies`, `Message(`, `Confirm(`, `Error(` in a page context, `Disabled`, `Invalid`, `Whitelist`, `Blacklist`, trailing punctuation patterns on captions).
|
||||
|
||||
A file enters the candidate worklist when its `keywords` intersect the extracted tokens or its topic (derived from the index entry's `path`, `title`, and `description`) matches a changed page element. Read an article's full file — its `## Best Practice` / `## Anti Pattern` bodies — only after it makes the worklist; candidate selection uses the index alone.
|
||||
|
||||
Apply these high-signal mappings before fuzzy topic ranking:
|
||||
|
||||
- A tableextension adds a field to `DropDown` while the corresponding lookup-page control remains `Visible = false` — `dropdown-fieldgroup-respects-lookup-page-visibility`.
|
||||
- An editable page part affects a total, FlowField, or FactBox on the parent but does not set `UpdatePropagation = Both` — `updatepropagation-both-refreshes-main-page`.
|
||||
|
||||
Once the candidate worklist is known, resolve layer-precedence conflicts per READ and record suppressions.
|
||||
|
||||
When the post-conflict worklist is empty because no applicable UI knowledge exists, or because configuration suppressed every candidate, emit `outcome: "no-knowledge"`. When the worklist is empty because no applicable UI knowledge matched the page changes, emit `outcome: "completed"` with an empty `findings` array.
|
||||
|
|
|
|||
|
|
@ -39,10 +39,12 @@ Narrow the relevant files to the subset that applies to the changes under review
|
|||
|
||||
- The changed AL object names and types — especially codeunits with `Subtype = Upgrade` or `Subtype = Install`, tables and tableextensions adding or changing fields, enums and enumextensions, and objects under `Hybrid*`/`Migration`/`Upgrade` namespaces.
|
||||
- The changed triggers and procedures, weighted toward `OnCheckPreconditionsPerCompany`/`PerDatabase`, `OnUpgradePerCompany`/`PerDatabase`, `OnValidateUpgradePerCompany`/`PerDatabase`, `OnInstallAppPerCompany`/`PerDatabase`, the `OnGetPerCompanyUpgradeTags`/`OnGetPerDatabaseUpgradeTags` subscribers, and helper procedures transitively reachable from those entry points.
|
||||
- Tokens extracted from the diff that relate to upgrade concerns (`Subtype = Upgrade`, `Subtype = Install`, `Upgrade Tag`, `HasUpgradeTag`, `SetUpgradeTag`, `OnCheckPreconditions`, `OnUpgrade`, `OnValidateUpgrade`, `OnInstallApp`, `DataTransfer`, `CopyFields`, `Insert`, `Modify`, `Delete`, `Rename`, `InitValue`, `ObsoleteState`, `ObsoleteReason`, `ObsoleteTag`, `DataVersion`, `ExecutionContext`, `PrimaryKey`, `key(`, `field(`, `value(`, `enum`, `enumextension`, `HybridSL`, `HybridGP`, `HybridBC`, `HybridBaseDeployment`).
|
||||
- Tokens extracted from the diff that relate to upgrade concerns (`Subtype = Upgrade`, `Subtype = Install`, `Upgrade Tag`, `HasUpgradeTag`, `SetUpgradeTag`, `OnCheckPreconditions`, `OnUpgrade`, `OnValidateUpgrade`, `OnInstallApp`, `DataTransfer`, `CopyFields`, `Insert`, `Modify`, `Delete`, `Rename`, `InitValue`, `ObsoleteState`, `ObsoleteReason`, `ObsoleteTag`, `ModuleInfo`, `AppVersion`, `DataVersion`, `NavApp.GetCurrentModuleInfo`, `ExecutionContext`, `PrimaryKey`, `key(`, `field(`, `value(`, `enum`, `enumextension`, `HybridSL`, `HybridGP`, `HybridBC`, `HybridBaseDeployment`).
|
||||
- For each `OnCheckPreconditions...` and `OnValidateUpgrade...` trigger, build the best available call graph from surrounding unchanged source as well as changed hunks, tracing resolved calls through reachable local or internal helpers. Worklist the check-only rule when a database write occurs either directly in the trigger or in any helper procedure reachable from it. Writes include `Insert`, `Modify`, `ModifyAll`, `Delete`, `DeleteAll`, `Rename`, and `DataTransfer`. Also perform the reverse check when a PR changes a writing helper body: worklist the rule when that helper is invoked directly or transitively by an unchanged check or validation trigger.
|
||||
- Treat a direct write or a fully resolved call chain as high-confidence evidence. When cross-object dispatch, unavailable declarations, or an incomplete call graph prevents proving the complete chain, cap confidence at `medium`, name the unresolved edge in the finding, and do not claim a violation without a resolved path from a check or validation trigger to a write.
|
||||
- Worklist the install-versus-upgrade rule when migration helpers are reachable only from an install codeunit.
|
||||
- Worklist `install-and-upgrade-codeunits-have-no-order.md` when a change adds multiple install or upgrade codeunits whose same-phase triggers share state or depend on one another.
|
||||
- Worklist `appversion-meaning-depends-on-execution-context.md` when install or upgrade code branches on `ModuleInfo.AppVersion()` or confuses it with `DataVersion()`.
|
||||
|
||||
A file enters the candidate worklist when its `keywords` intersect the extracted tokens or its topic (derived from the index entry's `path`, `title`, and `description`) matches a changed object type. Read an article's full file — its `## Best Practice` / `## Anti Pattern` bodies — only after it makes the worklist; candidate selection uses the index alone. When the diff contains no upgrade-related changes by any of the above signals, return `outcome: "not-applicable"` without evaluating files.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "bcquality",
|
||||
"description": "Quality skills and knowledge for Business Central development. Exposes a standalone AL review adapter backed by BCQuality's Entry protocol.",
|
||||
"version": "0.2.0",
|
||||
"description": "Quality skills and knowledge for Business Central. Exposes read-only AL plan enrichment and code-review adapters backed by BCQuality's Entry protocol.",
|
||||
"version": "0.3.0",
|
||||
"author": {
|
||||
"name": "microsoft/BCQuality",
|
||||
"url": "https://github.com/microsoft/BCQuality"
|
||||
|
|
@ -13,6 +13,7 @@
|
|||
"al",
|
||||
"business-central",
|
||||
"code-review",
|
||||
"plan-guidance",
|
||||
"quality"
|
||||
],
|
||||
"skills": [
|
||||
|
|
|
|||
|
|
@ -48,7 +48,9 @@
|
|||
"type": "array",
|
||||
"minItems": 1,
|
||||
"maxItems": 1,
|
||||
"items": { "const": "findings-report" }
|
||||
"items": {
|
||||
"enum": ["findings-report", "development-guidance-report"]
|
||||
}
|
||||
},
|
||||
"filters": {
|
||||
"type": "object",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# BCQuality global skills
|
||||
|
||||
This folder contains BCQuality's layer-independent protocol files and the
|
||||
host-native adapter used by standalone plugin installations.
|
||||
host-native adapters used by standalone plugin installations.
|
||||
|
||||
The protocol files have two kinds:
|
||||
|
||||
|
|
@ -31,8 +31,9 @@ READ and DO are read on demand — typically by the first action skill the agent
|
|||
| Path | Role |
|
||||
|---|---|
|
||||
| [`al-code-review/SKILL.md`](al-code-review/SKILL.md) | Exposes BCQuality through the standard `SKILL.md` format when this repository is installed as a plugin. |
|
||||
| [`al-development-plan/SKILL.md`](al-development-plan/SKILL.md) | Enriches an existing AL plan read-only through the standard `SKILL.md` format; does not generate a plan or implement code. |
|
||||
|
||||
The adapter is deliberately thin. It translates the caller's request into an
|
||||
Each adapter is deliberately thin. It translates the caller's request into an
|
||||
Entry task context, then follows Entry's dispatch without owning routing,
|
||||
review, index, or output policy. It is not an action skill, is not considered
|
||||
by Entry, and should not accumulate behavior already defined by `entry.md`,
|
||||
|
|
@ -40,18 +41,23 @@ by Entry, and should not accumulate behavior already defined by `entry.md`,
|
|||
|
||||
This gives the two skill formats distinct roles:
|
||||
|
||||
- `skills/al-code-review/SKILL.md` is the public host integration surface for a
|
||||
standalone plugin installation.
|
||||
- `skills/al-code-review/SKILL.md` and
|
||||
`skills/al-development-plan/SKILL.md` are the public host integration
|
||||
surfaces for a standalone plugin installation.
|
||||
- `microsoft/skills/review/al-code-review.md` is BCQuality's internal
|
||||
Microsoft-layer super-skill for coordinating a broad AL review.
|
||||
- `microsoft/skills/development/al-development-plan.md` is the read-only
|
||||
knowledge-enrichment interface for existing plans. Consumers own format
|
||||
normalization, planning, implementation, and delivery.
|
||||
|
||||
The host adapter and internal coordinator deliberately share the
|
||||
`al-code-review` name because they represent the same user-facing operation in
|
||||
their respective formats. Their locations distinguish their roles. The
|
||||
reference from the adapter to Entry, and from a dispatched super-skill to its
|
||||
leaf skills, is intentional progressive disclosure. It avoids registering
|
||||
every internal BCQuality protocol file as an ambient host skill while allowing
|
||||
each review domain to run in an isolated context.
|
||||
Each host adapter deliberately shares its name with the internal action skill
|
||||
for the same operation. Their locations distinguish the host integration from
|
||||
the layered policy. `al-code-review` remains distinct from BC-ALAgents'
|
||||
separately installed `al-review` skill, avoiding a collision in hosts that use
|
||||
one shared skill inventory. References from adapters to Entry, and from a
|
||||
dispatched super-skill to its leaves, are intentional progressive disclosure.
|
||||
This avoids registering every internal protocol file as an ambient host skill
|
||||
while allowing each review domain to run in an isolated context.
|
||||
|
||||
These contracts are stable. Changes require a PR approved by both maintainers.
|
||||
|
||||
|
|
|
|||
22
skills/al-development-plan/SKILL.md
Normal file
22
skills/al-development-plan/SKILL.md
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
name: al-development-plan
|
||||
description: Enrich an existing Business Central AL development plan with read-only BCQuality knowledge constraints. Does not generate a plan or implement code.
|
||||
---
|
||||
|
||||
# AL development plan guidance
|
||||
|
||||
This host-native adapter translates an existing plan and repository into Entry's task context. It does not plan new work, edit the target repository, run an implementation or review/fix loop, stage, commit, or publish changes.
|
||||
|
||||
## Execute
|
||||
|
||||
1. Resolve `PLUGIN_ROOT` to the directory containing this plugin's root `plugin.json`, two levels above this file.
|
||||
2. Preserve the caller's existing `development-plan` verbatim. Consumer-specific workflow payloads must be normalized by the consumer; do not interpret workflow state or manufacture a plan from a coding request.
|
||||
3. Build the task context for `PLUGIN_ROOT/skills/entry.md`:
|
||||
- Set `goal` to read-only BCQuality knowledge enrichment of the supplied plan, preserving the caller's intended change.
|
||||
- List only actually supplied inputs from `[development-plan, repository]` in `inputs-available`.
|
||||
- Set `technologies: [al]` only when established, and pass other applicability dimensions only when supplied or reliably determined.
|
||||
- Apply `BCQUALITY_ENABLED_LAYERS` and `BCQUALITY_DISABLED_SKILLS` as described in the `al-code-review` adapter.
|
||||
4. Read and execute Entry, including Preparation. Resolve its paths against `PLUGIN_ROOT`, not the target repository. Keep all generated index and scratch artifacts outside the target repository. Use READ's path-based fallback if index generation is unavailable; an unreadable corpus is a failure, not empty knowledge.
|
||||
5. Follow Entry's dispatch, checking its output metadata against the referenced skill before invocation. This operation accepts only `development-guidance-report`; return `failed` rather than execute another output kind. Pass the supplied existing plan and readable repository, and return the report unchanged. Return Entry's `no-match` or `failed` record unchanged when nothing is dispatched.
|
||||
|
||||
Missing inputs remain missing; the dispatched action skill returns `not-applicable` when it cannot proceed. A `no-knowledge` report is additive: it means no additional BCQuality constraints, not a refusal to let the consumer implement under its own gates. The consumer retains all implementation and delivery ownership.
|
||||
100
skills/do.md
100
skills/do.md
|
|
@ -7,7 +7,7 @@ title: Action Skill — the template every action skill follows
|
|||
|
||||
# DO
|
||||
|
||||
An action skill is a markdown file that tells an agent how to do one concrete job — review a pull request, audit telemetry usage, generate a skeleton — using knowledge files from BCQuality. This document is the template every action skill follows. Orchestrators rely on the template to consume any skill without skill-specific parsing.
|
||||
An action skill is a markdown file that tells an agent how to do one concrete job — review a pull request, audit telemetry usage, enrich an existing plan — using knowledge files from BCQuality. This document is the template every action skill follows. Orchestrators rely on the template to consume any skill without skill-specific parsing.
|
||||
|
||||
This contract is stable. Changes require a PR approved by both maintainers.
|
||||
|
||||
|
|
@ -23,7 +23,7 @@ Action skills do not live at the repo root. Layer-independent files in
|
|||
`/skills/` contain the three meta-skill contracts (READ, DO, WRITE), the
|
||||
entry-point skill (`entry.md`, `kind: entry-point`), and host-format adapters.
|
||||
Adapters are not action skills. Entry structurally follows the same
|
||||
four-step pattern but produces a dispatch record rather than a findings-report;
|
||||
four-step pattern but produces a dispatch record rather than an action-skill report;
|
||||
see [entry.md](entry.md) for its contract.
|
||||
|
||||
## Skills hold mechanics; knowledge files hold BC facts
|
||||
|
|
@ -62,12 +62,16 @@ application-area: [all]
|
|||
`bc-version`, `technologies`, `countries`, `application-area` are optional filters that let an orchestrator pre-select applicable skills for a task. They follow the same semantics as in READ.
|
||||
|
||||
`inputs` is a list of abstract input types the skill **accepts**. Standard values:
|
||||
`pr-diff`, `object-list`, `file-path`, `folder-path`, `repository`, and
|
||||
`telemetry-query`. Semantics are any-of: the orchestrator supplies whichever
|
||||
listed input types it has, and the skill is invoked with a non-empty subset of
|
||||
its declared `inputs`. A skill that cannot proceed with the supplied subset
|
||||
MUST return `outcome: "not-applicable"`. `outputs` is always a single-element
|
||||
list naming the output kind; today only `findings-report` is defined.
|
||||
`pr-diff`, `object-list`, `file-path`, `folder-path`, `repository`,
|
||||
`telemetry-query`, and `development-plan`. Semantics are any-of: the
|
||||
orchestrator supplies whichever listed input types it has, and the skill is
|
||||
invoked with a non-empty subset of its declared `inputs`. A skill that cannot
|
||||
proceed with the supplied subset MUST return `outcome: "not-applicable"`.
|
||||
|
||||
`outputs` is always a single-element list naming the output kind:
|
||||
|
||||
- `findings-report` — evaluates an input and reports defects or observations.
|
||||
- `development-guidance-report` — selects and summarizes applicable BCQuality knowledge for an existing development plan without changing the target repository.
|
||||
|
||||
`file-path` is one file. `folder-path` is a directory whose recursively
|
||||
contained files form the complete current-state input, such as a Business
|
||||
|
|
@ -98,13 +102,15 @@ Every action skill MUST contain these five sections, in order:
|
|||
|
||||
**Relevance.** Apply frontmatter filters to the candidates. Typical filters: match `bc-version` against the target environment, match `technologies` against the languages in scope, match `countries` and `application-area` against the consuming codebase's context. The exact matching rules are defined in READ (*Frontmatter matching semantics*). Files that do not match are discarded.
|
||||
|
||||
**Worklist.** Narrow the relevant candidates to the subset that applies to the current task. This is where the task-specific signal enters: the objects changed in the PR, the queries being audited, the skeleton being generated. Typical moves: match `keywords` against task vocabulary, match file topics against changed objects, deduplicate by concern.
|
||||
**Worklist.** Narrow the relevant candidates to the subset that applies to the current task. This is where the task-specific signal enters: the objects changed in the PR, the queries being audited, the existing plan being enriched. Typical moves: match `keywords` against task vocabulary, match file topics against changed objects, deduplicate by concern.
|
||||
|
||||
**Action.** Execute the skill's work against the worklist. Evaluate each item in the worklist against the task input and emit findings. The action step is where skill behavior differs; the preceding three steps are uniform.
|
||||
|
||||
## Output contract
|
||||
<a id="output-contract"></a>
|
||||
|
||||
Every action skill emits a single JSON document that conforms to this schema:
|
||||
## Findings-report contract
|
||||
|
||||
An action skill with `outputs: [findings-report]` emits a single JSON document that conforms to this schema:
|
||||
|
||||
The machine-readable structural schema is
|
||||
[`schemas/findings-report.schema.json`](../schemas/findings-report.schema.json).
|
||||
|
|
@ -335,6 +341,76 @@ Severity taxonomy:
|
|||
- `minor` — quality concern; worth flagging but not a gate.
|
||||
- `info` — observation or context; not actionable on its own.
|
||||
|
||||
## Development-guidance-report contract
|
||||
|
||||
An action skill with `outputs: [development-guidance-report]` emits one JSON document:
|
||||
|
||||
```json
|
||||
{
|
||||
"skill": { "id": "string", "version": 1 },
|
||||
"outcome": "completed | not-applicable | no-knowledge | partial | failed",
|
||||
"outcome-reason": "string",
|
||||
"summary": {
|
||||
"request": "string",
|
||||
"kind": "feature | bug | refactor | upgrade | maintenance",
|
||||
"candidates": 0,
|
||||
"selected": 0
|
||||
},
|
||||
"context": {
|
||||
"bc-version": "string",
|
||||
"technologies": ["string"],
|
||||
"countries": ["string"],
|
||||
"application-area": ["string"],
|
||||
"unknown": ["bc-version | technologies | countries | application-area"]
|
||||
},
|
||||
"knowledge": [
|
||||
{
|
||||
"path": "string",
|
||||
"sha": "string",
|
||||
"used-for": "string",
|
||||
"constraints": ["string"],
|
||||
"sample-paths": ["string"]
|
||||
}
|
||||
],
|
||||
"validation-considerations": [
|
||||
{
|
||||
"id": "string",
|
||||
"reason": "string",
|
||||
"evidence": "string"
|
||||
}
|
||||
],
|
||||
"suppressed": [
|
||||
{
|
||||
"reference": { "path": "string", "sha": "string" },
|
||||
"reason": "layer-precedence | configuration"
|
||||
}
|
||||
],
|
||||
"unresolved": ["string"]
|
||||
}
|
||||
```
|
||||
|
||||
The skill is read-only with respect to the target repository: no edits, generated files, staging, commits, or publication. Keep index, report, and scratch artifacts outside that repository. The report is strict JSON with no surrounding commentary. The caller supplies an existing plan and repository; consumer-specific input normalization and workflow state are outside this contract.
|
||||
|
||||
### Guidance outcome semantics
|
||||
|
||||
- `completed` — evaluation finished, at least one article was selected, every selected article was opened and faithfully converted into constraints, and no materially unresolved conditional guidance remains.
|
||||
- `not-applicable` — the required existing plan or readable repository is absent, or the task is outside the skill's applicability. No constraints are claimed.
|
||||
- `no-knowledge` — evaluation finished and there are **no additional applicable BCQuality constraints** for this plan. `knowledge` is empty. This is not a statement that the work is unsafe or unimplementable; the consuming workflow can proceed under its ordinary gates. Do not add generic or filler articles to avoid this outcome.
|
||||
- `partial` — evaluation is incomplete or conditional guidance remains materially unresolved. Name each gap in `outcome-reason` and `unresolved`; do not silently treat an unknown dimension as a match.
|
||||
- `failed` — retrieval, reference integrity, or another error prevents a reliable report. Set `outcome-reason`; consumers must not treat the result as reliable constraints or as `no-knowledge`.
|
||||
|
||||
`outcome-reason` is required for `partial` and `failed`, optional otherwise. These outcomes describe enrichment only, not permission to implement or deliver. The consumer owns handling of partial, failed, and unresolved guidance, including escalation, clarification, and re-enrichment; BCQuality does not impose a universal implementation gate.
|
||||
|
||||
### Guidance field semantics
|
||||
|
||||
`summary.request` preserves the planned intent and `kind` classifies it without replacing the plan. `candidates` and `selected` are non-negative integer counts: selected equals the number of unique `knowledge` entries and cannot exceed candidates. Counts are retrieval diagnostics, not capability or authoring-quality scores.
|
||||
|
||||
`knowledge[].constraints` is a non-empty list summarizing only normative `## Best Practice` and `## Anti Pattern` content from the referenced article. It must not introduce a Business Central fact absent from that article. `used-for` names the concrete plan decision. `sample-paths` contains only sibling samples that exist and were opened. All paths use forward slashes, are repository-relative, and must resolve inside the recorded BCQuality checkout; absolute paths, traversal, and links escaping that checkout are invalid. Every reference is subject to the reference-integrity gate.
|
||||
|
||||
`validation-considerations` states evidence the implementation workflow should obtain; it does not claim that a command or test has run. `suppressed` has the same shape and semantics as in a findings-report. `unresolved` records missing repository context or plan decisions that prevent a reliable constraint. Unknown applicability dimensions must appear in both `context.unknown` and a relevant unresolved entry, explaining whether they materially affect a candidate. An unknown dimension is not itself a failure or proof that relevant knowledge exists.
|
||||
|
||||
Reference SHAs, when present, identify the files read; they do not prove runtime pinning on their own. The consumer records and verifies the actual immutable BCQuality checkout used for both enrichment and final review, plus its filtering policy and run provenance outside the target repository. See [agent-consumption.md](../agent-consumption.md).
|
||||
|
||||
## Composition (super-skills)
|
||||
|
||||
A **super-skill** is an action skill whose frontmatter declares a non-empty `sub-skills: [...]`. A super-skill does not evaluate knowledge files directly; it invokes other action skills and composes their output.
|
||||
|
|
@ -441,4 +517,4 @@ Conforms to the DO output contract.
|
|||
|
||||
## How orchestrators consume output
|
||||
|
||||
An orchestrator invokes an action skill with an input appropriate to the skill's declared `inputs`, receives the JSON output, and maps findings to its delivery surface (PR comments, build gates, IDE diagnostics). The orchestrator MUST NOT interpret skill-specific fields beyond the schema above. Skills that need richer semantics MUST encode them within the schema (for example, by adding structured `message` text) rather than extending the output shape.
|
||||
An orchestrator invokes an action skill with an input appropriate to the skill's declared `inputs` and uses the single output kind declared in frontmatter. It maps a `findings-report` to PR comments, build gates, or IDE diagnostics, and a `development-guidance-report` to additional constraints for its existing implementation workflow. These are the two output schemas defined by this contract; the consumer retains ownership of implementation and delivery.
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ The agent invokes Entry with a **task context** supplied by the orchestrator:
|
|||
task-context:
|
||||
goal: string # free-text description of what needs doing
|
||||
inputs-available: # values the orchestrator has ready to pass to a chosen skill
|
||||
- development-plan
|
||||
- repository
|
||||
- pr-diff
|
||||
- file-path
|
||||
- folder-path
|
||||
|
|
@ -36,7 +38,7 @@ task-context:
|
|||
|
||||
## Preparation — knowledge index
|
||||
|
||||
Before routing, ensure the knowledge index is current for the **live** clone. The dispatched review skills read `knowledge-index.json` (at the clone root) at their Source step instead of opening every knowledge file — see READ's [Retrieval workflow](read.md). When a consumer prunes its clone to policy *before* the agent runs, the index MUST be built over the clone as it exists now, so it lists exactly the articles that survived pruning and never an article the consumer denied:
|
||||
Before routing, ensure the knowledge index is current for the **live** clone. The dispatched skills read `knowledge-index.json` (by default at the clone root) at their Source step instead of opening every knowledge file — see READ's [Retrieval workflow](read.md). When a consumer prunes its clone to policy *before* the agent runs, the index MUST be built over the clone as it exists now, so it lists exactly the articles that survived pruning and never an article the consumer denied:
|
||||
|
||||
- If `knowledge-index.json` is absent — or you cannot confirm it reflects the current knowledge tree — regenerate it by running, from the checkout root:
|
||||
|
||||
|
|
@ -46,6 +48,7 @@ Before routing, ensure the knowledge index is current for the **live** clone. Th
|
|||
|
||||
It defaults to indexing this checkout and writes `knowledge-index.json` at the root in well under a second. When in doubt, rebuild: a sub-second rebuild is always cheaper than a stale or over-listing index, which is a correctness risk.
|
||||
- The paths above assume the checkout root is the current directory. A caller that enters Entry from elsewhere — a plugin host, whose working directory is the user's own project — MUST resolve them against the BCQuality root it already knows instead. The generator resolves its own root, so invoking it by absolute path indexes and writes the right tree.
|
||||
- For read-only plan enrichment, generated artifacts MUST remain outside the target repository. When the target contains the BCQuality checkout, or that checkout is immutable, pass the generator's `-IndexPath` to an external runner-owned artifact location and supply that resolved index path to the dispatched skill. Do not regenerate inside the target or modify the immutable checkout. If generation is unavailable, use READ's path-based discovery; retrieval failure is not an empty corpus.
|
||||
- Pruning is the consumer's job, not Entry's, and not every consumer does it: an installation that ships the whole tree gets no deny guarantee from this step. There, `enabled-layers` narrows discovery only, and the unlisted layers' files remain on disk.
|
||||
- This is a side step. It MUST NOT change Entry's output — the dispatch record below is the only thing Entry emits, and build logs are never part of the dispatch JSON.
|
||||
|
||||
|
|
@ -97,7 +100,8 @@ Emit a single JSON document conforming to the output contract below. Entry does
|
|||
"path": "microsoft/skills/review/al-code-review.md"
|
||||
},
|
||||
"rationale": "string",
|
||||
"inputs": ["pr-diff"]
|
||||
"inputs": ["pr-diff"],
|
||||
"outputs": ["findings-report"]
|
||||
}
|
||||
],
|
||||
"skipped": [
|
||||
|
|
@ -120,10 +124,11 @@ Emit a single JSON document conforming to the output contract below. Entry does
|
|||
|
||||
**`dispatch[]`** — each entry names one action skill to invoke.
|
||||
|
||||
- `skill.path` — repo-relative, forward slashes. The agent fetches and executes the file directly from this path.
|
||||
- `skill.path` — repo-relative, forward slashes, copied from discovery. Resolve it inside the live BCQuality checkout; reject absolute paths, traversal, and links escaping that checkout. The agent reads the action skill at this exact path rather than constructing a plausible filename.
|
||||
- `skill.version` — copied from the dispatched skill's frontmatter so the orchestrator can detect drift between dispatch time and execution.
|
||||
- `rationale` — short human-readable string, for logs and traceability.
|
||||
- `inputs` — the intersection of `task-context.inputs-available` and the skill's declared `inputs`. The agent MUST pass exactly this subset when invoking the skill. Sending a strict intersection avoids accidental information leakage between skills.
|
||||
- `outputs` — the dispatched skill's complete, single-element `outputs` value copied from frontmatter. This lets an orchestrator distinguish `findings-report` from read-only `development-guidance-report` before invocation. Check the declared contract and actual skill; output metadata is not a sandbox or proof of side effects. Unknown output kinds must not be silently treated as a supported report.
|
||||
|
||||
Ordering of `dispatch[]` is not significant.
|
||||
|
||||
|
|
@ -161,7 +166,8 @@ Populated example (PR review on a repo where only `al-performance-review` is ena
|
|||
{
|
||||
"skill": { "id": "al-performance-review", "version": 1, "path": "microsoft/skills/review/al-performance-review.md" },
|
||||
"rationale": "Goal 'review pull request' matched; inputs-available contains pr-diff.",
|
||||
"inputs": ["pr-diff"]
|
||||
"inputs": ["pr-diff"],
|
||||
"outputs": ["findings-report"]
|
||||
}
|
||||
],
|
||||
"skipped": [
|
||||
|
|
@ -175,7 +181,7 @@ Populated example (PR review on a repo where only `al-performance-review` is ena
|
|||
|
||||
1. Invoke Entry with the orchestrator-supplied task context.
|
||||
2. Receive the dispatch record.
|
||||
3. For each entry in `dispatch[]`, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce a findings-report.
|
||||
4. Return the findings-reports to the orchestrator. When `outcome` is `no-match` or `failed`, return the dispatch record itself so the orchestrator can log the reason.
|
||||
3. For each entry in `dispatch[]`, inspect `outputs` before invocation, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce the declared report kind. Verify the file's frontmatter output still equals the dispatch value; return `failed` on drift rather than executing a different contract.
|
||||
4. Return the action-skill reports to the orchestrator. When Entry's `outcome` is `no-match` or `failed`, return the dispatch record itself so the orchestrator can log the reason.
|
||||
|
||||
READ and DO are the contracts that govern what the dispatched skills do. An agent that has not yet read READ and DO reads them when it executes the first dispatched skill — they are not prerequisites for invoking Entry.
|
||||
|
|
|
|||
499
tools/DevelopmentGuidance.Evidence.ps1
Normal file
499
tools/DevelopmentGuidance.Evidence.ps1
Normal file
|
|
@ -0,0 +1,499 @@
|
|||
# Helpers for Test-DevelopmentGuidanceFixtures.ps1 and its deterministic regressions.
|
||||
function Get-GuidanceDiagnostic {
|
||||
param($ErrorRecord)
|
||||
if ($ErrorRecord.Exception -is [Management.Automation.RuntimeException] -and
|
||||
$ErrorRecord.FullyQualifiedErrorId -eq $ErrorRecord.Exception.Message) {
|
||||
return $ErrorRecord.Exception.Message
|
||||
}
|
||||
return 'Evidence or report could not be read safely (missing, malformed, inaccessible or unsupported state).'
|
||||
}
|
||||
|
||||
function Assert-GuidanceObject {
|
||||
param($Value, [string] $Label, [string[]] $Fields = @())
|
||||
if ($Value -isnot [Collections.IDictionary]) { throw "$Label must be a JSON object." }
|
||||
foreach ($field in $Fields) {
|
||||
if (-not $Value.Contains($field)) { throw "$Label is missing required field '$field'." }
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-GuidanceString {
|
||||
param($Value, [string] $Label)
|
||||
if ($Value -isnot [string] -or [string]::IsNullOrWhiteSpace($Value)) { throw "$Label must be a non-empty string." }
|
||||
}
|
||||
|
||||
function Assert-GuidanceArray {
|
||||
param($Value, [string] $Label, [switch] $Strings, [switch] $NonEmpty)
|
||||
if ($Value -isnot [array]) { throw "$Label must be a JSON array." }
|
||||
if ($NonEmpty -and $Value.Count -eq 0) { throw "$Label must not be empty." }
|
||||
if ($Strings) { foreach ($item in $Value) { Assert-GuidanceString $item "$Label entry" } }
|
||||
}
|
||||
|
||||
function Assert-GuidanceInteger {
|
||||
param($Value, [string] $Label)
|
||||
if (($Value -isnot [long] -and $Value -isnot [int] -and $Value -isnot [bigint]) -or $Value -lt 0) {
|
||||
throw "$Label must be a non-negative integer."
|
||||
}
|
||||
}
|
||||
|
||||
function Read-GuidanceJson {
|
||||
param([string] $Path)
|
||||
function Assert-JsonMembers($Element) {
|
||||
if ($Element.ValueKind -eq [Text.Json.JsonValueKind]::Object) {
|
||||
$names = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($property in $Element.EnumerateObject()) {
|
||||
if (-not $names.Add($property.Name)) { throw 'JSON contains duplicate or case-ambiguous members.' }
|
||||
Assert-JsonMembers $property.Value
|
||||
}
|
||||
} elseif ($Element.ValueKind -eq [Text.Json.JsonValueKind]::Array) {
|
||||
foreach ($item in $Element.EnumerateArray()) { Assert-JsonMembers $item }
|
||||
}
|
||||
}
|
||||
try {
|
||||
if ((Get-Item -LiteralPath $Path -Force).Length -gt 32MB) { throw 'Oversized JSON.' }
|
||||
$text = [IO.File]::ReadAllText($Path)
|
||||
$document = [Text.Json.JsonDocument]::Parse($text)
|
||||
try { Assert-JsonMembers $document.RootElement } finally { $document.Dispose() }
|
||||
return ConvertFrom-Json -InputObject $text -AsHashtable -Depth 64 -NoEnumerate
|
||||
} catch { throw 'Input is not readable, strict JSON with unique members.' }
|
||||
}
|
||||
|
||||
function Get-GuidanceHash {
|
||||
param([string] $Path)
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash
|
||||
}
|
||||
|
||||
function Get-GuidanceCaseId {
|
||||
param([string] $Id)
|
||||
$bytes = [Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($Id))
|
||||
return "case-$([Convert]::ToHexString($bytes).Substring(0, 8).ToLowerInvariant())"
|
||||
}
|
||||
|
||||
function Test-GuidanceWithin {
|
||||
param([string] $Path, [string] $Parent)
|
||||
$comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal }
|
||||
$prefix = $Parent.TrimEnd([IO.Path]::DirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar
|
||||
return $Path.Equals($Parent, $comparison) -or $Path.StartsWith($prefix, $comparison)
|
||||
}
|
||||
|
||||
function Assert-GuidanceItem {
|
||||
param($Item)
|
||||
if (($Item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $Item.LinkType -or $Item.LinkTarget) {
|
||||
throw 'Links, junctions, hard links and reparse points are not supported.'
|
||||
}
|
||||
if (-not $Item.PSIsContainer -and $IsWindows) {
|
||||
$unsupportedStreams = @(
|
||||
Get-Item -LiteralPath $Item.FullName -Stream '*' -Force -ErrorAction Stop |
|
||||
Where-Object Stream -notin @(':$DATA', 'sec.endpointdlp')
|
||||
)
|
||||
if ($unsupportedStreams.Count) {
|
||||
throw 'Alternate data streams are not supported.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-GuidanceSafePath {
|
||||
param([string] $Path, [switch] $AllowMissing, [switch] $Directory, [switch] $File)
|
||||
Assert-GuidanceString $Path 'Filesystem path'
|
||||
$full = [IO.Path]::GetFullPath($Path)
|
||||
$driveRoot = [IO.Path]::GetPathRoot($full)
|
||||
if ($IsWindows -and ($driveRoot.StartsWith('\\') -or $full.Substring($driveRoot.Length).Contains(':'))) {
|
||||
throw 'Network paths and alternate stream paths are not supported.'
|
||||
}
|
||||
$cursor = $driveRoot
|
||||
$components = $full.Substring($driveRoot.Length).Split([IO.Path]::DirectorySeparatorChar, [StringSplitOptions]::RemoveEmptyEntries)
|
||||
foreach ($component in $components) {
|
||||
if ($component -match '[\. ]$') { throw 'Ambiguous filesystem path components are not supported.' }
|
||||
$cursor = Join-Path $cursor $component
|
||||
# Get-Item sees dangling links that Test-Path may treat as missing.
|
||||
$item = Get-Item -LiteralPath $cursor -Force -ErrorAction SilentlyContinue
|
||||
if ($null -ne $item) { Assert-GuidanceItem $item }
|
||||
elseif (-not $AllowMissing) { throw 'Required filesystem path is missing.' }
|
||||
}
|
||||
if ($Directory -and -not (Test-Path -LiteralPath $full -PathType Container)) { throw 'Required directory is missing.' }
|
||||
if ($File -and -not (Test-Path -LiteralPath $full -PathType Leaf)) { throw 'Required file is missing.' }
|
||||
return $full.TrimEnd([IO.Path]::DirectorySeparatorChar)
|
||||
}
|
||||
|
||||
function Assert-GuidanceTree {
|
||||
param([string] $Root)
|
||||
$pending = [Collections.Generic.Stack[string]]::new()
|
||||
$pending.Push($Root)
|
||||
while ($pending.Count) {
|
||||
foreach ($item in Get-ChildItem -LiteralPath $pending.Pop() -Force) {
|
||||
Assert-GuidanceItem $item
|
||||
if ($item.PSIsContainer) { $pending.Push($item.FullName) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-GuidanceGit {
|
||||
param([string] $Root, [string[]] $Arguments, [switch] $RawOutput)
|
||||
$start = [Diagnostics.ProcessStartInfo]::new('git')
|
||||
$start.UseShellExecute = $false
|
||||
$start.RedirectStandardOutput = $true
|
||||
$start.RedirectStandardError = $true
|
||||
foreach ($variable in @('GIT_DIR', 'GIT_WORK_TREE', 'GIT_COMMON_DIR', 'GIT_INDEX_FILE',
|
||||
'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_CONFIG',
|
||||
'GIT_CONFIG_COUNT', 'GIT_CONFIG_PARAMETERS', 'GIT_NAMESPACE')) {
|
||||
$null = $start.Environment.Remove($variable)
|
||||
}
|
||||
$start.Environment['GIT_CONFIG_NOSYSTEM'] = '1'
|
||||
$start.Environment['GIT_CONFIG_GLOBAL'] = ''
|
||||
$start.Environment['GIT_TERMINAL_PROMPT'] = '0'
|
||||
# In particular, do not execute a repository-supplied fsmonitor hook on reads.
|
||||
foreach ($argument in (@('--no-optional-locks', '-c', 'core.fsmonitor=false', '-C', $Root) + $Arguments)) {
|
||||
$start.ArgumentList.Add($argument)
|
||||
}
|
||||
$process = [Diagnostics.Process]::Start($start)
|
||||
try {
|
||||
$output = $process.StandardOutput.ReadToEndAsync()
|
||||
$errors = $process.StandardError.ReadToEndAsync()
|
||||
$process.WaitForExit()
|
||||
$null = $errors.GetAwaiter().GetResult()
|
||||
if ($process.ExitCode -ne 0) { throw 'Git evidence could not be read.' }
|
||||
$text = $output.GetAwaiter().GetResult()
|
||||
if ($RawOutput) { return $text }
|
||||
return $text.TrimEnd("`r", "`n")
|
||||
} finally { $process.Dispose() }
|
||||
}
|
||||
|
||||
function Get-GuidanceSnapshot {
|
||||
param([string] $Root, [switch] $Target)
|
||||
$Root = Get-GuidanceSafePath $Root -Directory
|
||||
$dotGit = Get-GuidanceSafePath (Join-Path $Root '.git')
|
||||
if (Test-Path -LiteralPath $dotGit -PathType Container) {
|
||||
$gitDir = $dotGit
|
||||
} else {
|
||||
if ($Target) { throw 'Target workspaces must be standalone repositories with internal Git storage.' }
|
||||
$pointer = [IO.File]::ReadAllText($dotGit)
|
||||
if ($pointer -notmatch '\Agitdir: ([^\r\n]+)\r?\n?\z') { throw 'Unsupported Git worktree pointer.' }
|
||||
$gitPath = $Matches[1]
|
||||
if (-not [IO.Path]::IsPathFullyQualified($gitPath)) { $gitPath = Join-Path $Root $gitPath }
|
||||
$gitDir = Get-GuidanceSafePath $gitPath -Directory
|
||||
}
|
||||
$commonDir = $gitDir
|
||||
$commonPointer = Get-GuidanceSafePath (Join-Path $gitDir 'commondir') -AllowMissing
|
||||
if (Test-Path -LiteralPath $commonPointer) {
|
||||
$commonPath = [IO.File]::ReadAllText($commonPointer).Trim()
|
||||
if (-not [IO.Path]::IsPathFullyQualified($commonPath)) { $commonPath = Join-Path $gitDir $commonPath }
|
||||
$commonDir = Get-GuidanceSafePath $commonPath -Directory
|
||||
}
|
||||
if ($Target -and ($gitDir -cne (Join-Path $Root '.git') -or $commonDir -cne $gitDir)) {
|
||||
throw 'Target workspaces must be standalone repositories with internal Git storage.'
|
||||
}
|
||||
$files = [Collections.Generic.List[object]]::new()
|
||||
$pending = [Collections.Generic.Stack[string]]::new()
|
||||
$pending.Push($Root)
|
||||
while ($pending.Count) {
|
||||
foreach ($item in @(Get-ChildItem -LiteralPath $pending.Pop() -Force | Sort-Object Name -CaseSensitive)) {
|
||||
Assert-GuidanceItem $item
|
||||
$relative = [IO.Path]::GetRelativePath($Root, $item.FullName).Replace('\', '/')
|
||||
$entry = [ordered]@{
|
||||
path = $relative
|
||||
kind = if ($item.PSIsContainer) { 'directory' } else { 'file' }
|
||||
attributes = [int]$item.Attributes
|
||||
unixMode = [int]$item.UnixFileMode
|
||||
creationUtcTicks = $item.CreationTimeUtc.Ticks
|
||||
}
|
||||
if ($item.PSIsContainer) {
|
||||
$pending.Push($item.FullName)
|
||||
} else {
|
||||
$entry.length = $item.Length
|
||||
$entry.lastWriteUtcTicks = $item.LastWriteTimeUtc.Ticks
|
||||
$entry.sha256 = Get-GuidanceHash $item.FullName
|
||||
}
|
||||
$files.Add($entry)
|
||||
}
|
||||
}
|
||||
# Linked knowledge worktrees have explicitly identified Git metadata outside
|
||||
# the content root. Record its meaningful configuration as well as HEAD/refs.
|
||||
$gitMetadata = [ordered]@{}
|
||||
foreach ($storage in @($gitDir, $commonDir) | Sort-Object -Unique) {
|
||||
foreach ($name in @('HEAD', 'commondir', 'config', 'config.worktree', 'packed-refs', 'refs', 'objects', 'index', 'info\exclude', 'shallow')) {
|
||||
$path = Get-GuidanceSafePath (Join-Path $storage $name) -AllowMissing
|
||||
if ((Test-Path -LiteralPath $path -PathType Container) -and -not (Test-GuidanceWithin $path $Root)) {
|
||||
Assert-GuidanceTree $path
|
||||
}
|
||||
if (Test-Path -LiteralPath $path -PathType Leaf) {
|
||||
$gitMetadata[$path] = Get-GuidanceHash $path
|
||||
}
|
||||
if ($name -in @('config', 'config.worktree') -and (Test-Path -LiteralPath $path) -and
|
||||
[IO.File]::ReadAllText($path) -match '(?im)^\s*\[include(?:If)?\b') {
|
||||
throw 'External Git configuration includes are not supported.'
|
||||
}
|
||||
}
|
||||
foreach ($name in @('objects\info\alternates', 'objects\info\http-alternates')) {
|
||||
if (Test-Path -LiteralPath (Join-Path $storage $name)) { throw 'External Git object stores are not supported.' }
|
||||
}
|
||||
}
|
||||
$top = Get-GuidanceSafePath (Invoke-GuidanceGit $Root @('rev-parse', '--show-toplevel')) -Directory
|
||||
if ($top -cne $Root) { throw 'Evidence requires the exact Git worktree root, not a subdirectory.' }
|
||||
$indexPath = Get-GuidanceSafePath (Join-Path $gitDir 'index') -AllowMissing
|
||||
$tracked = Invoke-GuidanceGit $Root @('ls-files', '--stage')
|
||||
if ($tracked -match '(?m)^160000 ') { throw 'Submodule workspaces are not supported.' }
|
||||
if ((Invoke-GuidanceGit $Root @('ls-files', '-t')) -match '(?m)^S ') { throw 'Sparse workspaces are not supported.' }
|
||||
$head = Invoke-GuidanceGit $Root @('rev-parse', '--verify', 'HEAD')
|
||||
$headFile = Get-GuidanceSafePath (Join-Path $gitDir 'HEAD') -File
|
||||
# Access times, Git status refreshes, and index-builder generatedAt are not evidence.
|
||||
return [ordered]@{
|
||||
version = 1
|
||||
root = $Root
|
||||
rootCreationUtcTicks = (Get-Item -LiteralPath $Root -Force).CreationTimeUtc.Ticks
|
||||
gitDir = $gitDir
|
||||
commonDir = $commonDir
|
||||
gitMetadata = $gitMetadata
|
||||
head = $head
|
||||
headFileSha256 = Get-GuidanceHash $headFile
|
||||
references = Invoke-GuidanceGit $Root @('for-each-ref', '--format=%(refname) %(objectname) %(symref)')
|
||||
indexPath = $indexPath
|
||||
indexSha256 = if (Test-Path -LiteralPath $indexPath) { Get-GuidanceHash $indexPath } else { $null }
|
||||
files = @($files | Sort-Object { $_.path } -CaseSensitive)
|
||||
}
|
||||
}
|
||||
|
||||
function Test-GuidanceSnapshotEqual {
|
||||
param($Before, $After)
|
||||
return ($Before | ConvertTo-Json -Depth 64 -Compress) -ceq ($After | ConvertTo-Json -Depth 64 -Compress)
|
||||
}
|
||||
|
||||
function Write-GuidanceNewJson {
|
||||
param([string] $Path, $Value)
|
||||
$parent = Split-Path -Parent $Path
|
||||
[IO.Directory]::CreateDirectory($parent) | Out-Null
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes(($Value | ConvertTo-Json -Depth 64))
|
||||
$stream = [IO.File]::Open($Path, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try { $stream.Write($bytes, 0, $bytes.Length) } finally { $stream.Dispose() }
|
||||
}
|
||||
|
||||
function Resolve-GuidanceReference {
|
||||
param([string] $Root, $Reference, [switch] $Knowledge, [string] $Article)
|
||||
Assert-GuidanceString $Reference 'Reference path'
|
||||
if ($Reference -cnotmatch '^[a-zA-Z0-9_-]+(?:/[a-zA-Z0-9_.-]+)+$' -or
|
||||
@($Reference.Split('/') | Where-Object { $_ -in @('.', '..') -or $_ -match '[\. ]$' }).Count) {
|
||||
throw 'Reference must be an unambiguous forward-slash repository-relative path without traversal.'
|
||||
}
|
||||
if ($Knowledge -and $Reference -cnotmatch '^(microsoft|community|custom)/knowledge/[a-z0-9-]+/(?:[a-z0-9-]+/)*[a-z0-9-]+\.md$') {
|
||||
throw 'Knowledge references must identify actual layered knowledge articles.'
|
||||
}
|
||||
if ($Article) {
|
||||
$stem = $Article.Substring(0, $Article.Length - 3)
|
||||
if ($Reference -cnotmatch ('^' + [regex]::Escape($stem) + '\.(good|bad)\.[a-zA-Z0-9]+$')) {
|
||||
throw 'Sample references must be good/bad siblings of their knowledge article.'
|
||||
}
|
||||
}
|
||||
$cursor = $Root
|
||||
foreach ($component in $Reference.Split('/')) {
|
||||
$cursor = Join-Path $cursor $component
|
||||
$cursor = Get-GuidanceSafePath $cursor
|
||||
if ((Get-Item -LiteralPath $cursor -Force).Name -cne $component) { throw 'Reference path casing must match the actual file.' }
|
||||
}
|
||||
if (-not (Test-GuidanceWithin $cursor $Root) -or -not (Test-Path -LiteralPath $cursor -PathType Leaf)) {
|
||||
throw 'Reference must resolve to an existing file inside the knowledge checkout.'
|
||||
}
|
||||
if ($Knowledge) {
|
||||
$text = [IO.File]::ReadAllText($cursor)
|
||||
if ($text -notmatch '(?s)^---\r?\n.*?\r?\n---' -or
|
||||
$text -notmatch '(?m)^domain:\s*\S+' -or $text -notmatch '(?m)^## (Best Practice|Anti Pattern)\s*$') {
|
||||
throw 'Knowledge reference does not contain a normative knowledge article.'
|
||||
}
|
||||
}
|
||||
return $cursor
|
||||
}
|
||||
|
||||
function Get-GuidancePlanRequest {
|
||||
param($Plan)
|
||||
if ($Plan -is [string]) { Assert-GuidanceString $Plan 'development-plan'; return $Plan }
|
||||
Assert-GuidanceObject $Plan 'development-plan' @('request')
|
||||
Assert-GuidanceString $Plan.request 'development-plan.request'
|
||||
return $Plan.request
|
||||
}
|
||||
|
||||
function Assert-GuidanceContext {
|
||||
param($Context)
|
||||
Assert-GuidanceObject $Context 'context' @('bc-version', 'technologies', 'countries', 'application-area', 'unknown')
|
||||
Assert-GuidanceString $Context.'bc-version' 'context.bc-version'
|
||||
foreach ($key in @('technologies', 'countries', 'application-area', 'unknown')) {
|
||||
Assert-GuidanceArray $Context[$key] "context.$key" -Strings
|
||||
if (@($Context[$key] | Sort-Object -Unique).Count -ne $Context[$key].Count) { throw "context.$key contains duplicates." }
|
||||
}
|
||||
foreach ($key in $Context.unknown) {
|
||||
if ($key -cnotin @('bc-version', 'technologies', 'countries', 'application-area')) { throw 'context.unknown contains an invalid dimension.' }
|
||||
}
|
||||
if ($Context.'bc-version' -eq 'unknown' -and 'bc-version' -cnotin $Context.unknown) {
|
||||
throw 'Unknown BC version must be recorded in context.unknown.'
|
||||
}
|
||||
foreach ($key in @('technologies', 'countries', 'application-area')) {
|
||||
if ((-not $Context[$key].Count -or 'unknown' -in $Context[$key]) -and $key -cnotin $Context.unknown) {
|
||||
throw 'Unavailable applicability dimensions must be recorded in context.unknown.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-GuidanceManifest {
|
||||
param($Manifest, [string] $Root)
|
||||
Assert-GuidanceObject $Manifest 'manifest' @('version', 'minimumKnowledgeRecall', 'minimumKnowledgePrecision', 'skill', 'cases')
|
||||
if ($Manifest.version -ne 1) { throw 'Unsupported guidance fixture manifest version.' }
|
||||
foreach ($name in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||
$value = $Manifest[$name]
|
||||
if (($value -isnot [double] -and $value -isnot [long] -and $value -isnot [int] -and $value -isnot [decimal]) -or
|
||||
$value -lt 0 -or $value -gt 1) { throw 'Manifest thresholds must be numbers between zero and one.' }
|
||||
}
|
||||
$null = Resolve-GuidanceReference $Root $Manifest.skill
|
||||
Assert-GuidanceArray $Manifest.cases 'manifest.cases' -NonEmpty
|
||||
$ids = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
$modelIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($case in $Manifest.cases) {
|
||||
Assert-GuidanceObject $case 'manifest case' @('id', 'expectedKind', 'expectedOutcome', 'development-plan', 'context', 'requiredKnowledge', 'optionalKnowledge', 'expectedUnknown', 'requiresUnresolved', 'requiresMaterialUnresolved')
|
||||
if ($case.id -isnot [string] -or $case.id -cnotmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') { throw 'Fixture id must be kebab-case.' }
|
||||
if (-not $ids.Add($case.id) -or -not $modelIds.Add((Get-GuidanceCaseId $case.id))) { throw 'Duplicate fixture or model case identity.' }
|
||||
if ($case.expectedKind -cnotin @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')) { throw 'Fixture expectedKind is invalid.' }
|
||||
if ($case.expectedOutcome -cnotin @('completed', 'not-applicable', 'no-knowledge', 'partial', 'failed')) { throw 'Fixture expectedOutcome is invalid.' }
|
||||
Assert-GuidanceString $case.expectedKind 'fixture.expectedKind'
|
||||
Assert-GuidanceString $case.expectedOutcome 'fixture.expectedOutcome'
|
||||
$null = Get-GuidancePlanRequest $case.'development-plan'
|
||||
Assert-GuidanceContext $case.context
|
||||
foreach ($name in @('requiredKnowledge', 'optionalKnowledge', 'expectedUnknown')) {
|
||||
Assert-GuidanceArray $case[$name] "fixture.$name" -Strings
|
||||
}
|
||||
if ($case.requiresUnresolved -isnot [bool]) { throw 'Fixture requiresUnresolved must be boolean.' }
|
||||
if ($case.requiresMaterialUnresolved -isnot [bool]) { throw 'Fixture requiresMaterialUnresolved must be boolean.' }
|
||||
if ($case.requiresMaterialUnresolved -and ($case.expectedOutcome -ne 'partial' -or -not $case.requiresUnresolved)) {
|
||||
throw 'Materially unresolved fixtures must expect partial and unresolved evidence.'
|
||||
}
|
||||
foreach ($key in $case.expectedUnknown) {
|
||||
if ($key -cnotin @('bc-version', 'technologies', 'countries', 'application-area')) { throw 'Fixture expectedUnknown is invalid.' }
|
||||
}
|
||||
$references = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||
if (@($references | Sort-Object -Unique).Count -ne $references.Count) { throw 'Fixture knowledge references contain duplicates.' }
|
||||
foreach ($reference in $references) { $null = Resolve-GuidanceReference $Root $reference -Knowledge }
|
||||
if ($case.expectedOutcome -in @('no-knowledge', 'not-applicable') -and $references.Count) {
|
||||
throw 'Empty-knowledge outcomes cannot require or accept knowledge.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-GuidanceResult {
|
||||
param($Result, $Case, $Manifest, [string] $Root, [string] $Workspace)
|
||||
Assert-GuidanceObject $Result 'result' @('caseId', 'guidanceReport')
|
||||
Assert-GuidanceString $Result.caseId 'result.caseId'
|
||||
if ($Result.caseId -cne (Get-GuidanceCaseId $Case.id)) { throw 'Result caseId mismatch.' }
|
||||
if ($Result.Contains('workspaceRoot')) {
|
||||
Assert-GuidanceString $Result.workspaceRoot 'result.workspaceRoot'
|
||||
if (-not [IO.Path]::IsPathFullyQualified($Result.workspaceRoot) -or
|
||||
(Get-GuidanceSafePath $Result.workspaceRoot -Directory) -cne $Workspace) {
|
||||
throw 'Result workspaceRoot disagrees with the runner binding.'
|
||||
}
|
||||
}
|
||||
$report = $Result.guidanceReport
|
||||
Assert-GuidanceObject $report 'guidanceReport' @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')
|
||||
Assert-GuidanceObject $report.skill 'skill' @('id', 'version')
|
||||
Assert-GuidanceString $report.skill.id 'skill.id'
|
||||
if ($report.skill.id -cne 'al-development-plan' -or $report.skill.version -isnot [long] -or $report.skill.version -ne 1) {
|
||||
throw 'Report skill identity/version is invalid.'
|
||||
}
|
||||
Assert-GuidanceString $report.outcome 'outcome'
|
||||
if ($report.outcome -cnotin @('completed', 'not-applicable', 'no-knowledge', 'partial', 'failed')) { throw 'Report outcome enum is invalid.' }
|
||||
if ($report.outcome -cne $Case.expectedOutcome) { throw 'Report outcome does not match fixture expectedOutcome.' }
|
||||
if ($report.outcome -in @('partial', 'failed') -or $report.Contains('outcome-reason')) {
|
||||
Assert-GuidanceString $report['outcome-reason'] 'outcome-reason'
|
||||
}
|
||||
Assert-GuidanceObject $report.summary 'summary' @('request', 'kind', 'candidates', 'selected')
|
||||
Assert-GuidanceString $report.summary.request 'summary.request'
|
||||
Assert-GuidanceString $report.summary.kind 'summary.kind'
|
||||
if ($report.summary.kind -cne $Case.expectedKind) { throw 'summary.kind does not match the intended change.' }
|
||||
Assert-GuidanceInteger $report.summary.candidates 'summary.candidates'
|
||||
Assert-GuidanceInteger $report.summary.selected 'summary.selected'
|
||||
Assert-GuidanceContext $report.context
|
||||
foreach ($name in @('knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
|
||||
Assert-GuidanceArray $report[$name] $name
|
||||
}
|
||||
Assert-GuidanceArray $report.unresolved 'unresolved' -Strings
|
||||
if ($report.summary.selected -ne $report.knowledge.Count -or $report.summary.selected -gt $report.summary.candidates) {
|
||||
throw 'Summary counts disagree with selected knowledge/candidates.'
|
||||
}
|
||||
if ($report.outcome -in @('no-knowledge', 'not-applicable') -and $report.knowledge.Count) { throw 'This outcome requires empty knowledge.' }
|
||||
if ($report.outcome -eq 'completed' -and -not $report.knowledge.Count) { throw 'Completed requires selected knowledge; empty evaluation is no-knowledge.' }
|
||||
if (($report.outcome -eq 'partial' -or $Case.requiresUnresolved) -and -not $report.unresolved.Count) {
|
||||
throw 'Partial/incomplete evaluation must explain unresolved gaps.'
|
||||
}
|
||||
foreach ($dimension in $Case.expectedUnknown) {
|
||||
if ($dimension -cnotin $report.context.unknown) { throw 'Expected unknown context was silently resolved.' }
|
||||
}
|
||||
foreach ($dimension in $report.context.unknown) {
|
||||
if (-not @($report.unresolved | Where-Object { $_ -match [regex]::Escape($dimension) }).Count) {
|
||||
throw 'Unknown dimensions require a corresponding unresolved explanation.'
|
||||
}
|
||||
}
|
||||
# Free-form unresolved text has no machine-readable materiality field in DO.
|
||||
# Known material fixture conditions are runner expectations, not model claims.
|
||||
if ($Case.requiresMaterialUnresolved -and $report.outcome -ne 'partial') { throw 'Material unknown guidance must remain partial.' }
|
||||
$used = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($entry in $report.knowledge) {
|
||||
Assert-GuidanceObject $entry 'knowledge entry' @('path', 'used-for', 'constraints', 'sample-paths')
|
||||
$null = Resolve-GuidanceReference $Root $entry.path -Knowledge
|
||||
if (-not $used.Add($entry.path)) { throw 'Duplicate knowledge reference.' }
|
||||
Assert-GuidanceString $entry.'used-for' 'knowledge.used-for'
|
||||
Assert-GuidanceArray $entry.constraints 'knowledge.constraints' -Strings -NonEmpty
|
||||
Assert-GuidanceArray $entry.'sample-paths' 'knowledge.sample-paths' -Strings
|
||||
if (@($entry.'sample-paths' | Sort-Object -Unique).Count -ne $entry.'sample-paths'.Count) { throw 'Duplicate sample reference.' }
|
||||
foreach ($sample in $entry.'sample-paths') { $null = Resolve-GuidanceReference $Root $sample -Article $entry.path }
|
||||
Assert-GuidanceReferenceSha $entry $Root
|
||||
}
|
||||
$validationIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($entry in $report.'validation-considerations') {
|
||||
Assert-GuidanceObject $entry 'validation consideration' @('id', 'reason', 'evidence')
|
||||
foreach ($key in @('id', 'reason', 'evidence')) { Assert-GuidanceString $entry[$key] "validation-considerations.$key" }
|
||||
if (-not $validationIds.Add($entry.id)) { throw 'Duplicate validation consideration id.' }
|
||||
}
|
||||
$suppressed = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||
foreach ($entry in $report.suppressed) {
|
||||
Assert-GuidanceObject $entry 'suppressed entry' @('reference', 'reason')
|
||||
Assert-GuidanceObject $entry.reference 'suppressed.reference' @('path')
|
||||
$null = Resolve-GuidanceReference $Root $entry.reference.path -Knowledge
|
||||
Assert-GuidanceReferenceSha $entry.reference $Root
|
||||
Assert-GuidanceString $entry.reason 'suppression reason'
|
||||
if ($entry.reason -cnotin @('layer-precedence', 'configuration')) { throw 'Suppression reason is invalid.' }
|
||||
if (-not $suppressed.Add($entry.reference.path) -or $used.Contains($entry.reference.path)) { throw 'Duplicate or selected suppressed reference.' }
|
||||
}
|
||||
$matched = @($Case.requiredKnowledge | Where-Object { $used.Contains($_) }).Count
|
||||
$recall = if ($Case.requiredKnowledge.Count) { $matched / $Case.requiredKnowledge.Count } else { 1.0 }
|
||||
$accepted = @($Case.requiredKnowledge) + @($Case.optionalKnowledge)
|
||||
$acceptedCount = @($used | Where-Object { $_ -cin $accepted }).Count
|
||||
$precision = if ($used.Count) { $acceptedCount / $used.Count } elseif (-not $Case.requiredKnowledge.Count) { 1.0 } else { 0.0 }
|
||||
if ($recall -lt $Manifest.minimumKnowledgeRecall) { throw 'Knowledge recall is below the manifest threshold.' }
|
||||
if ($precision -lt $Manifest.minimumKnowledgePrecision) { throw 'Knowledge precision is below the manifest threshold.' }
|
||||
}
|
||||
|
||||
function Assert-GuidanceReferenceSha {
|
||||
param($Entry, [string] $Root)
|
||||
if ($Entry.Contains('sha')) {
|
||||
if ($Entry.sha -isnot [string] -or $Entry.sha -cnotmatch '^([0-9a-fA-F]{40}|[0-9a-fA-F]{64})$') {
|
||||
throw 'Reference SHA must be a full commit object id.'
|
||||
}
|
||||
$head = Invoke-GuidanceGit $Root @('rev-parse', '--verify', 'HEAD')
|
||||
if ($Entry.sha -ne $head) { throw 'Reference SHA does not identify the recorded live checkout.' }
|
||||
$committed = Invoke-GuidanceGit $Root @('cat-file', 'blob', "$head`:$($Entry.path)") -RawOutput
|
||||
$live = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $Entry.path -Knowledge))
|
||||
if ($committed.Replace("`r`n", "`n") -cne $live.Replace("`r`n", "`n")) {
|
||||
throw 'Reference SHA content differs from the live knowledge article.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-GuidanceResultSchema {
|
||||
param([string] $CaseId)
|
||||
return [ordered]@{
|
||||
caseId = $CaseId
|
||||
guidanceReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
||||
'outcome-reason' = 'required for partial or failed'
|
||||
summary = [ordered]@{ request = 'planned intent'; kind = 'feature | bug | refactor | upgrade | maintenance'; candidates = 0; selected = 0 }
|
||||
context = [ordered]@{ 'bc-version' = 'resolved target or unknown'; technologies = @('al'); countries = @('w1'); 'application-area' = @('all'); unknown = @() }
|
||||
knowledge = @([ordered]@{ path = 'repo-relative knowledge article'; sha = 'optional full checkout commit id'; 'used-for' = 'plan decision'; constraints = @('faithful normative constraint'); 'sample-paths' = @() })
|
||||
'validation-considerations' = @([ordered]@{ id = 'stable id'; reason = 'why needed'; evidence = 'evidence implementation should obtain' })
|
||||
suppressed = @([ordered]@{ reference = [ordered]@{ path = 'suppressed knowledge path' }; reason = 'layer-precedence | configuration' })
|
||||
unresolved = @('Missing context/decision, affected candidate, and materiality; name unknown dimensions exactly.')
|
||||
}
|
||||
}
|
||||
}
|
||||
452
tools/Test-DevelopmentGuidanceEvaluator.ps1
Normal file
452
tools/Test-DevelopmentGuidanceEvaluator.ps1
Normal file
|
|
@ -0,0 +1,452 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Deterministic, offline regressions for the guidance evaluator (no model/AL run).
|
||||
.DESCRIPTION
|
||||
Creates only a uniquely named .guidance-evaluator-regression-* directory below
|
||||
the current checkout, with standalone Git repositories and sibling runner
|
||||
artifacts. Removes that exact directory in finally; never uses the OS temp
|
||||
directory or cleans a caller-provided repository.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
|
||||
$evaluator = Join-Path $PSScriptRoot 'Test-DevelopmentGuidanceFixtures.ps1'
|
||||
$sourceRoot = (Get-Item -LiteralPath (Join-Path $PSScriptRoot '..')).FullName
|
||||
$scratch = Join-Path $sourceRoot ".guidance-evaluator-regression-$([guid]::NewGuid().ToString('N'))"
|
||||
$root = Join-Path $scratch 'knowledge-checkout'
|
||||
$article = 'microsoft/knowledge/performance/pair-findset-with-next-loop.md'
|
||||
$otherArticle = 'microsoft/knowledge/performance/findset-true-applies-updlock-on-read.md'
|
||||
$sample = 'microsoft/knowledge/performance/pair-findset-with-next-loop.good.al'
|
||||
$tests = [Collections.Generic.List[string]]::new()
|
||||
$script:scenarioNumber = 0
|
||||
$script:reportScenario = $null
|
||||
|
||||
function Set-TestJson($Path, $Value) {
|
||||
[IO.File]::WriteAllText($Path, ($Value | ConvertTo-Json -Depth 64))
|
||||
}
|
||||
|
||||
function Invoke-TestGit([string] $Directory, [string[]] $Arguments) {
|
||||
$output = @(& git --no-optional-locks -C $Directory @Arguments 2>&1)
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Regression Git setup failed.' }
|
||||
}
|
||||
|
||||
function Initialize-TestRepository([string] $Directory) {
|
||||
[IO.Directory]::CreateDirectory($Directory) | Out-Null
|
||||
Invoke-TestGit $Directory @('init', '--quiet')
|
||||
Invoke-TestGit $Directory @('config', 'user.email', 'guidance-fixture@example.invalid')
|
||||
Invoke-TestGit $Directory @('config', 'user.name', 'Guidance fixture')
|
||||
Invoke-TestGit $Directory @('config', 'commit.gpgSign', 'false')
|
||||
Invoke-TestGit $Directory @('config', 'core.autocrlf', 'false')
|
||||
[IO.File]::WriteAllText((Join-Path $Directory 'app.json'), '{"name":"Synthetic AL fixture","application":"28.0.0.0"}')
|
||||
[IO.File]::WriteAllText((Join-Path $Directory 'tracked.al'), 'codeunit 50100 Example {}')
|
||||
[IO.File]::WriteAllText((Join-Path $Directory '.gitignore'), "ignored.txt`nignored-directory/`n")
|
||||
Invoke-TestGit $Directory @('add', '.')
|
||||
Invoke-TestGit $Directory @('commit', '--quiet', '-m', 'Synthetic fixture baseline')
|
||||
}
|
||||
|
||||
function Invoke-EvaluatorTest([string] $Name, [string[]] $Arguments, [bool] $ShouldPass, [string] $Diagnostic = '') {
|
||||
$output = @(& pwsh -NoProfile -File $evaluator @Arguments 2>&1) -join "`n"
|
||||
$code = $LASTEXITCODE
|
||||
if (($code -eq 0) -ne $ShouldPass -or $output -notmatch $(if ($ShouldPass) { 'PASSED|captured' } else { 'FAILED' })) {
|
||||
throw "Regression '$Name' unexpected exit $code. $output"
|
||||
}
|
||||
if ($Diagnostic -and $output -notmatch [regex]::Escape($Diagnostic)) {
|
||||
throw "Regression '$Name' missing diagnostic '$Diagnostic'. $output"
|
||||
}
|
||||
if ($output -match 'MODEL_SECRET_SENTINEL') { throw "Regression '$Name' leaked model content." }
|
||||
$tests.Add($Name)
|
||||
}
|
||||
|
||||
function New-TestScenario([string] $Outcome = 'completed', [switch] $Unknown, [switch] $SecondCase) {
|
||||
$script:scenarioNumber++
|
||||
$directory = Join-Path $scratch "scenario-$script:scenarioNumber"
|
||||
[IO.Directory]::CreateDirectory($directory) | Out-Null
|
||||
$workspace = Join-Path $directory 'target'
|
||||
Initialize-TestRepository $workspace
|
||||
[IO.File]::WriteAllText((Join-Path $workspace 'untracked.txt'), 'existing untracked content')
|
||||
[IO.File]::WriteAllText((Join-Path $workspace 'ignored.txt'), 'existing ignored content')
|
||||
[IO.Directory]::CreateDirectory((Join-Path $workspace 'ignored-directory')) | Out-Null
|
||||
[IO.File]::WriteAllText((Join-Path $workspace 'ignored-directory\child.txt'), 'ignored child')
|
||||
$results = Join-Path $directory 'results'
|
||||
[IO.Directory]::CreateDirectory($results) | Out-Null
|
||||
$hasKnowledge = $Outcome -in @('completed', 'partial')
|
||||
$case = [ordered]@{
|
||||
id = 'synthetic-case'
|
||||
expectedKind = 'bug'
|
||||
expectedOutcome = $Outcome
|
||||
expectedUnknown = @($(if ($Unknown) { 'bc-version' }))
|
||||
requiresUnresolved = ($Outcome -eq 'partial' -or $Unknown.IsPresent)
|
||||
requiresMaterialUnresolved = ($Outcome -eq 'partial')
|
||||
'development-plan' = [ordered]@{ kind = 'bug'; request = 'Iterate the supplied filtered record set.' }
|
||||
context = [ordered]@{
|
||||
'bc-version' = $(if ($Unknown) { 'unknown' } else { '28' })
|
||||
technologies = @('al')
|
||||
countries = @('w1')
|
||||
'application-area' = @('all')
|
||||
unknown = @($(if ($Unknown) { 'bc-version' }))
|
||||
}
|
||||
requiredKnowledge = @($(if ($hasKnowledge) { $article }))
|
||||
optionalKnowledge = @()
|
||||
}
|
||||
$manifest = [ordered]@{
|
||||
version = 1
|
||||
skill = 'microsoft/skills/development/al-development-plan.md'
|
||||
minimumKnowledgeRecall = 1.0
|
||||
minimumKnowledgePrecision = 1.0
|
||||
cases = @($case)
|
||||
}
|
||||
$result = [ordered]@{
|
||||
caseId = Get-GuidanceCaseId $case.id
|
||||
guidanceReport = [ordered]@{
|
||||
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||
outcome = $Outcome
|
||||
summary = [ordered]@{ request = 'Iterate all selected records.'; kind = 'bug'; candidates = [int]$hasKnowledge; selected = [int]$hasKnowledge }
|
||||
context = $case.context
|
||||
knowledge = @($(if ($hasKnowledge) {
|
||||
[ordered]@{ path = $article; 'used-for' = 'Choose the multi-record reader.'; constraints = @('Use FindSet when iterating with Next.'); 'sample-paths' = @($sample) }
|
||||
}))
|
||||
'validation-considerations' = @([ordered]@{ id = 'all-selected'; reason = 'Preserve selection.'; evidence = 'Test all selected records and an excluded record.' })
|
||||
suppressed = @()
|
||||
unresolved = @($(if ($Outcome -eq 'partial') {
|
||||
if ($Unknown) { 'bc-version is unknown and materially affects the candidate; clarify before completing guidance.' }
|
||||
else { 'The caller cardinality decision remains materially unresolved.' }
|
||||
} elseif ($Unknown) { 'bc-version is unknown but immaterial: selected loop guidance applies to all versions.' }))
|
||||
}
|
||||
}
|
||||
if ($Outcome -in @('partial', 'failed')) { $result.guidanceReport.'outcome-reason' = 'Fixture intentionally leaves evaluation incomplete.' }
|
||||
$manifestPath = Join-Path $directory 'manifest.json'
|
||||
$mapPath = Join-Path $directory 'workspace-map.json'
|
||||
$map = [ordered]@{ 'synthetic-case' = $workspace }
|
||||
if ($SecondCase) {
|
||||
$second = $case | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||
$second.id = 'second-case'
|
||||
$manifest.cases += $second
|
||||
$secondWorkspace = Join-Path $directory 'second-target'
|
||||
Initialize-TestRepository $secondWorkspace
|
||||
$map[$second.id] = $secondWorkspace
|
||||
$secondResult = $result | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||
$secondResult.caseId = Get-GuidanceCaseId $second.id
|
||||
Set-TestJson (Join-Path $results "result-$($secondResult.caseId).json") $secondResult
|
||||
}
|
||||
Set-TestJson $manifestPath $manifest
|
||||
Set-TestJson $mapPath $map
|
||||
$resultPath = Join-Path $results "result-$($result.caseId).json"
|
||||
Set-TestJson $resultPath $result
|
||||
$baselinePath = Join-Path $directory 'baseline.json'
|
||||
$captureArgs = @('-Root', $root, '-ManifestPath', $manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $mapPath, '-BaselinePath', $baselinePath)
|
||||
$output = @(& pwsh -NoProfile -File $evaluator @captureArgs 2>&1) -join "`n"
|
||||
if ($LASTEXITCODE -ne 0) { throw "Baseline setup failed. $output" }
|
||||
return [ordered]@{
|
||||
directory = $directory; workspace = $workspace; resultPath = $resultPath; result = $result
|
||||
results = $results; manifest = $manifest; manifestPath = $manifestPath; mapPath = $mapPath
|
||||
baselinePath = $baselinePath; captureArgs = $captureArgs
|
||||
scoreArgs = @('-Root', $root, '-ManifestPath', $manifestPath, '-ResultsDirectory', $results, '-BaselinePath', $baselinePath, '-BaselineSha256', (Get-GuidanceHash $baselinePath))
|
||||
}
|
||||
}
|
||||
|
||||
function Test-ReportMutation([string] $Name, [scriptblock] $Change, [string] $Diagnostic) {
|
||||
if ($null -eq $script:reportScenario) { $script:reportScenario = New-TestScenario }
|
||||
$scenario = $script:reportScenario
|
||||
$result = $scenario.result | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||
& $Change $result
|
||||
Set-TestJson $scenario.resultPath $result
|
||||
Invoke-EvaluatorTest $Name $scenario.scoreArgs $false $Diagnostic
|
||||
}
|
||||
|
||||
try {
|
||||
[IO.Directory]::CreateDirectory($root) | Out-Null
|
||||
foreach ($reference in @($article, $otherArticle, $sample, $otherArticle.Replace('.md', '.good.al'),
|
||||
'microsoft/skills/development/al-development-plan.md', 'tools/Build-KnowledgeIndex.ps1',
|
||||
'tools/Knowledge-Retrieval.ps1')) {
|
||||
$destination = Join-Path $root $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
|
||||
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
|
||||
}
|
||||
Initialize-TestRepository $root
|
||||
|
||||
$publicManifestPath = Join-Path $sourceRoot 'evaluation\development-guidance-fixtures.json'
|
||||
$publicManifest = Read-GuidanceJson $publicManifestPath
|
||||
$publicRoot = Join-Path $scratch 'public-fixture-checkout'
|
||||
$publicKnowledgeReferences = @(
|
||||
$publicManifest.cases |
|
||||
ForEach-Object { $_.requiredKnowledge; $_.optionalKnowledge } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
$publicSampleReferences = @(
|
||||
foreach ($articleReference in $publicKnowledgeReferences) {
|
||||
$articlePath = Join-Path $sourceRoot $articleReference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||
$articleDirectory = Split-Path $articleReference -Parent
|
||||
$articleText = [IO.File]::ReadAllText($articlePath)
|
||||
foreach ($match in [regex]::Matches(
|
||||
$articleText,
|
||||
'\[[^\]]+\]\((?<sample>[a-z0-9-]+\.(?:good|bad)\.[a-zA-Z0-9]+)\)'
|
||||
)) {
|
||||
"$($articleDirectory.Replace('\', '/'))/$($match.Groups['sample'].Value)"
|
||||
}
|
||||
}
|
||||
)
|
||||
$publicReferences = @(
|
||||
@(
|
||||
$publicManifest.skill,
|
||||
'tools/Build-KnowledgeIndex.ps1',
|
||||
'tools/Knowledge-Retrieval.ps1',
|
||||
'evaluation/development-guidance-fixtures.json'
|
||||
) +
|
||||
$publicKnowledgeReferences +
|
||||
$publicSampleReferences |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
foreach ($reference in @($publicReferences | Sort-Object -Unique)) {
|
||||
$destination = Join-Path $publicRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
|
||||
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
|
||||
}
|
||||
$publicPrepared = Join-Path $scratch 'public-prepared'
|
||||
Invoke-EvaluatorTest 'public five-case manifest validation' @('-Root', $publicRoot) $true
|
||||
Invoke-EvaluatorTest 'public five-case manifest preparation' @('-Root', $publicRoot, '-PrepareDirectory', $publicPrepared) $true
|
||||
$initialCase = $publicManifest.cases | Where-Object id -eq 'synthetic-normal-initial-plan'
|
||||
$initialRequest = Read-GuidanceJson (Join-Path $publicPrepared "request-$(Get-GuidanceCaseId $initialCase.id).json")
|
||||
if ($initialRequest.'development-plan' -cne $initialCase.'development-plan') { throw 'Preparation truncated the serialized initial plan.' }
|
||||
$document = $initialRequest.'development-plan' | ConvertFrom-Json -AsHashtable
|
||||
if ($document.metadata.kind -ne 'bug' -or
|
||||
@('Root cause and design', 'Proposed fix', 'Affected files', 'Test strategy', 'Acceptance criteria' |
|
||||
Where-Object { $document.body -notmatch [regex]::Escape($_) }).Count) {
|
||||
throw 'Synthetic consumer boundary lost metadata or markdown plan sections.'
|
||||
}
|
||||
$tests.Add('serialized synthetic initial-plan boundary preserves full metadata and markdown body')
|
||||
|
||||
$scenario = New-TestScenario
|
||||
Invoke-EvaluatorTest 'unchanged target with ignored and untracked files passes' $scenario.scoreArgs $true
|
||||
Invoke-EvaluatorTest 'scoring itself leaves index and content unchanged' $scenario.scoreArgs $true
|
||||
Invoke-EvaluatorTest 'existing baseline cannot silently recapture' $scenario.captureArgs $false 'Baseline already exists'
|
||||
Invoke-EvaluatorTest 'scoring without baseline fails' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results) $false 'require BaselinePath'
|
||||
Invoke-EvaluatorTest 'scoring requires independently retained baseline digest' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath) $false 'runner-retained pre-run BaselineSha256'
|
||||
Invoke-EvaluatorTest 'tampered baseline digest fails' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath, '-BaselineSha256', ('0' * 64)) $false 'digest mismatch'
|
||||
|
||||
$prepare = Join-Path $scenario.directory 'prepared'
|
||||
Invoke-EvaluatorTest 'prepare outside roots with runner workspace binding' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', $prepare, '-WorkspaceMapPath', $scenario.mapPath) $true
|
||||
$prepared = Read-GuidanceJson (Join-Path $prepare "request-$($scenario.result.caseId).json")
|
||||
if ($prepared.repository -cne $scenario.workspace -or $prepared.Contains('expectedOutcome') -or $prepared.Contains('requiredKnowledge')) {
|
||||
throw 'Prepared request lost runner binding or exposed answers.'
|
||||
}
|
||||
Invoke-EvaluatorTest 'preparation never overwrites requests' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', $prepare) $false 'new or empty'
|
||||
foreach ($option in @('PrepareDirectory', 'BaselinePath', 'ResultsDirectory')) {
|
||||
$inside = Join-Path $scenario.workspace 'unsafe-artifact'
|
||||
$arguments = @('-Root', $root, '-ManifestPath', $scenario.manifestPath)
|
||||
if ($option -eq 'PrepareDirectory') { $arguments += @('-PrepareDirectory', $inside, '-WorkspaceMapPath', $scenario.mapPath) }
|
||||
elseif ($option -eq 'BaselinePath') { $arguments += @('-CaptureBaseline', '-BaselinePath', $inside, '-WorkspaceMapPath', $scenario.mapPath) }
|
||||
else { $arguments += @('-BaselinePath', $scenario.baselinePath, '-BaselineSha256', (Get-GuidanceHash $scenario.baselinePath), '-ResultsDirectory', $inside) }
|
||||
Invoke-EvaluatorTest "$option inside target rejected" $arguments $false 'outside target workspaces'
|
||||
}
|
||||
|
||||
foreach ($mutation in @('uncommitted', 'committed', 'empty-commit', 'staged', 'index-only', 'untracked', 'ignored', 'ignored-child', 'added', 'deleted', 'directory', 'ref', 'metadata')) {
|
||||
$scenario = New-TestScenario
|
||||
switch ($mutation) {
|
||||
'uncommitted' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed") }
|
||||
'committed' {
|
||||
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||
Invoke-TestGit $scenario.workspace @('add', 'tracked.al')
|
||||
Invoke-TestGit $scenario.workspace @('commit', '--quiet', '-m', 'Committed forbidden edit')
|
||||
}
|
||||
'staged' {
|
||||
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||
Invoke-TestGit $scenario.workspace @('add', 'tracked.al')
|
||||
}
|
||||
'empty-commit' { Invoke-TestGit $scenario.workspace @('commit', '--quiet', '--allow-empty', '-m', 'Forbidden empty commit') }
|
||||
'index-only' { Invoke-TestGit $scenario.workspace @('update-index', '--assume-unchanged', 'tracked.al') }
|
||||
'untracked' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'untracked.txt'), 'changed') }
|
||||
'ignored' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'ignored.txt'), 'changed') }
|
||||
'ignored-child' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'ignored-directory\child.txt'), 'changed') }
|
||||
'added' { [IO.File]::WriteAllText((Join-Path $scenario.workspace 'new.txt'), 'new ignored/untracked payload') }
|
||||
'deleted' { [IO.File]::Delete((Join-Path $scenario.workspace 'untracked.txt')) }
|
||||
'directory' { [IO.Directory]::CreateDirectory((Join-Path $scenario.workspace 'new-empty-directory')) | Out-Null }
|
||||
'ref' { Invoke-TestGit $scenario.workspace @('branch', 'new-reference') }
|
||||
'metadata' {
|
||||
$path = Join-Path $scenario.workspace 'tracked.al'
|
||||
[IO.File]::SetLastWriteTimeUtc($path, [IO.File]::GetLastWriteTimeUtc($path).AddSeconds(5))
|
||||
}
|
||||
}
|
||||
Invoke-EvaluatorTest "$mutation mutation fails" $scenario.scoreArgs $false 'identity/content changed'
|
||||
}
|
||||
|
||||
$scenario = New-TestScenario -SecondCase
|
||||
Invoke-EvaluatorTest 'two independently bound workspaces pass' $scenario.scoreArgs $true
|
||||
$map = Read-GuidanceJson $scenario.mapPath
|
||||
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||
$scenario.result.workspaceRoot = $map['second-case']
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'self-reported clean workspace cannot hide changed runner target' $scenario.scoreArgs $false 'identity/content changed'
|
||||
$scenario = New-TestScenario -SecondCase
|
||||
$scenario.result.workspaceRoot = (Read-GuidanceJson $scenario.mapPath)['second-case']
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'result workspace swapping rejected even when both are clean' $scenario.scoreArgs $false 'runner binding'
|
||||
$scenario = New-TestScenario -SecondCase
|
||||
$scenario.result.caseId = Get-GuidanceCaseId 'second-case'
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'result case swapping rejected' $scenario.scoreArgs $false 'caseId mismatch'
|
||||
$scenario = New-TestScenario
|
||||
$scenario.manifest.cases[0].expectedKind = 'feature'
|
||||
Set-TestJson $scenario.manifestPath $scenario.manifest
|
||||
Invoke-EvaluatorTest 'changed manifest rejected' $scenario.scoreArgs $false 'root or manifest'
|
||||
$scenario = New-TestScenario
|
||||
$baseline = Read-GuidanceJson $scenario.baselinePath
|
||||
$baseline.workspaces['synthetic-case'].caseId = 'case-00000000'
|
||||
Set-TestJson $scenario.baselinePath $baseline
|
||||
Invoke-EvaluatorTest 'baseline case binding checked even with matching digest' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath, '-BaselineSha256', (Get-GuidanceHash $scenario.baselinePath)) $false 'case identity mismatch'
|
||||
|
||||
foreach ($outcome in @('no-knowledge', 'partial', 'failed', 'not-applicable')) {
|
||||
$scenario = New-TestScenario $outcome
|
||||
Invoke-EvaluatorTest "honest $outcome is distinguishable and passes" $scenario.scoreArgs $true
|
||||
}
|
||||
$scenario = New-TestScenario 'partial' -Unknown
|
||||
Invoke-EvaluatorTest 'material unknown stays partial' $scenario.scoreArgs $true
|
||||
$scenario.result.guidanceReport.outcome = 'completed'
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'material unknown cannot silently complete' $scenario.scoreArgs $false 'expectedOutcome'
|
||||
$scenario = New-TestScenario 'completed' -Unknown
|
||||
Invoke-EvaluatorTest 'nonmaterial unknown can complete with explanation' $scenario.scoreArgs $true
|
||||
$scenario.result.guidanceReport.unresolved = @()
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'unknown cannot disappear from unresolved evidence' $scenario.scoreArgs $false 'unresolved gaps'
|
||||
$scenario = New-TestScenario 'no-knowledge' -Unknown
|
||||
Invoke-EvaluatorTest 'immaterial unknown and no-knowledge are not failures' $scenario.scoreArgs $true
|
||||
foreach ($outcome in @('partial', 'failed')) {
|
||||
$scenario = New-TestScenario $outcome
|
||||
$scenario.result.guidanceReport.Remove('outcome-reason')
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest "$outcome requires outcome-reason" $scenario.scoreArgs $false 'outcome-reason'
|
||||
}
|
||||
$scenario = New-TestScenario 'no-knowledge'
|
||||
$scenario.result.guidanceReport.outcome = 'completed'
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'no-knowledge is not completed-empty' $scenario.scoreArgs $false 'expectedOutcome'
|
||||
$scenario = New-TestScenario 'no-knowledge'
|
||||
$scenario.result.guidanceReport.knowledge = @(@{ path = $article; 'used-for' = 'filler'; constraints = @('filler'); 'sample-paths' = @() })
|
||||
$scenario.result.guidanceReport.summary.candidates = 1
|
||||
$scenario.result.guidanceReport.summary.selected = 1
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'no-knowledge cannot contain filler knowledge' $scenario.scoreArgs $false 'requires empty knowledge'
|
||||
|
||||
Test-ReportMutation 'invalid outcome enum' { param($r) $r.guidanceReport.outcome = 'success' } 'outcome enum'
|
||||
Test-ReportMutation 'missing report fields' { param($r) $r.guidanceReport.Remove('knowledge') } 'missing required field'
|
||||
Test-ReportMutation 'null report object' { param($r) $r.guidanceReport = $null } 'JSON object'
|
||||
Test-ReportMutation 'wrong skill version type' { param($r) $r.guidanceReport.skill.version = '1' } 'identity/version'
|
||||
Test-ReportMutation 'wrong skill id type' { param($r) $r.guidanceReport.skill.id = @('al-development-plan') } 'non-empty string'
|
||||
Test-ReportMutation 'wrong kind type' { param($r) $r.guidanceReport.summary.kind = @('bug') } 'non-empty string'
|
||||
Test-ReportMutation 'wrong summary type' { param($r) $r.guidanceReport.summary = @() } 'JSON object'
|
||||
Test-ReportMutation 'fractional count' { param($r) $r.guidanceReport.summary.candidates = 1.5 } 'non-negative integer'
|
||||
Test-ReportMutation 'negative count' { param($r) $r.guidanceReport.summary.selected = -1 } 'non-negative integer'
|
||||
Test-ReportMutation 'inconsistent selected count' { param($r) $r.guidanceReport.summary.selected = 0 } 'Summary counts'
|
||||
Test-ReportMutation 'candidate count below selected' { param($r) $r.guidanceReport.summary.candidates = 0 } 'Summary counts'
|
||||
Test-ReportMutation 'wrong context list type' { param($r) $r.guidanceReport.context.technologies = 'al' } 'JSON array'
|
||||
Test-ReportMutation 'missing constraints' { param($r) $r.guidanceReport.knowledge[0].constraints = @() } 'must not be empty'
|
||||
Test-ReportMutation 'non-string constraints' { param($r) $r.guidanceReport.knowledge[0].constraints = @(@{ body = 'MODEL_SECRET_SENTINEL' }) } 'non-empty string'
|
||||
Test-ReportMutation 'missing sample array' { param($r) $r.guidanceReport.knowledge[0].Remove('sample-paths') } 'missing required field'
|
||||
Test-ReportMutation 'duplicate knowledge' {
|
||||
param($r)
|
||||
$r.guidanceReport.knowledge += $r.guidanceReport.knowledge[0]
|
||||
$r.guidanceReport.summary.selected = 2
|
||||
$r.guidanceReport.summary.candidates = 2
|
||||
} 'Duplicate knowledge'
|
||||
Test-ReportMutation 'bad validation entry' { param($r) $r.guidanceReport.'validation-considerations'[0].evidence = $false } 'non-empty string'
|
||||
Test-ReportMutation 'invalid suppression shape' { param($r) $r.guidanceReport.suppressed = @(@{ path = $article; reason = 'configuration' }) } 'missing required field'
|
||||
Test-ReportMutation 'invalid unresolved shape' { param($r) $r.guidanceReport.unresolved = @(@{ candidate = $article }) } 'non-empty string'
|
||||
Test-ReportMutation 'invalid SHA provenance' { param($r) $r.guidanceReport.knowledge[0].sha = '0' * 40 } 'recorded live checkout'
|
||||
$scenario = New-TestScenario
|
||||
$scenario.result.guidanceReport.knowledge[0].sha = Invoke-GuidanceGit $root @('rev-parse', 'HEAD')
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'actual pinned knowledge SHA accepted' $scenario.scoreArgs $true
|
||||
|
||||
foreach ($badPath in @('../outside.md', '/absolute.md', 'C:/external.md',
|
||||
'microsoft\knowledge\performance\pair-findset-with-next-loop.md',
|
||||
'microsoft/knowledge/performance/../performance/pair-findset-with-next-loop.md',
|
||||
'https://example.invalid/article.md', 'microsoft/knowledge/performance/missing.md',
|
||||
'microsoft/skills/development/al-development-plan.md')) {
|
||||
$scenario = New-TestScenario
|
||||
$scenario.result.guidanceReport.knowledge[0].path = $badPath
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'unsafe/nonexistent/non-knowledge citation rejected' $scenario.scoreArgs $false
|
||||
}
|
||||
foreach ($badSample in @('../outside.al', $article, $otherArticle.Replace('.md', '.good.al'), 'microsoft/knowledge/performance/other.good.al', 'microsoft\knowledge\performance\pair-findset-with-next-loop.good.al')) {
|
||||
$scenario = New-TestScenario
|
||||
$scenario.result.guidanceReport.knowledge[0].'sample-paths' = @($badSample)
|
||||
Set-TestJson $scenario.resultPath $scenario.result
|
||||
Invoke-EvaluatorTest 'unsafe/nonexistent/non-sibling sample rejected' $scenario.scoreArgs $false
|
||||
}
|
||||
foreach ($json in @('{', 'null', '[]', '{"caseId":"MODEL_SECRET_SENTINEL","caseId":"duplicate"}', '{"caseId":true,}', '// comment')) {
|
||||
$scenario = New-TestScenario
|
||||
[IO.File]::WriteAllText($scenario.resultPath, $json)
|
||||
Invoke-EvaluatorTest 'malformed model result fails without runtime crash or content leakage' $scenario.scoreArgs $false
|
||||
}
|
||||
|
||||
$scenario = New-TestScenario
|
||||
$outside = Join-Path $scenario.directory 'outside'
|
||||
[IO.Directory]::CreateDirectory($outside) | Out-Null
|
||||
$link = Join-Path $scenario.workspace 'escape'
|
||||
$linkKind = if ($IsWindows) { 'Junction' } else { 'SymbolicLink' }
|
||||
New-Item -ItemType $linkKind -Path $link -Target $outside | Out-Null
|
||||
try {
|
||||
Invoke-EvaluatorTest 'target junction/symlink rejected instead of followed' $scenario.scoreArgs $false 'Links, junctions'
|
||||
Invoke-EvaluatorTest 'baseline capture rejects junction/symlink target children' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', (Join-Path $scenario.directory 'linked-baseline.json')) $false 'Links, junctions'
|
||||
Invoke-EvaluatorTest 'prepared directory cannot escape through junction' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', (Join-Path $link 'prepared'), '-WorkspaceMapPath', $scenario.mapPath) $false 'Links, junctions'
|
||||
} finally { Remove-Item -LiteralPath $link -Force }
|
||||
$scenario = New-TestScenario
|
||||
$external = Join-Path $scenario.directory 'external.txt'
|
||||
[IO.File]::WriteAllText($external, 'external hard-link target')
|
||||
$hardLink = Join-Path $scenario.workspace 'hard-link.txt'
|
||||
New-Item -ItemType HardLink -Path $hardLink -Target $external | Out-Null
|
||||
try {
|
||||
Invoke-EvaluatorTest 'hard-link escape rejected' $scenario.scoreArgs $false 'Links, junctions'
|
||||
} finally { Remove-Item -LiteralPath $hardLink -Force }
|
||||
$scenario = New-TestScenario
|
||||
$externalArticle = Join-Path $scenario.directory 'outside.md'
|
||||
[IO.File]::Copy((Join-Path $root $article.Replace('/', [IO.Path]::DirectorySeparatorChar)), $externalArticle)
|
||||
$articleLink = Join-Path $root 'custom\knowledge\performance'
|
||||
[IO.Directory]::CreateDirectory((Split-Path $articleLink -Parent)) | Out-Null
|
||||
New-Item -ItemType $linkKind -Path $articleLink -Target $scenario.directory | Out-Null
|
||||
try {
|
||||
$rejected = $false
|
||||
try { $null = Resolve-GuidanceReference $root 'custom/knowledge/performance/outside.md' -Knowledge }
|
||||
catch { $rejected = $_.Exception.Message -match 'Links, junctions' }
|
||||
if (-not $rejected) { throw 'Linked knowledge reference was followed.' }
|
||||
$tests.Add('knowledge junction/symlink reference rejected')
|
||||
$rejected = $false
|
||||
try { $null = Resolve-GuidanceReference $root 'custom/knowledge/performance/pair-findset-with-next-loop.good.al' -Article 'custom/knowledge/performance/pair-findset-with-next-loop.md' }
|
||||
catch { $rejected = $_.Exception.Message -match 'Links, junctions' }
|
||||
if (-not $rejected) { throw 'Linked sample reference was followed.' }
|
||||
$tests.Add('sample junction/symlink reference rejected')
|
||||
} finally { Remove-Item -LiteralPath $articleLink -Force }
|
||||
$normativePath = Join-Path $root 'microsoft\knowledge\performance\single-normative-section.md'
|
||||
foreach ($heading in @('Best Practice', 'Anti Pattern')) {
|
||||
[IO.File]::WriteAllText($normativePath, "---`ndomain: performance`n---`n## Description`nSynthetic contract fixture.`n## $heading`nSynthetic normative constraint.`n")
|
||||
$null = Resolve-GuidanceReference $root 'microsoft/knowledge/performance/single-normative-section.md' -Knowledge
|
||||
$tests.Add("Knowledge article with only $heading accepted")
|
||||
}
|
||||
[IO.File]::Delete($normativePath)
|
||||
$scenario = New-TestScenario
|
||||
$linkedRoot = Join-Path $scratch 'linked-knowledge-checkout'
|
||||
Invoke-TestGit $root @('worktree', 'add', '--quiet', '--detach', $linkedRoot, 'HEAD')
|
||||
$linkedBaseline = Join-Path $scenario.directory 'linked-root-baseline.json'
|
||||
Invoke-EvaluatorTest 'linked knowledge checkout baseline capture' @('-Root', $linkedRoot, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', $linkedBaseline) $true
|
||||
Invoke-EvaluatorTest 'unchanged linked knowledge checkout scoring' @('-Root', $linkedRoot, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $linkedBaseline, '-BaselineSha256', (Get-GuidanceHash $linkedBaseline)) $true
|
||||
$scenario = New-TestScenario
|
||||
$map = Read-GuidanceJson $scenario.mapPath
|
||||
$map['synthetic-case'] = $linkedRoot
|
||||
Set-TestJson $scenario.mapPath $map
|
||||
Invoke-EvaluatorTest 'linked target checkout rejected as external Git storage' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', (Join-Path $scenario.directory 'external-git-baseline.json')) $false 'standalone repositories'
|
||||
$scenario = New-TestScenario
|
||||
[IO.File]::AppendAllText((Join-Path $root $sample.Replace('/', [IO.Path]::DirectorySeparatorChar)), "`n// changed knowledge sample")
|
||||
Invoke-EvaluatorTest 'knowledge checkout changes rejected' $scenario.scoreArgs $false 'Knowledge checkout identity/content changed'
|
||||
Write-Host "Development guidance evaluator regressions PASSED: $($tests.Count) checks."
|
||||
} finally {
|
||||
if (Test-Path -LiteralPath $scratch) {
|
||||
# Only our own uniquely named directory; links made by tests are removed above.
|
||||
Remove-Item -LiteralPath $scratch -Recurse -Force
|
||||
}
|
||||
}
|
||||
|
||||
# Intentional negative native-command probes leave LASTEXITCODE nonzero.
|
||||
exit 0
|
||||
219
tools/Test-DevelopmentGuidanceFixtures.ps1
Normal file
219
tools/Test-DevelopmentGuidanceFixtures.ps1
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Validates, prepares, and scores read-only AL development-guidance fixtures.
|
||||
.DESCRIPTION
|
||||
Capture runner-owned evidence BEFORE invoking an agent:
|
||||
-CaptureBaseline -WorkspaceMapPath <json> -BaselinePath <new-json>
|
||||
The workspace map is {"manifest-case-id":"absolute-standalone-git-root",...}.
|
||||
Score AFTER the agent finishes:
|
||||
-BaselinePath <json> -BaselineSha256 <runner-retained-digest> -ResultsDirectory <directory>
|
||||
Preparation (-PrepareDirectory) is independent; supply -WorkspaceMapPath to
|
||||
include runner-selected target paths. Never derive target paths from results.
|
||||
|
||||
Baseline, map, prepared requests, and results must be outside all targets and
|
||||
the knowledge checkout. Keep the baseline runner-only; retain the printed
|
||||
SHA256 and pass -BaselineSha256 when scoring to detect baseline tampering.
|
||||
Capture never overwrites an existing baseline. The runner must protect this
|
||||
script, the baseline/digest and its invocation from the agent.
|
||||
|
||||
This compares before/after evidence, NOT an OS sandbox or a write monitor.
|
||||
It cannot detect reverted transient writes, prove that articles were opened,
|
||||
or validate semantic faithfulness of prose. Files, directories, hashes,
|
||||
stable metadata, Git HEAD/refs/index and ignored/untracked files are compared.
|
||||
Links/reparse points, hard links, external Git storage in targets,
|
||||
submodules and sparse checkouts are rejected rather than followed. Windows
|
||||
alternate data streams are rejected except for the known endpoint-DLP
|
||||
metadata stream `sec.endpointdlp`, which is ignored because endpoint
|
||||
protection may add or refresh it asynchronously without changing file
|
||||
content. Direct stream paths remain rejected. Run in quiescent repositories.
|
||||
The knowledge checkout may itself be a linked Git worktree; its Git storage
|
||||
identity is recorded explicitly.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $Root = (Join-Path $PSScriptRoot '..'),
|
||||
[string] $ManifestPath,
|
||||
[string] $PrepareDirectory,
|
||||
[string] $ResultsDirectory,
|
||||
[switch] $CaptureBaseline,
|
||||
[string] $BaselinePath,
|
||||
[string] $BaselineSha256,
|
||||
[string] $WorkspaceMapPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
|
||||
|
||||
try {
|
||||
if ($CaptureBaseline -and ($ResultsDirectory -or $PrepareDirectory)) {
|
||||
throw 'CaptureBaseline is a separate pre-run operation.'
|
||||
}
|
||||
if (($CaptureBaseline -or $ResultsDirectory) -and -not $BaselinePath) {
|
||||
throw 'Capture and scoring require BaselinePath.'
|
||||
}
|
||||
if ($ResultsDirectory -and -not $BaselineSha256) {
|
||||
throw 'Scoring requires the runner-retained pre-run BaselineSha256.'
|
||||
}
|
||||
if ($CaptureBaseline -and -not $WorkspaceMapPath) {
|
||||
throw 'Capture requires a runner-owned WorkspaceMapPath.'
|
||||
}
|
||||
if ($ResultsDirectory -and ($WorkspaceMapPath -or $PrepareDirectory)) {
|
||||
throw 'Scoring uses only the recorded workspace map; run preparation separately.'
|
||||
}
|
||||
$Root = Get-GuidanceSafePath $Root -Directory
|
||||
if (-not $ManifestPath) { $ManifestPath = Join-Path $Root 'evaluation\development-guidance-fixtures.json' }
|
||||
$ManifestPath = Get-GuidanceSafePath $ManifestPath -File
|
||||
$manifest = Read-GuidanceJson $ManifestPath
|
||||
Assert-GuidanceManifest $manifest $Root
|
||||
$caseIds = @($manifest.cases | ForEach-Object { $_.id })
|
||||
$workspaces = [ordered]@{}
|
||||
$baseline = $null
|
||||
if ($WorkspaceMapPath) {
|
||||
$WorkspaceMapPath = Get-GuidanceSafePath $WorkspaceMapPath -File
|
||||
$map = Read-GuidanceJson $WorkspaceMapPath
|
||||
Assert-GuidanceObject $map 'workspace map'
|
||||
if ($map.Count -ne $caseIds.Count) { throw 'Workspace map must bind exactly every manifest case.' }
|
||||
foreach ($id in $caseIds) {
|
||||
Assert-GuidanceString $map[$id] 'workspace map value'
|
||||
if (-not [IO.Path]::IsPathFullyQualified($map[$id])) { throw 'Workspace roots must be absolute.' }
|
||||
$workspaces[$id] = Get-GuidanceSafePath $map[$id] -Directory
|
||||
}
|
||||
}
|
||||
if ($ResultsDirectory) {
|
||||
$BaselinePath = Get-GuidanceSafePath $BaselinePath -File
|
||||
if ($BaselineSha256 -cnotmatch '^[0-9A-Fa-f]{64}$') {
|
||||
throw 'BaselineSha256 must be a SHA256 digest.'
|
||||
}
|
||||
if ((Get-GuidanceHash $BaselinePath) -ne $BaselineSha256) {
|
||||
throw 'Runner baseline digest mismatch.'
|
||||
}
|
||||
$baseline = Read-GuidanceJson $BaselinePath
|
||||
Assert-GuidanceObject $baseline 'baseline' @('version', 'kind', 'root', 'manifestPath', 'manifestSha256', 'workspaces', 'rootSnapshot')
|
||||
if ($baseline.version -ne 1 -or $baseline.kind -cne 'bcquality-guidance-runner-baseline') {
|
||||
throw 'Unsupported runner baseline.'
|
||||
}
|
||||
if ($baseline.root -cne $Root -or $baseline.manifestPath -cne $ManifestPath -or
|
||||
$baseline.manifestSha256 -ne (Get-GuidanceHash $ManifestPath)) {
|
||||
throw 'Runner baseline does not match the knowledge root or manifest.'
|
||||
}
|
||||
Assert-GuidanceObject $baseline.workspaces 'baseline workspaces'
|
||||
if ($baseline.workspaces.Count -ne $caseIds.Count) { throw 'Runner baseline case set mismatch.' }
|
||||
foreach ($id in $caseIds) {
|
||||
$entry = $baseline.workspaces[$id]
|
||||
Assert-GuidanceObject $entry 'baseline workspace entry' @('caseId', 'root', 'snapshot')
|
||||
if ($entry.caseId -cne (Get-GuidanceCaseId $id)) { throw 'Runner baseline case identity mismatch.' }
|
||||
$workspaces[$id] = Get-GuidanceSafePath $entry.root -Directory
|
||||
}
|
||||
}
|
||||
$protectedRoots = @($Root) + @($workspaces.Values)
|
||||
for ($i = 0; $i -lt $protectedRoots.Count; $i++) {
|
||||
for ($j = $i + 1; $j -lt $protectedRoots.Count; $j++) {
|
||||
if ((Test-GuidanceWithin $protectedRoots[$i] $protectedRoots[$j]) -or
|
||||
(Test-GuidanceWithin $protectedRoots[$j] $protectedRoots[$i])) {
|
||||
throw 'Knowledge checkout and case workspaces must be distinct, non-overlapping roots.'
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach ($artifact in @($BaselinePath, $WorkspaceMapPath, $PrepareDirectory, $ResultsDirectory)) {
|
||||
if ($artifact) {
|
||||
$safeArtifact = Get-GuidanceSafePath $artifact -AllowMissing
|
||||
foreach ($protectedRoot in $protectedRoots) {
|
||||
if (Test-GuidanceWithin $safeArtifact $protectedRoot) {
|
||||
throw 'Runner artifacts must be outside target workspaces and the knowledge checkout.'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($CaptureBaseline) {
|
||||
$BaselinePath = Get-GuidanceSafePath $BaselinePath -AllowMissing
|
||||
if (Test-Path -LiteralPath $BaselinePath) { throw 'Baseline already exists; capture never overwrites evidence.' }
|
||||
$snapshots = [ordered]@{}
|
||||
foreach ($id in $caseIds) {
|
||||
$snapshots[$id] = [ordered]@{
|
||||
caseId = Get-GuidanceCaseId $id
|
||||
root = $workspaces[$id]
|
||||
snapshot = Get-GuidanceSnapshot $workspaces[$id] -Target
|
||||
}
|
||||
}
|
||||
$record = [ordered]@{
|
||||
kind = 'bcquality-guidance-runner-baseline'
|
||||
version = 1
|
||||
root = $Root
|
||||
manifestPath = $ManifestPath
|
||||
manifestSha256 = Get-GuidanceHash $ManifestPath
|
||||
rootSnapshot = Get-GuidanceSnapshot $Root
|
||||
workspaces = $snapshots
|
||||
}
|
||||
Write-GuidanceNewJson $BaselinePath $record
|
||||
Write-Host "Guidance baseline captured. Runner SHA256: $(Get-GuidanceHash $BaselinePath)"
|
||||
} elseif ($PrepareDirectory) {
|
||||
$PrepareDirectory = Get-GuidanceSafePath $PrepareDirectory -AllowMissing
|
||||
if ((Test-Path -LiteralPath $PrepareDirectory) -and
|
||||
@(Get-ChildItem -LiteralPath $PrepareDirectory -Force).Count) {
|
||||
throw 'PrepareDirectory must be new or empty; existing requests/evidence are never overwritten.'
|
||||
}
|
||||
[IO.Directory]::CreateDirectory($PrepareDirectory) | Out-Null
|
||||
# The index builder walks recursively; reject linked corpus paths first.
|
||||
Assert-GuidanceTree $Root
|
||||
& (Join-Path $Root 'tools\Build-KnowledgeIndex.ps1') -BCQualityRoot $Root `
|
||||
-IndexPath (Join-Path $PrepareDirectory 'knowledge-index.json') | Out-Null
|
||||
$skillInstructions = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $manifest.skill))
|
||||
foreach ($case in $manifest.cases) {
|
||||
$modelId = Get-GuidanceCaseId $case.id
|
||||
$request = [ordered]@{
|
||||
protocol = 'Run the supplied read-only skill on the runner-assigned repository and existing plan. Return only caseId and guidanceReport. The runner captures evidence before invocation; do not capture or modify it. Do not create artifacts in the target or knowledge checkout.'
|
||||
caseId = $modelId
|
||||
skill = $manifest.skill
|
||||
skillInstructions = $skillInstructions
|
||||
knowledgeIndex = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||
knowledgeRoot = $Root
|
||||
'task-context' = [ordered]@{
|
||||
goal = Get-GuidancePlanRequest $case.'development-plan'
|
||||
'inputs-available' = @('development-plan', 'repository')
|
||||
'bc-version' = $case.context.'bc-version'
|
||||
technologies = $case.context.technologies
|
||||
countries = $case.context.countries
|
||||
'application-area' = $case.context.'application-area'
|
||||
}
|
||||
'development-plan' = $case.'development-plan'
|
||||
resultSchema = Get-GuidanceResultSchema $modelId
|
||||
}
|
||||
if ($workspaces.Count) { $request.repository = $workspaces[$case.id] }
|
||||
Write-GuidanceNewJson (Join-Path $PrepareDirectory "request-$modelId.json") $request
|
||||
}
|
||||
Write-Host "Development guidance preparation PASSED: $($caseIds.Count) case(s)."
|
||||
} elseif ($ResultsDirectory) {
|
||||
$ResultsDirectory = Get-GuidanceSafePath $ResultsDirectory -Directory
|
||||
$failures = [Collections.Generic.List[string]]::new()
|
||||
if (-not (Test-GuidanceSnapshotEqual $baseline.rootSnapshot (Get-GuidanceSnapshot $Root))) {
|
||||
$failures.Add('Knowledge checkout identity/content changed after baseline capture.')
|
||||
}
|
||||
foreach ($case in $manifest.cases) {
|
||||
$id = $case.id
|
||||
try {
|
||||
if (-not (Test-GuidanceSnapshotEqual $baseline.workspaces[$id].snapshot `
|
||||
(Get-GuidanceSnapshot $workspaces[$id] -Target))) {
|
||||
throw 'Target repository identity/content changed after baseline capture.'
|
||||
}
|
||||
$resultPath = Get-GuidanceSafePath (Join-Path $ResultsDirectory "result-$(Get-GuidanceCaseId $id).json") -File
|
||||
$result = Read-GuidanceJson $resultPath
|
||||
Assert-GuidanceResult $result $case $manifest $Root $workspaces[$id]
|
||||
} catch {
|
||||
# Only evaluator-authored diagnostics are printed, never model or file contents.
|
||||
$failures.Add("${id}: $(Get-GuidanceDiagnostic $_)")
|
||||
}
|
||||
}
|
||||
if ($failures.Count) {
|
||||
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):"
|
||||
$failures | ForEach-Object { Write-Host " - $_" }
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Development guidance scoring PASSED: $($caseIds.Count) case(s)."
|
||||
} else {
|
||||
Write-Host "Development guidance fixture validation PASSED: $($caseIds.Count) case(s)."
|
||||
}
|
||||
} catch {
|
||||
Write-Host "Development guidance FAILED: $(Get-GuidanceDiagnostic $_)"
|
||||
exit 1
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue