Microsoft/bcquality
1
0
Fork 0
mirror of https://github.com/microsoft/BCQuality.git synced 2026-10-05 14:46:55 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Narrow AL development to read-only plan guidance

Browse source 
Retain shared knowledge enrichment and review guidance; defer standalone implementation and source-ingestion tracking. Add runner-owned baseline evidence, contract regressions, and explicit consumer/pilot boundaries.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Jesper Schulz-Wedde 2026-09-07 11:47:42 +02:00
parent f6fca1d56d
commit 1deac52a53
26 changed files with 1486 additions and 11037 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.claude-plugin/marketplace.json
View file

@ -8,7 +8,7 @@
{
"name": "bcquality",
"source": "./",
"description": "Business Central AL quality knowledge base and skills, packaged as an installable plugin. Exposes development and review adapters while preserving BCQuality's internal Entry and action-skill protocols.",
"description": "Business Central AL quality knowledge base and skills, packaged as an installable plugin. Exposes read-only plan enrichment and review adapters through BCQuality's Entry protocol.",
"version": "0.3.0",
"skills": [
"./skills/"

88
.github/scripts/validate_frontmatter.py vendored
View file

@ -39,7 +39,6 @@ ACTION_SKILL_REQUIRED_KEYS = {
}
ACTION_SKILL_OPTIONAL_KEYS = {
"bc-version", "technologies", "countries", "application-area", "sub-skills",
"quality-skill", "quality-round-limit", "guidance-skill",
}
META_SKILL_REQUIRED_KEYS = {"kind", "id", "version", "title"}
ENTRY_SKILL_REQUIRED_KEYS = {"kind", "id", "version", "title"}
@ -47,10 +46,10 @@ HOST_SKILL_REQUIRED_KEYS = {"name", "description"}
STANDARD_INPUTS = {
"pr-diff", "object-list", "file-path", "repository", "telemetry-query",
"development-request", "development-plan",
"development-plan",
}
ALLOWED_OUTPUTS = {
"findings-report", "implementation-report", "development-guidance-report",
"findings-report", "development-guidance-report",
}
VALID_SAMPLE_KINDS = {"good", "bad"}
@ -410,33 +409,6 @@ def validate_action_skill(path: Path, parsed: Parsed, report: Report) -> None:
if bad:
report.error(path, "R20", f"invalid sub-skills paths: {bad}", 1)
if "quality-skill" in fm:
quality_skill = fm["quality-skill"]
_, err = normalize_repo_md_path(quality_skill)
if err:
report.error(path, "R31", f"quality-skill {err}", 1)
if fm.get("outputs") != ["implementation-report"]:
report.error(path, "R31", "quality-skill is valid only with outputs: [implementation-report]", 1)
if "quality-round-limit" not in fm:
report.error(path, "R31", "quality-skill requires quality-round-limit", 1)
if "quality-round-limit" in fm:
limit = fm["quality-round-limit"]
if not isinstance(limit, int) or isinstance(limit, bool) or limit <= 0:
report.error(path, "R31", "quality-round-limit must be a positive integer", 1)
if "quality-skill" not in fm:
report.error(path, "R31", "quality-round-limit requires quality-skill", 1)
if fm.get("outputs") != ["implementation-report"]:
report.error(path, "R31", "quality-round-limit is valid only with outputs: [implementation-report]", 1)
if "guidance-skill" in fm:
guidance_skill = fm["guidance-skill"]
_, err = normalize_repo_md_path(guidance_skill)
if err:
report.error(path, "R32", f"guidance-skill {err}", 1)
if fm.get("outputs") != ["implementation-report"]:
report.error(path, "R32", "guidance-skill is valid only with outputs: [implementation-report]", 1)
# R21 five required sections, in order, each exactly once
heads = [h for h, _ in headings_in_order(parsed.body)]
indices: list[int] = []
@ -662,58 +634,6 @@ def validate_sub_skills_registry(path: Path, fm: dict[str, Any], root: Path, rep
report.error(path, "R26", f"leaf not registered in sub-skills: {leaf}", 1)
def validate_quality_skill(path: Path, fm: dict[str, Any], root: Path, report: Report) -> None:
"""R30: implementation quality-skill paths resolve to a findings producer."""
quality_skill = fm.get("quality-skill")
normalized, err = normalize_repo_md_path(quality_skill)
if err or normalized is None:
return
target = root / normalized
if not target.is_file():
report.error(path, "R30", f"quality-skill does not exist on disk: {normalized}", 1)
return
if target.resolve() == path.resolve():
report.error(path, "R30", "quality-skill must not reference the implementation skill itself", 1)
return
try:
target_parsed = parse_markdown(target.read_text(encoding="utf-8"))
except UnicodeDecodeError as e:
report.error(path, "R30", f"quality-skill is not valid UTF-8: {e}", 1)
return
target_outputs = (target_parsed.frontmatter or {}).get("outputs")
if target_outputs != ["findings-report"]:
report.error(path, "R30", f"quality-skill must emit findings-report: {normalized}", 1)
def validate_guidance_skill(path: Path, fm: dict[str, Any], root: Path, report: Report) -> None:
"""R33: implementation guidance-skill paths resolve to a read-only planner."""
guidance_skill = fm.get("guidance-skill")
normalized, err = normalize_repo_md_path(guidance_skill)
if err or normalized is None:
return
target = root / normalized
if not target.is_file():
report.error(path, "R33", f"guidance-skill does not exist on disk: {normalized}", 1)
return
if target.resolve() == path.resolve():
report.error(path, "R33", "guidance-skill must not reference the implementation skill itself", 1)
return
try:
target_parsed = parse_markdown(target.read_text(encoding="utf-8"))
except UnicodeDecodeError as e:
report.error(path, "R33", f"guidance-skill is not valid UTF-8: {e}", 1)
return
target_fm = target_parsed.frontmatter or {}
if target_fm.get("outputs") != ["development-guidance-report"]:
report.error(path, "R33", f"guidance-skill must emit development-guidance-report: {normalized}", 1)
if "development-plan" not in (target_fm.get("inputs") or []):
report.error(path, "R33", f"guidance-skill must accept development-plan: {normalized}", 1)
def run(root: Path) -> Report:
report = Report()
skill_records: list[SkillRecord] = []
@ -781,11 +701,9 @@ def run(root: Path) -> Report:
others = [q.relative_to(root).as_posix() for q in paths if q != p]
report.error(p, "R24", f"skill id '{sid}' ({kind}) is not unique; also defined in: {others}")
# Fourth pass: cross-skill references
# Fourth pass: R26 sub-skills registry matches leaf files on disk
for path, fm in action_skill_fms:
validate_sub_skills_registry(path, fm, root, report)
validate_quality_skill(path, fm, root, report)
validate_guidance_skill(path, fm, root, report)
return report

21
.github/workflows/development-coverage.yml → .github/workflows/development-guidance.yml vendored
View file

@ -1,4 +1,4 @@
name: Validate development coverage
name: Validate read-only development guidance
on:
pull_request:
@ -7,20 +7,19 @@ on:
branches: [main]
jobs:
validate-development-coverage:
runs-on: ubuntu-latest
validate-development-guidance:
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Validate Microsoft Learn coverage ledger
shell: pwsh
run: ./tools/Test-LearnCoverage.ps1 -Root .
- name: Validate and prepare development fixtures
shell: pwsh
run: ./tools/Test-DevelopmentFixtures.ps1 -Root . -PrepareDirectory "$env:RUNNER_TEMP/bcquality-development-fixtures"
- name: Validate and prepare development-guidance fixtures
shell: pwsh
run: ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory "$env:RUNNER_TEMP/bcquality-development-guidance-fixtures"
- name: Run credential-free guidance evaluator regressions
shell: pwsh
run: ./tools/Test-DevelopmentGuidanceEvaluator.ps1

59
README.md
View file

@ -56,7 +56,7 @@ Skills define how agents consume knowledge. They come in three flavors:
READ and DO are read on demand — typically when the first dispatched action skill runs. They are not prerequisites for invoking Entry. WRITE is only used when scaffolding new content.
- **Action skills** — concrete skills that follow the Action Skill template to do real work. Review skills emit findings reports; read-only planning skills emit development-guidance reports; implementation skills emit implementation reports. Action skills live inside the layers that own them (`/microsoft/skills/`, `/community/skills/`, `/custom/skills/`). [`microsoft/skills/development/al-development-plan.md`](microsoft/skills/development/al-development-plan.md) turns an existing plan into knowledge constraints, [`microsoft/skills/development/al-development.md`](microsoft/skills/development/al-development.md) consumes those constraints while implementing features, bugs, refactors, upgrades, and maintenance, and [`microsoft/skills/review/al-code-review.md`](microsoft/skills/review/al-code-review.md) provides the final quality gate.
- **Action skills** — concrete skills that follow the Action Skill template. Review skills emit findings reports; read-only plan enrichment emits development-guidance reports. Action skills live inside the layers that own them (`/microsoft/skills/`, `/community/skills/`, `/custom/skills/`). [`microsoft/skills/development/al-development-plan.md`](microsoft/skills/development/al-development-plan.md) turns an existing plan into additional knowledge constraints for the consumer's established workflow. [`microsoft/skills/review/al-code-review.md`](microsoft/skills/review/al-code-review.md) independently reviews the resulting changes.
### Agent bootstrapping
@ -66,7 +66,7 @@ An orchestrator (such as AL-Go) points the agent at BCQuality's URL and provides
BCQuality can also be installed directly as a plugin. The plugin registers
host-native adapters for [`al-code-review`](skills/al-code-review/SKILL.md) and
[`al-development`](skills/al-development/SKILL.md). Both adapt
[`al-development-plan`](skills/al-development-plan/SKILL.md). Both adapt
the caller's request to the same Entry protocol used by orchestrators.
For GitHub Copilot CLI:
@ -81,8 +81,8 @@ must be updated. The name remains distinct from BC-ALAgents' public
`al-review` skill because current hosts may load plugin skill names into one
shared inventory.
Plugin version `0.3.0` adds `al-development`, the knowledge-backed
implementation adapter for features, bugs, refactors, upgrades, and maintenance.
Plugin version `0.3.0` adds `al-development-plan`, a read-only adapter for
enriching an **existing** plan. It does not generate a plan or implement code.
The adapters are intentionally not second implementations:
@ -92,10 +92,10 @@ standalone host skill: skills/al-code-review/SKILL.md
-> review coordinator: microsoft/skills/review/al-code-review.md
-> domain review leaves
standalone host skill: skills/al-development/SKILL.md
standalone host skill: skills/al-development-plan/SKILL.md
-> routing contract: skills/entry.md
-> implementation skill: microsoft/skills/development/al-development.md
-> knowledge-guided implementation + AL review quality gate
-> enrichment skill: microsoft/skills/development/al-development-plan.md
-> referenced constraints for the consumer's existing workflow (read-only)
```
Only the files under `skills/*/SKILL.md` follow the host's packaging format.
@ -143,33 +143,31 @@ Code examples belong in separate files, not in the knowledge file itself. Knowle
## Scope
The current curated corpus covers technical AL concerns across Agents, AppSource and compatibility, data modeling, error handling, events, interfaces, performance, privacy, Query objects, security, style, telemetry, testing, UI, upgrade, and web services. Review skills evaluate existing changes against those domains. The `al-development` skill applies them before and during implementation, then runs the review coordinator as a final gate.
The current curated corpus covers technical AL concerns across Agents, AppSource and compatibility, data modeling, error handling, events, interfaces, performance, privacy, Query objects, security, style, telemetry, testing, UI, upgrade, and web services. Review skills evaluate existing changes against those domains. The read-only `al-development-plan` interface selects relevant constraints before the consumer implements its own plan.
Repository-specific orchestrators do not need to delegate implementation to
`al-development`. They can invoke `al-development-plan` with their existing
plan, feed its read-only guidance report into their own phases, and retain their
specialized environment, test, propagation, and delivery gates.
Repository-specific orchestrators retain planning, implementation, approvals,
tests, environment, propagation, and delivery ownership. The intended flow is
consumer analysis and normalized plan -> read-only BCQuality guidance ->
existing implementation phases -> independent final BCQuality review ->
delivery. Consumer uptake and a real runtime pilot are follow-up work, not
implemented integrations or demonstrated authoring improvements.
`al-development` does not silently fall back to generic generation when no
article applies. It returns `no-knowledge` without changing code, making corpus
coverage visible; callers can use their normal repository workflow or
contribute the missing Business Central-specific guidance.
`no-knowledge` means no additional applicable BCQuality constraints, with empty
`knowledge`; it does not make a plan unsafe or prevent the consumer from using
its ordinary gates. Retrieval failures and materially unresolved conditional
guidance are distinct outcomes, not empty knowledge. Do not add generic advice
just to avoid a `no-knowledge` result.
Business Central functional domains (Finance, Supply Chain Management, Manufacturing, Jobs, Warehousing, Service), PowerShell, pipelines, and Power Platform remain valid future repository scope, but they are **not current coverage claims** until corresponding knowledge and action skills exist. Consumers should derive supported review scope from the live knowledge index and dispatched skills, not from roadmap breadth.
## Tracking developer coverage
## Evidence and follow-up scope
BCQuality tracks source ingestion and implementation capability separately:
- [`coverage/microsoft-learn-developer-catalog.json`](coverage/microsoft-learn-developer-catalog.json) is the generated inventory of Business Central developer training.
- [`coverage/learn-coverage.json`](coverage/learn-coverage.json) records editorial progress and the disposition of each extracted concern.
- [`coverage/development-capabilities.json`](coverage/development-capabilities.json) tracks representative development capabilities and their evaluation fixtures.
Run `pwsh ./tools/Test-LearnCoverage.ps1` for current source progress and
`pwsh ./tools/Test-DevelopmentFixtures.ps1` for capability coverage. Article
count alone is not a completion metric: a capability becomes `validated` only
after its generated implementation passes compilation, tests, and the review
quality gate.
The [guidance evaluation](evaluation/README.md#read-only-plan-guidance) separates
credential-free contract/scorer regressions from external agent and runtime
evidence. Prepared requests and fixture counts do not establish compilation,
test execution, better repairs, or a capability percentage. Consumer adoption,
a pinned baseline comparison and runtime pilot, standalone authoring, and
source-ingestion catalog work remain separate follow-ups.
## How agents consume BCQuality
@ -180,7 +178,7 @@ Action skills follow a four-step pattern:
3. **Worklist** — narrow from N candidates to the M that apply to the current task
4. **Action** — apply the relevant knowledge and produce structured output
Every action skill declares one structured JSON output. Review skills emit a `findings-report`; planning skills emit a read-only `development-guidance-report`; implementation skills emit an `implementation-report` containing the plan, knowledge used, changed files, real validation results, and post-implementation review. All contracts are defined in the Action Skill meta-skill so orchestrators and action skills remain independently evolvable.
Every action skill declares one structured JSON output. Review skills emit a `findings-report`; plan enrichment emits a read-only `development-guidance-report`. Both contracts are defined in the Action Skill meta-skill so orchestrators and action skills remain independently evolvable.
BCQuality is an **additive** knowledge layer: it augments the agent's review judgement, it does not replace it. Super-skills (such as `al-code-review`) run a self-review pass alongside their sub-skills and surface concerns the agent identified on its own, marked with `from-sub-skill: "agent"` and an empty `references: []` so consumers can render them distinctly from knowledge-backed findings. See [agent-consumption.md](agent-consumption.md) and [`skills/do.md`](skills/do.md) for the full contract.
@ -192,8 +190,7 @@ For the end-to-end flow — from orchestrator trigger through to how output reac
```
├── /skills/ # Global: entry-point skill + meta-skill contracts (READ, DO, WRITE)
├── /coverage/ # Source-ingestion ledger and development capability matrix
├── /evaluation/ # Review and development evaluation fixtures
├── /evaluation/ # Review and read-only guidance evaluation fixtures
├── /.github/ # Actions and workflows
├── /microsoft/ # Microsoft-endorsed layer
│ ├── /knowledge/ # Knowledge files by domain

98
agent-consumption.md
View file

@ -14,7 +14,7 @@ For the high-level framing and repo structure, start with the [README](README.md
When BCQuality is installed as a standalone plugin, it additionally exposes
`skills/al-code-review/SKILL.md` and
`skills/al-development/SKILL.md`. These are host-format adapters, not
`skills/al-development-plan/SKILL.md`. These are host-format adapters, not
additional action skills: each creates the task context and enters the same
flow at Entry.
@ -27,7 +27,7 @@ flowchart LR
E -->|3 dispatch record| A
A -->|4 invoke dispatched skill| S[Action skill<br/>e.g. al-code-review]
S -->|5 execute| P[Source → Relevance<br/>→ Worklist → Action<br/>reading READ · DO on demand]
P -->|6 emit| R[Findings report<br/>or implementation report]
P -->|6 emit| R[Findings report<br/>or read-only guidance report]
R -->|7 integrate| O
```
@ -40,14 +40,14 @@ The agent reads `/skills/entry.md` and runs it against the task context. Entry a
For a standalone plugin installation, the host activates the matching adapter
first. The adapter preserves the caller's actual goal, constructs the task
context, and invokes Entry. It does not select the internal review or
development action skill itself or duplicate Entry's preparation, routing, and
plan-enrichment action skill itself or duplicate Entry's preparation, routing, and
failure semantics.
### 3. Agent consumes the dispatch record
The dispatch record names one or more action skills, the subset of inputs each
should receive, and each skill's output kind. The output kind identifies
read-only review or planning work versus repository-changing implementation
before invocation. If the outcome is `no-match` or `failed`, the agent returns
should receive, and each skill's output kind. The output kind distinguishes
findings from read-only plan guidance before invocation; it is not proof of
runtime side effects. If the outcome is `no-match` or `failed`, the agent returns
the record to the orchestrator unchanged.
### 4. Agent invokes each dispatched action skill
@ -81,40 +81,78 @@ The output contracts are defined in the DO meta-skill:
- A **findings report** carries review findings, domain labels, references, confidence, and suppressions.
- A **development guidance report** carries read-only knowledge constraints and validation considerations for an existing plan.
- An **implementation report** carries the development plan, classified mode, knowledge applied, changed files, validation results, final review, and remaining work.
The orchestrator parses this **without skill-specific logic**. This is the point of the contract: orchestrators and action skills evolve independently.
For development, the action happens before the report: the skill
first invokes the read-only planning skill to select applicable knowledge, then
changes the target repository, runs its native validation, and invokes the
configured review quality-skill over the resulting diff. A specialized
repository orchestrator may invoke only the planning skill and feed its
guidance report into its own implementation phases. The implementation report
is a machine-readable record of persisted work, not a code proposal for the
orchestrator to apply later.
For plan enrichment, the skill reads the existing plan and target repository,
selects applicable knowledge, and returns constraints without changing the
target. It does not generate a replacement plan, run tests, implement code, or
drive a review/fix loop. Implementation stays in the consuming workflow.
### 7. Orchestrator integrates
The orchestrator turns findings into PR comments, build gates, or IDE diagnostics. For implementation it presents the changed files and validation state, while the agent has already persisted the requested change in the target repository.
The orchestrator turns findings into PR comments, build gates, or IDE diagnostics. It can feed read-only guidance into its own implementation phases, preserving all existing approvals and delivery gates.
## Repository-specific development orchestrators
A repository-specific workflow can keep ownership of implementation and consume
BCQuality only for planning and review:
A repository-specific workflow can consume this read-only foundation before
authoring while retaining its independent final review. This is the intended
integration boundary, not a shipped consumer integration:
1. Produce its normal development plan after repository investigation.
2. Invoke Entry with `inputs-available: [development-plan, repository]` plus
the resolved applicability dimensions.
3. Execute the dispatched `al-development-plan` skill and preserve its
`development-guidance-report`.
4. Pass the selected article references, constraints, samples, and validation
considerations into its own test, implementation, and critique phases.
5. Run its existing BCQuality-backed review gate over the completed diff.
1. Investigate and produce the consumer's normal initial plan. Normalize any
consumer-specific format outside BCQuality. A full serialized plan document
containing metadata plus a markdown body (root cause or design intent,
proposed changes, affected files, test strategy, acceptance criteria) is a
valid boundary. A continuation/checkpoint payload is not a substitute for
initial-plan coverage; workflow identifiers and state stay with the consumer.
2. Resolve and record an immutable BCQuality checkout and filtering policy.
Invoke Entry with a read-only enrichment goal, the existing
`development-plan`, `repository`, and established applicability dimensions.
Keep index, guidance, and runner artifacts outside the target repository.
3. Execute the dispatched `al-development-plan` skill. Persist the unchanged
report and provenance **after** any consumer state initialization or cleanup
that could erase them. BCQuality does not own the state directory or lifecycle.
4. Inject relevant constraints and validation considerations into the existing
Baseline, Implement, propagation (such as MiApp), and Critique phases, or
equivalents. Re-enrich on material plan or applicability changes; preserve
the relationship between plan version, guidance, and implementation attempt.
5. Run an independent final BCQuality review against the completed diff using
the **same recorded immutable checkout** used for enrichment. Review the
actual changes, not the guidance report as proof of correctness, then apply
the consumer's ordinary delivery gates.
This is the integration model for specialized bug-fix or release workflows.
They keep environment provisioning, retries, state, commits, propagation, and
pull-request delivery; BCQuality supplies shared product knowledge before and
after the code change.
The consumer owns analysis, normalization, persistence, per-phase injection,
approvals, TDD and runtime execution, propagation, retries, commits, and PR
delivery. BCQuality supplies additional referenced product knowledge, not a
replacement orchestrator.
### Outcomes are additive, not a universal coding gate
`no-knowledge` with empty `knowledge` means no additional applicable BCQuality
constraints. The consumer may proceed under its ordinary gates. It must not be
conflated with failed retrieval/reference integrity (`failed`), incomplete
evaluation or materially unresolved conditional guidance (`partial`), or absent
required inputs (`not-applicable`). Consumers own the policy for handling those
outcomes and recorded unknowns: seek missing context, re-enrich, escalate, or
apply their existing risk controls without relabeling the report as successful.
Do not fill gaps with generic articles simply to unlock implementation.
### Pinning and pilot evidence
A configured tag or ref alone does not establish runtime pinning. Record the
resolved commit and actual checkout/content identity used at invocation, along
with enabled layers, pruning policy, index identity, plan version, and runner
provenance. Verify that identity at both enrichment and final review; fetching
default HEAD into an explicitly supplied checkout can bypass a configured ref.
Use an isolated checkout that cannot drift during the run.
Consumer rollout and an external pilot remain follow-up work. A pilot must
compare a pinned independent baseline run of the existing workflow without
enrichment against a matched enriched run, keeping starting code, task, model,
tools, runtime, and gates controlled and recording the actual BCQuality
checkout. Retain external logs, diffs, test/compile outcomes, and independent
final reviews, including failures and unresolved results. Credential-free
fixture preparation and scorer regressions do not demonstrate improved repair
quality, compilation, runtime success, or production integration.
## Knowledge-backed and agent findings

48
coverage/README.md
View file

@ -1,48 +0,0 @@
# Developer knowledge coverage
This directory separates **source coverage** from the knowledge corpus itself.
Microsoft Learn units are inputs to editorial work, not articles to import
one-for-one.
## Files
- `microsoft-learn-developer-catalog.json` is generated source metadata for all
Microsoft Learn modules tagged with both `dynamics-business-central` and
`developer`.
- `learn-coverage.json` is the maintained editorial ledger. Units absent from
this file are unreviewed.
- `development-capabilities.json` tracks whether representative Business
Central development capabilities have implementation fixtures.
The capability manifest declares `minimumFixtureCoverage`. CI fails when the
share of `fixture` or `validated` capabilities falls below that floor, so a
new planned capability cannot silently dilute generation coverage.
Each tracked unit has a `reviewStatus`:
- `in-progress` — at least one concern has been identified, but editorial
triage of the unit is not complete.
- `complete` — every relevant concern in the unit has a recorded outcome. A
complete unit may have no outcomes when it contains no remedial knowledge.
Each concern has one disposition:
- `candidate` — worth authoring or reconciling with existing knowledge.
- `authored` — produced one or more new knowledge articles.
- `covered-existing` — already represented by the linked article.
- `rejected` — fails BCQuality's remedial admission test.
- `deferred` — valid but intentionally postponed, with a rationale.
An authored article does not make its source unit complete automatically. One
unit can contain several independent concerns.
## Update and report
```powershell
pwsh ./tools/Update-LearnCatalog.ps1
pwsh ./tools/Test-LearnCoverage.ps1
```
The catalog updater also accepts `-CatalogPath` for an exported Microsoft Learn
Platform API response. CI validates the committed snapshot and editorial ledger
without network access.

132
coverage/development-capabilities.json
View file

@ -1,132 +0,0 @@
{
"version": 1,
"minimumFixtureCoverage": 0.5,
"capabilities": [
{
"id": "setup-and-master-data",
"title": "Setup and master data",
"status": "fixture",
"domains": [
"data-modeling",
"security",
"testing",
"ui"
],
"fixtureIds": [
"setup-backed-master-data"
]
},
{
"id": "document-workflows",
"title": "Document header and lines workflows",
"status": "fixture",
"domains": [
"data-modeling",
"events",
"performance",
"testing",
"ui"
],
"fixtureIds": [
"document-header-and-lines"
]
},
{
"id": "api-integrations",
"title": "Versioned API integrations",
"status": "fixture",
"domains": [
"security",
"testing",
"web-services"
],
"fixtureIds": [
"versioned-master-data-api"
]
},
{
"id": "bug-diagnosis-and-fix",
"title": "Bug diagnosis and surgical repair",
"status": "fixture",
"domains": [
"performance",
"testing"
],
"fixtureIds": [
"fix-filtered-batch-processing"
]
},
{
"id": "journals-and-posting",
"title": "Journals and posting routines",
"status": "planned",
"domains": [
"data-modeling",
"error-handling",
"events",
"performance",
"testing"
],
"fixtureIds": []
},
{
"id": "reports-and-documents",
"title": "Reports and document layouts",
"status": "planned",
"domains": [
"performance",
"testing",
"ui"
],
"fixtureIds": []
},
{
"id": "install-and-upgrade",
"title": "Installation and data upgrade",
"status": "fixture",
"domains": [
"breaking-changes",
"testing",
"upgrade"
],
"fixtureIds": [
"versioned-data-upgrade"
]
},
{
"id": "external-services",
"title": "Outbound services and authentication",
"status": "planned",
"domains": [
"error-handling",
"privacy",
"security",
"telemetry"
],
"fixtureIds": []
},
{
"id": "role-centers-and-onboarding",
"title": "Role Centers, setup, and onboarding",
"status": "planned",
"domains": [
"security",
"testing",
"ui"
],
"fixtureIds": []
},
{
"id": "appsource-lifecycle",
"title": "AppSource packaging and lifecycle",
"status": "planned",
"domains": [
"appsource",
"breaking-changes",
"testing",
"upgrade"
],
"fixtureIds": []
}
]
}

456
coverage/learn-coverage.json
View file

@ -1,456 +0,0 @@
{
"version": 1,
"catalog": "coverage/microsoft-learn-developer-catalog.json",
"units": [
{
"uid": "learn-dynamics.use-document-standards-business-central.4a-use-round-function",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "round-direction-symbols-use-magnitude",
"title": "Round direction symbols use magnitude rather than mathematical ordering",
"disposition": "authored",
"domain": "data-modeling",
"articlePaths": [
"microsoft/knowledge/data-modeling/round-direction-symbols-use-magnitude.md"
]
}
]
},
{
"uid": "learn-dynamics.use-document-standards-business-central.3-use-initrecord-function",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "initialize-document-defaults-in-initrecord",
"title": "Initialize document defaults in InitRecord after assigning the number",
"disposition": "authored",
"domain": "data-modeling",
"articlePaths": [
"microsoft/knowledge/data-modeling/initialize-document-defaults-in-initrecord.md"
]
}
]
},
{
"uid": "learn-dynamics.business-central-interfaces.type-testing",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "guard-interface-casts-with-is",
"title": "Guard optional interface casts with is",
"disposition": "authored",
"domain": "interfaces",
"articlePaths": [
"microsoft/knowledge/interfaces/guard-interface-casts-with-is.md"
]
}
]
},
{
"uid": "learn-dynamics.extend-modify-existing-table.add-field-group",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "dropdown-fieldgroup-respects-lookup-page-visibility",
"title": "DropDown fields remain hidden when their lookup-page controls are hidden",
"disposition": "authored",
"domain": "ui",
"articlePaths": [
"microsoft/knowledge/ui/dropdown-fieldgroup-respects-lookup-page-visibility.md"
]
}
]
},
{
"uid": "learn-dynamics.work-with-pages.8-controls",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "updatepropagation-both-refreshes-main-page",
"title": "UpdatePropagation Both refreshes the main page after line edits",
"disposition": "authored",
"domain": "ui",
"articlePaths": [
"microsoft/knowledge/ui/updatepropagation-both-refreshes-main-page.md"
]
},
{
"id": "applicationarea-parent-inheritance",
"title": "Page-level ApplicationArea inheritance excludes extension controls",
"disposition": "covered-existing",
"domain": "style",
"articlePaths": [
"microsoft/knowledge/style/applicationarea-required-on-page-controls.md"
]
}
]
},
{
"uid": "learn-dynamics.easy-application-upgrade.3-installation-upgrade-codeunits",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "install-and-upgrade-codeunits-have-no-order",
"title": "Separate install or upgrade codeunits have no execution order",
"disposition": "authored",
"domain": "upgrade",
"articlePaths": [
"microsoft/knowledge/upgrade/install-and-upgrade-codeunits-have-no-order.md"
]
},
{
"id": "appversion-meaning-depends-on-execution-context",
"title": "ModuleInfo AppVersion changes meaning with execution context",
"disposition": "authored",
"domain": "upgrade",
"articlePaths": [
"microsoft/knowledge/upgrade/appversion-meaning-depends-on-execution-context.md"
]
}
]
},
{
"uid": "learn-dynamics.work-with-tables.text-search",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "optimize-for-text-search-requires-double-ampersand-filter",
"title": "Optimized full-text search requires the double-ampersand filter operator",
"disposition": "candidate",
"domain": "query"
}
]
},
{
"uid": "learn-dynamics.extend-modify-existing-table.define-extension-objects",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "same-app-extension-references-follow-object-id-order",
"title": "Same-app extension objects can only reference lower-ID extension objects",
"disposition": "candidate",
"domain": "data-modeling"
}
]
},
{
"uid": "learn-dynamics.debug-deploy-extension.resource-policy-settings",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "resource-exposure-policy-is-not-nondebuggable",
"title": "Resource exposure policy and NonDebuggable protect different surfaces",
"disposition": "candidate",
"domain": "security"
}
]
},
{
"uid": "learn-dynamics.work-entitlements-permission-sets.3-create-entitlements-permission-sets",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "entitlement-objects-reference-same-app-permission-sets",
"title": "Entitlement objects are online-only and reference same-app permission sets",
"disposition": "candidate",
"domain": "security"
}
]
},
{
"uid": "learn-dynamics.work-entitlements-permission-sets.override-entitlements",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "inherent-permissions-cannot-elevate-other-extensions",
"title": "Inherent permissions cannot elevate access to another extension",
"disposition": "candidate",
"domain": "security"
}
]
},
{
"uid": "learn-dynamics.debug-deploy-extension.3a-snapshot",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "snapshot-debugging-captures-only-snappoints-and-exceptions",
"title": "Snapshot debugging captures state only at snappoints and exceptions",
"disposition": "candidate",
"domain": "testing"
}
]
},
{
"uid": "learn-dynamics.debug-deploy-extension.recovery-failures",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "publish-recovery-cannot-restore-every-extension-state",
"title": "Publish recovery cannot restore every upgrade or app-move failure",
"disposition": "candidate",
"domain": "upgrade"
}
]
},
{
"uid": "learn-dynamics.debug-deploy-extension.database-wait-statistics",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "database-wait-statistics-are-exposed-as-a-virtual-table",
"title": "Database wait statistics are exposed as a Business Central virtual table",
"disposition": "candidate",
"domain": "performance"
}
]
},
{
"uid": "learn-dynamics.work-with-pages.use-rich-text-editor",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "rich-text-controls-use-html-backed-blob-fields",
"title": "Rich text controls use HTML-backed Blob fields and isolated layout groups",
"disposition": "candidate",
"domain": "ui"
}
]
},
{
"uid": "learn-dynamics.work-with-pages.scan-barcodes",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "barcode-integration-scenarios-have-different-platform-support",
"title": "Barcode field, camera action, and hardware-scanner scenarios have different platform support",
"disposition": "candidate",
"domain": "ui"
}
]
},
{
"uid": "learn-dynamics.work-with-pages.9-search",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "card-pages-should-not-be-directly-searchable",
"title": "Card pages should be opened through their list rather than Tell Me",
"disposition": "candidate",
"domain": "ui"
}
]
},
{
"uid": "learn-dynamics.work-with-pages.hidden-fields",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "allowincustomizations-controls-the-add-field-pane",
"title": "AllowInCustomizations controls exposure through the Add field pane",
"disposition": "candidate",
"domain": "privacy"
}
]
},
{
"uid": "learn-dynamics.intro-development-environment.differentiate-apps",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "application-modules-depend-only-on-same-or-lower-layers",
"title": "Application modules depend only on the same or lower architectural layers",
"disposition": "candidate",
"domain": "data-modeling"
}
]
},
{
"uid": "learn-dynamics.manipulate-data-via-code.2-retrieve-data",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "legacy-find-directions-do-not-use-top-one",
"title": "Legacy Find direction calls do not have FindFirst or FindLast query behavior",
"disposition": "candidate",
"domain": "performance"
}
]
},
{
"uid": "learn-dynamics.easy-application-upgrade.2-upgrade-responsibilities",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "uninstall-preserves-extension-data",
"title": "Uninstall preserves extension data for reinstall or upgrade",
"disposition": "candidate",
"domain": "upgrade"
}
]
},
{
"uid": "learn-dynamics.easy-application-upgrade.consider-update-lifecycle",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "incompatible-cloud-extensions-have-a-remediation-window",
"title": "Incompatible cloud extensions have a fixed remediation window",
"disposition": "candidate",
"domain": "upgrade"
}
]
},
{
"uid": "learn-dynamics.easy-application-upgrade.6-answers-about-updates",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "minor-and-major-releases-update-apps-differently",
"title": "Minor and major Business Central releases update AppSource apps differently",
"disposition": "candidate",
"domain": "upgrade"
},
{
"id": "dependency-version-is-a-minimum",
"title": "An app.json dependency version is a minimum rather than an exact pin",
"disposition": "candidate",
"domain": "upgrade"
}
]
},
{
"uid": "learn-dynamics.application-types.3-library-dependency-applications",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "library-apps-install-through-the-dependency-chain",
"title": "Library apps install and update through the dependency chain",
"disposition": "candidate",
"domain": "appsource"
}
]
},
{
"uid": "learn-dynamics.business-central-interfaces.extending-interfaces",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "interfaces-can-compose-multiple-base-interfaces",
"title": "Interfaces can compose multiple base interfaces",
"disposition": "candidate",
"domain": "interfaces"
}
]
},
{
"uid": "learn-dynamics.test-automation.page-scripting-tool",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "page-scripting-cannot-drive-non-al-ui",
"title": "Page scripting cannot drive control add-ins or other non-AL UI",
"disposition": "candidate",
"domain": "testing"
}
]
},
{
"uid": "learn-dynamics.test-automation.2-test-automation-responsibilities",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "appsource-validation-must-use-the-current-build",
"title": "AppSource validation must use the current validation build",
"disposition": "candidate",
"domain": "appsource"
},
{
"id": "permission-testing-also-protects-the-essential-experience",
"title": "Permission testing must also protect the unextended Essential experience",
"disposition": "candidate",
"domain": "testing"
}
]
},
{
"uid": "learn-dynamics.test-automation.4-answers-about-testing",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "appsource-tests-run-for-every-supported-country",
"title": "AppSource tests must run separately for every supported country",
"disposition": "candidate",
"domain": "appsource"
},
{
"id": "upgrade-tests-cover-nonadjacent-versions",
"title": "Upgrade tests cover nonadjacent historical versions",
"disposition": "candidate",
"domain": "testing"
}
]
},
{
"uid": "learn-dynamics.bring-app-appsource.4-technical-validation",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "appsource-submissions-cannot-use-runtime-packages",
"title": "AppSource submissions cannot use runtime packages",
"disposition": "candidate",
"domain": "appsource"
},
{
"id": "per-tenant-and-marketplace-apps-need-distinct-identities",
"title": "Per-tenant and Marketplace variants need distinct app identities",
"disposition": "candidate",
"domain": "appsource"
},
{
"id": "profiles-are-declared-with-profile-objects",
"title": "Profiles are shipped with AL profile objects rather than table inserts",
"disposition": "candidate",
"domain": "appsource"
},
{
"id": "extension-layout-uses-relative-placement",
"title": "Extension layout uses named relative placement anchors",
"disposition": "candidate",
"domain": "breaking-changes"
}
]
},
{
"uid": "learn-dynamics.test-automation.3-documentation-examples",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "tests-use-reproducible-random-data",
"title": "Tests use reproducible randomized data instead of shared literals",
"disposition": "candidate",
"domain": "testing"
},
{
"id": "al-tests-run-through-vscode-test-explorer",
"title": "AL tests can run and debug through Visual Studio Code Test Explorer",
"disposition": "candidate",
"domain": "testing"
}
]
},
{
"uid": "learn-dynamics.easy-application-upgrade.4-manage-apps",
"reviewStatus": "in-progress",
"outcomes": [
{
"id": "manage-apps-separates-install-and-update-requirements",
"title": "Manage Apps separates missing dependencies from dependency updates",
"disposition": "candidate",
"domain": "upgrade"
}
]
}
]
}

8282
coverage/microsoft-learn-developer-catalog.json
View file

File diff suppressed because it is too large Load diff

150
evaluation/README.md
View file

@ -1,4 +1,4 @@
# AL review evaluation
# AL review and guidance evaluation
The evaluation is convention-driven. The harness discovers every `<layer>/skills/review/al-<domain>-review.md` leaf across the enabled `microsoft`, `community`, and `custom` layers. Duplicate domains resolve with `custom > community > microsoft` precedence. For each selected leaf, the harness finds paired knowledge across the same layers, applies the same precedence to duplicate article slugs, selects the first article (by filename) with both `.bad.al` and `.good.al` companions, and derives the expected positive and clean control automatically. Adding a conforming leaf requires no scoring-contract edit.
@ -55,50 +55,116 @@ This credential-free check proves every selected leaf maps to a same-named knowl
The committed gate requires full expected recall, the exact convention-derived article ID, and no findings on clean controls.
## AL development evaluation
`development-fixtures.json` defines end-to-end development requests rather than
prewritten good/bad snippets. Each case declares its execution mode, the
Business Central capabilities it exercises, the knowledge that should shape
the implementation, acceptance criteria, and the real checks an external
runner must perform.
Validate the fixture and capability manifests and prepare opaque requests:
```powershell
pwsh ./tools/Test-DevelopmentFixtures.ps1 -Root . -PrepareDirectory ./.development-evaluation
```
Each request runs `al-development` in a fresh writable AL repository. Cases
may exercise feature, bug, refactor, upgrade, or maintenance mode.
The runner compiles the generated project, runs its tests, invokes the review
quality gate, and stores the resulting implementation report using the opaque
`caseId` from its request, for example `result-case-a1b2c3d4.json`. It keeps the
generated repository available at the wrapper's `workspaceRoot` so scoring can
verify reported changed paths. Score all results with:
```powershell
pwsh ./tools/Test-DevelopmentFixtures.ps1 -Root . -ResultsDirectory ./.development-evaluation
```
The initial fixtures cover setup-backed master data, document header/line
workflows, versioned API integrations, and surgical diagnosis and repair of a
batch-processing bug, plus a rerunnable data upgrade. The capability manifest
enforces a minimum fixture-backed coverage ratio; the broader roadmap lives in
`coverage/development-capabilities.json`.
### Read-only plan guidance
## Read-only plan guidance
`development-guidance-fixtures.json` evaluates the planning interface used by
specialized orchestrators. It supplies an existing development plan and expects
a referenced set of implementation constraints without any target-repository
changes.
existing workflows. It supplies an existing plan and expects referenced
constraints without target-repository changes. The initial-plan fixture is
anonymized and synthetic: a full document with metadata and a markdown body
covering root cause, proposed fix, affected files, tests, and acceptance
criteria. It is integration-shaped input, not private consumer content, a
continuation checkpoint, or proof that any production consumer is integrated.
### Credential-free contract and scorer coverage
CI validates the manifest, prepares opaque model requests, and runs
deterministic scorer regressions with controlled reports and temporary Git
repositories. These checks cover report shape, outcomes, reference paths, and
the evaluator's pre/post read-only comparison. They do **not** run an agent,
compile AL, run Business Central tests, or establish better code authoring.
Prepare requests in a runner-owned artifact directory outside every target
workspace:
```powershell
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory ./.development-guidance-evaluation
$run = Join-Path ([IO.Path]::GetTempPath()) 'bcquality-guidance-run'
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory $run
```
An external runner stores `result-<case-id>.json` beside the generated request
and retains the clean fixture repository at `workspaceRoot`. Score the result
with `-ResultsDirectory`; the scorer verifies knowledge recall and precision
and fails if the planning pass changed the repository.
Preparation is not a model run. A scorer can validate a citation's path and
required fields, but only external agent traces and expert evaluation can
establish that the article was opened and its normative constraints faithfully
applied. Expected knowledge recall/precision is fixture-specific, not a corpus
coverage or authoring capability percentage.
`no-knowledge` means no additional applicable BCQuality constraints, not unsafe
work. It requires empty `knowledge`. Partial evaluation, failed retrieval,
unknown context, and materially unresolved applicability must remain visible
and distinct; they cannot be counted as successful enrichment merely because
the JSON is parseable.
Run the deterministic regression suite without an agent or AL environment:
```powershell
pwsh ./tools/Test-DevelopmentGuidanceEvaluator.ps1
```
### Runner-owned read-only evidence
For an external guidance run, first provision a representative, standalone Git
repository for each manifest case. The runner supplies a JSON workspace map
whose keys are the manifest IDs (not the hashed model IDs) and whose values are
absolute workspace roots. It may pass `-WorkspaceMapPath` during preparation
to bind the generated requests to those roots. The model must not select its
own workspace for scoring.
Capture evidence **before** invoking the agent, with the manifest, workspace
map, and source checkout already finalized:
```powershell
# Runner-selected paths, all outside the targets and BCQuality checkout.
$map = Join-Path $evidenceDirectory 'workspace-map.json'
$baseline = Join-Path $evidenceDirectory 'baseline.json'
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . `
-CaptureBaseline -WorkspaceMapPath $map -BaselinePath $baseline
# Retain the printed SHA256 in runner-only state BEFORE agent invocation.
# After the external agent writes result-case-<hash>.json files:
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . `
-ResultsDirectory $resultsDirectory -BaselinePath $baseline `
-BaselineSha256 $preRunDigest
```
`$evidenceDirectory`, `$resultsDirectory`, and `$preRunDigest` are supplied by
the runner; the digest must not be recomputed from potentially modified evidence
after the agent runs. Protect the baseline, digest, evaluator, and invocation
from agent changes. Capture refuses to overwrite an existing baseline. Results
contain only `caseId` and `guidanceReport`; a legacy `workspaceRoot`, if present,
must agree with the independently captured binding and never overrides it.
Missing baselines or digests, malformed reports, and escaped reference paths
fail scoring.
The comparison checks target identity, Git HEAD, refs and index, filesystem
content and stable metadata, including tracked, untracked, ignored files and
empty directories. Committing edits or making an empty commit does not evade
the check. It also compares the actual knowledge checkout and manifest identity.
Targets must have internal Git storage; linked target worktrees, submodules,
sparse checkouts, links/junctions/reparse points, hard links, and alternate data
streams are unsupported and rejected rather than silently excluded. A linked
**knowledge** checkout is supported with its Git storage identity recorded.
Use quiescent, isolated repositories; concurrent changes also fail the gate.
This is before/after evidence, not an OS sandbox or a complete write monitor.
It cannot prove that no transient write was reverted, that articles were opened,
or that constraints are semantically faithful. Reports and generated artifacts
must stay outside all target workspaces and the knowledge checkout. The
regression suite creates and removes its own uniquely named fixture directory;
it does not run against or clean a caller's target.
### External agent/runtime pilot (follow-up)
Consumer uptake and a real before-authoring pilot are not implemented by these
fixtures. The consumer must normalize its normal initial plan, persist guidance
after state initialization, inject it into existing phases, re-enrich on
material changes, and run an independent final review. See
[the integration boundary](../agent-consumption.md#repository-specific-development-orchestrators).
Before claiming improved repairs, run an independent pinned baseline without
enrichment and a matched enriched run. Hold starting code, task, model, tools,
runtime, and gates constant; record actual immutable BCQuality checkout and
policy identities rather than trusting a configured ref. Use that same
recorded checkout for enrichment and final review. Retain external logs,
article-read traces, resulting diffs, compile/test outcomes, and independent
review evidence, including failures, no-knowledge, partial, and unresolved
results. No compile/run or authoring-quality claim follows from the
credential-free checks above.

237
evaluation/development-fixtures.json
View file

@ -1,237 +0,0 @@
{
"version": 1,
"skill": "microsoft/skills/development/al-development.md",
"maximumReviewRounds": 3,
"minimumKnowledgeRecall": 1.0,
"minimumKnowledgePrecision": 0.5,
"cases": [
{
"id": "setup-backed-master-data",
"title": "Build setup-backed loyalty member master data",
"capabilities": [
"setup-and-master-data"
],
"expectedKind": "feature",
"development-request": {
"kind": "feature",
"description": "Add a Loyalty Member feature to an existing Business Central AL app. Administrators configure the member number series on a singleton setup card. Users create members through list and card pages, the table assigns numbers, and blocked members cannot be selected by consuming records. Include least-privilege permission sets and automated tests.",
"acceptance-criteria": [
"The setup is a blank-key singleton surfaced by a Card page.",
"Member numbers use the current No. Series codeunit and support manual numbers according to setup.",
"Blocked validation occurs where a member is consumed, not only on the member table.",
"The feature includes assignable least-privilege permissions and automated tests."
]
},
"context": {
"technologies": [
"al"
],
"countries": [
"w1"
],
"application-area": [
"all"
]
},
"requiredKnowledge": [
"microsoft/knowledge/data-modeling/setup-table-is-a-singleton.md",
"microsoft/knowledge/data-modeling/master-table-no-from-number-series-in-oninsert.md",
"microsoft/knowledge/data-modeling/check-blocked-in-referencing-code-not-in-master.md",
"microsoft/knowledge/security/permission-set-avoid-wildcard-grants.md",
"microsoft/knowledge/testing/use-library-codeunits-for-test-fixtures.md"
],
"optionalKnowledge": [
"microsoft/knowledge/data-modeling/use-no-series-codeunit-not-noseriesmanagement.md",
"microsoft/knowledge/security/compose-permission-sets-with-included-sets.md",
"microsoft/knowledge/style/applicationarea-required-on-page-controls.md",
"microsoft/knowledge/style/tooltip-required-on-page-fields.md",
"microsoft/knowledge/ui/showmandatory-on-code-required-page-fields.md"
],
"requiredChecks": [
"compile",
"tests",
"review"
]
},
{
"id": "document-header-and-lines",
"title": "Build a document header and lines workflow",
"capabilities": [
"document-workflows"
],
"expectedKind": "feature",
"development-request": {
"kind": "feature",
"description": "Implement a Service Quote feature with a header, lines, document page, number series, posting and document dates, calculated totals, and tests. Line edits must refresh the total shown on the header. Structure initialization so API, UI, and test creation paths behave consistently.",
"acceptance-criteria": [
"The header assigns its number before InitRecord establishes document defaults.",
"The document page links lines correctly and refreshes parent totals after edits.",
"Tests exercise creation outside the UI as well as the document-page behavior.",
"The implementation contains no obsolete NoSeriesManagement dependency."
]
},
"context": {
"technologies": [
"al"
],
"countries": [
"w1"
],
"application-area": [
"service"
]
},
"requiredKnowledge": [
"microsoft/knowledge/data-modeling/initialize-document-defaults-in-initrecord.md",
"microsoft/knowledge/data-modeling/use-no-series-codeunit-not-noseriesmanagement.md",
"microsoft/knowledge/ui/updatepropagation-both-refreshes-main-page.md",
"microsoft/knowledge/testing/use-library-codeunits-for-test-fixtures.md"
],
"optionalKnowledge": [
"microsoft/knowledge/events/publish-thin-onbefore-onafter-integration-events.md",
"microsoft/knowledge/style/applicationarea-required-on-page-controls.md",
"microsoft/knowledge/style/tooltip-required-on-page-fields.md"
],
"requiredChecks": [
"compile",
"tests",
"review"
]
},
{
"id": "versioned-master-data-api",
"title": "Expose master data through a versioned API",
"capabilities": [
"api-integrations"
],
"expectedKind": "feature",
"development-request": {
"kind": "auto",
"plan": "Expose Loyalty Member master data through a Business Central API page. Use a stable v1.0 contract, address records by SystemId, support insert and update, use conventional entity naming, and include permissions and automated API-oriented tests.",
"acceptance-criteria": [
"The API declares all routing properties and a stable APIVersion.",
"ODataKeyFields uses SystemId and the exposed SystemId field is not editable.",
"EntityName is singular, EntitySetName is plural, and both are lower camel case.",
"The API is covered by least-privilege permissions and automated tests."
]
},
"context": {
"technologies": [
"al"
],
"countries": [
"w1"
],
"application-area": [
"all"
]
},
"requiredKnowledge": [
"microsoft/knowledge/web-services/set-required-api-page-properties.md",
"microsoft/knowledge/web-services/expose-systemid-as-the-api-key.md",
"microsoft/knowledge/style/api-page-delayedinsert-true.md",
"microsoft/knowledge/style/api-page-entity-naming-singular-plural.md",
"microsoft/knowledge/security/permission-set-avoid-wildcard-grants.md"
],
"optionalKnowledge": [
"microsoft/knowledge/style/api-page-camelcase-properties.md",
"microsoft/knowledge/web-services/version-apis-by-adding-not-mutating-published-versions.md"
],
"requiredChecks": [
"compile",
"tests",
"review"
]
},
{
"id": "fix-filtered-batch-processing",
"title": "Fix a batch routine that processes only one record",
"capabilities": [
"bug-diagnosis-and-fix"
],
"expectedKind": "bug",
"development-request": {
"kind": "bug",
"description": "Users report that an existing filtered batch routine updates only the first matching record. Reproduce the defect, identify why iteration stops, make the smallest safe correction, and add a regression test that selects multiple records and proves every selected record is processed.",
"acceptance-criteria": [
"The defect is reproduced or demonstrated by a failing regression test before the fix.",
"The root cause is corrected without widening the supplied record filters.",
"The routine processes every selected record with the appropriate update locking behavior.",
"A regression test covers more than one selected record."
]
},
"context": {
"technologies": [
"al"
],
"countries": [
"w1"
],
"application-area": [
"all"
]
},
"requiredKnowledge": [
"microsoft/knowledge/performance/pair-findset-with-next-loop.md",
"microsoft/knowledge/testing/use-library-codeunits-for-test-fixtures.md"
],
"optionalKnowledge": [
"microsoft/knowledge/performance/findset-true-applies-updlock-on-read.md",
"microsoft/knowledge/performance/pass-var-record-to-preserve-partial-load-enumerator.md"
],
"requiredChecks": [
"compile",
"tests",
"review"
]
},
{
"id": "versioned-data-upgrade",
"title": "Implement a rerunnable data upgrade",
"capabilities": [
"install-and-upgrade"
],
"expectedKind": "upgrade",
"development-request": {
"kind": "upgrade",
"description": "Upgrade an existing app from a text-based Customer Tier field to a new enum-backed Tier field. Preserve existing customer data, support tenants that skip intermediate app versions, keep fresh installation separate from migration, and add upgrade tests.",
"acceptance-criteria": [
"Existing tier values are migrated without running field validation triggers.",
"The migration is guarded by an upgrade tag and is safe when the upgrade runs again.",
"Check and validation triggers do not write data.",
"Fresh installation does not run version-upgrade migration.",
"Tests cover more than one historical source data version."
]
},
"context": {
"technologies": [
"al"
],
"countries": [
"w1"
],
"application-area": [
"all"
]
},
"requiredKnowledge": [
"microsoft/knowledge/upgrade/appversion-meaning-depends-on-execution-context.md",
"microsoft/knowledge/upgrade/install-and-upgrade-codeunits-have-no-order.md",
"microsoft/knowledge/upgrade/use-upgrade-tags-not-version-checks.md",
"microsoft/knowledge/upgrade/check-only-triggers-do-not-migrate-data.md",
"microsoft/knowledge/upgrade/install-code-does-not-run-on-version-upgrade.md"
],
"optionalKnowledge": [
"microsoft/knowledge/upgrade/datatransfer-skips-triggers-and-subscribers.md",
"microsoft/knowledge/upgrade/datatransfer-for-bulk-init.md",
"microsoft/knowledge/upgrade/register-upgrade-tags-with-subscribers.md",
"microsoft/knowledge/testing/transactionmodel-attribute-governs-test-transactions.md"
],
"requiredChecks": [
"compile",
"tests",
"review"
]
}
]
}

133
evaluation/development-guidance-fixtures.json
View file

@ -5,36 +5,18 @@
"minimumKnowledgePrecision": 0.67,
"cases": [
{
"id": "bcapps-filtered-batch-bug-plan",
"title": "Select guidance from a BCFIX-HANDOFF v1 payload",
"id": "synthetic-normal-initial-plan",
"title": "Anonymized normal initial-plan consumer boundary",
"evidenceType": "integration-shaped-synthetic",
"boundary": "A synthetic consumer produces metadata plus a markdown plan body, serializes the full document, and passes that document as the generic existing development-plan. This is not an external real pilot and contains no consumer workflow-state schema.",
"expectedKind": "bug",
"development-plan": {
"format": "BCFIX-HANDOFF",
"version": 1,
"issue": 4312,
"phase": "implement",
"status": "paused",
"baton": 2,
"rootCause": "The routine calls FindFirst and updates the current record without entering an enumerator loop, so only the first record in the supplied filtered set is modified.",
"harnessMap": {
"testCodeunit": "Update Selected Entries Tests",
"libraries": [
"Library - Random"
],
"pages": [],
"handlers": []
},
"iterationsUsed": 1,
"filesCommitted": [
"test/Batch/UpdateSelectedEntries.Codeunit.al"
],
"lastTestResult": "1 failing, 4 passing; the red test shows only the first of three selected records is updated.",
"deadEnds": [
"Changing the page selection did not help because the codeunit discarded the supplied enumerator."
],
"nextStep": "Replace the single-record read with an update-safe FindSet/Next loop that preserves the supplied filters, then rerun the red test."
},
"expectedOutcome": "completed",
"expectedUnknown": [],
"requiresUnresolved": false,
"requiresMaterialUnresolved": false,
"development-plan": "{\"metadata\":{\"kind\":\"bug\",\"request\":\"Update every entry in the supplied filtered record set while preserving the caller's selection.\",\"origin\":\"anonymized synthetic initial plan\"},\"body\":\"## Root cause and design\\nThe routine reads and updates only the first record rather than iterating the supplied filtered set. Preserve the supplied filters and update each selected row.\\n\\n## Proposed fix\\nUse an update-safe FindSet/Next loop with an explicit update on each selected entry.\\n\\n## Affected files\\n- src/Batch/UpdateSelectedEntries.Codeunit.al\\n- test/Batch/UpdateSelectedEntriesTests.Codeunit.al\\n\\n## Test strategy\\nUse existing AL test library codeunits to arrange three selected rows and an excluded row. Assert all selected rows are updated and the excluded row is unchanged. This is a proposed test, not a reported result.\\n\\n## Acceptance criteria\\n- Every selected entry is updated exactly once.\\n- The supplied filters remain effective.\\n- No excluded entry changes.\\n- Empty selections cause no changes.\\n\"}",
"context": {
"bc-version": "28",
"technologies": [
"al"
],
@ -43,7 +25,8 @@
],
"application-area": [
"all"
]
],
"unknown": []
},
"requiredKnowledge": [
"microsoft/knowledge/performance/pair-findset-with-next-loop.md",
@ -58,6 +41,10 @@
"id": "versioned-upgrade-plan-guidance",
"title": "Select guidance for a versioned data upgrade",
"expectedKind": "upgrade",
"expectedOutcome": "completed",
"expectedUnknown": [],
"requiresUnresolved": false,
"requiresMaterialUnresolved": false,
"development-plan": {
"kind": "upgrade",
"request": "Migrate existing customer tier text values to a new enum field in an app upgrade.",
@ -78,6 +65,7 @@
]
},
"context": {
"bc-version": "28",
"technologies": [
"al"
],
@ -86,7 +74,8 @@
],
"application-area": [
"all"
]
],
"unknown": []
},
"requiredKnowledge": [
"microsoft/knowledge/upgrade/use-upgrade-tags-not-version-checks.md",
@ -97,6 +86,90 @@
"microsoft/knowledge/upgrade/datatransfer-skips-triggers-and-subscribers.md",
"microsoft/knowledge/upgrade/appversion-meaning-depends-on-execution-context.md"
]
},
{
"id": "no-additional-knowledge",
"title": "Honest empty enrichment does not prohibit ordinary work",
"expectedKind": "maintenance",
"expectedOutcome": "no-knowledge",
"expectedUnknown": [],
"requiresUnresolved": false,
"requiresMaterialUnresolved": false,
"development-plan": {
"kind": "maintenance",
"request": "Correct a spelling error in an existing internal explanatory comment in an AL procedure. Do not change the explanation, executable code, UI captions, schema, diagnostic text, configuration or behavior.",
"affected-files": ["src/Batch/EntryProcessor.Codeunit.al"],
"proposed-changes": ["Replace the misspelled word in the existing comment; add no new advice."],
"test-strategy": "Inspect the diff to confirm that only the comment spelling changes.",
"acceptance-criteria": ["Only the intended comment spelling changes; executable AL remains identical."]
},
"context": {
"bc-version": "28",
"technologies": ["al"],
"countries": ["w1"],
"application-area": ["all"],
"unknown": []
},
"requiredKnowledge": [],
"optionalKnowledge": []
},
{
"id": "unknown-material-version",
"title": "Materially unresolved version-sensitive guidance stays partial",
"expectedKind": "feature",
"expectedOutcome": "partial",
"expectedUnknown": ["bc-version"],
"requiresUnresolved": true,
"requiresMaterialUnresolved": true,
"development-plan": {
"kind": "feature",
"request": "Add an expensive Sum FlowField as the source of a usually-hidden page control. The proposed design relies on visibility suppressing calculation.",
"affected-files": ["src/Pages/EntryOverview.Page.al"],
"proposed-changes": ["Bind the page control directly to the FlowField and set Visible to a conditional expression."],
"test-strategy": "Verify aggregate queries are not executed while the control is hidden.",
"acceptance-criteria": ["Hidden controls do not cause expensive aggregate queries."],
"unknown": ["The deployment BC version and visible-only calculation feature state cannot be established from this fixture. Do not invent either."]
},
"context": {
"bc-version": "unknown",
"technologies": ["al"],
"countries": ["w1"],
"application-area": ["all"],
"unknown": ["bc-version"]
},
"requiredKnowledge": [
"microsoft/knowledge/performance/hidden-flowfields-still-calculate-before-bc26-opt-in.md"
],
"optionalKnowledge": []
},
{
"id": "partial-plan-decision",
"title": "Known platform context does not resolve an incomplete plan decision",
"expectedKind": "refactor",
"expectedOutcome": "partial",
"expectedUnknown": [],
"requiresUnresolved": true,
"requiresMaterialUnresolved": true,
"development-plan": {
"kind": "refactor",
"request": "Refactor a record read helper currently using FindFirst followed by Next. The caller contract does not establish whether to return one row or enumerate the entire filtered set.",
"affected-files": ["src/Queries/EntryReader.Codeunit.al"],
"proposed-changes": ["Choose the read method consistent with the intended cardinality after that decision is clarified."],
"test-strategy": "Add cardinality assertions after the caller contract is decided.",
"acceptance-criteria": ["The method and enumeration agree with the clarified caller contract."],
"unknown": ["Single-record versus multi-record caller intent remains materially unresolved."]
},
"context": {
"bc-version": "28",
"technologies": ["al"],
"countries": ["w1"],
"application-area": ["all"],
"unknown": []
},
"requiredKnowledge": [
"microsoft/knowledge/performance/pair-findset-with-next-loop.md"
],
"optionalKnowledge": []
}
]
}

11
microsoft/skills/development/al-development-plan.md
View file

@ -18,9 +18,11 @@ Selects the BCQuality knowledge that should constrain an existing AL development
Both a readable `repository` and a non-empty `development-plan` are required. The plan may be structured data or text, but it must identify the intended change. Return `not-applicable` without changing files when either input is absent or the repository is not an AL project.
The caller supplies its existing plan, not a request to generate one. Consumer-specific formats must be normalized by the consumer before invocation. This skill does not interpret issue records, continuation markers, batons, retries, or workflow state. A serialized document containing plan metadata and a markdown body is acceptable when it states the intended change, affected surfaces, proposed approach, test strategy, and acceptance criteria. Missing details remain unknown; do not invent them.
## Source
Read the BCQuality knowledge index once. Use entries from every enabled layer and domain. The index supplies candidate paths, applicability dimensions, keywords, titles, and descriptions; it never substitutes for opening selected articles in full.
Read the BCQuality knowledge index once, using the external path supplied by Entry when present. If no index is available, use READ's path-based discovery across enabled layers; inability to read the corpus is `failed`, not `no-knowledge`. Use entries from every enabled layer and domain. The index supplies candidate paths, applicability dimensions, keywords, titles, and descriptions; it never substitutes for opening selected articles in full.
Inspect the target repository read-only for `app.json`, affected files and symbols named by the plan, relevant tests, permission sets, dependencies, target/runtime versions, countries, application areas, and repository conventions. Do not create scratch or generated files inside the target repository.
@ -37,8 +39,7 @@ When a dimension cannot be resolved, retain conditionally applicable candidates
## Worklist
1. Normalize the plan into: request summary, development kind, assumptions, root cause or design intent, affected files and symbols, proposed changes, test strategy, and acceptance criteria. When the plan has no normalized kind, apply the same categories as `al-development`: new or expanded behavior is `feature`, a defect correction is `bug`, behavior-preserving restructuring is `refactor`, migration is `upgrade`, and other bounded work is `maintenance`. A repository-specific additive event or extensibility request maps to `feature`; retain its original work-item type in the request summary. Do not redesign the repository-specific workflow.
- For a `BCFIX-HANDOFF` v1 payload, map `rootCause` to root cause, `harnessMap` to test context, `filesCommitted` to affected files, `lastTestResult` to existing test evidence, `deadEnds` to rejected approaches, and `nextStep` to the immediate proposed change. Preserve `issue`, `phase`, `status`, `baton`, and `iterationsUsed` as workflow context only; they do not create Business Central constraints. A handoff with a non-empty root cause is `bug` unless the surrounding plan identifies an additive Event Request, which maps to `feature`.
1. Read the supplied plan for its request summary, development kind, assumptions, root cause or design intent, affected files and symbols, proposed changes, test strategy, and acceptance criteria. Preserve its intent; do not generate a replacement plan. If kind is not explicit, classify the stated intent: new or expanded behavior is `feature`, a defect correction is `bug`, behavior-preserving restructuring is `refactor`, migration is `upgrade`, and other bounded work is `maintenance`. Do not redesign the consumer's workflow.
2. Build retrieval vocabulary from the plan and confirmed repository symbols. Give exact object types, properties, methods, analyzers, errors, and affected domains more weight than broad business nouns.
3. Search the index in separate passes:
- data ownership, keys, setup, numbering, validation, transactions, and upgrade;
@ -66,4 +67,6 @@ Do not change the target repository. Before emitting, verify every knowledge and
## Output
Return one `development-guidance-report` conforming to DO. `completed` requires that every selected article was opened and faithfully converted into constraints. `no-knowledge` is valid when the plan is applicable but BCQuality contains no relevant article. `partial` names every unevaluated candidate or unresolved applicability gap.
Return one `development-guidance-report` conforming to DO. `completed` requires that every selected article was opened and faithfully converted into constraints, with no material unresolved applicability. `no-knowledge` means there are no additional applicable BCQuality constraints for this plan; emit empty `knowledge`. It does not mean the work is unsafe or unimplementable, and the consumer can proceed under its ordinary gates. Never add generic or filler guidance to avoid this outcome.
Return `partial` for incomplete evaluation or materially unresolved conditional guidance, naming every gap. Failed retrieval or reference-integrity checks are `failed`, never `no-knowledge`. Consumers own handling of partial, failed, and unresolved results, including clarification and re-enrichment; this read-only interface does not define a universal implementation gate.

98
microsoft/skills/development/al-development.md
View file

@ -1,98 +0,0 @@
---
kind: action-skill
id: al-development
version: 1
title: AL development
description: Implements Business Central AL features, bug fixes, refactors, upgrades, and maintenance changes using BCQuality knowledge.
inputs: [development-request, repository]
outputs: [implementation-report]
bc-version: [all]
technologies: [al]
countries: [w1]
application-area: [all]
guidance-skill: microsoft/skills/development/al-development-plan.md
quality-skill: microsoft/skills/review/al-code-review.md
quality-round-limit: 3
---
# AL development
Implements a Business Central change in an existing AL repository. Feature work, bug fixing, refactoring, upgrades, and maintenance share one public contract and one quality pipeline; their different investigation disciplines are execution modes within this skill.
Both a writable `repository` and a `development-request` are required. A structured request has this shape:
```yaml
development-request:
kind: auto # feature | bug | refactor | upgrade | maintenance
description: string # optional when plan states the requested outcome
plan: string # optional
acceptance-criteria: [string] # optional
```
A plain-text request is normalized to `kind: auto` with the text as `description`. A plan-only request is valid when the plan states the requested outcome. Return `not-applicable` without changing files when either input is absent, both description and plan are empty, or the repository is not an AL project.
## Source
Read the frontmatter `guidance-skill`; it owns BCQuality discovery and returns the knowledge constraints for the implementation plan. Inspect the target repository for `app.json`, existing objects, tests, permission sets, analyzers, build scripts, naming and object-ID conventions, dependencies, target/runtime versions, localization layout, and uncommitted user changes. For bugs, refactors, and upgrades, inspect enough history and surrounding code to establish the behavior being changed.
## Relevance
Resolve and pass this context to the guidance-skill:
- `bc-version` from the target application's platform/application/runtime settings or supplied context. For an upgrade, distinguish source and target versions.
- `technologies: [al]`, plus any additional technology actually required by the request.
- `countries` from `app.json`, workspace configuration, or supplied context.
- `application-area` from the request and affected objects.
Record unresolved dimensions in the development plan rather than silently substituting broad values. The guidance-skill applies READ's matching semantics and returns any conditional applicability in its report.
## Worklist
1. Normalize the request, deriving a concise description from a plan-only input, and classify `kind: auto` as:
- `feature` for new or intentionally expanded behavior;
- `bug` for observed behavior that contradicts an expected result;
- `refactor` for structural change with no intended behavior change;
- `upgrade` for schema, data, dependency, runtime, or application-version migration;
- `maintenance` for bounded development work that fits none of the above.
Preserve an explicit valid kind. When repository evidence conflicts with it, record the mismatch and ask for clarification before changing files rather than silently switching disciplines.
2. Establish the mode-specific implementation contract:
- **Feature:** define user-visible behavior and cover data lifecycle, UI/API, permissions, extensibility, upgrade impact, telemetry, and tests where applicable.
- **Bug:** state expected versus actual behavior, reproduce or otherwise prove the defect, trace the root cause, and define a regression test that fails for that cause.
- **Refactor:** identify the behavior and public contracts that must remain invariant, plus the checks that establish a before/after baseline.
- **Upgrade:** identify source and target states, data migration, compatibility, idempotency, and validation requirements.
- **Maintenance:** define the bounded outcome and the behavior that must not change.
3. Treat a supplied plan as an input constraint, not as proof. Reconcile it with repository reality and BCQuality; preserve its intent, correct unsafe assumptions, and record consequential deviations.
4. Discover existing implementation patterns and reusable objects before proposing new ones. Preserve repository conventions and current user changes.
5. Materialize a `development-plan` containing the classified kind, request, assumptions, affected files and symbols, design or root cause, proposed changes, validation strategy, and acceptance criteria.
6. Invoke the frontmatter `guidance-skill` with that plan, the repository, and the resolved context. It performs Source, Relevance, and knowledge worklisting independently and read-only.
7. Require a complete guidance result before editing product code:
- `completed` — use every returned constraint and validation consideration.
- `no-knowledge` — intentionally refuse to implement: return `no-knowledge` with no request changes, set `outcome-reason` to `No applicable BCQuality knowledge was found for this development plan.`, and add a `remaining` entry directing the caller to use a repository-specific workflow/general coding agent or contribute the missing BC-specific knowledge.
- `not-applicable`, `partial`, or `failed` — return the corresponding non-completed outcome without editing product code; preserve its reason in `remaining`.
8. Copy the guidance report's selected paths into the eventual implementation report only when the corresponding constraint materially shaped the implementation. Carry its suppression records forward.
## Action
1. Record the starting working-tree state so unrelated changes are preserved and excluded from `changes`.
2. Apply the execution mode:
- **Feature:** implement the smallest complete vertical slice; do not leave placeholder surfaces.
- **Bug:** reproduce first when feasible, fix the root cause rather than the symptom, keep the patch surgical, and add a regression test.
- **Refactor:** capture a behavioral baseline, avoid unrelated behavior changes, and prove the declared invariants afterward.
- **Upgrade:** make migrations rerunnable where required, preserve data and compatibility, and validate both upgraded and fresh-install paths when applicable.
- **Maintenance:** make only the bounded requested change and preserve surrounding behavior.
3. Produce a coherent design that satisfies the implementation contract and every constraint returned by the guidance-skill. Reuse existing abstractions and object ranges. Do not hard-code a Business Central fact in this skill or invent a rule absent from both the repository and reliable platform knowledge.
4. Implement the request end to end. Include all surfaces required by the mode, acceptance criteria, and repository conventions. Do not create success-shaped stubs.
5. Treat the guidance report as design constraints throughout implementation. Adapt its referenced companion samples to the target codebase; never copy demonstration IDs or names blindly.
6. Run the smallest existing build, analyzer, and test commands that cover the change. Fix failures caused by the implementation. Record every command and real outcome in `validation`; unavailable checks are `not-run`, never `passed`.
7. Invoke the frontmatter `quality-skill` against the final implementation diff, bounded by `quality-round-limit`:
- Record every invocation in `review-rounds`, including its gating `blocker` and `major` IDs.
- When no gating finding remains, mark the round `clean` and stop.
- Otherwise fix every justified, safely actionable gating finding, rerun affected validation, mark the round `fixing`, and start the next review round.
- Stop early as `stalled` when the gating ID set is unchanged from the preceding round, no gating finding can be fixed safely, or validation cannot be restored.
- When the final allowed round still has gating findings, mark it `limit-reached`.
Preserve the last complete findings-report in `review` and add a `validation` entry with `id: "review"`. A `stalled` or `limit-reached` loop returns `partial` with the unresolved gating findings in `remaining`. If review is disabled or unavailable, record `not-run` and return `partial`.
8. Verify the persisted files against the implementation contract, acceptance criteria, and mode-specific evidence. If behavior, validation, guidance, or review remains incomplete, return `partial` and list the exact gap in `remaining`.
## Output
Return one `implementation-report` conforming to DO. Set `plan.kind` to the classified execution mode. `knowledge` lists only articles opened in full and materially used. `changes` lists only files changed by this skill. `completed` requires a persisted implementation, passing required validation, and no unresolved `blocker` or `major` finding in `review`. `no-knowledge` is a visible coverage decision, not an error or silent fallback.

4
plugin.json
View file

@ -1,6 +1,6 @@
{
"name": "bcquality",
"description": "Quality skills and knowledge for Business Central development. Exposes AL development and code-review adapters backed by BCQuality's Entry protocol.",
"description": "Quality skills and knowledge for Business Central. Exposes read-only AL plan enrichment and code-review adapters backed by BCQuality's Entry protocol.",
"version": "0.3.0",
"author": {
"name": "microsoft/BCQuality",
@ -13,7 +13,7 @@
"al",
"business-central",
"code-review",
"development",
"plan-guidance",
"quality"
],
"skills": [

10
skills/README.md
View file

@ -31,7 +31,7 @@ READ and DO are read on demand — typically by the first action skill the agent
| Path | Role |
|---|---|
| [`al-code-review/SKILL.md`](al-code-review/SKILL.md) | Exposes BCQuality through the standard `SKILL.md` format when this repository is installed as a plugin. |
| [`al-development/SKILL.md`](al-development/SKILL.md) | Exposes knowledge-backed AL development through the standard `SKILL.md` format. |
| [`al-development-plan/SKILL.md`](al-development-plan/SKILL.md) | Enriches an existing AL plan read-only through the standard `SKILL.md` format; does not generate a plan or implement code. |
Each adapter is deliberately thin. It translates the caller's request into an
Entry task context, then follows Entry's dispatch without owning routing,
@ -42,15 +42,13 @@ by Entry, and should not accumulate behavior already defined by `entry.md`,
This gives the two skill formats distinct roles:
- `skills/al-code-review/SKILL.md` and
`skills/al-development/SKILL.md` are the public host integration
`skills/al-development-plan/SKILL.md` are the public host integration
surfaces for a standalone plugin installation.
- `microsoft/skills/review/al-code-review.md` is BCQuality's internal
Microsoft-layer super-skill for coordinating a broad AL review.
- `microsoft/skills/development/al-development.md` is the internal
Microsoft-layer implementation skill for all supported development modes.
- `microsoft/skills/development/al-development-plan.md` is the read-only
planning interface for repository-specific orchestrators that retain
implementation ownership.
knowledge-enrichment interface for existing plans. Consumers own format
normalization, planning, implementation, and delivery.
Each host adapter deliberately shares its name with the internal action skill
for the same operation. Their locations distinguish the host integration from

22
skills/al-development-plan/SKILL.md Normal file
View file

@ -0,0 +1,22 @@
---
name: al-development-plan
description: Enrich an existing Business Central AL development plan with read-only BCQuality knowledge constraints. Does not generate a plan or implement code.
---
# AL development plan guidance
This host-native adapter translates an existing plan and repository into Entry's task context. It does not plan new work, edit the target repository, run an implementation or review/fix loop, stage, commit, or publish changes.
## Execute
1. Resolve `PLUGIN_ROOT` to the directory containing this plugin's root `plugin.json`, two levels above this file.
2. Preserve the caller's existing `development-plan` verbatim. Consumer-specific workflow payloads must be normalized by the consumer; do not interpret workflow state or manufacture a plan from a coding request.
3. Build the task context for `PLUGIN_ROOT/skills/entry.md`:
- Set `goal` to read-only BCQuality knowledge enrichment of the supplied plan, preserving the caller's intended change.
- List only actually supplied inputs from `[development-plan, repository]` in `inputs-available`.
- Set `technologies: [al]` only when established, and pass other applicability dimensions only when supplied or reliably determined.
- Apply `BCQUALITY_ENABLED_LAYERS` and `BCQUALITY_DISABLED_SKILLS` as described in the `al-code-review` adapter.
4. Read and execute Entry, including Preparation. Resolve its paths against `PLUGIN_ROOT`, not the target repository. Keep all generated index and scratch artifacts outside the target repository. Use READ's path-based fallback if index generation is unavailable; an unreadable corpus is a failure, not empty knowledge.
5. Follow Entry's dispatch, checking its output metadata against the referenced skill before invocation. This operation accepts only `development-guidance-report`; return `failed` rather than execute another output kind. Pass the supplied existing plan and readable repository, and return the report unchanged. Return Entry's `no-match` or `failed` record unchanged when nothing is dispatched.
Missing inputs remain missing; the dispatched action skill returns `not-applicable` when it cannot proceed. A `no-knowledge` report is additive: it means no additional BCQuality constraints, not a refusal to let the consumer implement under its own gates. The consumer retains all implementation and delivery ownership.

34
skills/al-development/SKILL.md
View file

@ -1,34 +0,0 @@
---
name: al-development
description: Implement Business Central AL features, bug fixes, refactors, upgrades, and maintenance changes using BCQuality's curated platform knowledge.
---
# AL development
This is BCQuality's host-native adapter for standalone plugin installations. It translates a coding request into Entry's task context; the internal action skill owns classification, investigation, design, implementation, validation, and review policy.
When the target repository exposes a more specific local workflow for the request, such as an end-to-end bug-fix skill with its own environment and delivery gates, prefer that repository workflow unless the caller explicitly asks to use BCQuality's generic development skill.
This adapter is deliberately knowledge-backed: if BCQuality has no applicable
guidance, it returns a visible `no-knowledge` result without changing code. Use
the repository's normal coding workflow for unbacked requests, or add the
missing BC-specific knowledge before expecting this skill to implement them.
## Execute
1. Resolve `PLUGIN_ROOT` to the directory containing this plugin's root `plugin.json`. This file is `PLUGIN_ROOT/skills/al-development/SKILL.md`; when the host does not expose the plugin root, resolve it two levels above this file.
2. Build the `task-context` required by `PLUGIN_ROOT/skills/entry.md`:
- Copy the caller's request verbatim into `goal`.
- Set `inputs-available: [development-request, repository]`.
- Set `technologies: [al]` when the repository is an AL project.
- Pass `bc-version`, `countries`, and `application-area` only when supplied or reliably determined.
- Apply `BCQUALITY_ENABLED_LAYERS` and `BCQUALITY_DISABLED_SKILLS` exactly as the `al-code-review` adapter does.
3. Read and execute `PLUGIN_ROOT/skills/entry.md`, including Preparation. Resolve every path it names against `PLUGIN_ROOT`, not the user's repository. If knowledge-index generation is unavailable, use READ's path-based fallback.
4. Follow Entry's dispatch exactly. The normal result is `microsoft/skills/development/al-development.md`; do not select it directly or duplicate its behavior in this adapter.
5. Normalize the caller's input as `development-request`:
- Plain text becomes `{ kind: auto, description: <verbatim text> }`.
- Preserve an explicit `kind`, supplied plan, and `acceptance-criteria`.
- A plan-only input becomes `{ kind: auto, description: "Implement the supplied development plan.", plan: <verbatim plan> }`.
Pass the writable current workspace as `repository`, execute the dispatched skill, and return its `implementation-report` unchanged.
The adapter never edits BCQuality itself unless BCQuality is the caller's target repository. The target of implementation is the repository supplied by the caller.

112
skills/do.md
View file

@ -7,7 +7,7 @@ title: Action Skill — the template every action skill follows
# DO
An action skill is a markdown file that tells an agent how to do one concrete job — review a pull request, audit telemetry usage, generate a skeleton — using knowledge files from BCQuality. This document is the template every action skill follows. Orchestrators rely on the template to consume any skill without skill-specific parsing.
An action skill is a markdown file that tells an agent how to do one concrete job — review a pull request, audit telemetry usage, enrich an existing plan — using knowledge files from BCQuality. This document is the template every action skill follows. Orchestrators rely on the template to consume any skill without skill-specific parsing.
This contract is stable. Changes require a PR approved by both maintainers.
@ -56,20 +56,15 @@ application-area: [all]
`bc-version`, `technologies`, `countries`, `application-area` are optional filters that let an orchestrator pre-select applicable skills for a task. They follow the same semantics as in READ.
`inputs` is a list of abstract input types the skill **accepts**. Standard values: `pr-diff`, `object-list`, `file-path`, `repository`, `telemetry-query`, `development-request`, `development-plan`. Semantics are any-of: the orchestrator supplies whichever listed input types it has, and the skill is invoked with a non-empty subset of its declared `inputs`. A skill that cannot proceed with the supplied subset MUST return `outcome: "not-applicable"`.
`inputs` is a list of abstract input types the skill **accepts**. Standard values: `pr-diff`, `object-list`, `file-path`, `repository`, `telemetry-query`, `development-plan`. Semantics are any-of: the orchestrator supplies whichever listed input types it has, and the skill is invoked with a non-empty subset of its declared `inputs`. A skill that cannot proceed with the supplied subset MUST return `outcome: "not-applicable"`.
`outputs` is always a single-element list naming the output kind:
- `findings-report` — evaluates an input and reports defects or observations.
- `implementation-report` — changes a repository to satisfy a development request and reports the plan, knowledge used, changed files, validation, and post-implementation review.
- `development-guidance-report` — selects and summarizes applicable BCQuality knowledge for an existing development plan without changing the target repository.
`sub-skills` is an optional field. When present and non-empty, the skill is a **super-skill** that composes other action skills; see *Composition* below. Values are repo-relative paths to action-skill files.
`quality-skill` is optional on an action skill that emits an `implementation-report`. It names one repo-relative review action skill to run over the completed diff. It is a post-implementation gate, not a composed sub-skill: Entry does not route through it, and its final complete findings-report is returned in `review`. `quality-round-limit` is the required positive maximum number of review/fix rounds when a quality skill is declared. Consumer configuration still applies; if the named quality skill is disabled or unavailable, record its validation as `not-run` and do not claim `completed`.
`guidance-skill` is optional on an action skill that emits an `implementation-report`. It names one repo-relative read-only action skill that accepts a `development-plan` and emits a `development-guidance-report`. The implementation skill invokes it after forming its plan and before editing product code. Consumer configuration still applies; when guidance is disabled or unavailable, the implementation skill must not claim knowledge-backed development.
## Required sections
Every action skill MUST contain these five sections, in order:
@ -86,13 +81,15 @@ Every action skill MUST contain these five sections, in order:
**Relevance.** Apply frontmatter filters to the candidates. Typical filters: match `bc-version` against the target environment, match `technologies` against the languages in scope, match `countries` and `application-area` against the consuming codebase's context. The exact matching rules are defined in READ (*Frontmatter matching semantics*). Files that do not match are discarded.
**Worklist.** Narrow the relevant candidates to the subset that applies to the current task. This is where the task-specific signal enters: the objects changed in the PR, the queries being audited, the skeleton being generated. Typical moves: match `keywords` against task vocabulary, match file topics against changed objects, deduplicate by concern.
**Worklist.** Narrow the relevant candidates to the subset that applies to the current task. This is where the task-specific signal enters: the objects changed in the PR, the queries being audited, the existing plan being enriched. Typical moves: match `keywords` against task vocabulary, match file topics against changed objects, deduplicate by concern.
**Action.** Execute the skill's work against the worklist. Evaluate each item in the worklist against the task input and emit findings. The action step is where skill behavior differs; the preceding three steps are uniform.
<a id="output-contract"></a>
## Findings-report contract
Every action skill emits a single JSON document that conforms to this schema:
An action skill with `outputs: [findings-report]` emits a single JSON document that conforms to this schema:
```json
{
@ -300,96 +297,27 @@ An action skill with `outputs: [development-guidance-report]` emits one JSON doc
}
```
The skill is read-only with respect to the target repository. `completed` means every selected article was opened and converted into faithful implementation constraints. `no-knowledge` means no applicable article survived filtering; `knowledge` is empty. `partial` means candidate evaluation stopped early, with the gap named in `outcome-reason` and `unresolved`.
The skill is read-only with respect to the target repository: no edits, generated files, staging, commits, or publication. Keep index, report, and scratch artifacts outside that repository. The report is strict JSON with no surrounding commentary. The caller supplies an existing plan and repository; consumer-specific input normalization and workflow state are outside this contract.
`knowledge[].constraints` summarizes only normative `## Best Practice` and `## Anti Pattern` content from the referenced article. It must not introduce a Business Central fact absent from that article. `sample-paths` contains only sibling samples that exist and were opened. Every path is subject to the reference-integrity gate.
### Guidance outcome semantics
`validation-considerations` states evidence the implementation workflow should obtain; it does not claim that a command or test has run. `unresolved` records missing repository context or plan decisions that prevent a reliable constraint. Unknown applicability dimensions must appear in both `context.unknown` and a relevant unresolved entry.
- `completed` — evaluation finished, at least one article was selected, every selected article was opened and faithfully converted into constraints, and no materially unresolved conditional guidance remains.
- `not-applicable` — the required existing plan or readable repository is absent, or the task is outside the skill's applicability. No constraints are claimed.
- `no-knowledge` — evaluation finished and there are **no additional applicable BCQuality constraints** for this plan. `knowledge` is empty. This is not a statement that the work is unsafe or unimplementable; the consuming workflow can proceed under its ordinary gates. Do not add generic or filler articles to avoid this outcome.
- `partial` — evaluation is incomplete or conditional guidance remains materially unresolved. Name each gap in `outcome-reason` and `unresolved`; do not silently treat an unknown dimension as a match.
- `failed` — retrieval, reference integrity, or another error prevents a reliable report. Set `outcome-reason`; consumers must not treat the result as reliable constraints or as `no-knowledge`.
## Implementation-report contract
`outcome-reason` is required for `partial` and `failed`, optional otherwise. These outcomes describe enrichment only, not permission to implement or deliver. The consumer owns handling of partial, failed, and unresolved guidance, including escalation, clarification, and re-enrichment; BCQuality does not impose a universal implementation gate.
An action skill with `outputs: [implementation-report]` emits one JSON document:
### Guidance field semantics
```json
{
"skill": { "id": "string", "version": 1 },
"outcome": "completed | not-applicable | no-knowledge | partial | failed",
"outcome-reason": "string",
"summary": {
"request": "string",
"files-created": 0,
"files-modified": 0,
"files-deleted": 0
},
"plan": {
"kind": "feature | bug | refactor | upgrade | maintenance",
"assumptions": ["string"],
"decisions": ["string"],
"objects": ["string"]
},
"knowledge": [
{ "path": "string", "sha": "string", "used-for": "string" }
],
"changes": [
{
"path": "string",
"action": "created | modified | deleted",
"purpose": "string"
}
],
"validation": [
{
"id": "string",
"command": "string",
"status": "passed | failed | not-run",
"details": "string"
}
],
"review": { "...full findings-report from the post-implementation review..." : null },
"review-rounds": [
{
"round": 1,
"outcome": "clean | fixing | stalled | limit-reached",
"gating-finding-ids": ["string"]
}
],
"suppressed": [
{
"reference": { "path": "string", "sha": "string" },
"reason": "layer-precedence | configuration"
}
],
"remaining": ["string"]
}
```
`summary.request` preserves the planned intent and `kind` classifies it without replacing the plan. `candidates` and `selected` are non-negative integer counts: selected equals the number of unique `knowledge` entries and cannot exceed candidates. Counts are retrieval diagnostics, not capability or authoring-quality scores.
### Implementation outcome semantics
`knowledge[].constraints` is a non-empty list summarizing only normative `## Best Practice` and `## Anti Pattern` content from the referenced article. It must not introduce a Business Central fact absent from that article. `used-for` names the concrete plan decision. `sample-paths` contains only sibling samples that exist and were opened. All paths use forward slashes, are repository-relative, and must resolve inside the recorded BCQuality checkout; absolute paths, traversal, and links escaping that checkout are invalid. Every reference is subject to the reference-integrity gate.
- `completed` — the requested change is persisted in the repository, required validation passed, and the post-implementation review has no unresolved `blocker` or `major` finding.
- `not-applicable` — the request is not an implementation task accepted by the skill, or the supplied repository does not contain the required technology.
- `no-knowledge` — no applicable BCQuality knowledge survived filtering and the skill cannot safely implement the Business Central-specific request. No request changes are made.
- `partial` — useful changes were persisted, but part of the requested scope, validation, or post-implementation review could not be completed. `outcome-reason` and `remaining` identify the unfinished work.
- `failed` — the skill could not produce a reliable implementation. `outcome-reason` is required. Any working-tree changes remain visible and MUST still be listed in `changes`.
`validation-considerations` states evidence the implementation workflow should obtain; it does not claim that a command or test has run. `suppressed` has the same shape and semantics as in a findings-report. `unresolved` records missing repository context or plan decisions that prevent a reliable constraint. Unknown applicability dimensions must appear in both `context.unknown` and a relevant unresolved entry, explaining whether they materially affect a candidate. An unknown dimension is not itself a failure or proof that relevant knowledge exists.
### Implementation field semantics
**`summary.request`** is a concise statement of the implemented change. File counts describe only changes made by this skill; pre-existing user changes are excluded.
**`plan`** records the implementation decisions needed to understand the result. `kind` is the classified development mode: `feature`, `bug`, `refactor`, `upgrade`, or `maintenance`. `assumptions` contains only assumptions actually made; `decisions` captures consequential design choices; `objects` names the Business Central objects or other artifacts created or changed.
**`knowledge`** lists every knowledge file whose normative guidance materially shaped the implementation. `path` and optional `sha` follow the same reference format as a findings-report. `used-for` briefly names the design or implementation decision. The reference-integrity gate applies: every path must exist in the live checkout, be copied verbatim from discovery, and have been opened in full. Applicability alone is not enough to list an article.
**`changes`** is an exhaustive list of files created, modified, or deleted by the skill. Paths are repository-relative and use forward slashes. Do not include unrelated pre-existing changes.
**`validation`** records commands actually run. `passed` and `failed` require a real command result; unavailable tooling or an intentionally skipped check is `not-run` with `details`. A skill MUST NOT manufacture a successful check or replace a failed command with a success-shaped fallback.
**`review`** is optional for generic implementation skills and required when a skill's instructions mandate post-implementation review. When present, it is the complete findings-report returned by that review skill, not a rewritten summary.
**`review-rounds`** records every quality-skill invocation in order. `gating-finding-ids` contains the `blocker` and `major` IDs from that round. `clean` ends successfully; `fixing` means the skill applied justified fixes before another round; `stalled` means the same gating set persisted or no safe progress was possible; `limit-reached` means the configured round cap was exhausted. The array length MUST NOT exceed `quality-round-limit`. `stalled` or `limit-reached` requires implementation outcome `partial`, the final findings-report in `review`, and every unresolved gating item in `remaining`.
**`suppressed`** has the same semantics as in a findings-report and records applicable knowledge excluded by layer precedence or configuration.
**`remaining`** contains concrete unfinished work only. It is empty for `completed`.
Reference SHAs, when present, identify the files read; they do not prove runtime pinning on their own. The consumer records and verifies the actual immutable BCQuality checkout used for both enrichment and final review, plus its filtering policy and run provenance outside the target repository. See [agent-consumption.md](../agent-consumption.md).
## Composition (super-skills)
@ -465,4 +393,4 @@ Conforms to the DO output contract.
## How orchestrators consume output
An orchestrator invokes an action skill with an input appropriate to the skill's declared `inputs` and uses the single output kind declared in frontmatter. It maps a `findings-report` to PR comments, build gates, or IDE diagnostics; a `development-guidance-report` to constraints for a downstream implementation workflow; and an `implementation-report` to a coding-session summary, changed-file view, validation status, and any remaining work. The orchestrator MUST NOT interpret fields beyond the three schemas above.
An orchestrator invokes an action skill with an input appropriate to the skill's declared `inputs` and uses the single output kind declared in frontmatter. It maps a `findings-report` to PR comments, build gates, or IDE diagnostics, and a `development-guidance-report` to additional constraints for its existing implementation workflow. These are the two output schemas defined by this contract; the consumer retains ownership of implementation and delivery.

10
skills/entry.md
View file

@ -21,7 +21,6 @@ The agent invokes Entry with a **task context** supplied by the orchestrator:
task-context:
goal: string # free-text description of what needs doing
inputs-available: # values the orchestrator has ready to pass to a chosen skill
- development-request
- development-plan
- repository
- pr-diff
@ -38,7 +37,7 @@ task-context:
## Preparation — knowledge index
Before routing, ensure the knowledge index is current for the **live** clone. The dispatched review skills read `knowledge-index.json` (at the clone root) at their Source step instead of opening every knowledge file — see READ's [Retrieval workflow](read.md). When a consumer prunes its clone to policy *before* the agent runs, the index MUST be built over the clone as it exists now, so it lists exactly the articles that survived pruning and never an article the consumer denied:
Before routing, ensure the knowledge index is current for the **live** clone. The dispatched skills read `knowledge-index.json` (by default at the clone root) at their Source step instead of opening every knowledge file — see READ's [Retrieval workflow](read.md). When a consumer prunes its clone to policy *before* the agent runs, the index MUST be built over the clone as it exists now, so it lists exactly the articles that survived pruning and never an article the consumer denied:
- If `knowledge-index.json` is absent — or you cannot confirm it reflects the current knowledge tree — regenerate it by running, from the checkout root:
@ -48,6 +47,7 @@ Before routing, ensure the knowledge index is current for the **live** clone. Th
It defaults to indexing this checkout and writes `knowledge-index.json` at the root in well under a second. When in doubt, rebuild: a sub-second rebuild is always cheaper than a stale or over-listing index, which is a correctness risk.
- The paths above assume the checkout root is the current directory. A caller that enters Entry from elsewhere — a plugin host, whose working directory is the user's own project — MUST resolve them against the BCQuality root it already knows instead. The generator resolves its own root, so invoking it by absolute path indexes and writes the right tree.
- For read-only plan enrichment, generated artifacts MUST remain outside the target repository. When the target contains the BCQuality checkout, or that checkout is immutable, pass the generator's `-IndexPath` to an external runner-owned artifact location and supply that resolved index path to the dispatched skill. Do not regenerate inside the target or modify the immutable checkout. If generation is unavailable, use READ's path-based discovery; retrieval failure is not an empty corpus.
- Pruning is the consumer's job, not Entry's, and not every consumer does it: an installation that ships the whole tree gets no deny guarantee from this step. There, `enabled-layers` narrows discovery only, and the unlisted layers' files remain on disk.
- This is a side step. It MUST NOT change Entry's output — the dispatch record below is the only thing Entry emits, and build logs are never part of the dispatch JSON.
@ -123,11 +123,11 @@ Emit a single JSON document conforming to the output contract below. Entry does
**`dispatch[]`** — each entry names one action skill to invoke.
- `skill.path` — repo-relative, forward slashes. The agent fetches and executes the file directly from this path.
- `skill.path` — repo-relative, forward slashes, copied from discovery. Resolve it inside the live BCQuality checkout; reject absolute paths, traversal, and links escaping that checkout. The agent reads the action skill at this exact path rather than constructing a plausible filename.
- `skill.version` — copied from the dispatched skill's frontmatter so the orchestrator can detect drift between dispatch time and execution.
- `rationale` — short human-readable string, for logs and traceability.
- `inputs` — the intersection of `task-context.inputs-available` and the skill's declared `inputs`. The agent MUST pass exactly this subset when invoking the skill. Sending a strict intersection avoids accidental information leakage between skills.
- `outputs` — the dispatched skill's complete, single-element `outputs` value copied from frontmatter. This lets an orchestrator distinguish read-only `findings-report` and `development-guidance-report` work from repository-changing `implementation-report` work before invoking the skill. An orchestrator MAY require an additional write confirmation for `implementation-report`; it MUST NOT infer side effects from the skill ID or title.
- `outputs` — the dispatched skill's complete, single-element `outputs` value copied from frontmatter. This lets an orchestrator distinguish `findings-report` from read-only `development-guidance-report` before invocation. Check the declared contract and actual skill; output metadata is not a sandbox or proof of side effects. Unknown output kinds must not be silently treated as a supported report.
Ordering of `dispatch[]` is not significant.
@ -180,7 +180,7 @@ Populated example (PR review on a repo where only `al-performance-review` is ena
1. Invoke Entry with the orchestrator-supplied task context.
2. Receive the dispatch record.
3. For each entry in `dispatch[]`, inspect `outputs` before invocation, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce the declared report kind. Verify the file's frontmatter output still equals the dispatch value; return `failed` on drift rather than executing an unexpectedly mutating skill.
3. For each entry in `dispatch[]`, inspect `outputs` before invocation, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce the declared report kind. Verify the file's frontmatter output still equals the dispatch value; return `failed` on drift rather than executing a different contract.
4. Return the action-skill reports to the orchestrator. When Entry's `outcome` is `no-match` or `failed`, return the dispatch record itself so the orchestrator can log the reason.
READ and DO are the contracts that govern what the dispatched skills do. An agent that has not yet read READ and DO reads them when it executes the first dispatched skill — they are not prerequisites for invoking Entry.

496
tools/DevelopmentGuidance.Evidence.ps1 Normal file
View file

@ -0,0 +1,496 @@
# Helpers for Test-DevelopmentGuidanceFixtures.ps1 and its deterministic regressions.
function Get-GuidanceDiagnostic {
param($ErrorRecord)
if ($ErrorRecord.Exception -is [Management.Automation.RuntimeException] -and
$ErrorRecord.FullyQualifiedErrorId -eq $ErrorRecord.Exception.Message) {
return $ErrorRecord.Exception.Message
}
return 'Evidence or report could not be read safely (missing, malformed, inaccessible or unsupported state).'
}
function Assert-GuidanceObject {
param($Value, [string] $Label, [string[]] $Fields = @())
if ($Value -isnot [Collections.IDictionary]) { throw "$Label must be a JSON object." }
foreach ($field in $Fields) {
if (-not $Value.Contains($field)) { throw "$Label is missing required field '$field'." }
}
}
function Assert-GuidanceString {
param($Value, [string] $Label)
if ($Value -isnot [string] -or [string]::IsNullOrWhiteSpace($Value)) { throw "$Label must be a non-empty string." }
}
function Assert-GuidanceArray {
param($Value, [string] $Label, [switch] $Strings, [switch] $NonEmpty)
if ($Value -isnot [array]) { throw "$Label must be a JSON array." }
if ($NonEmpty -and $Value.Count -eq 0) { throw "$Label must not be empty." }
if ($Strings) { foreach ($item in $Value) { Assert-GuidanceString $item "$Label entry" } }
}
function Assert-GuidanceInteger {
param($Value, [string] $Label)
if (($Value -isnot [long] -and $Value -isnot [int] -and $Value -isnot [bigint]) -or $Value -lt 0) {
throw "$Label must be a non-negative integer."
}
}
function Read-GuidanceJson {
param([string] $Path)
function Assert-JsonMembers($Element) {
if ($Element.ValueKind -eq [Text.Json.JsonValueKind]::Object) {
$names = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
foreach ($property in $Element.EnumerateObject()) {
if (-not $names.Add($property.Name)) { throw 'JSON contains duplicate or case-ambiguous members.' }
Assert-JsonMembers $property.Value
}
} elseif ($Element.ValueKind -eq [Text.Json.JsonValueKind]::Array) {
foreach ($item in $Element.EnumerateArray()) { Assert-JsonMembers $item }
}
}
try {
if ((Get-Item -LiteralPath $Path -Force).Length -gt 32MB) { throw 'Oversized JSON.' }
$text = [IO.File]::ReadAllText($Path)
$document = [Text.Json.JsonDocument]::Parse($text)
try { Assert-JsonMembers $document.RootElement } finally { $document.Dispose() }
return ConvertFrom-Json -InputObject $text -AsHashtable -Depth 64 -NoEnumerate
} catch { throw 'Input is not readable, strict JSON with unique members.' }
}
function Get-GuidanceHash {
param([string] $Path)
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash
}
function Get-GuidanceCaseId {
param([string] $Id)
$bytes = [Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($Id))
return "case-$([Convert]::ToHexString($bytes).Substring(0, 8).ToLowerInvariant())"
}
function Test-GuidanceWithin {
param([string] $Path, [string] $Parent)
$comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal }
$prefix = $Parent.TrimEnd([IO.Path]::DirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar
return $Path.Equals($Parent, $comparison) -or $Path.StartsWith($prefix, $comparison)
}
function Assert-GuidanceItem {
param($Item)
if (($Item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $Item.LinkType -or $Item.LinkTarget) {
throw 'Links, junctions, hard links and reparse points are not supported.'
}
if (-not $Item.PSIsContainer -and $IsWindows) {
$streams = @(Get-Item -LiteralPath $Item.FullName -Stream '*' -Force -ErrorAction Stop)
if (@($streams | Where-Object Stream -ne ':$DATA').Count) {
throw 'Alternate data streams are not supported.'
}
}
}
function Get-GuidanceSafePath {
param([string] $Path, [switch] $AllowMissing, [switch] $Directory, [switch] $File)
Assert-GuidanceString $Path 'Filesystem path'
$full = [IO.Path]::GetFullPath($Path)
$driveRoot = [IO.Path]::GetPathRoot($full)
if ($IsWindows -and ($driveRoot.StartsWith('\\') -or $full.Substring($driveRoot.Length).Contains(':'))) {
throw 'Network paths and alternate stream paths are not supported.'
}
$cursor = $driveRoot
$components = $full.Substring($driveRoot.Length).Split([IO.Path]::DirectorySeparatorChar, [StringSplitOptions]::RemoveEmptyEntries)
foreach ($component in $components) {
if ($component -match '[\. ]$') { throw 'Ambiguous filesystem path components are not supported.' }
$cursor = Join-Path $cursor $component
# Get-Item sees dangling links that Test-Path may treat as missing.
$item = Get-Item -LiteralPath $cursor -Force -ErrorAction SilentlyContinue
if ($null -ne $item) { Assert-GuidanceItem $item }
elseif (-not $AllowMissing) { throw 'Required filesystem path is missing.' }
}
if ($Directory -and -not (Test-Path -LiteralPath $full -PathType Container)) { throw 'Required directory is missing.' }
if ($File -and -not (Test-Path -LiteralPath $full -PathType Leaf)) { throw 'Required file is missing.' }
return $full.TrimEnd([IO.Path]::DirectorySeparatorChar)
}
function Assert-GuidanceTree {
param([string] $Root)
$pending = [Collections.Generic.Stack[string]]::new()
$pending.Push($Root)
while ($pending.Count) {
foreach ($item in Get-ChildItem -LiteralPath $pending.Pop() -Force) {
Assert-GuidanceItem $item
if ($item.PSIsContainer) { $pending.Push($item.FullName) }
}
}
}
function Invoke-GuidanceGit {
param([string] $Root, [string[]] $Arguments, [switch] $RawOutput)
$start = [Diagnostics.ProcessStartInfo]::new('git')
$start.UseShellExecute = $false
$start.RedirectStandardOutput = $true
$start.RedirectStandardError = $true
foreach ($variable in @('GIT_DIR', 'GIT_WORK_TREE', 'GIT_COMMON_DIR', 'GIT_INDEX_FILE',
'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_CONFIG',
'GIT_CONFIG_COUNT', 'GIT_CONFIG_PARAMETERS', 'GIT_NAMESPACE')) {
$null = $start.Environment.Remove($variable)
}
$start.Environment['GIT_CONFIG_NOSYSTEM'] = '1'
$start.Environment['GIT_CONFIG_GLOBAL'] = ''
$start.Environment['GIT_TERMINAL_PROMPT'] = '0'
# In particular, do not execute a repository-supplied fsmonitor hook on reads.
foreach ($argument in (@('--no-optional-locks', '-c', 'core.fsmonitor=false', '-C', $Root) + $Arguments)) {
$start.ArgumentList.Add($argument)
}
$process = [Diagnostics.Process]::Start($start)
try {
$output = $process.StandardOutput.ReadToEndAsync()
$errors = $process.StandardError.ReadToEndAsync()
$process.WaitForExit()
$null = $errors.GetAwaiter().GetResult()
if ($process.ExitCode -ne 0) { throw 'Git evidence could not be read.' }
$text = $output.GetAwaiter().GetResult()
if ($RawOutput) { return $text }
return $text.TrimEnd("`r", "`n")
} finally { $process.Dispose() }
}
function Get-GuidanceSnapshot {
param([string] $Root, [switch] $Target)
$Root = Get-GuidanceSafePath $Root -Directory
$dotGit = Get-GuidanceSafePath (Join-Path $Root '.git')
if (Test-Path -LiteralPath $dotGit -PathType Container) {
$gitDir = $dotGit
} else {
if ($Target) { throw 'Target workspaces must be standalone repositories with internal Git storage.' }
$pointer = [IO.File]::ReadAllText($dotGit)
if ($pointer -notmatch '\Agitdir: ([^\r\n]+)\r?\n?\z') { throw 'Unsupported Git worktree pointer.' }
$gitPath = $Matches[1]
if (-not [IO.Path]::IsPathFullyQualified($gitPath)) { $gitPath = Join-Path $Root $gitPath }
$gitDir = Get-GuidanceSafePath $gitPath -Directory
}
$commonDir = $gitDir
$commonPointer = Get-GuidanceSafePath (Join-Path $gitDir 'commondir') -AllowMissing
if (Test-Path -LiteralPath $commonPointer) {
$commonPath = [IO.File]::ReadAllText($commonPointer).Trim()
if (-not [IO.Path]::IsPathFullyQualified($commonPath)) { $commonPath = Join-Path $gitDir $commonPath }
$commonDir = Get-GuidanceSafePath $commonPath -Directory
}
if ($Target -and ($gitDir -cne (Join-Path $Root '.git') -or $commonDir -cne $gitDir)) {
throw 'Target workspaces must be standalone repositories with internal Git storage.'
}
$files = [Collections.Generic.List[object]]::new()
$pending = [Collections.Generic.Stack[string]]::new()
$pending.Push($Root)
while ($pending.Count) {
foreach ($item in @(Get-ChildItem -LiteralPath $pending.Pop() -Force | Sort-Object Name -CaseSensitive)) {
Assert-GuidanceItem $item
$relative = [IO.Path]::GetRelativePath($Root, $item.FullName).Replace('\', '/')
$entry = [ordered]@{
path = $relative
kind = if ($item.PSIsContainer) { 'directory' } else { 'file' }
attributes = [int]$item.Attributes
unixMode = [int]$item.UnixFileMode
creationUtcTicks = $item.CreationTimeUtc.Ticks
}
if ($item.PSIsContainer) {
$pending.Push($item.FullName)
} else {
$entry.length = $item.Length
$entry.lastWriteUtcTicks = $item.LastWriteTimeUtc.Ticks
$entry.sha256 = Get-GuidanceHash $item.FullName
}
$files.Add($entry)
}
}
# Linked knowledge worktrees have explicitly identified Git metadata outside
# the content root. Record its meaningful configuration as well as HEAD/refs.
$gitMetadata = [ordered]@{}
foreach ($storage in @($gitDir, $commonDir) | Sort-Object -Unique) {
foreach ($name in @('HEAD', 'commondir', 'config', 'config.worktree', 'packed-refs', 'refs', 'objects', 'index', 'info\exclude', 'shallow')) {
$path = Get-GuidanceSafePath (Join-Path $storage $name) -AllowMissing
if ((Test-Path -LiteralPath $path -PathType Container) -and -not (Test-GuidanceWithin $path $Root)) {
Assert-GuidanceTree $path
}
if (Test-Path -LiteralPath $path -PathType Leaf) {
$gitMetadata[$path] = Get-GuidanceHash $path
}
if ($name -in @('config', 'config.worktree') -and (Test-Path -LiteralPath $path) -and
[IO.File]::ReadAllText($path) -match '(?im)^\s*\[include(?:If)?\b') {
throw 'External Git configuration includes are not supported.'
}
}
foreach ($name in @('objects\info\alternates', 'objects\info\http-alternates')) {
if (Test-Path -LiteralPath (Join-Path $storage $name)) { throw 'External Git object stores are not supported.' }
}
}
$top = Get-GuidanceSafePath (Invoke-GuidanceGit $Root @('rev-parse', '--show-toplevel')) -Directory
if ($top -cne $Root) { throw 'Evidence requires the exact Git worktree root, not a subdirectory.' }
$indexPath = Get-GuidanceSafePath (Join-Path $gitDir 'index') -AllowMissing
$tracked = Invoke-GuidanceGit $Root @('ls-files', '--stage')
if ($tracked -match '(?m)^160000 ') { throw 'Submodule workspaces are not supported.' }
if ((Invoke-GuidanceGit $Root @('ls-files', '-t')) -match '(?m)^S ') { throw 'Sparse workspaces are not supported.' }
$head = Invoke-GuidanceGit $Root @('rev-parse', '--verify', 'HEAD')
$headFile = Get-GuidanceSafePath (Join-Path $gitDir 'HEAD') -File
# Access times, Git status refreshes, and index-builder generatedAt are not evidence.
return [ordered]@{
version = 1
root = $Root
rootCreationUtcTicks = (Get-Item -LiteralPath $Root -Force).CreationTimeUtc.Ticks
gitDir = $gitDir
commonDir = $commonDir
gitMetadata = $gitMetadata
head = $head
headFileSha256 = Get-GuidanceHash $headFile
references = Invoke-GuidanceGit $Root @('for-each-ref', '--format=%(refname) %(objectname) %(symref)')
indexPath = $indexPath
indexSha256 = if (Test-Path -LiteralPath $indexPath) { Get-GuidanceHash $indexPath } else { $null }
files = @($files | Sort-Object { $_.path } -CaseSensitive)
}
}
function Test-GuidanceSnapshotEqual {
param($Before, $After)
return ($Before | ConvertTo-Json -Depth 64 -Compress) -ceq ($After | ConvertTo-Json -Depth 64 -Compress)
}
function Write-GuidanceNewJson {
param([string] $Path, $Value)
$parent = Split-Path -Parent $Path
[IO.Directory]::CreateDirectory($parent) | Out-Null
$bytes = [Text.Encoding]::UTF8.GetBytes(($Value | ConvertTo-Json -Depth 64))
$stream = [IO.File]::Open($Path, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
try { $stream.Write($bytes, 0, $bytes.Length) } finally { $stream.Dispose() }
}
function Resolve-GuidanceReference {
param([string] $Root, $Reference, [switch] $Knowledge, [string] $Article)
Assert-GuidanceString $Reference 'Reference path'
if ($Reference -cnotmatch '^[a-zA-Z0-9_-]+(?:/[a-zA-Z0-9_.-]+)+$' -or
@($Reference.Split('/') | Where-Object { $_ -in @('.', '..') -or $_ -match '[\. ]$' }).Count) {
throw 'Reference must be an unambiguous forward-slash repository-relative path without traversal.'
}
if ($Knowledge -and $Reference -cnotmatch '^(microsoft|community|custom)/knowledge/[a-z0-9-]+/(?:[a-z0-9-]+/)*[a-z0-9-]+\.md$') {
throw 'Knowledge references must identify actual layered knowledge articles.'
}
if ($Article) {
$stem = $Article.Substring(0, $Article.Length - 3)
if ($Reference -cnotmatch ('^' + [regex]::Escape($stem) + '\.(good|bad)\.[a-zA-Z0-9]+$')) {
throw 'Sample references must be good/bad siblings of their knowledge article.'
}
}
$cursor = $Root
foreach ($component in $Reference.Split('/')) {
$cursor = Join-Path $cursor $component
$cursor = Get-GuidanceSafePath $cursor
if ((Get-Item -LiteralPath $cursor -Force).Name -cne $component) { throw 'Reference path casing must match the actual file.' }
}
if (-not (Test-GuidanceWithin $cursor $Root) -or -not (Test-Path -LiteralPath $cursor -PathType Leaf)) {
throw 'Reference must resolve to an existing file inside the knowledge checkout.'
}
if ($Knowledge) {
$text = [IO.File]::ReadAllText($cursor)
if ($text -notmatch '(?s)^---\r?\n.*?\r?\n---' -or
$text -notmatch '(?m)^domain:\s*\S+' -or $text -notmatch '(?m)^## (Best Practice|Anti Pattern)\s*$') {
throw 'Knowledge reference does not contain a normative knowledge article.'
}
}
return $cursor
}
function Get-GuidancePlanRequest {
param($Plan)
if ($Plan -is [string]) { Assert-GuidanceString $Plan 'development-plan'; return $Plan }
Assert-GuidanceObject $Plan 'development-plan' @('request')
Assert-GuidanceString $Plan.request 'development-plan.request'
return $Plan.request
}
function Assert-GuidanceContext {
param($Context)
Assert-GuidanceObject $Context 'context' @('bc-version', 'technologies', 'countries', 'application-area', 'unknown')
Assert-GuidanceString $Context.'bc-version' 'context.bc-version'
foreach ($key in @('technologies', 'countries', 'application-area', 'unknown')) {
Assert-GuidanceArray $Context[$key] "context.$key" -Strings
if (@($Context[$key] | Sort-Object -Unique).Count -ne $Context[$key].Count) { throw "context.$key contains duplicates." }
}
foreach ($key in $Context.unknown) {
if ($key -cnotin @('bc-version', 'technologies', 'countries', 'application-area')) { throw 'context.unknown contains an invalid dimension.' }
}
if ($Context.'bc-version' -eq 'unknown' -and 'bc-version' -cnotin $Context.unknown) {
throw 'Unknown BC version must be recorded in context.unknown.'
}
foreach ($key in @('technologies', 'countries', 'application-area')) {
if ((-not $Context[$key].Count -or 'unknown' -in $Context[$key]) -and $key -cnotin $Context.unknown) {
throw 'Unavailable applicability dimensions must be recorded in context.unknown.'
}
}
}
function Assert-GuidanceManifest {
param($Manifest, [string] $Root)
Assert-GuidanceObject $Manifest 'manifest' @('version', 'minimumKnowledgeRecall', 'minimumKnowledgePrecision', 'skill', 'cases')
if ($Manifest.version -ne 1) { throw 'Unsupported guidance fixture manifest version.' }
foreach ($name in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
$value = $Manifest[$name]
if (($value -isnot [double] -and $value -isnot [long] -and $value -isnot [int] -and $value -isnot [decimal]) -or
$value -lt 0 -or $value -gt 1) { throw 'Manifest thresholds must be numbers between zero and one.' }
}
$null = Resolve-GuidanceReference $Root $Manifest.skill
Assert-GuidanceArray $Manifest.cases 'manifest.cases' -NonEmpty
$ids = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
$modelIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
foreach ($case in $Manifest.cases) {
Assert-GuidanceObject $case 'manifest case' @('id', 'expectedKind', 'expectedOutcome', 'development-plan', 'context', 'requiredKnowledge', 'optionalKnowledge', 'expectedUnknown', 'requiresUnresolved', 'requiresMaterialUnresolved')
if ($case.id -isnot [string] -or $case.id -cnotmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') { throw 'Fixture id must be kebab-case.' }
if (-not $ids.Add($case.id) -or -not $modelIds.Add((Get-GuidanceCaseId $case.id))) { throw 'Duplicate fixture or model case identity.' }
if ($case.expectedKind -cnotin @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')) { throw 'Fixture expectedKind is invalid.' }
if ($case.expectedOutcome -cnotin @('completed', 'not-applicable', 'no-knowledge', 'partial', 'failed')) { throw 'Fixture expectedOutcome is invalid.' }
Assert-GuidanceString $case.expectedKind 'fixture.expectedKind'
Assert-GuidanceString $case.expectedOutcome 'fixture.expectedOutcome'
$null = Get-GuidancePlanRequest $case.'development-plan'
Assert-GuidanceContext $case.context
foreach ($name in @('requiredKnowledge', 'optionalKnowledge', 'expectedUnknown')) {
Assert-GuidanceArray $case[$name] "fixture.$name" -Strings
}
if ($case.requiresUnresolved -isnot [bool]) { throw 'Fixture requiresUnresolved must be boolean.' }
if ($case.requiresMaterialUnresolved -isnot [bool]) { throw 'Fixture requiresMaterialUnresolved must be boolean.' }
if ($case.requiresMaterialUnresolved -and ($case.expectedOutcome -ne 'partial' -or -not $case.requiresUnresolved)) {
throw 'Materially unresolved fixtures must expect partial and unresolved evidence.'
}
foreach ($key in $case.expectedUnknown) {
if ($key -cnotin @('bc-version', 'technologies', 'countries', 'application-area')) { throw 'Fixture expectedUnknown is invalid.' }
}
$references = @($case.requiredKnowledge) + @($case.optionalKnowledge)
if (@($references | Sort-Object -Unique).Count -ne $references.Count) { throw 'Fixture knowledge references contain duplicates.' }
foreach ($reference in $references) { $null = Resolve-GuidanceReference $Root $reference -Knowledge }
if ($case.expectedOutcome -in @('no-knowledge', 'not-applicable') -and $references.Count) {
throw 'Empty-knowledge outcomes cannot require or accept knowledge.'
}
}
}
function Assert-GuidanceResult {
param($Result, $Case, $Manifest, [string] $Root, [string] $Workspace)
Assert-GuidanceObject $Result 'result' @('caseId', 'guidanceReport')
Assert-GuidanceString $Result.caseId 'result.caseId'
if ($Result.caseId -cne (Get-GuidanceCaseId $Case.id)) { throw 'Result caseId mismatch.' }
if ($Result.Contains('workspaceRoot')) {
Assert-GuidanceString $Result.workspaceRoot 'result.workspaceRoot'
if (-not [IO.Path]::IsPathFullyQualified($Result.workspaceRoot) -or
(Get-GuidanceSafePath $Result.workspaceRoot -Directory) -cne $Workspace) {
throw 'Result workspaceRoot disagrees with the runner binding.'
}
}
$report = $Result.guidanceReport
Assert-GuidanceObject $report 'guidanceReport' @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')
Assert-GuidanceObject $report.skill 'skill' @('id', 'version')
Assert-GuidanceString $report.skill.id 'skill.id'
if ($report.skill.id -cne 'al-development-plan' -or $report.skill.version -isnot [long] -or $report.skill.version -ne 1) {
throw 'Report skill identity/version is invalid.'
}
Assert-GuidanceString $report.outcome 'outcome'
if ($report.outcome -cnotin @('completed', 'not-applicable', 'no-knowledge', 'partial', 'failed')) { throw 'Report outcome enum is invalid.' }
if ($report.outcome -cne $Case.expectedOutcome) { throw 'Report outcome does not match fixture expectedOutcome.' }
if ($report.outcome -in @('partial', 'failed') -or $report.Contains('outcome-reason')) {
Assert-GuidanceString $report['outcome-reason'] 'outcome-reason'
}
Assert-GuidanceObject $report.summary 'summary' @('request', 'kind', 'candidates', 'selected')
Assert-GuidanceString $report.summary.request 'summary.request'
Assert-GuidanceString $report.summary.kind 'summary.kind'
if ($report.summary.kind -cne $Case.expectedKind) { throw 'summary.kind does not match the intended change.' }
Assert-GuidanceInteger $report.summary.candidates 'summary.candidates'
Assert-GuidanceInteger $report.summary.selected 'summary.selected'
Assert-GuidanceContext $report.context
foreach ($name in @('knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
Assert-GuidanceArray $report[$name] $name
}
Assert-GuidanceArray $report.unresolved 'unresolved' -Strings
if ($report.summary.selected -ne $report.knowledge.Count -or $report.summary.selected -gt $report.summary.candidates) {
throw 'Summary counts disagree with selected knowledge/candidates.'
}
if ($report.outcome -in @('no-knowledge', 'not-applicable') -and $report.knowledge.Count) { throw 'This outcome requires empty knowledge.' }
if ($report.outcome -eq 'completed' -and -not $report.knowledge.Count) { throw 'Completed requires selected knowledge; empty evaluation is no-knowledge.' }
if (($report.outcome -eq 'partial' -or $Case.requiresUnresolved) -and -not $report.unresolved.Count) {
throw 'Partial/incomplete evaluation must explain unresolved gaps.'
}
foreach ($dimension in $Case.expectedUnknown) {
if ($dimension -cnotin $report.context.unknown) { throw 'Expected unknown context was silently resolved.' }
}
foreach ($dimension in $report.context.unknown) {
if (-not @($report.unresolved | Where-Object { $_ -match [regex]::Escape($dimension) }).Count) {
throw 'Unknown dimensions require a corresponding unresolved explanation.'
}
}
# Free-form unresolved text has no machine-readable materiality field in DO.
# Known material fixture conditions are runner expectations, not model claims.
if ($Case.requiresMaterialUnresolved -and $report.outcome -ne 'partial') { throw 'Material unknown guidance must remain partial.' }
$used = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
foreach ($entry in $report.knowledge) {
Assert-GuidanceObject $entry 'knowledge entry' @('path', 'used-for', 'constraints', 'sample-paths')
$null = Resolve-GuidanceReference $Root $entry.path -Knowledge
if (-not $used.Add($entry.path)) { throw 'Duplicate knowledge reference.' }
Assert-GuidanceString $entry.'used-for' 'knowledge.used-for'
Assert-GuidanceArray $entry.constraints 'knowledge.constraints' -Strings -NonEmpty
Assert-GuidanceArray $entry.'sample-paths' 'knowledge.sample-paths' -Strings
if (@($entry.'sample-paths' | Sort-Object -Unique).Count -ne $entry.'sample-paths'.Count) { throw 'Duplicate sample reference.' }
foreach ($sample in $entry.'sample-paths') { $null = Resolve-GuidanceReference $Root $sample -Article $entry.path }
Assert-GuidanceReferenceSha $entry $Root
}
$validationIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
foreach ($entry in $report.'validation-considerations') {
Assert-GuidanceObject $entry 'validation consideration' @('id', 'reason', 'evidence')
foreach ($key in @('id', 'reason', 'evidence')) { Assert-GuidanceString $entry[$key] "validation-considerations.$key" }
if (-not $validationIds.Add($entry.id)) { throw 'Duplicate validation consideration id.' }
}
$suppressed = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
foreach ($entry in $report.suppressed) {
Assert-GuidanceObject $entry 'suppressed entry' @('reference', 'reason')
Assert-GuidanceObject $entry.reference 'suppressed.reference' @('path')
$null = Resolve-GuidanceReference $Root $entry.reference.path -Knowledge
Assert-GuidanceReferenceSha $entry.reference $Root
Assert-GuidanceString $entry.reason 'suppression reason'
if ($entry.reason -cnotin @('layer-precedence', 'configuration')) { throw 'Suppression reason is invalid.' }
if (-not $suppressed.Add($entry.reference.path) -or $used.Contains($entry.reference.path)) { throw 'Duplicate or selected suppressed reference.' }
}
$matched = @($Case.requiredKnowledge | Where-Object { $used.Contains($_) }).Count
$recall = if ($Case.requiredKnowledge.Count) { $matched / $Case.requiredKnowledge.Count } else { 1.0 }
$accepted = @($Case.requiredKnowledge) + @($Case.optionalKnowledge)
$acceptedCount = @($used | Where-Object { $_ -cin $accepted }).Count
$precision = if ($used.Count) { $acceptedCount / $used.Count } elseif (-not $Case.requiredKnowledge.Count) { 1.0 } else { 0.0 }
if ($recall -lt $Manifest.minimumKnowledgeRecall) { throw 'Knowledge recall is below the manifest threshold.' }
if ($precision -lt $Manifest.minimumKnowledgePrecision) { throw 'Knowledge precision is below the manifest threshold.' }
}
function Assert-GuidanceReferenceSha {
param($Entry, [string] $Root)
if ($Entry.Contains('sha')) {
if ($Entry.sha -isnot [string] -or $Entry.sha -cnotmatch '^([0-9a-fA-F]{40}|[0-9a-fA-F]{64})$') {
throw 'Reference SHA must be a full commit object id.'
}
$head = Invoke-GuidanceGit $Root @('rev-parse', '--verify', 'HEAD')
if ($Entry.sha -ne $head) { throw 'Reference SHA does not identify the recorded live checkout.' }
$committed = Invoke-GuidanceGit $Root @('cat-file', 'blob', "$head`:$($Entry.path)") -RawOutput
$live = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $Entry.path -Knowledge))
if ($committed.Replace("`r`n", "`n") -cne $live.Replace("`r`n", "`n")) {
throw 'Reference SHA content differs from the live knowledge article.'
}
}
}
function Get-GuidanceResultSchema {
param([string] $CaseId)
return [ordered]@{
caseId = $CaseId
guidanceReport = [ordered]@{
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
'outcome-reason' = 'required for partial or failed'
summary = [ordered]@{ request = 'planned intent'; kind = 'feature | bug | refactor | upgrade | maintenance'; candidates = 0; selected = 0 }
context = [ordered]@{ 'bc-version' = 'resolved target or unknown'; technologies = @('al'); countries = @('w1'); 'application-area' = @('all'); unknown = @() }
knowledge = @([ordered]@{ path = 'repo-relative knowledge article'; sha = 'optional full checkout commit id'; 'used-for' = 'plan decision'; constraints = @('faithful normative constraint'); 'sample-paths' = @() })
'validation-considerations' = @([ordered]@{ id = 'stable id'; reason = 'why needed'; evidence = 'evidence implementation should obtain' })
suppressed = @([ordered]@{ reference = [ordered]@{ path = 'suppressed knowledge path' }; reason = 'layer-precedence | configuration' })
unresolved = @('Missing context/decision, affected candidate, and materiality; name unknown dimensions exactly.')
}
}
}

584
tools/Test-DevelopmentFixtures.ps1
View file

@ -1,584 +0,0 @@
<#
.SYNOPSIS
Validates and prepares BCQuality AL development fixtures.
.DESCRIPTION
Static validation checks fixture IDs, capability links, knowledge references,
and the development skill. -PrepareDirectory emits opaque requests for model
runs. -ResultsDirectory scores implementation reports produced by an
external runner that performed the declared compile, test, and review checks.
#>
[CmdletBinding()]
param(
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
[string] $ManifestPath,
[string] $CapabilitiesPath,
[string] $PrepareDirectory,
[string] $ResultsDirectory
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$Root = (Resolve-Path -LiteralPath $Root).Path
if (-not $ManifestPath) {
$ManifestPath = Join-Path $Root 'evaluation/development-fixtures.json'
}
if (-not $CapabilitiesPath) {
$CapabilitiesPath = Join-Path $Root 'coverage/development-capabilities.json'
}
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
$capabilityManifest = Get-Content -LiteralPath $CapabilitiesPath -Raw | ConvertFrom-Json
$problems = [System.Collections.Generic.List[string]]::new()
function Get-ModelCaseId {
param([string] $ManifestId)
$sha = [System.Security.Cryptography.SHA256]::Create()
try {
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
$hash = $sha.ComputeHash($bytes)
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
return "case-$token"
} finally {
$sha.Dispose()
}
}
if ($manifest.version -ne 1) {
$problems.Add("Unsupported development fixture version: $($manifest.version)") | Out-Null
}
if ($capabilityManifest.version -ne 1) {
$problems.Add("Unsupported capability manifest version: $($capabilityManifest.version)") | Out-Null
}
$minimumFixtureCoverage = if ($capabilityManifest.PSObject.Properties.Name -contains 'minimumFixtureCoverage') {
[double]$capabilityManifest.minimumFixtureCoverage
} else {
-1
}
if ($minimumFixtureCoverage -lt 0 -or $minimumFixtureCoverage -gt 1) {
$problems.Add("minimumFixtureCoverage must be between 0 and 1.") | Out-Null
}
$maximumReviewRounds = if ($manifest.PSObject.Properties.Name -contains 'maximumReviewRounds') {
[int]$manifest.maximumReviewRounds
} else {
0
}
if ($maximumReviewRounds -le 0) {
$problems.Add("maximumReviewRounds must be a positive integer.") | Out-Null
}
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
$threshold = [double]$manifest.$thresholdName
if ($threshold -lt 0 -or $threshold -gt 1) {
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
}
}
$skillPath = [string]$manifest.skill
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
$problems.Add("Development skill does not exist: $skillPath") | Out-Null
} else {
$skillText = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
$limitMatch = [regex]::Match($skillText, '(?m)^quality-round-limit:\s*(\d+)\s*$')
if (-not $limitMatch.Success -or [int]$limitMatch.Groups[1].Value -ne $maximumReviewRounds) {
$problems.Add("maximumReviewRounds must match the development skill quality-round-limit.") | Out-Null
}
}
$validChecks = @('compile', 'tests', 'review')
$validInputKinds = @('auto', 'feature', 'bug', 'refactor', 'upgrade', 'maintenance')
$validOutputKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
$caseById = @{}
foreach ($case in @($manifest.cases)) {
$id = [string]$case.id
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
} elseif ($caseById.ContainsKey($id)) {
$problems.Add("Duplicate fixture id: $id") | Out-Null
} else {
$caseById[$id] = $case
}
if ($case.PSObject.Properties.Name -notcontains 'development-request') {
$problems.Add("${id}: development-request is required.") | Out-Null
continue
}
$request = $case.'development-request'
if ($validInputKinds -notcontains [string]$request.kind) {
$problems.Add("${id}: development-request.kind must be one of $($validInputKinds -join ', ').") | Out-Null
}
$description = if ($request.PSObject.Properties.Name -contains 'description') {
[string]$request.description
} else {
''
}
$planText = if ($request.PSObject.Properties.Name -contains 'plan') {
[string]$request.plan
} else {
''
}
if ([string]::IsNullOrWhiteSpace($description) -and [string]::IsNullOrWhiteSpace($planText)) {
$problems.Add("${id}: development-request requires description or plan.") | Out-Null
}
if ($request.PSObject.Properties.Name -notcontains 'acceptance-criteria' -or
-not @($request.'acceptance-criteria').Count) {
$problems.Add("${id}: development-request.acceptance-criteria must not be empty.") | Out-Null
}
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
[string]$case.expectedKind
} else {
''
}
if ($validOutputKinds -notcontains $expectedKind) {
$problems.Add("${id}: expectedKind must be one of $($validOutputKinds -join ', ').") | Out-Null
}
if ([string]$request.kind -ne 'auto' -and [string]$request.kind -ne $expectedKind) {
$problems.Add("${id}: explicit request kind '$($request.kind)' must equal expectedKind '$expectedKind'.") | Out-Null
}
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
}
foreach ($reference in $expectedKnowledge) {
$reference = [string]$reference
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
}
}
foreach ($check in @($case.requiredChecks)) {
if ($validChecks -notcontains [string]$check) {
$problems.Add("${id}: unsupported required check '$check'.") | Out-Null
}
}
}
$validCapabilityStatuses = @('planned', 'fixture', 'validated')
$capabilityIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
foreach ($capability in @($capabilityManifest.capabilities)) {
$id = [string]$capability.id
if (-not $capabilityIds.Add($id)) {
$problems.Add("Duplicate capability id: $id") | Out-Null
}
if ($validCapabilityStatuses -notcontains [string]$capability.status) {
$problems.Add("${id}: invalid capability status '$($capability.status)'.") | Out-Null
}
$fixtureIds = @($capability.fixtureIds)
if ($capability.status -in @('fixture', 'validated') -and -not $fixtureIds.Count) {
$problems.Add("${id}: status '$($capability.status)' requires at least one fixture.") | Out-Null
}
foreach ($fixtureId in $fixtureIds) {
if (-not $caseById.ContainsKey([string]$fixtureId)) {
$problems.Add("${id}: unknown fixture id '$fixtureId'.") | Out-Null
} elseif (@($caseById[[string]$fixtureId].capabilities) -notcontains $id) {
$problems.Add("${id}: fixture '$fixtureId' does not link back to the capability.") | Out-Null
}
}
}
foreach ($case in @($manifest.cases)) {
foreach ($capabilityId in @($case.capabilities)) {
if (-not $capabilityIds.Contains([string]$capabilityId)) {
$problems.Add("$($case.id): unknown capability '$capabilityId'.") | Out-Null
continue
}
$capability = @(
$capabilityManifest.capabilities |
Where-Object id -eq ([string]$capabilityId)
)[0]
if (@($capability.fixtureIds) -notcontains [string]$case.id) {
$problems.Add("$($case.id): capability '$capabilityId' does not link back to the fixture.") | Out-Null
}
}
}
$fixtureBackedCount = @(
$capabilityManifest.capabilities |
Where-Object status -in @('fixture', 'validated')
).Count
$capabilityCount = @($capabilityManifest.capabilities).Count
$fixtureCoverage = if ($capabilityCount) { $fixtureBackedCount / $capabilityCount } else { 0.0 }
if ($fixtureCoverage -lt $minimumFixtureCoverage) {
$problems.Add("Fixture-backed capability coverage $fixtureCoverage is below minimumFixtureCoverage $minimumFixtureCoverage.") | Out-Null
}
if ($problems.Count) {
Write-Host "Development fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
exit 1
}
if ($PrepareDirectory) {
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-evaluation'
if (Test-Path -LiteralPath $PrepareDirectory) {
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
throw "PrepareDirectory is not empty and is not a BCQuality development evaluation directory: $PrepareDirectory"
}
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
Get-ChildItem -LiteralPath $PrepareDirectory -File |
Where-Object {
$_.Name -eq 'knowledge-index.json' -or
$_.Name -like 'request-*.json' -or
$_.Name -like 'result-*.json'
} |
Remove-Item -Force
}
} else {
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
}
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development evaluation directory' -Encoding UTF8
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
foreach ($case in @($manifest.cases)) {
$modelId = Get-ModelCaseId ([string]$case.id)
$request = [ordered]@{
protocol = 'Run the supplied AL development skill in a fresh writable fixture repository. Persist the implementation, run real checks, and return only implementationReport using the supplied schema.'
caseId = $modelId
skill = $skillPath
skillInstructions = $skillInstructions
knowledgeIndex = 'knowledge-index.json'
'task-context' = [ordered]@{
goal = if ($case.'development-request'.PSObject.Properties.Name -contains 'description') {
[string]$case.'development-request'.description
} else {
[string]$case.'development-request'.plan
}
'inputs-available' = @('development-request', 'repository')
technologies = @($case.context.technologies)
countries = @($case.context.countries)
'application-area' = @($case.context.'application-area')
}
'development-request' = [ordered]@{
kind = [string]$case.'development-request'.kind
description = if ($case.'development-request'.PSObject.Properties.Name -contains 'description') {
[string]$case.'development-request'.description
} else {
$null
}
plan = if ($case.'development-request'.PSObject.Properties.Name -contains 'plan') {
[string]$case.'development-request'.plan
} else {
$null
}
'acceptance-criteria' = @($case.'development-request'.'acceptance-criteria')
}
resultSchema = [ordered]@{
caseId = $modelId
workspaceRoot = 'absolute path to the retained fixture repository'
implementationReport = [ordered]@{
skill = [ordered]@{ id = 'al-development'; version = 1 }
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
'outcome-reason' = 'required for partial or failed'
summary = [ordered]@{
request = 'implemented change'
'files-created' = 0
'files-modified' = 0
'files-deleted' = 0
}
plan = [ordered]@{
kind = 'feature | bug | refactor | upgrade | maintenance'
assumptions = @()
decisions = @()
objects = @()
}
knowledge = @([ordered]@{ path = 'repo-relative knowledge article path'; sha = 'optional commit sha'; 'used-for' = 'decision' })
changes = @([ordered]@{ path = 'repo-relative changed file'; action = 'created | modified | deleted'; purpose = 'reason' })
validation = @([ordered]@{ id = 'compile | tests | review'; command = 'command or quality-skill path'; status = 'passed | failed | not-run'; details = 'non-empty evidence' })
review = [ordered]@{
skill = [ordered]@{ id = 'al-code-review'; version = 1 }
outcome = 'completed | partial | failed'
summary = [ordered]@{
counts = [ordered]@{ blocker = 0; major = 0; minor = 0; info = 0 }
coverage = [ordered]@{ 'worklist-size' = 0; 'items-evaluated' = 0 }
}
findings = @()
suppressed = @()
}
'review-rounds' = @([ordered]@{
round = 1
outcome = 'clean | fixing | stalled | limit-reached'
'gating-finding-ids' = @()
})
suppressed = @()
remaining = @()
}
}
}
$request | ConvertTo-Json -Depth 12 |
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
}
}
if ($ResultsDirectory) {
$failures = [System.Collections.Generic.List[string]]::new()
foreach ($case in @($manifest.cases)) {
$modelId = Get-ModelCaseId ([string]$case.id)
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
$failures.Add("$($case.id): missing result file.") | Out-Null
continue
}
try {
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
} catch {
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
continue
}
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
continue
}
if ($result.PSObject.Properties.Name -notcontains 'implementationReport') {
$failures.Add("$($case.id): implementationReport is missing.") | Out-Null
continue
}
$report = $result.implementationReport
foreach ($requiredField in @('skill', 'outcome', 'summary', 'plan', 'knowledge', 'changes', 'validation', 'review', 'review-rounds', 'suppressed', 'remaining')) {
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
$failures.Add("$($case.id): implementation report is missing '$requiredField'.") | Out-Null
}
}
$reportedSkillId = if (
$report.PSObject.Properties.Name -contains 'skill' -and
$report.skill.PSObject.Properties.Name -contains 'id'
) {
[string]$report.skill.id
} else {
''
}
if ($reportedSkillId -ne 'al-development') {
$failures.Add("$($case.id): implementation report skill is '$reportedSkillId'.") | Out-Null
}
$summaryRequest = if (
$report.PSObject.Properties.Name -contains 'summary' -and
$report.summary.PSObject.Properties.Name -contains 'request'
) {
[string]$report.summary.request
} else {
''
}
if ([string]::IsNullOrWhiteSpace($summaryRequest)) {
$failures.Add("$($case.id): summary.request is missing or empty.") | Out-Null
}
$reportedKind = if (
$report.PSObject.Properties.Name -contains 'plan' -and
$report.plan.PSObject.Properties.Name -contains 'kind'
) {
[string]$report.plan.kind
} else {
''
}
if ($reportedKind -ne [string]$case.expectedKind) {
$failures.Add("$($case.id): plan.kind '$reportedKind' does not match expected mode '$($case.expectedKind)'.") | Out-Null
}
$outcome = if ($report.PSObject.Properties.Name -contains 'outcome') { [string]$report.outcome } else { '' }
if ($outcome -ne 'completed') {
$failures.Add("$($case.id): implementation outcome is '$outcome'.") | Out-Null
}
[object[]]$knowledgeEntries = @()
if ($report.PSObject.Properties.Name -contains 'knowledge') {
$knowledgeEntries = @($report.knowledge)
}
if (-not $knowledgeEntries.Count) {
$failures.Add("$($case.id): implementation report contains no knowledge entries.") | Out-Null
}
$usedKnowledge = @(
$knowledgeEntries |
Where-Object { $_.PSObject.Properties.Name -contains 'path' } |
ForEach-Object { [string]$_.path }
)
if (@($usedKnowledge | Sort-Object -Unique).Count -ne $usedKnowledge.Count) {
$failures.Add("$($case.id): knowledge contains duplicate paths.") | Out-Null
}
foreach ($entry in $knowledgeEntries) {
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
if ([string]::IsNullOrWhiteSpace($path) -or
[System.IO.Path]::IsPathRooted($path) -or
@($path -split '/|\\') -contains '..' -or
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
$failures.Add("$($case.id): knowledge entry has a missing or invalid path '$path'.") | Out-Null
}
if ([string]::IsNullOrWhiteSpace($usedFor)) {
$failures.Add("$($case.id): knowledge entry '$path' has no used-for explanation.") | Out-Null
}
}
$requiredKnowledge = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
$matched = @($requiredKnowledge | Where-Object { $usedKnowledge -contains $_ }).Count
$recall = if ($requiredKnowledge.Count) { $matched / $requiredKnowledge.Count } else { 1.0 }
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
}
$acceptedKnowledge = @(
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
ForEach-Object { [string]$_ } |
Sort-Object -Unique
)
$acceptedUsed = @($usedKnowledge | Where-Object { $acceptedKnowledge -contains $_ }).Count
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
}
[object[]]$validationEntries = @()
if ($report.PSObject.Properties.Name -contains 'validation') {
$validationEntries = @($report.validation)
}
foreach ($requiredCheck in @($case.requiredChecks)) {
$check = @($validationEntries | Where-Object {
$_.PSObject.Properties.Name -contains 'id' -and [string]$_.id -eq $requiredCheck
})
$checkStatus = if ($check.Count -eq 1 -and $check[0].PSObject.Properties.Name -contains 'status') {
[string]$check[0].status
} else {
''
}
if ($check.Count -ne 1 -or $checkStatus -ne 'passed') {
$failures.Add("$($case.id): required check '$requiredCheck' did not pass exactly once.") | Out-Null
continue
}
$command = if ($check[0].PSObject.Properties.Name -contains 'command') { [string]$check[0].command } else { '' }
$details = if ($check[0].PSObject.Properties.Name -contains 'details') { [string]$check[0].details } else { '' }
if ([string]::IsNullOrWhiteSpace($command) -or [string]::IsNullOrWhiteSpace($details)) {
$failures.Add("$($case.id): required check '$requiredCheck' lacks command or evidence details.") | Out-Null
}
}
[object[]]$changes = @()
if ($report.PSObject.Properties.Name -contains 'changes') {
$changes = @($report.changes)
}
if (-not $changes.Count) {
$failures.Add("$($case.id): implementation report contains no changed files.") | Out-Null
}
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') { [string]$result.workspaceRoot } else { '' }
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
} else {
& git -C $workspaceRoot rev-parse --is-inside-work-tree 2>$null | Out-Null
if ($LASTEXITCODE -ne 0) {
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
continue
}
$actualChanges = @(
@(& git -C $workspaceRoot diff --name-only HEAD) +
@(& git -C $workspaceRoot ls-files --others --exclude-standard) |
Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } |
ForEach-Object { ([string]$_).Replace('\', '/') } |
Sort-Object -Unique
)
if ($LASTEXITCODE -ne 0) {
$failures.Add("$($case.id): unable to read the workspace diff.") | Out-Null
} else {
$reportedChanges = @(
$changes |
Where-Object { $_.PSObject.Properties.Name -contains 'path' } |
ForEach-Object {
$path = ([string]$_.path).Replace('\', '/')
if ([System.IO.Path]::IsPathRooted($path) -or @($path -split '/') -contains '..') {
$failures.Add("$($case.id): changed path is not repository-relative: $path") | Out-Null
}
$path
} |
Sort-Object -Unique
)
foreach ($path in @($reportedChanges | Where-Object { $actualChanges -notcontains $_ })) {
$failures.Add("$($case.id): reported change is absent from the worktree diff: $path") | Out-Null
}
foreach ($path in @($actualChanges | Where-Object { $reportedChanges -notcontains $_ })) {
$failures.Add("$($case.id): worktree change is absent from the implementation report: $path") | Out-Null
}
}
}
if ($report.PSObject.Properties.Name -notcontains 'review') {
$failures.Add("$($case.id): complete post-implementation review is missing.") | Out-Null
} else {
$review = $report.review
foreach ($requiredField in @('skill', 'outcome', 'summary', 'findings', 'suppressed')) {
if ($review.PSObject.Properties.Name -notcontains $requiredField) {
$failures.Add("$($case.id): review is missing '$requiredField'.") | Out-Null
}
}
[object[]]$reviewFindings = @()
if ($review.PSObject.Properties.Name -contains 'findings') {
$reviewFindings = @($review.findings)
}
$reviewOutcome = if ($review.PSObject.Properties.Name -contains 'outcome') { [string]$review.outcome } else { '' }
if ($reviewOutcome -ne 'completed') {
$failures.Add("$($case.id): post-implementation review outcome is '$reviewOutcome'.") | Out-Null
}
$gatingFindings = @(
$reviewFindings |
Where-Object {
$_.PSObject.Properties.Name -contains 'severity' -and
[string]$_.severity -in @('blocker', 'major')
}
)
if ($gatingFindings.Count) {
$failures.Add("$($case.id): post-implementation review has $($gatingFindings.Count) gating finding(s).") | Out-Null
}
}
[object[]]$reviewRounds = @()
if ($report.PSObject.Properties.Name -contains 'review-rounds') {
$reviewRounds = @($report.'review-rounds')
}
if (-not $reviewRounds.Count -or $reviewRounds.Count -gt $maximumReviewRounds) {
$failures.Add("$($case.id): review-rounds must contain 1..$maximumReviewRounds entries.") | Out-Null
} else {
for ($index = 0; $index -lt $reviewRounds.Count; $index++) {
$round = $reviewRounds[$index]
$roundNumber = if ($round.PSObject.Properties.Name -contains 'round') { [int]$round.round } else { 0 }
$roundOutcome = if ($round.PSObject.Properties.Name -contains 'outcome') { [string]$round.outcome } else { '' }
if ($roundNumber -ne ($index + 1)) {
$failures.Add("$($case.id): review round numbering is not contiguous.") | Out-Null
}
if ($roundOutcome -notin @('clean', 'fixing', 'stalled', 'limit-reached')) {
$failures.Add("$($case.id): review round $roundNumber has invalid outcome '$roundOutcome'.") | Out-Null
}
[object[]]$roundGatingIds = @()
if ($round.PSObject.Properties.Name -contains 'gating-finding-ids') {
$roundGatingIds = @($round.'gating-finding-ids')
}
if ($roundOutcome -eq 'clean' -and $roundGatingIds.Count) {
$failures.Add("$($case.id): clean review round $roundNumber must have no gating IDs.") | Out-Null
}
if ($roundOutcome -in @('fixing', 'stalled', 'limit-reached') -and -not $roundGatingIds.Count) {
$failures.Add("$($case.id): review round $roundNumber outcome '$roundOutcome' requires gating IDs.") | Out-Null
}
if (@($roundGatingIds | Sort-Object -Unique).Count -ne $roundGatingIds.Count) {
$failures.Add("$($case.id): review round $roundNumber has duplicate gating IDs.") | Out-Null
}
}
if ([string]$reviewRounds[-1].outcome -ne 'clean') {
$failures.Add("$($case.id): completed implementation must end with a clean review round.") | Out-Null
}
}
[object[]]$remaining = @()
if ($report.PSObject.Properties.Name -contains 'remaining') {
$remaining = @($report.remaining)
}
if ($remaining.Count) {
$failures.Add("$($case.id): completed report still lists remaining work.") | Out-Null
}
}
if ($failures.Count) {
Write-Host "Development fixture scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
exit 1
}
Write-Host "Development fixture scoring PASSED: $(@($manifest.cases).Count) case(s)."
} else {
Write-Host "Development fixture validation PASSED: $(@($manifest.cases).Count) cases; $fixtureBackedCount of $($capabilityIds.Count) capabilities have fixtures (minimum $minimumFixtureCoverage)."
}

421
tools/Test-DevelopmentGuidanceEvaluator.ps1 Normal file
View file

@ -0,0 +1,421 @@
<#
.SYNOPSIS
Deterministic, offline regressions for the guidance evaluator (no model/AL run).
.DESCRIPTION
Creates only a uniquely named .guidance-evaluator-regression-* directory below
the current checkout, with standalone Git repositories and sibling runner
artifacts. Removes that exact directory in finally; never uses the OS temp
directory or cleans a caller-provided repository.
#>
[CmdletBinding()]
param()
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
$evaluator = Join-Path $PSScriptRoot 'Test-DevelopmentGuidanceFixtures.ps1'
$sourceRoot = (Get-Item -LiteralPath (Join-Path $PSScriptRoot '..')).FullName
$scratch = Join-Path $sourceRoot ".guidance-evaluator-regression-$([guid]::NewGuid().ToString('N'))"
$root = Join-Path $scratch 'knowledge-checkout'
$article = 'microsoft/knowledge/performance/pair-findset-with-next-loop.md'
$otherArticle = 'microsoft/knowledge/performance/findset-true-applies-updlock-on-read.md'
$sample = 'microsoft/knowledge/performance/pair-findset-with-next-loop.good.al'
$tests = [Collections.Generic.List[string]]::new()
$script:scenarioNumber = 0
$script:reportScenario = $null
function Set-TestJson($Path, $Value) {
[IO.File]::WriteAllText($Path, ($Value | ConvertTo-Json -Depth 64))
}
function Invoke-TestGit([string] $Directory, [string[]] $Arguments) {
$output = @(& git --no-optional-locks -C $Directory @Arguments 2>&1)
if ($LASTEXITCODE -ne 0) { throw 'Regression Git setup failed.' }
}
function Initialize-TestRepository([string] $Directory) {
[IO.Directory]::CreateDirectory($Directory) | Out-Null
Invoke-TestGit $Directory @('init', '--quiet')
Invoke-TestGit $Directory @('config', 'user.email', 'guidance-fixture@example.invalid')
Invoke-TestGit $Directory @('config', 'user.name', 'Guidance fixture')
Invoke-TestGit $Directory @('config', 'commit.gpgSign', 'false')
Invoke-TestGit $Directory @('config', 'core.autocrlf', 'false')
[IO.File]::WriteAllText((Join-Path $Directory 'app.json'), '{"name":"Synthetic AL fixture","application":"28.0.0.0"}')
[IO.File]::WriteAllText((Join-Path $Directory 'tracked.al'), 'codeunit 50100 Example {}')
[IO.File]::WriteAllText((Join-Path $Directory '.gitignore'), "ignored.txt`nignored-directory/`n")
Invoke-TestGit $Directory @('add', '.')
Invoke-TestGit $Directory @('commit', '--quiet', '-m', 'Synthetic fixture baseline')
}
function Invoke-EvaluatorTest([string] $Name, [string[]] $Arguments, [bool] $ShouldPass, [string] $Diagnostic = '') {
$output = @(& pwsh -NoProfile -File $evaluator @Arguments 2>&1) -join "`n"
$code = $LASTEXITCODE
if (($code -eq 0) -ne $ShouldPass -or $output -notmatch $(if ($ShouldPass) { 'PASSED|captured' } else { 'FAILED' })) {
throw "Regression '$Name' unexpected exit $code. $output"
}
if ($Diagnostic -and $output -notmatch [regex]::Escape($Diagnostic)) {
throw "Regression '$Name' missing diagnostic '$Diagnostic'. $output"
}
if ($output -match 'MODEL_SECRET_SENTINEL') { throw "Regression '$Name' leaked model content." }
$tests.Add($Name)
}
function New-TestScenario([string] $Outcome = 'completed', [switch] $Unknown, [switch] $SecondCase) {
$script:scenarioNumber++
$directory = Join-Path $scratch "scenario-$script:scenarioNumber"
[IO.Directory]::CreateDirectory($directory) | Out-Null
$workspace = Join-Path $directory 'target'
Initialize-TestRepository $workspace
[IO.File]::WriteAllText((Join-Path $workspace 'untracked.txt'), 'existing untracked content')
[IO.File]::WriteAllText((Join-Path $workspace 'ignored.txt'), 'existing ignored content')
[IO.Directory]::CreateDirectory((Join-Path $workspace 'ignored-directory')) | Out-Null
[IO.File]::WriteAllText((Join-Path $workspace 'ignored-directory\child.txt'), 'ignored child')
$results = Join-Path $directory 'results'
[IO.Directory]::CreateDirectory($results) | Out-Null
$hasKnowledge = $Outcome -in @('completed', 'partial')
$case = [ordered]@{
id = 'synthetic-case'
expectedKind = 'bug'
expectedOutcome = $Outcome
expectedUnknown = @($(if ($Unknown) { 'bc-version' }))
requiresUnresolved = ($Outcome -eq 'partial' -or $Unknown.IsPresent)
requiresMaterialUnresolved = ($Outcome -eq 'partial')
'development-plan' = [ordered]@{ kind = 'bug'; request = 'Iterate the supplied filtered record set.' }
context = [ordered]@{
'bc-version' = $(if ($Unknown) { 'unknown' } else { '28' })
technologies = @('al')
countries = @('w1')
'application-area' = @('all')
unknown = @($(if ($Unknown) { 'bc-version' }))
}
requiredKnowledge = @($(if ($hasKnowledge) { $article }))
optionalKnowledge = @()
}
$manifest = [ordered]@{
version = 1
skill = 'microsoft/skills/development/al-development-plan.md'
minimumKnowledgeRecall = 1.0
minimumKnowledgePrecision = 1.0
cases = @($case)
}
$result = [ordered]@{
caseId = Get-GuidanceCaseId $case.id
guidanceReport = [ordered]@{
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
outcome = $Outcome
summary = [ordered]@{ request = 'Iterate all selected records.'; kind = 'bug'; candidates = [int]$hasKnowledge; selected = [int]$hasKnowledge }
context = $case.context
knowledge = @($(if ($hasKnowledge) {
[ordered]@{ path = $article; 'used-for' = 'Choose the multi-record reader.'; constraints = @('Use FindSet when iterating with Next.'); 'sample-paths' = @($sample) }
}))
'validation-considerations' = @([ordered]@{ id = 'all-selected'; reason = 'Preserve selection.'; evidence = 'Test all selected records and an excluded record.' })
suppressed = @()
unresolved = @($(if ($Outcome -eq 'partial') {
if ($Unknown) { 'bc-version is unknown and materially affects the candidate; clarify before completing guidance.' }
else { 'The caller cardinality decision remains materially unresolved.' }
} elseif ($Unknown) { 'bc-version is unknown but immaterial: selected loop guidance applies to all versions.' }))
}
}
if ($Outcome -in @('partial', 'failed')) { $result.guidanceReport.'outcome-reason' = 'Fixture intentionally leaves evaluation incomplete.' }
$manifestPath = Join-Path $directory 'manifest.json'
$mapPath = Join-Path $directory 'workspace-map.json'
$map = [ordered]@{ 'synthetic-case' = $workspace }
if ($SecondCase) {
$second = $case | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
$second.id = 'second-case'
$manifest.cases += $second
$secondWorkspace = Join-Path $directory 'second-target'
Initialize-TestRepository $secondWorkspace
$map[$second.id] = $secondWorkspace
$secondResult = $result | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
$secondResult.caseId = Get-GuidanceCaseId $second.id
Set-TestJson (Join-Path $results "result-$($secondResult.caseId).json") $secondResult
}
Set-TestJson $manifestPath $manifest
Set-TestJson $mapPath $map
$resultPath = Join-Path $results "result-$($result.caseId).json"
Set-TestJson $resultPath $result
$baselinePath = Join-Path $directory 'baseline.json'
$captureArgs = @('-Root', $root, '-ManifestPath', $manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $mapPath, '-BaselinePath', $baselinePath)
$output = @(& pwsh -NoProfile -File $evaluator @captureArgs 2>&1) -join "`n"
if ($LASTEXITCODE -ne 0) { throw "Baseline setup failed. $output" }
return [ordered]@{
directory = $directory; workspace = $workspace; resultPath = $resultPath; result = $result
results = $results; manifest = $manifest; manifestPath = $manifestPath; mapPath = $mapPath
baselinePath = $baselinePath; captureArgs = $captureArgs
scoreArgs = @('-Root', $root, '-ManifestPath', $manifestPath, '-ResultsDirectory', $results, '-BaselinePath', $baselinePath, '-BaselineSha256', (Get-GuidanceHash $baselinePath))
}
}
function Test-ReportMutation([string] $Name, [scriptblock] $Change, [string] $Diagnostic) {
if ($null -eq $script:reportScenario) { $script:reportScenario = New-TestScenario }
$scenario = $script:reportScenario
$result = $scenario.result | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
& $Change $result
Set-TestJson $scenario.resultPath $result
Invoke-EvaluatorTest $Name $scenario.scoreArgs $false $Diagnostic
}
try {
[IO.Directory]::CreateDirectory($root) | Out-Null
foreach ($reference in @($article, $otherArticle, $sample, $otherArticle.Replace('.md', '.good.al'),
'microsoft/skills/development/al-development-plan.md', 'tools/Build-KnowledgeIndex.ps1')) {
$destination = Join-Path $root $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
}
Initialize-TestRepository $root
$publicManifestPath = Join-Path $sourceRoot 'evaluation\development-guidance-fixtures.json'
$publicManifest = Read-GuidanceJson $publicManifestPath
$publicRoot = Join-Path $scratch 'public-fixture-checkout'
$publicReferences = @($publicManifest.skill, 'tools/Build-KnowledgeIndex.ps1', 'evaluation/development-guidance-fixtures.json') +
@($publicManifest.cases | ForEach-Object { $_.requiredKnowledge; $_.optionalKnowledge })
foreach ($reference in @($publicReferences | Sort-Object -Unique)) {
$destination = Join-Path $publicRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
}
$publicPrepared = Join-Path $scratch 'public-prepared'
Invoke-EvaluatorTest 'public five-case manifest validation' @('-Root', $publicRoot) $true
Invoke-EvaluatorTest 'public five-case manifest preparation' @('-Root', $publicRoot, '-PrepareDirectory', $publicPrepared) $true
$initialCase = $publicManifest.cases | Where-Object id -eq 'synthetic-normal-initial-plan'
$initialRequest = Read-GuidanceJson (Join-Path $publicPrepared "request-$(Get-GuidanceCaseId $initialCase.id).json")
if ($initialRequest.'development-plan' -cne $initialCase.'development-plan') { throw 'Preparation truncated the serialized initial plan.' }
$document = $initialRequest.'development-plan' | ConvertFrom-Json -AsHashtable
if ($document.metadata.kind -ne 'bug' -or
@('Root cause and design', 'Proposed fix', 'Affected files', 'Test strategy', 'Acceptance criteria' |
Where-Object { $document.body -notmatch [regex]::Escape($_) }).Count) {
throw 'Synthetic consumer boundary lost metadata or markdown plan sections.'
}
$tests.Add('serialized synthetic initial-plan boundary preserves full metadata and markdown body')
$scenario = New-TestScenario
Invoke-EvaluatorTest 'unchanged target with ignored and untracked files passes' $scenario.scoreArgs $true
Invoke-EvaluatorTest 'scoring itself leaves index and content unchanged' $scenario.scoreArgs $true
Invoke-EvaluatorTest 'existing baseline cannot silently recapture' $scenario.captureArgs $false 'Baseline already exists'
Invoke-EvaluatorTest 'scoring without baseline fails' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results) $false 'require BaselinePath'
Invoke-EvaluatorTest 'scoring requires independently retained baseline digest' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath) $false 'runner-retained pre-run BaselineSha256'
Invoke-EvaluatorTest 'tampered baseline digest fails' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath, '-BaselineSha256', ('0' * 64)) $false 'digest mismatch'
$prepare = Join-Path $scenario.directory 'prepared'
Invoke-EvaluatorTest 'prepare outside roots with runner workspace binding' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', $prepare, '-WorkspaceMapPath', $scenario.mapPath) $true
$prepared = Read-GuidanceJson (Join-Path $prepare "request-$($scenario.result.caseId).json")
if ($prepared.repository -cne $scenario.workspace -or $prepared.Contains('expectedOutcome') -or $prepared.Contains('requiredKnowledge')) {
throw 'Prepared request lost runner binding or exposed answers.'
}
Invoke-EvaluatorTest 'preparation never overwrites requests' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', $prepare) $false 'new or empty'
foreach ($option in @('PrepareDirectory', 'BaselinePath', 'ResultsDirectory')) {
$inside = Join-Path $scenario.workspace 'unsafe-artifact'
$arguments = @('-Root', $root, '-ManifestPath', $scenario.manifestPath)
if ($option -eq 'PrepareDirectory') { $arguments += @('-PrepareDirectory', $inside, '-WorkspaceMapPath', $scenario.mapPath) }
elseif ($option -eq 'BaselinePath') { $arguments += @('-CaptureBaseline', '-BaselinePath', $inside, '-WorkspaceMapPath', $scenario.mapPath) }
else { $arguments += @('-BaselinePath', $scenario.baselinePath, '-BaselineSha256', (Get-GuidanceHash $scenario.baselinePath), '-ResultsDirectory', $inside) }
Invoke-EvaluatorTest "$option inside target rejected" $arguments $false 'outside target workspaces'
}
foreach ($mutation in @('uncommitted', 'committed', 'empty-commit', 'staged', 'index-only', 'untracked', 'ignored', 'ignored-child', 'added', 'deleted', 'directory', 'ref', 'metadata')) {
$scenario = New-TestScenario
switch ($mutation) {
'uncommitted' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed") }
'committed' {
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
Invoke-TestGit $scenario.workspace @('add', 'tracked.al')
Invoke-TestGit $scenario.workspace @('commit', '--quiet', '-m', 'Committed forbidden edit')
}
'staged' {
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
Invoke-TestGit $scenario.workspace @('add', 'tracked.al')
}
'empty-commit' { Invoke-TestGit $scenario.workspace @('commit', '--quiet', '--allow-empty', '-m', 'Forbidden empty commit') }
'index-only' { Invoke-TestGit $scenario.workspace @('update-index', '--assume-unchanged', 'tracked.al') }
'untracked' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'untracked.txt'), 'changed') }
'ignored' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'ignored.txt'), 'changed') }
'ignored-child' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'ignored-directory\child.txt'), 'changed') }
'added' { [IO.File]::WriteAllText((Join-Path $scenario.workspace 'new.txt'), 'new ignored/untracked payload') }
'deleted' { [IO.File]::Delete((Join-Path $scenario.workspace 'untracked.txt')) }
'directory' { [IO.Directory]::CreateDirectory((Join-Path $scenario.workspace 'new-empty-directory')) | Out-Null }
'ref' { Invoke-TestGit $scenario.workspace @('branch', 'new-reference') }
'metadata' {
$path = Join-Path $scenario.workspace 'tracked.al'
[IO.File]::SetLastWriteTimeUtc($path, [IO.File]::GetLastWriteTimeUtc($path).AddSeconds(5))
}
}
Invoke-EvaluatorTest "$mutation mutation fails" $scenario.scoreArgs $false 'identity/content changed'
}
$scenario = New-TestScenario -SecondCase
Invoke-EvaluatorTest 'two independently bound workspaces pass' $scenario.scoreArgs $true
$map = Read-GuidanceJson $scenario.mapPath
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
$scenario.result.workspaceRoot = $map['second-case']
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'self-reported clean workspace cannot hide changed runner target' $scenario.scoreArgs $false 'identity/content changed'
$scenario = New-TestScenario -SecondCase
$scenario.result.workspaceRoot = (Read-GuidanceJson $scenario.mapPath)['second-case']
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'result workspace swapping rejected even when both are clean' $scenario.scoreArgs $false 'runner binding'
$scenario = New-TestScenario -SecondCase
$scenario.result.caseId = Get-GuidanceCaseId 'second-case'
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'result case swapping rejected' $scenario.scoreArgs $false 'caseId mismatch'
$scenario = New-TestScenario
$scenario.manifest.cases[0].expectedKind = 'feature'
Set-TestJson $scenario.manifestPath $scenario.manifest
Invoke-EvaluatorTest 'changed manifest rejected' $scenario.scoreArgs $false 'root or manifest'
$scenario = New-TestScenario
$baseline = Read-GuidanceJson $scenario.baselinePath
$baseline.workspaces['synthetic-case'].caseId = 'case-00000000'
Set-TestJson $scenario.baselinePath $baseline
Invoke-EvaluatorTest 'baseline case binding checked even with matching digest' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath, '-BaselineSha256', (Get-GuidanceHash $scenario.baselinePath)) $false 'case identity mismatch'
foreach ($outcome in @('no-knowledge', 'partial', 'failed', 'not-applicable')) {
$scenario = New-TestScenario $outcome
Invoke-EvaluatorTest "honest $outcome is distinguishable and passes" $scenario.scoreArgs $true
}
$scenario = New-TestScenario 'partial' -Unknown
Invoke-EvaluatorTest 'material unknown stays partial' $scenario.scoreArgs $true
$scenario.result.guidanceReport.outcome = 'completed'
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'material unknown cannot silently complete' $scenario.scoreArgs $false 'expectedOutcome'
$scenario = New-TestScenario 'completed' -Unknown
Invoke-EvaluatorTest 'nonmaterial unknown can complete with explanation' $scenario.scoreArgs $true
$scenario.result.guidanceReport.unresolved = @()
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'unknown cannot disappear from unresolved evidence' $scenario.scoreArgs $false 'unresolved gaps'
$scenario = New-TestScenario 'no-knowledge' -Unknown
Invoke-EvaluatorTest 'immaterial unknown and no-knowledge are not failures' $scenario.scoreArgs $true
foreach ($outcome in @('partial', 'failed')) {
$scenario = New-TestScenario $outcome
$scenario.result.guidanceReport.Remove('outcome-reason')
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest "$outcome requires outcome-reason" $scenario.scoreArgs $false 'outcome-reason'
}
$scenario = New-TestScenario 'no-knowledge'
$scenario.result.guidanceReport.outcome = 'completed'
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'no-knowledge is not completed-empty' $scenario.scoreArgs $false 'expectedOutcome'
$scenario = New-TestScenario 'no-knowledge'
$scenario.result.guidanceReport.knowledge = @(@{ path = $article; 'used-for' = 'filler'; constraints = @('filler'); 'sample-paths' = @() })
$scenario.result.guidanceReport.summary.candidates = 1
$scenario.result.guidanceReport.summary.selected = 1
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'no-knowledge cannot contain filler knowledge' $scenario.scoreArgs $false 'requires empty knowledge'
Test-ReportMutation 'invalid outcome enum' { param($r) $r.guidanceReport.outcome = 'success' } 'outcome enum'
Test-ReportMutation 'missing report fields' { param($r) $r.guidanceReport.Remove('knowledge') } 'missing required field'
Test-ReportMutation 'null report object' { param($r) $r.guidanceReport = $null } 'JSON object'
Test-ReportMutation 'wrong skill version type' { param($r) $r.guidanceReport.skill.version = '1' } 'identity/version'
Test-ReportMutation 'wrong skill id type' { param($r) $r.guidanceReport.skill.id = @('al-development-plan') } 'non-empty string'
Test-ReportMutation 'wrong kind type' { param($r) $r.guidanceReport.summary.kind = @('bug') } 'non-empty string'
Test-ReportMutation 'wrong summary type' { param($r) $r.guidanceReport.summary = @() } 'JSON object'
Test-ReportMutation 'fractional count' { param($r) $r.guidanceReport.summary.candidates = 1.5 } 'non-negative integer'
Test-ReportMutation 'negative count' { param($r) $r.guidanceReport.summary.selected = -1 } 'non-negative integer'
Test-ReportMutation 'inconsistent selected count' { param($r) $r.guidanceReport.summary.selected = 0 } 'Summary counts'
Test-ReportMutation 'candidate count below selected' { param($r) $r.guidanceReport.summary.candidates = 0 } 'Summary counts'
Test-ReportMutation 'wrong context list type' { param($r) $r.guidanceReport.context.technologies = 'al' } 'JSON array'
Test-ReportMutation 'missing constraints' { param($r) $r.guidanceReport.knowledge[0].constraints = @() } 'must not be empty'
Test-ReportMutation 'non-string constraints' { param($r) $r.guidanceReport.knowledge[0].constraints = @(@{ body = 'MODEL_SECRET_SENTINEL' }) } 'non-empty string'
Test-ReportMutation 'missing sample array' { param($r) $r.guidanceReport.knowledge[0].Remove('sample-paths') } 'missing required field'
Test-ReportMutation 'duplicate knowledge' {
param($r)
$r.guidanceReport.knowledge += $r.guidanceReport.knowledge[0]
$r.guidanceReport.summary.selected = 2
$r.guidanceReport.summary.candidates = 2
} 'Duplicate knowledge'
Test-ReportMutation 'bad validation entry' { param($r) $r.guidanceReport.'validation-considerations'[0].evidence = $false } 'non-empty string'
Test-ReportMutation 'invalid suppression shape' { param($r) $r.guidanceReport.suppressed = @(@{ path = $article; reason = 'configuration' }) } 'missing required field'
Test-ReportMutation 'invalid unresolved shape' { param($r) $r.guidanceReport.unresolved = @(@{ candidate = $article }) } 'non-empty string'
Test-ReportMutation 'invalid SHA provenance' { param($r) $r.guidanceReport.knowledge[0].sha = '0' * 40 } 'recorded live checkout'
$scenario = New-TestScenario
$scenario.result.guidanceReport.knowledge[0].sha = Invoke-GuidanceGit $root @('rev-parse', 'HEAD')
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'actual pinned knowledge SHA accepted' $scenario.scoreArgs $true
foreach ($badPath in @('../outside.md', '/absolute.md', 'C:/external.md',
'microsoft\knowledge\performance\pair-findset-with-next-loop.md',
'microsoft/knowledge/performance/../performance/pair-findset-with-next-loop.md',
'https://example.invalid/article.md', 'microsoft/knowledge/performance/missing.md',
'microsoft/skills/development/al-development-plan.md')) {
$scenario = New-TestScenario
$scenario.result.guidanceReport.knowledge[0].path = $badPath
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'unsafe/nonexistent/non-knowledge citation rejected' $scenario.scoreArgs $false
}
foreach ($badSample in @('../outside.al', $article, $otherArticle.Replace('.md', '.good.al'), 'microsoft/knowledge/performance/other.good.al', 'microsoft\knowledge\performance\pair-findset-with-next-loop.good.al')) {
$scenario = New-TestScenario
$scenario.result.guidanceReport.knowledge[0].'sample-paths' = @($badSample)
Set-TestJson $scenario.resultPath $scenario.result
Invoke-EvaluatorTest 'unsafe/nonexistent/non-sibling sample rejected' $scenario.scoreArgs $false
}
foreach ($json in @('{', 'null', '[]', '{"caseId":"MODEL_SECRET_SENTINEL","caseId":"duplicate"}', '{"caseId":true,}', '// comment')) {
$scenario = New-TestScenario
[IO.File]::WriteAllText($scenario.resultPath, $json)
Invoke-EvaluatorTest 'malformed model result fails without runtime crash or content leakage' $scenario.scoreArgs $false
}
$scenario = New-TestScenario
$outside = Join-Path $scenario.directory 'outside'
[IO.Directory]::CreateDirectory($outside) | Out-Null
$link = Join-Path $scenario.workspace 'escape'
$linkKind = if ($IsWindows) { 'Junction' } else { 'SymbolicLink' }
New-Item -ItemType $linkKind -Path $link -Target $outside | Out-Null
try {
Invoke-EvaluatorTest 'target junction/symlink rejected instead of followed' $scenario.scoreArgs $false 'Links, junctions'
Invoke-EvaluatorTest 'baseline capture rejects junction/symlink target children' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', (Join-Path $scenario.directory 'linked-baseline.json')) $false 'Links, junctions'
Invoke-EvaluatorTest 'prepared directory cannot escape through junction' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', (Join-Path $link 'prepared'), '-WorkspaceMapPath', $scenario.mapPath) $false 'Links, junctions'
} finally { Remove-Item -LiteralPath $link -Force }
$scenario = New-TestScenario
$external = Join-Path $scenario.directory 'external.txt'
[IO.File]::WriteAllText($external, 'external hard-link target')
$hardLink = Join-Path $scenario.workspace 'hard-link.txt'
New-Item -ItemType HardLink -Path $hardLink -Target $external | Out-Null
try {
Invoke-EvaluatorTest 'hard-link escape rejected' $scenario.scoreArgs $false 'Links, junctions'
} finally { Remove-Item -LiteralPath $hardLink -Force }
$scenario = New-TestScenario
$externalArticle = Join-Path $scenario.directory 'outside.md'
[IO.File]::Copy((Join-Path $root $article.Replace('/', [IO.Path]::DirectorySeparatorChar)), $externalArticle)
$articleLink = Join-Path $root 'custom\knowledge\performance'
[IO.Directory]::CreateDirectory((Split-Path $articleLink -Parent)) | Out-Null
New-Item -ItemType $linkKind -Path $articleLink -Target $scenario.directory | Out-Null
try {
$rejected = $false
try { $null = Resolve-GuidanceReference $root 'custom/knowledge/performance/outside.md' -Knowledge }
catch { $rejected = $_.Exception.Message -match 'Links, junctions' }
if (-not $rejected) { throw 'Linked knowledge reference was followed.' }
$tests.Add('knowledge junction/symlink reference rejected')
$rejected = $false
try { $null = Resolve-GuidanceReference $root 'custom/knowledge/performance/pair-findset-with-next-loop.good.al' -Article 'custom/knowledge/performance/pair-findset-with-next-loop.md' }
catch { $rejected = $_.Exception.Message -match 'Links, junctions' }
if (-not $rejected) { throw 'Linked sample reference was followed.' }
$tests.Add('sample junction/symlink reference rejected')
} finally { Remove-Item -LiteralPath $articleLink -Force }
$normativePath = Join-Path $root 'microsoft\knowledge\performance\single-normative-section.md'
foreach ($heading in @('Best Practice', 'Anti Pattern')) {
[IO.File]::WriteAllText($normativePath, "---`ndomain: performance`n---`n## Description`nSynthetic contract fixture.`n## $heading`nSynthetic normative constraint.`n")
$null = Resolve-GuidanceReference $root 'microsoft/knowledge/performance/single-normative-section.md' -Knowledge
$tests.Add("Knowledge article with only $heading accepted")
}
[IO.File]::Delete($normativePath)
$scenario = New-TestScenario
$linkedRoot = Join-Path $scratch 'linked-knowledge-checkout'
Invoke-TestGit $root @('worktree', 'add', '--quiet', '--detach', $linkedRoot, 'HEAD')
$linkedBaseline = Join-Path $scenario.directory 'linked-root-baseline.json'
Invoke-EvaluatorTest 'linked knowledge checkout baseline capture' @('-Root', $linkedRoot, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', $linkedBaseline) $true
Invoke-EvaluatorTest 'unchanged linked knowledge checkout scoring' @('-Root', $linkedRoot, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $linkedBaseline, '-BaselineSha256', (Get-GuidanceHash $linkedBaseline)) $true
$scenario = New-TestScenario
$map = Read-GuidanceJson $scenario.mapPath
$map['synthetic-case'] = $linkedRoot
Set-TestJson $scenario.mapPath $map
Invoke-EvaluatorTest 'linked target checkout rejected as external Git storage' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', (Join-Path $scenario.directory 'external-git-baseline.json')) $false 'standalone repositories'
$scenario = New-TestScenario
[IO.File]::AppendAllText((Join-Path $root $sample.Replace('/', [IO.Path]::DirectorySeparatorChar)), "`n// changed knowledge sample")
Invoke-EvaluatorTest 'knowledge checkout changes rejected' $scenario.scoreArgs $false 'Knowledge checkout identity/content changed'
Write-Host "Development guidance evaluator regressions PASSED: $($tests.Count) checks."
} finally {
if (Test-Path -LiteralPath $scratch) {
# Only our own uniquely named directory; links made by tests are removed above.
Remove-Item -LiteralPath $scratch -Recurse -Force
}
}

513
tools/Test-DevelopmentGuidanceFixtures.ps1
View file

@ -1,352 +1,215 @@
<#
.SYNOPSIS
Validates, prepares, and scores read-only AL development-guidance fixtures.
.DESCRIPTION
Capture runner-owned evidence BEFORE invoking an agent:
-CaptureBaseline -WorkspaceMapPath <json> -BaselinePath <new-json>
The workspace map is {"manifest-case-id":"absolute-standalone-git-root",...}.
Score AFTER the agent finishes:
-BaselinePath <json> -BaselineSha256 <runner-retained-digest> -ResultsDirectory <directory>
Preparation (-PrepareDirectory) is independent; supply -WorkspaceMapPath to
include runner-selected target paths. Never derive target paths from results.
Baseline, map, prepared requests, and results must be outside all targets and
the knowledge checkout. Keep the baseline runner-only; retain the printed
SHA256 and pass -BaselineSha256 when scoring to detect baseline tampering.
Capture never overwrites an existing baseline. The runner must protect this
script, the baseline/digest and its invocation from the agent.
This compares before/after evidence, NOT an OS sandbox or a write monitor.
It cannot detect reverted transient writes, prove that articles were opened,
or validate semantic faithfulness of prose. Files, directories, hashes,
stable metadata, Git HEAD/refs/index and ignored/untracked files are compared.
Links/reparse points, hard links, alternate data streams, external Git
storage in targets, submodules and sparse checkouts are rejected rather than
followed. Run in quiescent repositories. The knowledge checkout may itself
be a linked Git worktree; its Git storage identity is recorded explicitly.
#>
[CmdletBinding()]
param(
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
[string] $Root = (Join-Path $PSScriptRoot '..'),
[string] $ManifestPath,
[string] $PrepareDirectory,
[string] $ResultsDirectory
[string] $ResultsDirectory,
[switch] $CaptureBaseline,
[string] $BaselinePath,
[string] $BaselineSha256,
[string] $WorkspaceMapPath
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
$Root = (Resolve-Path -LiteralPath $Root).Path
if (-not $ManifestPath) {
$ManifestPath = Join-Path $Root 'evaluation/development-guidance-fixtures.json'
}
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
$problems = [System.Collections.Generic.List[string]]::new()
$validKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
function Get-ModelCaseId {
param([string] $ManifestId)
$sha = [System.Security.Cryptography.SHA256]::Create()
try {
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
$hash = $sha.ComputeHash($bytes)
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
return "case-$token"
} finally {
$sha.Dispose()
try {
if ($CaptureBaseline -and ($ResultsDirectory -or $PrepareDirectory)) {
throw 'CaptureBaseline is a separate pre-run operation.'
}
}
function Get-PlanRequest {
param([object] $Plan)
if ($Plan.PSObject.Properties.Name -contains 'request' -and
-not [string]::IsNullOrWhiteSpace([string]$Plan.request)) {
return [string]$Plan.request
if (($CaptureBaseline -or $ResultsDirectory) -and -not $BaselinePath) {
throw 'Capture and scoring require BaselinePath.'
}
if ($Plan.PSObject.Properties.Name -contains 'format' -and
[string]$Plan.format -eq 'BCFIX-HANDOFF' -and
$Plan.PSObject.Properties.Name -contains 'nextStep') {
$issue = if ($Plan.PSObject.Properties.Name -contains 'issue') { [string]$Plan.issue } else { 'unknown' }
return "Continue BCFIX issue #${issue}: $($Plan.nextStep)"
if ($ResultsDirectory -and -not $BaselineSha256) {
throw 'Scoring requires the runner-retained pre-run BaselineSha256.'
}
return ''
}
if ($manifest.version -ne 1) {
$problems.Add("Unsupported guidance fixture version: $($manifest.version)") | Out-Null
}
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
$threshold = [double]$manifest.$thresholdName
if ($threshold -lt 0 -or $threshold -gt 1) {
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
if ($CaptureBaseline -and -not $WorkspaceMapPath) {
throw 'Capture requires a runner-owned WorkspaceMapPath.'
}
}
$skillPath = [string]$manifest.skill
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
$problems.Add("Guidance skill does not exist: $skillPath") | Out-Null
}
$seenIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
foreach ($case in @($manifest.cases)) {
$id = [string]$case.id
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
} elseif (-not $seenIds.Add($id)) {
$problems.Add("Duplicate fixture id: $id") | Out-Null
if ($ResultsDirectory -and ($WorkspaceMapPath -or $PrepareDirectory)) {
throw 'Scoring uses only the recorded workspace map; run preparation separately.'
}
if ($case.PSObject.Properties.Name -notcontains 'development-plan') {
$problems.Add("${id}: development-plan is required.") | Out-Null
continue
$Root = Get-GuidanceSafePath $Root -Directory
if (-not $ManifestPath) { $ManifestPath = Join-Path $Root 'evaluation\development-guidance-fixtures.json' }
$ManifestPath = Get-GuidanceSafePath $ManifestPath -File
$manifest = Read-GuidanceJson $ManifestPath
Assert-GuidanceManifest $manifest $Root
$caseIds = @($manifest.cases | ForEach-Object { $_.id })
$workspaces = [ordered]@{}
$baseline = $null
if ($WorkspaceMapPath) {
$WorkspaceMapPath = Get-GuidanceSafePath $WorkspaceMapPath -File
$map = Read-GuidanceJson $WorkspaceMapPath
Assert-GuidanceObject $map 'workspace map'
if ($map.Count -ne $caseIds.Count) { throw 'Workspace map must bind exactly every manifest case.' }
foreach ($id in $caseIds) {
Assert-GuidanceString $map[$id] 'workspace map value'
if (-not [IO.Path]::IsPathFullyQualified($map[$id])) { throw 'Workspace roots must be absolute.' }
$workspaces[$id] = Get-GuidanceSafePath $map[$id] -Directory
}
}
$plan = $case.'development-plan'
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
[string]$case.expectedKind
} else {
''
if ($ResultsDirectory) {
$BaselinePath = Get-GuidanceSafePath $BaselinePath -File
if ($BaselineSha256 -cnotmatch '^[0-9A-Fa-f]{64}$') {
throw 'BaselineSha256 must be a SHA256 digest.'
}
if ((Get-GuidanceHash $BaselinePath) -ne $BaselineSha256) {
throw 'Runner baseline digest mismatch.'
}
$baseline = Read-GuidanceJson $BaselinePath
Assert-GuidanceObject $baseline 'baseline' @('version', 'kind', 'root', 'manifestPath', 'manifestSha256', 'workspaces', 'rootSnapshot')
if ($baseline.version -ne 1 -or $baseline.kind -cne 'bcquality-guidance-runner-baseline') {
throw 'Unsupported runner baseline.'
}
if ($baseline.root -cne $Root -or $baseline.manifestPath -cne $ManifestPath -or
$baseline.manifestSha256 -ne (Get-GuidanceHash $ManifestPath)) {
throw 'Runner baseline does not match the knowledge root or manifest.'
}
Assert-GuidanceObject $baseline.workspaces 'baseline workspaces'
if ($baseline.workspaces.Count -ne $caseIds.Count) { throw 'Runner baseline case set mismatch.' }
foreach ($id in $caseIds) {
$entry = $baseline.workspaces[$id]
Assert-GuidanceObject $entry 'baseline workspace entry' @('caseId', 'root', 'snapshot')
if ($entry.caseId -cne (Get-GuidanceCaseId $id)) { throw 'Runner baseline case identity mismatch.' }
$workspaces[$id] = Get-GuidanceSafePath $entry.root -Directory
}
}
if ($validKinds -notcontains $expectedKind) {
$problems.Add("${id}: expectedKind is invalid.") | Out-Null
}
$isBcfixHandoff = (
$plan.PSObject.Properties.Name -contains 'format' -and
[string]$plan.format -eq 'BCFIX-HANDOFF'
)
if ($isBcfixHandoff) {
$requiredHandoffFields = @(
'version', 'issue', 'phase', 'status', 'baton', 'rootCause',
'harnessMap', 'iterationsUsed', 'filesCommitted', 'lastTestResult',
'deadEnds', 'nextStep'
)
foreach ($field in $requiredHandoffFields) {
if ($plan.PSObject.Properties.Name -notcontains $field) {
$problems.Add("${id}: BCFIX-HANDOFF is missing '$field'.") | Out-Null
$protectedRoots = @($Root) + @($workspaces.Values)
for ($i = 0; $i -lt $protectedRoots.Count; $i++) {
for ($j = $i + 1; $j -lt $protectedRoots.Count; $j++) {
if ((Test-GuidanceWithin $protectedRoots[$i] $protectedRoots[$j]) -or
(Test-GuidanceWithin $protectedRoots[$j] $protectedRoots[$i])) {
throw 'Knowledge checkout and case workspaces must be distinct, non-overlapping roots.'
}
}
$handoffVersion = if ($plan.PSObject.Properties.Name -contains 'version') { [int]$plan.version } else { 0 }
$handoffPhase = if ($plan.PSObject.Properties.Name -contains 'phase') { [string]$plan.phase } else { '' }
$handoffStatus = if ($plan.PSObject.Properties.Name -contains 'status') { [string]$plan.status } else { '' }
if ($handoffVersion -ne 1) {
$problems.Add("${id}: only BCFIX-HANDOFF version 1 is supported.") | Out-Null
}
if ($handoffPhase -notin @('plan', 'baseline', 'implement', 'pr')) {
$problems.Add("${id}: BCFIX-HANDOFF phase is invalid.") | Out-Null
}
if ($handoffStatus -notin @('in-progress', 'paused', 'done')) {
$problems.Add("${id}: BCFIX-HANDOFF status is invalid.") | Out-Null
}
} elseif (
$plan.PSObject.Properties.Name -notcontains 'kind' -or
$validKinds -notcontains [string]$plan.kind
) {
$problems.Add("${id}: development-plan.kind is invalid.") | Out-Null
}
if ([string]::IsNullOrWhiteSpace((Get-PlanRequest $plan))) {
$problems.Add("${id}: development-plan must provide request or BCFIX nextStep.") | Out-Null
}
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
}
foreach ($reference in $expectedKnowledge) {
$reference = [string]$reference
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
foreach ($artifact in @($BaselinePath, $WorkspaceMapPath, $PrepareDirectory, $ResultsDirectory)) {
if ($artifact) {
$safeArtifact = Get-GuidanceSafePath $artifact -AllowMissing
foreach ($protectedRoot in $protectedRoots) {
if (Test-GuidanceWithin $safeArtifact $protectedRoot) {
throw 'Runner artifacts must be outside target workspaces and the knowledge checkout.'
}
}
}
}
}
if ($problems.Count) {
Write-Host "Development guidance fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
if ($CaptureBaseline) {
$BaselinePath = Get-GuidanceSafePath $BaselinePath -AllowMissing
if (Test-Path -LiteralPath $BaselinePath) { throw 'Baseline already exists; capture never overwrites evidence.' }
$snapshots = [ordered]@{}
foreach ($id in $caseIds) {
$snapshots[$id] = [ordered]@{
caseId = Get-GuidanceCaseId $id
root = $workspaces[$id]
snapshot = Get-GuidanceSnapshot $workspaces[$id] -Target
}
}
$record = [ordered]@{
kind = 'bcquality-guidance-runner-baseline'
version = 1
root = $Root
manifestPath = $ManifestPath
manifestSha256 = Get-GuidanceHash $ManifestPath
rootSnapshot = Get-GuidanceSnapshot $Root
workspaces = $snapshots
}
Write-GuidanceNewJson $BaselinePath $record
Write-Host "Guidance baseline captured. Runner SHA256: $(Get-GuidanceHash $BaselinePath)"
} elseif ($PrepareDirectory) {
$PrepareDirectory = Get-GuidanceSafePath $PrepareDirectory -AllowMissing
if ((Test-Path -LiteralPath $PrepareDirectory) -and
@(Get-ChildItem -LiteralPath $PrepareDirectory -Force).Count) {
throw 'PrepareDirectory must be new or empty; existing requests/evidence are never overwritten.'
}
[IO.Directory]::CreateDirectory($PrepareDirectory) | Out-Null
# The index builder walks recursively; reject linked corpus paths first.
Assert-GuidanceTree $Root
& (Join-Path $Root 'tools\Build-KnowledgeIndex.ps1') -BCQualityRoot $Root `
-IndexPath (Join-Path $PrepareDirectory 'knowledge-index.json') | Out-Null
$skillInstructions = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $manifest.skill))
foreach ($case in $manifest.cases) {
$modelId = Get-GuidanceCaseId $case.id
$request = [ordered]@{
protocol = 'Run the supplied read-only skill on the runner-assigned repository and existing plan. Return only caseId and guidanceReport. The runner captures evidence before invocation; do not capture or modify it. Do not create artifacts in the target or knowledge checkout.'
caseId = $modelId
skill = $manifest.skill
skillInstructions = $skillInstructions
knowledgeIndex = Join-Path $PrepareDirectory 'knowledge-index.json'
knowledgeRoot = $Root
'task-context' = [ordered]@{
goal = Get-GuidancePlanRequest $case.'development-plan'
'inputs-available' = @('development-plan', 'repository')
'bc-version' = $case.context.'bc-version'
technologies = $case.context.technologies
countries = $case.context.countries
'application-area' = $case.context.'application-area'
}
'development-plan' = $case.'development-plan'
resultSchema = Get-GuidanceResultSchema $modelId
}
if ($workspaces.Count) { $request.repository = $workspaces[$case.id] }
Write-GuidanceNewJson (Join-Path $PrepareDirectory "request-$modelId.json") $request
}
Write-Host "Development guidance preparation PASSED: $($caseIds.Count) case(s)."
} elseif ($ResultsDirectory) {
$ResultsDirectory = Get-GuidanceSafePath $ResultsDirectory -Directory
$failures = [Collections.Generic.List[string]]::new()
if (-not (Test-GuidanceSnapshotEqual $baseline.rootSnapshot (Get-GuidanceSnapshot $Root))) {
$failures.Add('Knowledge checkout identity/content changed after baseline capture.')
}
foreach ($case in $manifest.cases) {
$id = $case.id
try {
if (-not (Test-GuidanceSnapshotEqual $baseline.workspaces[$id].snapshot `
(Get-GuidanceSnapshot $workspaces[$id] -Target))) {
throw 'Target repository identity/content changed after baseline capture.'
}
$resultPath = Get-GuidanceSafePath (Join-Path $ResultsDirectory "result-$(Get-GuidanceCaseId $id).json") -File
$result = Read-GuidanceJson $resultPath
Assert-GuidanceResult $result $case $manifest $Root $workspaces[$id]
} catch {
# Only evaluator-authored diagnostics are printed, never model or file contents.
$failures.Add("${id}: $(Get-GuidanceDiagnostic $_)")
}
}
if ($failures.Count) {
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):"
$failures | ForEach-Object { Write-Host " - $_" }
exit 1
}
Write-Host "Development guidance scoring PASSED: $($caseIds.Count) case(s)."
} else {
Write-Host "Development guidance fixture validation PASSED: $($caseIds.Count) case(s)."
}
} catch {
Write-Host "Development guidance FAILED: $(Get-GuidanceDiagnostic $_)"
exit 1
}
if ($PrepareDirectory) {
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-guidance-evaluation'
if (Test-Path -LiteralPath $PrepareDirectory) {
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
throw "PrepareDirectory is not empty and is not a BCQuality development-guidance evaluation directory: $PrepareDirectory"
}
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
Get-ChildItem -LiteralPath $PrepareDirectory -File |
Where-Object {
$_.Name -eq 'knowledge-index.json' -or
$_.Name -like 'request-*.json' -or
$_.Name -like 'result-*.json'
} |
Remove-Item -Force
}
} else {
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
}
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development-guidance evaluation directory' -Encoding UTF8
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
foreach ($case in @($manifest.cases)) {
$modelId = Get-ModelCaseId ([string]$case.id)
[ordered]@{
protocol = 'Run the supplied AL development-plan skill read-only in a clean fixture repository. Return only guidanceReport using the supplied schema and retain the repository for read-only verification.'
caseId = $modelId
skill = $skillPath
skillInstructions = $skillInstructions
knowledgeIndex = 'knowledge-index.json'
'task-context' = [ordered]@{
goal = Get-PlanRequest $case.'development-plan'
'inputs-available' = @('development-plan', 'repository')
technologies = @($case.context.technologies)
countries = @($case.context.countries)
'application-area' = @($case.context.'application-area')
}
'development-plan' = $case.'development-plan'
resultSchema = [ordered]@{
caseId = $modelId
workspaceRoot = 'absolute path to the retained clean fixture repository'
guidanceReport = [ordered]@{
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
'outcome-reason' = 'required for partial or failed'
summary = [ordered]@{
request = 'planned change'
kind = 'feature | bug | refactor | upgrade | maintenance'
candidates = 0
selected = 0
}
context = [ordered]@{
'bc-version' = 'resolved target or unknown'
technologies = @('al')
countries = @('w1')
'application-area' = @('all')
unknown = @()
}
knowledge = @([ordered]@{
path = 'repo-relative article path'
sha = 'optional commit sha'
'used-for' = 'plan decision'
constraints = @('faithful normative constraint')
'sample-paths' = @()
})
'validation-considerations' = @([ordered]@{
id = 'stable id'
reason = 'why evidence is needed'
evidence = 'evidence implementation should obtain'
})
suppressed = @()
unresolved = @()
}
}
} | ConvertTo-Json -Depth 15 |
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
}
}
if ($ResultsDirectory) {
$failures = [System.Collections.Generic.List[string]]::new()
foreach ($case in @($manifest.cases)) {
$modelId = Get-ModelCaseId ([string]$case.id)
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
$failures.Add("$($case.id): missing result file.") | Out-Null
continue
}
try {
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
} catch {
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
continue
}
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
continue
}
if ($result.PSObject.Properties.Name -notcontains 'guidanceReport') {
$failures.Add("$($case.id): guidanceReport is missing.") | Out-Null
continue
}
$report = $result.guidanceReport
foreach ($requiredField in @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
$failures.Add("$($case.id): guidance report is missing '$requiredField'.") | Out-Null
}
}
$skillId = if (
$report.PSObject.Properties.Name -contains 'skill' -and
$report.skill.PSObject.Properties.Name -contains 'id'
) { [string]$report.skill.id } else { '' }
if ($skillId -ne 'al-development-plan') {
$failures.Add("$($case.id): guidance skill is '$skillId'.") | Out-Null
}
if ($report.PSObject.Properties.Name -notcontains 'outcome' -or [string]$report.outcome -ne 'completed') {
$failures.Add("$($case.id): guidance outcome is not completed.") | Out-Null
}
$kind = if (
$report.PSObject.Properties.Name -contains 'summary' -and
$report.summary.PSObject.Properties.Name -contains 'kind'
) { [string]$report.summary.kind } else { '' }
if ($kind -ne [string]$case.expectedKind) {
$failures.Add("$($case.id): summary.kind '$kind' does not match the plan.") | Out-Null
}
[object[]]$knowledgeEntries = @()
if ($report.PSObject.Properties.Name -contains 'knowledge') {
$knowledgeEntries = @($report.knowledge)
}
$usedKnowledge = @()
foreach ($entry in $knowledgeEntries) {
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
[object[]]$constraints = @()
if ($entry.PSObject.Properties.Name -contains 'constraints') {
$constraints = @($entry.constraints)
}
if ([string]::IsNullOrWhiteSpace($path) -or
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
$failures.Add("$($case.id): invalid knowledge path '$path'.") | Out-Null
} else {
$usedKnowledge += $path
}
if ([string]::IsNullOrWhiteSpace($usedFor) -or -not $constraints.Count) {
$failures.Add("$($case.id): '$path' lacks used-for or constraints.") | Out-Null
}
if ($entry.PSObject.Properties.Name -contains 'sample-paths') {
foreach ($samplePath in @($entry.'sample-paths')) {
$samplePath = [string]$samplePath
if ([string]::IsNullOrWhiteSpace($samplePath) -or
-not (Test-Path -LiteralPath (Join-Path $Root $samplePath) -PathType Leaf)) {
$failures.Add("$($case.id): invalid sample path '$samplePath'.") | Out-Null
}
}
}
}
$required = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
$matched = @($required | Where-Object { $usedKnowledge -contains $_ }).Count
$recall = if ($required.Count) { $matched / $required.Count } else { 1.0 }
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
}
$accepted = @(
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
ForEach-Object { [string]$_ } |
Sort-Object -Unique
)
$acceptedUsed = @($usedKnowledge | Where-Object { $accepted -contains $_ }).Count
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
}
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') {
[string]$result.workspaceRoot
} else {
''
}
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
} else {
$status = @(& git -C $workspaceRoot status --porcelain)
if ($LASTEXITCODE -ne 0) {
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
} elseif ($status.Count) {
$failures.Add("$($case.id): guidance skill changed the target repository.") | Out-Null
}
}
}
if ($failures.Count) {
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
exit 1
}
Write-Host "Development guidance scoring PASSED: $(@($manifest.cases).Count) case(s)."
} else {
Write-Host "Development guidance fixture validation PASSED: $(@($manifest.cases).Count) case(s)."
}

240
tools/Test-LearnCoverage.ps1
View file

@ -1,240 +0,0 @@
<#
.SYNOPSIS
Validates Microsoft Learn ingestion coverage and reports progress.
#>
[CmdletBinding()]
param(
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
[string] $CatalogPath,
[string] $CoveragePath,
[switch] $Json
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$Root = (Resolve-Path -LiteralPath $Root).Path
if (-not $CatalogPath) {
$CatalogPath = Join-Path $Root 'coverage/microsoft-learn-developer-catalog.json'
}
if (-not $CoveragePath) {
$CoveragePath = Join-Path $Root 'coverage/learn-coverage.json'
}
foreach ($path in @($CatalogPath, $CoveragePath)) {
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
throw "Coverage input not found: $path"
}
}
$catalog = Get-Content -LiteralPath $CatalogPath -Raw | ConvertFrom-Json
$coverage = Get-Content -LiteralPath $CoveragePath -Raw | ConvertFrom-Json
$problems = [System.Collections.Generic.List[string]]::new()
if ($catalog.version -ne 1) {
$problems.Add("Unsupported catalog version: $($catalog.version)") | Out-Null
}
if ($coverage.version -ne 1) {
$problems.Add("Unsupported coverage version: $($coverage.version)") | Out-Null
}
if ([string]$coverage.catalog -ne 'coverage/microsoft-learn-developer-catalog.json') {
$problems.Add("Coverage catalog path must be coverage/microsoft-learn-developer-catalog.json.") | Out-Null
}
$catalogUnits = @{}
foreach ($unit in @($catalog.units)) {
$uid = [string]$unit.uid
if ($catalogUnits.ContainsKey($uid)) {
$problems.Add("Duplicate catalog unit uid: $uid") | Out-Null
} else {
$catalogUnits[$uid] = $unit
}
if ([string]::IsNullOrWhiteSpace([string]$unit.title)) {
$problems.Add("${uid}: catalog unit title is empty.") | Out-Null
}
if ([string]::IsNullOrWhiteSpace([string]$unit.url) -or
-not ([string]$unit.url).StartsWith('https://learn.microsoft.com/', [System.StringComparison]::OrdinalIgnoreCase)) {
$problems.Add("${uid}: catalog unit URL is missing or not a Microsoft Learn URL.") | Out-Null
}
}
$catalogModules = @{}
foreach ($module in @($catalog.modules)) {
$uid = [string]$module.uid
if ($catalogModules.ContainsKey($uid)) {
$problems.Add("Duplicate catalog module uid: $uid") | Out-Null
} else {
$catalogModules[$uid] = $module
}
foreach ($unitUid in @($module.unitUids)) {
if (-not $catalogUnits.ContainsKey([string]$unitUid)) {
$problems.Add("${uid}: references unknown unit '$unitUid'.") | Out-Null
}
}
}
$catalogPaths = @{}
foreach ($path in @($catalog.learningPaths)) {
$uid = [string]$path.uid
if ($catalogPaths.ContainsKey($uid)) {
$problems.Add("Duplicate catalog learning-path uid: $uid") | Out-Null
} else {
$catalogPaths[$uid] = $path
}
foreach ($moduleUid in @($path.moduleUids)) {
if (-not $catalogModules.ContainsKey([string]$moduleUid)) {
$problems.Add("${uid}: references unknown module '$moduleUid'.") | Out-Null
}
}
}
if ([int]$catalog.counts.units -ne $catalogUnits.Count) {
$problems.Add("Catalog unit count does not match units array.") | Out-Null
}
if ([int]$catalog.counts.modules -ne $catalogModules.Count) {
$problems.Add("Catalog module count does not match modules array.") | Out-Null
}
if ([int]$catalog.counts.learningPaths -ne $catalogPaths.Count) {
$problems.Add("Catalog learning-path count does not match learningPaths array.") | Out-Null
}
foreach ($unit in @($catalog.units)) {
foreach ($moduleUid in @($unit.moduleUids)) {
if (-not $catalogModules.ContainsKey([string]$moduleUid)) {
$problems.Add("$($unit.uid): references unknown module '$moduleUid'.") | Out-Null
} elseif (@($catalogModules[[string]$moduleUid].unitUids) -notcontains [string]$unit.uid) {
$problems.Add("$($unit.uid): module '$moduleUid' does not link back to the unit.") | Out-Null
}
}
}
$validStatuses = @('in-progress', 'complete')
$validDispositions = @('candidate', 'authored', 'covered-existing', 'rejected', 'deferred')
$seenUnits = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
$seenOutcomes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
$articlePaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
$dispositionCounts = @{}
foreach ($disposition in $validDispositions) {
$dispositionCounts[$disposition] = 0
}
foreach ($unitCoverage in @($coverage.units)) {
$uid = [string]$unitCoverage.uid
if (-not $seenUnits.Add($uid)) {
$problems.Add("Duplicate coverage unit uid: $uid") | Out-Null
}
if (-not $catalogUnits.ContainsKey($uid)) {
$problems.Add("Coverage unit is absent from the catalog: $uid") | Out-Null
}
$reviewStatus = [string]$unitCoverage.reviewStatus
if ($validStatuses -notcontains $reviewStatus) {
$problems.Add("${uid}: invalid reviewStatus '$reviewStatus'.") | Out-Null
}
[object[]]$outcomes = @()
if ($unitCoverage.PSObject.Properties.Name -contains 'outcomes') {
$outcomes = @($unitCoverage.outcomes)
}
if ($reviewStatus -eq 'in-progress' -and -not $outcomes.Count) {
$problems.Add("${uid}: an in-progress unit must contain at least one outcome.") | Out-Null
}
foreach ($outcome in $outcomes) {
$id = [string]$outcome.id
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
$problems.Add("${uid}: outcome id must be kebab-case: '$id'.") | Out-Null
} elseif (-not $seenOutcomes.Add($id)) {
$problems.Add("Duplicate outcome id: $id") | Out-Null
}
$disposition = [string]$outcome.disposition
if ($validDispositions -notcontains $disposition) {
$problems.Add("${id}: invalid disposition '$disposition'.") | Out-Null
continue
}
$dispositionCounts[$disposition]++
if ([string]::IsNullOrWhiteSpace([string]$outcome.title)) {
$problems.Add("${id}: title is required.") | Out-Null
}
if ($disposition -in @('candidate', 'authored', 'covered-existing')) {
if ([string]::IsNullOrWhiteSpace([string]$outcome.domain)) {
$problems.Add("${id}: domain is required for disposition '$disposition'.") | Out-Null
}
}
[object[]]$paths = @()
if ($outcome.PSObject.Properties.Name -contains 'articlePaths') {
$paths = @($outcome.articlePaths)
}
if ($disposition -in @('authored', 'covered-existing')) {
if (-not $paths.Count) {
$problems.Add("${id}: articlePaths is required for disposition '$disposition'.") | Out-Null
}
foreach ($relativePath in $paths) {
$relativePath = [string]$relativePath
if ($relativePath.Contains('\') -or -not $relativePath.EndsWith('.md')) {
$problems.Add("${id}: article path must be a forward-slash .md path: $relativePath") | Out-Null
continue
}
if (-not (Test-Path -LiteralPath (Join-Path $Root $relativePath) -PathType Leaf)) {
$problems.Add("${id}: article does not exist: $relativePath") | Out-Null
}
$articlePaths.Add($relativePath) | Out-Null
}
} elseif ($paths.Count) {
$problems.Add("${id}: disposition '$disposition' must not set articlePaths.") | Out-Null
}
$rationale = if ($outcome.PSObject.Properties.Name -contains 'rationale') {
[string]$outcome.rationale
} else {
''
}
if ($disposition -in @('rejected', 'deferred') -and
[string]::IsNullOrWhiteSpace($rationale)) {
$problems.Add("${id}: rationale is required for disposition '$disposition'.") | Out-Null
}
}
}
if ($problems.Count) {
Write-Host "Learn coverage validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
exit 1
}
$complete = @($coverage.units | Where-Object reviewStatus -eq 'complete').Count
$inProgress = @($coverage.units | Where-Object reviewStatus -eq 'in-progress').Count
$summary = [ordered]@{
catalogUnits = $catalogUnits.Count
completeUnits = $complete
inProgressUnits = $inProgress
unreviewedUnits = $catalogUnits.Count - $complete - $inProgress
candidateOutcomes = $dispositionCounts.candidate
authoredOutcomes = $dispositionCounts.authored
coveredExistingOutcomes = $dispositionCounts.'covered-existing'
rejectedOutcomes = $dispositionCounts.rejected
deferredOutcomes = $dispositionCounts.deferred
referencedArticles = $articlePaths.Count
}
if ($Json) {
$summary | ConvertTo-Json
} else {
$message = (
'Learn coverage: {0} units; {1} complete, {2} in progress, {3} unreviewed; ' +
'{4} candidate outcomes, {5} authored outcomes, {6} existing-coverage outcomes.'
) -f @(
$summary.catalogUnits,
$summary.completeUnits,
$summary.inProgressUnits,
$summary.unreviewedUnits,
$summary.candidateOutcomes,
$summary.authoredOutcomes,
$summary.coveredExistingOutcomes
)
Write-Host $message
}

262
tools/Update-LearnCatalog.ps1
View file

@ -1,262 +0,0 @@
<#
.SYNOPSIS
Updates the committed Microsoft Learn Business Central developer catalog.
.DESCRIPTION
Filters a Microsoft Learn catalog payload to modules tagged for both
dynamics-business-central and developer. The output is intentionally only
source metadata; editorial dispositions live in coverage/learn-coverage.json.
The legacy unauthenticated catalog endpoint remains the default while it is
available. Use -CatalogPath with an export from the authenticated Learn
Platform API when the legacy endpoint is retired.
#>
[CmdletBinding(DefaultParameterSetName = 'Remote')]
param(
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
[Parameter(ParameterSetName = 'Remote')]
[uri] $CatalogUri = 'https://learn.microsoft.com/api/catalog/?locale=en-us',
[Parameter(Mandatory, ParameterSetName = 'File')]
[string] $CatalogPath,
[string] $OutputPath
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$Root = (Resolve-Path -LiteralPath $Root).Path
if (-not $OutputPath) {
$OutputPath = Join-Path $Root 'coverage/microsoft-learn-developer-catalog.json'
}
function Get-CanonicalUrl {
param([string] $Url)
if ([string]::IsNullOrWhiteSpace($Url)) {
return $null
}
$parsed = [uri]$Url
return "$($parsed.Scheme)://$($parsed.Host)$($parsed.AbsolutePath)".TrimEnd('/')
}
function Get-CatalogProperty {
param(
[object] $Object,
[string] $Name,
[object] $Default = $null
)
$property = $Object.PSObject.Properties[$Name]
if ($property) {
return $property.Value
}
return $Default
}
function Get-LatestTimestamp {
param([object[]] $Values)
$timestamps = @(
$Values |
Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } |
ForEach-Object { [datetimeoffset]::Parse([string]$_) } |
Sort-Object
)
if (-not $timestamps.Count) {
return $null
}
return $timestamps[-1].ToUniversalTime().ToString(
"yyyy-MM-dd'T'HH:mm:ss'Z'",
[System.Globalization.CultureInfo]::InvariantCulture
)
}
function Convert-ToUtcTimestamp {
param([object] $Value)
return ([datetimeoffset]$Value).ToUniversalTime().ToString(
"yyyy-MM-dd'T'HH:mm:ss'Z'",
[System.Globalization.CultureInfo]::InvariantCulture
)
}
$catalog = if ($PSCmdlet.ParameterSetName -eq 'File') {
Get-Content -LiteralPath $CatalogPath -Raw | ConvertFrom-Json
} else {
Invoke-RestMethod -Uri $CatalogUri
}
$product = 'dynamics-business-central'
$role = 'developer'
$modules = @(
$catalog.modules |
Where-Object {
@(Get-CatalogProperty $_ 'products' @()) -contains $product -and
@(Get-CatalogProperty $_ 'roles' @()) -contains $role
} |
Sort-Object uid
)
$moduleIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
foreach ($module in $modules) {
$moduleIds.Add([string]$module.uid) | Out-Null
}
$learningPaths = @(
$catalog.learningPaths |
Where-Object {
$path = $_
$pathModules = @(Get-CatalogProperty $path 'modules' @())
@($pathModules | Where-Object { $moduleIds.Contains([string]$_) }).Count -gt 0 -and
@(Get-CatalogProperty $path 'products' @()) -contains $product -and
@(Get-CatalogProperty $path 'roles' @()) -contains $role
} |
Sort-Object uid
)
$pathIdsByModule = @{}
foreach ($path in $learningPaths) {
foreach ($moduleId in @(Get-CatalogProperty $path 'modules' @())) {
if (-not $moduleIds.Contains([string]$moduleId)) {
continue
}
if (-not $pathIdsByModule.ContainsKey([string]$moduleId)) {
$pathIdsByModule[[string]$moduleId] = [System.Collections.Generic.List[string]]::new()
}
$pathIdsByModule[[string]$moduleId].Add([string]$path.uid)
}
}
$unitById = @{}
foreach ($unit in @(Get-CatalogProperty $catalog 'units' @())) {
$unitById[[string]$unit.uid] = $unit
}
$moduleIdsByUnit = @{}
$moduleById = @{}
$unitIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
foreach ($module in $modules) {
$moduleById[[string]$module.uid] = $module
foreach ($unitId in @(Get-CatalogProperty $module 'units' @())) {
$unitId = [string]$unitId
$unitIds.Add($unitId) | Out-Null
if (-not $moduleIdsByUnit.ContainsKey($unitId)) {
$moduleIdsByUnit[$unitId] = [System.Collections.Generic.List[string]]::new()
}
$moduleIdsByUnit[$unitId].Add([string]$module.uid)
}
}
$pathRecords = @(
foreach ($path in $learningPaths) {
[ordered]@{
uid = [string]$path.uid
title = [string]$path.title
url = Get-CanonicalUrl ([string]$path.url)
lastModified = if (Get-CatalogProperty $path 'last_modified') {
Convert-ToUtcTimestamp (Get-CatalogProperty $path 'last_modified')
} else {
$null
}
durationInMinutes = [int](Get-CatalogProperty $path 'duration_in_minutes' 0)
moduleUids = @(
@(Get-CatalogProperty $path 'modules' @()) |
Where-Object { $moduleIds.Contains([string]$_) } |
ForEach-Object { [string]$_ } |
Sort-Object -Unique
)
}
}
)
$moduleRecords = @(
foreach ($module in $modules) {
[ordered]@{
uid = [string]$module.uid
title = [string]$module.title
summary = [string](Get-CatalogProperty $module 'summary' '')
url = Get-CanonicalUrl ([string]$module.url)
lastModified = Convert-ToUtcTimestamp (Get-CatalogProperty $module 'last_modified')
durationInMinutes = [int](Get-CatalogProperty $module 'duration_in_minutes' 0)
levels = @(@(Get-CatalogProperty $module 'levels' @()) | ForEach-Object { [string]$_ } | Sort-Object -Unique)
subjects = @(@(Get-CatalogProperty $module 'subjects' @()) | ForEach-Object { [string]$_ } | Sort-Object -Unique)
learningPathUids = if ($pathIdsByModule.ContainsKey([string]$module.uid)) {
@($pathIdsByModule[[string]$module.uid] | Sort-Object -Unique)
} else {
@()
}
unitUids = @(@(Get-CatalogProperty $module 'units' @()) | ForEach-Object { [string]$_ })
}
}
)
$unitRecords = @(
foreach ($unitId in @($unitIds | Sort-Object)) {
if (-not $unitById.ContainsKey($unitId)) {
throw "Catalog module references missing unit '$unitId'."
}
$unit = $unitById[$unitId]
$unitUrl = [string](Get-CatalogProperty $unit 'url' '')
if ([string]::IsNullOrWhiteSpace($unitUrl)) {
$moduleId = [string]@($moduleIdsByUnit[$unitId])[0]
$moduleUrl = Get-CanonicalUrl ([string]$moduleById[$moduleId].url)
$unitSlug = if ($unitId.StartsWith("$moduleId.", [System.StringComparison]::Ordinal)) {
$unitId.Substring($moduleId.Length + 1)
} else {
@($unitId -split '\.')[-1]
}
$unitUrl = "$moduleUrl/$unitSlug"
}
[ordered]@{
uid = $unitId
title = [string]$unit.title
url = Get-CanonicalUrl $unitUrl
lastModified = if (Get-CatalogProperty $unit 'last_modified') {
Convert-ToUtcTimestamp (Get-CatalogProperty $unit 'last_modified')
} else {
$null
}
durationInMinutes = [int](Get-CatalogProperty $unit 'duration_in_minutes' 0)
moduleUids = @($moduleIdsByUnit[$unitId] | Sort-Object -Unique)
}
}
)
$snapshot = [ordered]@{
version = 1
source = [ordered]@{
provider = 'Microsoft Learn'
catalogUri = if ($PSCmdlet.ParameterSetName -eq 'Remote') { [string]$CatalogUri } else { $null }
locale = 'en-us'
filters = [ordered]@{
product = $product
role = $role
}
catalogLastModified = Get-LatestTimestamp @(
@($moduleRecords | ForEach-Object lastModified) +
@($unitRecords | ForEach-Object lastModified) +
@($pathRecords | ForEach-Object lastModified)
)
}
counts = [ordered]@{
learningPaths = $pathRecords.Count
modules = $moduleRecords.Count
units = $unitRecords.Count
}
learningPaths = $pathRecords
modules = $moduleRecords
units = $unitRecords
}
$outputDirectory = Split-Path -Parent $OutputPath
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null
}
$snapshot | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $OutputPath -Encoding UTF8
Write-Host "Microsoft Learn developer catalog: $($pathRecords.Count) paths, $($moduleRecords.Count) modules, $($unitRecords.Count) units."
Write-Host "Catalog: $OutputPath"