mirror of
https://github.com/microsoft/BCQuality.git
synced 2026-10-05 14:46:55 +01:00
Narrow AL development to read-only plan guidance
Retain shared knowledge enrichment and review guidance; defer standalone implementation and source-ingestion tracking. Add runner-owned baseline evidence, contract regressions, and explicit consumer/pilot boundaries. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
parent
f6fca1d56d
commit
1deac52a53
26 changed files with 1486 additions and 11037 deletions
|
|
@ -8,7 +8,7 @@
|
||||||
{
|
{
|
||||||
"name": "bcquality",
|
"name": "bcquality",
|
||||||
"source": "./",
|
"source": "./",
|
||||||
"description": "Business Central AL quality knowledge base and skills, packaged as an installable plugin. Exposes development and review adapters while preserving BCQuality's internal Entry and action-skill protocols.",
|
"description": "Business Central AL quality knowledge base and skills, packaged as an installable plugin. Exposes read-only plan enrichment and review adapters through BCQuality's Entry protocol.",
|
||||||
"version": "0.3.0",
|
"version": "0.3.0",
|
||||||
"skills": [
|
"skills": [
|
||||||
"./skills/"
|
"./skills/"
|
||||||
|
|
|
||||||
88
.github/scripts/validate_frontmatter.py
vendored
88
.github/scripts/validate_frontmatter.py
vendored
|
|
@ -39,7 +39,6 @@ ACTION_SKILL_REQUIRED_KEYS = {
|
||||||
}
|
}
|
||||||
ACTION_SKILL_OPTIONAL_KEYS = {
|
ACTION_SKILL_OPTIONAL_KEYS = {
|
||||||
"bc-version", "technologies", "countries", "application-area", "sub-skills",
|
"bc-version", "technologies", "countries", "application-area", "sub-skills",
|
||||||
"quality-skill", "quality-round-limit", "guidance-skill",
|
|
||||||
}
|
}
|
||||||
META_SKILL_REQUIRED_KEYS = {"kind", "id", "version", "title"}
|
META_SKILL_REQUIRED_KEYS = {"kind", "id", "version", "title"}
|
||||||
ENTRY_SKILL_REQUIRED_KEYS = {"kind", "id", "version", "title"}
|
ENTRY_SKILL_REQUIRED_KEYS = {"kind", "id", "version", "title"}
|
||||||
|
|
@ -47,10 +46,10 @@ HOST_SKILL_REQUIRED_KEYS = {"name", "description"}
|
||||||
|
|
||||||
STANDARD_INPUTS = {
|
STANDARD_INPUTS = {
|
||||||
"pr-diff", "object-list", "file-path", "repository", "telemetry-query",
|
"pr-diff", "object-list", "file-path", "repository", "telemetry-query",
|
||||||
"development-request", "development-plan",
|
"development-plan",
|
||||||
}
|
}
|
||||||
ALLOWED_OUTPUTS = {
|
ALLOWED_OUTPUTS = {
|
||||||
"findings-report", "implementation-report", "development-guidance-report",
|
"findings-report", "development-guidance-report",
|
||||||
}
|
}
|
||||||
VALID_SAMPLE_KINDS = {"good", "bad"}
|
VALID_SAMPLE_KINDS = {"good", "bad"}
|
||||||
|
|
||||||
|
|
@ -410,33 +409,6 @@ def validate_action_skill(path: Path, parsed: Parsed, report: Report) -> None:
|
||||||
if bad:
|
if bad:
|
||||||
report.error(path, "R20", f"invalid sub-skills paths: {bad}", 1)
|
report.error(path, "R20", f"invalid sub-skills paths: {bad}", 1)
|
||||||
|
|
||||||
if "quality-skill" in fm:
|
|
||||||
quality_skill = fm["quality-skill"]
|
|
||||||
_, err = normalize_repo_md_path(quality_skill)
|
|
||||||
if err:
|
|
||||||
report.error(path, "R31", f"quality-skill {err}", 1)
|
|
||||||
if fm.get("outputs") != ["implementation-report"]:
|
|
||||||
report.error(path, "R31", "quality-skill is valid only with outputs: [implementation-report]", 1)
|
|
||||||
if "quality-round-limit" not in fm:
|
|
||||||
report.error(path, "R31", "quality-skill requires quality-round-limit", 1)
|
|
||||||
|
|
||||||
if "quality-round-limit" in fm:
|
|
||||||
limit = fm["quality-round-limit"]
|
|
||||||
if not isinstance(limit, int) or isinstance(limit, bool) or limit <= 0:
|
|
||||||
report.error(path, "R31", "quality-round-limit must be a positive integer", 1)
|
|
||||||
if "quality-skill" not in fm:
|
|
||||||
report.error(path, "R31", "quality-round-limit requires quality-skill", 1)
|
|
||||||
if fm.get("outputs") != ["implementation-report"]:
|
|
||||||
report.error(path, "R31", "quality-round-limit is valid only with outputs: [implementation-report]", 1)
|
|
||||||
|
|
||||||
if "guidance-skill" in fm:
|
|
||||||
guidance_skill = fm["guidance-skill"]
|
|
||||||
_, err = normalize_repo_md_path(guidance_skill)
|
|
||||||
if err:
|
|
||||||
report.error(path, "R32", f"guidance-skill {err}", 1)
|
|
||||||
if fm.get("outputs") != ["implementation-report"]:
|
|
||||||
report.error(path, "R32", "guidance-skill is valid only with outputs: [implementation-report]", 1)
|
|
||||||
|
|
||||||
# R21 five required sections, in order, each exactly once
|
# R21 five required sections, in order, each exactly once
|
||||||
heads = [h for h, _ in headings_in_order(parsed.body)]
|
heads = [h for h, _ in headings_in_order(parsed.body)]
|
||||||
indices: list[int] = []
|
indices: list[int] = []
|
||||||
|
|
@ -662,58 +634,6 @@ def validate_sub_skills_registry(path: Path, fm: dict[str, Any], root: Path, rep
|
||||||
report.error(path, "R26", f"leaf not registered in sub-skills: {leaf}", 1)
|
report.error(path, "R26", f"leaf not registered in sub-skills: {leaf}", 1)
|
||||||
|
|
||||||
|
|
||||||
def validate_quality_skill(path: Path, fm: dict[str, Any], root: Path, report: Report) -> None:
|
|
||||||
"""R30: implementation quality-skill paths resolve to a findings producer."""
|
|
||||||
quality_skill = fm.get("quality-skill")
|
|
||||||
normalized, err = normalize_repo_md_path(quality_skill)
|
|
||||||
if err or normalized is None:
|
|
||||||
return
|
|
||||||
|
|
||||||
target = root / normalized
|
|
||||||
if not target.is_file():
|
|
||||||
report.error(path, "R30", f"quality-skill does not exist on disk: {normalized}", 1)
|
|
||||||
return
|
|
||||||
if target.resolve() == path.resolve():
|
|
||||||
report.error(path, "R30", "quality-skill must not reference the implementation skill itself", 1)
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
target_parsed = parse_markdown(target.read_text(encoding="utf-8"))
|
|
||||||
except UnicodeDecodeError as e:
|
|
||||||
report.error(path, "R30", f"quality-skill is not valid UTF-8: {e}", 1)
|
|
||||||
return
|
|
||||||
target_outputs = (target_parsed.frontmatter or {}).get("outputs")
|
|
||||||
if target_outputs != ["findings-report"]:
|
|
||||||
report.error(path, "R30", f"quality-skill must emit findings-report: {normalized}", 1)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_guidance_skill(path: Path, fm: dict[str, Any], root: Path, report: Report) -> None:
|
|
||||||
"""R33: implementation guidance-skill paths resolve to a read-only planner."""
|
|
||||||
guidance_skill = fm.get("guidance-skill")
|
|
||||||
normalized, err = normalize_repo_md_path(guidance_skill)
|
|
||||||
if err or normalized is None:
|
|
||||||
return
|
|
||||||
|
|
||||||
target = root / normalized
|
|
||||||
if not target.is_file():
|
|
||||||
report.error(path, "R33", f"guidance-skill does not exist on disk: {normalized}", 1)
|
|
||||||
return
|
|
||||||
if target.resolve() == path.resolve():
|
|
||||||
report.error(path, "R33", "guidance-skill must not reference the implementation skill itself", 1)
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
target_parsed = parse_markdown(target.read_text(encoding="utf-8"))
|
|
||||||
except UnicodeDecodeError as e:
|
|
||||||
report.error(path, "R33", f"guidance-skill is not valid UTF-8: {e}", 1)
|
|
||||||
return
|
|
||||||
target_fm = target_parsed.frontmatter or {}
|
|
||||||
if target_fm.get("outputs") != ["development-guidance-report"]:
|
|
||||||
report.error(path, "R33", f"guidance-skill must emit development-guidance-report: {normalized}", 1)
|
|
||||||
if "development-plan" not in (target_fm.get("inputs") or []):
|
|
||||||
report.error(path, "R33", f"guidance-skill must accept development-plan: {normalized}", 1)
|
|
||||||
|
|
||||||
|
|
||||||
def run(root: Path) -> Report:
|
def run(root: Path) -> Report:
|
||||||
report = Report()
|
report = Report()
|
||||||
skill_records: list[SkillRecord] = []
|
skill_records: list[SkillRecord] = []
|
||||||
|
|
@ -781,11 +701,9 @@ def run(root: Path) -> Report:
|
||||||
others = [q.relative_to(root).as_posix() for q in paths if q != p]
|
others = [q.relative_to(root).as_posix() for q in paths if q != p]
|
||||||
report.error(p, "R24", f"skill id '{sid}' ({kind}) is not unique; also defined in: {others}")
|
report.error(p, "R24", f"skill id '{sid}' ({kind}) is not unique; also defined in: {others}")
|
||||||
|
|
||||||
# Fourth pass: cross-skill references
|
# Fourth pass: R26 sub-skills registry matches leaf files on disk
|
||||||
for path, fm in action_skill_fms:
|
for path, fm in action_skill_fms:
|
||||||
validate_sub_skills_registry(path, fm, root, report)
|
validate_sub_skills_registry(path, fm, root, report)
|
||||||
validate_quality_skill(path, fm, root, report)
|
|
||||||
validate_guidance_skill(path, fm, root, report)
|
|
||||||
|
|
||||||
return report
|
return report
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
name: Validate development coverage
|
name: Validate read-only development guidance
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
|
|
@ -7,20 +7,19 @@ on:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
validate-development-coverage:
|
validate-development-guidance:
|
||||||
runs-on: ubuntu-latest
|
strategy:
|
||||||
|
matrix:
|
||||||
|
os: [ubuntu-latest, windows-latest]
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
steps:
|
steps:
|
||||||
- name: Check out repository
|
- name: Check out repository
|
||||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
|
|
||||||
- name: Validate Microsoft Learn coverage ledger
|
|
||||||
shell: pwsh
|
|
||||||
run: ./tools/Test-LearnCoverage.ps1 -Root .
|
|
||||||
|
|
||||||
- name: Validate and prepare development fixtures
|
|
||||||
shell: pwsh
|
|
||||||
run: ./tools/Test-DevelopmentFixtures.ps1 -Root . -PrepareDirectory "$env:RUNNER_TEMP/bcquality-development-fixtures"
|
|
||||||
|
|
||||||
- name: Validate and prepare development-guidance fixtures
|
- name: Validate and prepare development-guidance fixtures
|
||||||
shell: pwsh
|
shell: pwsh
|
||||||
run: ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory "$env:RUNNER_TEMP/bcquality-development-guidance-fixtures"
|
run: ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory "$env:RUNNER_TEMP/bcquality-development-guidance-fixtures"
|
||||||
|
|
||||||
|
- name: Run credential-free guidance evaluator regressions
|
||||||
|
shell: pwsh
|
||||||
|
run: ./tools/Test-DevelopmentGuidanceEvaluator.ps1
|
||||||
59
README.md
59
README.md
|
|
@ -56,7 +56,7 @@ Skills define how agents consume knowledge. They come in three flavors:
|
||||||
|
|
||||||
READ and DO are read on demand — typically when the first dispatched action skill runs. They are not prerequisites for invoking Entry. WRITE is only used when scaffolding new content.
|
READ and DO are read on demand — typically when the first dispatched action skill runs. They are not prerequisites for invoking Entry. WRITE is only used when scaffolding new content.
|
||||||
|
|
||||||
- **Action skills** — concrete skills that follow the Action Skill template to do real work. Review skills emit findings reports; read-only planning skills emit development-guidance reports; implementation skills emit implementation reports. Action skills live inside the layers that own them (`/microsoft/skills/`, `/community/skills/`, `/custom/skills/`). [`microsoft/skills/development/al-development-plan.md`](microsoft/skills/development/al-development-plan.md) turns an existing plan into knowledge constraints, [`microsoft/skills/development/al-development.md`](microsoft/skills/development/al-development.md) consumes those constraints while implementing features, bugs, refactors, upgrades, and maintenance, and [`microsoft/skills/review/al-code-review.md`](microsoft/skills/review/al-code-review.md) provides the final quality gate.
|
- **Action skills** — concrete skills that follow the Action Skill template. Review skills emit findings reports; read-only plan enrichment emits development-guidance reports. Action skills live inside the layers that own them (`/microsoft/skills/`, `/community/skills/`, `/custom/skills/`). [`microsoft/skills/development/al-development-plan.md`](microsoft/skills/development/al-development-plan.md) turns an existing plan into additional knowledge constraints for the consumer's established workflow. [`microsoft/skills/review/al-code-review.md`](microsoft/skills/review/al-code-review.md) independently reviews the resulting changes.
|
||||||
|
|
||||||
### Agent bootstrapping
|
### Agent bootstrapping
|
||||||
|
|
||||||
|
|
@ -66,7 +66,7 @@ An orchestrator (such as AL-Go) points the agent at BCQuality's URL and provides
|
||||||
|
|
||||||
BCQuality can also be installed directly as a plugin. The plugin registers
|
BCQuality can also be installed directly as a plugin. The plugin registers
|
||||||
host-native adapters for [`al-code-review`](skills/al-code-review/SKILL.md) and
|
host-native adapters for [`al-code-review`](skills/al-code-review/SKILL.md) and
|
||||||
[`al-development`](skills/al-development/SKILL.md). Both adapt
|
[`al-development-plan`](skills/al-development-plan/SKILL.md). Both adapt
|
||||||
the caller's request to the same Entry protocol used by orchestrators.
|
the caller's request to the same Entry protocol used by orchestrators.
|
||||||
|
|
||||||
For GitHub Copilot CLI:
|
For GitHub Copilot CLI:
|
||||||
|
|
@ -81,8 +81,8 @@ must be updated. The name remains distinct from BC-ALAgents' public
|
||||||
`al-review` skill because current hosts may load plugin skill names into one
|
`al-review` skill because current hosts may load plugin skill names into one
|
||||||
shared inventory.
|
shared inventory.
|
||||||
|
|
||||||
Plugin version `0.3.0` adds `al-development`, the knowledge-backed
|
Plugin version `0.3.0` adds `al-development-plan`, a read-only adapter for
|
||||||
implementation adapter for features, bugs, refactors, upgrades, and maintenance.
|
enriching an **existing** plan. It does not generate a plan or implement code.
|
||||||
|
|
||||||
The adapters are intentionally not second implementations:
|
The adapters are intentionally not second implementations:
|
||||||
|
|
||||||
|
|
@ -92,10 +92,10 @@ standalone host skill: skills/al-code-review/SKILL.md
|
||||||
-> review coordinator: microsoft/skills/review/al-code-review.md
|
-> review coordinator: microsoft/skills/review/al-code-review.md
|
||||||
-> domain review leaves
|
-> domain review leaves
|
||||||
|
|
||||||
standalone host skill: skills/al-development/SKILL.md
|
standalone host skill: skills/al-development-plan/SKILL.md
|
||||||
-> routing contract: skills/entry.md
|
-> routing contract: skills/entry.md
|
||||||
-> implementation skill: microsoft/skills/development/al-development.md
|
-> enrichment skill: microsoft/skills/development/al-development-plan.md
|
||||||
-> knowledge-guided implementation + AL review quality gate
|
-> referenced constraints for the consumer's existing workflow (read-only)
|
||||||
```
|
```
|
||||||
|
|
||||||
Only the files under `skills/*/SKILL.md` follow the host's packaging format.
|
Only the files under `skills/*/SKILL.md` follow the host's packaging format.
|
||||||
|
|
@ -143,33 +143,31 @@ Code examples belong in separate files, not in the knowledge file itself. Knowle
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
|
|
||||||
The current curated corpus covers technical AL concerns across Agents, AppSource and compatibility, data modeling, error handling, events, interfaces, performance, privacy, Query objects, security, style, telemetry, testing, UI, upgrade, and web services. Review skills evaluate existing changes against those domains. The `al-development` skill applies them before and during implementation, then runs the review coordinator as a final gate.
|
The current curated corpus covers technical AL concerns across Agents, AppSource and compatibility, data modeling, error handling, events, interfaces, performance, privacy, Query objects, security, style, telemetry, testing, UI, upgrade, and web services. Review skills evaluate existing changes against those domains. The read-only `al-development-plan` interface selects relevant constraints before the consumer implements its own plan.
|
||||||
|
|
||||||
Repository-specific orchestrators do not need to delegate implementation to
|
Repository-specific orchestrators retain planning, implementation, approvals,
|
||||||
`al-development`. They can invoke `al-development-plan` with their existing
|
tests, environment, propagation, and delivery ownership. The intended flow is
|
||||||
plan, feed its read-only guidance report into their own phases, and retain their
|
consumer analysis and normalized plan -> read-only BCQuality guidance ->
|
||||||
specialized environment, test, propagation, and delivery gates.
|
existing implementation phases -> independent final BCQuality review ->
|
||||||
|
delivery. Consumer uptake and a real runtime pilot are follow-up work, not
|
||||||
|
implemented integrations or demonstrated authoring improvements.
|
||||||
|
|
||||||
`al-development` does not silently fall back to generic generation when no
|
`no-knowledge` means no additional applicable BCQuality constraints, with empty
|
||||||
article applies. It returns `no-knowledge` without changing code, making corpus
|
`knowledge`; it does not make a plan unsafe or prevent the consumer from using
|
||||||
coverage visible; callers can use their normal repository workflow or
|
its ordinary gates. Retrieval failures and materially unresolved conditional
|
||||||
contribute the missing Business Central-specific guidance.
|
guidance are distinct outcomes, not empty knowledge. Do not add generic advice
|
||||||
|
just to avoid a `no-knowledge` result.
|
||||||
|
|
||||||
Business Central functional domains (Finance, Supply Chain Management, Manufacturing, Jobs, Warehousing, Service), PowerShell, pipelines, and Power Platform remain valid future repository scope, but they are **not current coverage claims** until corresponding knowledge and action skills exist. Consumers should derive supported review scope from the live knowledge index and dispatched skills, not from roadmap breadth.
|
Business Central functional domains (Finance, Supply Chain Management, Manufacturing, Jobs, Warehousing, Service), PowerShell, pipelines, and Power Platform remain valid future repository scope, but they are **not current coverage claims** until corresponding knowledge and action skills exist. Consumers should derive supported review scope from the live knowledge index and dispatched skills, not from roadmap breadth.
|
||||||
|
|
||||||
## Tracking developer coverage
|
## Evidence and follow-up scope
|
||||||
|
|
||||||
BCQuality tracks source ingestion and implementation capability separately:
|
The [guidance evaluation](evaluation/README.md#read-only-plan-guidance) separates
|
||||||
|
credential-free contract/scorer regressions from external agent and runtime
|
||||||
- [`coverage/microsoft-learn-developer-catalog.json`](coverage/microsoft-learn-developer-catalog.json) is the generated inventory of Business Central developer training.
|
evidence. Prepared requests and fixture counts do not establish compilation,
|
||||||
- [`coverage/learn-coverage.json`](coverage/learn-coverage.json) records editorial progress and the disposition of each extracted concern.
|
test execution, better repairs, or a capability percentage. Consumer adoption,
|
||||||
- [`coverage/development-capabilities.json`](coverage/development-capabilities.json) tracks representative development capabilities and their evaluation fixtures.
|
a pinned baseline comparison and runtime pilot, standalone authoring, and
|
||||||
|
source-ingestion catalog work remain separate follow-ups.
|
||||||
Run `pwsh ./tools/Test-LearnCoverage.ps1` for current source progress and
|
|
||||||
`pwsh ./tools/Test-DevelopmentFixtures.ps1` for capability coverage. Article
|
|
||||||
count alone is not a completion metric: a capability becomes `validated` only
|
|
||||||
after its generated implementation passes compilation, tests, and the review
|
|
||||||
quality gate.
|
|
||||||
|
|
||||||
## How agents consume BCQuality
|
## How agents consume BCQuality
|
||||||
|
|
||||||
|
|
@ -180,7 +178,7 @@ Action skills follow a four-step pattern:
|
||||||
3. **Worklist** — narrow from N candidates to the M that apply to the current task
|
3. **Worklist** — narrow from N candidates to the M that apply to the current task
|
||||||
4. **Action** — apply the relevant knowledge and produce structured output
|
4. **Action** — apply the relevant knowledge and produce structured output
|
||||||
|
|
||||||
Every action skill declares one structured JSON output. Review skills emit a `findings-report`; planning skills emit a read-only `development-guidance-report`; implementation skills emit an `implementation-report` containing the plan, knowledge used, changed files, real validation results, and post-implementation review. All contracts are defined in the Action Skill meta-skill so orchestrators and action skills remain independently evolvable.
|
Every action skill declares one structured JSON output. Review skills emit a `findings-report`; plan enrichment emits a read-only `development-guidance-report`. Both contracts are defined in the Action Skill meta-skill so orchestrators and action skills remain independently evolvable.
|
||||||
|
|
||||||
BCQuality is an **additive** knowledge layer: it augments the agent's review judgement, it does not replace it. Super-skills (such as `al-code-review`) run a self-review pass alongside their sub-skills and surface concerns the agent identified on its own, marked with `from-sub-skill: "agent"` and an empty `references: []` so consumers can render them distinctly from knowledge-backed findings. See [agent-consumption.md](agent-consumption.md) and [`skills/do.md`](skills/do.md) for the full contract.
|
BCQuality is an **additive** knowledge layer: it augments the agent's review judgement, it does not replace it. Super-skills (such as `al-code-review`) run a self-review pass alongside their sub-skills and surface concerns the agent identified on its own, marked with `from-sub-skill: "agent"` and an empty `references: []` so consumers can render them distinctly from knowledge-backed findings. See [agent-consumption.md](agent-consumption.md) and [`skills/do.md`](skills/do.md) for the full contract.
|
||||||
|
|
||||||
|
|
@ -192,8 +190,7 @@ For the end-to-end flow — from orchestrator trigger through to how output reac
|
||||||
|
|
||||||
```
|
```
|
||||||
├── /skills/ # Global: entry-point skill + meta-skill contracts (READ, DO, WRITE)
|
├── /skills/ # Global: entry-point skill + meta-skill contracts (READ, DO, WRITE)
|
||||||
├── /coverage/ # Source-ingestion ledger and development capability matrix
|
├── /evaluation/ # Review and read-only guidance evaluation fixtures
|
||||||
├── /evaluation/ # Review and development evaluation fixtures
|
|
||||||
├── /.github/ # Actions and workflows
|
├── /.github/ # Actions and workflows
|
||||||
├── /microsoft/ # Microsoft-endorsed layer
|
├── /microsoft/ # Microsoft-endorsed layer
|
||||||
│ ├── /knowledge/ # Knowledge files by domain
|
│ ├── /knowledge/ # Knowledge files by domain
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ For the high-level framing and repo structure, start with the [README](README.md
|
||||||
|
|
||||||
When BCQuality is installed as a standalone plugin, it additionally exposes
|
When BCQuality is installed as a standalone plugin, it additionally exposes
|
||||||
`skills/al-code-review/SKILL.md` and
|
`skills/al-code-review/SKILL.md` and
|
||||||
`skills/al-development/SKILL.md`. These are host-format adapters, not
|
`skills/al-development-plan/SKILL.md`. These are host-format adapters, not
|
||||||
additional action skills: each creates the task context and enters the same
|
additional action skills: each creates the task context and enters the same
|
||||||
flow at Entry.
|
flow at Entry.
|
||||||
|
|
||||||
|
|
@ -27,7 +27,7 @@ flowchart LR
|
||||||
E -->|3 dispatch record| A
|
E -->|3 dispatch record| A
|
||||||
A -->|4 invoke dispatched skill| S[Action skill<br/>e.g. al-code-review]
|
A -->|4 invoke dispatched skill| S[Action skill<br/>e.g. al-code-review]
|
||||||
S -->|5 execute| P[Source → Relevance<br/>→ Worklist → Action<br/>reading READ · DO on demand]
|
S -->|5 execute| P[Source → Relevance<br/>→ Worklist → Action<br/>reading READ · DO on demand]
|
||||||
P -->|6 emit| R[Findings report<br/>or implementation report]
|
P -->|6 emit| R[Findings report<br/>or read-only guidance report]
|
||||||
R -->|7 integrate| O
|
R -->|7 integrate| O
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -40,14 +40,14 @@ The agent reads `/skills/entry.md` and runs it against the task context. Entry a
|
||||||
For a standalone plugin installation, the host activates the matching adapter
|
For a standalone plugin installation, the host activates the matching adapter
|
||||||
first. The adapter preserves the caller's actual goal, constructs the task
|
first. The adapter preserves the caller's actual goal, constructs the task
|
||||||
context, and invokes Entry. It does not select the internal review or
|
context, and invokes Entry. It does not select the internal review or
|
||||||
development action skill itself or duplicate Entry's preparation, routing, and
|
plan-enrichment action skill itself or duplicate Entry's preparation, routing, and
|
||||||
failure semantics.
|
failure semantics.
|
||||||
|
|
||||||
### 3. Agent consumes the dispatch record
|
### 3. Agent consumes the dispatch record
|
||||||
The dispatch record names one or more action skills, the subset of inputs each
|
The dispatch record names one or more action skills, the subset of inputs each
|
||||||
should receive, and each skill's output kind. The output kind identifies
|
should receive, and each skill's output kind. The output kind distinguishes
|
||||||
read-only review or planning work versus repository-changing implementation
|
findings from read-only plan guidance before invocation; it is not proof of
|
||||||
before invocation. If the outcome is `no-match` or `failed`, the agent returns
|
runtime side effects. If the outcome is `no-match` or `failed`, the agent returns
|
||||||
the record to the orchestrator unchanged.
|
the record to the orchestrator unchanged.
|
||||||
|
|
||||||
### 4. Agent invokes each dispatched action skill
|
### 4. Agent invokes each dispatched action skill
|
||||||
|
|
@ -81,40 +81,78 @@ The output contracts are defined in the DO meta-skill:
|
||||||
|
|
||||||
- A **findings report** carries review findings, domain labels, references, confidence, and suppressions.
|
- A **findings report** carries review findings, domain labels, references, confidence, and suppressions.
|
||||||
- A **development guidance report** carries read-only knowledge constraints and validation considerations for an existing plan.
|
- A **development guidance report** carries read-only knowledge constraints and validation considerations for an existing plan.
|
||||||
- An **implementation report** carries the development plan, classified mode, knowledge applied, changed files, validation results, final review, and remaining work.
|
|
||||||
|
|
||||||
The orchestrator parses this **without skill-specific logic**. This is the point of the contract: orchestrators and action skills evolve independently.
|
The orchestrator parses this **without skill-specific logic**. This is the point of the contract: orchestrators and action skills evolve independently.
|
||||||
|
|
||||||
For development, the action happens before the report: the skill
|
For plan enrichment, the skill reads the existing plan and target repository,
|
||||||
first invokes the read-only planning skill to select applicable knowledge, then
|
selects applicable knowledge, and returns constraints without changing the
|
||||||
changes the target repository, runs its native validation, and invokes the
|
target. It does not generate a replacement plan, run tests, implement code, or
|
||||||
configured review quality-skill over the resulting diff. A specialized
|
drive a review/fix loop. Implementation stays in the consuming workflow.
|
||||||
repository orchestrator may invoke only the planning skill and feed its
|
|
||||||
guidance report into its own implementation phases. The implementation report
|
|
||||||
is a machine-readable record of persisted work, not a code proposal for the
|
|
||||||
orchestrator to apply later.
|
|
||||||
|
|
||||||
### 7. Orchestrator integrates
|
### 7. Orchestrator integrates
|
||||||
The orchestrator turns findings into PR comments, build gates, or IDE diagnostics. For implementation it presents the changed files and validation state, while the agent has already persisted the requested change in the target repository.
|
The orchestrator turns findings into PR comments, build gates, or IDE diagnostics. It can feed read-only guidance into its own implementation phases, preserving all existing approvals and delivery gates.
|
||||||
|
|
||||||
## Repository-specific development orchestrators
|
## Repository-specific development orchestrators
|
||||||
|
|
||||||
A repository-specific workflow can keep ownership of implementation and consume
|
A repository-specific workflow can consume this read-only foundation before
|
||||||
BCQuality only for planning and review:
|
authoring while retaining its independent final review. This is the intended
|
||||||
|
integration boundary, not a shipped consumer integration:
|
||||||
|
|
||||||
1. Produce its normal development plan after repository investigation.
|
1. Investigate and produce the consumer's normal initial plan. Normalize any
|
||||||
2. Invoke Entry with `inputs-available: [development-plan, repository]` plus
|
consumer-specific format outside BCQuality. A full serialized plan document
|
||||||
the resolved applicability dimensions.
|
containing metadata plus a markdown body (root cause or design intent,
|
||||||
3. Execute the dispatched `al-development-plan` skill and preserve its
|
proposed changes, affected files, test strategy, acceptance criteria) is a
|
||||||
`development-guidance-report`.
|
valid boundary. A continuation/checkpoint payload is not a substitute for
|
||||||
4. Pass the selected article references, constraints, samples, and validation
|
initial-plan coverage; workflow identifiers and state stay with the consumer.
|
||||||
considerations into its own test, implementation, and critique phases.
|
2. Resolve and record an immutable BCQuality checkout and filtering policy.
|
||||||
5. Run its existing BCQuality-backed review gate over the completed diff.
|
Invoke Entry with a read-only enrichment goal, the existing
|
||||||
|
`development-plan`, `repository`, and established applicability dimensions.
|
||||||
|
Keep index, guidance, and runner artifacts outside the target repository.
|
||||||
|
3. Execute the dispatched `al-development-plan` skill. Persist the unchanged
|
||||||
|
report and provenance **after** any consumer state initialization or cleanup
|
||||||
|
that could erase them. BCQuality does not own the state directory or lifecycle.
|
||||||
|
4. Inject relevant constraints and validation considerations into the existing
|
||||||
|
Baseline, Implement, propagation (such as MiApp), and Critique phases, or
|
||||||
|
equivalents. Re-enrich on material plan or applicability changes; preserve
|
||||||
|
the relationship between plan version, guidance, and implementation attempt.
|
||||||
|
5. Run an independent final BCQuality review against the completed diff using
|
||||||
|
the **same recorded immutable checkout** used for enrichment. Review the
|
||||||
|
actual changes, not the guidance report as proof of correctness, then apply
|
||||||
|
the consumer's ordinary delivery gates.
|
||||||
|
|
||||||
This is the integration model for specialized bug-fix or release workflows.
|
The consumer owns analysis, normalization, persistence, per-phase injection,
|
||||||
They keep environment provisioning, retries, state, commits, propagation, and
|
approvals, TDD and runtime execution, propagation, retries, commits, and PR
|
||||||
pull-request delivery; BCQuality supplies shared product knowledge before and
|
delivery. BCQuality supplies additional referenced product knowledge, not a
|
||||||
after the code change.
|
replacement orchestrator.
|
||||||
|
|
||||||
|
### Outcomes are additive, not a universal coding gate
|
||||||
|
|
||||||
|
`no-knowledge` with empty `knowledge` means no additional applicable BCQuality
|
||||||
|
constraints. The consumer may proceed under its ordinary gates. It must not be
|
||||||
|
conflated with failed retrieval/reference integrity (`failed`), incomplete
|
||||||
|
evaluation or materially unresolved conditional guidance (`partial`), or absent
|
||||||
|
required inputs (`not-applicable`). Consumers own the policy for handling those
|
||||||
|
outcomes and recorded unknowns: seek missing context, re-enrich, escalate, or
|
||||||
|
apply their existing risk controls without relabeling the report as successful.
|
||||||
|
Do not fill gaps with generic articles simply to unlock implementation.
|
||||||
|
|
||||||
|
### Pinning and pilot evidence
|
||||||
|
|
||||||
|
A configured tag or ref alone does not establish runtime pinning. Record the
|
||||||
|
resolved commit and actual checkout/content identity used at invocation, along
|
||||||
|
with enabled layers, pruning policy, index identity, plan version, and runner
|
||||||
|
provenance. Verify that identity at both enrichment and final review; fetching
|
||||||
|
default HEAD into an explicitly supplied checkout can bypass a configured ref.
|
||||||
|
Use an isolated checkout that cannot drift during the run.
|
||||||
|
|
||||||
|
Consumer rollout and an external pilot remain follow-up work. A pilot must
|
||||||
|
compare a pinned independent baseline run of the existing workflow without
|
||||||
|
enrichment against a matched enriched run, keeping starting code, task, model,
|
||||||
|
tools, runtime, and gates controlled and recording the actual BCQuality
|
||||||
|
checkout. Retain external logs, diffs, test/compile outcomes, and independent
|
||||||
|
final reviews, including failures and unresolved results. Credential-free
|
||||||
|
fixture preparation and scorer regressions do not demonstrate improved repair
|
||||||
|
quality, compilation, runtime success, or production integration.
|
||||||
|
|
||||||
## Knowledge-backed and agent findings
|
## Knowledge-backed and agent findings
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,48 +0,0 @@
|
||||||
# Developer knowledge coverage
|
|
||||||
|
|
||||||
This directory separates **source coverage** from the knowledge corpus itself.
|
|
||||||
Microsoft Learn units are inputs to editorial work, not articles to import
|
|
||||||
one-for-one.
|
|
||||||
|
|
||||||
## Files
|
|
||||||
|
|
||||||
- `microsoft-learn-developer-catalog.json` is generated source metadata for all
|
|
||||||
Microsoft Learn modules tagged with both `dynamics-business-central` and
|
|
||||||
`developer`.
|
|
||||||
- `learn-coverage.json` is the maintained editorial ledger. Units absent from
|
|
||||||
this file are unreviewed.
|
|
||||||
- `development-capabilities.json` tracks whether representative Business
|
|
||||||
Central development capabilities have implementation fixtures.
|
|
||||||
|
|
||||||
The capability manifest declares `minimumFixtureCoverage`. CI fails when the
|
|
||||||
share of `fixture` or `validated` capabilities falls below that floor, so a
|
|
||||||
new planned capability cannot silently dilute generation coverage.
|
|
||||||
|
|
||||||
Each tracked unit has a `reviewStatus`:
|
|
||||||
|
|
||||||
- `in-progress` — at least one concern has been identified, but editorial
|
|
||||||
triage of the unit is not complete.
|
|
||||||
- `complete` — every relevant concern in the unit has a recorded outcome. A
|
|
||||||
complete unit may have no outcomes when it contains no remedial knowledge.
|
|
||||||
|
|
||||||
Each concern has one disposition:
|
|
||||||
|
|
||||||
- `candidate` — worth authoring or reconciling with existing knowledge.
|
|
||||||
- `authored` — produced one or more new knowledge articles.
|
|
||||||
- `covered-existing` — already represented by the linked article.
|
|
||||||
- `rejected` — fails BCQuality's remedial admission test.
|
|
||||||
- `deferred` — valid but intentionally postponed, with a rationale.
|
|
||||||
|
|
||||||
An authored article does not make its source unit complete automatically. One
|
|
||||||
unit can contain several independent concerns.
|
|
||||||
|
|
||||||
## Update and report
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
pwsh ./tools/Update-LearnCatalog.ps1
|
|
||||||
pwsh ./tools/Test-LearnCoverage.ps1
|
|
||||||
```
|
|
||||||
|
|
||||||
The catalog updater also accepts `-CatalogPath` for an exported Microsoft Learn
|
|
||||||
Platform API response. CI validates the committed snapshot and editorial ledger
|
|
||||||
without network access.
|
|
||||||
|
|
@ -1,132 +0,0 @@
|
||||||
{
|
|
||||||
"version": 1,
|
|
||||||
"minimumFixtureCoverage": 0.5,
|
|
||||||
"capabilities": [
|
|
||||||
{
|
|
||||||
"id": "setup-and-master-data",
|
|
||||||
"title": "Setup and master data",
|
|
||||||
"status": "fixture",
|
|
||||||
"domains": [
|
|
||||||
"data-modeling",
|
|
||||||
"security",
|
|
||||||
"testing",
|
|
||||||
"ui"
|
|
||||||
],
|
|
||||||
"fixtureIds": [
|
|
||||||
"setup-backed-master-data"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "document-workflows",
|
|
||||||
"title": "Document header and lines workflows",
|
|
||||||
"status": "fixture",
|
|
||||||
"domains": [
|
|
||||||
"data-modeling",
|
|
||||||
"events",
|
|
||||||
"performance",
|
|
||||||
"testing",
|
|
||||||
"ui"
|
|
||||||
],
|
|
||||||
"fixtureIds": [
|
|
||||||
"document-header-and-lines"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "api-integrations",
|
|
||||||
"title": "Versioned API integrations",
|
|
||||||
"status": "fixture",
|
|
||||||
"domains": [
|
|
||||||
"security",
|
|
||||||
"testing",
|
|
||||||
"web-services"
|
|
||||||
],
|
|
||||||
"fixtureIds": [
|
|
||||||
"versioned-master-data-api"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "bug-diagnosis-and-fix",
|
|
||||||
"title": "Bug diagnosis and surgical repair",
|
|
||||||
"status": "fixture",
|
|
||||||
"domains": [
|
|
||||||
"performance",
|
|
||||||
"testing"
|
|
||||||
],
|
|
||||||
"fixtureIds": [
|
|
||||||
"fix-filtered-batch-processing"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "journals-and-posting",
|
|
||||||
"title": "Journals and posting routines",
|
|
||||||
"status": "planned",
|
|
||||||
"domains": [
|
|
||||||
"data-modeling",
|
|
||||||
"error-handling",
|
|
||||||
"events",
|
|
||||||
"performance",
|
|
||||||
"testing"
|
|
||||||
],
|
|
||||||
"fixtureIds": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "reports-and-documents",
|
|
||||||
"title": "Reports and document layouts",
|
|
||||||
"status": "planned",
|
|
||||||
"domains": [
|
|
||||||
"performance",
|
|
||||||
"testing",
|
|
||||||
"ui"
|
|
||||||
],
|
|
||||||
"fixtureIds": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "install-and-upgrade",
|
|
||||||
"title": "Installation and data upgrade",
|
|
||||||
"status": "fixture",
|
|
||||||
"domains": [
|
|
||||||
"breaking-changes",
|
|
||||||
"testing",
|
|
||||||
"upgrade"
|
|
||||||
],
|
|
||||||
"fixtureIds": [
|
|
||||||
"versioned-data-upgrade"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "external-services",
|
|
||||||
"title": "Outbound services and authentication",
|
|
||||||
"status": "planned",
|
|
||||||
"domains": [
|
|
||||||
"error-handling",
|
|
||||||
"privacy",
|
|
||||||
"security",
|
|
||||||
"telemetry"
|
|
||||||
],
|
|
||||||
"fixtureIds": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "role-centers-and-onboarding",
|
|
||||||
"title": "Role Centers, setup, and onboarding",
|
|
||||||
"status": "planned",
|
|
||||||
"domains": [
|
|
||||||
"security",
|
|
||||||
"testing",
|
|
||||||
"ui"
|
|
||||||
],
|
|
||||||
"fixtureIds": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "appsource-lifecycle",
|
|
||||||
"title": "AppSource packaging and lifecycle",
|
|
||||||
"status": "planned",
|
|
||||||
"domains": [
|
|
||||||
"appsource",
|
|
||||||
"breaking-changes",
|
|
||||||
"testing",
|
|
||||||
"upgrade"
|
|
||||||
],
|
|
||||||
"fixtureIds": []
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,456 +0,0 @@
|
||||||
{
|
|
||||||
"version": 1,
|
|
||||||
"catalog": "coverage/microsoft-learn-developer-catalog.json",
|
|
||||||
"units": [
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.use-document-standards-business-central.4a-use-round-function",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "round-direction-symbols-use-magnitude",
|
|
||||||
"title": "Round direction symbols use magnitude rather than mathematical ordering",
|
|
||||||
"disposition": "authored",
|
|
||||||
"domain": "data-modeling",
|
|
||||||
"articlePaths": [
|
|
||||||
"microsoft/knowledge/data-modeling/round-direction-symbols-use-magnitude.md"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.use-document-standards-business-central.3-use-initrecord-function",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "initialize-document-defaults-in-initrecord",
|
|
||||||
"title": "Initialize document defaults in InitRecord after assigning the number",
|
|
||||||
"disposition": "authored",
|
|
||||||
"domain": "data-modeling",
|
|
||||||
"articlePaths": [
|
|
||||||
"microsoft/knowledge/data-modeling/initialize-document-defaults-in-initrecord.md"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.business-central-interfaces.type-testing",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "guard-interface-casts-with-is",
|
|
||||||
"title": "Guard optional interface casts with is",
|
|
||||||
"disposition": "authored",
|
|
||||||
"domain": "interfaces",
|
|
||||||
"articlePaths": [
|
|
||||||
"microsoft/knowledge/interfaces/guard-interface-casts-with-is.md"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.extend-modify-existing-table.add-field-group",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "dropdown-fieldgroup-respects-lookup-page-visibility",
|
|
||||||
"title": "DropDown fields remain hidden when their lookup-page controls are hidden",
|
|
||||||
"disposition": "authored",
|
|
||||||
"domain": "ui",
|
|
||||||
"articlePaths": [
|
|
||||||
"microsoft/knowledge/ui/dropdown-fieldgroup-respects-lookup-page-visibility.md"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.work-with-pages.8-controls",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "updatepropagation-both-refreshes-main-page",
|
|
||||||
"title": "UpdatePropagation Both refreshes the main page after line edits",
|
|
||||||
"disposition": "authored",
|
|
||||||
"domain": "ui",
|
|
||||||
"articlePaths": [
|
|
||||||
"microsoft/knowledge/ui/updatepropagation-both-refreshes-main-page.md"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "applicationarea-parent-inheritance",
|
|
||||||
"title": "Page-level ApplicationArea inheritance excludes extension controls",
|
|
||||||
"disposition": "covered-existing",
|
|
||||||
"domain": "style",
|
|
||||||
"articlePaths": [
|
|
||||||
"microsoft/knowledge/style/applicationarea-required-on-page-controls.md"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.easy-application-upgrade.3-installation-upgrade-codeunits",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "install-and-upgrade-codeunits-have-no-order",
|
|
||||||
"title": "Separate install or upgrade codeunits have no execution order",
|
|
||||||
"disposition": "authored",
|
|
||||||
"domain": "upgrade",
|
|
||||||
"articlePaths": [
|
|
||||||
"microsoft/knowledge/upgrade/install-and-upgrade-codeunits-have-no-order.md"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "appversion-meaning-depends-on-execution-context",
|
|
||||||
"title": "ModuleInfo AppVersion changes meaning with execution context",
|
|
||||||
"disposition": "authored",
|
|
||||||
"domain": "upgrade",
|
|
||||||
"articlePaths": [
|
|
||||||
"microsoft/knowledge/upgrade/appversion-meaning-depends-on-execution-context.md"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.work-with-tables.text-search",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "optimize-for-text-search-requires-double-ampersand-filter",
|
|
||||||
"title": "Optimized full-text search requires the double-ampersand filter operator",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "query"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.extend-modify-existing-table.define-extension-objects",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "same-app-extension-references-follow-object-id-order",
|
|
||||||
"title": "Same-app extension objects can only reference lower-ID extension objects",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "data-modeling"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.debug-deploy-extension.resource-policy-settings",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "resource-exposure-policy-is-not-nondebuggable",
|
|
||||||
"title": "Resource exposure policy and NonDebuggable protect different surfaces",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "security"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.work-entitlements-permission-sets.3-create-entitlements-permission-sets",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "entitlement-objects-reference-same-app-permission-sets",
|
|
||||||
"title": "Entitlement objects are online-only and reference same-app permission sets",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "security"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.work-entitlements-permission-sets.override-entitlements",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "inherent-permissions-cannot-elevate-other-extensions",
|
|
||||||
"title": "Inherent permissions cannot elevate access to another extension",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "security"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.debug-deploy-extension.3a-snapshot",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "snapshot-debugging-captures-only-snappoints-and-exceptions",
|
|
||||||
"title": "Snapshot debugging captures state only at snappoints and exceptions",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "testing"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.debug-deploy-extension.recovery-failures",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "publish-recovery-cannot-restore-every-extension-state",
|
|
||||||
"title": "Publish recovery cannot restore every upgrade or app-move failure",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "upgrade"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.debug-deploy-extension.database-wait-statistics",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "database-wait-statistics-are-exposed-as-a-virtual-table",
|
|
||||||
"title": "Database wait statistics are exposed as a Business Central virtual table",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "performance"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.work-with-pages.use-rich-text-editor",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "rich-text-controls-use-html-backed-blob-fields",
|
|
||||||
"title": "Rich text controls use HTML-backed Blob fields and isolated layout groups",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "ui"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.work-with-pages.scan-barcodes",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "barcode-integration-scenarios-have-different-platform-support",
|
|
||||||
"title": "Barcode field, camera action, and hardware-scanner scenarios have different platform support",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "ui"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.work-with-pages.9-search",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "card-pages-should-not-be-directly-searchable",
|
|
||||||
"title": "Card pages should be opened through their list rather than Tell Me",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "ui"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.work-with-pages.hidden-fields",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "allowincustomizations-controls-the-add-field-pane",
|
|
||||||
"title": "AllowInCustomizations controls exposure through the Add field pane",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "privacy"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.intro-development-environment.differentiate-apps",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "application-modules-depend-only-on-same-or-lower-layers",
|
|
||||||
"title": "Application modules depend only on the same or lower architectural layers",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "data-modeling"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.manipulate-data-via-code.2-retrieve-data",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "legacy-find-directions-do-not-use-top-one",
|
|
||||||
"title": "Legacy Find direction calls do not have FindFirst or FindLast query behavior",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "performance"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.easy-application-upgrade.2-upgrade-responsibilities",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "uninstall-preserves-extension-data",
|
|
||||||
"title": "Uninstall preserves extension data for reinstall or upgrade",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "upgrade"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.easy-application-upgrade.consider-update-lifecycle",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "incompatible-cloud-extensions-have-a-remediation-window",
|
|
||||||
"title": "Incompatible cloud extensions have a fixed remediation window",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "upgrade"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.easy-application-upgrade.6-answers-about-updates",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "minor-and-major-releases-update-apps-differently",
|
|
||||||
"title": "Minor and major Business Central releases update AppSource apps differently",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "upgrade"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "dependency-version-is-a-minimum",
|
|
||||||
"title": "An app.json dependency version is a minimum rather than an exact pin",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "upgrade"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.application-types.3-library-dependency-applications",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "library-apps-install-through-the-dependency-chain",
|
|
||||||
"title": "Library apps install and update through the dependency chain",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "appsource"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.business-central-interfaces.extending-interfaces",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "interfaces-can-compose-multiple-base-interfaces",
|
|
||||||
"title": "Interfaces can compose multiple base interfaces",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "interfaces"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.test-automation.page-scripting-tool",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "page-scripting-cannot-drive-non-al-ui",
|
|
||||||
"title": "Page scripting cannot drive control add-ins or other non-AL UI",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "testing"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.test-automation.2-test-automation-responsibilities",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "appsource-validation-must-use-the-current-build",
|
|
||||||
"title": "AppSource validation must use the current validation build",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "appsource"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "permission-testing-also-protects-the-essential-experience",
|
|
||||||
"title": "Permission testing must also protect the unextended Essential experience",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "testing"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.test-automation.4-answers-about-testing",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "appsource-tests-run-for-every-supported-country",
|
|
||||||
"title": "AppSource tests must run separately for every supported country",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "appsource"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "upgrade-tests-cover-nonadjacent-versions",
|
|
||||||
"title": "Upgrade tests cover nonadjacent historical versions",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "testing"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.bring-app-appsource.4-technical-validation",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "appsource-submissions-cannot-use-runtime-packages",
|
|
||||||
"title": "AppSource submissions cannot use runtime packages",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "appsource"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "per-tenant-and-marketplace-apps-need-distinct-identities",
|
|
||||||
"title": "Per-tenant and Marketplace variants need distinct app identities",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "appsource"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "profiles-are-declared-with-profile-objects",
|
|
||||||
"title": "Profiles are shipped with AL profile objects rather than table inserts",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "appsource"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "extension-layout-uses-relative-placement",
|
|
||||||
"title": "Extension layout uses named relative placement anchors",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "breaking-changes"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.test-automation.3-documentation-examples",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "tests-use-reproducible-random-data",
|
|
||||||
"title": "Tests use reproducible randomized data instead of shared literals",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "testing"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "al-tests-run-through-vscode-test-explorer",
|
|
||||||
"title": "AL tests can run and debug through Visual Studio Code Test Explorer",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "testing"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uid": "learn-dynamics.easy-application-upgrade.4-manage-apps",
|
|
||||||
"reviewStatus": "in-progress",
|
|
||||||
"outcomes": [
|
|
||||||
{
|
|
||||||
"id": "manage-apps-separates-install-and-update-requirements",
|
|
||||||
"title": "Manage Apps separates missing dependencies from dependency updates",
|
|
||||||
"disposition": "candidate",
|
|
||||||
"domain": "upgrade"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,4 @@
|
||||||
# AL review evaluation
|
# AL review and guidance evaluation
|
||||||
|
|
||||||
The evaluation is convention-driven. The harness discovers every `<layer>/skills/review/al-<domain>-review.md` leaf across the enabled `microsoft`, `community`, and `custom` layers. Duplicate domains resolve with `custom > community > microsoft` precedence. For each selected leaf, the harness finds paired knowledge across the same layers, applies the same precedence to duplicate article slugs, selects the first article (by filename) with both `.bad.al` and `.good.al` companions, and derives the expected positive and clean control automatically. Adding a conforming leaf requires no scoring-contract edit.
|
The evaluation is convention-driven. The harness discovers every `<layer>/skills/review/al-<domain>-review.md` leaf across the enabled `microsoft`, `community`, and `custom` layers. Duplicate domains resolve with `custom > community > microsoft` precedence. For each selected leaf, the harness finds paired knowledge across the same layers, applies the same precedence to duplicate article slugs, selects the first article (by filename) with both `.bad.al` and `.good.al` companions, and derives the expected positive and clean control automatically. Adding a conforming leaf requires no scoring-contract edit.
|
||||||
|
|
||||||
|
|
@ -55,50 +55,116 @@ This credential-free check proves every selected leaf maps to a same-named knowl
|
||||||
|
|
||||||
The committed gate requires full expected recall, the exact convention-derived article ID, and no findings on clean controls.
|
The committed gate requires full expected recall, the exact convention-derived article ID, and no findings on clean controls.
|
||||||
|
|
||||||
## AL development evaluation
|
## Read-only plan guidance
|
||||||
|
|
||||||
`development-fixtures.json` defines end-to-end development requests rather than
|
|
||||||
prewritten good/bad snippets. Each case declares its execution mode, the
|
|
||||||
Business Central capabilities it exercises, the knowledge that should shape
|
|
||||||
the implementation, acceptance criteria, and the real checks an external
|
|
||||||
runner must perform.
|
|
||||||
|
|
||||||
Validate the fixture and capability manifests and prepare opaque requests:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
pwsh ./tools/Test-DevelopmentFixtures.ps1 -Root . -PrepareDirectory ./.development-evaluation
|
|
||||||
```
|
|
||||||
|
|
||||||
Each request runs `al-development` in a fresh writable AL repository. Cases
|
|
||||||
may exercise feature, bug, refactor, upgrade, or maintenance mode.
|
|
||||||
The runner compiles the generated project, runs its tests, invokes the review
|
|
||||||
quality gate, and stores the resulting implementation report using the opaque
|
|
||||||
`caseId` from its request, for example `result-case-a1b2c3d4.json`. It keeps the
|
|
||||||
generated repository available at the wrapper's `workspaceRoot` so scoring can
|
|
||||||
verify reported changed paths. Score all results with:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
pwsh ./tools/Test-DevelopmentFixtures.ps1 -Root . -ResultsDirectory ./.development-evaluation
|
|
||||||
```
|
|
||||||
|
|
||||||
The initial fixtures cover setup-backed master data, document header/line
|
|
||||||
workflows, versioned API integrations, and surgical diagnosis and repair of a
|
|
||||||
batch-processing bug, plus a rerunnable data upgrade. The capability manifest
|
|
||||||
enforces a minimum fixture-backed coverage ratio; the broader roadmap lives in
|
|
||||||
`coverage/development-capabilities.json`.
|
|
||||||
|
|
||||||
### Read-only plan guidance
|
|
||||||
|
|
||||||
`development-guidance-fixtures.json` evaluates the planning interface used by
|
`development-guidance-fixtures.json` evaluates the planning interface used by
|
||||||
specialized orchestrators. It supplies an existing development plan and expects
|
existing workflows. It supplies an existing plan and expects referenced
|
||||||
a referenced set of implementation constraints without any target-repository
|
constraints without target-repository changes. The initial-plan fixture is
|
||||||
changes.
|
anonymized and synthetic: a full document with metadata and a markdown body
|
||||||
|
covering root cause, proposed fix, affected files, tests, and acceptance
|
||||||
|
criteria. It is integration-shaped input, not private consumer content, a
|
||||||
|
continuation checkpoint, or proof that any production consumer is integrated.
|
||||||
|
|
||||||
|
### Credential-free contract and scorer coverage
|
||||||
|
|
||||||
|
CI validates the manifest, prepares opaque model requests, and runs
|
||||||
|
deterministic scorer regressions with controlled reports and temporary Git
|
||||||
|
repositories. These checks cover report shape, outcomes, reference paths, and
|
||||||
|
the evaluator's pre/post read-only comparison. They do **not** run an agent,
|
||||||
|
compile AL, run Business Central tests, or establish better code authoring.
|
||||||
|
|
||||||
|
Prepare requests in a runner-owned artifact directory outside every target
|
||||||
|
workspace:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory ./.development-guidance-evaluation
|
$run = Join-Path ([IO.Path]::GetTempPath()) 'bcquality-guidance-run'
|
||||||
|
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . -PrepareDirectory $run
|
||||||
```
|
```
|
||||||
|
|
||||||
An external runner stores `result-<case-id>.json` beside the generated request
|
Preparation is not a model run. A scorer can validate a citation's path and
|
||||||
and retains the clean fixture repository at `workspaceRoot`. Score the result
|
required fields, but only external agent traces and expert evaluation can
|
||||||
with `-ResultsDirectory`; the scorer verifies knowledge recall and precision
|
establish that the article was opened and its normative constraints faithfully
|
||||||
and fails if the planning pass changed the repository.
|
applied. Expected knowledge recall/precision is fixture-specific, not a corpus
|
||||||
|
coverage or authoring capability percentage.
|
||||||
|
|
||||||
|
`no-knowledge` means no additional applicable BCQuality constraints, not unsafe
|
||||||
|
work. It requires empty `knowledge`. Partial evaluation, failed retrieval,
|
||||||
|
unknown context, and materially unresolved applicability must remain visible
|
||||||
|
and distinct; they cannot be counted as successful enrichment merely because
|
||||||
|
the JSON is parseable.
|
||||||
|
|
||||||
|
Run the deterministic regression suite without an agent or AL environment:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
pwsh ./tools/Test-DevelopmentGuidanceEvaluator.ps1
|
||||||
|
```
|
||||||
|
|
||||||
|
### Runner-owned read-only evidence
|
||||||
|
|
||||||
|
For an external guidance run, first provision a representative, standalone Git
|
||||||
|
repository for each manifest case. The runner supplies a JSON workspace map
|
||||||
|
whose keys are the manifest IDs (not the hashed model IDs) and whose values are
|
||||||
|
absolute workspace roots. It may pass `-WorkspaceMapPath` during preparation
|
||||||
|
to bind the generated requests to those roots. The model must not select its
|
||||||
|
own workspace for scoring.
|
||||||
|
|
||||||
|
Capture evidence **before** invoking the agent, with the manifest, workspace
|
||||||
|
map, and source checkout already finalized:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# Runner-selected paths, all outside the targets and BCQuality checkout.
|
||||||
|
$map = Join-Path $evidenceDirectory 'workspace-map.json'
|
||||||
|
$baseline = Join-Path $evidenceDirectory 'baseline.json'
|
||||||
|
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . `
|
||||||
|
-CaptureBaseline -WorkspaceMapPath $map -BaselinePath $baseline
|
||||||
|
|
||||||
|
# Retain the printed SHA256 in runner-only state BEFORE agent invocation.
|
||||||
|
# After the external agent writes result-case-<hash>.json files:
|
||||||
|
pwsh ./tools/Test-DevelopmentGuidanceFixtures.ps1 -Root . `
|
||||||
|
-ResultsDirectory $resultsDirectory -BaselinePath $baseline `
|
||||||
|
-BaselineSha256 $preRunDigest
|
||||||
|
```
|
||||||
|
|
||||||
|
`$evidenceDirectory`, `$resultsDirectory`, and `$preRunDigest` are supplied by
|
||||||
|
the runner; the digest must not be recomputed from potentially modified evidence
|
||||||
|
after the agent runs. Protect the baseline, digest, evaluator, and invocation
|
||||||
|
from agent changes. Capture refuses to overwrite an existing baseline. Results
|
||||||
|
contain only `caseId` and `guidanceReport`; a legacy `workspaceRoot`, if present,
|
||||||
|
must agree with the independently captured binding and never overrides it.
|
||||||
|
Missing baselines or digests, malformed reports, and escaped reference paths
|
||||||
|
fail scoring.
|
||||||
|
|
||||||
|
The comparison checks target identity, Git HEAD, refs and index, filesystem
|
||||||
|
content and stable metadata, including tracked, untracked, ignored files and
|
||||||
|
empty directories. Committing edits or making an empty commit does not evade
|
||||||
|
the check. It also compares the actual knowledge checkout and manifest identity.
|
||||||
|
Targets must have internal Git storage; linked target worktrees, submodules,
|
||||||
|
sparse checkouts, links/junctions/reparse points, hard links, and alternate data
|
||||||
|
streams are unsupported and rejected rather than silently excluded. A linked
|
||||||
|
**knowledge** checkout is supported with its Git storage identity recorded.
|
||||||
|
Use quiescent, isolated repositories; concurrent changes also fail the gate.
|
||||||
|
|
||||||
|
This is before/after evidence, not an OS sandbox or a complete write monitor.
|
||||||
|
It cannot prove that no transient write was reverted, that articles were opened,
|
||||||
|
or that constraints are semantically faithful. Reports and generated artifacts
|
||||||
|
must stay outside all target workspaces and the knowledge checkout. The
|
||||||
|
regression suite creates and removes its own uniquely named fixture directory;
|
||||||
|
it does not run against or clean a caller's target.
|
||||||
|
|
||||||
|
### External agent/runtime pilot (follow-up)
|
||||||
|
|
||||||
|
Consumer uptake and a real before-authoring pilot are not implemented by these
|
||||||
|
fixtures. The consumer must normalize its normal initial plan, persist guidance
|
||||||
|
after state initialization, inject it into existing phases, re-enrich on
|
||||||
|
material changes, and run an independent final review. See
|
||||||
|
[the integration boundary](../agent-consumption.md#repository-specific-development-orchestrators).
|
||||||
|
|
||||||
|
Before claiming improved repairs, run an independent pinned baseline without
|
||||||
|
enrichment and a matched enriched run. Hold starting code, task, model, tools,
|
||||||
|
runtime, and gates constant; record actual immutable BCQuality checkout and
|
||||||
|
policy identities rather than trusting a configured ref. Use that same
|
||||||
|
recorded checkout for enrichment and final review. Retain external logs,
|
||||||
|
article-read traces, resulting diffs, compile/test outcomes, and independent
|
||||||
|
review evidence, including failures, no-knowledge, partial, and unresolved
|
||||||
|
results. No compile/run or authoring-quality claim follows from the
|
||||||
|
credential-free checks above.
|
||||||
|
|
|
||||||
|
|
@ -1,237 +0,0 @@
|
||||||
{
|
|
||||||
"version": 1,
|
|
||||||
"skill": "microsoft/skills/development/al-development.md",
|
|
||||||
"maximumReviewRounds": 3,
|
|
||||||
"minimumKnowledgeRecall": 1.0,
|
|
||||||
"minimumKnowledgePrecision": 0.5,
|
|
||||||
"cases": [
|
|
||||||
{
|
|
||||||
"id": "setup-backed-master-data",
|
|
||||||
"title": "Build setup-backed loyalty member master data",
|
|
||||||
"capabilities": [
|
|
||||||
"setup-and-master-data"
|
|
||||||
],
|
|
||||||
"expectedKind": "feature",
|
|
||||||
"development-request": {
|
|
||||||
"kind": "feature",
|
|
||||||
"description": "Add a Loyalty Member feature to an existing Business Central AL app. Administrators configure the member number series on a singleton setup card. Users create members through list and card pages, the table assigns numbers, and blocked members cannot be selected by consuming records. Include least-privilege permission sets and automated tests.",
|
|
||||||
"acceptance-criteria": [
|
|
||||||
"The setup is a blank-key singleton surfaced by a Card page.",
|
|
||||||
"Member numbers use the current No. Series codeunit and support manual numbers according to setup.",
|
|
||||||
"Blocked validation occurs where a member is consumed, not only on the member table.",
|
|
||||||
"The feature includes assignable least-privilege permissions and automated tests."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"context": {
|
|
||||||
"technologies": [
|
|
||||||
"al"
|
|
||||||
],
|
|
||||||
"countries": [
|
|
||||||
"w1"
|
|
||||||
],
|
|
||||||
"application-area": [
|
|
||||||
"all"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"requiredKnowledge": [
|
|
||||||
"microsoft/knowledge/data-modeling/setup-table-is-a-singleton.md",
|
|
||||||
"microsoft/knowledge/data-modeling/master-table-no-from-number-series-in-oninsert.md",
|
|
||||||
"microsoft/knowledge/data-modeling/check-blocked-in-referencing-code-not-in-master.md",
|
|
||||||
"microsoft/knowledge/security/permission-set-avoid-wildcard-grants.md",
|
|
||||||
"microsoft/knowledge/testing/use-library-codeunits-for-test-fixtures.md"
|
|
||||||
],
|
|
||||||
"optionalKnowledge": [
|
|
||||||
"microsoft/knowledge/data-modeling/use-no-series-codeunit-not-noseriesmanagement.md",
|
|
||||||
"microsoft/knowledge/security/compose-permission-sets-with-included-sets.md",
|
|
||||||
"microsoft/knowledge/style/applicationarea-required-on-page-controls.md",
|
|
||||||
"microsoft/knowledge/style/tooltip-required-on-page-fields.md",
|
|
||||||
"microsoft/knowledge/ui/showmandatory-on-code-required-page-fields.md"
|
|
||||||
],
|
|
||||||
"requiredChecks": [
|
|
||||||
"compile",
|
|
||||||
"tests",
|
|
||||||
"review"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "document-header-and-lines",
|
|
||||||
"title": "Build a document header and lines workflow",
|
|
||||||
"capabilities": [
|
|
||||||
"document-workflows"
|
|
||||||
],
|
|
||||||
"expectedKind": "feature",
|
|
||||||
"development-request": {
|
|
||||||
"kind": "feature",
|
|
||||||
"description": "Implement a Service Quote feature with a header, lines, document page, number series, posting and document dates, calculated totals, and tests. Line edits must refresh the total shown on the header. Structure initialization so API, UI, and test creation paths behave consistently.",
|
|
||||||
"acceptance-criteria": [
|
|
||||||
"The header assigns its number before InitRecord establishes document defaults.",
|
|
||||||
"The document page links lines correctly and refreshes parent totals after edits.",
|
|
||||||
"Tests exercise creation outside the UI as well as the document-page behavior.",
|
|
||||||
"The implementation contains no obsolete NoSeriesManagement dependency."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"context": {
|
|
||||||
"technologies": [
|
|
||||||
"al"
|
|
||||||
],
|
|
||||||
"countries": [
|
|
||||||
"w1"
|
|
||||||
],
|
|
||||||
"application-area": [
|
|
||||||
"service"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"requiredKnowledge": [
|
|
||||||
"microsoft/knowledge/data-modeling/initialize-document-defaults-in-initrecord.md",
|
|
||||||
"microsoft/knowledge/data-modeling/use-no-series-codeunit-not-noseriesmanagement.md",
|
|
||||||
"microsoft/knowledge/ui/updatepropagation-both-refreshes-main-page.md",
|
|
||||||
"microsoft/knowledge/testing/use-library-codeunits-for-test-fixtures.md"
|
|
||||||
],
|
|
||||||
"optionalKnowledge": [
|
|
||||||
"microsoft/knowledge/events/publish-thin-onbefore-onafter-integration-events.md",
|
|
||||||
"microsoft/knowledge/style/applicationarea-required-on-page-controls.md",
|
|
||||||
"microsoft/knowledge/style/tooltip-required-on-page-fields.md"
|
|
||||||
],
|
|
||||||
"requiredChecks": [
|
|
||||||
"compile",
|
|
||||||
"tests",
|
|
||||||
"review"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "versioned-master-data-api",
|
|
||||||
"title": "Expose master data through a versioned API",
|
|
||||||
"capabilities": [
|
|
||||||
"api-integrations"
|
|
||||||
],
|
|
||||||
"expectedKind": "feature",
|
|
||||||
"development-request": {
|
|
||||||
"kind": "auto",
|
|
||||||
"plan": "Expose Loyalty Member master data through a Business Central API page. Use a stable v1.0 contract, address records by SystemId, support insert and update, use conventional entity naming, and include permissions and automated API-oriented tests.",
|
|
||||||
"acceptance-criteria": [
|
|
||||||
"The API declares all routing properties and a stable APIVersion.",
|
|
||||||
"ODataKeyFields uses SystemId and the exposed SystemId field is not editable.",
|
|
||||||
"EntityName is singular, EntitySetName is plural, and both are lower camel case.",
|
|
||||||
"The API is covered by least-privilege permissions and automated tests."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"context": {
|
|
||||||
"technologies": [
|
|
||||||
"al"
|
|
||||||
],
|
|
||||||
"countries": [
|
|
||||||
"w1"
|
|
||||||
],
|
|
||||||
"application-area": [
|
|
||||||
"all"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"requiredKnowledge": [
|
|
||||||
"microsoft/knowledge/web-services/set-required-api-page-properties.md",
|
|
||||||
"microsoft/knowledge/web-services/expose-systemid-as-the-api-key.md",
|
|
||||||
"microsoft/knowledge/style/api-page-delayedinsert-true.md",
|
|
||||||
"microsoft/knowledge/style/api-page-entity-naming-singular-plural.md",
|
|
||||||
"microsoft/knowledge/security/permission-set-avoid-wildcard-grants.md"
|
|
||||||
],
|
|
||||||
"optionalKnowledge": [
|
|
||||||
"microsoft/knowledge/style/api-page-camelcase-properties.md",
|
|
||||||
"microsoft/knowledge/web-services/version-apis-by-adding-not-mutating-published-versions.md"
|
|
||||||
],
|
|
||||||
"requiredChecks": [
|
|
||||||
"compile",
|
|
||||||
"tests",
|
|
||||||
"review"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "fix-filtered-batch-processing",
|
|
||||||
"title": "Fix a batch routine that processes only one record",
|
|
||||||
"capabilities": [
|
|
||||||
"bug-diagnosis-and-fix"
|
|
||||||
],
|
|
||||||
"expectedKind": "bug",
|
|
||||||
"development-request": {
|
|
||||||
"kind": "bug",
|
|
||||||
"description": "Users report that an existing filtered batch routine updates only the first matching record. Reproduce the defect, identify why iteration stops, make the smallest safe correction, and add a regression test that selects multiple records and proves every selected record is processed.",
|
|
||||||
"acceptance-criteria": [
|
|
||||||
"The defect is reproduced or demonstrated by a failing regression test before the fix.",
|
|
||||||
"The root cause is corrected without widening the supplied record filters.",
|
|
||||||
"The routine processes every selected record with the appropriate update locking behavior.",
|
|
||||||
"A regression test covers more than one selected record."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"context": {
|
|
||||||
"technologies": [
|
|
||||||
"al"
|
|
||||||
],
|
|
||||||
"countries": [
|
|
||||||
"w1"
|
|
||||||
],
|
|
||||||
"application-area": [
|
|
||||||
"all"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"requiredKnowledge": [
|
|
||||||
"microsoft/knowledge/performance/pair-findset-with-next-loop.md",
|
|
||||||
"microsoft/knowledge/testing/use-library-codeunits-for-test-fixtures.md"
|
|
||||||
],
|
|
||||||
"optionalKnowledge": [
|
|
||||||
"microsoft/knowledge/performance/findset-true-applies-updlock-on-read.md",
|
|
||||||
"microsoft/knowledge/performance/pass-var-record-to-preserve-partial-load-enumerator.md"
|
|
||||||
],
|
|
||||||
"requiredChecks": [
|
|
||||||
"compile",
|
|
||||||
"tests",
|
|
||||||
"review"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "versioned-data-upgrade",
|
|
||||||
"title": "Implement a rerunnable data upgrade",
|
|
||||||
"capabilities": [
|
|
||||||
"install-and-upgrade"
|
|
||||||
],
|
|
||||||
"expectedKind": "upgrade",
|
|
||||||
"development-request": {
|
|
||||||
"kind": "upgrade",
|
|
||||||
"description": "Upgrade an existing app from a text-based Customer Tier field to a new enum-backed Tier field. Preserve existing customer data, support tenants that skip intermediate app versions, keep fresh installation separate from migration, and add upgrade tests.",
|
|
||||||
"acceptance-criteria": [
|
|
||||||
"Existing tier values are migrated without running field validation triggers.",
|
|
||||||
"The migration is guarded by an upgrade tag and is safe when the upgrade runs again.",
|
|
||||||
"Check and validation triggers do not write data.",
|
|
||||||
"Fresh installation does not run version-upgrade migration.",
|
|
||||||
"Tests cover more than one historical source data version."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"context": {
|
|
||||||
"technologies": [
|
|
||||||
"al"
|
|
||||||
],
|
|
||||||
"countries": [
|
|
||||||
"w1"
|
|
||||||
],
|
|
||||||
"application-area": [
|
|
||||||
"all"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"requiredKnowledge": [
|
|
||||||
"microsoft/knowledge/upgrade/appversion-meaning-depends-on-execution-context.md",
|
|
||||||
"microsoft/knowledge/upgrade/install-and-upgrade-codeunits-have-no-order.md",
|
|
||||||
"microsoft/knowledge/upgrade/use-upgrade-tags-not-version-checks.md",
|
|
||||||
"microsoft/knowledge/upgrade/check-only-triggers-do-not-migrate-data.md",
|
|
||||||
"microsoft/knowledge/upgrade/install-code-does-not-run-on-version-upgrade.md"
|
|
||||||
],
|
|
||||||
"optionalKnowledge": [
|
|
||||||
"microsoft/knowledge/upgrade/datatransfer-skips-triggers-and-subscribers.md",
|
|
||||||
"microsoft/knowledge/upgrade/datatransfer-for-bulk-init.md",
|
|
||||||
"microsoft/knowledge/upgrade/register-upgrade-tags-with-subscribers.md",
|
|
||||||
"microsoft/knowledge/testing/transactionmodel-attribute-governs-test-transactions.md"
|
|
||||||
],
|
|
||||||
"requiredChecks": [
|
|
||||||
"compile",
|
|
||||||
"tests",
|
|
||||||
"review"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -5,36 +5,18 @@
|
||||||
"minimumKnowledgePrecision": 0.67,
|
"minimumKnowledgePrecision": 0.67,
|
||||||
"cases": [
|
"cases": [
|
||||||
{
|
{
|
||||||
"id": "bcapps-filtered-batch-bug-plan",
|
"id": "synthetic-normal-initial-plan",
|
||||||
"title": "Select guidance from a BCFIX-HANDOFF v1 payload",
|
"title": "Anonymized normal initial-plan consumer boundary",
|
||||||
|
"evidenceType": "integration-shaped-synthetic",
|
||||||
|
"boundary": "A synthetic consumer produces metadata plus a markdown plan body, serializes the full document, and passes that document as the generic existing development-plan. This is not an external real pilot and contains no consumer workflow-state schema.",
|
||||||
"expectedKind": "bug",
|
"expectedKind": "bug",
|
||||||
"development-plan": {
|
"expectedOutcome": "completed",
|
||||||
"format": "BCFIX-HANDOFF",
|
"expectedUnknown": [],
|
||||||
"version": 1,
|
"requiresUnresolved": false,
|
||||||
"issue": 4312,
|
"requiresMaterialUnresolved": false,
|
||||||
"phase": "implement",
|
"development-plan": "{\"metadata\":{\"kind\":\"bug\",\"request\":\"Update every entry in the supplied filtered record set while preserving the caller's selection.\",\"origin\":\"anonymized synthetic initial plan\"},\"body\":\"## Root cause and design\\nThe routine reads and updates only the first record rather than iterating the supplied filtered set. Preserve the supplied filters and update each selected row.\\n\\n## Proposed fix\\nUse an update-safe FindSet/Next loop with an explicit update on each selected entry.\\n\\n## Affected files\\n- src/Batch/UpdateSelectedEntries.Codeunit.al\\n- test/Batch/UpdateSelectedEntriesTests.Codeunit.al\\n\\n## Test strategy\\nUse existing AL test library codeunits to arrange three selected rows and an excluded row. Assert all selected rows are updated and the excluded row is unchanged. This is a proposed test, not a reported result.\\n\\n## Acceptance criteria\\n- Every selected entry is updated exactly once.\\n- The supplied filters remain effective.\\n- No excluded entry changes.\\n- Empty selections cause no changes.\\n\"}",
|
||||||
"status": "paused",
|
|
||||||
"baton": 2,
|
|
||||||
"rootCause": "The routine calls FindFirst and updates the current record without entering an enumerator loop, so only the first record in the supplied filtered set is modified.",
|
|
||||||
"harnessMap": {
|
|
||||||
"testCodeunit": "Update Selected Entries Tests",
|
|
||||||
"libraries": [
|
|
||||||
"Library - Random"
|
|
||||||
],
|
|
||||||
"pages": [],
|
|
||||||
"handlers": []
|
|
||||||
},
|
|
||||||
"iterationsUsed": 1,
|
|
||||||
"filesCommitted": [
|
|
||||||
"test/Batch/UpdateSelectedEntries.Codeunit.al"
|
|
||||||
],
|
|
||||||
"lastTestResult": "1 failing, 4 passing; the red test shows only the first of three selected records is updated.",
|
|
||||||
"deadEnds": [
|
|
||||||
"Changing the page selection did not help because the codeunit discarded the supplied enumerator."
|
|
||||||
],
|
|
||||||
"nextStep": "Replace the single-record read with an update-safe FindSet/Next loop that preserves the supplied filters, then rerun the red test."
|
|
||||||
},
|
|
||||||
"context": {
|
"context": {
|
||||||
|
"bc-version": "28",
|
||||||
"technologies": [
|
"technologies": [
|
||||||
"al"
|
"al"
|
||||||
],
|
],
|
||||||
|
|
@ -43,7 +25,8 @@
|
||||||
],
|
],
|
||||||
"application-area": [
|
"application-area": [
|
||||||
"all"
|
"all"
|
||||||
]
|
],
|
||||||
|
"unknown": []
|
||||||
},
|
},
|
||||||
"requiredKnowledge": [
|
"requiredKnowledge": [
|
||||||
"microsoft/knowledge/performance/pair-findset-with-next-loop.md",
|
"microsoft/knowledge/performance/pair-findset-with-next-loop.md",
|
||||||
|
|
@ -58,6 +41,10 @@
|
||||||
"id": "versioned-upgrade-plan-guidance",
|
"id": "versioned-upgrade-plan-guidance",
|
||||||
"title": "Select guidance for a versioned data upgrade",
|
"title": "Select guidance for a versioned data upgrade",
|
||||||
"expectedKind": "upgrade",
|
"expectedKind": "upgrade",
|
||||||
|
"expectedOutcome": "completed",
|
||||||
|
"expectedUnknown": [],
|
||||||
|
"requiresUnresolved": false,
|
||||||
|
"requiresMaterialUnresolved": false,
|
||||||
"development-plan": {
|
"development-plan": {
|
||||||
"kind": "upgrade",
|
"kind": "upgrade",
|
||||||
"request": "Migrate existing customer tier text values to a new enum field in an app upgrade.",
|
"request": "Migrate existing customer tier text values to a new enum field in an app upgrade.",
|
||||||
|
|
@ -78,6 +65,7 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"context": {
|
"context": {
|
||||||
|
"bc-version": "28",
|
||||||
"technologies": [
|
"technologies": [
|
||||||
"al"
|
"al"
|
||||||
],
|
],
|
||||||
|
|
@ -86,7 +74,8 @@
|
||||||
],
|
],
|
||||||
"application-area": [
|
"application-area": [
|
||||||
"all"
|
"all"
|
||||||
]
|
],
|
||||||
|
"unknown": []
|
||||||
},
|
},
|
||||||
"requiredKnowledge": [
|
"requiredKnowledge": [
|
||||||
"microsoft/knowledge/upgrade/use-upgrade-tags-not-version-checks.md",
|
"microsoft/knowledge/upgrade/use-upgrade-tags-not-version-checks.md",
|
||||||
|
|
@ -97,6 +86,90 @@
|
||||||
"microsoft/knowledge/upgrade/datatransfer-skips-triggers-and-subscribers.md",
|
"microsoft/knowledge/upgrade/datatransfer-skips-triggers-and-subscribers.md",
|
||||||
"microsoft/knowledge/upgrade/appversion-meaning-depends-on-execution-context.md"
|
"microsoft/knowledge/upgrade/appversion-meaning-depends-on-execution-context.md"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "no-additional-knowledge",
|
||||||
|
"title": "Honest empty enrichment does not prohibit ordinary work",
|
||||||
|
"expectedKind": "maintenance",
|
||||||
|
"expectedOutcome": "no-knowledge",
|
||||||
|
"expectedUnknown": [],
|
||||||
|
"requiresUnresolved": false,
|
||||||
|
"requiresMaterialUnresolved": false,
|
||||||
|
"development-plan": {
|
||||||
|
"kind": "maintenance",
|
||||||
|
"request": "Correct a spelling error in an existing internal explanatory comment in an AL procedure. Do not change the explanation, executable code, UI captions, schema, diagnostic text, configuration or behavior.",
|
||||||
|
"affected-files": ["src/Batch/EntryProcessor.Codeunit.al"],
|
||||||
|
"proposed-changes": ["Replace the misspelled word in the existing comment; add no new advice."],
|
||||||
|
"test-strategy": "Inspect the diff to confirm that only the comment spelling changes.",
|
||||||
|
"acceptance-criteria": ["Only the intended comment spelling changes; executable AL remains identical."]
|
||||||
|
},
|
||||||
|
"context": {
|
||||||
|
"bc-version": "28",
|
||||||
|
"technologies": ["al"],
|
||||||
|
"countries": ["w1"],
|
||||||
|
"application-area": ["all"],
|
||||||
|
"unknown": []
|
||||||
|
},
|
||||||
|
"requiredKnowledge": [],
|
||||||
|
"optionalKnowledge": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unknown-material-version",
|
||||||
|
"title": "Materially unresolved version-sensitive guidance stays partial",
|
||||||
|
"expectedKind": "feature",
|
||||||
|
"expectedOutcome": "partial",
|
||||||
|
"expectedUnknown": ["bc-version"],
|
||||||
|
"requiresUnresolved": true,
|
||||||
|
"requiresMaterialUnresolved": true,
|
||||||
|
"development-plan": {
|
||||||
|
"kind": "feature",
|
||||||
|
"request": "Add an expensive Sum FlowField as the source of a usually-hidden page control. The proposed design relies on visibility suppressing calculation.",
|
||||||
|
"affected-files": ["src/Pages/EntryOverview.Page.al"],
|
||||||
|
"proposed-changes": ["Bind the page control directly to the FlowField and set Visible to a conditional expression."],
|
||||||
|
"test-strategy": "Verify aggregate queries are not executed while the control is hidden.",
|
||||||
|
"acceptance-criteria": ["Hidden controls do not cause expensive aggregate queries."],
|
||||||
|
"unknown": ["The deployment BC version and visible-only calculation feature state cannot be established from this fixture. Do not invent either."]
|
||||||
|
},
|
||||||
|
"context": {
|
||||||
|
"bc-version": "unknown",
|
||||||
|
"technologies": ["al"],
|
||||||
|
"countries": ["w1"],
|
||||||
|
"application-area": ["all"],
|
||||||
|
"unknown": ["bc-version"]
|
||||||
|
},
|
||||||
|
"requiredKnowledge": [
|
||||||
|
"microsoft/knowledge/performance/hidden-flowfields-still-calculate-before-bc26-opt-in.md"
|
||||||
|
],
|
||||||
|
"optionalKnowledge": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "partial-plan-decision",
|
||||||
|
"title": "Known platform context does not resolve an incomplete plan decision",
|
||||||
|
"expectedKind": "refactor",
|
||||||
|
"expectedOutcome": "partial",
|
||||||
|
"expectedUnknown": [],
|
||||||
|
"requiresUnresolved": true,
|
||||||
|
"requiresMaterialUnresolved": true,
|
||||||
|
"development-plan": {
|
||||||
|
"kind": "refactor",
|
||||||
|
"request": "Refactor a record read helper currently using FindFirst followed by Next. The caller contract does not establish whether to return one row or enumerate the entire filtered set.",
|
||||||
|
"affected-files": ["src/Queries/EntryReader.Codeunit.al"],
|
||||||
|
"proposed-changes": ["Choose the read method consistent with the intended cardinality after that decision is clarified."],
|
||||||
|
"test-strategy": "Add cardinality assertions after the caller contract is decided.",
|
||||||
|
"acceptance-criteria": ["The method and enumeration agree with the clarified caller contract."],
|
||||||
|
"unknown": ["Single-record versus multi-record caller intent remains materially unresolved."]
|
||||||
|
},
|
||||||
|
"context": {
|
||||||
|
"bc-version": "28",
|
||||||
|
"technologies": ["al"],
|
||||||
|
"countries": ["w1"],
|
||||||
|
"application-area": ["all"],
|
||||||
|
"unknown": []
|
||||||
|
},
|
||||||
|
"requiredKnowledge": [
|
||||||
|
"microsoft/knowledge/performance/pair-findset-with-next-loop.md"
|
||||||
|
],
|
||||||
|
"optionalKnowledge": []
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,9 +18,11 @@ Selects the BCQuality knowledge that should constrain an existing AL development
|
||||||
|
|
||||||
Both a readable `repository` and a non-empty `development-plan` are required. The plan may be structured data or text, but it must identify the intended change. Return `not-applicable` without changing files when either input is absent or the repository is not an AL project.
|
Both a readable `repository` and a non-empty `development-plan` are required. The plan may be structured data or text, but it must identify the intended change. Return `not-applicable` without changing files when either input is absent or the repository is not an AL project.
|
||||||
|
|
||||||
|
The caller supplies its existing plan, not a request to generate one. Consumer-specific formats must be normalized by the consumer before invocation. This skill does not interpret issue records, continuation markers, batons, retries, or workflow state. A serialized document containing plan metadata and a markdown body is acceptable when it states the intended change, affected surfaces, proposed approach, test strategy, and acceptance criteria. Missing details remain unknown; do not invent them.
|
||||||
|
|
||||||
## Source
|
## Source
|
||||||
|
|
||||||
Read the BCQuality knowledge index once. Use entries from every enabled layer and domain. The index supplies candidate paths, applicability dimensions, keywords, titles, and descriptions; it never substitutes for opening selected articles in full.
|
Read the BCQuality knowledge index once, using the external path supplied by Entry when present. If no index is available, use READ's path-based discovery across enabled layers; inability to read the corpus is `failed`, not `no-knowledge`. Use entries from every enabled layer and domain. The index supplies candidate paths, applicability dimensions, keywords, titles, and descriptions; it never substitutes for opening selected articles in full.
|
||||||
|
|
||||||
Inspect the target repository read-only for `app.json`, affected files and symbols named by the plan, relevant tests, permission sets, dependencies, target/runtime versions, countries, application areas, and repository conventions. Do not create scratch or generated files inside the target repository.
|
Inspect the target repository read-only for `app.json`, affected files and symbols named by the plan, relevant tests, permission sets, dependencies, target/runtime versions, countries, application areas, and repository conventions. Do not create scratch or generated files inside the target repository.
|
||||||
|
|
||||||
|
|
@ -37,8 +39,7 @@ When a dimension cannot be resolved, retain conditionally applicable candidates
|
||||||
|
|
||||||
## Worklist
|
## Worklist
|
||||||
|
|
||||||
1. Normalize the plan into: request summary, development kind, assumptions, root cause or design intent, affected files and symbols, proposed changes, test strategy, and acceptance criteria. When the plan has no normalized kind, apply the same categories as `al-development`: new or expanded behavior is `feature`, a defect correction is `bug`, behavior-preserving restructuring is `refactor`, migration is `upgrade`, and other bounded work is `maintenance`. A repository-specific additive event or extensibility request maps to `feature`; retain its original work-item type in the request summary. Do not redesign the repository-specific workflow.
|
1. Read the supplied plan for its request summary, development kind, assumptions, root cause or design intent, affected files and symbols, proposed changes, test strategy, and acceptance criteria. Preserve its intent; do not generate a replacement plan. If kind is not explicit, classify the stated intent: new or expanded behavior is `feature`, a defect correction is `bug`, behavior-preserving restructuring is `refactor`, migration is `upgrade`, and other bounded work is `maintenance`. Do not redesign the consumer's workflow.
|
||||||
- For a `BCFIX-HANDOFF` v1 payload, map `rootCause` to root cause, `harnessMap` to test context, `filesCommitted` to affected files, `lastTestResult` to existing test evidence, `deadEnds` to rejected approaches, and `nextStep` to the immediate proposed change. Preserve `issue`, `phase`, `status`, `baton`, and `iterationsUsed` as workflow context only; they do not create Business Central constraints. A handoff with a non-empty root cause is `bug` unless the surrounding plan identifies an additive Event Request, which maps to `feature`.
|
|
||||||
2. Build retrieval vocabulary from the plan and confirmed repository symbols. Give exact object types, properties, methods, analyzers, errors, and affected domains more weight than broad business nouns.
|
2. Build retrieval vocabulary from the plan and confirmed repository symbols. Give exact object types, properties, methods, analyzers, errors, and affected domains more weight than broad business nouns.
|
||||||
3. Search the index in separate passes:
|
3. Search the index in separate passes:
|
||||||
- data ownership, keys, setup, numbering, validation, transactions, and upgrade;
|
- data ownership, keys, setup, numbering, validation, transactions, and upgrade;
|
||||||
|
|
@ -66,4 +67,6 @@ Do not change the target repository. Before emitting, verify every knowledge and
|
||||||
|
|
||||||
## Output
|
## Output
|
||||||
|
|
||||||
Return one `development-guidance-report` conforming to DO. `completed` requires that every selected article was opened and faithfully converted into constraints. `no-knowledge` is valid when the plan is applicable but BCQuality contains no relevant article. `partial` names every unevaluated candidate or unresolved applicability gap.
|
Return one `development-guidance-report` conforming to DO. `completed` requires that every selected article was opened and faithfully converted into constraints, with no material unresolved applicability. `no-knowledge` means there are no additional applicable BCQuality constraints for this plan; emit empty `knowledge`. It does not mean the work is unsafe or unimplementable, and the consumer can proceed under its ordinary gates. Never add generic or filler guidance to avoid this outcome.
|
||||||
|
|
||||||
|
Return `partial` for incomplete evaluation or materially unresolved conditional guidance, naming every gap. Failed retrieval or reference-integrity checks are `failed`, never `no-knowledge`. Consumers own handling of partial, failed, and unresolved results, including clarification and re-enrichment; this read-only interface does not define a universal implementation gate.
|
||||||
|
|
|
||||||
|
|
@ -1,98 +0,0 @@
|
||||||
---
|
|
||||||
kind: action-skill
|
|
||||||
id: al-development
|
|
||||||
version: 1
|
|
||||||
title: AL development
|
|
||||||
description: Implements Business Central AL features, bug fixes, refactors, upgrades, and maintenance changes using BCQuality knowledge.
|
|
||||||
inputs: [development-request, repository]
|
|
||||||
outputs: [implementation-report]
|
|
||||||
bc-version: [all]
|
|
||||||
technologies: [al]
|
|
||||||
countries: [w1]
|
|
||||||
application-area: [all]
|
|
||||||
guidance-skill: microsoft/skills/development/al-development-plan.md
|
|
||||||
quality-skill: microsoft/skills/review/al-code-review.md
|
|
||||||
quality-round-limit: 3
|
|
||||||
---
|
|
||||||
|
|
||||||
# AL development
|
|
||||||
|
|
||||||
Implements a Business Central change in an existing AL repository. Feature work, bug fixing, refactoring, upgrades, and maintenance share one public contract and one quality pipeline; their different investigation disciplines are execution modes within this skill.
|
|
||||||
|
|
||||||
Both a writable `repository` and a `development-request` are required. A structured request has this shape:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
development-request:
|
|
||||||
kind: auto # feature | bug | refactor | upgrade | maintenance
|
|
||||||
description: string # optional when plan states the requested outcome
|
|
||||||
plan: string # optional
|
|
||||||
acceptance-criteria: [string] # optional
|
|
||||||
```
|
|
||||||
|
|
||||||
A plain-text request is normalized to `kind: auto` with the text as `description`. A plan-only request is valid when the plan states the requested outcome. Return `not-applicable` without changing files when either input is absent, both description and plan are empty, or the repository is not an AL project.
|
|
||||||
|
|
||||||
## Source
|
|
||||||
|
|
||||||
Read the frontmatter `guidance-skill`; it owns BCQuality discovery and returns the knowledge constraints for the implementation plan. Inspect the target repository for `app.json`, existing objects, tests, permission sets, analyzers, build scripts, naming and object-ID conventions, dependencies, target/runtime versions, localization layout, and uncommitted user changes. For bugs, refactors, and upgrades, inspect enough history and surrounding code to establish the behavior being changed.
|
|
||||||
|
|
||||||
## Relevance
|
|
||||||
|
|
||||||
Resolve and pass this context to the guidance-skill:
|
|
||||||
|
|
||||||
- `bc-version` from the target application's platform/application/runtime settings or supplied context. For an upgrade, distinguish source and target versions.
|
|
||||||
- `technologies: [al]`, plus any additional technology actually required by the request.
|
|
||||||
- `countries` from `app.json`, workspace configuration, or supplied context.
|
|
||||||
- `application-area` from the request and affected objects.
|
|
||||||
|
|
||||||
Record unresolved dimensions in the development plan rather than silently substituting broad values. The guidance-skill applies READ's matching semantics and returns any conditional applicability in its report.
|
|
||||||
|
|
||||||
## Worklist
|
|
||||||
|
|
||||||
1. Normalize the request, deriving a concise description from a plan-only input, and classify `kind: auto` as:
|
|
||||||
- `feature` for new or intentionally expanded behavior;
|
|
||||||
- `bug` for observed behavior that contradicts an expected result;
|
|
||||||
- `refactor` for structural change with no intended behavior change;
|
|
||||||
- `upgrade` for schema, data, dependency, runtime, or application-version migration;
|
|
||||||
- `maintenance` for bounded development work that fits none of the above.
|
|
||||||
Preserve an explicit valid kind. When repository evidence conflicts with it, record the mismatch and ask for clarification before changing files rather than silently switching disciplines.
|
|
||||||
2. Establish the mode-specific implementation contract:
|
|
||||||
- **Feature:** define user-visible behavior and cover data lifecycle, UI/API, permissions, extensibility, upgrade impact, telemetry, and tests where applicable.
|
|
||||||
- **Bug:** state expected versus actual behavior, reproduce or otherwise prove the defect, trace the root cause, and define a regression test that fails for that cause.
|
|
||||||
- **Refactor:** identify the behavior and public contracts that must remain invariant, plus the checks that establish a before/after baseline.
|
|
||||||
- **Upgrade:** identify source and target states, data migration, compatibility, idempotency, and validation requirements.
|
|
||||||
- **Maintenance:** define the bounded outcome and the behavior that must not change.
|
|
||||||
3. Treat a supplied plan as an input constraint, not as proof. Reconcile it with repository reality and BCQuality; preserve its intent, correct unsafe assumptions, and record consequential deviations.
|
|
||||||
4. Discover existing implementation patterns and reusable objects before proposing new ones. Preserve repository conventions and current user changes.
|
|
||||||
5. Materialize a `development-plan` containing the classified kind, request, assumptions, affected files and symbols, design or root cause, proposed changes, validation strategy, and acceptance criteria.
|
|
||||||
6. Invoke the frontmatter `guidance-skill` with that plan, the repository, and the resolved context. It performs Source, Relevance, and knowledge worklisting independently and read-only.
|
|
||||||
7. Require a complete guidance result before editing product code:
|
|
||||||
- `completed` — use every returned constraint and validation consideration.
|
|
||||||
- `no-knowledge` — intentionally refuse to implement: return `no-knowledge` with no request changes, set `outcome-reason` to `No applicable BCQuality knowledge was found for this development plan.`, and add a `remaining` entry directing the caller to use a repository-specific workflow/general coding agent or contribute the missing BC-specific knowledge.
|
|
||||||
- `not-applicable`, `partial`, or `failed` — return the corresponding non-completed outcome without editing product code; preserve its reason in `remaining`.
|
|
||||||
8. Copy the guidance report's selected paths into the eventual implementation report only when the corresponding constraint materially shaped the implementation. Carry its suppression records forward.
|
|
||||||
|
|
||||||
## Action
|
|
||||||
|
|
||||||
1. Record the starting working-tree state so unrelated changes are preserved and excluded from `changes`.
|
|
||||||
2. Apply the execution mode:
|
|
||||||
- **Feature:** implement the smallest complete vertical slice; do not leave placeholder surfaces.
|
|
||||||
- **Bug:** reproduce first when feasible, fix the root cause rather than the symptom, keep the patch surgical, and add a regression test.
|
|
||||||
- **Refactor:** capture a behavioral baseline, avoid unrelated behavior changes, and prove the declared invariants afterward.
|
|
||||||
- **Upgrade:** make migrations rerunnable where required, preserve data and compatibility, and validate both upgraded and fresh-install paths when applicable.
|
|
||||||
- **Maintenance:** make only the bounded requested change and preserve surrounding behavior.
|
|
||||||
3. Produce a coherent design that satisfies the implementation contract and every constraint returned by the guidance-skill. Reuse existing abstractions and object ranges. Do not hard-code a Business Central fact in this skill or invent a rule absent from both the repository and reliable platform knowledge.
|
|
||||||
4. Implement the request end to end. Include all surfaces required by the mode, acceptance criteria, and repository conventions. Do not create success-shaped stubs.
|
|
||||||
5. Treat the guidance report as design constraints throughout implementation. Adapt its referenced companion samples to the target codebase; never copy demonstration IDs or names blindly.
|
|
||||||
6. Run the smallest existing build, analyzer, and test commands that cover the change. Fix failures caused by the implementation. Record every command and real outcome in `validation`; unavailable checks are `not-run`, never `passed`.
|
|
||||||
7. Invoke the frontmatter `quality-skill` against the final implementation diff, bounded by `quality-round-limit`:
|
|
||||||
- Record every invocation in `review-rounds`, including its gating `blocker` and `major` IDs.
|
|
||||||
- When no gating finding remains, mark the round `clean` and stop.
|
|
||||||
- Otherwise fix every justified, safely actionable gating finding, rerun affected validation, mark the round `fixing`, and start the next review round.
|
|
||||||
- Stop early as `stalled` when the gating ID set is unchanged from the preceding round, no gating finding can be fixed safely, or validation cannot be restored.
|
|
||||||
- When the final allowed round still has gating findings, mark it `limit-reached`.
|
|
||||||
Preserve the last complete findings-report in `review` and add a `validation` entry with `id: "review"`. A `stalled` or `limit-reached` loop returns `partial` with the unresolved gating findings in `remaining`. If review is disabled or unavailable, record `not-run` and return `partial`.
|
|
||||||
8. Verify the persisted files against the implementation contract, acceptance criteria, and mode-specific evidence. If behavior, validation, guidance, or review remains incomplete, return `partial` and list the exact gap in `remaining`.
|
|
||||||
|
|
||||||
## Output
|
|
||||||
|
|
||||||
Return one `implementation-report` conforming to DO. Set `plan.kind` to the classified execution mode. `knowledge` lists only articles opened in full and materially used. `changes` lists only files changed by this skill. `completed` requires a persisted implementation, passing required validation, and no unresolved `blocker` or `major` finding in `review`. `no-knowledge` is a visible coverage decision, not an error or silent fallback.
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "bcquality",
|
"name": "bcquality",
|
||||||
"description": "Quality skills and knowledge for Business Central development. Exposes AL development and code-review adapters backed by BCQuality's Entry protocol.",
|
"description": "Quality skills and knowledge for Business Central. Exposes read-only AL plan enrichment and code-review adapters backed by BCQuality's Entry protocol.",
|
||||||
"version": "0.3.0",
|
"version": "0.3.0",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "microsoft/BCQuality",
|
"name": "microsoft/BCQuality",
|
||||||
|
|
@ -13,7 +13,7 @@
|
||||||
"al",
|
"al",
|
||||||
"business-central",
|
"business-central",
|
||||||
"code-review",
|
"code-review",
|
||||||
"development",
|
"plan-guidance",
|
||||||
"quality"
|
"quality"
|
||||||
],
|
],
|
||||||
"skills": [
|
"skills": [
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ READ and DO are read on demand — typically by the first action skill the agent
|
||||||
| Path | Role |
|
| Path | Role |
|
||||||
|---|---|
|
|---|---|
|
||||||
| [`al-code-review/SKILL.md`](al-code-review/SKILL.md) | Exposes BCQuality through the standard `SKILL.md` format when this repository is installed as a plugin. |
|
| [`al-code-review/SKILL.md`](al-code-review/SKILL.md) | Exposes BCQuality through the standard `SKILL.md` format when this repository is installed as a plugin. |
|
||||||
| [`al-development/SKILL.md`](al-development/SKILL.md) | Exposes knowledge-backed AL development through the standard `SKILL.md` format. |
|
| [`al-development-plan/SKILL.md`](al-development-plan/SKILL.md) | Enriches an existing AL plan read-only through the standard `SKILL.md` format; does not generate a plan or implement code. |
|
||||||
|
|
||||||
Each adapter is deliberately thin. It translates the caller's request into an
|
Each adapter is deliberately thin. It translates the caller's request into an
|
||||||
Entry task context, then follows Entry's dispatch without owning routing,
|
Entry task context, then follows Entry's dispatch without owning routing,
|
||||||
|
|
@ -42,15 +42,13 @@ by Entry, and should not accumulate behavior already defined by `entry.md`,
|
||||||
This gives the two skill formats distinct roles:
|
This gives the two skill formats distinct roles:
|
||||||
|
|
||||||
- `skills/al-code-review/SKILL.md` and
|
- `skills/al-code-review/SKILL.md` and
|
||||||
`skills/al-development/SKILL.md` are the public host integration
|
`skills/al-development-plan/SKILL.md` are the public host integration
|
||||||
surfaces for a standalone plugin installation.
|
surfaces for a standalone plugin installation.
|
||||||
- `microsoft/skills/review/al-code-review.md` is BCQuality's internal
|
- `microsoft/skills/review/al-code-review.md` is BCQuality's internal
|
||||||
Microsoft-layer super-skill for coordinating a broad AL review.
|
Microsoft-layer super-skill for coordinating a broad AL review.
|
||||||
- `microsoft/skills/development/al-development.md` is the internal
|
|
||||||
Microsoft-layer implementation skill for all supported development modes.
|
|
||||||
- `microsoft/skills/development/al-development-plan.md` is the read-only
|
- `microsoft/skills/development/al-development-plan.md` is the read-only
|
||||||
planning interface for repository-specific orchestrators that retain
|
knowledge-enrichment interface for existing plans. Consumers own format
|
||||||
implementation ownership.
|
normalization, planning, implementation, and delivery.
|
||||||
|
|
||||||
Each host adapter deliberately shares its name with the internal action skill
|
Each host adapter deliberately shares its name with the internal action skill
|
||||||
for the same operation. Their locations distinguish the host integration from
|
for the same operation. Their locations distinguish the host integration from
|
||||||
|
|
|
||||||
22
skills/al-development-plan/SKILL.md
Normal file
22
skills/al-development-plan/SKILL.md
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
---
|
||||||
|
name: al-development-plan
|
||||||
|
description: Enrich an existing Business Central AL development plan with read-only BCQuality knowledge constraints. Does not generate a plan or implement code.
|
||||||
|
---
|
||||||
|
|
||||||
|
# AL development plan guidance
|
||||||
|
|
||||||
|
This host-native adapter translates an existing plan and repository into Entry's task context. It does not plan new work, edit the target repository, run an implementation or review/fix loop, stage, commit, or publish changes.
|
||||||
|
|
||||||
|
## Execute
|
||||||
|
|
||||||
|
1. Resolve `PLUGIN_ROOT` to the directory containing this plugin's root `plugin.json`, two levels above this file.
|
||||||
|
2. Preserve the caller's existing `development-plan` verbatim. Consumer-specific workflow payloads must be normalized by the consumer; do not interpret workflow state or manufacture a plan from a coding request.
|
||||||
|
3. Build the task context for `PLUGIN_ROOT/skills/entry.md`:
|
||||||
|
- Set `goal` to read-only BCQuality knowledge enrichment of the supplied plan, preserving the caller's intended change.
|
||||||
|
- List only actually supplied inputs from `[development-plan, repository]` in `inputs-available`.
|
||||||
|
- Set `technologies: [al]` only when established, and pass other applicability dimensions only when supplied or reliably determined.
|
||||||
|
- Apply `BCQUALITY_ENABLED_LAYERS` and `BCQUALITY_DISABLED_SKILLS` as described in the `al-code-review` adapter.
|
||||||
|
4. Read and execute Entry, including Preparation. Resolve its paths against `PLUGIN_ROOT`, not the target repository. Keep all generated index and scratch artifacts outside the target repository. Use READ's path-based fallback if index generation is unavailable; an unreadable corpus is a failure, not empty knowledge.
|
||||||
|
5. Follow Entry's dispatch, checking its output metadata against the referenced skill before invocation. This operation accepts only `development-guidance-report`; return `failed` rather than execute another output kind. Pass the supplied existing plan and readable repository, and return the report unchanged. Return Entry's `no-match` or `failed` record unchanged when nothing is dispatched.
|
||||||
|
|
||||||
|
Missing inputs remain missing; the dispatched action skill returns `not-applicable` when it cannot proceed. A `no-knowledge` report is additive: it means no additional BCQuality constraints, not a refusal to let the consumer implement under its own gates. The consumer retains all implementation and delivery ownership.
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
---
|
|
||||||
name: al-development
|
|
||||||
description: Implement Business Central AL features, bug fixes, refactors, upgrades, and maintenance changes using BCQuality's curated platform knowledge.
|
|
||||||
---
|
|
||||||
|
|
||||||
# AL development
|
|
||||||
|
|
||||||
This is BCQuality's host-native adapter for standalone plugin installations. It translates a coding request into Entry's task context; the internal action skill owns classification, investigation, design, implementation, validation, and review policy.
|
|
||||||
|
|
||||||
When the target repository exposes a more specific local workflow for the request, such as an end-to-end bug-fix skill with its own environment and delivery gates, prefer that repository workflow unless the caller explicitly asks to use BCQuality's generic development skill.
|
|
||||||
|
|
||||||
This adapter is deliberately knowledge-backed: if BCQuality has no applicable
|
|
||||||
guidance, it returns a visible `no-knowledge` result without changing code. Use
|
|
||||||
the repository's normal coding workflow for unbacked requests, or add the
|
|
||||||
missing BC-specific knowledge before expecting this skill to implement them.
|
|
||||||
|
|
||||||
## Execute
|
|
||||||
|
|
||||||
1. Resolve `PLUGIN_ROOT` to the directory containing this plugin's root `plugin.json`. This file is `PLUGIN_ROOT/skills/al-development/SKILL.md`; when the host does not expose the plugin root, resolve it two levels above this file.
|
|
||||||
2. Build the `task-context` required by `PLUGIN_ROOT/skills/entry.md`:
|
|
||||||
- Copy the caller's request verbatim into `goal`.
|
|
||||||
- Set `inputs-available: [development-request, repository]`.
|
|
||||||
- Set `technologies: [al]` when the repository is an AL project.
|
|
||||||
- Pass `bc-version`, `countries`, and `application-area` only when supplied or reliably determined.
|
|
||||||
- Apply `BCQUALITY_ENABLED_LAYERS` and `BCQUALITY_DISABLED_SKILLS` exactly as the `al-code-review` adapter does.
|
|
||||||
3. Read and execute `PLUGIN_ROOT/skills/entry.md`, including Preparation. Resolve every path it names against `PLUGIN_ROOT`, not the user's repository. If knowledge-index generation is unavailable, use READ's path-based fallback.
|
|
||||||
4. Follow Entry's dispatch exactly. The normal result is `microsoft/skills/development/al-development.md`; do not select it directly or duplicate its behavior in this adapter.
|
|
||||||
5. Normalize the caller's input as `development-request`:
|
|
||||||
- Plain text becomes `{ kind: auto, description: <verbatim text> }`.
|
|
||||||
- Preserve an explicit `kind`, supplied plan, and `acceptance-criteria`.
|
|
||||||
- A plan-only input becomes `{ kind: auto, description: "Implement the supplied development plan.", plan: <verbatim plan> }`.
|
|
||||||
Pass the writable current workspace as `repository`, execute the dispatched skill, and return its `implementation-report` unchanged.
|
|
||||||
|
|
||||||
The adapter never edits BCQuality itself unless BCQuality is the caller's target repository. The target of implementation is the repository supplied by the caller.
|
|
||||||
112
skills/do.md
112
skills/do.md
|
|
@ -7,7 +7,7 @@ title: Action Skill — the template every action skill follows
|
||||||
|
|
||||||
# DO
|
# DO
|
||||||
|
|
||||||
An action skill is a markdown file that tells an agent how to do one concrete job — review a pull request, audit telemetry usage, generate a skeleton — using knowledge files from BCQuality. This document is the template every action skill follows. Orchestrators rely on the template to consume any skill without skill-specific parsing.
|
An action skill is a markdown file that tells an agent how to do one concrete job — review a pull request, audit telemetry usage, enrich an existing plan — using knowledge files from BCQuality. This document is the template every action skill follows. Orchestrators rely on the template to consume any skill without skill-specific parsing.
|
||||||
|
|
||||||
This contract is stable. Changes require a PR approved by both maintainers.
|
This contract is stable. Changes require a PR approved by both maintainers.
|
||||||
|
|
||||||
|
|
@ -56,20 +56,15 @@ application-area: [all]
|
||||||
|
|
||||||
`bc-version`, `technologies`, `countries`, `application-area` are optional filters that let an orchestrator pre-select applicable skills for a task. They follow the same semantics as in READ.
|
`bc-version`, `technologies`, `countries`, `application-area` are optional filters that let an orchestrator pre-select applicable skills for a task. They follow the same semantics as in READ.
|
||||||
|
|
||||||
`inputs` is a list of abstract input types the skill **accepts**. Standard values: `pr-diff`, `object-list`, `file-path`, `repository`, `telemetry-query`, `development-request`, `development-plan`. Semantics are any-of: the orchestrator supplies whichever listed input types it has, and the skill is invoked with a non-empty subset of its declared `inputs`. A skill that cannot proceed with the supplied subset MUST return `outcome: "not-applicable"`.
|
`inputs` is a list of abstract input types the skill **accepts**. Standard values: `pr-diff`, `object-list`, `file-path`, `repository`, `telemetry-query`, `development-plan`. Semantics are any-of: the orchestrator supplies whichever listed input types it has, and the skill is invoked with a non-empty subset of its declared `inputs`. A skill that cannot proceed with the supplied subset MUST return `outcome: "not-applicable"`.
|
||||||
|
|
||||||
`outputs` is always a single-element list naming the output kind:
|
`outputs` is always a single-element list naming the output kind:
|
||||||
|
|
||||||
- `findings-report` — evaluates an input and reports defects or observations.
|
- `findings-report` — evaluates an input and reports defects or observations.
|
||||||
- `implementation-report` — changes a repository to satisfy a development request and reports the plan, knowledge used, changed files, validation, and post-implementation review.
|
|
||||||
- `development-guidance-report` — selects and summarizes applicable BCQuality knowledge for an existing development plan without changing the target repository.
|
- `development-guidance-report` — selects and summarizes applicable BCQuality knowledge for an existing development plan without changing the target repository.
|
||||||
|
|
||||||
`sub-skills` is an optional field. When present and non-empty, the skill is a **super-skill** that composes other action skills; see *Composition* below. Values are repo-relative paths to action-skill files.
|
`sub-skills` is an optional field. When present and non-empty, the skill is a **super-skill** that composes other action skills; see *Composition* below. Values are repo-relative paths to action-skill files.
|
||||||
|
|
||||||
`quality-skill` is optional on an action skill that emits an `implementation-report`. It names one repo-relative review action skill to run over the completed diff. It is a post-implementation gate, not a composed sub-skill: Entry does not route through it, and its final complete findings-report is returned in `review`. `quality-round-limit` is the required positive maximum number of review/fix rounds when a quality skill is declared. Consumer configuration still applies; if the named quality skill is disabled or unavailable, record its validation as `not-run` and do not claim `completed`.
|
|
||||||
|
|
||||||
`guidance-skill` is optional on an action skill that emits an `implementation-report`. It names one repo-relative read-only action skill that accepts a `development-plan` and emits a `development-guidance-report`. The implementation skill invokes it after forming its plan and before editing product code. Consumer configuration still applies; when guidance is disabled or unavailable, the implementation skill must not claim knowledge-backed development.
|
|
||||||
|
|
||||||
## Required sections
|
## Required sections
|
||||||
|
|
||||||
Every action skill MUST contain these five sections, in order:
|
Every action skill MUST contain these five sections, in order:
|
||||||
|
|
@ -86,13 +81,15 @@ Every action skill MUST contain these five sections, in order:
|
||||||
|
|
||||||
**Relevance.** Apply frontmatter filters to the candidates. Typical filters: match `bc-version` against the target environment, match `technologies` against the languages in scope, match `countries` and `application-area` against the consuming codebase's context. The exact matching rules are defined in READ (*Frontmatter matching semantics*). Files that do not match are discarded.
|
**Relevance.** Apply frontmatter filters to the candidates. Typical filters: match `bc-version` against the target environment, match `technologies` against the languages in scope, match `countries` and `application-area` against the consuming codebase's context. The exact matching rules are defined in READ (*Frontmatter matching semantics*). Files that do not match are discarded.
|
||||||
|
|
||||||
**Worklist.** Narrow the relevant candidates to the subset that applies to the current task. This is where the task-specific signal enters: the objects changed in the PR, the queries being audited, the skeleton being generated. Typical moves: match `keywords` against task vocabulary, match file topics against changed objects, deduplicate by concern.
|
**Worklist.** Narrow the relevant candidates to the subset that applies to the current task. This is where the task-specific signal enters: the objects changed in the PR, the queries being audited, the existing plan being enriched. Typical moves: match `keywords` against task vocabulary, match file topics against changed objects, deduplicate by concern.
|
||||||
|
|
||||||
**Action.** Execute the skill's work against the worklist. Evaluate each item in the worklist against the task input and emit findings. The action step is where skill behavior differs; the preceding three steps are uniform.
|
**Action.** Execute the skill's work against the worklist. Evaluate each item in the worklist against the task input and emit findings. The action step is where skill behavior differs; the preceding three steps are uniform.
|
||||||
|
|
||||||
|
<a id="output-contract"></a>
|
||||||
|
|
||||||
## Findings-report contract
|
## Findings-report contract
|
||||||
|
|
||||||
Every action skill emits a single JSON document that conforms to this schema:
|
An action skill with `outputs: [findings-report]` emits a single JSON document that conforms to this schema:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
|
|
@ -300,96 +297,27 @@ An action skill with `outputs: [development-guidance-report]` emits one JSON doc
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
The skill is read-only with respect to the target repository. `completed` means every selected article was opened and converted into faithful implementation constraints. `no-knowledge` means no applicable article survived filtering; `knowledge` is empty. `partial` means candidate evaluation stopped early, with the gap named in `outcome-reason` and `unresolved`.
|
The skill is read-only with respect to the target repository: no edits, generated files, staging, commits, or publication. Keep index, report, and scratch artifacts outside that repository. The report is strict JSON with no surrounding commentary. The caller supplies an existing plan and repository; consumer-specific input normalization and workflow state are outside this contract.
|
||||||
|
|
||||||
`knowledge[].constraints` summarizes only normative `## Best Practice` and `## Anti Pattern` content from the referenced article. It must not introduce a Business Central fact absent from that article. `sample-paths` contains only sibling samples that exist and were opened. Every path is subject to the reference-integrity gate.
|
### Guidance outcome semantics
|
||||||
|
|
||||||
`validation-considerations` states evidence the implementation workflow should obtain; it does not claim that a command or test has run. `unresolved` records missing repository context or plan decisions that prevent a reliable constraint. Unknown applicability dimensions must appear in both `context.unknown` and a relevant unresolved entry.
|
- `completed` — evaluation finished, at least one article was selected, every selected article was opened and faithfully converted into constraints, and no materially unresolved conditional guidance remains.
|
||||||
|
- `not-applicable` — the required existing plan or readable repository is absent, or the task is outside the skill's applicability. No constraints are claimed.
|
||||||
|
- `no-knowledge` — evaluation finished and there are **no additional applicable BCQuality constraints** for this plan. `knowledge` is empty. This is not a statement that the work is unsafe or unimplementable; the consuming workflow can proceed under its ordinary gates. Do not add generic or filler articles to avoid this outcome.
|
||||||
|
- `partial` — evaluation is incomplete or conditional guidance remains materially unresolved. Name each gap in `outcome-reason` and `unresolved`; do not silently treat an unknown dimension as a match.
|
||||||
|
- `failed` — retrieval, reference integrity, or another error prevents a reliable report. Set `outcome-reason`; consumers must not treat the result as reliable constraints or as `no-knowledge`.
|
||||||
|
|
||||||
## Implementation-report contract
|
`outcome-reason` is required for `partial` and `failed`, optional otherwise. These outcomes describe enrichment only, not permission to implement or deliver. The consumer owns handling of partial, failed, and unresolved guidance, including escalation, clarification, and re-enrichment; BCQuality does not impose a universal implementation gate.
|
||||||
|
|
||||||
An action skill with `outputs: [implementation-report]` emits one JSON document:
|
### Guidance field semantics
|
||||||
|
|
||||||
```json
|
`summary.request` preserves the planned intent and `kind` classifies it without replacing the plan. `candidates` and `selected` are non-negative integer counts: selected equals the number of unique `knowledge` entries and cannot exceed candidates. Counts are retrieval diagnostics, not capability or authoring-quality scores.
|
||||||
{
|
|
||||||
"skill": { "id": "string", "version": 1 },
|
|
||||||
"outcome": "completed | not-applicable | no-knowledge | partial | failed",
|
|
||||||
"outcome-reason": "string",
|
|
||||||
"summary": {
|
|
||||||
"request": "string",
|
|
||||||
"files-created": 0,
|
|
||||||
"files-modified": 0,
|
|
||||||
"files-deleted": 0
|
|
||||||
},
|
|
||||||
"plan": {
|
|
||||||
"kind": "feature | bug | refactor | upgrade | maintenance",
|
|
||||||
"assumptions": ["string"],
|
|
||||||
"decisions": ["string"],
|
|
||||||
"objects": ["string"]
|
|
||||||
},
|
|
||||||
"knowledge": [
|
|
||||||
{ "path": "string", "sha": "string", "used-for": "string" }
|
|
||||||
],
|
|
||||||
"changes": [
|
|
||||||
{
|
|
||||||
"path": "string",
|
|
||||||
"action": "created | modified | deleted",
|
|
||||||
"purpose": "string"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"validation": [
|
|
||||||
{
|
|
||||||
"id": "string",
|
|
||||||
"command": "string",
|
|
||||||
"status": "passed | failed | not-run",
|
|
||||||
"details": "string"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"review": { "...full findings-report from the post-implementation review..." : null },
|
|
||||||
"review-rounds": [
|
|
||||||
{
|
|
||||||
"round": 1,
|
|
||||||
"outcome": "clean | fixing | stalled | limit-reached",
|
|
||||||
"gating-finding-ids": ["string"]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"suppressed": [
|
|
||||||
{
|
|
||||||
"reference": { "path": "string", "sha": "string" },
|
|
||||||
"reason": "layer-precedence | configuration"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"remaining": ["string"]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Implementation outcome semantics
|
`knowledge[].constraints` is a non-empty list summarizing only normative `## Best Practice` and `## Anti Pattern` content from the referenced article. It must not introduce a Business Central fact absent from that article. `used-for` names the concrete plan decision. `sample-paths` contains only sibling samples that exist and were opened. All paths use forward slashes, are repository-relative, and must resolve inside the recorded BCQuality checkout; absolute paths, traversal, and links escaping that checkout are invalid. Every reference is subject to the reference-integrity gate.
|
||||||
|
|
||||||
- `completed` — the requested change is persisted in the repository, required validation passed, and the post-implementation review has no unresolved `blocker` or `major` finding.
|
`validation-considerations` states evidence the implementation workflow should obtain; it does not claim that a command or test has run. `suppressed` has the same shape and semantics as in a findings-report. `unresolved` records missing repository context or plan decisions that prevent a reliable constraint. Unknown applicability dimensions must appear in both `context.unknown` and a relevant unresolved entry, explaining whether they materially affect a candidate. An unknown dimension is not itself a failure or proof that relevant knowledge exists.
|
||||||
- `not-applicable` — the request is not an implementation task accepted by the skill, or the supplied repository does not contain the required technology.
|
|
||||||
- `no-knowledge` — no applicable BCQuality knowledge survived filtering and the skill cannot safely implement the Business Central-specific request. No request changes are made.
|
|
||||||
- `partial` — useful changes were persisted, but part of the requested scope, validation, or post-implementation review could not be completed. `outcome-reason` and `remaining` identify the unfinished work.
|
|
||||||
- `failed` — the skill could not produce a reliable implementation. `outcome-reason` is required. Any working-tree changes remain visible and MUST still be listed in `changes`.
|
|
||||||
|
|
||||||
### Implementation field semantics
|
Reference SHAs, when present, identify the files read; they do not prove runtime pinning on their own. The consumer records and verifies the actual immutable BCQuality checkout used for both enrichment and final review, plus its filtering policy and run provenance outside the target repository. See [agent-consumption.md](../agent-consumption.md).
|
||||||
|
|
||||||
**`summary.request`** is a concise statement of the implemented change. File counts describe only changes made by this skill; pre-existing user changes are excluded.
|
|
||||||
|
|
||||||
**`plan`** records the implementation decisions needed to understand the result. `kind` is the classified development mode: `feature`, `bug`, `refactor`, `upgrade`, or `maintenance`. `assumptions` contains only assumptions actually made; `decisions` captures consequential design choices; `objects` names the Business Central objects or other artifacts created or changed.
|
|
||||||
|
|
||||||
**`knowledge`** lists every knowledge file whose normative guidance materially shaped the implementation. `path` and optional `sha` follow the same reference format as a findings-report. `used-for` briefly names the design or implementation decision. The reference-integrity gate applies: every path must exist in the live checkout, be copied verbatim from discovery, and have been opened in full. Applicability alone is not enough to list an article.
|
|
||||||
|
|
||||||
**`changes`** is an exhaustive list of files created, modified, or deleted by the skill. Paths are repository-relative and use forward slashes. Do not include unrelated pre-existing changes.
|
|
||||||
|
|
||||||
**`validation`** records commands actually run. `passed` and `failed` require a real command result; unavailable tooling or an intentionally skipped check is `not-run` with `details`. A skill MUST NOT manufacture a successful check or replace a failed command with a success-shaped fallback.
|
|
||||||
|
|
||||||
**`review`** is optional for generic implementation skills and required when a skill's instructions mandate post-implementation review. When present, it is the complete findings-report returned by that review skill, not a rewritten summary.
|
|
||||||
|
|
||||||
**`review-rounds`** records every quality-skill invocation in order. `gating-finding-ids` contains the `blocker` and `major` IDs from that round. `clean` ends successfully; `fixing` means the skill applied justified fixes before another round; `stalled` means the same gating set persisted or no safe progress was possible; `limit-reached` means the configured round cap was exhausted. The array length MUST NOT exceed `quality-round-limit`. `stalled` or `limit-reached` requires implementation outcome `partial`, the final findings-report in `review`, and every unresolved gating item in `remaining`.
|
|
||||||
|
|
||||||
**`suppressed`** has the same semantics as in a findings-report and records applicable knowledge excluded by layer precedence or configuration.
|
|
||||||
|
|
||||||
**`remaining`** contains concrete unfinished work only. It is empty for `completed`.
|
|
||||||
|
|
||||||
## Composition (super-skills)
|
## Composition (super-skills)
|
||||||
|
|
||||||
|
|
@ -465,4 +393,4 @@ Conforms to the DO output contract.
|
||||||
|
|
||||||
## How orchestrators consume output
|
## How orchestrators consume output
|
||||||
|
|
||||||
An orchestrator invokes an action skill with an input appropriate to the skill's declared `inputs` and uses the single output kind declared in frontmatter. It maps a `findings-report` to PR comments, build gates, or IDE diagnostics; a `development-guidance-report` to constraints for a downstream implementation workflow; and an `implementation-report` to a coding-session summary, changed-file view, validation status, and any remaining work. The orchestrator MUST NOT interpret fields beyond the three schemas above.
|
An orchestrator invokes an action skill with an input appropriate to the skill's declared `inputs` and uses the single output kind declared in frontmatter. It maps a `findings-report` to PR comments, build gates, or IDE diagnostics, and a `development-guidance-report` to additional constraints for its existing implementation workflow. These are the two output schemas defined by this contract; the consumer retains ownership of implementation and delivery.
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,6 @@ The agent invokes Entry with a **task context** supplied by the orchestrator:
|
||||||
task-context:
|
task-context:
|
||||||
goal: string # free-text description of what needs doing
|
goal: string # free-text description of what needs doing
|
||||||
inputs-available: # values the orchestrator has ready to pass to a chosen skill
|
inputs-available: # values the orchestrator has ready to pass to a chosen skill
|
||||||
- development-request
|
|
||||||
- development-plan
|
- development-plan
|
||||||
- repository
|
- repository
|
||||||
- pr-diff
|
- pr-diff
|
||||||
|
|
@ -38,7 +37,7 @@ task-context:
|
||||||
|
|
||||||
## Preparation — knowledge index
|
## Preparation — knowledge index
|
||||||
|
|
||||||
Before routing, ensure the knowledge index is current for the **live** clone. The dispatched review skills read `knowledge-index.json` (at the clone root) at their Source step instead of opening every knowledge file — see READ's [Retrieval workflow](read.md). When a consumer prunes its clone to policy *before* the agent runs, the index MUST be built over the clone as it exists now, so it lists exactly the articles that survived pruning and never an article the consumer denied:
|
Before routing, ensure the knowledge index is current for the **live** clone. The dispatched skills read `knowledge-index.json` (by default at the clone root) at their Source step instead of opening every knowledge file — see READ's [Retrieval workflow](read.md). When a consumer prunes its clone to policy *before* the agent runs, the index MUST be built over the clone as it exists now, so it lists exactly the articles that survived pruning and never an article the consumer denied:
|
||||||
|
|
||||||
- If `knowledge-index.json` is absent — or you cannot confirm it reflects the current knowledge tree — regenerate it by running, from the checkout root:
|
- If `knowledge-index.json` is absent — or you cannot confirm it reflects the current knowledge tree — regenerate it by running, from the checkout root:
|
||||||
|
|
||||||
|
|
@ -48,6 +47,7 @@ Before routing, ensure the knowledge index is current for the **live** clone. Th
|
||||||
|
|
||||||
It defaults to indexing this checkout and writes `knowledge-index.json` at the root in well under a second. When in doubt, rebuild: a sub-second rebuild is always cheaper than a stale or over-listing index, which is a correctness risk.
|
It defaults to indexing this checkout and writes `knowledge-index.json` at the root in well under a second. When in doubt, rebuild: a sub-second rebuild is always cheaper than a stale or over-listing index, which is a correctness risk.
|
||||||
- The paths above assume the checkout root is the current directory. A caller that enters Entry from elsewhere — a plugin host, whose working directory is the user's own project — MUST resolve them against the BCQuality root it already knows instead. The generator resolves its own root, so invoking it by absolute path indexes and writes the right tree.
|
- The paths above assume the checkout root is the current directory. A caller that enters Entry from elsewhere — a plugin host, whose working directory is the user's own project — MUST resolve them against the BCQuality root it already knows instead. The generator resolves its own root, so invoking it by absolute path indexes and writes the right tree.
|
||||||
|
- For read-only plan enrichment, generated artifacts MUST remain outside the target repository. When the target contains the BCQuality checkout, or that checkout is immutable, pass the generator's `-IndexPath` to an external runner-owned artifact location and supply that resolved index path to the dispatched skill. Do not regenerate inside the target or modify the immutable checkout. If generation is unavailable, use READ's path-based discovery; retrieval failure is not an empty corpus.
|
||||||
- Pruning is the consumer's job, not Entry's, and not every consumer does it: an installation that ships the whole tree gets no deny guarantee from this step. There, `enabled-layers` narrows discovery only, and the unlisted layers' files remain on disk.
|
- Pruning is the consumer's job, not Entry's, and not every consumer does it: an installation that ships the whole tree gets no deny guarantee from this step. There, `enabled-layers` narrows discovery only, and the unlisted layers' files remain on disk.
|
||||||
- This is a side step. It MUST NOT change Entry's output — the dispatch record below is the only thing Entry emits, and build logs are never part of the dispatch JSON.
|
- This is a side step. It MUST NOT change Entry's output — the dispatch record below is the only thing Entry emits, and build logs are never part of the dispatch JSON.
|
||||||
|
|
||||||
|
|
@ -123,11 +123,11 @@ Emit a single JSON document conforming to the output contract below. Entry does
|
||||||
|
|
||||||
**`dispatch[]`** — each entry names one action skill to invoke.
|
**`dispatch[]`** — each entry names one action skill to invoke.
|
||||||
|
|
||||||
- `skill.path` — repo-relative, forward slashes. The agent fetches and executes the file directly from this path.
|
- `skill.path` — repo-relative, forward slashes, copied from discovery. Resolve it inside the live BCQuality checkout; reject absolute paths, traversal, and links escaping that checkout. The agent reads the action skill at this exact path rather than constructing a plausible filename.
|
||||||
- `skill.version` — copied from the dispatched skill's frontmatter so the orchestrator can detect drift between dispatch time and execution.
|
- `skill.version` — copied from the dispatched skill's frontmatter so the orchestrator can detect drift between dispatch time and execution.
|
||||||
- `rationale` — short human-readable string, for logs and traceability.
|
- `rationale` — short human-readable string, for logs and traceability.
|
||||||
- `inputs` — the intersection of `task-context.inputs-available` and the skill's declared `inputs`. The agent MUST pass exactly this subset when invoking the skill. Sending a strict intersection avoids accidental information leakage between skills.
|
- `inputs` — the intersection of `task-context.inputs-available` and the skill's declared `inputs`. The agent MUST pass exactly this subset when invoking the skill. Sending a strict intersection avoids accidental information leakage between skills.
|
||||||
- `outputs` — the dispatched skill's complete, single-element `outputs` value copied from frontmatter. This lets an orchestrator distinguish read-only `findings-report` and `development-guidance-report` work from repository-changing `implementation-report` work before invoking the skill. An orchestrator MAY require an additional write confirmation for `implementation-report`; it MUST NOT infer side effects from the skill ID or title.
|
- `outputs` — the dispatched skill's complete, single-element `outputs` value copied from frontmatter. This lets an orchestrator distinguish `findings-report` from read-only `development-guidance-report` before invocation. Check the declared contract and actual skill; output metadata is not a sandbox or proof of side effects. Unknown output kinds must not be silently treated as a supported report.
|
||||||
|
|
||||||
Ordering of `dispatch[]` is not significant.
|
Ordering of `dispatch[]` is not significant.
|
||||||
|
|
||||||
|
|
@ -180,7 +180,7 @@ Populated example (PR review on a repo where only `al-performance-review` is ena
|
||||||
|
|
||||||
1. Invoke Entry with the orchestrator-supplied task context.
|
1. Invoke Entry with the orchestrator-supplied task context.
|
||||||
2. Receive the dispatch record.
|
2. Receive the dispatch record.
|
||||||
3. For each entry in `dispatch[]`, inspect `outputs` before invocation, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce the declared report kind. Verify the file's frontmatter output still equals the dispatch value; return `failed` on drift rather than executing an unexpectedly mutating skill.
|
3. For each entry in `dispatch[]`, inspect `outputs` before invocation, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce the declared report kind. Verify the file's frontmatter output still equals the dispatch value; return `failed` on drift rather than executing a different contract.
|
||||||
4. Return the action-skill reports to the orchestrator. When Entry's `outcome` is `no-match` or `failed`, return the dispatch record itself so the orchestrator can log the reason.
|
4. Return the action-skill reports to the orchestrator. When Entry's `outcome` is `no-match` or `failed`, return the dispatch record itself so the orchestrator can log the reason.
|
||||||
|
|
||||||
READ and DO are the contracts that govern what the dispatched skills do. An agent that has not yet read READ and DO reads them when it executes the first dispatched skill — they are not prerequisites for invoking Entry.
|
READ and DO are the contracts that govern what the dispatched skills do. An agent that has not yet read READ and DO reads them when it executes the first dispatched skill — they are not prerequisites for invoking Entry.
|
||||||
|
|
|
||||||
496
tools/DevelopmentGuidance.Evidence.ps1
Normal file
496
tools/DevelopmentGuidance.Evidence.ps1
Normal file
|
|
@ -0,0 +1,496 @@
|
||||||
|
# Helpers for Test-DevelopmentGuidanceFixtures.ps1 and its deterministic regressions.
|
||||||
|
function Get-GuidanceDiagnostic {
|
||||||
|
param($ErrorRecord)
|
||||||
|
if ($ErrorRecord.Exception -is [Management.Automation.RuntimeException] -and
|
||||||
|
$ErrorRecord.FullyQualifiedErrorId -eq $ErrorRecord.Exception.Message) {
|
||||||
|
return $ErrorRecord.Exception.Message
|
||||||
|
}
|
||||||
|
return 'Evidence or report could not be read safely (missing, malformed, inaccessible or unsupported state).'
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceObject {
|
||||||
|
param($Value, [string] $Label, [string[]] $Fields = @())
|
||||||
|
if ($Value -isnot [Collections.IDictionary]) { throw "$Label must be a JSON object." }
|
||||||
|
foreach ($field in $Fields) {
|
||||||
|
if (-not $Value.Contains($field)) { throw "$Label is missing required field '$field'." }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceString {
|
||||||
|
param($Value, [string] $Label)
|
||||||
|
if ($Value -isnot [string] -or [string]::IsNullOrWhiteSpace($Value)) { throw "$Label must be a non-empty string." }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceArray {
|
||||||
|
param($Value, [string] $Label, [switch] $Strings, [switch] $NonEmpty)
|
||||||
|
if ($Value -isnot [array]) { throw "$Label must be a JSON array." }
|
||||||
|
if ($NonEmpty -and $Value.Count -eq 0) { throw "$Label must not be empty." }
|
||||||
|
if ($Strings) { foreach ($item in $Value) { Assert-GuidanceString $item "$Label entry" } }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceInteger {
|
||||||
|
param($Value, [string] $Label)
|
||||||
|
if (($Value -isnot [long] -and $Value -isnot [int] -and $Value -isnot [bigint]) -or $Value -lt 0) {
|
||||||
|
throw "$Label must be a non-negative integer."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Read-GuidanceJson {
|
||||||
|
param([string] $Path)
|
||||||
|
function Assert-JsonMembers($Element) {
|
||||||
|
if ($Element.ValueKind -eq [Text.Json.JsonValueKind]::Object) {
|
||||||
|
$names = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
|
||||||
|
foreach ($property in $Element.EnumerateObject()) {
|
||||||
|
if (-not $names.Add($property.Name)) { throw 'JSON contains duplicate or case-ambiguous members.' }
|
||||||
|
Assert-JsonMembers $property.Value
|
||||||
|
}
|
||||||
|
} elseif ($Element.ValueKind -eq [Text.Json.JsonValueKind]::Array) {
|
||||||
|
foreach ($item in $Element.EnumerateArray()) { Assert-JsonMembers $item }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if ((Get-Item -LiteralPath $Path -Force).Length -gt 32MB) { throw 'Oversized JSON.' }
|
||||||
|
$text = [IO.File]::ReadAllText($Path)
|
||||||
|
$document = [Text.Json.JsonDocument]::Parse($text)
|
||||||
|
try { Assert-JsonMembers $document.RootElement } finally { $document.Dispose() }
|
||||||
|
return ConvertFrom-Json -InputObject $text -AsHashtable -Depth 64 -NoEnumerate
|
||||||
|
} catch { throw 'Input is not readable, strict JSON with unique members.' }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-GuidanceHash {
|
||||||
|
param([string] $Path)
|
||||||
|
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-GuidanceCaseId {
|
||||||
|
param([string] $Id)
|
||||||
|
$bytes = [Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($Id))
|
||||||
|
return "case-$([Convert]::ToHexString($bytes).Substring(0, 8).ToLowerInvariant())"
|
||||||
|
}
|
||||||
|
|
||||||
|
function Test-GuidanceWithin {
|
||||||
|
param([string] $Path, [string] $Parent)
|
||||||
|
$comparison = if ($IsWindows) { [StringComparison]::OrdinalIgnoreCase } else { [StringComparison]::Ordinal }
|
||||||
|
$prefix = $Parent.TrimEnd([IO.Path]::DirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar
|
||||||
|
return $Path.Equals($Parent, $comparison) -or $Path.StartsWith($prefix, $comparison)
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceItem {
|
||||||
|
param($Item)
|
||||||
|
if (($Item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or $Item.LinkType -or $Item.LinkTarget) {
|
||||||
|
throw 'Links, junctions, hard links and reparse points are not supported.'
|
||||||
|
}
|
||||||
|
if (-not $Item.PSIsContainer -and $IsWindows) {
|
||||||
|
$streams = @(Get-Item -LiteralPath $Item.FullName -Stream '*' -Force -ErrorAction Stop)
|
||||||
|
if (@($streams | Where-Object Stream -ne ':$DATA').Count) {
|
||||||
|
throw 'Alternate data streams are not supported.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-GuidanceSafePath {
|
||||||
|
param([string] $Path, [switch] $AllowMissing, [switch] $Directory, [switch] $File)
|
||||||
|
Assert-GuidanceString $Path 'Filesystem path'
|
||||||
|
$full = [IO.Path]::GetFullPath($Path)
|
||||||
|
$driveRoot = [IO.Path]::GetPathRoot($full)
|
||||||
|
if ($IsWindows -and ($driveRoot.StartsWith('\\') -or $full.Substring($driveRoot.Length).Contains(':'))) {
|
||||||
|
throw 'Network paths and alternate stream paths are not supported.'
|
||||||
|
}
|
||||||
|
$cursor = $driveRoot
|
||||||
|
$components = $full.Substring($driveRoot.Length).Split([IO.Path]::DirectorySeparatorChar, [StringSplitOptions]::RemoveEmptyEntries)
|
||||||
|
foreach ($component in $components) {
|
||||||
|
if ($component -match '[\. ]$') { throw 'Ambiguous filesystem path components are not supported.' }
|
||||||
|
$cursor = Join-Path $cursor $component
|
||||||
|
# Get-Item sees dangling links that Test-Path may treat as missing.
|
||||||
|
$item = Get-Item -LiteralPath $cursor -Force -ErrorAction SilentlyContinue
|
||||||
|
if ($null -ne $item) { Assert-GuidanceItem $item }
|
||||||
|
elseif (-not $AllowMissing) { throw 'Required filesystem path is missing.' }
|
||||||
|
}
|
||||||
|
if ($Directory -and -not (Test-Path -LiteralPath $full -PathType Container)) { throw 'Required directory is missing.' }
|
||||||
|
if ($File -and -not (Test-Path -LiteralPath $full -PathType Leaf)) { throw 'Required file is missing.' }
|
||||||
|
return $full.TrimEnd([IO.Path]::DirectorySeparatorChar)
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceTree {
|
||||||
|
param([string] $Root)
|
||||||
|
$pending = [Collections.Generic.Stack[string]]::new()
|
||||||
|
$pending.Push($Root)
|
||||||
|
while ($pending.Count) {
|
||||||
|
foreach ($item in Get-ChildItem -LiteralPath $pending.Pop() -Force) {
|
||||||
|
Assert-GuidanceItem $item
|
||||||
|
if ($item.PSIsContainer) { $pending.Push($item.FullName) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Invoke-GuidanceGit {
|
||||||
|
param([string] $Root, [string[]] $Arguments, [switch] $RawOutput)
|
||||||
|
$start = [Diagnostics.ProcessStartInfo]::new('git')
|
||||||
|
$start.UseShellExecute = $false
|
||||||
|
$start.RedirectStandardOutput = $true
|
||||||
|
$start.RedirectStandardError = $true
|
||||||
|
foreach ($variable in @('GIT_DIR', 'GIT_WORK_TREE', 'GIT_COMMON_DIR', 'GIT_INDEX_FILE',
|
||||||
|
'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_CONFIG',
|
||||||
|
'GIT_CONFIG_COUNT', 'GIT_CONFIG_PARAMETERS', 'GIT_NAMESPACE')) {
|
||||||
|
$null = $start.Environment.Remove($variable)
|
||||||
|
}
|
||||||
|
$start.Environment['GIT_CONFIG_NOSYSTEM'] = '1'
|
||||||
|
$start.Environment['GIT_CONFIG_GLOBAL'] = ''
|
||||||
|
$start.Environment['GIT_TERMINAL_PROMPT'] = '0'
|
||||||
|
# In particular, do not execute a repository-supplied fsmonitor hook on reads.
|
||||||
|
foreach ($argument in (@('--no-optional-locks', '-c', 'core.fsmonitor=false', '-C', $Root) + $Arguments)) {
|
||||||
|
$start.ArgumentList.Add($argument)
|
||||||
|
}
|
||||||
|
$process = [Diagnostics.Process]::Start($start)
|
||||||
|
try {
|
||||||
|
$output = $process.StandardOutput.ReadToEndAsync()
|
||||||
|
$errors = $process.StandardError.ReadToEndAsync()
|
||||||
|
$process.WaitForExit()
|
||||||
|
$null = $errors.GetAwaiter().GetResult()
|
||||||
|
if ($process.ExitCode -ne 0) { throw 'Git evidence could not be read.' }
|
||||||
|
$text = $output.GetAwaiter().GetResult()
|
||||||
|
if ($RawOutput) { return $text }
|
||||||
|
return $text.TrimEnd("`r", "`n")
|
||||||
|
} finally { $process.Dispose() }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-GuidanceSnapshot {
|
||||||
|
param([string] $Root, [switch] $Target)
|
||||||
|
$Root = Get-GuidanceSafePath $Root -Directory
|
||||||
|
$dotGit = Get-GuidanceSafePath (Join-Path $Root '.git')
|
||||||
|
if (Test-Path -LiteralPath $dotGit -PathType Container) {
|
||||||
|
$gitDir = $dotGit
|
||||||
|
} else {
|
||||||
|
if ($Target) { throw 'Target workspaces must be standalone repositories with internal Git storage.' }
|
||||||
|
$pointer = [IO.File]::ReadAllText($dotGit)
|
||||||
|
if ($pointer -notmatch '\Agitdir: ([^\r\n]+)\r?\n?\z') { throw 'Unsupported Git worktree pointer.' }
|
||||||
|
$gitPath = $Matches[1]
|
||||||
|
if (-not [IO.Path]::IsPathFullyQualified($gitPath)) { $gitPath = Join-Path $Root $gitPath }
|
||||||
|
$gitDir = Get-GuidanceSafePath $gitPath -Directory
|
||||||
|
}
|
||||||
|
$commonDir = $gitDir
|
||||||
|
$commonPointer = Get-GuidanceSafePath (Join-Path $gitDir 'commondir') -AllowMissing
|
||||||
|
if (Test-Path -LiteralPath $commonPointer) {
|
||||||
|
$commonPath = [IO.File]::ReadAllText($commonPointer).Trim()
|
||||||
|
if (-not [IO.Path]::IsPathFullyQualified($commonPath)) { $commonPath = Join-Path $gitDir $commonPath }
|
||||||
|
$commonDir = Get-GuidanceSafePath $commonPath -Directory
|
||||||
|
}
|
||||||
|
if ($Target -and ($gitDir -cne (Join-Path $Root '.git') -or $commonDir -cne $gitDir)) {
|
||||||
|
throw 'Target workspaces must be standalone repositories with internal Git storage.'
|
||||||
|
}
|
||||||
|
$files = [Collections.Generic.List[object]]::new()
|
||||||
|
$pending = [Collections.Generic.Stack[string]]::new()
|
||||||
|
$pending.Push($Root)
|
||||||
|
while ($pending.Count) {
|
||||||
|
foreach ($item in @(Get-ChildItem -LiteralPath $pending.Pop() -Force | Sort-Object Name -CaseSensitive)) {
|
||||||
|
Assert-GuidanceItem $item
|
||||||
|
$relative = [IO.Path]::GetRelativePath($Root, $item.FullName).Replace('\', '/')
|
||||||
|
$entry = [ordered]@{
|
||||||
|
path = $relative
|
||||||
|
kind = if ($item.PSIsContainer) { 'directory' } else { 'file' }
|
||||||
|
attributes = [int]$item.Attributes
|
||||||
|
unixMode = [int]$item.UnixFileMode
|
||||||
|
creationUtcTicks = $item.CreationTimeUtc.Ticks
|
||||||
|
}
|
||||||
|
if ($item.PSIsContainer) {
|
||||||
|
$pending.Push($item.FullName)
|
||||||
|
} else {
|
||||||
|
$entry.length = $item.Length
|
||||||
|
$entry.lastWriteUtcTicks = $item.LastWriteTimeUtc.Ticks
|
||||||
|
$entry.sha256 = Get-GuidanceHash $item.FullName
|
||||||
|
}
|
||||||
|
$files.Add($entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# Linked knowledge worktrees have explicitly identified Git metadata outside
|
||||||
|
# the content root. Record its meaningful configuration as well as HEAD/refs.
|
||||||
|
$gitMetadata = [ordered]@{}
|
||||||
|
foreach ($storage in @($gitDir, $commonDir) | Sort-Object -Unique) {
|
||||||
|
foreach ($name in @('HEAD', 'commondir', 'config', 'config.worktree', 'packed-refs', 'refs', 'objects', 'index', 'info\exclude', 'shallow')) {
|
||||||
|
$path = Get-GuidanceSafePath (Join-Path $storage $name) -AllowMissing
|
||||||
|
if ((Test-Path -LiteralPath $path -PathType Container) -and -not (Test-GuidanceWithin $path $Root)) {
|
||||||
|
Assert-GuidanceTree $path
|
||||||
|
}
|
||||||
|
if (Test-Path -LiteralPath $path -PathType Leaf) {
|
||||||
|
$gitMetadata[$path] = Get-GuidanceHash $path
|
||||||
|
}
|
||||||
|
if ($name -in @('config', 'config.worktree') -and (Test-Path -LiteralPath $path) -and
|
||||||
|
[IO.File]::ReadAllText($path) -match '(?im)^\s*\[include(?:If)?\b') {
|
||||||
|
throw 'External Git configuration includes are not supported.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foreach ($name in @('objects\info\alternates', 'objects\info\http-alternates')) {
|
||||||
|
if (Test-Path -LiteralPath (Join-Path $storage $name)) { throw 'External Git object stores are not supported.' }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$top = Get-GuidanceSafePath (Invoke-GuidanceGit $Root @('rev-parse', '--show-toplevel')) -Directory
|
||||||
|
if ($top -cne $Root) { throw 'Evidence requires the exact Git worktree root, not a subdirectory.' }
|
||||||
|
$indexPath = Get-GuidanceSafePath (Join-Path $gitDir 'index') -AllowMissing
|
||||||
|
$tracked = Invoke-GuidanceGit $Root @('ls-files', '--stage')
|
||||||
|
if ($tracked -match '(?m)^160000 ') { throw 'Submodule workspaces are not supported.' }
|
||||||
|
if ((Invoke-GuidanceGit $Root @('ls-files', '-t')) -match '(?m)^S ') { throw 'Sparse workspaces are not supported.' }
|
||||||
|
$head = Invoke-GuidanceGit $Root @('rev-parse', '--verify', 'HEAD')
|
||||||
|
$headFile = Get-GuidanceSafePath (Join-Path $gitDir 'HEAD') -File
|
||||||
|
# Access times, Git status refreshes, and index-builder generatedAt are not evidence.
|
||||||
|
return [ordered]@{
|
||||||
|
version = 1
|
||||||
|
root = $Root
|
||||||
|
rootCreationUtcTicks = (Get-Item -LiteralPath $Root -Force).CreationTimeUtc.Ticks
|
||||||
|
gitDir = $gitDir
|
||||||
|
commonDir = $commonDir
|
||||||
|
gitMetadata = $gitMetadata
|
||||||
|
head = $head
|
||||||
|
headFileSha256 = Get-GuidanceHash $headFile
|
||||||
|
references = Invoke-GuidanceGit $Root @('for-each-ref', '--format=%(refname) %(objectname) %(symref)')
|
||||||
|
indexPath = $indexPath
|
||||||
|
indexSha256 = if (Test-Path -LiteralPath $indexPath) { Get-GuidanceHash $indexPath } else { $null }
|
||||||
|
files = @($files | Sort-Object { $_.path } -CaseSensitive)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Test-GuidanceSnapshotEqual {
|
||||||
|
param($Before, $After)
|
||||||
|
return ($Before | ConvertTo-Json -Depth 64 -Compress) -ceq ($After | ConvertTo-Json -Depth 64 -Compress)
|
||||||
|
}
|
||||||
|
|
||||||
|
function Write-GuidanceNewJson {
|
||||||
|
param([string] $Path, $Value)
|
||||||
|
$parent = Split-Path -Parent $Path
|
||||||
|
[IO.Directory]::CreateDirectory($parent) | Out-Null
|
||||||
|
$bytes = [Text.Encoding]::UTF8.GetBytes(($Value | ConvertTo-Json -Depth 64))
|
||||||
|
$stream = [IO.File]::Open($Path, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||||
|
try { $stream.Write($bytes, 0, $bytes.Length) } finally { $stream.Dispose() }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Resolve-GuidanceReference {
|
||||||
|
param([string] $Root, $Reference, [switch] $Knowledge, [string] $Article)
|
||||||
|
Assert-GuidanceString $Reference 'Reference path'
|
||||||
|
if ($Reference -cnotmatch '^[a-zA-Z0-9_-]+(?:/[a-zA-Z0-9_.-]+)+$' -or
|
||||||
|
@($Reference.Split('/') | Where-Object { $_ -in @('.', '..') -or $_ -match '[\. ]$' }).Count) {
|
||||||
|
throw 'Reference must be an unambiguous forward-slash repository-relative path without traversal.'
|
||||||
|
}
|
||||||
|
if ($Knowledge -and $Reference -cnotmatch '^(microsoft|community|custom)/knowledge/[a-z0-9-]+/(?:[a-z0-9-]+/)*[a-z0-9-]+\.md$') {
|
||||||
|
throw 'Knowledge references must identify actual layered knowledge articles.'
|
||||||
|
}
|
||||||
|
if ($Article) {
|
||||||
|
$stem = $Article.Substring(0, $Article.Length - 3)
|
||||||
|
if ($Reference -cnotmatch ('^' + [regex]::Escape($stem) + '\.(good|bad)\.[a-zA-Z0-9]+$')) {
|
||||||
|
throw 'Sample references must be good/bad siblings of their knowledge article.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$cursor = $Root
|
||||||
|
foreach ($component in $Reference.Split('/')) {
|
||||||
|
$cursor = Join-Path $cursor $component
|
||||||
|
$cursor = Get-GuidanceSafePath $cursor
|
||||||
|
if ((Get-Item -LiteralPath $cursor -Force).Name -cne $component) { throw 'Reference path casing must match the actual file.' }
|
||||||
|
}
|
||||||
|
if (-not (Test-GuidanceWithin $cursor $Root) -or -not (Test-Path -LiteralPath $cursor -PathType Leaf)) {
|
||||||
|
throw 'Reference must resolve to an existing file inside the knowledge checkout.'
|
||||||
|
}
|
||||||
|
if ($Knowledge) {
|
||||||
|
$text = [IO.File]::ReadAllText($cursor)
|
||||||
|
if ($text -notmatch '(?s)^---\r?\n.*?\r?\n---' -or
|
||||||
|
$text -notmatch '(?m)^domain:\s*\S+' -or $text -notmatch '(?m)^## (Best Practice|Anti Pattern)\s*$') {
|
||||||
|
throw 'Knowledge reference does not contain a normative knowledge article.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $cursor
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-GuidancePlanRequest {
|
||||||
|
param($Plan)
|
||||||
|
if ($Plan -is [string]) { Assert-GuidanceString $Plan 'development-plan'; return $Plan }
|
||||||
|
Assert-GuidanceObject $Plan 'development-plan' @('request')
|
||||||
|
Assert-GuidanceString $Plan.request 'development-plan.request'
|
||||||
|
return $Plan.request
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceContext {
|
||||||
|
param($Context)
|
||||||
|
Assert-GuidanceObject $Context 'context' @('bc-version', 'technologies', 'countries', 'application-area', 'unknown')
|
||||||
|
Assert-GuidanceString $Context.'bc-version' 'context.bc-version'
|
||||||
|
foreach ($key in @('technologies', 'countries', 'application-area', 'unknown')) {
|
||||||
|
Assert-GuidanceArray $Context[$key] "context.$key" -Strings
|
||||||
|
if (@($Context[$key] | Sort-Object -Unique).Count -ne $Context[$key].Count) { throw "context.$key contains duplicates." }
|
||||||
|
}
|
||||||
|
foreach ($key in $Context.unknown) {
|
||||||
|
if ($key -cnotin @('bc-version', 'technologies', 'countries', 'application-area')) { throw 'context.unknown contains an invalid dimension.' }
|
||||||
|
}
|
||||||
|
if ($Context.'bc-version' -eq 'unknown' -and 'bc-version' -cnotin $Context.unknown) {
|
||||||
|
throw 'Unknown BC version must be recorded in context.unknown.'
|
||||||
|
}
|
||||||
|
foreach ($key in @('technologies', 'countries', 'application-area')) {
|
||||||
|
if ((-not $Context[$key].Count -or 'unknown' -in $Context[$key]) -and $key -cnotin $Context.unknown) {
|
||||||
|
throw 'Unavailable applicability dimensions must be recorded in context.unknown.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceManifest {
|
||||||
|
param($Manifest, [string] $Root)
|
||||||
|
Assert-GuidanceObject $Manifest 'manifest' @('version', 'minimumKnowledgeRecall', 'minimumKnowledgePrecision', 'skill', 'cases')
|
||||||
|
if ($Manifest.version -ne 1) { throw 'Unsupported guidance fixture manifest version.' }
|
||||||
|
foreach ($name in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
||||||
|
$value = $Manifest[$name]
|
||||||
|
if (($value -isnot [double] -and $value -isnot [long] -and $value -isnot [int] -and $value -isnot [decimal]) -or
|
||||||
|
$value -lt 0 -or $value -gt 1) { throw 'Manifest thresholds must be numbers between zero and one.' }
|
||||||
|
}
|
||||||
|
$null = Resolve-GuidanceReference $Root $Manifest.skill
|
||||||
|
Assert-GuidanceArray $Manifest.cases 'manifest.cases' -NonEmpty
|
||||||
|
$ids = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||||
|
$modelIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||||
|
foreach ($case in $Manifest.cases) {
|
||||||
|
Assert-GuidanceObject $case 'manifest case' @('id', 'expectedKind', 'expectedOutcome', 'development-plan', 'context', 'requiredKnowledge', 'optionalKnowledge', 'expectedUnknown', 'requiresUnresolved', 'requiresMaterialUnresolved')
|
||||||
|
if ($case.id -isnot [string] -or $case.id -cnotmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') { throw 'Fixture id must be kebab-case.' }
|
||||||
|
if (-not $ids.Add($case.id) -or -not $modelIds.Add((Get-GuidanceCaseId $case.id))) { throw 'Duplicate fixture or model case identity.' }
|
||||||
|
if ($case.expectedKind -cnotin @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')) { throw 'Fixture expectedKind is invalid.' }
|
||||||
|
if ($case.expectedOutcome -cnotin @('completed', 'not-applicable', 'no-knowledge', 'partial', 'failed')) { throw 'Fixture expectedOutcome is invalid.' }
|
||||||
|
Assert-GuidanceString $case.expectedKind 'fixture.expectedKind'
|
||||||
|
Assert-GuidanceString $case.expectedOutcome 'fixture.expectedOutcome'
|
||||||
|
$null = Get-GuidancePlanRequest $case.'development-plan'
|
||||||
|
Assert-GuidanceContext $case.context
|
||||||
|
foreach ($name in @('requiredKnowledge', 'optionalKnowledge', 'expectedUnknown')) {
|
||||||
|
Assert-GuidanceArray $case[$name] "fixture.$name" -Strings
|
||||||
|
}
|
||||||
|
if ($case.requiresUnresolved -isnot [bool]) { throw 'Fixture requiresUnresolved must be boolean.' }
|
||||||
|
if ($case.requiresMaterialUnresolved -isnot [bool]) { throw 'Fixture requiresMaterialUnresolved must be boolean.' }
|
||||||
|
if ($case.requiresMaterialUnresolved -and ($case.expectedOutcome -ne 'partial' -or -not $case.requiresUnresolved)) {
|
||||||
|
throw 'Materially unresolved fixtures must expect partial and unresolved evidence.'
|
||||||
|
}
|
||||||
|
foreach ($key in $case.expectedUnknown) {
|
||||||
|
if ($key -cnotin @('bc-version', 'technologies', 'countries', 'application-area')) { throw 'Fixture expectedUnknown is invalid.' }
|
||||||
|
}
|
||||||
|
$references = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
||||||
|
if (@($references | Sort-Object -Unique).Count -ne $references.Count) { throw 'Fixture knowledge references contain duplicates.' }
|
||||||
|
foreach ($reference in $references) { $null = Resolve-GuidanceReference $Root $reference -Knowledge }
|
||||||
|
if ($case.expectedOutcome -in @('no-knowledge', 'not-applicable') -and $references.Count) {
|
||||||
|
throw 'Empty-knowledge outcomes cannot require or accept knowledge.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceResult {
|
||||||
|
param($Result, $Case, $Manifest, [string] $Root, [string] $Workspace)
|
||||||
|
Assert-GuidanceObject $Result 'result' @('caseId', 'guidanceReport')
|
||||||
|
Assert-GuidanceString $Result.caseId 'result.caseId'
|
||||||
|
if ($Result.caseId -cne (Get-GuidanceCaseId $Case.id)) { throw 'Result caseId mismatch.' }
|
||||||
|
if ($Result.Contains('workspaceRoot')) {
|
||||||
|
Assert-GuidanceString $Result.workspaceRoot 'result.workspaceRoot'
|
||||||
|
if (-not [IO.Path]::IsPathFullyQualified($Result.workspaceRoot) -or
|
||||||
|
(Get-GuidanceSafePath $Result.workspaceRoot -Directory) -cne $Workspace) {
|
||||||
|
throw 'Result workspaceRoot disagrees with the runner binding.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$report = $Result.guidanceReport
|
||||||
|
Assert-GuidanceObject $report 'guidanceReport' @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')
|
||||||
|
Assert-GuidanceObject $report.skill 'skill' @('id', 'version')
|
||||||
|
Assert-GuidanceString $report.skill.id 'skill.id'
|
||||||
|
if ($report.skill.id -cne 'al-development-plan' -or $report.skill.version -isnot [long] -or $report.skill.version -ne 1) {
|
||||||
|
throw 'Report skill identity/version is invalid.'
|
||||||
|
}
|
||||||
|
Assert-GuidanceString $report.outcome 'outcome'
|
||||||
|
if ($report.outcome -cnotin @('completed', 'not-applicable', 'no-knowledge', 'partial', 'failed')) { throw 'Report outcome enum is invalid.' }
|
||||||
|
if ($report.outcome -cne $Case.expectedOutcome) { throw 'Report outcome does not match fixture expectedOutcome.' }
|
||||||
|
if ($report.outcome -in @('partial', 'failed') -or $report.Contains('outcome-reason')) {
|
||||||
|
Assert-GuidanceString $report['outcome-reason'] 'outcome-reason'
|
||||||
|
}
|
||||||
|
Assert-GuidanceObject $report.summary 'summary' @('request', 'kind', 'candidates', 'selected')
|
||||||
|
Assert-GuidanceString $report.summary.request 'summary.request'
|
||||||
|
Assert-GuidanceString $report.summary.kind 'summary.kind'
|
||||||
|
if ($report.summary.kind -cne $Case.expectedKind) { throw 'summary.kind does not match the intended change.' }
|
||||||
|
Assert-GuidanceInteger $report.summary.candidates 'summary.candidates'
|
||||||
|
Assert-GuidanceInteger $report.summary.selected 'summary.selected'
|
||||||
|
Assert-GuidanceContext $report.context
|
||||||
|
foreach ($name in @('knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
|
||||||
|
Assert-GuidanceArray $report[$name] $name
|
||||||
|
}
|
||||||
|
Assert-GuidanceArray $report.unresolved 'unresolved' -Strings
|
||||||
|
if ($report.summary.selected -ne $report.knowledge.Count -or $report.summary.selected -gt $report.summary.candidates) {
|
||||||
|
throw 'Summary counts disagree with selected knowledge/candidates.'
|
||||||
|
}
|
||||||
|
if ($report.outcome -in @('no-knowledge', 'not-applicable') -and $report.knowledge.Count) { throw 'This outcome requires empty knowledge.' }
|
||||||
|
if ($report.outcome -eq 'completed' -and -not $report.knowledge.Count) { throw 'Completed requires selected knowledge; empty evaluation is no-knowledge.' }
|
||||||
|
if (($report.outcome -eq 'partial' -or $Case.requiresUnresolved) -and -not $report.unresolved.Count) {
|
||||||
|
throw 'Partial/incomplete evaluation must explain unresolved gaps.'
|
||||||
|
}
|
||||||
|
foreach ($dimension in $Case.expectedUnknown) {
|
||||||
|
if ($dimension -cnotin $report.context.unknown) { throw 'Expected unknown context was silently resolved.' }
|
||||||
|
}
|
||||||
|
foreach ($dimension in $report.context.unknown) {
|
||||||
|
if (-not @($report.unresolved | Where-Object { $_ -match [regex]::Escape($dimension) }).Count) {
|
||||||
|
throw 'Unknown dimensions require a corresponding unresolved explanation.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# Free-form unresolved text has no machine-readable materiality field in DO.
|
||||||
|
# Known material fixture conditions are runner expectations, not model claims.
|
||||||
|
if ($Case.requiresMaterialUnresolved -and $report.outcome -ne 'partial') { throw 'Material unknown guidance must remain partial.' }
|
||||||
|
$used = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||||
|
foreach ($entry in $report.knowledge) {
|
||||||
|
Assert-GuidanceObject $entry 'knowledge entry' @('path', 'used-for', 'constraints', 'sample-paths')
|
||||||
|
$null = Resolve-GuidanceReference $Root $entry.path -Knowledge
|
||||||
|
if (-not $used.Add($entry.path)) { throw 'Duplicate knowledge reference.' }
|
||||||
|
Assert-GuidanceString $entry.'used-for' 'knowledge.used-for'
|
||||||
|
Assert-GuidanceArray $entry.constraints 'knowledge.constraints' -Strings -NonEmpty
|
||||||
|
Assert-GuidanceArray $entry.'sample-paths' 'knowledge.sample-paths' -Strings
|
||||||
|
if (@($entry.'sample-paths' | Sort-Object -Unique).Count -ne $entry.'sample-paths'.Count) { throw 'Duplicate sample reference.' }
|
||||||
|
foreach ($sample in $entry.'sample-paths') { $null = Resolve-GuidanceReference $Root $sample -Article $entry.path }
|
||||||
|
Assert-GuidanceReferenceSha $entry $Root
|
||||||
|
}
|
||||||
|
$validationIds = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||||
|
foreach ($entry in $report.'validation-considerations') {
|
||||||
|
Assert-GuidanceObject $entry 'validation consideration' @('id', 'reason', 'evidence')
|
||||||
|
foreach ($key in @('id', 'reason', 'evidence')) { Assert-GuidanceString $entry[$key] "validation-considerations.$key" }
|
||||||
|
if (-not $validationIds.Add($entry.id)) { throw 'Duplicate validation consideration id.' }
|
||||||
|
}
|
||||||
|
$suppressed = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
|
||||||
|
foreach ($entry in $report.suppressed) {
|
||||||
|
Assert-GuidanceObject $entry 'suppressed entry' @('reference', 'reason')
|
||||||
|
Assert-GuidanceObject $entry.reference 'suppressed.reference' @('path')
|
||||||
|
$null = Resolve-GuidanceReference $Root $entry.reference.path -Knowledge
|
||||||
|
Assert-GuidanceReferenceSha $entry.reference $Root
|
||||||
|
Assert-GuidanceString $entry.reason 'suppression reason'
|
||||||
|
if ($entry.reason -cnotin @('layer-precedence', 'configuration')) { throw 'Suppression reason is invalid.' }
|
||||||
|
if (-not $suppressed.Add($entry.reference.path) -or $used.Contains($entry.reference.path)) { throw 'Duplicate or selected suppressed reference.' }
|
||||||
|
}
|
||||||
|
$matched = @($Case.requiredKnowledge | Where-Object { $used.Contains($_) }).Count
|
||||||
|
$recall = if ($Case.requiredKnowledge.Count) { $matched / $Case.requiredKnowledge.Count } else { 1.0 }
|
||||||
|
$accepted = @($Case.requiredKnowledge) + @($Case.optionalKnowledge)
|
||||||
|
$acceptedCount = @($used | Where-Object { $_ -cin $accepted }).Count
|
||||||
|
$precision = if ($used.Count) { $acceptedCount / $used.Count } elseif (-not $Case.requiredKnowledge.Count) { 1.0 } else { 0.0 }
|
||||||
|
if ($recall -lt $Manifest.minimumKnowledgeRecall) { throw 'Knowledge recall is below the manifest threshold.' }
|
||||||
|
if ($precision -lt $Manifest.minimumKnowledgePrecision) { throw 'Knowledge precision is below the manifest threshold.' }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-GuidanceReferenceSha {
|
||||||
|
param($Entry, [string] $Root)
|
||||||
|
if ($Entry.Contains('sha')) {
|
||||||
|
if ($Entry.sha -isnot [string] -or $Entry.sha -cnotmatch '^([0-9a-fA-F]{40}|[0-9a-fA-F]{64})$') {
|
||||||
|
throw 'Reference SHA must be a full commit object id.'
|
||||||
|
}
|
||||||
|
$head = Invoke-GuidanceGit $Root @('rev-parse', '--verify', 'HEAD')
|
||||||
|
if ($Entry.sha -ne $head) { throw 'Reference SHA does not identify the recorded live checkout.' }
|
||||||
|
$committed = Invoke-GuidanceGit $Root @('cat-file', 'blob', "$head`:$($Entry.path)") -RawOutput
|
||||||
|
$live = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $Entry.path -Knowledge))
|
||||||
|
if ($committed.Replace("`r`n", "`n") -cne $live.Replace("`r`n", "`n")) {
|
||||||
|
throw 'Reference SHA content differs from the live knowledge article.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-GuidanceResultSchema {
|
||||||
|
param([string] $CaseId)
|
||||||
|
return [ordered]@{
|
||||||
|
caseId = $CaseId
|
||||||
|
guidanceReport = [ordered]@{
|
||||||
|
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||||
|
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
||||||
|
'outcome-reason' = 'required for partial or failed'
|
||||||
|
summary = [ordered]@{ request = 'planned intent'; kind = 'feature | bug | refactor | upgrade | maintenance'; candidates = 0; selected = 0 }
|
||||||
|
context = [ordered]@{ 'bc-version' = 'resolved target or unknown'; technologies = @('al'); countries = @('w1'); 'application-area' = @('all'); unknown = @() }
|
||||||
|
knowledge = @([ordered]@{ path = 'repo-relative knowledge article'; sha = 'optional full checkout commit id'; 'used-for' = 'plan decision'; constraints = @('faithful normative constraint'); 'sample-paths' = @() })
|
||||||
|
'validation-considerations' = @([ordered]@{ id = 'stable id'; reason = 'why needed'; evidence = 'evidence implementation should obtain' })
|
||||||
|
suppressed = @([ordered]@{ reference = [ordered]@{ path = 'suppressed knowledge path' }; reason = 'layer-precedence | configuration' })
|
||||||
|
unresolved = @('Missing context/decision, affected candidate, and materiality; name unknown dimensions exactly.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,584 +0,0 @@
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Validates and prepares BCQuality AL development fixtures.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
|
||||||
Static validation checks fixture IDs, capability links, knowledge references,
|
|
||||||
and the development skill. -PrepareDirectory emits opaque requests for model
|
|
||||||
runs. -ResultsDirectory scores implementation reports produced by an
|
|
||||||
external runner that performed the declared compile, test, and review checks.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param(
|
|
||||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
|
||||||
[string] $ManifestPath,
|
|
||||||
[string] $CapabilitiesPath,
|
|
||||||
[string] $PrepareDirectory,
|
|
||||||
[string] $ResultsDirectory
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
|
|
||||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
|
||||||
if (-not $ManifestPath) {
|
|
||||||
$ManifestPath = Join-Path $Root 'evaluation/development-fixtures.json'
|
|
||||||
}
|
|
||||||
if (-not $CapabilitiesPath) {
|
|
||||||
$CapabilitiesPath = Join-Path $Root 'coverage/development-capabilities.json'
|
|
||||||
}
|
|
||||||
|
|
||||||
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
|
|
||||||
$capabilityManifest = Get-Content -LiteralPath $CapabilitiesPath -Raw | ConvertFrom-Json
|
|
||||||
$problems = [System.Collections.Generic.List[string]]::new()
|
|
||||||
|
|
||||||
function Get-ModelCaseId {
|
|
||||||
param([string] $ManifestId)
|
|
||||||
|
|
||||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
|
||||||
try {
|
|
||||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
|
|
||||||
$hash = $sha.ComputeHash($bytes)
|
|
||||||
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
|
|
||||||
return "case-$token"
|
|
||||||
} finally {
|
|
||||||
$sha.Dispose()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($manifest.version -ne 1) {
|
|
||||||
$problems.Add("Unsupported development fixture version: $($manifest.version)") | Out-Null
|
|
||||||
}
|
|
||||||
if ($capabilityManifest.version -ne 1) {
|
|
||||||
$problems.Add("Unsupported capability manifest version: $($capabilityManifest.version)") | Out-Null
|
|
||||||
}
|
|
||||||
$minimumFixtureCoverage = if ($capabilityManifest.PSObject.Properties.Name -contains 'minimumFixtureCoverage') {
|
|
||||||
[double]$capabilityManifest.minimumFixtureCoverage
|
|
||||||
} else {
|
|
||||||
-1
|
|
||||||
}
|
|
||||||
if ($minimumFixtureCoverage -lt 0 -or $minimumFixtureCoverage -gt 1) {
|
|
||||||
$problems.Add("minimumFixtureCoverage must be between 0 and 1.") | Out-Null
|
|
||||||
}
|
|
||||||
$maximumReviewRounds = if ($manifest.PSObject.Properties.Name -contains 'maximumReviewRounds') {
|
|
||||||
[int]$manifest.maximumReviewRounds
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
if ($maximumReviewRounds -le 0) {
|
|
||||||
$problems.Add("maximumReviewRounds must be a positive integer.") | Out-Null
|
|
||||||
}
|
|
||||||
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
|
||||||
$threshold = [double]$manifest.$thresholdName
|
|
||||||
if ($threshold -lt 0 -or $threshold -gt 1) {
|
|
||||||
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$skillPath = [string]$manifest.skill
|
|
||||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
|
|
||||||
$problems.Add("Development skill does not exist: $skillPath") | Out-Null
|
|
||||||
} else {
|
|
||||||
$skillText = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
|
||||||
$limitMatch = [regex]::Match($skillText, '(?m)^quality-round-limit:\s*(\d+)\s*$')
|
|
||||||
if (-not $limitMatch.Success -or [int]$limitMatch.Groups[1].Value -ne $maximumReviewRounds) {
|
|
||||||
$problems.Add("maximumReviewRounds must match the development skill quality-round-limit.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$validChecks = @('compile', 'tests', 'review')
|
|
||||||
$validInputKinds = @('auto', 'feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
|
||||||
$validOutputKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
|
||||||
$caseById = @{}
|
|
||||||
foreach ($case in @($manifest.cases)) {
|
|
||||||
$id = [string]$case.id
|
|
||||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
|
||||||
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
|
|
||||||
} elseif ($caseById.ContainsKey($id)) {
|
|
||||||
$problems.Add("Duplicate fixture id: $id") | Out-Null
|
|
||||||
} else {
|
|
||||||
$caseById[$id] = $case
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($case.PSObject.Properties.Name -notcontains 'development-request') {
|
|
||||||
$problems.Add("${id}: development-request is required.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
$request = $case.'development-request'
|
|
||||||
if ($validInputKinds -notcontains [string]$request.kind) {
|
|
||||||
$problems.Add("${id}: development-request.kind must be one of $($validInputKinds -join ', ').") | Out-Null
|
|
||||||
}
|
|
||||||
$description = if ($request.PSObject.Properties.Name -contains 'description') {
|
|
||||||
[string]$request.description
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
$planText = if ($request.PSObject.Properties.Name -contains 'plan') {
|
|
||||||
[string]$request.plan
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace($description) -and [string]::IsNullOrWhiteSpace($planText)) {
|
|
||||||
$problems.Add("${id}: development-request requires description or plan.") | Out-Null
|
|
||||||
}
|
|
||||||
if ($request.PSObject.Properties.Name -notcontains 'acceptance-criteria' -or
|
|
||||||
-not @($request.'acceptance-criteria').Count) {
|
|
||||||
$problems.Add("${id}: development-request.acceptance-criteria must not be empty.") | Out-Null
|
|
||||||
}
|
|
||||||
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
|
|
||||||
[string]$case.expectedKind
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ($validOutputKinds -notcontains $expectedKind) {
|
|
||||||
$problems.Add("${id}: expectedKind must be one of $($validOutputKinds -join ', ').") | Out-Null
|
|
||||||
}
|
|
||||||
if ([string]$request.kind -ne 'auto' -and [string]$request.kind -ne $expectedKind) {
|
|
||||||
$problems.Add("${id}: explicit request kind '$($request.kind)' must equal expectedKind '$expectedKind'.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
|
||||||
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
|
|
||||||
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
|
|
||||||
}
|
|
||||||
foreach ($reference in $expectedKnowledge) {
|
|
||||||
$reference = [string]$reference
|
|
||||||
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
|
|
||||||
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
|
|
||||||
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
|
|
||||||
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
foreach ($check in @($case.requiredChecks)) {
|
|
||||||
if ($validChecks -notcontains [string]$check) {
|
|
||||||
$problems.Add("${id}: unsupported required check '$check'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$validCapabilityStatuses = @('planned', 'fixture', 'validated')
|
|
||||||
$capabilityIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
|
||||||
foreach ($capability in @($capabilityManifest.capabilities)) {
|
|
||||||
$id = [string]$capability.id
|
|
||||||
if (-not $capabilityIds.Add($id)) {
|
|
||||||
$problems.Add("Duplicate capability id: $id") | Out-Null
|
|
||||||
}
|
|
||||||
if ($validCapabilityStatuses -notcontains [string]$capability.status) {
|
|
||||||
$problems.Add("${id}: invalid capability status '$($capability.status)'.") | Out-Null
|
|
||||||
}
|
|
||||||
$fixtureIds = @($capability.fixtureIds)
|
|
||||||
if ($capability.status -in @('fixture', 'validated') -and -not $fixtureIds.Count) {
|
|
||||||
$problems.Add("${id}: status '$($capability.status)' requires at least one fixture.") | Out-Null
|
|
||||||
}
|
|
||||||
foreach ($fixtureId in $fixtureIds) {
|
|
||||||
if (-not $caseById.ContainsKey([string]$fixtureId)) {
|
|
||||||
$problems.Add("${id}: unknown fixture id '$fixtureId'.") | Out-Null
|
|
||||||
} elseif (@($caseById[[string]$fixtureId].capabilities) -notcontains $id) {
|
|
||||||
$problems.Add("${id}: fixture '$fixtureId' does not link back to the capability.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($case in @($manifest.cases)) {
|
|
||||||
foreach ($capabilityId in @($case.capabilities)) {
|
|
||||||
if (-not $capabilityIds.Contains([string]$capabilityId)) {
|
|
||||||
$problems.Add("$($case.id): unknown capability '$capabilityId'.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
$capability = @(
|
|
||||||
$capabilityManifest.capabilities |
|
|
||||||
Where-Object id -eq ([string]$capabilityId)
|
|
||||||
)[0]
|
|
||||||
if (@($capability.fixtureIds) -notcontains [string]$case.id) {
|
|
||||||
$problems.Add("$($case.id): capability '$capabilityId' does not link back to the fixture.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$fixtureBackedCount = @(
|
|
||||||
$capabilityManifest.capabilities |
|
|
||||||
Where-Object status -in @('fixture', 'validated')
|
|
||||||
).Count
|
|
||||||
$capabilityCount = @($capabilityManifest.capabilities).Count
|
|
||||||
$fixtureCoverage = if ($capabilityCount) { $fixtureBackedCount / $capabilityCount } else { 0.0 }
|
|
||||||
if ($fixtureCoverage -lt $minimumFixtureCoverage) {
|
|
||||||
$problems.Add("Fixture-backed capability coverage $fixtureCoverage is below minimumFixtureCoverage $minimumFixtureCoverage.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($problems.Count) {
|
|
||||||
Write-Host "Development fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
|
||||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($PrepareDirectory) {
|
|
||||||
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-evaluation'
|
|
||||||
if (Test-Path -LiteralPath $PrepareDirectory) {
|
|
||||||
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
|
|
||||||
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
|
|
||||||
throw "PrepareDirectory is not empty and is not a BCQuality development evaluation directory: $PrepareDirectory"
|
|
||||||
}
|
|
||||||
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
|
|
||||||
Get-ChildItem -LiteralPath $PrepareDirectory -File |
|
|
||||||
Where-Object {
|
|
||||||
$_.Name -eq 'knowledge-index.json' -or
|
|
||||||
$_.Name -like 'request-*.json' -or
|
|
||||||
$_.Name -like 'result-*.json'
|
|
||||||
} |
|
|
||||||
Remove-Item -Force
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
|
|
||||||
}
|
|
||||||
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development evaluation directory' -Encoding UTF8
|
|
||||||
|
|
||||||
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
|
|
||||||
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
|
||||||
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
|
||||||
|
|
||||||
foreach ($case in @($manifest.cases)) {
|
|
||||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
|
||||||
$request = [ordered]@{
|
|
||||||
protocol = 'Run the supplied AL development skill in a fresh writable fixture repository. Persist the implementation, run real checks, and return only implementationReport using the supplied schema.'
|
|
||||||
caseId = $modelId
|
|
||||||
skill = $skillPath
|
|
||||||
skillInstructions = $skillInstructions
|
|
||||||
knowledgeIndex = 'knowledge-index.json'
|
|
||||||
'task-context' = [ordered]@{
|
|
||||||
goal = if ($case.'development-request'.PSObject.Properties.Name -contains 'description') {
|
|
||||||
[string]$case.'development-request'.description
|
|
||||||
} else {
|
|
||||||
[string]$case.'development-request'.plan
|
|
||||||
}
|
|
||||||
'inputs-available' = @('development-request', 'repository')
|
|
||||||
technologies = @($case.context.technologies)
|
|
||||||
countries = @($case.context.countries)
|
|
||||||
'application-area' = @($case.context.'application-area')
|
|
||||||
}
|
|
||||||
'development-request' = [ordered]@{
|
|
||||||
kind = [string]$case.'development-request'.kind
|
|
||||||
description = if ($case.'development-request'.PSObject.Properties.Name -contains 'description') {
|
|
||||||
[string]$case.'development-request'.description
|
|
||||||
} else {
|
|
||||||
$null
|
|
||||||
}
|
|
||||||
plan = if ($case.'development-request'.PSObject.Properties.Name -contains 'plan') {
|
|
||||||
[string]$case.'development-request'.plan
|
|
||||||
} else {
|
|
||||||
$null
|
|
||||||
}
|
|
||||||
'acceptance-criteria' = @($case.'development-request'.'acceptance-criteria')
|
|
||||||
}
|
|
||||||
resultSchema = [ordered]@{
|
|
||||||
caseId = $modelId
|
|
||||||
workspaceRoot = 'absolute path to the retained fixture repository'
|
|
||||||
implementationReport = [ordered]@{
|
|
||||||
skill = [ordered]@{ id = 'al-development'; version = 1 }
|
|
||||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
|
||||||
'outcome-reason' = 'required for partial or failed'
|
|
||||||
summary = [ordered]@{
|
|
||||||
request = 'implemented change'
|
|
||||||
'files-created' = 0
|
|
||||||
'files-modified' = 0
|
|
||||||
'files-deleted' = 0
|
|
||||||
}
|
|
||||||
plan = [ordered]@{
|
|
||||||
kind = 'feature | bug | refactor | upgrade | maintenance'
|
|
||||||
assumptions = @()
|
|
||||||
decisions = @()
|
|
||||||
objects = @()
|
|
||||||
}
|
|
||||||
knowledge = @([ordered]@{ path = 'repo-relative knowledge article path'; sha = 'optional commit sha'; 'used-for' = 'decision' })
|
|
||||||
changes = @([ordered]@{ path = 'repo-relative changed file'; action = 'created | modified | deleted'; purpose = 'reason' })
|
|
||||||
validation = @([ordered]@{ id = 'compile | tests | review'; command = 'command or quality-skill path'; status = 'passed | failed | not-run'; details = 'non-empty evidence' })
|
|
||||||
review = [ordered]@{
|
|
||||||
skill = [ordered]@{ id = 'al-code-review'; version = 1 }
|
|
||||||
outcome = 'completed | partial | failed'
|
|
||||||
summary = [ordered]@{
|
|
||||||
counts = [ordered]@{ blocker = 0; major = 0; minor = 0; info = 0 }
|
|
||||||
coverage = [ordered]@{ 'worklist-size' = 0; 'items-evaluated' = 0 }
|
|
||||||
}
|
|
||||||
findings = @()
|
|
||||||
suppressed = @()
|
|
||||||
}
|
|
||||||
'review-rounds' = @([ordered]@{
|
|
||||||
round = 1
|
|
||||||
outcome = 'clean | fixing | stalled | limit-reached'
|
|
||||||
'gating-finding-ids' = @()
|
|
||||||
})
|
|
||||||
suppressed = @()
|
|
||||||
remaining = @()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$request | ConvertTo-Json -Depth 12 |
|
|
||||||
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($ResultsDirectory) {
|
|
||||||
$failures = [System.Collections.Generic.List[string]]::new()
|
|
||||||
foreach ($case in @($manifest.cases)) {
|
|
||||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
|
||||||
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
|
|
||||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
|
||||||
$failures.Add("$($case.id): missing result file.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
|
||||||
} catch {
|
|
||||||
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
|
|
||||||
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if ($result.PSObject.Properties.Name -notcontains 'implementationReport') {
|
|
||||||
$failures.Add("$($case.id): implementationReport is missing.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
$report = $result.implementationReport
|
|
||||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'plan', 'knowledge', 'changes', 'validation', 'review', 'review-rounds', 'suppressed', 'remaining')) {
|
|
||||||
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
|
|
||||||
$failures.Add("$($case.id): implementation report is missing '$requiredField'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$reportedSkillId = if (
|
|
||||||
$report.PSObject.Properties.Name -contains 'skill' -and
|
|
||||||
$report.skill.PSObject.Properties.Name -contains 'id'
|
|
||||||
) {
|
|
||||||
[string]$report.skill.id
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ($reportedSkillId -ne 'al-development') {
|
|
||||||
$failures.Add("$($case.id): implementation report skill is '$reportedSkillId'.") | Out-Null
|
|
||||||
}
|
|
||||||
$summaryRequest = if (
|
|
||||||
$report.PSObject.Properties.Name -contains 'summary' -and
|
|
||||||
$report.summary.PSObject.Properties.Name -contains 'request'
|
|
||||||
) {
|
|
||||||
[string]$report.summary.request
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace($summaryRequest)) {
|
|
||||||
$failures.Add("$($case.id): summary.request is missing or empty.") | Out-Null
|
|
||||||
}
|
|
||||||
$reportedKind = if (
|
|
||||||
$report.PSObject.Properties.Name -contains 'plan' -and
|
|
||||||
$report.plan.PSObject.Properties.Name -contains 'kind'
|
|
||||||
) {
|
|
||||||
[string]$report.plan.kind
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ($reportedKind -ne [string]$case.expectedKind) {
|
|
||||||
$failures.Add("$($case.id): plan.kind '$reportedKind' does not match expected mode '$($case.expectedKind)'.") | Out-Null
|
|
||||||
}
|
|
||||||
$outcome = if ($report.PSObject.Properties.Name -contains 'outcome') { [string]$report.outcome } else { '' }
|
|
||||||
if ($outcome -ne 'completed') {
|
|
||||||
$failures.Add("$($case.id): implementation outcome is '$outcome'.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
[object[]]$knowledgeEntries = @()
|
|
||||||
if ($report.PSObject.Properties.Name -contains 'knowledge') {
|
|
||||||
$knowledgeEntries = @($report.knowledge)
|
|
||||||
}
|
|
||||||
if (-not $knowledgeEntries.Count) {
|
|
||||||
$failures.Add("$($case.id): implementation report contains no knowledge entries.") | Out-Null
|
|
||||||
}
|
|
||||||
$usedKnowledge = @(
|
|
||||||
$knowledgeEntries |
|
|
||||||
Where-Object { $_.PSObject.Properties.Name -contains 'path' } |
|
|
||||||
ForEach-Object { [string]$_.path }
|
|
||||||
)
|
|
||||||
if (@($usedKnowledge | Sort-Object -Unique).Count -ne $usedKnowledge.Count) {
|
|
||||||
$failures.Add("$($case.id): knowledge contains duplicate paths.") | Out-Null
|
|
||||||
}
|
|
||||||
foreach ($entry in $knowledgeEntries) {
|
|
||||||
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
|
|
||||||
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
|
|
||||||
if ([string]::IsNullOrWhiteSpace($path) -or
|
|
||||||
[System.IO.Path]::IsPathRooted($path) -or
|
|
||||||
@($path -split '/|\\') -contains '..' -or
|
|
||||||
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
|
||||||
$failures.Add("$($case.id): knowledge entry has a missing or invalid path '$path'.") | Out-Null
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace($usedFor)) {
|
|
||||||
$failures.Add("$($case.id): knowledge entry '$path' has no used-for explanation.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$requiredKnowledge = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
|
|
||||||
$matched = @($requiredKnowledge | Where-Object { $usedKnowledge -contains $_ }).Count
|
|
||||||
$recall = if ($requiredKnowledge.Count) { $matched / $requiredKnowledge.Count } else { 1.0 }
|
|
||||||
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
|
|
||||||
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
|
|
||||||
}
|
|
||||||
$acceptedKnowledge = @(
|
|
||||||
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
|
|
||||||
ForEach-Object { [string]$_ } |
|
|
||||||
Sort-Object -Unique
|
|
||||||
)
|
|
||||||
$acceptedUsed = @($usedKnowledge | Where-Object { $acceptedKnowledge -contains $_ }).Count
|
|
||||||
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
|
|
||||||
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
|
|
||||||
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
[object[]]$validationEntries = @()
|
|
||||||
if ($report.PSObject.Properties.Name -contains 'validation') {
|
|
||||||
$validationEntries = @($report.validation)
|
|
||||||
}
|
|
||||||
foreach ($requiredCheck in @($case.requiredChecks)) {
|
|
||||||
$check = @($validationEntries | Where-Object {
|
|
||||||
$_.PSObject.Properties.Name -contains 'id' -and [string]$_.id -eq $requiredCheck
|
|
||||||
})
|
|
||||||
$checkStatus = if ($check.Count -eq 1 -and $check[0].PSObject.Properties.Name -contains 'status') {
|
|
||||||
[string]$check[0].status
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ($check.Count -ne 1 -or $checkStatus -ne 'passed') {
|
|
||||||
$failures.Add("$($case.id): required check '$requiredCheck' did not pass exactly once.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
$command = if ($check[0].PSObject.Properties.Name -contains 'command') { [string]$check[0].command } else { '' }
|
|
||||||
$details = if ($check[0].PSObject.Properties.Name -contains 'details') { [string]$check[0].details } else { '' }
|
|
||||||
if ([string]::IsNullOrWhiteSpace($command) -or [string]::IsNullOrWhiteSpace($details)) {
|
|
||||||
$failures.Add("$($case.id): required check '$requiredCheck' lacks command or evidence details.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[object[]]$changes = @()
|
|
||||||
if ($report.PSObject.Properties.Name -contains 'changes') {
|
|
||||||
$changes = @($report.changes)
|
|
||||||
}
|
|
||||||
if (-not $changes.Count) {
|
|
||||||
$failures.Add("$($case.id): implementation report contains no changed files.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') { [string]$result.workspaceRoot } else { '' }
|
|
||||||
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
|
|
||||||
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
|
|
||||||
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
|
|
||||||
} else {
|
|
||||||
& git -C $workspaceRoot rev-parse --is-inside-work-tree 2>$null | Out-Null
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
$actualChanges = @(
|
|
||||||
@(& git -C $workspaceRoot diff --name-only HEAD) +
|
|
||||||
@(& git -C $workspaceRoot ls-files --others --exclude-standard) |
|
|
||||||
Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } |
|
|
||||||
ForEach-Object { ([string]$_).Replace('\', '/') } |
|
|
||||||
Sort-Object -Unique
|
|
||||||
)
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
$failures.Add("$($case.id): unable to read the workspace diff.") | Out-Null
|
|
||||||
} else {
|
|
||||||
$reportedChanges = @(
|
|
||||||
$changes |
|
|
||||||
Where-Object { $_.PSObject.Properties.Name -contains 'path' } |
|
|
||||||
ForEach-Object {
|
|
||||||
$path = ([string]$_.path).Replace('\', '/')
|
|
||||||
if ([System.IO.Path]::IsPathRooted($path) -or @($path -split '/') -contains '..') {
|
|
||||||
$failures.Add("$($case.id): changed path is not repository-relative: $path") | Out-Null
|
|
||||||
}
|
|
||||||
$path
|
|
||||||
} |
|
|
||||||
Sort-Object -Unique
|
|
||||||
)
|
|
||||||
foreach ($path in @($reportedChanges | Where-Object { $actualChanges -notcontains $_ })) {
|
|
||||||
$failures.Add("$($case.id): reported change is absent from the worktree diff: $path") | Out-Null
|
|
||||||
}
|
|
||||||
foreach ($path in @($actualChanges | Where-Object { $reportedChanges -notcontains $_ })) {
|
|
||||||
$failures.Add("$($case.id): worktree change is absent from the implementation report: $path") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($report.PSObject.Properties.Name -notcontains 'review') {
|
|
||||||
$failures.Add("$($case.id): complete post-implementation review is missing.") | Out-Null
|
|
||||||
} else {
|
|
||||||
$review = $report.review
|
|
||||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'findings', 'suppressed')) {
|
|
||||||
if ($review.PSObject.Properties.Name -notcontains $requiredField) {
|
|
||||||
$failures.Add("$($case.id): review is missing '$requiredField'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
[object[]]$reviewFindings = @()
|
|
||||||
if ($review.PSObject.Properties.Name -contains 'findings') {
|
|
||||||
$reviewFindings = @($review.findings)
|
|
||||||
}
|
|
||||||
$reviewOutcome = if ($review.PSObject.Properties.Name -contains 'outcome') { [string]$review.outcome } else { '' }
|
|
||||||
if ($reviewOutcome -ne 'completed') {
|
|
||||||
$failures.Add("$($case.id): post-implementation review outcome is '$reviewOutcome'.") | Out-Null
|
|
||||||
}
|
|
||||||
$gatingFindings = @(
|
|
||||||
$reviewFindings |
|
|
||||||
Where-Object {
|
|
||||||
$_.PSObject.Properties.Name -contains 'severity' -and
|
|
||||||
[string]$_.severity -in @('blocker', 'major')
|
|
||||||
}
|
|
||||||
)
|
|
||||||
if ($gatingFindings.Count) {
|
|
||||||
$failures.Add("$($case.id): post-implementation review has $($gatingFindings.Count) gating finding(s).") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
[object[]]$reviewRounds = @()
|
|
||||||
if ($report.PSObject.Properties.Name -contains 'review-rounds') {
|
|
||||||
$reviewRounds = @($report.'review-rounds')
|
|
||||||
}
|
|
||||||
if (-not $reviewRounds.Count -or $reviewRounds.Count -gt $maximumReviewRounds) {
|
|
||||||
$failures.Add("$($case.id): review-rounds must contain 1..$maximumReviewRounds entries.") | Out-Null
|
|
||||||
} else {
|
|
||||||
for ($index = 0; $index -lt $reviewRounds.Count; $index++) {
|
|
||||||
$round = $reviewRounds[$index]
|
|
||||||
$roundNumber = if ($round.PSObject.Properties.Name -contains 'round') { [int]$round.round } else { 0 }
|
|
||||||
$roundOutcome = if ($round.PSObject.Properties.Name -contains 'outcome') { [string]$round.outcome } else { '' }
|
|
||||||
if ($roundNumber -ne ($index + 1)) {
|
|
||||||
$failures.Add("$($case.id): review round numbering is not contiguous.") | Out-Null
|
|
||||||
}
|
|
||||||
if ($roundOutcome -notin @('clean', 'fixing', 'stalled', 'limit-reached')) {
|
|
||||||
$failures.Add("$($case.id): review round $roundNumber has invalid outcome '$roundOutcome'.") | Out-Null
|
|
||||||
}
|
|
||||||
[object[]]$roundGatingIds = @()
|
|
||||||
if ($round.PSObject.Properties.Name -contains 'gating-finding-ids') {
|
|
||||||
$roundGatingIds = @($round.'gating-finding-ids')
|
|
||||||
}
|
|
||||||
if ($roundOutcome -eq 'clean' -and $roundGatingIds.Count) {
|
|
||||||
$failures.Add("$($case.id): clean review round $roundNumber must have no gating IDs.") | Out-Null
|
|
||||||
}
|
|
||||||
if ($roundOutcome -in @('fixing', 'stalled', 'limit-reached') -and -not $roundGatingIds.Count) {
|
|
||||||
$failures.Add("$($case.id): review round $roundNumber outcome '$roundOutcome' requires gating IDs.") | Out-Null
|
|
||||||
}
|
|
||||||
if (@($roundGatingIds | Sort-Object -Unique).Count -ne $roundGatingIds.Count) {
|
|
||||||
$failures.Add("$($case.id): review round $roundNumber has duplicate gating IDs.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ([string]$reviewRounds[-1].outcome -ne 'clean') {
|
|
||||||
$failures.Add("$($case.id): completed implementation must end with a clean review round.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
[object[]]$remaining = @()
|
|
||||||
if ($report.PSObject.Properties.Name -contains 'remaining') {
|
|
||||||
$remaining = @($report.remaining)
|
|
||||||
}
|
|
||||||
if ($remaining.Count) {
|
|
||||||
$failures.Add("$($case.id): completed report still lists remaining work.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($failures.Count) {
|
|
||||||
Write-Host "Development fixture scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
|
|
||||||
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
Write-Host "Development fixture scoring PASSED: $(@($manifest.cases).Count) case(s)."
|
|
||||||
} else {
|
|
||||||
Write-Host "Development fixture validation PASSED: $(@($manifest.cases).Count) cases; $fixtureBackedCount of $($capabilityIds.Count) capabilities have fixtures (minimum $minimumFixtureCoverage)."
|
|
||||||
}
|
|
||||||
421
tools/Test-DevelopmentGuidanceEvaluator.ps1
Normal file
421
tools/Test-DevelopmentGuidanceEvaluator.ps1
Normal file
|
|
@ -0,0 +1,421 @@
|
||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Deterministic, offline regressions for the guidance evaluator (no model/AL run).
|
||||||
|
.DESCRIPTION
|
||||||
|
Creates only a uniquely named .guidance-evaluator-regression-* directory below
|
||||||
|
the current checkout, with standalone Git repositories and sibling runner
|
||||||
|
artifacts. Removes that exact directory in finally; never uses the OS temp
|
||||||
|
directory or cleans a caller-provided repository.
|
||||||
|
#>
|
||||||
|
[CmdletBinding()]
|
||||||
|
param()
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = 'Stop'
|
||||||
|
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
|
||||||
|
$evaluator = Join-Path $PSScriptRoot 'Test-DevelopmentGuidanceFixtures.ps1'
|
||||||
|
$sourceRoot = (Get-Item -LiteralPath (Join-Path $PSScriptRoot '..')).FullName
|
||||||
|
$scratch = Join-Path $sourceRoot ".guidance-evaluator-regression-$([guid]::NewGuid().ToString('N'))"
|
||||||
|
$root = Join-Path $scratch 'knowledge-checkout'
|
||||||
|
$article = 'microsoft/knowledge/performance/pair-findset-with-next-loop.md'
|
||||||
|
$otherArticle = 'microsoft/knowledge/performance/findset-true-applies-updlock-on-read.md'
|
||||||
|
$sample = 'microsoft/knowledge/performance/pair-findset-with-next-loop.good.al'
|
||||||
|
$tests = [Collections.Generic.List[string]]::new()
|
||||||
|
$script:scenarioNumber = 0
|
||||||
|
$script:reportScenario = $null
|
||||||
|
|
||||||
|
function Set-TestJson($Path, $Value) {
|
||||||
|
[IO.File]::WriteAllText($Path, ($Value | ConvertTo-Json -Depth 64))
|
||||||
|
}
|
||||||
|
|
||||||
|
function Invoke-TestGit([string] $Directory, [string[]] $Arguments) {
|
||||||
|
$output = @(& git --no-optional-locks -C $Directory @Arguments 2>&1)
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw 'Regression Git setup failed.' }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Initialize-TestRepository([string] $Directory) {
|
||||||
|
[IO.Directory]::CreateDirectory($Directory) | Out-Null
|
||||||
|
Invoke-TestGit $Directory @('init', '--quiet')
|
||||||
|
Invoke-TestGit $Directory @('config', 'user.email', 'guidance-fixture@example.invalid')
|
||||||
|
Invoke-TestGit $Directory @('config', 'user.name', 'Guidance fixture')
|
||||||
|
Invoke-TestGit $Directory @('config', 'commit.gpgSign', 'false')
|
||||||
|
Invoke-TestGit $Directory @('config', 'core.autocrlf', 'false')
|
||||||
|
[IO.File]::WriteAllText((Join-Path $Directory 'app.json'), '{"name":"Synthetic AL fixture","application":"28.0.0.0"}')
|
||||||
|
[IO.File]::WriteAllText((Join-Path $Directory 'tracked.al'), 'codeunit 50100 Example {}')
|
||||||
|
[IO.File]::WriteAllText((Join-Path $Directory '.gitignore'), "ignored.txt`nignored-directory/`n")
|
||||||
|
Invoke-TestGit $Directory @('add', '.')
|
||||||
|
Invoke-TestGit $Directory @('commit', '--quiet', '-m', 'Synthetic fixture baseline')
|
||||||
|
}
|
||||||
|
|
||||||
|
function Invoke-EvaluatorTest([string] $Name, [string[]] $Arguments, [bool] $ShouldPass, [string] $Diagnostic = '') {
|
||||||
|
$output = @(& pwsh -NoProfile -File $evaluator @Arguments 2>&1) -join "`n"
|
||||||
|
$code = $LASTEXITCODE
|
||||||
|
if (($code -eq 0) -ne $ShouldPass -or $output -notmatch $(if ($ShouldPass) { 'PASSED|captured' } else { 'FAILED' })) {
|
||||||
|
throw "Regression '$Name' unexpected exit $code. $output"
|
||||||
|
}
|
||||||
|
if ($Diagnostic -and $output -notmatch [regex]::Escape($Diagnostic)) {
|
||||||
|
throw "Regression '$Name' missing diagnostic '$Diagnostic'. $output"
|
||||||
|
}
|
||||||
|
if ($output -match 'MODEL_SECRET_SENTINEL') { throw "Regression '$Name' leaked model content." }
|
||||||
|
$tests.Add($Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
function New-TestScenario([string] $Outcome = 'completed', [switch] $Unknown, [switch] $SecondCase) {
|
||||||
|
$script:scenarioNumber++
|
||||||
|
$directory = Join-Path $scratch "scenario-$script:scenarioNumber"
|
||||||
|
[IO.Directory]::CreateDirectory($directory) | Out-Null
|
||||||
|
$workspace = Join-Path $directory 'target'
|
||||||
|
Initialize-TestRepository $workspace
|
||||||
|
[IO.File]::WriteAllText((Join-Path $workspace 'untracked.txt'), 'existing untracked content')
|
||||||
|
[IO.File]::WriteAllText((Join-Path $workspace 'ignored.txt'), 'existing ignored content')
|
||||||
|
[IO.Directory]::CreateDirectory((Join-Path $workspace 'ignored-directory')) | Out-Null
|
||||||
|
[IO.File]::WriteAllText((Join-Path $workspace 'ignored-directory\child.txt'), 'ignored child')
|
||||||
|
$results = Join-Path $directory 'results'
|
||||||
|
[IO.Directory]::CreateDirectory($results) | Out-Null
|
||||||
|
$hasKnowledge = $Outcome -in @('completed', 'partial')
|
||||||
|
$case = [ordered]@{
|
||||||
|
id = 'synthetic-case'
|
||||||
|
expectedKind = 'bug'
|
||||||
|
expectedOutcome = $Outcome
|
||||||
|
expectedUnknown = @($(if ($Unknown) { 'bc-version' }))
|
||||||
|
requiresUnresolved = ($Outcome -eq 'partial' -or $Unknown.IsPresent)
|
||||||
|
requiresMaterialUnresolved = ($Outcome -eq 'partial')
|
||||||
|
'development-plan' = [ordered]@{ kind = 'bug'; request = 'Iterate the supplied filtered record set.' }
|
||||||
|
context = [ordered]@{
|
||||||
|
'bc-version' = $(if ($Unknown) { 'unknown' } else { '28' })
|
||||||
|
technologies = @('al')
|
||||||
|
countries = @('w1')
|
||||||
|
'application-area' = @('all')
|
||||||
|
unknown = @($(if ($Unknown) { 'bc-version' }))
|
||||||
|
}
|
||||||
|
requiredKnowledge = @($(if ($hasKnowledge) { $article }))
|
||||||
|
optionalKnowledge = @()
|
||||||
|
}
|
||||||
|
$manifest = [ordered]@{
|
||||||
|
version = 1
|
||||||
|
skill = 'microsoft/skills/development/al-development-plan.md'
|
||||||
|
minimumKnowledgeRecall = 1.0
|
||||||
|
minimumKnowledgePrecision = 1.0
|
||||||
|
cases = @($case)
|
||||||
|
}
|
||||||
|
$result = [ordered]@{
|
||||||
|
caseId = Get-GuidanceCaseId $case.id
|
||||||
|
guidanceReport = [ordered]@{
|
||||||
|
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
||||||
|
outcome = $Outcome
|
||||||
|
summary = [ordered]@{ request = 'Iterate all selected records.'; kind = 'bug'; candidates = [int]$hasKnowledge; selected = [int]$hasKnowledge }
|
||||||
|
context = $case.context
|
||||||
|
knowledge = @($(if ($hasKnowledge) {
|
||||||
|
[ordered]@{ path = $article; 'used-for' = 'Choose the multi-record reader.'; constraints = @('Use FindSet when iterating with Next.'); 'sample-paths' = @($sample) }
|
||||||
|
}))
|
||||||
|
'validation-considerations' = @([ordered]@{ id = 'all-selected'; reason = 'Preserve selection.'; evidence = 'Test all selected records and an excluded record.' })
|
||||||
|
suppressed = @()
|
||||||
|
unresolved = @($(if ($Outcome -eq 'partial') {
|
||||||
|
if ($Unknown) { 'bc-version is unknown and materially affects the candidate; clarify before completing guidance.' }
|
||||||
|
else { 'The caller cardinality decision remains materially unresolved.' }
|
||||||
|
} elseif ($Unknown) { 'bc-version is unknown but immaterial: selected loop guidance applies to all versions.' }))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($Outcome -in @('partial', 'failed')) { $result.guidanceReport.'outcome-reason' = 'Fixture intentionally leaves evaluation incomplete.' }
|
||||||
|
$manifestPath = Join-Path $directory 'manifest.json'
|
||||||
|
$mapPath = Join-Path $directory 'workspace-map.json'
|
||||||
|
$map = [ordered]@{ 'synthetic-case' = $workspace }
|
||||||
|
if ($SecondCase) {
|
||||||
|
$second = $case | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||||
|
$second.id = 'second-case'
|
||||||
|
$manifest.cases += $second
|
||||||
|
$secondWorkspace = Join-Path $directory 'second-target'
|
||||||
|
Initialize-TestRepository $secondWorkspace
|
||||||
|
$map[$second.id] = $secondWorkspace
|
||||||
|
$secondResult = $result | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||||
|
$secondResult.caseId = Get-GuidanceCaseId $second.id
|
||||||
|
Set-TestJson (Join-Path $results "result-$($secondResult.caseId).json") $secondResult
|
||||||
|
}
|
||||||
|
Set-TestJson $manifestPath $manifest
|
||||||
|
Set-TestJson $mapPath $map
|
||||||
|
$resultPath = Join-Path $results "result-$($result.caseId).json"
|
||||||
|
Set-TestJson $resultPath $result
|
||||||
|
$baselinePath = Join-Path $directory 'baseline.json'
|
||||||
|
$captureArgs = @('-Root', $root, '-ManifestPath', $manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $mapPath, '-BaselinePath', $baselinePath)
|
||||||
|
$output = @(& pwsh -NoProfile -File $evaluator @captureArgs 2>&1) -join "`n"
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw "Baseline setup failed. $output" }
|
||||||
|
return [ordered]@{
|
||||||
|
directory = $directory; workspace = $workspace; resultPath = $resultPath; result = $result
|
||||||
|
results = $results; manifest = $manifest; manifestPath = $manifestPath; mapPath = $mapPath
|
||||||
|
baselinePath = $baselinePath; captureArgs = $captureArgs
|
||||||
|
scoreArgs = @('-Root', $root, '-ManifestPath', $manifestPath, '-ResultsDirectory', $results, '-BaselinePath', $baselinePath, '-BaselineSha256', (Get-GuidanceHash $baselinePath))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Test-ReportMutation([string] $Name, [scriptblock] $Change, [string] $Diagnostic) {
|
||||||
|
if ($null -eq $script:reportScenario) { $script:reportScenario = New-TestScenario }
|
||||||
|
$scenario = $script:reportScenario
|
||||||
|
$result = $scenario.result | ConvertTo-Json -Depth 64 | ConvertFrom-Json -AsHashtable
|
||||||
|
& $Change $result
|
||||||
|
Set-TestJson $scenario.resultPath $result
|
||||||
|
Invoke-EvaluatorTest $Name $scenario.scoreArgs $false $Diagnostic
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
[IO.Directory]::CreateDirectory($root) | Out-Null
|
||||||
|
foreach ($reference in @($article, $otherArticle, $sample, $otherArticle.Replace('.md', '.good.al'),
|
||||||
|
'microsoft/skills/development/al-development-plan.md', 'tools/Build-KnowledgeIndex.ps1')) {
|
||||||
|
$destination = Join-Path $root $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||||
|
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
|
||||||
|
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
|
||||||
|
}
|
||||||
|
Initialize-TestRepository $root
|
||||||
|
|
||||||
|
$publicManifestPath = Join-Path $sourceRoot 'evaluation\development-guidance-fixtures.json'
|
||||||
|
$publicManifest = Read-GuidanceJson $publicManifestPath
|
||||||
|
$publicRoot = Join-Path $scratch 'public-fixture-checkout'
|
||||||
|
$publicReferences = @($publicManifest.skill, 'tools/Build-KnowledgeIndex.ps1', 'evaluation/development-guidance-fixtures.json') +
|
||||||
|
@($publicManifest.cases | ForEach-Object { $_.requiredKnowledge; $_.optionalKnowledge })
|
||||||
|
foreach ($reference in @($publicReferences | Sort-Object -Unique)) {
|
||||||
|
$destination = Join-Path $publicRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)
|
||||||
|
[IO.Directory]::CreateDirectory((Split-Path $destination -Parent)) | Out-Null
|
||||||
|
[IO.File]::Copy((Join-Path $sourceRoot $reference.Replace('/', [IO.Path]::DirectorySeparatorChar)), $destination)
|
||||||
|
}
|
||||||
|
$publicPrepared = Join-Path $scratch 'public-prepared'
|
||||||
|
Invoke-EvaluatorTest 'public five-case manifest validation' @('-Root', $publicRoot) $true
|
||||||
|
Invoke-EvaluatorTest 'public five-case manifest preparation' @('-Root', $publicRoot, '-PrepareDirectory', $publicPrepared) $true
|
||||||
|
$initialCase = $publicManifest.cases | Where-Object id -eq 'synthetic-normal-initial-plan'
|
||||||
|
$initialRequest = Read-GuidanceJson (Join-Path $publicPrepared "request-$(Get-GuidanceCaseId $initialCase.id).json")
|
||||||
|
if ($initialRequest.'development-plan' -cne $initialCase.'development-plan') { throw 'Preparation truncated the serialized initial plan.' }
|
||||||
|
$document = $initialRequest.'development-plan' | ConvertFrom-Json -AsHashtable
|
||||||
|
if ($document.metadata.kind -ne 'bug' -or
|
||||||
|
@('Root cause and design', 'Proposed fix', 'Affected files', 'Test strategy', 'Acceptance criteria' |
|
||||||
|
Where-Object { $document.body -notmatch [regex]::Escape($_) }).Count) {
|
||||||
|
throw 'Synthetic consumer boundary lost metadata or markdown plan sections.'
|
||||||
|
}
|
||||||
|
$tests.Add('serialized synthetic initial-plan boundary preserves full metadata and markdown body')
|
||||||
|
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
Invoke-EvaluatorTest 'unchanged target with ignored and untracked files passes' $scenario.scoreArgs $true
|
||||||
|
Invoke-EvaluatorTest 'scoring itself leaves index and content unchanged' $scenario.scoreArgs $true
|
||||||
|
Invoke-EvaluatorTest 'existing baseline cannot silently recapture' $scenario.captureArgs $false 'Baseline already exists'
|
||||||
|
Invoke-EvaluatorTest 'scoring without baseline fails' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results) $false 'require BaselinePath'
|
||||||
|
Invoke-EvaluatorTest 'scoring requires independently retained baseline digest' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath) $false 'runner-retained pre-run BaselineSha256'
|
||||||
|
Invoke-EvaluatorTest 'tampered baseline digest fails' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath, '-BaselineSha256', ('0' * 64)) $false 'digest mismatch'
|
||||||
|
|
||||||
|
$prepare = Join-Path $scenario.directory 'prepared'
|
||||||
|
Invoke-EvaluatorTest 'prepare outside roots with runner workspace binding' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', $prepare, '-WorkspaceMapPath', $scenario.mapPath) $true
|
||||||
|
$prepared = Read-GuidanceJson (Join-Path $prepare "request-$($scenario.result.caseId).json")
|
||||||
|
if ($prepared.repository -cne $scenario.workspace -or $prepared.Contains('expectedOutcome') -or $prepared.Contains('requiredKnowledge')) {
|
||||||
|
throw 'Prepared request lost runner binding or exposed answers.'
|
||||||
|
}
|
||||||
|
Invoke-EvaluatorTest 'preparation never overwrites requests' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', $prepare) $false 'new or empty'
|
||||||
|
foreach ($option in @('PrepareDirectory', 'BaselinePath', 'ResultsDirectory')) {
|
||||||
|
$inside = Join-Path $scenario.workspace 'unsafe-artifact'
|
||||||
|
$arguments = @('-Root', $root, '-ManifestPath', $scenario.manifestPath)
|
||||||
|
if ($option -eq 'PrepareDirectory') { $arguments += @('-PrepareDirectory', $inside, '-WorkspaceMapPath', $scenario.mapPath) }
|
||||||
|
elseif ($option -eq 'BaselinePath') { $arguments += @('-CaptureBaseline', '-BaselinePath', $inside, '-WorkspaceMapPath', $scenario.mapPath) }
|
||||||
|
else { $arguments += @('-BaselinePath', $scenario.baselinePath, '-BaselineSha256', (Get-GuidanceHash $scenario.baselinePath), '-ResultsDirectory', $inside) }
|
||||||
|
Invoke-EvaluatorTest "$option inside target rejected" $arguments $false 'outside target workspaces'
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($mutation in @('uncommitted', 'committed', 'empty-commit', 'staged', 'index-only', 'untracked', 'ignored', 'ignored-child', 'added', 'deleted', 'directory', 'ref', 'metadata')) {
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
switch ($mutation) {
|
||||||
|
'uncommitted' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed") }
|
||||||
|
'committed' {
|
||||||
|
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||||
|
Invoke-TestGit $scenario.workspace @('add', 'tracked.al')
|
||||||
|
Invoke-TestGit $scenario.workspace @('commit', '--quiet', '-m', 'Committed forbidden edit')
|
||||||
|
}
|
||||||
|
'staged' {
|
||||||
|
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||||
|
Invoke-TestGit $scenario.workspace @('add', 'tracked.al')
|
||||||
|
}
|
||||||
|
'empty-commit' { Invoke-TestGit $scenario.workspace @('commit', '--quiet', '--allow-empty', '-m', 'Forbidden empty commit') }
|
||||||
|
'index-only' { Invoke-TestGit $scenario.workspace @('update-index', '--assume-unchanged', 'tracked.al') }
|
||||||
|
'untracked' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'untracked.txt'), 'changed') }
|
||||||
|
'ignored' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'ignored.txt'), 'changed') }
|
||||||
|
'ignored-child' { [IO.File]::AppendAllText((Join-Path $scenario.workspace 'ignored-directory\child.txt'), 'changed') }
|
||||||
|
'added' { [IO.File]::WriteAllText((Join-Path $scenario.workspace 'new.txt'), 'new ignored/untracked payload') }
|
||||||
|
'deleted' { [IO.File]::Delete((Join-Path $scenario.workspace 'untracked.txt')) }
|
||||||
|
'directory' { [IO.Directory]::CreateDirectory((Join-Path $scenario.workspace 'new-empty-directory')) | Out-Null }
|
||||||
|
'ref' { Invoke-TestGit $scenario.workspace @('branch', 'new-reference') }
|
||||||
|
'metadata' {
|
||||||
|
$path = Join-Path $scenario.workspace 'tracked.al'
|
||||||
|
[IO.File]::SetLastWriteTimeUtc($path, [IO.File]::GetLastWriteTimeUtc($path).AddSeconds(5))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Invoke-EvaluatorTest "$mutation mutation fails" $scenario.scoreArgs $false 'identity/content changed'
|
||||||
|
}
|
||||||
|
|
||||||
|
$scenario = New-TestScenario -SecondCase
|
||||||
|
Invoke-EvaluatorTest 'two independently bound workspaces pass' $scenario.scoreArgs $true
|
||||||
|
$map = Read-GuidanceJson $scenario.mapPath
|
||||||
|
[IO.File]::AppendAllText((Join-Path $scenario.workspace 'tracked.al'), "`n// changed")
|
||||||
|
$scenario.result.workspaceRoot = $map['second-case']
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'self-reported clean workspace cannot hide changed runner target' $scenario.scoreArgs $false 'identity/content changed'
|
||||||
|
$scenario = New-TestScenario -SecondCase
|
||||||
|
$scenario.result.workspaceRoot = (Read-GuidanceJson $scenario.mapPath)['second-case']
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'result workspace swapping rejected even when both are clean' $scenario.scoreArgs $false 'runner binding'
|
||||||
|
$scenario = New-TestScenario -SecondCase
|
||||||
|
$scenario.result.caseId = Get-GuidanceCaseId 'second-case'
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'result case swapping rejected' $scenario.scoreArgs $false 'caseId mismatch'
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$scenario.manifest.cases[0].expectedKind = 'feature'
|
||||||
|
Set-TestJson $scenario.manifestPath $scenario.manifest
|
||||||
|
Invoke-EvaluatorTest 'changed manifest rejected' $scenario.scoreArgs $false 'root or manifest'
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$baseline = Read-GuidanceJson $scenario.baselinePath
|
||||||
|
$baseline.workspaces['synthetic-case'].caseId = 'case-00000000'
|
||||||
|
Set-TestJson $scenario.baselinePath $baseline
|
||||||
|
Invoke-EvaluatorTest 'baseline case binding checked even with matching digest' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $scenario.baselinePath, '-BaselineSha256', (Get-GuidanceHash $scenario.baselinePath)) $false 'case identity mismatch'
|
||||||
|
|
||||||
|
foreach ($outcome in @('no-knowledge', 'partial', 'failed', 'not-applicable')) {
|
||||||
|
$scenario = New-TestScenario $outcome
|
||||||
|
Invoke-EvaluatorTest "honest $outcome is distinguishable and passes" $scenario.scoreArgs $true
|
||||||
|
}
|
||||||
|
$scenario = New-TestScenario 'partial' -Unknown
|
||||||
|
Invoke-EvaluatorTest 'material unknown stays partial' $scenario.scoreArgs $true
|
||||||
|
$scenario.result.guidanceReport.outcome = 'completed'
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'material unknown cannot silently complete' $scenario.scoreArgs $false 'expectedOutcome'
|
||||||
|
$scenario = New-TestScenario 'completed' -Unknown
|
||||||
|
Invoke-EvaluatorTest 'nonmaterial unknown can complete with explanation' $scenario.scoreArgs $true
|
||||||
|
$scenario.result.guidanceReport.unresolved = @()
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'unknown cannot disappear from unresolved evidence' $scenario.scoreArgs $false 'unresolved gaps'
|
||||||
|
$scenario = New-TestScenario 'no-knowledge' -Unknown
|
||||||
|
Invoke-EvaluatorTest 'immaterial unknown and no-knowledge are not failures' $scenario.scoreArgs $true
|
||||||
|
foreach ($outcome in @('partial', 'failed')) {
|
||||||
|
$scenario = New-TestScenario $outcome
|
||||||
|
$scenario.result.guidanceReport.Remove('outcome-reason')
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest "$outcome requires outcome-reason" $scenario.scoreArgs $false 'outcome-reason'
|
||||||
|
}
|
||||||
|
$scenario = New-TestScenario 'no-knowledge'
|
||||||
|
$scenario.result.guidanceReport.outcome = 'completed'
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'no-knowledge is not completed-empty' $scenario.scoreArgs $false 'expectedOutcome'
|
||||||
|
$scenario = New-TestScenario 'no-knowledge'
|
||||||
|
$scenario.result.guidanceReport.knowledge = @(@{ path = $article; 'used-for' = 'filler'; constraints = @('filler'); 'sample-paths' = @() })
|
||||||
|
$scenario.result.guidanceReport.summary.candidates = 1
|
||||||
|
$scenario.result.guidanceReport.summary.selected = 1
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'no-knowledge cannot contain filler knowledge' $scenario.scoreArgs $false 'requires empty knowledge'
|
||||||
|
|
||||||
|
Test-ReportMutation 'invalid outcome enum' { param($r) $r.guidanceReport.outcome = 'success' } 'outcome enum'
|
||||||
|
Test-ReportMutation 'missing report fields' { param($r) $r.guidanceReport.Remove('knowledge') } 'missing required field'
|
||||||
|
Test-ReportMutation 'null report object' { param($r) $r.guidanceReport = $null } 'JSON object'
|
||||||
|
Test-ReportMutation 'wrong skill version type' { param($r) $r.guidanceReport.skill.version = '1' } 'identity/version'
|
||||||
|
Test-ReportMutation 'wrong skill id type' { param($r) $r.guidanceReport.skill.id = @('al-development-plan') } 'non-empty string'
|
||||||
|
Test-ReportMutation 'wrong kind type' { param($r) $r.guidanceReport.summary.kind = @('bug') } 'non-empty string'
|
||||||
|
Test-ReportMutation 'wrong summary type' { param($r) $r.guidanceReport.summary = @() } 'JSON object'
|
||||||
|
Test-ReportMutation 'fractional count' { param($r) $r.guidanceReport.summary.candidates = 1.5 } 'non-negative integer'
|
||||||
|
Test-ReportMutation 'negative count' { param($r) $r.guidanceReport.summary.selected = -1 } 'non-negative integer'
|
||||||
|
Test-ReportMutation 'inconsistent selected count' { param($r) $r.guidanceReport.summary.selected = 0 } 'Summary counts'
|
||||||
|
Test-ReportMutation 'candidate count below selected' { param($r) $r.guidanceReport.summary.candidates = 0 } 'Summary counts'
|
||||||
|
Test-ReportMutation 'wrong context list type' { param($r) $r.guidanceReport.context.technologies = 'al' } 'JSON array'
|
||||||
|
Test-ReportMutation 'missing constraints' { param($r) $r.guidanceReport.knowledge[0].constraints = @() } 'must not be empty'
|
||||||
|
Test-ReportMutation 'non-string constraints' { param($r) $r.guidanceReport.knowledge[0].constraints = @(@{ body = 'MODEL_SECRET_SENTINEL' }) } 'non-empty string'
|
||||||
|
Test-ReportMutation 'missing sample array' { param($r) $r.guidanceReport.knowledge[0].Remove('sample-paths') } 'missing required field'
|
||||||
|
Test-ReportMutation 'duplicate knowledge' {
|
||||||
|
param($r)
|
||||||
|
$r.guidanceReport.knowledge += $r.guidanceReport.knowledge[0]
|
||||||
|
$r.guidanceReport.summary.selected = 2
|
||||||
|
$r.guidanceReport.summary.candidates = 2
|
||||||
|
} 'Duplicate knowledge'
|
||||||
|
Test-ReportMutation 'bad validation entry' { param($r) $r.guidanceReport.'validation-considerations'[0].evidence = $false } 'non-empty string'
|
||||||
|
Test-ReportMutation 'invalid suppression shape' { param($r) $r.guidanceReport.suppressed = @(@{ path = $article; reason = 'configuration' }) } 'missing required field'
|
||||||
|
Test-ReportMutation 'invalid unresolved shape' { param($r) $r.guidanceReport.unresolved = @(@{ candidate = $article }) } 'non-empty string'
|
||||||
|
Test-ReportMutation 'invalid SHA provenance' { param($r) $r.guidanceReport.knowledge[0].sha = '0' * 40 } 'recorded live checkout'
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$scenario.result.guidanceReport.knowledge[0].sha = Invoke-GuidanceGit $root @('rev-parse', 'HEAD')
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'actual pinned knowledge SHA accepted' $scenario.scoreArgs $true
|
||||||
|
|
||||||
|
foreach ($badPath in @('../outside.md', '/absolute.md', 'C:/external.md',
|
||||||
|
'microsoft\knowledge\performance\pair-findset-with-next-loop.md',
|
||||||
|
'microsoft/knowledge/performance/../performance/pair-findset-with-next-loop.md',
|
||||||
|
'https://example.invalid/article.md', 'microsoft/knowledge/performance/missing.md',
|
||||||
|
'microsoft/skills/development/al-development-plan.md')) {
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$scenario.result.guidanceReport.knowledge[0].path = $badPath
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'unsafe/nonexistent/non-knowledge citation rejected' $scenario.scoreArgs $false
|
||||||
|
}
|
||||||
|
foreach ($badSample in @('../outside.al', $article, $otherArticle.Replace('.md', '.good.al'), 'microsoft/knowledge/performance/other.good.al', 'microsoft\knowledge\performance\pair-findset-with-next-loop.good.al')) {
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$scenario.result.guidanceReport.knowledge[0].'sample-paths' = @($badSample)
|
||||||
|
Set-TestJson $scenario.resultPath $scenario.result
|
||||||
|
Invoke-EvaluatorTest 'unsafe/nonexistent/non-sibling sample rejected' $scenario.scoreArgs $false
|
||||||
|
}
|
||||||
|
foreach ($json in @('{', 'null', '[]', '{"caseId":"MODEL_SECRET_SENTINEL","caseId":"duplicate"}', '{"caseId":true,}', '// comment')) {
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
[IO.File]::WriteAllText($scenario.resultPath, $json)
|
||||||
|
Invoke-EvaluatorTest 'malformed model result fails without runtime crash or content leakage' $scenario.scoreArgs $false
|
||||||
|
}
|
||||||
|
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$outside = Join-Path $scenario.directory 'outside'
|
||||||
|
[IO.Directory]::CreateDirectory($outside) | Out-Null
|
||||||
|
$link = Join-Path $scenario.workspace 'escape'
|
||||||
|
$linkKind = if ($IsWindows) { 'Junction' } else { 'SymbolicLink' }
|
||||||
|
New-Item -ItemType $linkKind -Path $link -Target $outside | Out-Null
|
||||||
|
try {
|
||||||
|
Invoke-EvaluatorTest 'target junction/symlink rejected instead of followed' $scenario.scoreArgs $false 'Links, junctions'
|
||||||
|
Invoke-EvaluatorTest 'baseline capture rejects junction/symlink target children' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', (Join-Path $scenario.directory 'linked-baseline.json')) $false 'Links, junctions'
|
||||||
|
Invoke-EvaluatorTest 'prepared directory cannot escape through junction' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-PrepareDirectory', (Join-Path $link 'prepared'), '-WorkspaceMapPath', $scenario.mapPath) $false 'Links, junctions'
|
||||||
|
} finally { Remove-Item -LiteralPath $link -Force }
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$external = Join-Path $scenario.directory 'external.txt'
|
||||||
|
[IO.File]::WriteAllText($external, 'external hard-link target')
|
||||||
|
$hardLink = Join-Path $scenario.workspace 'hard-link.txt'
|
||||||
|
New-Item -ItemType HardLink -Path $hardLink -Target $external | Out-Null
|
||||||
|
try {
|
||||||
|
Invoke-EvaluatorTest 'hard-link escape rejected' $scenario.scoreArgs $false 'Links, junctions'
|
||||||
|
} finally { Remove-Item -LiteralPath $hardLink -Force }
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$externalArticle = Join-Path $scenario.directory 'outside.md'
|
||||||
|
[IO.File]::Copy((Join-Path $root $article.Replace('/', [IO.Path]::DirectorySeparatorChar)), $externalArticle)
|
||||||
|
$articleLink = Join-Path $root 'custom\knowledge\performance'
|
||||||
|
[IO.Directory]::CreateDirectory((Split-Path $articleLink -Parent)) | Out-Null
|
||||||
|
New-Item -ItemType $linkKind -Path $articleLink -Target $scenario.directory | Out-Null
|
||||||
|
try {
|
||||||
|
$rejected = $false
|
||||||
|
try { $null = Resolve-GuidanceReference $root 'custom/knowledge/performance/outside.md' -Knowledge }
|
||||||
|
catch { $rejected = $_.Exception.Message -match 'Links, junctions' }
|
||||||
|
if (-not $rejected) { throw 'Linked knowledge reference was followed.' }
|
||||||
|
$tests.Add('knowledge junction/symlink reference rejected')
|
||||||
|
$rejected = $false
|
||||||
|
try { $null = Resolve-GuidanceReference $root 'custom/knowledge/performance/pair-findset-with-next-loop.good.al' -Article 'custom/knowledge/performance/pair-findset-with-next-loop.md' }
|
||||||
|
catch { $rejected = $_.Exception.Message -match 'Links, junctions' }
|
||||||
|
if (-not $rejected) { throw 'Linked sample reference was followed.' }
|
||||||
|
$tests.Add('sample junction/symlink reference rejected')
|
||||||
|
} finally { Remove-Item -LiteralPath $articleLink -Force }
|
||||||
|
$normativePath = Join-Path $root 'microsoft\knowledge\performance\single-normative-section.md'
|
||||||
|
foreach ($heading in @('Best Practice', 'Anti Pattern')) {
|
||||||
|
[IO.File]::WriteAllText($normativePath, "---`ndomain: performance`n---`n## Description`nSynthetic contract fixture.`n## $heading`nSynthetic normative constraint.`n")
|
||||||
|
$null = Resolve-GuidanceReference $root 'microsoft/knowledge/performance/single-normative-section.md' -Knowledge
|
||||||
|
$tests.Add("Knowledge article with only $heading accepted")
|
||||||
|
}
|
||||||
|
[IO.File]::Delete($normativePath)
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$linkedRoot = Join-Path $scratch 'linked-knowledge-checkout'
|
||||||
|
Invoke-TestGit $root @('worktree', 'add', '--quiet', '--detach', $linkedRoot, 'HEAD')
|
||||||
|
$linkedBaseline = Join-Path $scenario.directory 'linked-root-baseline.json'
|
||||||
|
Invoke-EvaluatorTest 'linked knowledge checkout baseline capture' @('-Root', $linkedRoot, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', $linkedBaseline) $true
|
||||||
|
Invoke-EvaluatorTest 'unchanged linked knowledge checkout scoring' @('-Root', $linkedRoot, '-ManifestPath', $scenario.manifestPath, '-ResultsDirectory', $scenario.results, '-BaselinePath', $linkedBaseline, '-BaselineSha256', (Get-GuidanceHash $linkedBaseline)) $true
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
$map = Read-GuidanceJson $scenario.mapPath
|
||||||
|
$map['synthetic-case'] = $linkedRoot
|
||||||
|
Set-TestJson $scenario.mapPath $map
|
||||||
|
Invoke-EvaluatorTest 'linked target checkout rejected as external Git storage' @('-Root', $root, '-ManifestPath', $scenario.manifestPath, '-CaptureBaseline', '-WorkspaceMapPath', $scenario.mapPath, '-BaselinePath', (Join-Path $scenario.directory 'external-git-baseline.json')) $false 'standalone repositories'
|
||||||
|
$scenario = New-TestScenario
|
||||||
|
[IO.File]::AppendAllText((Join-Path $root $sample.Replace('/', [IO.Path]::DirectorySeparatorChar)), "`n// changed knowledge sample")
|
||||||
|
Invoke-EvaluatorTest 'knowledge checkout changes rejected' $scenario.scoreArgs $false 'Knowledge checkout identity/content changed'
|
||||||
|
Write-Host "Development guidance evaluator regressions PASSED: $($tests.Count) checks."
|
||||||
|
} finally {
|
||||||
|
if (Test-Path -LiteralPath $scratch) {
|
||||||
|
# Only our own uniquely named directory; links made by tests are removed above.
|
||||||
|
Remove-Item -LiteralPath $scratch -Recurse -Force
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,352 +1,215 @@
|
||||||
<#
|
<#
|
||||||
.SYNOPSIS
|
.SYNOPSIS
|
||||||
Validates, prepares, and scores read-only AL development-guidance fixtures.
|
Validates, prepares, and scores read-only AL development-guidance fixtures.
|
||||||
|
.DESCRIPTION
|
||||||
|
Capture runner-owned evidence BEFORE invoking an agent:
|
||||||
|
-CaptureBaseline -WorkspaceMapPath <json> -BaselinePath <new-json>
|
||||||
|
The workspace map is {"manifest-case-id":"absolute-standalone-git-root",...}.
|
||||||
|
Score AFTER the agent finishes:
|
||||||
|
-BaselinePath <json> -BaselineSha256 <runner-retained-digest> -ResultsDirectory <directory>
|
||||||
|
Preparation (-PrepareDirectory) is independent; supply -WorkspaceMapPath to
|
||||||
|
include runner-selected target paths. Never derive target paths from results.
|
||||||
|
|
||||||
|
Baseline, map, prepared requests, and results must be outside all targets and
|
||||||
|
the knowledge checkout. Keep the baseline runner-only; retain the printed
|
||||||
|
SHA256 and pass -BaselineSha256 when scoring to detect baseline tampering.
|
||||||
|
Capture never overwrites an existing baseline. The runner must protect this
|
||||||
|
script, the baseline/digest and its invocation from the agent.
|
||||||
|
|
||||||
|
This compares before/after evidence, NOT an OS sandbox or a write monitor.
|
||||||
|
It cannot detect reverted transient writes, prove that articles were opened,
|
||||||
|
or validate semantic faithfulness of prose. Files, directories, hashes,
|
||||||
|
stable metadata, Git HEAD/refs/index and ignored/untracked files are compared.
|
||||||
|
Links/reparse points, hard links, alternate data streams, external Git
|
||||||
|
storage in targets, submodules and sparse checkouts are rejected rather than
|
||||||
|
followed. Run in quiescent repositories. The knowledge checkout may itself
|
||||||
|
be a linked Git worktree; its Git storage identity is recorded explicitly.
|
||||||
#>
|
#>
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
[string] $Root = (Join-Path $PSScriptRoot '..'),
|
||||||
[string] $ManifestPath,
|
[string] $ManifestPath,
|
||||||
[string] $PrepareDirectory,
|
[string] $PrepareDirectory,
|
||||||
[string] $ResultsDirectory
|
[string] $ResultsDirectory,
|
||||||
|
[switch] $CaptureBaseline,
|
||||||
|
[string] $BaselinePath,
|
||||||
|
[string] $BaselineSha256,
|
||||||
|
[string] $WorkspaceMapPath
|
||||||
)
|
)
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
Set-StrictMode -Version Latest
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
|
. (Join-Path $PSScriptRoot 'DevelopmentGuidance.Evidence.ps1')
|
||||||
|
|
||||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
|
||||||
if (-not $ManifestPath) {
|
|
||||||
$ManifestPath = Join-Path $Root 'evaluation/development-guidance-fixtures.json'
|
|
||||||
}
|
|
||||||
|
|
||||||
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
|
|
||||||
$problems = [System.Collections.Generic.List[string]]::new()
|
|
||||||
$validKinds = @('feature', 'bug', 'refactor', 'upgrade', 'maintenance')
|
|
||||||
|
|
||||||
function Get-ModelCaseId {
|
|
||||||
param([string] $ManifestId)
|
|
||||||
|
|
||||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
|
||||||
try {
|
try {
|
||||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($ManifestId)
|
if ($CaptureBaseline -and ($ResultsDirectory -or $PrepareDirectory)) {
|
||||||
$hash = $sha.ComputeHash($bytes)
|
throw 'CaptureBaseline is a separate pre-run operation.'
|
||||||
$token = ([System.BitConverter]::ToString($hash) -replace '-', '').Substring(0, 8).ToLowerInvariant()
|
}
|
||||||
return "case-$token"
|
if (($CaptureBaseline -or $ResultsDirectory) -and -not $BaselinePath) {
|
||||||
} finally {
|
throw 'Capture and scoring require BaselinePath.'
|
||||||
$sha.Dispose()
|
}
|
||||||
|
if ($ResultsDirectory -and -not $BaselineSha256) {
|
||||||
|
throw 'Scoring requires the runner-retained pre-run BaselineSha256.'
|
||||||
|
}
|
||||||
|
if ($CaptureBaseline -and -not $WorkspaceMapPath) {
|
||||||
|
throw 'Capture requires a runner-owned WorkspaceMapPath.'
|
||||||
|
}
|
||||||
|
if ($ResultsDirectory -and ($WorkspaceMapPath -or $PrepareDirectory)) {
|
||||||
|
throw 'Scoring uses only the recorded workspace map; run preparation separately.'
|
||||||
|
}
|
||||||
|
$Root = Get-GuidanceSafePath $Root -Directory
|
||||||
|
if (-not $ManifestPath) { $ManifestPath = Join-Path $Root 'evaluation\development-guidance-fixtures.json' }
|
||||||
|
$ManifestPath = Get-GuidanceSafePath $ManifestPath -File
|
||||||
|
$manifest = Read-GuidanceJson $ManifestPath
|
||||||
|
Assert-GuidanceManifest $manifest $Root
|
||||||
|
$caseIds = @($manifest.cases | ForEach-Object { $_.id })
|
||||||
|
$workspaces = [ordered]@{}
|
||||||
|
$baseline = $null
|
||||||
|
if ($WorkspaceMapPath) {
|
||||||
|
$WorkspaceMapPath = Get-GuidanceSafePath $WorkspaceMapPath -File
|
||||||
|
$map = Read-GuidanceJson $WorkspaceMapPath
|
||||||
|
Assert-GuidanceObject $map 'workspace map'
|
||||||
|
if ($map.Count -ne $caseIds.Count) { throw 'Workspace map must bind exactly every manifest case.' }
|
||||||
|
foreach ($id in $caseIds) {
|
||||||
|
Assert-GuidanceString $map[$id] 'workspace map value'
|
||||||
|
if (-not [IO.Path]::IsPathFullyQualified($map[$id])) { throw 'Workspace roots must be absolute.' }
|
||||||
|
$workspaces[$id] = Get-GuidanceSafePath $map[$id] -Directory
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if ($ResultsDirectory) {
|
||||||
function Get-PlanRequest {
|
$BaselinePath = Get-GuidanceSafePath $BaselinePath -File
|
||||||
param([object] $Plan)
|
if ($BaselineSha256 -cnotmatch '^[0-9A-Fa-f]{64}$') {
|
||||||
|
throw 'BaselineSha256 must be a SHA256 digest.'
|
||||||
if ($Plan.PSObject.Properties.Name -contains 'request' -and
|
|
||||||
-not [string]::IsNullOrWhiteSpace([string]$Plan.request)) {
|
|
||||||
return [string]$Plan.request
|
|
||||||
}
|
}
|
||||||
if ($Plan.PSObject.Properties.Name -contains 'format' -and
|
if ((Get-GuidanceHash $BaselinePath) -ne $BaselineSha256) {
|
||||||
[string]$Plan.format -eq 'BCFIX-HANDOFF' -and
|
throw 'Runner baseline digest mismatch.'
|
||||||
$Plan.PSObject.Properties.Name -contains 'nextStep') {
|
|
||||||
$issue = if ($Plan.PSObject.Properties.Name -contains 'issue') { [string]$Plan.issue } else { 'unknown' }
|
|
||||||
return "Continue BCFIX issue #${issue}: $($Plan.nextStep)"
|
|
||||||
}
|
}
|
||||||
return ''
|
$baseline = Read-GuidanceJson $BaselinePath
|
||||||
|
Assert-GuidanceObject $baseline 'baseline' @('version', 'kind', 'root', 'manifestPath', 'manifestSha256', 'workspaces', 'rootSnapshot')
|
||||||
|
if ($baseline.version -ne 1 -or $baseline.kind -cne 'bcquality-guidance-runner-baseline') {
|
||||||
|
throw 'Unsupported runner baseline.'
|
||||||
}
|
}
|
||||||
|
if ($baseline.root -cne $Root -or $baseline.manifestPath -cne $ManifestPath -or
|
||||||
if ($manifest.version -ne 1) {
|
$baseline.manifestSha256 -ne (Get-GuidanceHash $ManifestPath)) {
|
||||||
$problems.Add("Unsupported guidance fixture version: $($manifest.version)") | Out-Null
|
throw 'Runner baseline does not match the knowledge root or manifest.'
|
||||||
}
|
}
|
||||||
foreach ($thresholdName in @('minimumKnowledgeRecall', 'minimumKnowledgePrecision')) {
|
Assert-GuidanceObject $baseline.workspaces 'baseline workspaces'
|
||||||
$threshold = [double]$manifest.$thresholdName
|
if ($baseline.workspaces.Count -ne $caseIds.Count) { throw 'Runner baseline case set mismatch.' }
|
||||||
if ($threshold -lt 0 -or $threshold -gt 1) {
|
foreach ($id in $caseIds) {
|
||||||
$problems.Add("$thresholdName must be between 0 and 1.") | Out-Null
|
$entry = $baseline.workspaces[$id]
|
||||||
|
Assert-GuidanceObject $entry 'baseline workspace entry' @('caseId', 'root', 'snapshot')
|
||||||
|
if ($entry.caseId -cne (Get-GuidanceCaseId $id)) { throw 'Runner baseline case identity mismatch.' }
|
||||||
|
$workspaces[$id] = Get-GuidanceSafePath $entry.root -Directory
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
$protectedRoots = @($Root) + @($workspaces.Values)
|
||||||
$skillPath = [string]$manifest.skill
|
for ($i = 0; $i -lt $protectedRoots.Count; $i++) {
|
||||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $skillPath) -PathType Leaf)) {
|
for ($j = $i + 1; $j -lt $protectedRoots.Count; $j++) {
|
||||||
$problems.Add("Guidance skill does not exist: $skillPath") | Out-Null
|
if ((Test-GuidanceWithin $protectedRoots[$i] $protectedRoots[$j]) -or
|
||||||
}
|
(Test-GuidanceWithin $protectedRoots[$j] $protectedRoots[$i])) {
|
||||||
|
throw 'Knowledge checkout and case workspaces must be distinct, non-overlapping roots.'
|
||||||
$seenIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
|
||||||
foreach ($case in @($manifest.cases)) {
|
|
||||||
$id = [string]$case.id
|
|
||||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
|
||||||
$problems.Add("Fixture id must be kebab-case: '$id'.") | Out-Null
|
|
||||||
} elseif (-not $seenIds.Add($id)) {
|
|
||||||
$problems.Add("Duplicate fixture id: $id") | Out-Null
|
|
||||||
}
|
|
||||||
if ($case.PSObject.Properties.Name -notcontains 'development-plan') {
|
|
||||||
$problems.Add("${id}: development-plan is required.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
$plan = $case.'development-plan'
|
|
||||||
$expectedKind = if ($case.PSObject.Properties.Name -contains 'expectedKind') {
|
|
||||||
[string]$case.expectedKind
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ($validKinds -notcontains $expectedKind) {
|
|
||||||
$problems.Add("${id}: expectedKind is invalid.") | Out-Null
|
|
||||||
}
|
|
||||||
$isBcfixHandoff = (
|
|
||||||
$plan.PSObject.Properties.Name -contains 'format' -and
|
|
||||||
[string]$plan.format -eq 'BCFIX-HANDOFF'
|
|
||||||
)
|
|
||||||
if ($isBcfixHandoff) {
|
|
||||||
$requiredHandoffFields = @(
|
|
||||||
'version', 'issue', 'phase', 'status', 'baton', 'rootCause',
|
|
||||||
'harnessMap', 'iterationsUsed', 'filesCommitted', 'lastTestResult',
|
|
||||||
'deadEnds', 'nextStep'
|
|
||||||
)
|
|
||||||
foreach ($field in $requiredHandoffFields) {
|
|
||||||
if ($plan.PSObject.Properties.Name -notcontains $field) {
|
|
||||||
$problems.Add("${id}: BCFIX-HANDOFF is missing '$field'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$handoffVersion = if ($plan.PSObject.Properties.Name -contains 'version') { [int]$plan.version } else { 0 }
|
|
||||||
$handoffPhase = if ($plan.PSObject.Properties.Name -contains 'phase') { [string]$plan.phase } else { '' }
|
|
||||||
$handoffStatus = if ($plan.PSObject.Properties.Name -contains 'status') { [string]$plan.status } else { '' }
|
|
||||||
if ($handoffVersion -ne 1) {
|
|
||||||
$problems.Add("${id}: only BCFIX-HANDOFF version 1 is supported.") | Out-Null
|
|
||||||
}
|
|
||||||
if ($handoffPhase -notin @('plan', 'baseline', 'implement', 'pr')) {
|
|
||||||
$problems.Add("${id}: BCFIX-HANDOFF phase is invalid.") | Out-Null
|
|
||||||
}
|
|
||||||
if ($handoffStatus -notin @('in-progress', 'paused', 'done')) {
|
|
||||||
$problems.Add("${id}: BCFIX-HANDOFF status is invalid.") | Out-Null
|
|
||||||
}
|
|
||||||
} elseif (
|
|
||||||
$plan.PSObject.Properties.Name -notcontains 'kind' -or
|
|
||||||
$validKinds -notcontains [string]$plan.kind
|
|
||||||
) {
|
|
||||||
$problems.Add("${id}: development-plan.kind is invalid.") | Out-Null
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace((Get-PlanRequest $plan))) {
|
|
||||||
$problems.Add("${id}: development-plan must provide request or BCFIX nextStep.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$expectedKnowledge = @($case.requiredKnowledge) + @($case.optionalKnowledge)
|
|
||||||
if (@($expectedKnowledge | Sort-Object -Unique).Count -ne $expectedKnowledge.Count) {
|
|
||||||
$problems.Add("${id}: requiredKnowledge and optionalKnowledge contain duplicates.") | Out-Null
|
|
||||||
}
|
|
||||||
foreach ($reference in $expectedKnowledge) {
|
|
||||||
$reference = [string]$reference
|
|
||||||
if ($reference.Contains('\') -or -not $reference.EndsWith('.md')) {
|
|
||||||
$problems.Add("${id}: invalid knowledge path: $reference") | Out-Null
|
|
||||||
} elseif (-not (Test-Path -LiteralPath (Join-Path $Root $reference) -PathType Leaf)) {
|
|
||||||
$problems.Add("${id}: knowledge article does not exist: $reference") | Out-Null
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
foreach ($artifact in @($BaselinePath, $WorkspaceMapPath, $PrepareDirectory, $ResultsDirectory)) {
|
||||||
if ($problems.Count) {
|
if ($artifact) {
|
||||||
Write-Host "Development guidance fixture validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
$safeArtifact = Get-GuidanceSafePath $artifact -AllowMissing
|
||||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
foreach ($protectedRoot in $protectedRoots) {
|
||||||
exit 1
|
if (Test-GuidanceWithin $safeArtifact $protectedRoot) {
|
||||||
|
throw 'Runner artifacts must be outside target workspaces and the knowledge checkout.'
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($PrepareDirectory) {
|
|
||||||
$markerPath = Join-Path $PrepareDirectory '.bcquality-development-guidance-evaluation'
|
|
||||||
if (Test-Path -LiteralPath $PrepareDirectory) {
|
|
||||||
$existing = @(Get-ChildItem -LiteralPath $PrepareDirectory -Force)
|
|
||||||
if ($existing.Count -and -not (Test-Path -LiteralPath $markerPath -PathType Leaf)) {
|
|
||||||
throw "PrepareDirectory is not empty and is not a BCQuality development-guidance evaluation directory: $PrepareDirectory"
|
|
||||||
}
|
}
|
||||||
if (Test-Path -LiteralPath $markerPath -PathType Leaf) {
|
|
||||||
Get-ChildItem -LiteralPath $PrepareDirectory -File |
|
|
||||||
Where-Object {
|
|
||||||
$_.Name -eq 'knowledge-index.json' -or
|
|
||||||
$_.Name -like 'request-*.json' -or
|
|
||||||
$_.Name -like 'result-*.json'
|
|
||||||
} |
|
|
||||||
Remove-Item -Force
|
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
New-Item -ItemType Directory -Force -Path $PrepareDirectory | Out-Null
|
|
||||||
}
|
}
|
||||||
Set-Content -LiteralPath $markerPath -Value 'BCQuality generated development-guidance evaluation directory' -Encoding UTF8
|
if ($CaptureBaseline) {
|
||||||
|
$BaselinePath = Get-GuidanceSafePath $BaselinePath -AllowMissing
|
||||||
$indexPath = Join-Path $PrepareDirectory 'knowledge-index.json'
|
if (Test-Path -LiteralPath $BaselinePath) { throw 'Baseline already exists; capture never overwrites evidence.' }
|
||||||
& (Join-Path $Root 'tools/Build-KnowledgeIndex.ps1') -BCQualityRoot $Root -IndexPath $indexPath | Out-Null
|
$snapshots = [ordered]@{}
|
||||||
$skillInstructions = Get-Content -LiteralPath (Join-Path $Root $skillPath) -Raw
|
foreach ($id in $caseIds) {
|
||||||
|
$snapshots[$id] = [ordered]@{
|
||||||
foreach ($case in @($manifest.cases)) {
|
caseId = Get-GuidanceCaseId $id
|
||||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
root = $workspaces[$id]
|
||||||
[ordered]@{
|
snapshot = Get-GuidanceSnapshot $workspaces[$id] -Target
|
||||||
protocol = 'Run the supplied AL development-plan skill read-only in a clean fixture repository. Return only guidanceReport using the supplied schema and retain the repository for read-only verification.'
|
}
|
||||||
|
}
|
||||||
|
$record = [ordered]@{
|
||||||
|
kind = 'bcquality-guidance-runner-baseline'
|
||||||
|
version = 1
|
||||||
|
root = $Root
|
||||||
|
manifestPath = $ManifestPath
|
||||||
|
manifestSha256 = Get-GuidanceHash $ManifestPath
|
||||||
|
rootSnapshot = Get-GuidanceSnapshot $Root
|
||||||
|
workspaces = $snapshots
|
||||||
|
}
|
||||||
|
Write-GuidanceNewJson $BaselinePath $record
|
||||||
|
Write-Host "Guidance baseline captured. Runner SHA256: $(Get-GuidanceHash $BaselinePath)"
|
||||||
|
} elseif ($PrepareDirectory) {
|
||||||
|
$PrepareDirectory = Get-GuidanceSafePath $PrepareDirectory -AllowMissing
|
||||||
|
if ((Test-Path -LiteralPath $PrepareDirectory) -and
|
||||||
|
@(Get-ChildItem -LiteralPath $PrepareDirectory -Force).Count) {
|
||||||
|
throw 'PrepareDirectory must be new or empty; existing requests/evidence are never overwritten.'
|
||||||
|
}
|
||||||
|
[IO.Directory]::CreateDirectory($PrepareDirectory) | Out-Null
|
||||||
|
# The index builder walks recursively; reject linked corpus paths first.
|
||||||
|
Assert-GuidanceTree $Root
|
||||||
|
& (Join-Path $Root 'tools\Build-KnowledgeIndex.ps1') -BCQualityRoot $Root `
|
||||||
|
-IndexPath (Join-Path $PrepareDirectory 'knowledge-index.json') | Out-Null
|
||||||
|
$skillInstructions = [IO.File]::ReadAllText((Resolve-GuidanceReference $Root $manifest.skill))
|
||||||
|
foreach ($case in $manifest.cases) {
|
||||||
|
$modelId = Get-GuidanceCaseId $case.id
|
||||||
|
$request = [ordered]@{
|
||||||
|
protocol = 'Run the supplied read-only skill on the runner-assigned repository and existing plan. Return only caseId and guidanceReport. The runner captures evidence before invocation; do not capture or modify it. Do not create artifacts in the target or knowledge checkout.'
|
||||||
caseId = $modelId
|
caseId = $modelId
|
||||||
skill = $skillPath
|
skill = $manifest.skill
|
||||||
skillInstructions = $skillInstructions
|
skillInstructions = $skillInstructions
|
||||||
knowledgeIndex = 'knowledge-index.json'
|
knowledgeIndex = Join-Path $PrepareDirectory 'knowledge-index.json'
|
||||||
|
knowledgeRoot = $Root
|
||||||
'task-context' = [ordered]@{
|
'task-context' = [ordered]@{
|
||||||
goal = Get-PlanRequest $case.'development-plan'
|
goal = Get-GuidancePlanRequest $case.'development-plan'
|
||||||
'inputs-available' = @('development-plan', 'repository')
|
'inputs-available' = @('development-plan', 'repository')
|
||||||
technologies = @($case.context.technologies)
|
'bc-version' = $case.context.'bc-version'
|
||||||
countries = @($case.context.countries)
|
technologies = $case.context.technologies
|
||||||
'application-area' = @($case.context.'application-area')
|
countries = $case.context.countries
|
||||||
|
'application-area' = $case.context.'application-area'
|
||||||
}
|
}
|
||||||
'development-plan' = $case.'development-plan'
|
'development-plan' = $case.'development-plan'
|
||||||
resultSchema = [ordered]@{
|
resultSchema = Get-GuidanceResultSchema $modelId
|
||||||
caseId = $modelId
|
|
||||||
workspaceRoot = 'absolute path to the retained clean fixture repository'
|
|
||||||
guidanceReport = [ordered]@{
|
|
||||||
skill = [ordered]@{ id = 'al-development-plan'; version = 1 }
|
|
||||||
outcome = 'completed | not-applicable | no-knowledge | partial | failed'
|
|
||||||
'outcome-reason' = 'required for partial or failed'
|
|
||||||
summary = [ordered]@{
|
|
||||||
request = 'planned change'
|
|
||||||
kind = 'feature | bug | refactor | upgrade | maintenance'
|
|
||||||
candidates = 0
|
|
||||||
selected = 0
|
|
||||||
}
|
}
|
||||||
context = [ordered]@{
|
if ($workspaces.Count) { $request.repository = $workspaces[$case.id] }
|
||||||
'bc-version' = 'resolved target or unknown'
|
Write-GuidanceNewJson (Join-Path $PrepareDirectory "request-$modelId.json") $request
|
||||||
technologies = @('al')
|
|
||||||
countries = @('w1')
|
|
||||||
'application-area' = @('all')
|
|
||||||
unknown = @()
|
|
||||||
}
|
}
|
||||||
knowledge = @([ordered]@{
|
Write-Host "Development guidance preparation PASSED: $($caseIds.Count) case(s)."
|
||||||
path = 'repo-relative article path'
|
} elseif ($ResultsDirectory) {
|
||||||
sha = 'optional commit sha'
|
$ResultsDirectory = Get-GuidanceSafePath $ResultsDirectory -Directory
|
||||||
'used-for' = 'plan decision'
|
$failures = [Collections.Generic.List[string]]::new()
|
||||||
constraints = @('faithful normative constraint')
|
if (-not (Test-GuidanceSnapshotEqual $baseline.rootSnapshot (Get-GuidanceSnapshot $Root))) {
|
||||||
'sample-paths' = @()
|
$failures.Add('Knowledge checkout identity/content changed after baseline capture.')
|
||||||
})
|
|
||||||
'validation-considerations' = @([ordered]@{
|
|
||||||
id = 'stable id'
|
|
||||||
reason = 'why evidence is needed'
|
|
||||||
evidence = 'evidence implementation should obtain'
|
|
||||||
})
|
|
||||||
suppressed = @()
|
|
||||||
unresolved = @()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} | ConvertTo-Json -Depth 15 |
|
|
||||||
Set-Content -LiteralPath (Join-Path $PrepareDirectory "request-$modelId.json") -Encoding UTF8
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($ResultsDirectory) {
|
|
||||||
$failures = [System.Collections.Generic.List[string]]::new()
|
|
||||||
foreach ($case in @($manifest.cases)) {
|
|
||||||
$modelId = Get-ModelCaseId ([string]$case.id)
|
|
||||||
$resultPath = Join-Path $ResultsDirectory "result-$modelId.json"
|
|
||||||
if (-not (Test-Path -LiteralPath $resultPath -PathType Leaf)) {
|
|
||||||
$failures.Add("$($case.id): missing result file.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
foreach ($case in $manifest.cases) {
|
||||||
|
$id = $case.id
|
||||||
try {
|
try {
|
||||||
$result = Get-Content -LiteralPath $resultPath -Raw | ConvertFrom-Json
|
if (-not (Test-GuidanceSnapshotEqual $baseline.workspaces[$id].snapshot `
|
||||||
|
(Get-GuidanceSnapshot $workspaces[$id] -Target))) {
|
||||||
|
throw 'Target repository identity/content changed after baseline capture.'
|
||||||
|
}
|
||||||
|
$resultPath = Get-GuidanceSafePath (Join-Path $ResultsDirectory "result-$(Get-GuidanceCaseId $id).json") -File
|
||||||
|
$result = Read-GuidanceJson $resultPath
|
||||||
|
Assert-GuidanceResult $result $case $manifest $Root $workspaces[$id]
|
||||||
} catch {
|
} catch {
|
||||||
$failures.Add("$($case.id): result is not valid JSON: $($_.Exception.Message)") | Out-Null
|
# Only evaluator-authored diagnostics are printed, never model or file contents.
|
||||||
continue
|
$failures.Add("${id}: $(Get-GuidanceDiagnostic $_)")
|
||||||
}
|
|
||||||
if ($result.PSObject.Properties.Name -notcontains 'caseId' -or [string]$result.caseId -ne $modelId) {
|
|
||||||
$failures.Add("$($case.id): result caseId mismatch.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if ($result.PSObject.Properties.Name -notcontains 'guidanceReport') {
|
|
||||||
$failures.Add("$($case.id): guidanceReport is missing.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
$report = $result.guidanceReport
|
|
||||||
foreach ($requiredField in @('skill', 'outcome', 'summary', 'context', 'knowledge', 'validation-considerations', 'suppressed', 'unresolved')) {
|
|
||||||
if ($report.PSObject.Properties.Name -notcontains $requiredField) {
|
|
||||||
$failures.Add("$($case.id): guidance report is missing '$requiredField'.") | Out-Null
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$skillId = if (
|
|
||||||
$report.PSObject.Properties.Name -contains 'skill' -and
|
|
||||||
$report.skill.PSObject.Properties.Name -contains 'id'
|
|
||||||
) { [string]$report.skill.id } else { '' }
|
|
||||||
if ($skillId -ne 'al-development-plan') {
|
|
||||||
$failures.Add("$($case.id): guidance skill is '$skillId'.") | Out-Null
|
|
||||||
}
|
|
||||||
if ($report.PSObject.Properties.Name -notcontains 'outcome' -or [string]$report.outcome -ne 'completed') {
|
|
||||||
$failures.Add("$($case.id): guidance outcome is not completed.") | Out-Null
|
|
||||||
}
|
|
||||||
$kind = if (
|
|
||||||
$report.PSObject.Properties.Name -contains 'summary' -and
|
|
||||||
$report.summary.PSObject.Properties.Name -contains 'kind'
|
|
||||||
) { [string]$report.summary.kind } else { '' }
|
|
||||||
if ($kind -ne [string]$case.expectedKind) {
|
|
||||||
$failures.Add("$($case.id): summary.kind '$kind' does not match the plan.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
[object[]]$knowledgeEntries = @()
|
|
||||||
if ($report.PSObject.Properties.Name -contains 'knowledge') {
|
|
||||||
$knowledgeEntries = @($report.knowledge)
|
|
||||||
}
|
|
||||||
$usedKnowledge = @()
|
|
||||||
foreach ($entry in $knowledgeEntries) {
|
|
||||||
$path = if ($entry.PSObject.Properties.Name -contains 'path') { [string]$entry.path } else { '' }
|
|
||||||
$usedFor = if ($entry.PSObject.Properties.Name -contains 'used-for') { [string]$entry.'used-for' } else { '' }
|
|
||||||
[object[]]$constraints = @()
|
|
||||||
if ($entry.PSObject.Properties.Name -contains 'constraints') {
|
|
||||||
$constraints = @($entry.constraints)
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace($path) -or
|
|
||||||
-not (Test-Path -LiteralPath (Join-Path $Root $path) -PathType Leaf)) {
|
|
||||||
$failures.Add("$($case.id): invalid knowledge path '$path'.") | Out-Null
|
|
||||||
} else {
|
|
||||||
$usedKnowledge += $path
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace($usedFor) -or -not $constraints.Count) {
|
|
||||||
$failures.Add("$($case.id): '$path' lacks used-for or constraints.") | Out-Null
|
|
||||||
}
|
|
||||||
if ($entry.PSObject.Properties.Name -contains 'sample-paths') {
|
|
||||||
foreach ($samplePath in @($entry.'sample-paths')) {
|
|
||||||
$samplePath = [string]$samplePath
|
|
||||||
if ([string]::IsNullOrWhiteSpace($samplePath) -or
|
|
||||||
-not (Test-Path -LiteralPath (Join-Path $Root $samplePath) -PathType Leaf)) {
|
|
||||||
$failures.Add("$($case.id): invalid sample path '$samplePath'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$required = @($case.requiredKnowledge | ForEach-Object { [string]$_ })
|
|
||||||
$matched = @($required | Where-Object { $usedKnowledge -contains $_ }).Count
|
|
||||||
$recall = if ($required.Count) { $matched / $required.Count } else { 1.0 }
|
|
||||||
if ($recall -lt [double]$manifest.minimumKnowledgeRecall) {
|
|
||||||
$failures.Add("$($case.id): knowledge recall $recall is below $($manifest.minimumKnowledgeRecall).") | Out-Null
|
|
||||||
}
|
|
||||||
$accepted = @(
|
|
||||||
@($case.requiredKnowledge) + @($case.optionalKnowledge) |
|
|
||||||
ForEach-Object { [string]$_ } |
|
|
||||||
Sort-Object -Unique
|
|
||||||
)
|
|
||||||
$acceptedUsed = @($usedKnowledge | Where-Object { $accepted -contains $_ }).Count
|
|
||||||
$precision = if ($usedKnowledge.Count) { $acceptedUsed / $usedKnowledge.Count } else { 0.0 }
|
|
||||||
if ($precision -lt [double]$manifest.minimumKnowledgePrecision) {
|
|
||||||
$failures.Add("$($case.id): knowledge precision $precision is below $($manifest.minimumKnowledgePrecision).") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$workspaceRoot = if ($result.PSObject.Properties.Name -contains 'workspaceRoot') {
|
|
||||||
[string]$result.workspaceRoot
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace($workspaceRoot) -or
|
|
||||||
-not (Test-Path -LiteralPath $workspaceRoot -PathType Container)) {
|
|
||||||
$failures.Add("$($case.id): workspaceRoot is missing or unavailable.") | Out-Null
|
|
||||||
} else {
|
|
||||||
$status = @(& git -C $workspaceRoot status --porcelain)
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
$failures.Add("$($case.id): workspaceRoot is not a readable git worktree.") | Out-Null
|
|
||||||
} elseif ($status.Count) {
|
|
||||||
$failures.Add("$($case.id): guidance skill changed the target repository.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($failures.Count) {
|
if ($failures.Count) {
|
||||||
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):" -ForegroundColor Red
|
Write-Host "Development guidance scoring FAILED ($($failures.Count) problem(s)):"
|
||||||
$failures | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
$failures | ForEach-Object { Write-Host " - $_" }
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
Write-Host "Development guidance scoring PASSED: $(@($manifest.cases).Count) case(s)."
|
Write-Host "Development guidance scoring PASSED: $($caseIds.Count) case(s)."
|
||||||
} else {
|
} else {
|
||||||
Write-Host "Development guidance fixture validation PASSED: $(@($manifest.cases).Count) case(s)."
|
Write-Host "Development guidance fixture validation PASSED: $($caseIds.Count) case(s)."
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
Write-Host "Development guidance FAILED: $(Get-GuidanceDiagnostic $_)"
|
||||||
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,240 +0,0 @@
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Validates Microsoft Learn ingestion coverage and reports progress.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param(
|
|
||||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
|
||||||
[string] $CatalogPath,
|
|
||||||
[string] $CoveragePath,
|
|
||||||
[switch] $Json
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
|
|
||||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
|
||||||
if (-not $CatalogPath) {
|
|
||||||
$CatalogPath = Join-Path $Root 'coverage/microsoft-learn-developer-catalog.json'
|
|
||||||
}
|
|
||||||
if (-not $CoveragePath) {
|
|
||||||
$CoveragePath = Join-Path $Root 'coverage/learn-coverage.json'
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($path in @($CatalogPath, $CoveragePath)) {
|
|
||||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
|
||||||
throw "Coverage input not found: $path"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$catalog = Get-Content -LiteralPath $CatalogPath -Raw | ConvertFrom-Json
|
|
||||||
$coverage = Get-Content -LiteralPath $CoveragePath -Raw | ConvertFrom-Json
|
|
||||||
$problems = [System.Collections.Generic.List[string]]::new()
|
|
||||||
|
|
||||||
if ($catalog.version -ne 1) {
|
|
||||||
$problems.Add("Unsupported catalog version: $($catalog.version)") | Out-Null
|
|
||||||
}
|
|
||||||
if ($coverage.version -ne 1) {
|
|
||||||
$problems.Add("Unsupported coverage version: $($coverage.version)") | Out-Null
|
|
||||||
}
|
|
||||||
if ([string]$coverage.catalog -ne 'coverage/microsoft-learn-developer-catalog.json') {
|
|
||||||
$problems.Add("Coverage catalog path must be coverage/microsoft-learn-developer-catalog.json.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$catalogUnits = @{}
|
|
||||||
foreach ($unit in @($catalog.units)) {
|
|
||||||
$uid = [string]$unit.uid
|
|
||||||
if ($catalogUnits.ContainsKey($uid)) {
|
|
||||||
$problems.Add("Duplicate catalog unit uid: $uid") | Out-Null
|
|
||||||
} else {
|
|
||||||
$catalogUnits[$uid] = $unit
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace([string]$unit.title)) {
|
|
||||||
$problems.Add("${uid}: catalog unit title is empty.") | Out-Null
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace([string]$unit.url) -or
|
|
||||||
-not ([string]$unit.url).StartsWith('https://learn.microsoft.com/', [System.StringComparison]::OrdinalIgnoreCase)) {
|
|
||||||
$problems.Add("${uid}: catalog unit URL is missing or not a Microsoft Learn URL.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$catalogModules = @{}
|
|
||||||
foreach ($module in @($catalog.modules)) {
|
|
||||||
$uid = [string]$module.uid
|
|
||||||
if ($catalogModules.ContainsKey($uid)) {
|
|
||||||
$problems.Add("Duplicate catalog module uid: $uid") | Out-Null
|
|
||||||
} else {
|
|
||||||
$catalogModules[$uid] = $module
|
|
||||||
}
|
|
||||||
foreach ($unitUid in @($module.unitUids)) {
|
|
||||||
if (-not $catalogUnits.ContainsKey([string]$unitUid)) {
|
|
||||||
$problems.Add("${uid}: references unknown unit '$unitUid'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$catalogPaths = @{}
|
|
||||||
foreach ($path in @($catalog.learningPaths)) {
|
|
||||||
$uid = [string]$path.uid
|
|
||||||
if ($catalogPaths.ContainsKey($uid)) {
|
|
||||||
$problems.Add("Duplicate catalog learning-path uid: $uid") | Out-Null
|
|
||||||
} else {
|
|
||||||
$catalogPaths[$uid] = $path
|
|
||||||
}
|
|
||||||
foreach ($moduleUid in @($path.moduleUids)) {
|
|
||||||
if (-not $catalogModules.ContainsKey([string]$moduleUid)) {
|
|
||||||
$problems.Add("${uid}: references unknown module '$moduleUid'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ([int]$catalog.counts.units -ne $catalogUnits.Count) {
|
|
||||||
$problems.Add("Catalog unit count does not match units array.") | Out-Null
|
|
||||||
}
|
|
||||||
if ([int]$catalog.counts.modules -ne $catalogModules.Count) {
|
|
||||||
$problems.Add("Catalog module count does not match modules array.") | Out-Null
|
|
||||||
}
|
|
||||||
if ([int]$catalog.counts.learningPaths -ne $catalogPaths.Count) {
|
|
||||||
$problems.Add("Catalog learning-path count does not match learningPaths array.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($unit in @($catalog.units)) {
|
|
||||||
foreach ($moduleUid in @($unit.moduleUids)) {
|
|
||||||
if (-not $catalogModules.ContainsKey([string]$moduleUid)) {
|
|
||||||
$problems.Add("$($unit.uid): references unknown module '$moduleUid'.") | Out-Null
|
|
||||||
} elseif (@($catalogModules[[string]$moduleUid].unitUids) -notcontains [string]$unit.uid) {
|
|
||||||
$problems.Add("$($unit.uid): module '$moduleUid' does not link back to the unit.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$validStatuses = @('in-progress', 'complete')
|
|
||||||
$validDispositions = @('candidate', 'authored', 'covered-existing', 'rejected', 'deferred')
|
|
||||||
$seenUnits = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
|
||||||
$seenOutcomes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
|
||||||
$articlePaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
|
||||||
$dispositionCounts = @{}
|
|
||||||
foreach ($disposition in $validDispositions) {
|
|
||||||
$dispositionCounts[$disposition] = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($unitCoverage in @($coverage.units)) {
|
|
||||||
$uid = [string]$unitCoverage.uid
|
|
||||||
if (-not $seenUnits.Add($uid)) {
|
|
||||||
$problems.Add("Duplicate coverage unit uid: $uid") | Out-Null
|
|
||||||
}
|
|
||||||
if (-not $catalogUnits.ContainsKey($uid)) {
|
|
||||||
$problems.Add("Coverage unit is absent from the catalog: $uid") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$reviewStatus = [string]$unitCoverage.reviewStatus
|
|
||||||
if ($validStatuses -notcontains $reviewStatus) {
|
|
||||||
$problems.Add("${uid}: invalid reviewStatus '$reviewStatus'.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
[object[]]$outcomes = @()
|
|
||||||
if ($unitCoverage.PSObject.Properties.Name -contains 'outcomes') {
|
|
||||||
$outcomes = @($unitCoverage.outcomes)
|
|
||||||
}
|
|
||||||
if ($reviewStatus -eq 'in-progress' -and -not $outcomes.Count) {
|
|
||||||
$problems.Add("${uid}: an in-progress unit must contain at least one outcome.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($outcome in $outcomes) {
|
|
||||||
$id = [string]$outcome.id
|
|
||||||
if ($id -notmatch '^[a-z0-9]+(?:-[a-z0-9]+)*$') {
|
|
||||||
$problems.Add("${uid}: outcome id must be kebab-case: '$id'.") | Out-Null
|
|
||||||
} elseif (-not $seenOutcomes.Add($id)) {
|
|
||||||
$problems.Add("Duplicate outcome id: $id") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$disposition = [string]$outcome.disposition
|
|
||||||
if ($validDispositions -notcontains $disposition) {
|
|
||||||
$problems.Add("${id}: invalid disposition '$disposition'.") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
$dispositionCounts[$disposition]++
|
|
||||||
|
|
||||||
if ([string]::IsNullOrWhiteSpace([string]$outcome.title)) {
|
|
||||||
$problems.Add("${id}: title is required.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($disposition -in @('candidate', 'authored', 'covered-existing')) {
|
|
||||||
if ([string]::IsNullOrWhiteSpace([string]$outcome.domain)) {
|
|
||||||
$problems.Add("${id}: domain is required for disposition '$disposition'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[object[]]$paths = @()
|
|
||||||
if ($outcome.PSObject.Properties.Name -contains 'articlePaths') {
|
|
||||||
$paths = @($outcome.articlePaths)
|
|
||||||
}
|
|
||||||
if ($disposition -in @('authored', 'covered-existing')) {
|
|
||||||
if (-not $paths.Count) {
|
|
||||||
$problems.Add("${id}: articlePaths is required for disposition '$disposition'.") | Out-Null
|
|
||||||
}
|
|
||||||
foreach ($relativePath in $paths) {
|
|
||||||
$relativePath = [string]$relativePath
|
|
||||||
if ($relativePath.Contains('\') -or -not $relativePath.EndsWith('.md')) {
|
|
||||||
$problems.Add("${id}: article path must be a forward-slash .md path: $relativePath") | Out-Null
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (-not (Test-Path -LiteralPath (Join-Path $Root $relativePath) -PathType Leaf)) {
|
|
||||||
$problems.Add("${id}: article does not exist: $relativePath") | Out-Null
|
|
||||||
}
|
|
||||||
$articlePaths.Add($relativePath) | Out-Null
|
|
||||||
}
|
|
||||||
} elseif ($paths.Count) {
|
|
||||||
$problems.Add("${id}: disposition '$disposition' must not set articlePaths.") | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$rationale = if ($outcome.PSObject.Properties.Name -contains 'rationale') {
|
|
||||||
[string]$outcome.rationale
|
|
||||||
} else {
|
|
||||||
''
|
|
||||||
}
|
|
||||||
if ($disposition -in @('rejected', 'deferred') -and
|
|
||||||
[string]::IsNullOrWhiteSpace($rationale)) {
|
|
||||||
$problems.Add("${id}: rationale is required for disposition '$disposition'.") | Out-Null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($problems.Count) {
|
|
||||||
Write-Host "Learn coverage validation FAILED ($($problems.Count) problem(s)):" -ForegroundColor Red
|
|
||||||
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
$complete = @($coverage.units | Where-Object reviewStatus -eq 'complete').Count
|
|
||||||
$inProgress = @($coverage.units | Where-Object reviewStatus -eq 'in-progress').Count
|
|
||||||
$summary = [ordered]@{
|
|
||||||
catalogUnits = $catalogUnits.Count
|
|
||||||
completeUnits = $complete
|
|
||||||
inProgressUnits = $inProgress
|
|
||||||
unreviewedUnits = $catalogUnits.Count - $complete - $inProgress
|
|
||||||
candidateOutcomes = $dispositionCounts.candidate
|
|
||||||
authoredOutcomes = $dispositionCounts.authored
|
|
||||||
coveredExistingOutcomes = $dispositionCounts.'covered-existing'
|
|
||||||
rejectedOutcomes = $dispositionCounts.rejected
|
|
||||||
deferredOutcomes = $dispositionCounts.deferred
|
|
||||||
referencedArticles = $articlePaths.Count
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($Json) {
|
|
||||||
$summary | ConvertTo-Json
|
|
||||||
} else {
|
|
||||||
$message = (
|
|
||||||
'Learn coverage: {0} units; {1} complete, {2} in progress, {3} unreviewed; ' +
|
|
||||||
'{4} candidate outcomes, {5} authored outcomes, {6} existing-coverage outcomes.'
|
|
||||||
) -f @(
|
|
||||||
$summary.catalogUnits,
|
|
||||||
$summary.completeUnits,
|
|
||||||
$summary.inProgressUnits,
|
|
||||||
$summary.unreviewedUnits,
|
|
||||||
$summary.candidateOutcomes,
|
|
||||||
$summary.authoredOutcomes,
|
|
||||||
$summary.coveredExistingOutcomes
|
|
||||||
)
|
|
||||||
Write-Host $message
|
|
||||||
}
|
|
||||||
|
|
@ -1,262 +0,0 @@
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Updates the committed Microsoft Learn Business Central developer catalog.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
|
||||||
Filters a Microsoft Learn catalog payload to modules tagged for both
|
|
||||||
dynamics-business-central and developer. The output is intentionally only
|
|
||||||
source metadata; editorial dispositions live in coverage/learn-coverage.json.
|
|
||||||
|
|
||||||
The legacy unauthenticated catalog endpoint remains the default while it is
|
|
||||||
available. Use -CatalogPath with an export from the authenticated Learn
|
|
||||||
Platform API when the legacy endpoint is retired.
|
|
||||||
#>
|
|
||||||
[CmdletBinding(DefaultParameterSetName = 'Remote')]
|
|
||||||
param(
|
|
||||||
[string] $Root = (Resolve-Path (Join-Path $PSScriptRoot '..')),
|
|
||||||
|
|
||||||
[Parameter(ParameterSetName = 'Remote')]
|
|
||||||
[uri] $CatalogUri = 'https://learn.microsoft.com/api/catalog/?locale=en-us',
|
|
||||||
|
|
||||||
[Parameter(Mandatory, ParameterSetName = 'File')]
|
|
||||||
[string] $CatalogPath,
|
|
||||||
|
|
||||||
[string] $OutputPath
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
|
|
||||||
$Root = (Resolve-Path -LiteralPath $Root).Path
|
|
||||||
if (-not $OutputPath) {
|
|
||||||
$OutputPath = Join-Path $Root 'coverage/microsoft-learn-developer-catalog.json'
|
|
||||||
}
|
|
||||||
|
|
||||||
function Get-CanonicalUrl {
|
|
||||||
param([string] $Url)
|
|
||||||
|
|
||||||
if ([string]::IsNullOrWhiteSpace($Url)) {
|
|
||||||
return $null
|
|
||||||
}
|
|
||||||
|
|
||||||
$parsed = [uri]$Url
|
|
||||||
return "$($parsed.Scheme)://$($parsed.Host)$($parsed.AbsolutePath)".TrimEnd('/')
|
|
||||||
}
|
|
||||||
|
|
||||||
function Get-CatalogProperty {
|
|
||||||
param(
|
|
||||||
[object] $Object,
|
|
||||||
[string] $Name,
|
|
||||||
[object] $Default = $null
|
|
||||||
)
|
|
||||||
|
|
||||||
$property = $Object.PSObject.Properties[$Name]
|
|
||||||
if ($property) {
|
|
||||||
return $property.Value
|
|
||||||
}
|
|
||||||
return $Default
|
|
||||||
}
|
|
||||||
|
|
||||||
function Get-LatestTimestamp {
|
|
||||||
param([object[]] $Values)
|
|
||||||
|
|
||||||
$timestamps = @(
|
|
||||||
$Values |
|
|
||||||
Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } |
|
|
||||||
ForEach-Object { [datetimeoffset]::Parse([string]$_) } |
|
|
||||||
Sort-Object
|
|
||||||
)
|
|
||||||
if (-not $timestamps.Count) {
|
|
||||||
return $null
|
|
||||||
}
|
|
||||||
|
|
||||||
return $timestamps[-1].ToUniversalTime().ToString(
|
|
||||||
"yyyy-MM-dd'T'HH:mm:ss'Z'",
|
|
||||||
[System.Globalization.CultureInfo]::InvariantCulture
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
function Convert-ToUtcTimestamp {
|
|
||||||
param([object] $Value)
|
|
||||||
|
|
||||||
return ([datetimeoffset]$Value).ToUniversalTime().ToString(
|
|
||||||
"yyyy-MM-dd'T'HH:mm:ss'Z'",
|
|
||||||
[System.Globalization.CultureInfo]::InvariantCulture
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
$catalog = if ($PSCmdlet.ParameterSetName -eq 'File') {
|
|
||||||
Get-Content -LiteralPath $CatalogPath -Raw | ConvertFrom-Json
|
|
||||||
} else {
|
|
||||||
Invoke-RestMethod -Uri $CatalogUri
|
|
||||||
}
|
|
||||||
|
|
||||||
$product = 'dynamics-business-central'
|
|
||||||
$role = 'developer'
|
|
||||||
$modules = @(
|
|
||||||
$catalog.modules |
|
|
||||||
Where-Object {
|
|
||||||
@(Get-CatalogProperty $_ 'products' @()) -contains $product -and
|
|
||||||
@(Get-CatalogProperty $_ 'roles' @()) -contains $role
|
|
||||||
} |
|
|
||||||
Sort-Object uid
|
|
||||||
)
|
|
||||||
|
|
||||||
$moduleIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
|
||||||
foreach ($module in $modules) {
|
|
||||||
$moduleIds.Add([string]$module.uid) | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$learningPaths = @(
|
|
||||||
$catalog.learningPaths |
|
|
||||||
Where-Object {
|
|
||||||
$path = $_
|
|
||||||
$pathModules = @(Get-CatalogProperty $path 'modules' @())
|
|
||||||
@($pathModules | Where-Object { $moduleIds.Contains([string]$_) }).Count -gt 0 -and
|
|
||||||
@(Get-CatalogProperty $path 'products' @()) -contains $product -and
|
|
||||||
@(Get-CatalogProperty $path 'roles' @()) -contains $role
|
|
||||||
} |
|
|
||||||
Sort-Object uid
|
|
||||||
)
|
|
||||||
|
|
||||||
$pathIdsByModule = @{}
|
|
||||||
foreach ($path in $learningPaths) {
|
|
||||||
foreach ($moduleId in @(Get-CatalogProperty $path 'modules' @())) {
|
|
||||||
if (-not $moduleIds.Contains([string]$moduleId)) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (-not $pathIdsByModule.ContainsKey([string]$moduleId)) {
|
|
||||||
$pathIdsByModule[[string]$moduleId] = [System.Collections.Generic.List[string]]::new()
|
|
||||||
}
|
|
||||||
$pathIdsByModule[[string]$moduleId].Add([string]$path.uid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$unitById = @{}
|
|
||||||
foreach ($unit in @(Get-CatalogProperty $catalog 'units' @())) {
|
|
||||||
$unitById[[string]$unit.uid] = $unit
|
|
||||||
}
|
|
||||||
|
|
||||||
$moduleIdsByUnit = @{}
|
|
||||||
$moduleById = @{}
|
|
||||||
$unitIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
|
|
||||||
foreach ($module in $modules) {
|
|
||||||
$moduleById[[string]$module.uid] = $module
|
|
||||||
foreach ($unitId in @(Get-CatalogProperty $module 'units' @())) {
|
|
||||||
$unitId = [string]$unitId
|
|
||||||
$unitIds.Add($unitId) | Out-Null
|
|
||||||
if (-not $moduleIdsByUnit.ContainsKey($unitId)) {
|
|
||||||
$moduleIdsByUnit[$unitId] = [System.Collections.Generic.List[string]]::new()
|
|
||||||
}
|
|
||||||
$moduleIdsByUnit[$unitId].Add([string]$module.uid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$pathRecords = @(
|
|
||||||
foreach ($path in $learningPaths) {
|
|
||||||
[ordered]@{
|
|
||||||
uid = [string]$path.uid
|
|
||||||
title = [string]$path.title
|
|
||||||
url = Get-CanonicalUrl ([string]$path.url)
|
|
||||||
lastModified = if (Get-CatalogProperty $path 'last_modified') {
|
|
||||||
Convert-ToUtcTimestamp (Get-CatalogProperty $path 'last_modified')
|
|
||||||
} else {
|
|
||||||
$null
|
|
||||||
}
|
|
||||||
durationInMinutes = [int](Get-CatalogProperty $path 'duration_in_minutes' 0)
|
|
||||||
moduleUids = @(
|
|
||||||
@(Get-CatalogProperty $path 'modules' @()) |
|
|
||||||
Where-Object { $moduleIds.Contains([string]$_) } |
|
|
||||||
ForEach-Object { [string]$_ } |
|
|
||||||
Sort-Object -Unique
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
$moduleRecords = @(
|
|
||||||
foreach ($module in $modules) {
|
|
||||||
[ordered]@{
|
|
||||||
uid = [string]$module.uid
|
|
||||||
title = [string]$module.title
|
|
||||||
summary = [string](Get-CatalogProperty $module 'summary' '')
|
|
||||||
url = Get-CanonicalUrl ([string]$module.url)
|
|
||||||
lastModified = Convert-ToUtcTimestamp (Get-CatalogProperty $module 'last_modified')
|
|
||||||
durationInMinutes = [int](Get-CatalogProperty $module 'duration_in_minutes' 0)
|
|
||||||
levels = @(@(Get-CatalogProperty $module 'levels' @()) | ForEach-Object { [string]$_ } | Sort-Object -Unique)
|
|
||||||
subjects = @(@(Get-CatalogProperty $module 'subjects' @()) | ForEach-Object { [string]$_ } | Sort-Object -Unique)
|
|
||||||
learningPathUids = if ($pathIdsByModule.ContainsKey([string]$module.uid)) {
|
|
||||||
@($pathIdsByModule[[string]$module.uid] | Sort-Object -Unique)
|
|
||||||
} else {
|
|
||||||
@()
|
|
||||||
}
|
|
||||||
unitUids = @(@(Get-CatalogProperty $module 'units' @()) | ForEach-Object { [string]$_ })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
$unitRecords = @(
|
|
||||||
foreach ($unitId in @($unitIds | Sort-Object)) {
|
|
||||||
if (-not $unitById.ContainsKey($unitId)) {
|
|
||||||
throw "Catalog module references missing unit '$unitId'."
|
|
||||||
}
|
|
||||||
$unit = $unitById[$unitId]
|
|
||||||
$unitUrl = [string](Get-CatalogProperty $unit 'url' '')
|
|
||||||
if ([string]::IsNullOrWhiteSpace($unitUrl)) {
|
|
||||||
$moduleId = [string]@($moduleIdsByUnit[$unitId])[0]
|
|
||||||
$moduleUrl = Get-CanonicalUrl ([string]$moduleById[$moduleId].url)
|
|
||||||
$unitSlug = if ($unitId.StartsWith("$moduleId.", [System.StringComparison]::Ordinal)) {
|
|
||||||
$unitId.Substring($moduleId.Length + 1)
|
|
||||||
} else {
|
|
||||||
@($unitId -split '\.')[-1]
|
|
||||||
}
|
|
||||||
$unitUrl = "$moduleUrl/$unitSlug"
|
|
||||||
}
|
|
||||||
[ordered]@{
|
|
||||||
uid = $unitId
|
|
||||||
title = [string]$unit.title
|
|
||||||
url = Get-CanonicalUrl $unitUrl
|
|
||||||
lastModified = if (Get-CatalogProperty $unit 'last_modified') {
|
|
||||||
Convert-ToUtcTimestamp (Get-CatalogProperty $unit 'last_modified')
|
|
||||||
} else {
|
|
||||||
$null
|
|
||||||
}
|
|
||||||
durationInMinutes = [int](Get-CatalogProperty $unit 'duration_in_minutes' 0)
|
|
||||||
moduleUids = @($moduleIdsByUnit[$unitId] | Sort-Object -Unique)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
$snapshot = [ordered]@{
|
|
||||||
version = 1
|
|
||||||
source = [ordered]@{
|
|
||||||
provider = 'Microsoft Learn'
|
|
||||||
catalogUri = if ($PSCmdlet.ParameterSetName -eq 'Remote') { [string]$CatalogUri } else { $null }
|
|
||||||
locale = 'en-us'
|
|
||||||
filters = [ordered]@{
|
|
||||||
product = $product
|
|
||||||
role = $role
|
|
||||||
}
|
|
||||||
catalogLastModified = Get-LatestTimestamp @(
|
|
||||||
@($moduleRecords | ForEach-Object lastModified) +
|
|
||||||
@($unitRecords | ForEach-Object lastModified) +
|
|
||||||
@($pathRecords | ForEach-Object lastModified)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
counts = [ordered]@{
|
|
||||||
learningPaths = $pathRecords.Count
|
|
||||||
modules = $moduleRecords.Count
|
|
||||||
units = $unitRecords.Count
|
|
||||||
}
|
|
||||||
learningPaths = $pathRecords
|
|
||||||
modules = $moduleRecords
|
|
||||||
units = $unitRecords
|
|
||||||
}
|
|
||||||
|
|
||||||
$outputDirectory = Split-Path -Parent $OutputPath
|
|
||||||
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
|
||||||
New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null
|
|
||||||
}
|
|
||||||
$snapshot | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $OutputPath -Encoding UTF8
|
|
||||||
Write-Host "Microsoft Learn developer catalog: $($pathRecords.Count) paths, $($moduleRecords.Count) modules, $($unitRecords.Count) units."
|
|
||||||
Write-Host "Catalog: $OutputPath"
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue