feat(ng-dev): add ai skills validation command

josephperrott · josephperrott · commit 6fa596ff434e · 2026-02-02T13:48:18.000-08:00
Implements a new `ng-dev ai skills validate` command to validate `SKILL.md` files within the repository.
The command enforces the Agent Skills specification by validating YAML frontmatter against a Zod schema, ensuring correct directory structure, and checking for required fields.
diff --git a/.prettierignore b/.prettierignore
@@ -22,3 +22,6 @@ bazel/map-size-tracking/test/size-golden.json
 
 **/test/goldens/**/*.md
 **/pnpm-lock.yaml
+
+# AI Skills intentionally not formatted to ensure validation failures.
+ng-dev/ai/skills/fixtures/invalid-frontmatter-location/skills/test-skill/SKILL.md
diff --git a/ng-dev/ai/BUILD.bazel b/ng-dev/ai/BUILD.bazel
@@ -14,6 +14,7 @@ ts_project(
         "//ng-dev:node_modules/cli-progress",
         "//ng-dev:node_modules/fast-glob",
         "//ng-dev:node_modules/yargs",
+        "//ng-dev/ai/skills",
         "//ng-dev/utils",
     ],
 )
diff --git a/ng-dev/ai/cli.ts b/ng-dev/ai/cli.ts
@@ -8,8 +8,9 @@
 import {Argv} from 'yargs';
 import {MigrateModule} from './migrate.js';
 import {FixModule} from './fix.js';
+import {SkillsModule} from './skills/cli.js';
 
 /** Build the parser for the AI commands. */
 export function buildAiParser(localYargs: Argv) {
-  return localYargs.command(MigrateModule).command(FixModule);
+  return localYargs.command(MigrateModule).command(FixModule).command(SkillsModule);
 }
diff --git a/ng-dev/ai/skills/BUILD.bazel b/ng-dev/ai/skills/BUILD.bazel
@@ -0,0 +1,39 @@
+load("//tools:defaults.bzl", "jasmine_test", "ts_project")
+
+ts_project(
+    name = "skills",
+    srcs = glob(
+        ["**/*.ts"],
+        exclude = ["**/*.spec.ts"],
+    ),
+    visibility = ["//ng-dev:__subpackages__"],
+    deps = [
+        "//ng-dev:node_modules/@types/node",
+        "//ng-dev:node_modules/@types/yargs",
+        "//ng-dev:node_modules/fast-glob",
+        "//ng-dev:node_modules/yaml",
+        "//ng-dev:node_modules/yargs",
+        "//ng-dev:node_modules/zod",
+        "//ng-dev/utils",
+    ],
+)
+
+ts_project(
+    name = "test_lib",
+    testonly = True,
+    srcs = ["validate.spec.ts"],
+    deps = [
+        ":skills",
+        "//ng-dev:node_modules/@types/jasmine",
+        "//ng-dev:node_modules/@types/node",
+        "//ng-dev/utils",
+        "//ng-dev/utils/testing",
+    ],
+)
+
+jasmine_test(
+    name = "test",
+    data = glob(["fixtures/**"]) + [
+        ":test_lib",
+    ],
+)
diff --git a/ng-dev/ai/skills/cli.ts b/ng-dev/ai/skills/cli.ts
@@ -0,0 +1,39 @@
+/**
+ * @license
+ * Copyright Google LLC
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Arguments, Argv, CommandModule} from 'yargs';
+
+import {validateSkills} from './validate.js';
+import {determineRepoBaseDirFromCwd} from '../../utils/repo-directory.js';
+
+interface Options {
+  baseDir: string;
+}
+
+async function builder(yargs: Argv) {
+  return yargs.option('base-dir' as 'baseDir', {
+    type: 'string',
+    default: determineRepoBaseDirFromCwd(),
+    hidden: true,
+    description: 'The base directory to look for skills in',
+  });
+}
+
+async function handler({baseDir}: Arguments<Options>) {
+  process.exitCode = (await validateSkills(baseDir)).exitCode;
+}
+
+/**
+ * Validates all skills found in the `skills/` directory.
+ */
+export const SkillsModule: CommandModule<{}, Options> = {
+  command: 'skills validate',
+  describe: 'Validate agent skills in the repository',
+  builder,
+  handler,
+};
diff --git a/ng-dev/ai/skills/fixtures/complex-skill/skills/complex-skill/SKILL.md b/ng-dev/ai/skills/fixtures/complex-skill/skills/complex-skill/SKILL.md
@@ -0,0 +1,11 @@
+---
+name: complex-skill
+description: Valid description
+license: MIT
+compatibility: Any
+metadata:
+  key: value
+allowed-tools: tool1
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/invalid-frontmatter-location/skills/test-skill/SKILL.md b/ng-dev/ai/skills/fixtures/invalid-frontmatter-location/skills/test-skill/SKILL.md
@@ -0,0 +1,7 @@
+<!-- This skill is invalid because the frontmatter is not at the very beginning of the file -->
+---
+name: test-skill
+description: Valid description
+license: MIT
+---
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/invalid-name-format/skills/InvalidName/SKILL.md b/ng-dev/ai/skills/fixtures/invalid-name-format/skills/InvalidName/SKILL.md
@@ -0,0 +1,7 @@
+---
+name: InvalidName
+description: Valid description
+license: MIT
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/invalid-schema/skills/bad-schema/SKILL.md b/ng-dev/ai/skills/fixtures/invalid-schema/skills/bad-schema/SKILL.md
@@ -0,0 +1,6 @@
+---
+name: bad-schema
+description: Valid description
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/missing-frontmatter/skills/bad-skill/SKILL.md b/ng-dev/ai/skills/fixtures/missing-frontmatter/skills/bad-skill/SKILL.md
@@ -0,0 +1 @@
+Just text
diff --git a/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill1/SKILL.md b/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill1/SKILL.md
@@ -0,0 +1,7 @@
+---
+name: skill1
+description: Valid description
+license: MIT
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill2/SKILL.md b/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill2/SKILL.md
@@ -0,0 +1,13 @@
+---
+name: skill2
+description: Invalid description
+license: MIT
+allowed-tools:
+  - should
+  - have
+  - been
+  - a
+  - string
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill1/SKILL.md b/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill1/SKILL.md
@@ -0,0 +1,7 @@
+---
+name: skill1
+description: Valid description
+license: MIT
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill2/SKILL.md b/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill2/SKILL.md
@@ -0,0 +1,7 @@
+---
+name: skill2
+description: Valid description
+license: MIT
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/name-mismatch/skills/wrong-name/SKILL.md b/ng-dev/ai/skills/fixtures/name-mismatch/skills/wrong-name/SKILL.md
@@ -0,0 +1,7 @@
+---
+name: right-name
+description: Valid description
+license: MIT
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/fixtures/valid-skill/skills/test-skill/SKILL.md b/ng-dev/ai/skills/fixtures/valid-skill/skills/test-skill/SKILL.md
@@ -0,0 +1,7 @@
+---
+name: test-skill
+description: Valid description
+license: MIT
+---
+
+Instruction
diff --git a/ng-dev/ai/skills/validate.spec.ts b/ng-dev/ai/skills/validate.spec.ts
@@ -0,0 +1,71 @@
+import {join} from 'path';
+import {validateSkill, validateSkills} from './validate.js';
+
+function getFixturePath(relativePath: string): string {
+  return join(process.cwd(), 'ng-dev/ai/skills/fixtures', relativePath);
+}
+
+describe('validateSkills', () => {
+  it('should pass for valid skill', async () => {
+    const skillPath = getFixturePath('valid-skill/skills/test-skill/SKILL.md');
+    const {failures} = await validateSkill(skillPath);
+    expect(failures.length).toBe(0);
+  });
+
+  it('should fail for missing frontmatter', async () => {
+    const skillPath = getFixturePath('missing-frontmatter/skills/bad-skill/SKILL.md');
+    const {failures} = await validateSkill(skillPath);
+
+    expect(failures.length).toBe(1);
+    expect(failures).toContain(jasmine.stringMatching('Missing or invalid frontmatter'));
+  });
+
+  it('should fail if frontmatter is not at the start of the file', async () => {
+    const skillPath = getFixturePath('invalid-frontmatter-location/skills/test-skill/SKILL.md');
+    const {failures} = await validateSkill(skillPath);
+
+    expect(failures).toContain(jasmine.stringMatching('Missing or invalid frontmatter'));
+  });
+
+  it('should fail for invalid schema (missing license)', async () => {
+    const skillPath = getFixturePath('invalid-schema/skills/bad-schema/SKILL.md');
+    const {failures} = await validateSkill(skillPath);
+
+    expect(failures.length).toBe(1);
+    expect(failures).toContain(jasmine.stringMatching('Schema validation failure'));
+  });
+
+  it('should fail for name mismatch', async () => {
+    const skillPath = getFixturePath('name-mismatch/skills/wrong-name/SKILL.md');
+    const {failures} = await validateSkill(skillPath);
+
+    expect(failures.length).toBe(1);
+    expect(failures).toContain(jasmine.stringMatching('Name mismatch'));
+  });
+
+  it('should fail for invalid name format', async () => {
+    const skillPath = getFixturePath('invalid-name-format/skills/InvalidName/SKILL.md');
+    const {failures} = await validateSkill(skillPath);
+
+    expect(failures.length).toBe(1);
+    expect(failures).toContain(jasmine.stringMatching('Schema validation failure'));
+  });
+
+  it('should pass with valid optional fields', async () => {
+    const skillPath = getFixturePath('complex-skill/skills/complex-skill/SKILL.md');
+    const {failures} = await validateSkill(skillPath);
+    expect(failures.length).toBe(0);
+  });
+
+  it('should validate multiple skills', async () => {
+    const repoRoot = getFixturePath('multiple-valid');
+    const {exitCode} = await validateSkills(repoRoot);
+    expect(exitCode).toBe(0);
+  });
+
+  it('should fail if one of multiple skills is invalid', async () => {
+    const repoRoot = getFixturePath('multiple-mixed');
+    const {exitCode} = await validateSkills(repoRoot);
+    expect(exitCode).toBe(1);
+  });
+});
diff --git a/ng-dev/ai/skills/validate.ts b/ng-dev/ai/skills/validate.ts
@@ -0,0 +1,129 @@
+/**
+ * @license
+ * Copyright Google LLC
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {z} from 'zod';
+import {readFile} from 'node:fs/promises';
+import {join, basename} from 'node:path';
+import glob from 'fast-glob';
+import {parse} from 'yaml';
+import {bold, green, Log, red, yellow} from '../../utils/logging.js';
+
+/**
+ * Validation schema for the SKILL.md frontmatter.
+ * Based on https://agentskills.io/specification
+ */
+const skillFrontmatterSchema = z.object({
+  name: z
+    .string()
+    .min(1)
+    .max(64)
+    .regex(/^[a-z0-9-]+$/, 'Name must only contain lowercase alphanumeric characters and hyphens')
+    .refine(
+      (val) => !val.startsWith('-') && !val.endsWith('-') && !val.includes('--'),
+      'Name must not start/end with hyphens or contain consecutive hyphens',
+    ),
+  description: z.string().min(1).max(1024),
+  license: z.string().min(1),
+  compatibility: z.string().min(1).max(500).optional(),
+  metadata: z.record(z.string(), z.string()).optional(),
+  'allowed-tools': z
+    .string()
+    .transform((val) =>
+      val
+        .split(',')
+        .map((s) => s.trim())
+        .filter(Boolean),
+    )
+    .optional(),
+});
+
+export interface ValidationResult {
+  name: string;
+  failures: string[];
+}
+
+/** Validates all skills found in the `skills/` directory. */
+export async function validateSkills(
+  repoRoot: string,
+): Promise<{results: ValidationResult[]; exitCode: number}> {
+  let errorCount = 0;
+  const skillFiles = await glob('**/SKILL.md', {cwd: join(repoRoot, 'skills'), absolute: true});
+
+  if (skillFiles.length === 0) {
+    Log.info(` ${yellow('⚠')}  No skills found in skills/ directory.`);
+    return {results: [], exitCode: 0};
+  }
+
+  Log.info(`Found ${skillFiles.length} skills. Validating...`);
+
+  const validationResults = await Promise.all(skillFiles.map(validateSkill));
+
+  for (const result of validationResults.sort((a, b) => a.failures.length - b.failures.length)) {
+    if (result.failures.length > 0) {
+      Log.info(` ${red('✘')}  ${bold(result.name)} (${join('skills', result.name, 'SKILL.md')})`);
+      result.failures.forEach((failure) => {
+        Log.info(`  -  ${failure}`);
+        errorCount++;
+      });
+    } else {
+      Log.info(` ${green('✔')}  ${bold(result.name)} (${join('skills', result.name, 'SKILL.md')})`);
+    }
+  }
+
+  Log.info();
+  if (errorCount > 0) {
+    Log.error(` ${red('✘')}  Validation failed with ${errorCount} errors.`);
+    return {results: validationResults, exitCode: 1};
+  } else {
+    Log.info(` ${green('✔')} All skills validated successfully.`);
+    return {results: validationResults, exitCode: 0};
+  }
+}
+
+/** Validates a single skill file. */
+export async function validateSkill(filePath: string): Promise<ValidationResult> {
+  const name = basename(join(filePath, '..'));
+  const failures: string[] = [];
+
+  try {
+    const content = await readFile(filePath, {encoding: 'utf8'});
+    const frontmatterRaw = content.match(/^---\n([\s\S]*?)\n---/);
+
+    if (frontmatterRaw === null) {
+      failures.push('Missing or invalid frontmatter in SKILL.md');
+      return {name, failures};
+    }
+
+    let frontmatter: unknown;
+    try {
+      frontmatter = parse(frontmatterRaw[1]);
+    } catch (e) {
+      failures.push(`Failed to parse YAML frontmatter: ${(e as Error).message}`);
+      return {name, failures};
+    }
+
+    const frontmatterData = skillFrontmatterSchema.safeParse(frontmatter);
+    if (frontmatterData.success === false) {
+      for (const issue of frontmatterData.error.issues) {
+        failures.push(`Schema validation failure: [${issue.path.join('.')}] ${issue.message}`);
+      }
+      return {name, failures};
+    }
+
+    // Check name match if data looks like an object with a name
+    if (frontmatterData.data?.name !== name) {
+      failures.push(
+        `Name mismatch. Expected "${name}", found "${frontmatterData.data?.name ?? '<UNKNOWN>'}"`,
+      );
+    }
+  } catch (e) {
+    failures.push(`Unexpected error: ${(e as Error).message}`);
+  }
+
+  return {name, failures};
+}
diff --git a/ng-dev/package.json b/ng-dev/package.json
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,7 @@ ts_project(`
`14`	`14`	`"//ng-dev:node_modules/cli-progress",`
`15`	`15`	`"//ng-dev:node_modules/fast-glob",`
`16`	`16`	`"//ng-dev:node_modules/yargs",`
	`17`	`+ "//ng-dev/ai/skills",`
`17`	`18`	`"//ng-dev/utils",`
`18`	`19`	`],`
`19`	`20`	`)`