From 633fbdfbdb7f213bef7d071737acbb83137bf886 Mon Sep 17 00:00:00 2001 From: Naman Swami Date: Sat, 26 Sep 2026 21:03:20 +0530 Subject: [PATCH] feat(validate): add passport preflight checks for Checkpoint 1 & 2 (EXPLAINABILITY.md and DUTIES.md) --- src/commands/validate-passport.test.ts | 176 ++++++++++++++++++++ src/commands/validate.ts | 212 ++++++++++++++++++++++++- 2 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 src/commands/validate-passport.test.ts diff --git a/src/commands/validate-passport.test.ts b/src/commands/validate-passport.test.ts new file mode 100644 index 0000000..b9063ae --- /dev/null +++ b/src/commands/validate-passport.test.ts @@ -0,0 +1,176 @@ +/** + * Tests for GitAgent Passport Preflight Validation (Checkpoints 1 & 2). + * + * Uses Node.js built-in test runner (node --test). + */ +import { test, describe } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { + validateExplainabilityMd, + validateDutiesMd, + validatePassport +} from './validate.js'; + +function createTempAgentDir(files: Record): string { + const dir = mkdtempSync(join(tmpdir(), 'gitagent-passport-test-')); + for (const [name, content] of Object.entries(files)) { + writeFileSync(join(dir, name), content, 'utf-8'); + } + return dir; +} + +describe('GitAgent Passport: validateExplainabilityMd (Checkpoint 2)', () => { + test('passes with valid decision, data sources, and limitations sections', () => { + const dir = createTempAgentDir({ + 'EXPLAINABILITY.md': `# Explainability + +## Decision Making and Reasoning +The agent parses user queries using first-principles reasoning. It decomposes complex problems into atomic steps before evaluating. + +## Data Sources and Inputs +All calculations are grounded in standard peer-reviewed literature. Numerical parameters are ingested through validated schemas. + +## Limitations and Known Constraints +The agent cannot provide medical diagnostics. It will gracefully refuse questions outside of theoretical physics. +` + }); + + const result = validateExplainabilityMd(dir); + assert.equal(result.valid, true); + assert.equal(result.errors.length, 0); + }); + + test('fails if EXPLAINABILITY.md is missing', () => { + const dir = createTempAgentDir({}); + const result = validateExplainabilityMd(dir); + assert.equal(result.valid, false); + assert.match(result.errors[0], /required for Passport Checkpoint 2 but not found/i); + }); + + test('fails if decision/reasoning section is missing', () => { + const dir = createTempAgentDir({ + 'EXPLAINABILITY.md': `# Explainability + +## Data Sources +The agent reads reference papers. Numerical inputs are checked. + +## Limitations +The agent has bounded context windows. Hallucinations are actively filtered. +` + }); + + const result = validateExplainabilityMd(dir); + assert.equal(result.valid, false); + assert.ok(result.errors.some(e => e.includes('Decision/Reasoning'))); + }); + + test('fails if data sources/inputs section is missing', () => { + const dir = createTempAgentDir({ + 'EXPLAINABILITY.md': `# Explainability + +## Reasoning Approach +The agent employs chain of thought reasoning. It validates each step logically. + +## Known Limitations +The agent requires external tool invocation. Latency may increase under load. +` + }); + + const result = validateExplainabilityMd(dir); + assert.equal(result.valid, false); + assert.ok(result.errors.some(e => e.includes('Data Sources/Inputs'))); + }); + + test('fails if limitations section is missing', () => { + const dir = createTempAgentDir({ + 'EXPLAINABILITY.md': `# Explainability + +## How It Decides +The agent follows strict deterministic rules. It verifies intermediate steps. + +## Input Data Used +Inputs are sanitized against standard regex. All inputs are UTF-8 encoded. +` + }); + + const result = validateExplainabilityMd(dir); + assert.equal(result.valid, false); + assert.ok(result.errors.some(e => e.includes('Limitations'))); + }); + + test('fails if a section contains fewer than 2 sentences', () => { + const dir = createTempAgentDir({ + 'EXPLAINABILITY.md': `# Explainability + +## Decision Logic +Only one sentence here. + +## Data Sources +First valid sentence. Second valid sentence. + +## Constraints +First limitation sentence. Second limitation sentence. +` + }); + + const result = validateExplainabilityMd(dir); + assert.equal(result.valid, false); + assert.ok(result.errors.some(e => e.includes('at least 2 sentences explaining decision logic'))); + }); +}); + +describe('GitAgent Passport: validateDutiesMd (Checkpoint 1)', () => { + test('passes with segregated maker and checker duties', () => { + const dir = createTempAgentDir({ + 'DUTIES.md': `# Separation of Duties Policy + +- **Sage (Maker):** Formulates step-by-step STEM derivations and drafting responses. +- **Reviewer (Checker):** Independently audits calculations, checks assumptions, and approves release. +` + }); + + const result = validateDutiesMd(dir); + assert.equal(result.valid, true); + assert.equal(result.errors.length, 0); + }); + + test('fails if maker and checker are combined in the same role definition', () => { + const dir = createTempAgentDir({ + 'DUTIES.md': `# Separation of Duties Policy + +- Role: Maker and Checker (Single agent creates and validates output) +` + }); + + const result = validateDutiesMd(dir); + assert.equal(result.valid, false); + assert.ok(result.errors.some(e => e.includes('Maker and Checker cannot be assigned to the same entity or role'))); + }); + + test('returns warning if DUTIES.md is absent', () => { + const dir = createTempAgentDir({}); + const result = validateDutiesMd(dir); + assert.equal(result.valid, true); + assert.equal(result.errors.length, 0); + assert.ok(result.warnings.length > 0); + }); +}); + +describe('GitAgent Passport: validatePassport preflight aggregator', () => { + test('validates both checkpoints together', () => { + const dir = createTempAgentDir({ + 'DUTIES.md': `- Maker: Author\n- Checker: Auditor`, + 'EXPLAINABILITY.md': `## How It Decides\nSentence one about logic. Sentence two about logic.\n## Data Used\nSentence one about data. Sentence two about data.\n## Known Issues\nSentence one about constraints. Sentence two about constraints.` + }); + + const result = validatePassport(dir); + assert.equal(result.valid, true); + assert.equal(result.errors.length, 0); + assert.equal(result.checkpoint1.valid, true); + assert.equal(result.checkpoint2.valid, true); + }); +}); diff --git a/src/commands/validate.ts b/src/commands/validate.ts index dfcd7a9..8f7f1ae 100644 --- a/src/commands/validate.ts +++ b/src/commands/validate.ts @@ -10,12 +10,13 @@ import { loadSchema } from '../utils/schemas.js'; import { parseSkillMd } from '../utils/skill-loader.js'; import { success, error, warn, info, heading, divider } from '../utils/format.js'; -interface ValidateOptions { +export interface ValidateOptions { dir: string; compliance: boolean; + passport: boolean; } -interface ValidationResult { +export interface ValidationResult { valid: boolean; errors: string[]; warnings: string[]; @@ -114,6 +115,178 @@ function validateSoulMd(dir: string): ValidationResult { return result; } +export function validateExplainabilityMd(dir: string): ValidationResult { + const result: ValidationResult = { valid: true, errors: [], warnings: [] }; + const explainPath = join(dir, 'EXPLAINABILITY.md'); + + if (!existsSync(explainPath)) { + result.valid = false; + result.errors.push('EXPLAINABILITY.md is required for Passport Checkpoint 2 but not found'); + return result; + } + + const raw = readFileSync(explainPath, 'utf-8').trim(); + if (raw.length === 0) { + result.valid = false; + result.errors.push('EXPLAINABILITY.md is empty — must contain structured explanation sections'); + return result; + } + + const lines = raw.split('\n'); + const sections: Array<{ heading: string; body: string }> = []; + let currentHeading = ''; + let currentBody: string[] = []; + + for (const line of lines) { + const match = line.match(/^#{1,4}\s+(.+)$/); + if (match) { + if (currentHeading) { + sections.push({ heading: currentHeading, body: currentBody.join('\n').trim() }); + } + currentHeading = match[1].trim(); + currentBody = []; + } else { + currentBody.push(line); + } + } + if (currentHeading) { + sections.push({ heading: currentHeading, body: currentBody.join('\n').trim() }); + } + + if (sections.length === 0) { + result.valid = false; + result.errors.push('EXPLAINABILITY.md contains no markdown headings — must organize topics under headings'); + return result; + } + + const countSentences = (text: string): number => { + const clean = text + .replace(/```[\s\S]*?```/g, '') + .replace(/\|.*\|/g, '') + .replace(/^#.*$/gm, '') + .trim(); + if (!clean) return 0; + const sentences = clean.split(/[.!?]+(?:\s+|$)/).filter(s => s.trim().length > 5); + return sentences.length; + }; + + // 1. Decision / Reasoning logic check (Checkpoint 2) + const decisionSection = sections.find(s => /decision|reasoning|how it decides/i.test(s.heading)); + if (!decisionSection) { + result.valid = false; + result.errors.push( + 'EXPLAINABILITY.md missing required Decision/Reasoning section (heading must include "decision", "reasoning", or "how it decides")' + ); + } else { + const sentences = countSentences(decisionSection.body); + if (sentences < 2) { + result.valid = false; + result.errors.push( + `EXPLAINABILITY.md section "${decisionSection.heading}" must contain at least 2 sentences explaining decision logic (found ${sentences})` + ); + } + } + + // 2. Data Sources / Inputs check (Checkpoint 2) + const inputSection = sections.find(s => /data source|input|data used/i.test(s.heading)); + if (!inputSection) { + result.valid = false; + result.errors.push( + 'EXPLAINABILITY.md missing required Data Sources/Inputs section (heading must include "data source", "input", or "data used")' + ); + } else { + const sentences = countSentences(inputSection.body); + if (sentences < 2) { + result.valid = false; + result.errors.push( + `EXPLAINABILITY.md section "${inputSection.heading}" must contain at least 2 sentences describing data sources or inputs (found ${sentences})` + ); + } + } + + // 3. Limitations / Constraints check (Checkpoint 2) + const limitationSection = sections.find(s => /limitation|constraint|known issue/i.test(s.heading)); + if (!limitationSection) { + result.valid = false; + result.errors.push( + 'EXPLAINABILITY.md missing required Limitations section (heading must include "limitation", "constraint", or "known issue")' + ); + } else { + const sentences = countSentences(limitationSection.body); + if (sentences < 2) { + result.valid = false; + result.errors.push( + `EXPLAINABILITY.md section "${limitationSection.heading}" must contain at least 2 sentences detailing limitations or failure modes (found ${sentences})` + ); + } + } + + return result; +} + +export function validateDutiesMd(dir: string): ValidationResult { + const result: ValidationResult = { valid: true, errors: [], warnings: [] }; + const dutiesPath = join(dir, 'DUTIES.md'); + const agentsPath = join(dir, 'AGENTS.md'); + + const filesToCheck = [ + { path: dutiesPath, name: 'DUTIES.md' }, + { path: agentsPath, name: 'AGENTS.md' } + ].filter(f => existsSync(f.path)); + + if (filesToCheck.length === 0) { + result.warnings.push('DUTIES.md not found — recommended for multi-agent segregation of duties (Passport Checkpoint 1)'); + return result; + } + + for (const { path, name } of filesToCheck) { + const content = readFileSync(path, 'utf-8'); + const lines = content.split('\n'); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (/\bmaker\b/i.test(line) && /\b(checker|reviewer)\b/i.test(line)) { + // Exclude legitimate architectural phrases + if (/boundary|separation|segregation|distinct|separate|differ|between|versus|\bvs\b/i.test(line)) { + continue; + } + + // Flag actual role combinations or conflations + if ( + /[:=\-–—|].*(?:assign|hold|act|role|persona).*\bmaker\b.*(?:and|&|\/|\+).*\b(?:checker|reviewer)\b/i.test(line) || + /[:=\-–—|].*(?:assign|hold|act|role|persona).*\b(?:checker|reviewer)\b.*(?:and|&|\/|\+).*\bmaker\b/i.test(line) || + /\b(same|single)\s+(?:agent|role|entity|individual|person|user).*\b(?:maker\b.*\b(?:checker|reviewer)|(?:checker|reviewer)\b.*\bmaker)\b/i.test(line) || + /\b(?:maker\s*(?:and|&|\/|\+)\s*(?:checker|reviewer)|(?:checker|reviewer)\s*(?:and|&|\/|\+)\s*maker)\s*(?:role|agent|persona|responsibility)\b/i.test(line) + ) { + result.valid = false; + result.errors.push( + `${name} (line ${i + 1}): Maker and Checker cannot be assigned to the same entity or role (violates Separation of Duties)` + ); + } + } + } + } + + return result; +} + +export function validatePassport(dir: string): { + valid: boolean; + errors: string[]; + warnings: string[]; + checkpoint1: ValidationResult; + checkpoint2: ValidationResult; +} { + const cp1 = validateDutiesMd(dir); + const cp2 = validateExplainabilityMd(dir); + + const valid = cp1.valid && cp2.valid; + const errors = [...cp1.errors, ...cp2.errors]; + const warnings = [...cp1.warnings, ...cp2.warnings]; + + return { valid, errors, warnings, checkpoint1: cp1, checkpoint2: cp2 }; +} + function validateCompliance(dir: string): ValidationResult { const result: ValidationResult = { valid: true, errors: [], warnings: [] }; @@ -477,6 +650,7 @@ export const validateCommand = new Command('validate') .description('Validate a gitagent repository against the specification') .option('-d, --dir ', 'Agent directory', '.') .option('-c, --compliance', 'Include regulatory compliance validation', false) + .option('-p, --passport', 'Validate against GitAgent Passport requirements (Checkpoints 1 & 2)', false) .action(async (options: ValidateOptions) => { const dir = resolve(options.dir); heading('Validating gitagent'); @@ -603,6 +777,40 @@ export const validateCommand = new Command('validate') success('skills/ — valid'); } + // GitAgent Passport Preflight Validation (Checkpoints 1 & 2) + if (options.passport) { + divider(); + heading('GitAgent Passport Preflight Validation'); + + const passportResult = validatePassport(dir); + + // Checkpoint 1: DUTIES & Separation of Duties + if (passportResult.checkpoint1.valid && passportResult.checkpoint1.errors.length === 0) { + success('Checkpoint 1 (DUTIES / Separation of Duties) — valid'); + } else if (!passportResult.checkpoint1.valid) { + error('Checkpoint 1 (DUTIES / Separation of Duties) — invalid'); + passportResult.checkpoint1.errors.forEach(e => error(` ${e}`)); + allValid = false; + } + passportResult.checkpoint1.warnings.forEach(w => warn(` ${w}`)); + totalErrors += passportResult.checkpoint1.errors.length; + totalWarnings += passportResult.checkpoint1.warnings.length; + + // Checkpoint 2: EXPLAINABILITY.md Transparency + if (passportResult.checkpoint2.valid && passportResult.checkpoint2.errors.length === 0) { + success('Checkpoint 2 (EXPLAINABILITY / Transparency) — valid'); + } else if (!passportResult.checkpoint2.valid) { + error('Checkpoint 2 (EXPLAINABILITY / Transparency) — invalid'); + passportResult.checkpoint2.errors.forEach(e => error(` ${e}`)); + allValid = false; + } + passportResult.checkpoint2.warnings.forEach(w => warn(` ${w}`)); + totalErrors += passportResult.checkpoint2.errors.length; + totalWarnings += passportResult.checkpoint2.warnings.length; + } else if (existsSync(join(dir, 'EXPLAINABILITY.md')) || existsSync(join(dir, 'DUTIES.md'))) { + info('Tip: Run with --passport (-p) to verify GitAgent Passport Checkpoints 1 & 2 compliance.'); + } + // Compliance validation if (options.compliance) { divider();