feat: classifyDiffAgainstPlan — pure diff vs plan classifier (VerifyResult)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

__tests__/verify/classify.test.ts

@@ -0,0 +1,126 @@
+import { describe, it, expect } from 'vitest'
+import { classifyDiffAgainstPlan } from '../../src/verify/classify.js'
+
+// Helpers to build minimal unified diff snippets
+function makeDiff(files: string[], linesPerFile = 5): string {
+  return files
+    .map(
+      (f) =>
+        `diff --git a/${f} b/${f}\n--- a/${f}\n+++ b/${f}\n` +
+        Array.from({ length: linesPerFile }, (_, i) => `+added line ${i}`).join('\n'),
+    )
+    .join('\n')
+}
+
+function largeFileDiff(file: string, addedLines = 60): string {
+  return (
+    `diff --git a/${file} b/${file}\n--- a/${file}\n+++ b/${file}\n` +
+    Array.from({ length: addedLines }, (_, i) => `+line ${i}`).join('\n')
+  )
+}
+
+describe('classifyDiffAgainstPlan — empty diff', () => {
+  it('returns EMPTY for blank diff string', () => {
+    const r = classifyDiffAgainstPlan({ diff: '', plan: 'some plan' })
+    expect(r.result).toBe('EMPTY')
+    expect(r.reasoning).toMatch(/geen bestandswijzigingen/i)
+  })
+
+  it('returns EMPTY for diff with no +++ b/ lines', () => {
+    const r = classifyDiffAgainstPlan({ diff: 'Binary files differ\n', plan: 'plan' })
+    expect(r.result).toBe('EMPTY')
+  })
+})
+
+describe('classifyDiffAgainstPlan — no plan baseline', () => {
+  it('returns ALIGNED for null plan with small diff', () => {
+    const r = classifyDiffAgainstPlan({ diff: makeDiff(['src/foo.ts']), plan: null })
+    expect(r.result).toBe('ALIGNED')
+  })
+
+  it('returns ALIGNED for empty string plan with small diff', () => {
+    const r = classifyDiffAgainstPlan({ diff: makeDiff(['src/foo.ts']), plan: '   ' })
+    expect(r.result).toBe('ALIGNED')
+  })
+
+  it('returns DIVERGENT for null plan with large diff (>50 changed lines)', () => {
+    const r = classifyDiffAgainstPlan({ diff: largeFileDiff('src/big.ts', 60), plan: null })
+    expect(r.result).toBe('DIVERGENT')
+  })
+})
+
+describe('classifyDiffAgainstPlan — plan has no extractable paths', () => {
+  const plan = 'Add a new feature. Update the logic. Write tests.'
+
+  it('returns ALIGNED for small diff when plan has no paths', () => {
+    const r = classifyDiffAgainstPlan({ diff: makeDiff(['src/feature.ts']), plan })
+    expect(r.result).toBe('ALIGNED')
+  })
+
+  it('returns DIVERGENT for large diff when plan has no paths', () => {
+    const r = classifyDiffAgainstPlan({ diff: largeFileDiff('src/feature.ts', 60), plan })
+    expect(r.result).toBe('DIVERGENT')
+  })
+})
+
+describe('classifyDiffAgainstPlan — PARTIAL coverage', () => {
+  const plan = 'Edit `src/api.ts` and `src/ui.ts`.'
+
+  it('returns PARTIAL when only one of two plan paths is in diff', () => {
+    const diff = makeDiff(['src/api.ts'])
+    const r = classifyDiffAgainstPlan({ diff, plan })
+    expect(r.result).toBe('PARTIAL')
+    expect(r.reasoning).toMatch(/1\/2/)
+    expect(r.reasoning).toMatch(/ontbrekend/i)
+  })
+
+  it('returns PARTIAL when none of the plan paths are in diff', () => {
+    const diff = makeDiff(['src/unrelated.ts'])
+    const r = classifyDiffAgainstPlan({ diff, plan })
+    expect(r.result).toBe('PARTIAL')
+    expect(r.reasoning).toMatch(/0\/2/)
+  })
+})
+
+describe('classifyDiffAgainstPlan — ALIGNED', () => {
+  it('returns ALIGNED when all plan paths are in diff with ratio < 3', () => {
+    const plan = 'Modify `src/a.ts` and `src/b.ts`.'
+    const diff = makeDiff(['src/a.ts', 'src/b.ts', 'src/c.ts']) // ratio 3/2 = 1.5 < 3
+    const r = classifyDiffAgainstPlan({ diff, plan })
+    expect(r.result).toBe('ALIGNED')
+    expect(r.reasoning).toMatch(/alle 2/i)
+  })
+
+  it('returns ALIGNED for exact 1-to-1 match', () => {
+    const plan = 'Update `src/verify/classify.ts`.'
+    const diff = makeDiff(['src/verify/classify.ts'])
+    const r = classifyDiffAgainstPlan({ diff, plan })
+    expect(r.result).toBe('ALIGNED')
+  })
+
+  it('suffix-matches short plan path against full diff path', () => {
+    const plan = 'Edit `classify.ts` in the verify module.'
+    const diff = makeDiff(['src/verify/classify.ts'])
+    const r = classifyDiffAgainstPlan({ diff, plan })
+    expect(r.result).toBe('ALIGNED')
+  })
+})
+
+describe('classifyDiffAgainstPlan — DIVERGENT (scope creep)', () => {
+  it('returns DIVERGENT when diff has 3x+ more paths than plan', () => {
+    const plan = 'Update `src/a.ts`.'
+    // 1 plan path, 4 diff paths → ratio 4.0 >= 3
+    const diff = makeDiff(['src/a.ts', 'src/b.ts', 'src/c.ts', 'src/d.ts'])
+    const r = classifyDiffAgainstPlan({ diff, plan })
+    expect(r.result).toBe('DIVERGENT')
+    expect(r.reasoning).toMatch(/4\.0x/)
+  })
+
+  it('shows extra file paths in reasoning', () => {
+    const plan = 'Modify `src/core.ts`.'
+    const diff = makeDiff(['src/core.ts', 'src/extra1.ts', 'src/extra2.ts', 'src/extra3.ts', 'src/extra4.ts'])
+    const r = classifyDiffAgainstPlan({ diff, plan })
+    expect(r.result).toBe('DIVERGENT')
+    expect(r.reasoning).toMatch(/extra/i)
+  })
+})

src/verify/classify.ts

@@ -0,0 +1,125 @@
+export type VerifyResultValue = 'ALIGNED' | 'PARTIAL' | 'EMPTY' | 'DIVERGENT'
+
+export interface ClassifyResult {
+  result: VerifyResultValue
+  reasoning: string
+}
+
+// Extract changed file paths from a unified diff ("+++ b/<path>" lines).
+function extractDiffPaths(diff: string): string[] {
+  const paths = new Set<string>()
+  for (const line of diff.split('\n')) {
+    const m = line.match(/^\+\+\+ b\/(.+)$/)
+    if (m && m[1].trim() !== '/dev/null') paths.add(m[1].trim())
+  }
+  return [...paths]
+}
+
+// Extract file paths mentioned in a plan (backtick-quoted, parenthesised, or bullet-list headings).
+function extractPlanPaths(plan: string): string[] {
+  const paths = new Set<string>()
+
+  const backtickRe = /`([^`\s][^`]*[^`\s]|[^`\s])`/g
+  let m: RegExpExecArray | null
+  while ((m = backtickRe.exec(plan)) !== null) {
+    const p = m[1].trim()
+    if ((p.includes('/') || p.includes('.')) && !p.includes(' ') && p.length > 3) paths.add(p)
+  }
+
+  const bulletRe = /^[-*]\s+\*{0,2}([^\s*][^\s]*)\.([a-zA-Z]{1,6})\*{0,2}\s*[:\n]/gm
+  while ((m = bulletRe.exec(plan)) !== null) {
+    paths.add(`${m[1]}.${m[2]}`)
+  }
+
+  return [...paths]
+}
+
+// Path match: exact or suffix match so "classify.ts" matches "src/verify/classify.ts".
+function pathMatches(planPath: string, diffPaths: string[]): boolean {
+  const norm = planPath.replace(/\\/g, '/')
+  return diffPaths.some((dp) => {
+    const ndp = dp.replace(/\\/g, '/')
+    return ndp === norm || ndp.endsWith(`/${norm}`) || norm.endsWith(`/${ndp}`)
+  })
+}
+
+/**
+ * Classify a unified git diff against an implementation plan.
+ * Returns a VerifyResult (ALIGNED|PARTIAL|EMPTY|DIVERGENT) plus human-readable reasoning.
+ *
+ * v1 heuristic — no LLM required:
+ *   - No file changes in diff                              → EMPTY
+ *   - Plan empty, diff ≤50 changed lines                  → ALIGNED (targeted fix)
+ *   - Plan empty, diff >50 changed lines                  → DIVERGENT (too large to trust)
+ *   - Plan paths < 100% covered in diff                   → PARTIAL
+ *   - Plan paths 100% covered, diff ≥3× more paths        → DIVERGENT (scope creep)
+ *   - Plan paths 100% covered, diff <3× more paths        → ALIGNED
+ */
+export function classifyDiffAgainstPlan(opts: {
+  diff: string
+  plan: string | null
+}): ClassifyResult {
+  const { diff, plan } = opts
+
+  const diffPaths = extractDiffPaths(diff)
+  if (diffPaths.length === 0) {
+    return { result: 'EMPTY', reasoning: 'Geen bestandswijzigingen in de diff.' }
+  }
+
+  const changedLines = diff.split('\n').filter((l) => l.startsWith('+') || l.startsWith('-')).length
+
+  if (!plan || plan.trim().length === 0) {
+    if (changedLines > 50) {
+      return {
+        result: 'DIVERGENT',
+        reasoning: `Geen plan-baseline aanwezig; ${changedLines} gewijzigde regels — te groot om als aligned te bestempelen.`,
+      }
+    }
+    return {
+      result: 'ALIGNED',
+      reasoning: `Geen plan-baseline; ${diffPaths.length} bestand(en) gewijzigd — kleine gerichte wijziging.`,
+    }
+  }
+
+  const planPaths = extractPlanPaths(plan)
+
+  if (planPaths.length === 0) {
+    if (changedLines > 50) {
+      return {
+        result: 'DIVERGENT',
+        reasoning: `Plan vermeldt geen specifieke paden; ${changedLines} gewijzigde regels — te groot om als aligned te bestempelen.`,
+      }
+    }
+    return {
+      result: 'ALIGNED',
+      reasoning: `Plan vermeldt geen specifieke paden; ${diffPaths.length} bestand(en) gewijzigd.`,
+    }
+  }
+
+  const covered = planPaths.filter((pp) => pathMatches(pp, diffPaths))
+  const coverage = covered.length / planPaths.length
+  const ratio = diffPaths.length / planPaths.length
+
+  if (coverage < 1) {
+    const missing = planPaths.filter((pp) => !pathMatches(pp, diffPaths))
+    const missingStr = missing.slice(0, 3).join(', ') + (missing.length > 3 ? ` + ${missing.length - 3} meer` : '')
+    return {
+      result: 'PARTIAL',
+      reasoning: `${covered.length}/${planPaths.length} plan-paden aanwezig in diff. Ontbrekend: ${missingStr}.`,
+    }
+  }
+
+  if (ratio >= 3) {
+    const extra = diffPaths.filter((dp) => !planPaths.some((pp) => pathMatches(pp, [dp])))
+    const extraStr = extra.slice(0, 3).join(', ') + (extra.length > 3 ? ` + ${extra.length - 3} meer` : '')
+    return {
+      result: 'DIVERGENT',
+      reasoning: `Alle ${planPaths.length} plan-paden aanwezig, maar diff bevat ${diffPaths.length} paden (${ratio.toFixed(1)}x). Extra: ${extraStr}.`,
+    }
+  }
+
+  return {
+    result: 'ALIGNED',
+    reasoning: `Alle ${planPaths.length} plan-paden aanwezig in diff (${diffPaths.length} totaal; ${ratio.toFixed(1)}x).`,
+  }
+}