PBI-9 + PBI-47: worktree foundation, product-worktrees, P0 fixes, PAUSED flow

Adds two interlocking PBIs:

PBI-9 — Worktree foundation + persistent product-worktrees for idea-jobs
  - src/git/worktree-paths.ts: centralised root + skip-set + lock-path helpers
  - src/git/file-lock.ts: proper-lockfile wrapper, deadlock-safe ordered acquire
  - src/git/product-worktree.ts: detached-HEAD worktree per product, .scratch/
    excluded via git rev-parse --git-path (handles linked .git file)
  - src/git/job-locks.ts: setupProductWorktrees + releaseLocksOnTerminal
  - wait-for-job.ts: idea-branch wires product-worktrees for IDEA_GRILL/MAKE_PLAN
  - update-job-status.ts + pbi-cascade.ts + stale-reset: release on all four
    server-side terminal transitions (DONE/FAILED/CANCELLED/stale)
  - cleanup-my-worktrees: skip _products/ + *.lock
  - README: worktrees section with single-host invariant + advisory-lock path

PBI-47 — Sprint-flow P0 corrections + PAUSED flow with rich pause_context
  - prisma schema: ClaudeJob.{base_sha,head_sha} + SprintRun.pause_context
  - tryClaimJob captures base_sha; prepareDoneUpdate captures head_sha
  - verify-task-against-plan diffs vs base_sha (no more origin/main fallback);
    rejects with MISSING_BASE_SHA when null — fixes per-task verify-scope P0
  - pr.ts: createPullRequest enableAutoMerge default false; new
    enableAutoMergeOnPr with --match-head-commit guard + 5-category typed
    EnableAutoMergeResult — fixes STORY auto-merge timing P0
  - src/flow/{effects,worktree-lease,pr-flow,sprint-run}.ts: pure transition
    modules + idempotent declarative effects executor
  - update-job-status: STORY auto-merge fires only on the last task of the
    story (story.status === DONE), with head_sha as merge guard; MERGE_CONFLICT
    routes to sprint-run flow which produces CREATE_CLAUDE_QUESTION +
    SET_SPRINT_RUN_STATUS effects with rich pause_context

Tests: 31 test files, 242 passing. Pure-transition tests cover STORY 3-tasks
auto-merge timing, SPRINT draft→ready, MERGE_CONFLICT pause/resume, file-lock
deadlock prevention, worktree-lease lifecycle, delete-only verify (ALIGNED),
per-job verify scope (base_sha isolation), 5-category auto-merge errors.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/tools/cleanup-my-worktrees.ts

@@ -1,12 +1,11 @@
 import { z } from 'zod'
 import * as fs from 'node:fs/promises'
-import * as path from 'node:path'
-import * as os from 'node:os'
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
 import { prisma } from '../prisma.js'
 import { requireWriteAccess } from '../auth.js'
 import { toolJson, withToolErrors } from '../errors.js'
 import { removeWorktreeForJob } from '../git/worktree.js'
+import { getWorktreeRoot, SYSTEM_WORKTREE_DIRS } from '../git/worktree-paths.js'
 import { resolveRepoRoot } from './wait-for-job.js'
 
 const TERMINAL_STATUSES = new Set(['DONE', 'FAILED', 'CANCELLED'])
@@ -15,16 +14,20 @@ const ACTIVE_STATUSES = new Set(['QUEUED', 'CLAIMED', 'RUNNING'])
 const inputSchema = z.object({})
 
 export async function getWorktreeParent(): Promise<string> {
-  return (
-    process.env.SCRUM4ME_AGENT_WORKTREE_DIR ??
-    path.join(os.homedir(), '.scrum4me-agent-worktrees')
-  )
+  return getWorktreeRoot()
 }
 
 export async function listWorktreeJobIds(worktreeParent: string): Promise<string[]> {
   try {
     const entries = await fs.readdir(worktreeParent, { withFileTypes: true })
-    return entries.filter((e) => e.isDirectory()).map((e) => e.name)
+    return entries
+      .filter(
+        (e) =>
+          e.isDirectory()
+          && !SYSTEM_WORKTREE_DIRS.has(e.name)
+          && !e.name.endsWith('.lock'),
+      )
+      .map((e) => e.name)
   } catch {
     return []
   }

src/tools/update-job-status.ts

@@ -11,11 +11,30 @@ import { prisma } from '../prisma.js'
 import { requireWriteAccess } from '../auth.js'
 import { toolJson, toolError, withToolErrors } from '../errors.js'
 import { removeWorktreeForJob } from '../git/worktree.js'
+import { getWorktreeRoot } from '../git/worktree-paths.js'
+import { releaseLocksOnTerminal } from '../git/job-locks.js'
 import { resolveRepoRoot } from './wait-for-job.js'
 import { pushBranchForJob } from '../git/push.js'
 import { createPullRequest, markPullRequestReady } from '../git/pr.js'
 import { cancelPbiOnFailure } from '../cancel/pbi-cascade.js'
 import { propagateStatusUpwards } from '../lib/tasks-status-update.js'
+import { transition as prFlowTransition } from '../flow/pr-flow.js'
+import { transition as sprintRunTransition } from '../flow/sprint-run.js'
+import { executeEffects } from '../flow/effects.js'
+import { execFile as execFileCb } from 'node:child_process'
+import { promisify } from 'node:util'
+
+const execGh = promisify(execFileCb)
+
+async function fetchConflictFiles(prUrl: string): Promise<string[]> {
+  try {
+    const { stdout } = await execGh('gh', ['pr', 'view', prUrl, '--json', 'files'])
+    const parsed = JSON.parse(stdout) as { files?: Array<{ path: string }> }
+    return parsed.files?.map((f) => f.path) ?? []
+  } catch {
+    return []
+  }
+}
 
 const inputSchema = z.object({
   job_id: z.string().min(1),
@@ -85,26 +104,37 @@ export type DoneUpdatePlan = {
   branchOverride: string | undefined
   errorOverride: string | undefined
   skipWorktreeCleanup: boolean
+  headSha: string | undefined
 }
 
 export async function prepareDoneUpdate(
   jobId: string,
   branch: string | undefined,
 ): Promise<DoneUpdatePlan> {
-  const worktreeDir =
-    process.env.SCRUM4ME_AGENT_WORKTREE_DIR ?? path.join(os.homedir(), '.scrum4me-agent-worktrees')
+  const worktreeDir = getWorktreeRoot()
   const worktreePath = path.join(worktreeDir, jobId)
   const branchName = branch ?? `feat/job-${jobId.slice(-8)}`
 
   const pushResult = await pushBranchForJob({ worktreePath, branchName })
 
   if (pushResult.pushed) {
+    let headSha: string | undefined
+    try {
+      const { execFile } = await import('node:child_process')
+      const { promisify } = await import('node:util')
+      const exec = promisify(execFile)
+      const { stdout } = await exec('git', ['rev-parse', 'HEAD'], { cwd: worktreePath })
+      headSha = stdout.trim()
+    } catch (err) {
+      console.warn(`[prepareDoneUpdate] failed to resolve HEAD sha for job ${jobId}:`, err)
+    }
     return {
       dbStatus: 'DONE',
       pushedAt: new Date(),
       branchOverride: branchName,
       errorOverride: undefined,
       skipWorktreeCleanup: false,
+      headSha,
     }
   }
 
@@ -115,6 +145,7 @@ export async function prepareDoneUpdate(
       branchOverride: undefined,
       errorOverride: undefined,
       skipWorktreeCleanup: false,
+      headSha: undefined,
     }
   }
 
@@ -126,6 +157,7 @@ export async function prepareDoneUpdate(
     branchOverride: undefined,
     errorOverride: `push failed (${pushResult.reason}): ${snippet}`,
     skipWorktreeCleanup: true,
+    headSha: undefined,
   }
 }
 
@@ -376,6 +408,7 @@ export function registerUpdateJobStatusTool(server: McpServer) {
         let branchToWrite = branch
         let errorToWrite = error
         let skipWorktreeCleanup = false
+        let headShaToWrite: string | undefined
 
         if (status === 'done') {
           // M12: idea-jobs hebben geen task/plan_snapshot/branch — skip de
@@ -401,6 +434,7 @@ export function registerUpdateJobStatusTool(server: McpServer) {
             if (plan.branchOverride !== undefined) branchToWrite = plan.branchOverride
             if (plan.errorOverride !== undefined) errorToWrite = plan.errorOverride
             skipWorktreeCleanup = plan.skipWorktreeCleanup
+            headShaToWrite = plan.headSha
           }
         }
 
@@ -414,9 +448,7 @@ export function registerUpdateJobStatusTool(server: McpServer) {
           job.kind === 'TASK_IMPLEMENTATION' &&
           job.task_id
         ) {
-          const worktreeDir =
-            process.env.SCRUM4ME_AGENT_WORKTREE_DIR ??
-            path.join(os.homedir(), '.scrum4me-agent-worktrees')
+          const worktreeDir = getWorktreeRoot()
           prUrl = await maybeCreateAutoPr({
             jobId: job_id,
             productId: job.product_id,
@@ -443,6 +475,7 @@ export function registerUpdateJobStatusTool(server: McpServer) {
             ...(summary !== undefined ? { summary } : {}),
             ...(errorToWrite !== undefined ? { error: errorToWrite } : {}),
             ...(prUrl !== null ? { pr_url: prUrl } : {}),
+            ...(headShaToWrite !== undefined ? { head_sha: headShaToWrite } : {}),
             ...(model_id !== undefined ? { model_id } : {}),
             ...(input_tokens !== undefined ? { input_tokens } : {}),
             ...(output_tokens !== undefined ? { output_tokens } : {}),
@@ -468,6 +501,7 @@ export function registerUpdateJobStatusTool(server: McpServer) {
         // sibling-cancel binnen dezelfde SprintRun af bij FAILED.
         // Idea-jobs hebben geen task_id en worden hier overgeslagen.
         let sprintRunBecameDone = false
+        let storyBecameDone = false
         if (
           (actualStatus === 'done' || actualStatus === 'failed') &&
           job.kind === 'TASK_IMPLEMENTATION' &&
@@ -479,6 +513,7 @@ export function registerUpdateJobStatusTool(server: McpServer) {
               actualStatus === 'done' ? 'DONE' : 'FAILED',
             )
             sprintRunBecameDone = actualStatus === 'done' && propagation.sprintRunChanged
+            storyBecameDone = actualStatus === 'done' && propagation.storyChanged
           } catch (err) {
             console.warn(
               `[update_job_status] propagateStatusUpwards error for task ${job.task_id}:`,
@@ -487,6 +522,72 @@ export function registerUpdateJobStatusTool(server: McpServer) {
           }
         }
 
+        // PBI-47 (P0): STORY-mode auto-merge timing fix.
+        // Only enable auto-merge when this DONE was the *last* task of a STORY
+        // (story.status flipped to DONE) and pr_strategy === STORY. The
+        // pr-flow transition emits ENABLE_AUTO_MERGE with the head_sha guard.
+        if (
+          storyBecameDone &&
+          updated.pr_url &&
+          headShaToWrite &&
+          job.kind === 'TASK_IMPLEMENTATION'
+        ) {
+          const storyCtx = await prisma.claudeJob.findUnique({
+            where: { id: job_id },
+            select: {
+              task: { select: { story: { select: { status: true } } } },
+              sprint_run: { select: { pr_strategy: true } },
+            },
+          })
+          if (
+            storyCtx?.sprint_run?.pr_strategy === 'STORY'
+            && storyCtx.task?.story.status === 'DONE'
+          ) {
+            const result = prFlowTransition(
+              { kind: 'pr_opened', strategy: 'STORY', prUrl: updated.pr_url },
+              {
+                type: 'STORY_COMPLETED',
+                storyId: '',
+                headSha: headShaToWrite,
+              },
+            )
+            const outcomes = await executeEffects(result.effects)
+            // PBI-47 (C2): route MERGE_CONFLICT to sprint-run flow → PAUSED.
+            // Other reasons (CHECKS_FAILED, GH_AUTH_ERROR, AUTO_MERGE_NOT_ALLOWED, UNKNOWN)
+            // remain warnings; CHECKS_FAILED is already covered by the task-FAIL cascade.
+            for (const o of outcomes) {
+              if (o.effect === 'ENABLE_AUTO_MERGE' && !o.ok) {
+                console.warn(
+                  `[update_job_status] auto-merge fail for ${updated.pr_url}: ${o.reason} ${o.stderr.slice(0, 200)}`,
+                )
+                if (o.reason === 'MERGE_CONFLICT') {
+                  const sprintRunId = await prisma.claudeJob
+                    .findUnique({
+                      where: { id: job_id },
+                      select: { sprint_run_id: true },
+                    })
+                    .then((j) => j?.sprint_run_id)
+                  if (sprintRunId) {
+                    const conflictFiles = await fetchConflictFiles(updated.pr_url)
+                    const conflictResult = sprintRunTransition(
+                      { kind: 'running', sprintRunId },
+                      {
+                        type: 'MERGE_CONFLICT',
+                        prUrl: updated.pr_url,
+                        prHeadSha: headShaToWrite ?? '',
+                        conflictFiles,
+                        resumeInstructions:
+                          'Resolve the conflict on this branch, push, then resume the sprint via the UI.',
+                      },
+                    )
+                    await executeEffects(conflictResult.effects)
+                  }
+                }
+              }
+            }
+          }
+        }
+
         // SPRINT-mode: bij sprint-DONE de draft-PR ready-for-review zetten.
         // Mens reviewt + mergt zelf — geen auto-merge in deze modus.
         if (sprintRunBecameDone && updated.pr_url) {
@@ -581,6 +682,12 @@ export function registerUpdateJobStatusTool(server: McpServer) {
           await cancelPbiOnFailure(job_id)
         }
 
+        // PBI-9: release product-worktree locks on terminal transitions.
+        // No-op for jobs without registered locks (i.e. TASK_IMPLEMENTATION).
+        if (actualStatus === 'done' || actualStatus === 'failed') {
+          await releaseLocksOnTerminal(job_id)
+        }
+
         const queueCount = await prisma.claudeJob.count({
           where: { user_id: userId, status: 'QUEUED' },
         })

src/tools/verify-task-against-plan.ts

@@ -15,8 +15,15 @@ const inputSchema = z.object({
   worktree_path: z.string().min(1),
 })
 
-export async function getDiffInWorktree(worktreePath: string): Promise<string> {
-  const { stdout } = await exec('git', ['diff', 'origin/main...HEAD'], { cwd: worktreePath })
+export async function getDiffInWorktree(
+  worktreePath: string,
+  baseSha?: string,
+): Promise<string> {
+  // PBI-47 (P0): when base_sha is provided, diff against the per-job base
+  // captured at claim-time so verify only sees the current task's changes.
+  // Falls back to origin/main only for legacy callers without base_sha.
+  const range = baseSha ? `${baseSha}...HEAD` : 'origin/main...HEAD'
+  const { stdout } = await exec('git', ['diff', range], { cwd: worktreePath })
   return stdout
 }
 
@@ -58,7 +65,7 @@ export function registerVerifyTaskAgainstPlanTool(server: McpServer) {
               where: { status: { in: ['CLAIMED', 'RUNNING'] } },
               orderBy: { created_at: 'desc' },
               take: 1,
-              select: { id: true, plan_snapshot: true },
+              select: { id: true, plan_snapshot: true, base_sha: true },
             },
           },
         })
@@ -67,9 +74,19 @@ export function registerVerifyTaskAgainstPlanTool(server: McpServer) {
 
         const activeJob = task.claude_jobs[0] ?? null
 
+        // PBI-47 (P0): require base_sha so diff is scoped to this job's work,
+        // not the full origin/main...HEAD which would include sibling commits
+        // on a reused story/sprint branch.
+        if (activeJob && !activeJob.base_sha) {
+          return toolError(
+            'MISSING_BASE_SHA: This claim has no base_sha. '
+              + 'Re-claim the task (cancel + wait_for_job) so a fresh base_sha is captured.',
+          )
+        }
+
         let diff: string
         try {
-          diff = await getDiffInWorktree(worktree_path)
+          diff = await getDiffInWorktree(worktree_path, activeJob?.base_sha ?? undefined)
         } catch (err) {
           return toolError(
             `git diff failed in worktree (${worktree_path}): ${(err as Error).message ?? 'unknown error'}`,

src/tools/wait-for-job.ts

@@ -7,10 +7,15 @@ import { Client } from 'pg'
 import * as fs from 'node:fs/promises'
 import * as os from 'node:os'
 import * as path from 'node:path'
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
 import { prisma } from '../prisma.js'
+
+const execFileP = promisify(execFile)
 import { requireWriteAccess } from '../auth.js'
 import { toolJson, toolError, withToolErrors } from '../errors.js'
 import { createWorktreeForJob } from '../git/worktree.js'
+import { setupProductWorktrees, releaseLocksOnTerminal } from '../git/job-locks.js'
 
 /** Parse `https://github.com/<owner>/<name>(.git)?` → `<name>`. */
 export function repoNameFromUrl(repoUrl: string | null | undefined): string | null {
@@ -181,6 +186,26 @@ export async function attachWorktreeToJob(
       branchName,
       reuseBranch: reused,
     })
+
+    // PBI-47 (P0): capture base_sha so verify_task_against_plan can diff
+    // against the claim-time HEAD instead of origin/main. For reused branches
+    // (siblings already pushed), base_sha = SHA of the worktree HEAD now.
+    // For fresh branches, base_sha = origin/main HEAD which createWorktreeForJob
+    // just checked out.
+    let baseSha: string | null = null
+    try {
+      const { stdout } = await execFileP('git', ['rev-parse', 'HEAD'], { cwd: worktreePath })
+      baseSha = stdout.trim()
+    } catch (err) {
+      console.warn(`[attachWorktreeToJob] failed to resolve base_sha for ${jobId}:`, err)
+    }
+    if (baseSha) {
+      await prisma.claudeJob.update({
+        where: { id: jobId },
+        data: { base_sha: baseSha },
+      })
+    }
+
     return { worktree_path: worktreePath, branch_name: actualBranch, reused_branch: reused }
   } catch (err) {
     await rollbackClaim(jobId)
@@ -234,6 +259,11 @@ export async function resetStaleClaimedJobs(userId: string): Promise<void> {
 
   if (failedRows.length === 0 && requeuedRows.length === 0) return
 
+  // PBI-9: release any product-worktree locks held by these stale jobs.
+  // No-op for jobs without registered locks (TASK_IMPLEMENTATION).
+  for (const j of failedRows) await releaseLocksOnTerminal(j.id)
+  for (const j of requeuedRows) await releaseLocksOnTerminal(j.id)
+
   // Notify UI via SSE for each transition (best-effort)
   try {
     const pg = new Client({ connectionString: process.env.DATABASE_URL })
@@ -371,6 +401,9 @@ async function getFullJobContext(jobId: string) {
       idea: {
         include: {
           pbi: { select: { id: true, code: true, title: true } },
+          secondary_products: {
+            include: { product: { select: { id: true, repo_url: true } } },
+          },
         },
       },
       product: { select: { id: true, name: true, repo_url: true, definition_of_done: true } },
@@ -384,6 +417,21 @@ async function getFullJobContext(jobId: string) {
     if (!job.idea) return null
     const { idea } = job
     const { getIdeaPromptText } = await import('../lib/idea-prompts.js')
+
+    // Setup persistent product-worktrees for this idea-job (PBI-9).
+    // Primary product is gated by repo_url via resolveRepoRoot returning null.
+    // Secondary products from IdeaProduct[] need explicit repo_url filter.
+    const involvedProductIds: string[] = []
+    if (idea.product_id) involvedProductIds.push(idea.product_id)
+    for (const ip of idea.secondary_products ?? []) {
+      if (ip.product?.repo_url && !involvedProductIds.includes(ip.product_id)) {
+        involvedProductIds.push(ip.product_id)
+      }
+    }
+    const worktrees = involvedProductIds.length > 0
+      ? await setupProductWorktrees(job.id, involvedProductIds, (pid) => resolveRepoRoot(pid))
+      : []
+
     return {
       job_id: job.id,
       kind: job.kind,
@@ -408,6 +456,11 @@ async function getFullJobContext(jobId: string) {
       repo_url: job.product.repo_url,
       prompt_text: getIdeaPromptText(job.kind),
       branch_suggestion: `feat/idea-${idea.code.toLowerCase()}-${job.kind === 'IDEA_GRILL' ? 'grill' : 'plan'}`,
+      product_worktrees: worktrees.map((w) => ({
+        product_id: w.productId,
+        worktree_path: w.worktreePath,
+      })),
+      primary_worktree_path: worktrees[0]?.worktreePath ?? null,
     }
   }