From 30d6aad2c09460ef7070cba2e2f454907274c391 Mon Sep 17 00:00:00 2001 From: Val Alexander Date: Sun, 18 Oct 2026 11:42:10 -0500 Subject: [PATCH 1/4] Add the request variants to the fidelity path Four cases in e2e/conversation-fidelity.spec.ts, through the real app: - A native withdrawal: the Claude fixture takes its own request back (control_cancel_request). The card closes as withdrawn, no answer reaches the CLI, a late answer is refused, and the file is unchanged. - The desktop and the phone answer at once, over the bridge and the companion's HTTPS. Exactly one is accepted, the CLI reads one decision, the card names who gave it, and the file matches the winner. - With the journal on, a phone that loaded the open card reconnects after the desktop's decision. Its delta shows the card answered by the desktop and sent, its late answer is a 409, and every count stays at one. - A clarifying question inside the path, from a codex-cli 0.163.0-shaped fixture: answered on its card, heard once, followed by a real edit and check, saved to Notes, and inert after a restart that re-runs nothing. The fixture now counts every decision the CLI reads, so the earlier cases assert that too. The golden path is marked slow: since #301 it also takes 18 reference captures and relaunches, and it ran past 90 s on a Mac at load average 100. These cases found the companion's 404 for threads started on the Mac, fixed in the CF-6 branch. Refs #281 Coven-Harness: Claude Code / claude-opus-5-5 --- .../desktop/e2e/conversation-fidelity.spec.ts | 229 +++++++++++++++++- 1 file changed, 222 insertions(+), 7 deletions(-) diff --git a/apps/desktop/e2e/conversation-fidelity.spec.ts b/apps/desktop/e2e/conversation-fidelity.spec.ts index b52df515..6504f7d7 100644 --- a/apps/desktop/e2e/conversation-fidelity.spec.ts +++ b/apps/desktop/e2e/conversation-fidelity.spec.ts @@ -1,5 +1,7 @@ import { test, expect, type ElectronApplication, type Page } from "@playwright/test"; +import crypto from "node:crypto"; import fs from "node:fs"; +import https from "node:https"; import path from "node:path"; import { currentRow, items, launch, resizeWindow, seedHome, setTheme, tid } from "./support"; @@ -10,7 +12,7 @@ import { currentRow, items, launch, resizeWindow, seedHome, setTheme, tid } from * typed outcomes and note revisions, not on success text. Not live-provider acceptance. */ -type Variant = "golden" | "deny" | "fail" | "stop"; +type Variant = "golden" | "deny" | "fail" | "stop" | "withdraw"; const BEFORE = "hello\n"; const AFTER = "hello, fidelity\n"; @@ -30,6 +32,7 @@ const rl = require('node:readline').createInterface({ input: process.stdin }); rl.on('close', () => process.exit(0)); rl.on('line', (line) => { const msg = JSON.parse(line); + if (msg.type === 'control_response') count('decisions'); if (msg.type === 'control_response' && pending) { const go = pending; pending = null; go(msg.response.response.behavior === 'allow'); return; } if (msg.type !== 'user') return; emit({ type: 'system', subtype: 'init', session_id: 'fidelity-session', claude_code_version: '2.1.302', model: 'fixture' }); @@ -71,11 +74,18 @@ rl.on('line', (line) => { if (${JSON.stringify(variant)} === 'stop') setTimeout(finish, 60000); else finish(); }; emit({ type: 'control_request', request_id: 'perm-1', request: { subtype: 'can_use_tool', tool_name: 'Edit', tool_use_id: 'edit-1', input: edit } }); + // The CLI takes its own request back, as Claude Code does when it no longer needs the answer. + if (${JSON.stringify(variant)} === 'withdraw') setTimeout(() => { + pending = null; + emit({ type: 'control_cancel_request', request_id: 'perm-1' }); + emit({ type: 'assistant', message: { id: 'm-final', content: [{ type: 'text', text: 'Left the file as it was.' }] } }); + emit({ type: 'result', subtype: 'success', is_error: false, session_id: 'fidelity-session', result: 'Left the file as it was.', num_turns: 2 }); + }, 1500); }); `; } -async function open(variant: Variant): Promise<{ app: ElectronApplication; page: Page; home: string; repo: string; counts: string; read: () => Record }> { +async function open(variant: Variant, opts: { journal?: boolean } = {}): Promise<{ app: ElectronApplication; page: Page; home: string; repo: string; counts: string; read: () => Record }> { const { home, repo } = seedHome({ default_backend: "claude" }); fs.writeFileSync(path.join(repo, "greeting.txt"), BEFORE); const counts = path.join(home, "fixture-counts.json"); @@ -85,7 +95,7 @@ async function open(variant: Variant): Promise<{ app: ElectronApplication; page: const state = JSON.parse(fs.readFileSync(statePath, "utf8")); state.settings.claude_bin = cli; fs.writeFileSync(statePath, JSON.stringify(state)); - const { app, page } = await launch(home); + const { app, page } = await withJournal(Boolean(opts.journal), () => launch(home)); await page.keyboard.press("Meta+n"); await expect(tid(page, "composer-input")).toBeFocused(); await tid(page, "composer-input").fill("Change the greeting to mention fidelity and check it"); @@ -97,6 +107,35 @@ async function open(variant: Variant): Promise<{ app: ElectronApplication; page: const approval = (page: Page) => items(page, "approval").filter({ hasText: "Edit" }); +/** Launches with the event journal on or off; the env is read at launch. */ +async function withJournal(on: boolean, start: () => Promise): Promise { + const before = process.env.MODEX_EVENT_JOURNAL; + if (on) process.env.MODEX_EVENT_JOURNAL = "1"; else delete process.env.MODEX_EVENT_JOURNAL; + try { return await start(); } finally { if (before === undefined) delete process.env.MODEX_EVENT_JOURNAL; else process.env.MODEX_EVENT_JOURNAL = before; } +} + +/** Turns on the iPhone companion and returns a phone: HTTPS with the pinned certificate and token, as the app makes them. */ +async function phone(page: Page, home: string) { + const status = await page.evaluate(() => window.modex!.invoke("companion:start", undefined)) as { pairingUri?: string }; + const pairing = JSON.parse(Buffer.from(new URL(status.pairingUri!).searchParams.get("data")!, "base64url").toString()) as { url: string; token: string; fingerprint: string }; + return (route: string, method = "GET", body?: unknown) => new Promise<{ status: number; body: any }>((resolve, reject) => { + const req = https.request(new URL(route, pairing.url), { + method, headers: { authorization: `Bearer ${pairing.token}`, ...(body ? { "content-type": "application/json" } : {}) }, + ca: fs.readFileSync(path.join(home, "companion", "cert.pem")), + checkServerIdentity: (_host, cert) => crypto.createHash("sha256").update(cert.raw).digest("hex") === pairing.fingerprint ? undefined : new Error("unpinned"), + }, (res) => { + let text = ""; + res.on("data", (chunk) => { text += chunk; }); + res.on("end", () => resolve({ status: res.statusCode!, body: JSON.parse(text) })); + }); + req.on("error", reject); + req.end(body ? JSON.stringify(body) : undefined); + }); +} +type Card = { id: string; kind: string; answer?: string; answeredBy?: string; delivery?: string; withdrawn?: string }; +const cards = async (page: Page, threadId: string, kind = "approval") => + ((await page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId)) as Card[]).filter((item) => item.kind === kind); + /** * Reference captures (docs/screenshots/conversation-fidelity/): one state in every theme, at the 900×600 * minimum and at a fixed wider viewport, each with no sideways scroll. MODEX_SCREENSHOTS= keeps them. @@ -129,6 +168,8 @@ const close = async (run: { app: ElectronApplication; home: string; repo: string }; test("golden path: plan, approval, a real edit and check, the answer, Save to Notes, Open, and a restart that restores without re-running", async () => { + // It also takes 18 reference captures across three states, nine theme switches and a relaunch. + test.slow(); let run = await open("golden"); try { await expect(approval(run.page)).toBeVisible(); @@ -138,7 +179,7 @@ test("golden path: plan, approval, a real edit and check, the answer, Save to No await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); await capture(run, "completed"); expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER); - expect(run.read()).toEqual({ edits: 1, checksStarted: 1, checksFinished: 1 }); + expect(run.read()).toEqual({ decisions: 1, edits: 1, checksStarted: 1, checksFinished: 1 }); // Typed outcomes, the reported plan and the final answer, as recorded rather than as prose says. await expect(tid(items(run.page, "plan"), "plan-step")).toHaveText(["Reported done Edit greeting.txt", "Reported done Check the file"]); await tid(run.page, "turn-label").last().click(); @@ -171,7 +212,7 @@ test("golden path: plan, approval, a real edit and check, the answer, Save to No run = { ...run, app: relaunched.app, page: relaunched.page }; const after = await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId); expect(after).toEqual(before); - expect(run.read()).toEqual({ edits: 1, checksStarted: 1, checksFinished: 1 }); + expect(run.read()).toEqual({ decisions: 1, edits: 1, checksStarted: 1, checksFinished: 1 }); expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER); await run.page.locator(`[data-testid="thread-row"][data-thread-id="${threadId}"] button.thread-main`).click(); await expect(run.page.locator('.turn-content > div[data-final="true"]').last()).toContainText("Changed the greeting; the check passed."); @@ -185,7 +226,7 @@ test("a denial leaves the file as it was and runs nothing", async () => { await approval(run.page).getByRole("button", { name: "Deny" }).click(); await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(BEFORE); - expect(run.read()).toEqual({}); + expect(run.read()).toEqual({ decisions: 1 }); await tid(run.page, "turn-label").last().click(); await expect(tid(items(run.page, "tool").filter({ hasText: "edit" }).first(), "tool-meta")).toContainText("denied"); } finally { await close(run); } @@ -215,7 +256,181 @@ test("Stop while a check runs ends it as stopped; the check never reports, and n await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); const check = items(run.page, "tool").filter({ hasText: "grep -q fidelity" }); await expect(tid(check, "tool-meta")).toContainText("stopped"); - expect(run.read()).toEqual({ edits: 1, checksStarted: 1 }); + expect(run.read()).toEqual({ decisions: 1, edits: 1, checksStarted: 1 }); await expect(run.page.getByText("Stopped.")).toBeVisible(); } finally { await close(run); } }); + +// CF-3's request variants (#276, #279): a native withdrawal, two clients answering at once, a reconnect after +// the decision was sent, and a clarifying question inside the path. `decisions` counts every answer the CLI read. + +test("a native withdrawal closes the card, sends no answer, and leaves the file as it was", async () => { + const run = await open("withdraw"); + try { + await expect(approval(run.page)).toBeVisible(); + await expect(tid(approval(run.page), "approval-answer")).toHaveText("Withdrawn — the agent no longer needs an answer", { timeout: 15_000 }); + await expect(approval(run.page).getByRole("button", { name: "Approve" })).toHaveCount(0); + await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); + const threadId = (await currentRow(run.page).getAttribute("data-thread-id"))!; + const [card] = await cards(run.page, threadId); + expect([card!.withdrawn, card!.answeredBy]).toEqual(["native", undefined]); + expect(await run.page.evaluate(({ id, item }) => window.modex!.invoke("thread:answer", { threadId: id, itemId: item, answer: "yes" }), { id: threadId, item: card!.id })).toEqual({ ok: false, reason: "withdrawn" }); + expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(BEFORE); + expect(run.read()).toEqual({}); + } finally { await close(run); } +}); + +test("the desktop and the phone answer at once: exactly one decision reaches the CLI, and the card names who gave it", async () => { + const run = await open("golden"); + try { + const request = await phone(run.page, run.home); + await expect(approval(run.page)).toBeVisible(); + const threadId = (await currentRow(run.page).getAttribute("data-thread-id"))!; + const [open] = await cards(run.page, threadId); + // Both in flight at once: the phone over HTTPS, the desktop through the bridge its Approve button uses. + const [fromPhone, fromDesktop] = await Promise.all([ + request(`/v1/threads/${threadId}/answer`, "POST", { itemId: open!.id, answer: "no" }), + run.page.evaluate(({ id, item }) => window.modex!.invoke("thread:answer", { threadId: id, itemId: item, answer: "yes" }), { id: threadId, item: open!.id }) as Promise<{ ok: boolean; reason?: string }>, + ]); + await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); + const [card] = await cards(run.page, threadId); + expect([fromPhone.status === 200, fromDesktop.ok].filter(Boolean)).toHaveLength(1); + expect(run.read().decisions).toBe(1); + expect(card!.delivery).toBe("sent"); + if (card!.answeredBy === "phone") { + expect([fromPhone.status, fromDesktop, card!.answer]).toEqual([200, { ok: false, reason: "already-resolved" }, "no"]); + expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(BEFORE); + await expect(tid(approval(run.page), "answer-note")).toHaveText("Answered on your phone."); + } else { + expect([card!.answeredBy, fromDesktop.ok, fromPhone.status, card!.answer]).toEqual(["desktop", true, 409, "yes"]); + expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER); + } + } finally { await close(run); } +}); + +test("with the journal on, a phone that reconnects after the decision was sent sees it answered, and its late answer is refused", async () => { + const run = await open("golden", { journal: true }); + try { + const request = await phone(run.page, run.home); + await expect(approval(run.page)).toBeVisible(); + const threadId = (await currentRow(run.page).getAttribute("data-thread-id"))!; + // The phone loads the thread while the card is open, then sleeps through the decision. + const before = (await request(`/v1/snapshot?threadId=${threadId}`)).body as { items: Card[]; journal?: { id: string; cursor: number } }; + expect(before.journal, "the journal gives the phone a cursor").toBeTruthy(); + const card = before.items.find((item) => item.kind === "approval")!; + expect(card.answer).toBeUndefined(); + await approval(run.page).getByRole("button", { name: "Approve" }).click(); + await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); + // It wakes and asks for what changed since its cursor. + const after = (await request(`/v1/snapshot?threadId=${threadId}&journal=${before.journal!.id}&after=${before.journal!.cursor}`)).body as { items: Card[]; delta?: true }; + expect(after.delta).toBe(true); + const changed = after.items.find((item) => item.id === card.id)!; + expect([changed.answer, changed.answeredBy, changed.delivery]).toEqual(["yes", "desktop", "sent"]); + // A tap queued before the sleep is refused, and the CLI hears nothing more. + const late = await request(`/v1/threads/${threadId}/answer`, "POST", { itemId: card.id, answer: "no" }); + expect(late.status, JSON.stringify({ threadId, body: late.body })).toBe(409); + expect(run.read()).toEqual({ decisions: 1, edits: 1, checksStarted: 1, checksFinished: 1 }); + } finally { await close(run); } +}); + +/** A codex-cli 0.163.0-shaped app-server that asks which greeting, then edits `greeting.txt` for real and checks it. */ +function fakeCodex(repo: string, counts: string): string { + const file = path.join(repo, "greeting.txt"); + return `#!${process.execPath} +const fs = require('node:fs'); +const { spawnSync } = require('node:child_process'); +if (process.argv.includes('--version')) { console.log('codex-cli 0.163.0'); process.exit(0); } +if (process.argv.includes('login') || process.argv.includes('auth')) { console.log('Logged in using ChatGPT'); process.exit(0); } +const emit = (o) => process.stdout.write(JSON.stringify(o) + '\\n'); +const count = (key) => { const c = fs.existsSync(${JSON.stringify(counts)}) ? JSON.parse(fs.readFileSync(${JSON.stringify(counts)}, 'utf8')) : {}; c[key] = (c[key] || 0) + 1; fs.writeFileSync(${JSON.stringify(counts)}, JSON.stringify(c)); }; +let turn = 0, open = null; +const cwds = new Map(); +require('node:readline').createInterface({ input: process.stdin }).on('line', (line) => { + const msg = JSON.parse(line); + const p = msg.params || {}; + if (msg.method === 'initialize') return emit({ id: msg.id, result: { userAgent: 'codex-fidelity/0.163.0' } }); + if (msg.method === 'initialized') return; + if (msg.method === 'account/read') return emit({ id: msg.id, result: { account: { type: 'chatgpt' }, requiresOpenaiAuth: true } }); + if (msg.method === 'model/list') return emit({ id: msg.id, result: { data: [{ id: 'fixture', model: 'fixture', displayName: 'Fixture', isDefault: true }], nextCursor: null } }); + if (msg.method === 'thread/start' || msg.method === 'thread/resume') { const id = p.threadId || 'thr-' + cwds.size; cwds.set(id, p.cwd || ''); return emit({ id: msg.id, result: { thread: { id } } }); } + const n = (method, params) => emit({ method, params: { threadId: open.thread, turnId: open.turn, ...params } }); + if (msg.method === 'turn/start') { + const id = 'turn-' + (++turn); + emit({ id: msg.id, result: { turn: { id } } }); + if (/modex-title-/.test(cwds.get(p.threadId) || '')) { + emit({ method: 'item/completed', params: { threadId: p.threadId, turnId: id, item: { id: 'name-' + turn, type: 'agentMessage', text: 'Fidelity greeting', phase: 'final_answer' } } }); + return emit({ method: 'turn/completed', params: { threadId: p.threadId, turn: { id, status: 'completed', error: null } } }); + } + open = { thread: p.threadId, turn: id, request: 700 + turn }; + emit({ method: 'turn/started', params: { threadId: p.threadId, turn: { id } } }); + n('turn/plan/updated', { explanation: null, plan: [{ step: 'Ask which greeting', status: 'inProgress' }, { step: 'Edit greeting.txt', status: 'pending' }, { step: 'Check the file', status: 'pending' }] }); + return emit({ id: open.request, method: 'item/tool/requestUserInput', params: { threadId: p.threadId, turnId: id, itemId: 'ask-' + turn, isBlocking: true, autoResolutionMs: null, + questions: [{ id: 'q-word', header: 'Greeting', question: 'Which word should the greeting add?', isOther: false, isSecret: false, options: [{ label: 'fidelity', description: '' }, { label: 'world', description: '' }] }] } }); + } + if (open && msg.id === open.request && !msg.method) { + count('decisions'); + const word = msg.result.answers['q-word'].answers[0]; + const change = { path: ${JSON.stringify(file)}, kind: { type: 'update' }, diff: '-hello\\n+hello, ' + word }; + n('item/started', { item: { type: 'fileChange', id: 'edit-1', changes: [change], status: 'inProgress' } }); + count('edits'); + fs.writeFileSync(${JSON.stringify(file)}, 'hello, ' + word + '\\n'); + n('item/completed', { item: { type: 'fileChange', id: 'edit-1', changes: [change], status: 'completed' } }); + const check = 'grep -q ' + word + ' greeting.txt'; + n('item/started', { item: { type: 'commandExecution', id: 'check-1', command: check, cwd: ${JSON.stringify(repo)}, status: 'inProgress' } }); + count('checksStarted'); + const ran = spawnSync('/bin/sh', ['-c', check], { cwd: ${JSON.stringify(repo)} }); + count('checksFinished'); + n('item/completed', { item: { type: 'commandExecution', id: 'check-1', command: check, cwd: ${JSON.stringify(repo)}, status: 'completed', exitCode: ran.status, aggregatedOutput: '' } }); + n('turn/plan/updated', { explanation: null, plan: [{ step: 'Ask which greeting', status: 'completed' }, { step: 'Edit greeting.txt', status: 'completed' }, { step: 'Check the file', status: 'completed' }] }); + n('item/completed', { item: { type: 'agentMessage', id: 'final-' + turn, text: 'Changed the greeting to ' + word + '; the check passed.', phase: 'final_answer' } }); + return emit({ method: 'turn/completed', params: { threadId: open.thread, turn: { id: open.turn, status: 'completed', error: null } } }); + } + if (msg.id !== undefined && msg.method) emit({ id: msg.id, error: { code: -32601, message: 'fixture does not implement ' + msg.method } }); +}); +`; +} + +test("a clarifying question in the path: answered on its card, heard once, saved to Notes, and inert after a restart", async () => { + const { home, repo } = seedHome({ default_backend: "codex", default_mode: "agent" }); + fs.writeFileSync(path.join(repo, "greeting.txt"), BEFORE); + const counts = path.join(home, "fixture-counts.json"); + const cli = path.join(home, "codex-fidelity.cjs"); + fs.writeFileSync(cli, fakeCodex(repo, counts), { mode: 0o755 }); + const statePath = path.join(home, "app/state.json"); + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.settings.codex_bin = cli; + fs.writeFileSync(statePath, JSON.stringify(state)); + const read = () => (fs.existsSync(counts) ? JSON.parse(fs.readFileSync(counts, "utf8")) as Record : {}); + let run = { ...(await launch(home)), home, repo }; + try { + await run.page.keyboard.press("Meta+n"); + await tid(run.page, "composer-input").fill("Change the greeting and check it"); + await run.page.keyboard.press("Meta+Enter"); + const question = items(run.page, "question").first(); + await expect(tid(question, "question-submit")).toBeVisible({ timeout: 15_000 }); + await expect(currentRow(run.page)).toHaveAttribute("data-status", "waiting"); + expect(read()).toEqual({}); + await question.getByRole("radio", { name: "fidelity" }).check(); + await tid(question, "question-submit").click(); + await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); + expect(fs.readFileSync(path.join(repo, "greeting.txt"), "utf8")).toBe(AFTER); + expect(read()).toEqual({ decisions: 1, edits: 1, checksStarted: 1, checksFinished: 1 }); + await expect(tid(items(run.page, "plan"), "plan-step")).toHaveText(["Reported done Ask which greeting", "Reported done Edit greeting.txt", "Reported done Check the file"]); + await tid(run.page, "save-task-note").last().click(); + await expect(tid(run.page, "task-note-saved").last()).toContainText("Saved to Notes · Task:"); + const note = ((await run.page.evaluate(() => window.modex!.invoke("space:list", undefined))) as { title: string; markdown: string }[]).find((p) => p.title.startsWith("Task:"))!; + expect(note.markdown).toContain("- Which word should the greeting add? — fidelity"); + expect(note.markdown).toContain("`greeting.txt`"); + expect(note.markdown).toContain("`grep -q fidelity greeting.txt` — succeeded, exit code 0"); + // Restart: the card stays answered, nothing re-asks or re-runs, and a late answer is refused. + const threadId = (await currentRow(run.page).getAttribute("data-thread-id"))!; + const before = await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId); + await run.app.close(); + run = { ...(await launch(home)), home, repo }; + expect(await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId)).toEqual(before); + const [card] = await cards(run.page, threadId, "question"); + expect([card!.answeredBy, card!.delivery]).toEqual(["desktop", "sent"]); + expect(await run.page.evaluate(({ id, item }) => window.modex!.invoke("thread:answer-question", { threadId: id, itemId: item, answers: { "q-word": ["world"] } }), { id: threadId, item: card!.id })).toEqual({ ok: false, reason: "already-resolved" }); + expect(read()).toEqual({ decisions: 1, edits: 1, checksStarted: 1, checksFinished: 1 }); + } finally { await close(run); } +}); From c7b4725f8594202ad1ea9862bd86231b7d69c52c Mon Sep 17 00:00:00 2001 From: Val Alexander Date: Sun, 18 Oct 2026 11:45:42 -0500 Subject: [PATCH 2/4] Record the request variants in the acceptance receipt The matrix rows for a clarifying question, a native withdrawal, simultaneous answers and a reconnect after a decision move from open to verified with a fixture, with what each case asserts. The defects list adds the companion's 404 for threads started on the Mac. The fidelity summary gains rows for questions and approvals. Refs #281 Coven-Harness: Claude Code / claude-opus-5-5 --- docs/conversation-fidelity.md | 2 ++ ...-10-18-conversation-fidelity-acceptance.md | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/conversation-fidelity.md b/docs/conversation-fidelity.md index bd96d2cc..c1e8f69c 100644 --- a/docs/conversation-fidelity.md +++ b/docs/conversation-fidelity.md @@ -21,6 +21,8 @@ replays or re-sends anything by being shown, reloaded or reconnected. | Exit code | Reported | Not reported | | Citations | Memory citations | None; links in the reply are labelled as the model's | | Plan | `turn/plan/updated` | `TodoWrite` | +| Questions | A card for each `item/tool/requestUserInput` (experimental); its answers in the task record | None: the CLI sends Modex no structured questions | +| Approvals | Answered once, from the Mac or the phone; the card says who answered and whether the CLI received it | The same | | Save to Notes | Yes | Yes | | Stop | Native interrupt | Process stop (CF-2, [#275](https://github.com/TypeSafeAI/modex/issues/275), refines this) | diff --git a/docs/reviews/2026-10-18-conversation-fidelity-acceptance.md b/docs/reviews/2026-10-18-conversation-fidelity-acceptance.md index 4a2f1d41..dc7f1c79 100644 --- a/docs/reviews/2026-10-18-conversation-fidelity-acceptance.md +++ b/docs/reviews/2026-10-18-conversation-fidelity-acceptance.md @@ -27,13 +27,15 @@ several children have parts still open. Each row names the build it was checked | Requirement | Evidence | State | | --- | --- | --- | | Golden path: request → plan → approval → real edit → check → answer → Save to Notes → Open → restart restores, nothing re-runs | `e2e/conversation-fidelity.spec.ts` "golden path", against the real app, main process and `claude -p` adapter with a fake Claude CLI. Asserts file bytes, execution counts `{edits: 1, checksStarted: 1, checksFinished: 1}` before and after restart, plan statuses, outcomes, note revision 1 → 2 on the same page, and identical `thread:items` after restart | Verified with a fixture CLI, not a live provider | -| A plan or clarifying question answered during the path | — | Open: needs CF-3 | -| Paired-client reconnect in the path | CF-6 unit tests (journal replay, the phone's full and delta snapshots); `journal-resume.test.ts` | Verified in unit tests; not yet inside the golden fixture | +| A plan or clarifying question answered during the path | "a clarifying question in the path", with a codex-cli 0.163.0-shaped fixture: answered on its card, the CLI reads one decision, then a real edit and check, Save to Notes with the clarification, and a restart where the card is inert and counts stay `{decisions: 1, edits: 1, checksStarted: 1, checksFinished: 1}`. `e2e/plan-clarification.spec.ts` adds a plan revision, a secret answer kept out of every file, and a stopped clarification saved as not answered | Verified (fixture) | +| Paired-client reconnect in the path | The phone, journal on: "a phone that reconnects after the decision was sent" below. A paired desktop: `test/request-journal.test.ts`, over TLS with a real journal | Verified (fixture for the phone, unit for a paired desktop) | | Denial: file unchanged, nothing runs | "a denial leaves the file as it was and runs nothing": bytes unchanged, counts `{}` | Verified (fixture) | | Command failure | "a failing check is a failed command": the tool is failed, and the plan step stays in progress | Verified (fixture) | | Unknown outcome | CF-1 and CF-6 unit tests: a dropped stream; work left running at a restart | Verified (unit) | | Stop while executing | "Stop while a check runs": stopped, the check never reports, nothing retried | Verified (fixture); CF-2 refines Stop | -| Request withdrawal; simultaneous desktop and phone answers; reconnect after a decision send | — | Open: needs CF-3 | +| Native request withdrawal | "a native withdrawal": Claude's `control_cancel_request` closes the card as withdrawn, no answer reaches the CLI (`{}`), a late answer is refused, the file is unchanged | Verified (fixture) | +| Simultaneous desktop and phone answers | "the desktop and the phone answer at once", over the bridge and the companion's HTTPS: exactly one is accepted, the CLI reads one decision, the card names who gave it, the loser gets `already-resolved` or a 409, and the file matches the winner | Verified (fixture) | +| Reconnect after a decision send | "a phone that reconnects after the decision was sent", journal on: its delta shows the card answered by the desktop and sent, its late answer is a 409, and counts stay at one | Verified (fixture) | | Old snapshot migration | `test/conversation-migration.test.ts` | Verified (unit) | | Journal off | The golden fixture runs with the journal off, the default | Verified (fixture) | | Expired cursor | Existing `event-journal.test.ts` resume and reload cases | Verified (unit, existing) | @@ -58,12 +60,10 @@ These are acceptance tests of behaviour CF-1 through CF-7 already built, so they ## Next -1. Land CF-2 (#286) and CF-3. Then add the question, withdrawal, simultaneous-answer and - reconnect-after-decision variants to the golden fixture. -2. Pin a candidate SHA and run fresh full checks on it. -3. Run authorized live Codex and Claude smoke tasks through the app, and record versions, models and +1. Pin a candidate SHA and run fresh full checks on it. +2. Run authorized live Codex and Claude smoke tasks through the app, and record versions, models and limits. -4. Get human keyboard and VoiceOver acceptance. +3. Get human keyboard and VoiceOver acceptance. ## Measurements on a named Mac @@ -116,6 +116,11 @@ Each one has a failing test first, then the fix, in the PR that added these chec - **An approval card scrolled sideways at 900×600** (CF-3 surface, #276). With the Review pane open the transcript column is 319 px, and a long path plus the action row pushed Deny 160 px out of view. The question now wraps anywhere and the actions wrap. +- **The phone could not act on a thread started on the Mac** (companion, since #181 in v0.0.12; fixed in + CF-6's request half, #279). A thread created from a Mac draft keeps the full UUID its draft reserved, + but the companion's thread routes took only 8 hex characters, so answering, sending, setting the policy + or archiving from the phone got a 404. The simultaneous-answer and reconnect variants found it; + `test/companion.test.ts` failed first with a 404 for a UUID thread. - **The typed-turns theme test checked Graphite three times** (CF-4, #277). It set the theme with `settings:update`, which the window applies only on its next refresh. Specs now switch themes through Settings (`setTheme` in `e2e/support.ts`) and wait for the window to show it. From 7e2411e7dc407b43ea55a0c11ab05d6e6f9517e2 Mon Sep 17 00:00:00 2001 From: Val Alexander Date: Sun, 18 Oct 2026 12:11:32 -0500 Subject: [PATCH 3/4] Record CF-8's request variants in status Refs #281 Coven-Harness: Claude Code / claude-opus-5-5 --- docs/status.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/status.md b/docs/status.md index b98a1ed7..6ac4c54f 100644 --- a/docs/status.md +++ b/docs/status.md @@ -105,11 +105,12 @@ and [landing verification](reviews/2026-10-11-landing-clarity.md). - **The phone acts on threads started on the Mac again.** Since v0.0.12, answering, sending, setting the approval policy or archiving from the phone failed on any thread created from a Mac draft. - **Old transcripts** read unchanged, with no invented provenance. - - **Acceptance (CF-8 #281, first part).** A fixture Claude CLI drives the real app through a - plan, an approval, a real edit, a check, the answer and Save to Notes, then a restart that - re-runs nothing. Denial, a failing check and Stop are covered too. Each of those states has - reference captures in every theme at 900×600 and 1440×900, and a long transcript and a 3 MiB log - are measured on a named Mac. [Fidelity](conversation-fidelity.md), + - **Acceptance (CF-8 #281).** A fixture Claude CLI drives the real app through a plan, an + approval, a real edit, a check, the answer and Save to Notes, then a restart that re-runs + nothing. Denial, a failing check and Stop are covered too, and so are a native withdrawal, the + Mac and the phone answering at once, a phone reconnecting after the decision, and a clarifying + question inside the path. The path's states have reference captures in every theme at 900×600 + and 1440×900, and a long transcript and a 3 MiB log are measured on a named Mac. [Fidelity](conversation-fidelity.md), [review](reviews/2026-10-18-conversation-fidelity-acceptance.md). - **Fixes those checks found.** Opening a thread for the first time no longer resends and rewrites every item in it, which cost a 1,400-item transcript about 55 ms. An approval card in a narrow From 3377deeba7df8080002b96aa9daa9ca80075e43d Mon Sep 17 00:00:00 2001 From: Val Alexander Date: Sun, 18 Oct 2026 20:29:57 -0500 Subject: [PATCH 4/4] Record the landed children in the acceptance receipt CF-2 (#286), CF-3 (#293), CF-4's Markdown slice (#283), CF-6's request half (#308) and CF-7's clarifications (#312) are on main, with their merge commits. The candidate note says the fixture rows now run on main, and no candidate is pinned while the live-provider and human gates stay open. Refs #281 Coven-Harness: Claude Code / claude-opus-5-5 --- ...2026-10-18-conversation-fidelity-acceptance.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/reviews/2026-10-18-conversation-fidelity-acceptance.md b/docs/reviews/2026-10-18-conversation-fidelity-acceptance.md index dc7f1c79..5981ce58 100644 --- a/docs/reviews/2026-10-18-conversation-fidelity-acceptance.md +++ b/docs/reviews/2026-10-18-conversation-fidelity-acceptance.md @@ -7,20 +7,21 @@ recorded as live-provider or human acceptance. ## Candidate -No integrated candidate is pinned yet: CF-2 (#275, PR #286) and CF-3 (#276) have not landed, and -several children have parts still open. Each row names the build it was checked on. +Every child's slice for this gate has landed, so the fixture rows below run on `main`. No candidate +is pinned yet, because the live-provider and human gates are still open. Each row names the build it +was checked on. ## Children | Child | Landed | State | | --- | --- | --- | | CF-1 #274: identity, phases, outcomes | #282 → `7780093` | Done; closed | -| CF-2 #275: Claude Stop and cleanup | PR #286, green, waiting to update over CF-4 | Owned by another session | -| CF-3 #276: human requests | In progress, stacked on PR #286 | Not landed | -| CF-4 #277: typed turns, Markdown | #284 → `8e26534` (typed turns); Markdown slice is PR #283 | Partial | +| CF-2 #275: Claude Stop and cleanup | #286 → `6db0818` | Landed | +| CF-3 #276: human requests | #293 → `57e7607` | Landed; a live Codex question capture is open | +| CF-4 #277: typed turns, Markdown | #284 → `8e26534` (typed turns); #283 → `a4039b2` (Markdown) | Partial: remaining Markdown checks | | CF-5 #278: evidence | #287 → `d3711c6` (first part) | Partial | -| CF-6 #279: journal and clients | #285 → `fb76aac` (CF-1 half) | Partial; request half waits for CF-3 | -| CF-7 #280: plans and Notes | #289 → `f6976df` (first part) | Partial; question cards wait for CF-3 | +| CF-6 #279: journal and clients | #285 → `fb76aac` (CF-1 half); #308 → `d625eda` (request half) | Landed; phone question answers are open | +| CF-7 #280: plans and Notes | #289 → `f6976df` (first part); #312 → `f4cd67f` (clarifications) | Landed; verified steps are open | ## Integration matrix