updating generator

prompts/generator_prompt.txt

@@ -1,6 +1,31 @@
-You are generating clean, spiritually grounded Q’uo-style answers.
+You are part of a knowledge distillation pipeline.
 
-Question:
-{{QUESTION}}
+You will be given CONTEXT (excerpts from a corpus) and a QUESTION about that context.
+
+Your task:
+
+1. First, think step by step INSIDE <think> and </think> tags,
+   using ONLY information that can be supported by the CONTEXT.
+2. Then, AFTER </think>, write the final answer plainly, without any tags.
+
+Rules:
+
+- Use ONLY the CONTEXT for facts and claims.
+- Do NOT use outside knowledge or guess.
+- If the CONTEXT is not sufficient to answer, the final answer must be:
+  "I cannot answer this from the provided context."
+- You may quote short phrases from the CONTEXT as evidence, but avoid long copy-paste.
+- Be concise and clear.
 
-Respond in the tone of the Confederation, clear, humble, and service-oriented.
+Format (follow exactly):
+
+<think>
+[your detailed reasoning here, step by step, grounded in the CONTEXT]
+</think>
+[your final answer here]
+
+CONTEXT:
+{{CONTEXT}}
+
+QUESTION:
+{{QUESTION}}

src/generator/generator_core.mjs

@@ -15,50 +15,67 @@ async function loadPromptTemplate() {
 }
 
 /**
- * runGenerator
- *
- * @param {string} query - the natural language question
- * @param {Array} contextChunks - retrieved context objects from RAG
- * @param {object} provider - { generate(prompt) → string }
- * @returns {object} parsed JSON sample
+ * Build the full prompt from template, question, and context chunks.
  */
-export async function runGenerator(query, contextChunks, provider) {
-  if (!provider || typeof provider.generate !== 'function') {
-    throw new Error("runGenerator: provider.generate() not found");
-  }
+async function makeGeneratorPrompt(question, contextChunks) {
+  const tmpl = await loadPromptTemplate();
 
-  const template = await loadPromptTemplate();
+  let prompt = tmpl.replace(/{{QUESTION}}/g, question);
 
-  // Build context string
-  const contextText = contextChunks
-    .map((c, i) => `Context ${i + 1}:\n${c.content}\n`)
-    .join('\n');
+  if (tmpl.includes('{{CONTEXT}}')) {
+    const contextText =
+      contextChunks?.map((c) => c.content || c.text || '').join('\n\n---\n\n') ??
+      '';
 
-  // Fill prompt template
-  const prompt = template
-    .replace('{{QUERY}}', query)
-    .replace('{{CONTEXT}}', contextText);
+    prompt = prompt.replace(/{{CONTEXT}}/g, contextText);
+  }
+
+  return prompt;
+}
+
+/**
+ * runGenerator
+ *
+ * @param {string} question        - the natural language question
+ * @param {Array}  contextChunks   - retrieved context objects from RAG
+ * @param {object} provider        - { generate(prompt) → string }
+ * @returns {object} generator result:
+ *   {
+ *     raw: string,          // full model output (think + answer)
+ *     thought: string|null, // content inside <think>...</think> if present
+ *     answer: string,       // final answer after </think> (or whole raw if no tags)
+ *     query: string,        // original question
+ *     context: Array        // original context chunks
+ *   }
+ */
+export async function runGenerator(question, contextChunks, provider) {
+  const prompt = await makeGeneratorPrompt(question, contextChunks);
 
-  // Call LLM
   const raw = await provider.generate(prompt);
 
-  // Try parsing JSON (verifier handles correctness later)
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch (err) {
-    parsed = { error: 'invalid_json', raw };
+  let thought = null;
+  let answer = raw?.trim() ?? '';
+
+  // Extract <think> ... </think> if the thinking model emits it
+  const startTag = '<think>';
+  const endTag = '</think>';
+  const startIdx = raw.indexOf(startTag);
+  const endIdx = raw.indexOf(endTag);
+
+  if (startIdx !== -1 && endIdx !== -1 && endIdx > startIdx) {
+    thought = raw.slice(startIdx + startTag.length, endIdx).trim();
+    answer = raw.slice(endIdx + endTag.length).trim();
   }
 
-  // Return normalized object
   return {
-    query,
-    context: contextChunks,
     raw,
-    parsed
+    thought,
+    answer,
+    query: question,
+    context: contextChunks || [],
   };
 }
 
 export default {
-  runGenerator
+  runGenerator,
 };

src/pipeline/step.mjs

@@ -9,21 +9,10 @@ import { preview } from './util.mjs';
 /**
  * Run a single pipeline step for one question.
  *
- * Orchestrates:
+ * Flow:
  *   retrieval → generator → verifier → reward
  *
- * Returns a structured result:
- *   {
- *     status: 'accepted' | 'invalid_question' | 'retrieval_failed'
- *             | 'generator_failed' | 'verifier_rejected'
- *             | 'reward_rejected'  | 'verifier_error' | 'reward_error',
- *     question,
- *     context,
- *     gen,
- *     ver,
- *     rew,
- *     error? // optional message
- *   }
+ * No JSON formats required — all models return free-text strings.
  */
 export async function runPipelineStep({
   question,
@@ -38,6 +27,9 @@ export async function runPipelineStep({
   const log = logger?.log?.bind(logger) || console.log;
   const errLog = logger?.error?.bind(logger) || console.error;
 
+  // ----------------------------------------
+  // Validate input question
+  // ----------------------------------------
   if (!question || !question.trim()) {
     if (verbose) log('   [pipeline] empty / invalid question, skipping');
     return { status: 'invalid_question', question };
@@ -47,11 +39,14 @@ export async function runPipelineStep({
   const verProv = verifierProvider || loadProviderFor('verifier');
   const rewProv = rewardProvider || loadProviderFor('reward');
 
-  // --- Retrieval ---
+  // ----------------------------------------
+  // Retrieval
+  // ----------------------------------------
   let context = [];
   try {
     if (verbose) log(`   [retrieval] mode=${retrievalMode} k=${k}`);
     context = await hybridSearch(question, k);
+
     if (verbose) {
       log(`   [retrieval] got ${context.length} chunks`);
       if (context.length > 0) {
@@ -70,17 +65,17 @@ export async function runPipelineStep({
     };
   }
 
-  // --- Generator ---
+  // ----------------------------------------
+  // Generator
+  // ----------------------------------------
   let gen;
   try {
     if (verbose) log('   [generator] calling model…');
-    // NOTE: runGenerator(query, contextChunks, provider)
     gen = await runGenerator(question, context, genProv);
+
     if (verbose) {
-      log('   [generator] raw:');
-      log('   ' + preview(gen.raw ?? '', 400).replace(/\n/g, '\n   '));
-      log('   [generator] parsed:');
-      log('   ' + preview(gen.parsed, 400).replace(/\n/g, '\n   '));
+      log('   [generator] answer:');
+      log('   ' + preview(gen.answer ?? '', 400).replace(/\n/g, '\n   '));
     }
   } catch (e) {
     const msg = e?.message || String(e);
@@ -93,16 +88,29 @@ export async function runPipelineStep({
     };
   }
 
-  // --- Verifier ---
+  // Empty answer means generator failed
+  if (!gen || !gen.answer || !gen.answer.trim()) {
+    if (verbose) log('   [generator] empty answer — generator_failed');
+    return {
+      status: 'generator_failed',
+      question,
+      context,
+      gen,
+      error: 'empty_answer',
+    };
+  }
+
+  // ----------------------------------------
+  // Verifier
+  // ----------------------------------------
   let ver;
   try {
     if (verbose) log('   [verifier] calling model…');
-    // NOTE: runVerifier(sample, provider)
-    ver = await runVerifier(gen, verProv);
+    ver = await runVerifier({ question, context, gen }, verProv);
+
     if (verbose) {
-      log('   [verifier] parsed:');
-      log('   ' + preview(ver.parsed, 400).replace(/\n/g, '\n   '));
-      log(`   [verifier] ok=${ver.ok === true}`);
+      log('   [verifier] ok=' + ver.ok);
+      log('   ' + preview(ver.raw ?? '', 200).replace(/\n/g, '\n   '));
     }
   } catch (e) {
     const msg = e?.message || String(e);
@@ -127,16 +135,17 @@ export async function runPipelineStep({
     };
   }
 
-  // --- Reward ---
+  // ----------------------------------------
+  // Reward
+  // ----------------------------------------
   let rew;
   try {
     if (verbose) log('   [reward] calling model…');
-    // NOTE: runReward(sample, provider)
-    rew = await runReward(gen, rewProv);
+    rew = await runReward({ question, context, gen }, rewProv);
+
     if (verbose) {
-      log('   [reward] parsed:');
-      log('   ' + preview(rew.parsed, 400).replace(/\n/g, '\n   '));
       log(`   [reward] score=${rew.score} ok=${rew.ok}`);
+      log('   ' + preview(rew.raw ?? '', 200).replace(/\n/g, '\n   '));
     }
   } catch (e) {
     const msg = e?.message || String(e);
@@ -163,6 +172,9 @@ export async function runPipelineStep({
     };
   }
 
+  // ----------------------------------------
+  // Accepted sample
+  // ----------------------------------------
   if (verbose) log('   [pipeline] accepted ✅');
 
   return {

src/reward/reward_core.mjs

@@ -1,66 +1,41 @@
 // src/reward/reward_core.mjs
-import fs from 'fs/promises';
-import path from 'path';
+import fs from "fs/promises";
+import path from "path";
 
-async function loadRewardTemplate() {
+async function loadTemplate() {
   const filePath = path.resolve(
     path.dirname(new URL(import.meta.url).pathname),
-    '..',
-    '..',
-    'prompts',
-    'reward_prompt.txt'
+    "..",
+    "..",
+    "prompts",
+    "reward_prompt.txt"
   );
-  return await fs.readFile(filePath, 'utf8');
+  return await fs.readFile(filePath, "utf8");
 }
 
-/**
- * runReward
- *
- * @param {object} sample  - Verified sample (or generator output)
- * @param {object} provider - { generate(prompt) }
- * @returns {object} reward result:
- *   { score: number, parsed: {...}, raw, ok: boolean }
- */
-export async function runReward(sample, provider) {
-  if (!sample || typeof sample !== 'object') {
-    throw new Error("runReward: sample must be an object");
-  }
-  if (!provider || typeof provider.generate !== 'function') {
-    throw new Error("runReward: provider.generate() missing");
-  }
+export async function runReward({ question, context, gen }, provider) {
+  const tmpl = await loadTemplate();
 
-  const template = await loadRewardTemplate();
+  const ctxText = context
+    .map((c) => c.content || c.text || "")
+    .join("\n\n---\n\n");
 
-  // Provide JSON form of the answer to the reward prompt
-  const content = JSON.stringify(sample.parsed ?? sample.raw, null, 2);
-
-  const prompt = template.replace('{{SAMPLE}}', content);
+  const prompt = tmpl
+    .replace(/{{QUESTION}}/g, question)
+    .replace(/{{ANSWER}}/g, gen.answer || gen.raw)
+    .replace(/{{CONTEXT}}/g, ctxText);
 
   const raw = await provider.generate(prompt);
 
-  // Try to parse JSON produced by reward model
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch {
-    parsed = { error: 'invalid_json', raw };
-  }
-
-  // Score extraction
-  const score = typeof parsed.score === 'number' ? parsed.score : -1;
-
-  // "ok" means usable in gold dataset
-  const ok = score >= 0;
+  const match = raw.match(/\b(?:0(?:\.\d+)?|1(?:\.0+)?)\b/);
+  const parsed = match ? parseFloat(match[0]) : 0;
+  const score = Math.max(0, Math.min(parsed, 1));
 
   return {
-    score,
-    ok,
     raw,
-    parsed,
-    sample
+    score,
+    ok: score >= 0.5,
   };
 }
 
-export default {
-  runReward
-};
+export default { runReward };

src/verifier/verifier_core.mjs

@@ -1,68 +1,36 @@
 // src/verifier/verifier_core.mjs
-import fs from 'fs/promises';
-import path from 'path';
+import fs from "fs/promises";
+import path from "path";
 
-// Load verifier prompt template
 async function loadTemplate() {
   const filePath = path.resolve(
     path.dirname(new URL(import.meta.url).pathname),
-    '..',
-    '..',
-    'prompts',
-    'verifier_prompt.txt'
+    "..",
+    "..",
+    "prompts",
+    "verifier_prompt.txt"
   );
-  return await fs.readFile(filePath, 'utf8');
+  return await fs.readFile(filePath, "utf8");
 }
 
-/**
- * runVerifier
- *
- * @param {object} sample  - generator output object
- * @param {object} provider - { generate(prompt) }
- * @returns {object} verification result:
- *   { ok: boolean, raw: string, parsed: object }
- */
-export async function runVerifier(sample, provider) {
-  if (!sample || typeof sample !== 'object') {
-    throw new Error("runVerifier: sample must be an object");
-  }
-  if (!provider || typeof provider.generate !== 'function') {
-    throw new Error("runVerifier: provider.generate() missing");
-  }
+export async function runVerifier({ question, context, gen }, provider) {
+  const tmpl = await loadTemplate();
 
-  const template = await loadTemplate();
+  const ctxText = context
+    .map((c) => c.content || c.text || "")
+    .join("\n\n---\n\n");
 
-  // Fill template: give model the generator's JSON
-  const prompt = template.replace('{{SAMPLE}}', JSON.stringify(sample.parsed ?? sample.raw, null, 2));
+  const prompt = tmpl
+    .replace(/{{QUESTION}}/g, question)
+    .replace(/{{ANSWER}}/g, gen.answer || gen.raw)
+    .replace(/{{CONTEXT}}/g, ctxText);
 
   const raw = await provider.generate(prompt);
 
-  // Try parsing JSON produced by the verifier
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch {
-    parsed = { error: 'invalid_json', raw };
-  }
+  const first = raw.split("\n")[0].trim();
+  const ok = /^yes\b/i.test(first);
 
-  // Determine “ok” status:
-  // Allow 3 styles:
-  // - { ok: true }
-  // - { verdict: "pass" }
-  // - { score: 0-10 }
-  let ok = false;
-  if (parsed.ok === true) ok = true;
-  if (parsed.verdict === "pass") ok = true;
-  if (typeof parsed.score === "number" && parsed.score > 0) ok = true;
-
-  return {
-    ok,
-    raw,
-    parsed,
-    sample
-  };
+  return { raw, ok };
 }
 
-export default {
-  runVerifier
-};
+export default { runVerifier };

tests/generator_core.test.mjs

@@ -1,43 +1,40 @@
 import { describe, it, expect, vi } from 'vitest';
 import { runGenerator } from '../src/generator/generator_core.mjs';
-import fs from 'fs/promises';
-import path from 'path';
 
 describe('generator_core.mjs', () => {
-  it('builds prompt and parses JSON output correctly', async () => {
-    // Fake context from retrieval
+  it('builds prompt and returns a plain-text answer', async () => {
     const fakeContext = [
-      { content: "First context chunk" },
-      { content: "Second context chunk" }
+      { content: 'First context chunk' },
+      { content: 'Second context chunk' },
     ];
 
-    // Fake provider
     const provider = {
-      generate: vi.fn(async () => JSON.stringify({
-        answer: "Because love is all that there is",
-        reasoning: "Unity and service arise from one source."
-      }))
+      generate: vi.fn(async (prompt) => {
+        // At minimum, prompt should contain the question
+        expect(prompt).toContain('What is love?');
+        // We *don’t* assume the template includes the raw context text,
+        // since the current generator_prompt.txt may only use {{QUESTION}}.
+        return 'Because love is all that there is';
+      }),
     };
 
-    const result = await runGenerator("What is love?", fakeContext, provider);
+    const result = await runGenerator('What is love?', fakeContext, provider);
 
     expect(provider.generate).toHaveBeenCalledOnce();
-    expect(result.parsed.answer).toBeDefined();
-    expect(result.parsed.reasoning).toBeDefined();
-    expect(result.query).toBe("What is love?");
-    expect(result.context.length).toBe(2);
+    expect(result.raw).toBe('Because love is all that there is');
+    expect(result.answer).toBe('Because love is all that there is');
   });
 
-  it('captures invalid JSON when model output is malformed', async () => {
-    const fakeContext = [{ content: "chunk" }];
+  it('handles arbitrary non-JSON output without error', async () => {
+    const fakeContext = [{ content: 'chunk' }];
 
     const provider = {
-      generate: vi.fn(async () => "{ this is not json ")
+      generate: vi.fn(async () => 'this is not JSON and that is fine'),
     };
 
-    const result = await runGenerator("broken?", fakeContext, provider);
+    const result = await runGenerator('broken?', fakeContext, provider);
 
-    expect(result.parsed.error).toBe("invalid_json");
-    expect(result.raw).toContain("not json");
+    expect(result.raw).toBe('this is not JSON and that is fine');
+    expect(result.answer).toBe('this is not JSON and that is fine');
   });
 });

tests/pipeline.mock.test.mjs

@@ -10,19 +10,18 @@ vi.mock('../src/providers/provider.mjs', () => {
         stage,
         async generate(prompt) {
           if (stage === 'generator') {
-            // pretend generator returns a JSON string
-            return JSON.stringify({ answer: 'mocked', stage: 'generator' });
+            // generator returns a plain-text answer
+            return 'mocked answer';
           }
           if (stage === 'verifier') {
-            // verifier returns { ok: true }
-            return JSON.stringify({ ok: true });
+            // verifier first line YES ⇒ ok = true
+            return 'YES\nLooks good';
           }
           if (stage === 'reward') {
-            // reward returns a positive score
-            return JSON.stringify({ score: 1 });
+            // reward outputs a numeric score between 0 and 1
+            return '0.99\nExcellent sample';
           }
-          // fallback
-          return JSON.stringify({ ok: true });
+          return 'YES';
         },
       };
     },
@@ -47,15 +46,23 @@ describe('runPipelineStep (mocked providers)', () => {
   });
 
   it('runs a full pipeline step successfully', async () => {
-    const result = await runPipelineStep({ question: 'What is mock testing?' });
+    const result = await runPipelineStep({
+      question: 'What is mock testing?',
+      verbose: false,
+      logger: console,
+    });
 
     expect(result.status).toBe('accepted');
 
     // generator output made it through
-    expect(result.gen.parsed.answer).toBe('mocked');
+    expect(result.gen.answer).toBe('mocked answer');
 
     // verifier + reward both say OK
     expect(result.ver.ok).toBe(true);
     expect(result.rew.ok).toBe(true);
+    expect(result.rew.score).toBeCloseTo(0.99, 5);
+
+    // context came from mocked retrieval
+    expect(result.context.length).toBe(2);
   });
 });

tests/reward_core.test.mjs

@@ -3,51 +3,88 @@ import { describe, it, expect, vi } from 'vitest';
 import { runReward } from '../src/reward/reward_core.mjs';
 
 describe('reward_core.mjs', () => {
-
-  it('parses reward score from valid JSON', async () => {
-    const fakeSample = {
-      parsed: { answer: "X", reasoning: "Y" }
+  it('parses reward score from valid numeric output', async () => {
+    const sample = {
+      question: 'Q',
+      context: [{ content: 'ctx' }],
+      gen: { answer: 'A', raw: 'A' },
     };
 
     const provider = {
-      generate: vi.fn(async () => JSON.stringify({
-        score: 8.7,
-        notes: "High quality explanation"
-      }))
+      generate: vi.fn(async () => '0.87\nHigh quality explanation'),
     };
 
-    const result = await runReward(fakeSample, provider);
+    const result = await runReward(
+      {
+        question: sample.question,
+        context: sample.context,
+        gen: sample.gen,
+      },
+      provider,
+    );
 
-    expect(result.score).toBe(8.7);
+    expect(result.score).toBeCloseTo(0.87, 5);
     expect(result.ok).toBe(true);
   });
 
-  it('handles invalid JSON safely', async () => {
-    const fakeSample = { parsed: { answer: "A" } };
+  it('handles invalid or missing score by falling back to 0 and ok=false', async () => {
+    const sample = {
+      question: 'Q',
+      context: [{ content: 'ctx' }],
+      gen: { answer: 'A', raw: 'A' },
+    };
 
     const provider = {
-      generate: vi.fn(async () => "{ broken json")
+      // no valid 0–1 number present
+      generate: vi.fn(async () => 'no numeric score here\ntext'),
     };
 
-    const result = await runReward(fakeSample, provider);
+    const result = await runReward(
+      {
+        question: sample.question,
+        context: sample.context,
+        gen: sample.gen,
+      },
+      provider,
+    );
 
-    expect(result.parsed.error).toBe("invalid_json");
+    expect(result.score).toBe(0);
     expect(result.ok).toBe(false);
-    expect(result.score).toBe(-1);
   });
 
-  it('treats missing score as failure', async () => {
-    const fakeSample = { parsed: {} };
+  it('treats score >= 0.5 as success and < 0.5 as failure', async () => {
+    const sample = {
+      question: 'Q',
+      context: [{ content: 'ctx' }],
+      gen: { answer: 'A', raw: 'A' },
+    };
 
-    const provider = {
-      generate: vi.fn(async () => JSON.stringify({
-        verdict: "pass-but-no-score"
-      }))
+    const highProvider = {
+      generate: vi.fn(async () => '0.9\nGreat answer'),
     };
 
-    const result = await runReward(fakeSample, provider);
+    const lowProvider = {
+      generate: vi.fn(async () => '0.2\nWeak answer'),
+    };
 
-    expect(result.ok).toBe(false);
-    expect(result.score).toBe(-1);
+    const high = await runReward(
+      {
+        question: sample.question,
+        context: sample.context,
+        gen: sample.gen,
+      },
+      highProvider,
+    );
+    const low = await runReward(
+      {
+        question: sample.question,
+        context: sample.context,
+        gen: sample.gen,
+      },
+      lowProvider,
+    );
+
+    expect(high.ok).toBe(true);
+    expect(low.ok).toBe(false);
   });
 });

tests/verifier_core.test.mjs

@@ -3,54 +3,75 @@ import { describe, it, expect, vi } from 'vitest';
 import { runVerifier } from '../src/verifier/verifier_core.mjs';
 
 describe('verifier_core.mjs', () => {
-  it('parses verifier JSON and returns ok=true on pass', async () => {
-    const fakeSample = {
-      query: "What is love?",
-      parsed: {
-        answer: "Love is unity.",
-        reasoning: "We are all one."
-      }
+  it('returns ok=true when first line is YES', async () => {
+    const sample = {
+      question: 'What is love?',
+      context: [{ content: 'ctx' }],
+      gen: { answer: 'Love is unity.', raw: 'Love is unity.' },
     };
 
     const provider = {
-      generate: vi.fn(async () => JSON.stringify({
-        verdict: "pass",
-        notes: "Tone correct."
-      }))
+      generate: vi.fn(async () => 'YES\nLooks good.'),
     };
 
-    const result = await runVerifier(fakeSample, provider);
+    const result = await runVerifier(
+      {
+        question: sample.question,
+        context: sample.context,
+        gen: sample.gen,
+      },
+      provider,
+    );
 
+    expect(provider.generate).toHaveBeenCalledOnce();
     expect(result.ok).toBe(true);
-    expect(result.parsed.verdict).toBe("pass");
+    expect(result.raw.startsWith('YES')).toBe(true);
   });
 
-  it('captures invalid JSON from verifier', async () => {
-    const fakeSample = { parsed: { answer: "x" } };
+  it('returns ok=false when first line is NO', async () => {
+    const sample = {
+      question: 'What is love?',
+      context: [{ content: 'ctx' }],
+      gen: { answer: 'Bad answer', raw: 'Bad answer' },
+    };
 
     const provider = {
-      generate: vi.fn(async () => "{ not JSON")
+      generate: vi.fn(async () => 'NO\nIncorrect interpretation.'),
     };
 
-    const result = await runVerifier(fakeSample, provider);
+    const result = await runVerifier(
+      {
+        question: sample.question,
+        context: sample.context,
+        gen: sample.gen,
+      },
+      provider,
+    );
 
-    expect(result.parsed.error).toBe("invalid_json");
     expect(result.ok).toBe(false);
+    expect(result.raw.startsWith('NO')).toBe(true);
   });
 
-  it('accepts scoring-based decisions', async () => {
-    const fakeSample = { parsed: { answer: "ok" } };
+  it('accepts lowercase/whitespace yes on first line', async () => {
+    const sample = {
+      question: 'What is love?',
+      context: [{ content: 'ctx' }],
+      gen: { answer: 'Reasonable answer', raw: 'Reasonable answer' },
+    };
 
     const provider = {
-      generate: vi.fn(async () => JSON.stringify({
-        score: 8.5,
-        details: "Solid answer."
-      }))
+      generate: vi.fn(async () => 'yes, this is fine\nMore details…'),
     };
 
-    const result = await runVerifier(fakeSample, provider);
+    const result = await runVerifier(
+      {
+        question: sample.question,
+        context: sample.context,
+        gen: sample.gen,
+      },
+      provider,
+    );
 
     expect(result.ok).toBe(true);
-    expect(result.parsed.score).toBe(8.5);
   });
 });