diff --git a/README.md b/README.md index 1de8ddd..bbb61e4 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,14 @@ Additional coverage lives in the [private eval repository](https://github.com/al Eight fictional cases inspired by [Humanizer](https://github.com/blader/humanizer/blob/225a6f39ac85f76ee48dbad772ea4abe4ed6c9d8/SKILL.md) cover rhetorical framing, inflated claims, formatting, voice and replies to readers who already know the context. Controls keep useful checklists and plain technical wording. Run the same cases with the skill using `bun run eval:write-naturally --filter-metadata group=humanizer`, then without it using `bun run eval:write-naturally:baseline`. Both conditions use the same writer model, reasoning setting and graders. The baseline rejects recorded tool calls, including file reads, and missing traces. This checks the model turn; it cannot rule out instructions supplied by the surrounding Codex configuration. +### Compare with the Humanizer skill + +`bun run eval:humanizer` runs the full public suite with Humanizer using the same writer settings, case criteria and graders as `bun run eval:write-naturally`. To compare only the eight Humanizer-inspired cases, add `--filter-metadata group=humanizer` to both commands. Use `bun run eval:humanizer --validate` to check the generated configuration without downloads or model calls. + +The extension downloads Humanizer's `SKILL.md` and MIT license from revision `225a6f39ac85f76ee48dbad772ea4abe4ed6c9d8` before evaluation and verifies both SHA-256 hashes. The writer reads that skill in a separate temporary workspace; the common rewrite instructions stay the same. The prompt requests Humanizer's embedded mode so graders receive only the final text. Downloads require host networking; networking remains disabled inside the model workspace. Download or checksum failures stop the run. The runner cleans up downloaded files and temporary configuration after normal completion or a reported failure; interruption can leave ignored temporary files. The result description records the revision and skill hash. + +Use the same model overrides and filters for each condition. Compare individual outputs and separate grader errors from writing failures before making claims about either skill. These runs can still receive instructions from the surrounding Codex configuration. + Readability has separate style criteria for these cases; fact preservation is graded independently. Calibration answers stay in metadata and are never supplied to the writer. Compare individual outputs and repeat runs before attributing a gain to the skill. Changes are reviewed through pull requests. See [AGENTS.md](AGENTS.md) for contribution guidance. diff --git a/evals/productivity/write-naturally/extensions/humanizer.mjs b/evals/productivity/write-naturally/extensions/humanizer.mjs new file mode 100644 index 0000000..6e28bf1 --- /dev/null +++ b/evals/productivity/write-naturally/extensions/humanizer.mjs @@ -0,0 +1,42 @@ +import { createHash } from 'node:crypto'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +export const humanizerSource = { + repository: 'https://github.com/blader/humanizer', + revision: '225a6f39ac85f76ee48dbad772ea4abe4ed6c9d8', + files: { + 'SKILL.md': '0612f1dfb1672b0ea9b97e139bf1f06cabe98d8b27424fe8ff01e1fb4cc99cad', + LICENSE: '4ac4810254ab36d45419141aeb8e69bf50652cfafe5b2dab947d06d44e5cbf96', + }, +}; + +export function createHumanizerHook(workspace, fetchSource = fetch) { + const agents = resolve(workspace, '.agents'); + const target = resolve(agents, 'skills/humanizer'); + return async function workspaceHook(hookName, context) { + if (hookName === 'beforeAll') { + await rm(agents, { recursive: true, force: true }); + try { + const files = await Promise.all(Object.entries(humanizerSource.files).map(async ([name, hash]) => { + const url = `https://raw.githubusercontent.com/blader/humanizer/${humanizerSource.revision}/${name}`; + const response = await fetchSource(url, { signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw new Error(`Humanizer download failed (${response.status}): ${url}`); + const bytes = Buffer.from(await response.arrayBuffer()); + if (createHash('sha256').update(bytes).digest('hex') !== hash) { + throw new Error(`Humanizer checksum mismatch: ${name}`); + } + return { name, bytes }; + })); + await mkdir(target, { recursive: true }); + await Promise.all(files.map(({ name, bytes }) => writeFile(resolve(target, name), bytes))); + await writeFile(resolve(target, 'source.json'), JSON.stringify(humanizerSource, null, 2)); + } catch (error) { + await rm(agents, { recursive: true, force: true }); + throw error; + } + } + if (hookName === 'afterAll') await rm(agents, { recursive: true, force: true }); + return context; + }; +} diff --git a/package.json b/package.json index 791ae6b..177bffb 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "test": "bun test", "eval:validate": "promptfoo validate config -c evals/productivity/write-naturally/promptfooconfig.yaml", "eval:write-naturally": "promptfoo eval -c evals/productivity/write-naturally/promptfooconfig.yaml --no-cache --no-share", - "eval:write-naturally:baseline": "bun run scripts/eval-writing-baseline.ts evals/productivity/write-naturally" + "eval:write-naturally:baseline": "bun run scripts/eval-writing-baseline.ts evals/productivity/write-naturally", + "eval:humanizer": "bun run scripts/eval-writing-humanizer.ts evals/productivity/write-naturally" }, "devDependencies": { "@openai/codex-sdk": "0.162.1", diff --git a/scripts/eval-writing-humanizer.ts b/scripts/eval-writing-humanizer.ts new file mode 100644 index 0000000..3376de8 --- /dev/null +++ b/scripts/eval-writing-humanizer.ts @@ -0,0 +1,82 @@ +import { mkdir, rm } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { randomUUID } from 'node:crypto'; +import { humanizerSource } from '../evals/productivity/write-naturally/extensions/humanizer.mjs'; + +// Derive the comparison from the caller's suite, including private fixtures. +export async function runHumanizerComparison(suitePath: string, args: string[]) { + const suiteRoot = resolve(suitePath); + const config = Bun.YAML.parse(await Bun.file(resolve(suiteRoot, 'promptfooconfig.yaml')).text()) as any; + const comparison = structuredClone(config); + const runDir = resolve(suiteRoot, 'outputs', `humanizer-${randomUUID()}`); + const workspace = resolve(runDir, 'workspace'); + const generatedConfig = resolve(runDir, 'config.json'); + const hookFile = resolve(runDir, 'extension.mjs'); + const promptFile = resolve(runDir, 'rewrite.md'); + const hookSource = pathToFileURL(resolve(import.meta.dir, '../evals/productivity/write-naturally/extensions/humanizer.mjs')).href; + comparison.description = `${config.description} — Humanizer ${humanizerSource.revision}, SHA-256 ${humanizerSource.files['SKILL.md']}`; + comparison.extensions = [`file://${hookFile}:workspaceHook`]; + comparison.prompts = [{ id: `file://${promptFile}`, label: `humanizer@${humanizerSource.revision}` }]; + const testPattern = config.tests.replace(/^file:\/\//, ''); + const testFiles = [...new Bun.Glob(testPattern).scanSync({ cwd: suiteRoot, absolute: true })].sort(); + if (!testFiles.length) throw new Error(`No cases matched ${testPattern}`); + comparison.tests = (await Promise.all(testFiles.map(async filename => + Bun.YAML.parse(await Bun.file(filename).text()), + ))).flat(); + for (const provider of comparison.providers) { + provider.label = 'codex-humanizer'; + provider.config.working_dir = workspace; + } + for (const assertion of comparison.defaultTest.assert) { + if (assertion.metric === 'repository-skill-used') { + assertion.metric = 'humanizer-skill-used'; + assertion.value = 'humanizer'; + } else if (assertion.metric === 'repository-skill-path') { + assertion.metric = 'humanizer-skill-path'; + assertion.value = ` + const expected = ${JSON.stringify(resolve(workspace, '.agents/skills/humanizer/SKILL.md'))}; + const calls = context.metadata?.skillCalls || []; + const pass = calls.some(call => { + const path = String(call.path || '').replace(/\\\\/g, '/'); + return path === '.agents/skills/humanizer/SKILL.md' || path === expected; + }); + return { pass, score: pass ? 1 : 0, reason: pass ? 'Read the pinned Humanizer skill.' : 'Expected a skill read from the Humanizer comparison workspace.' }; + `; + } + } + // Generated config lives under outputs; keep every judge in the original suite workspace. + const rebaseJudge = (provider: any) => { + if (provider?.config?.working_dir) { + provider.config.working_dir = resolve(suiteRoot, provider.config.working_dir); + } + }; + for (const test of [comparison.defaultTest, ...comparison.tests]) { + rebaseJudge(test.options?.provider); + for (const assertion of test.assert || []) rebaseJudge(assertion.provider); + } + const originalPrompt = await Bun.file(resolve(suiteRoot, 'prompts/rewrite.md')).text(); + const separator = originalPrompt.indexOf('\n\n'); + if (separator < 0) throw new Error('Expected the suite prompt to begin with a skill-loading paragraph.'); + const prompt = 'First run `cat .agents/skills/humanizer/SKILL.md` and read its output. Then use $humanizer from that file to write the requested text. Use its embedded mode and return only the final rewritten text.\n\n' + originalPrompt.slice(separator + 2); + await mkdir(workspace, { recursive: true }); + try { + await Bun.write(hookFile, `import { createHumanizerHook } from ${JSON.stringify(hookSource)};\nexport const workspaceHook = createHumanizerHook(${JSON.stringify(workspace)});\n`); + await Bun.write(promptFile, prompt); + await Bun.write(generatedConfig, JSON.stringify(comparison, null, 2)); + const cliArgs = args.includes('--validate') + ? ['validate', 'config', '-c', generatedConfig] + : ['eval', '-c', generatedConfig, ...args, '--no-cache', '--no-share']; + return await Bun.spawn([resolve('node_modules/.bin/promptfoo'), ...cliArgs], { + stdout: 'inherit', stderr: 'inherit', + }).exited; + } finally { + await rm(runDir, { recursive: true, force: true }); + } +} + +if (import.meta.main) { + const [suitePath, ...args] = process.argv.slice(2); + if (!suitePath) throw new Error('Expected an eval suite directory.'); + process.exit(await runHumanizerComparison(suitePath, args)); +}