Skip to content

fix(evals): distinguish permission status from credential disclosures #188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vriesd merged 13 commits into from
Oct 9, 2026
Merged

fix(evals): distinguish permission status from credential disclosures #188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
10ef3ea
test(evals): reproduce ordinary authorization status rejection
vriesdcrv Oct 9, 2026
cc94822
test(evals): keep heading authorization statuses private
vriesdcrv Oct 9, 2026
d573547
fix(evals): distinguish task permission status from credential values
vriesdcrv Oct 9, 2026
6ab9096
test(evals): protect indented permission predicate context
vriesdcrv Oct 9, 2026
b279c68
fix(evals): reject indented permission predicates
vriesdcrv Oct 9, 2026
db491fb
test(evals): reproduce mixed tab permission indentation
vriesdcrv Oct 9, 2026
5eaae88
fix(evals): keep mixed tab indentation outside prose permission
vriesdcrv Oct 9, 2026
23a3e3b
test(evals): cover permission capture across Markdown line endings
vriesdcrv Oct 9, 2026
a702377
fix(evals): parse permission context with Markdown line endings
vriesdcrv Oct 9, 2026
797dc79
test(evals): reproduce quoted permission retention
vriesdcrv Oct 9, 2026
d23dbfb
fix(evals): exclude quoted permission paragraphs from capture
vriesdcrv Oct 9, 2026
513f2ef
test(evals): cover quote markers inside closed code fences
vriesdcrv Oct 9, 2026
fd17491
fix(evals): reset quoted paragraph state at code fence boundaries
vriesdcrv Oct 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 76 additions & 2 deletions evals/harness.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ const SENSITIVE_DOCUMENT_ASSIGNMENT =
const SENSITIVE_DOCUMENT_FIELD_NAME =
/\b(?:[A-Za-z_][A-Za-z0-9_-]*?)?(?:token|password|passwd|secret|key|authorization|credential)[A-Za-z0-9_-]*\b/i;
const SENSITIVE_DOCUMENT_DISCLOSURE =
/(?:^|[\s"'`{,])(?:[A-Za-z_][A-Za-z0-9_-]*?)?(?:token|password|passwd|secret|key|authorization|credential)[A-Za-z0-9_-]*\s*["'`]?\s+(?:is|was|are|were|equals?|contains?)\s+["'`]?[\S]+/im;
/(?:^|[\s"'`{,])((?:[A-Za-z_][A-Za-z0-9_-]*?)?(?:token|password|passwd|secret|key|authorization|credential)[A-Za-z0-9_-]*)\s*["'`]?\s+((?:is|was|are|were|equals?|contains?)\s+["'`]?[\S]+)/gim;
const SENSITIVE_INLINE_ASSIGNMENT =
/(?:token|password|passwd|secret|key|authorization)\s*=/i;

Expand All @@ -403,8 +403,82 @@ function documentPrefixHasCodeFence(prefix: string): boolean {
return fence !== null;
}

function documentPrefixHasBlockQuote(prefix: string): boolean {
const lines = prefix.split(/\r\n?|\n/);
const current = lines.pop() ?? "";
const quotePrefix = /^ {0,3}(?:(?:[-*+]|\d{1,9}[.)])[ \t]+)*(?:>[ \t]*)+/;
let quotedParagraph = false;
for (const line of lines) {
const quote = quotePrefix.exec(line);
if (quote) {
quotedParagraph = line.slice(quote[0].length).trim() !== "";
} else if (
!line.trim() ||
/^ {0,3}(?:`{3,}|~{3,})/.test(line) ||
/^ {0,3}(?:#{1,6}(?:[ \t]|$)|(?:[-*+]|1[.)])[ \t]+\S)/.test(line)
) {
quotedParagraph = false;
}
}
if (quotePrefix.test(current)) return true;
return (
quotedParagraph &&
!/^ {0,3}(?:#{1,6}[ \t]|(?:[-*+]|1[.)])[ \t]+)/.test(current)
);
}

function isPermissionStatusClause(
content: string,
fieldStart: number,
predicateStart: number,
): boolean {
const lineStart =
Math.max(
content.lastIndexOf("\n", fieldStart),
content.lastIndexOf("\r", fieldStart),
) + 1;
const linePrefix = content.slice(lineStart, fieldStart);
const predicateLineStart =
Math.max(
content.lastIndexOf("\n", predicateStart),
content.lastIndexOf("\r", predicateStart),
) + 1;
const predicateLinePrefix = content.slice(predicateLineStart, predicateStart);
const prefix = linePrefix.split(/[.;!?](?=\s|$)/).at(-1) ?? "";
Comment thread
vriesd marked this conversation as resolved.
if (
documentPrefixHasBlockQuote(content.slice(0, fieldStart)) ||
/[`"']/.test(linePrefix) ||
/^(?: {4}| {0,3}\t)/.test(linePrefix) ||
/^(?: {4}| {0,3}\t)/.test(predicateLinePrefix) ||
/^ {0,3}#{1,6}\s/.test(linePrefix) ||
documentPrefixHasCodeFence(
content.slice(0, fieldStart).replace(/\r\n?/g, "\n"),
) ||
!/^\s*(?:[-*+]\s+)?(?:(?:the|this|separate|explicit|prior|additional|user|human)\s+)*$/i.test(
prefix,
)
)
return false;
const remainder = content.slice(predicateStart);
const boundary = remainder.search(/[;.!?](?=\s|$)|[\r\n]/);
const predicate = (
boundary < 0 ? remainder : remainder.slice(0, boundary)
).trim();
return /^(?:is|was|are|were)\s+(?:not\s+)?(?:required|needed|necessary|pending|requested|granted|denied|obtained)\s+(?:before|for|to)\s+(?:(?:(?:the|this|product|code|test|gate)\s+)*(?:repairs?|inspection|review|modification|deployment)(?:\s+(?:work|task))?|(?:repair(?:ing)?|inspect(?:ing)?|review(?:ing)?|modify|modifying|change|changing|deploy(?:ing)?)\s+(?:(?:the|this|product|code|test|gate)\s+)*(?:files|code|product|tests|gate))$/i.test(
predicate,
);
}

function hasSensitiveDocumentAssignment(content: string): boolean {
if (SENSITIVE_DOCUMENT_DISCLOSURE.test(content)) return true;
for (const match of content.matchAll(SENSITIVE_DOCUMENT_DISCLOSURE)) {
const fieldStart = match.index + match[0].indexOf(match[1] ?? "");
const predicateStart = match.index + match[0].lastIndexOf(match[2] ?? "");
if (
match[1]?.toLowerCase() !== "authorization" ||
!isPermissionStatusClause(content, fieldStart, predicateStart)
)
return true;
}
for (const match of content.matchAll(SENSITIVE_DOCUMENT_ASSIGNMENT)) {
const keyStart = match.index + match[0].indexOf(match[1] ?? "");
const prefix = content.slice(
Expand Down
213 changes: 213 additions & 0 deletions tests/inspection-capture.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,3 +291,216 @@ for (const suffix of [
});
});
}

for (const content of [
"Separate authorization is required before product repairs.\n",
"Authorization\nis required before repairs.\n",
"Authorization is needed before changing product files.\n",
"Explicit authorization was requested before repair work.\n",
"Authorization is pending for repairs.\n",
"Authorization is granted for the repair task.\n",
"Authorization is denied for deployment.\n",
"Prior authorization was obtained to change product files.\n",
"The authorization is necessary before code modification.\n",
"Authorization is not required before this inspection.\n",
"Authorization was not granted for repairs.\n",
"- Separate authorization is required before product repairs.\n",
"## Repair scope\nSeparate authorization is required before product repairs.\n",
"Separate authorization is required before product repairs.\n\n## Key findings\nThe parser has a defect.\n",
"Separate authorization is required before repairs; authorization is needed to modify code.\n",
]) {
test(`complete permission status is retained ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "observed",
content,
sha256: `sha256:${createHash("sha256").update(content).digest("hex")}`,
});
});
}

for (const content of [
"Authorization is required.\n",
"Authorization is opaque-short.\n",
"Authorization is required opaque-short.\n",
"Authorization is required before repairs opaque-short.\n",
"Authorization is required before repairs.opaque-short\n",
"Authorization is required before\nopaque-short.\n",
"Authorization is required before repairs; the token is opaque-short.\n",
"Authorization is required before repairs. The authorization is required.\n",
"Authorization is required before repairs.\n\nThe authorization is opaque-short.\n",
"Authorization is required before repairs.\n\nThe token is opaque-short.\n",
"API_AUTHORIZATION is required before repairs.\n",
"authorizationToken is required before repairs.\n",
"The credential is required before repairs.\n",
'"authorization" is required before repairs.\n',
"`authorization` is required before repairs.\n",
"`Authorization is required before repairs.`\n",
"```text\nAuthorization is required before repairs.\n```\n",
"~~~text\nAuthorization is required before repairs.\n~~~\n",
"````text\n```\nAuthorization is required before repairs.\n````\n",
" Authorization is required before repairs.\n",
"\tAuthorization is required before repairs.\n",
"Authorization\n is required before repairs.\n",
"Authorization\n\tis required before repairs.\n",
"## Authorization is required before repairs.\n",
"## Repair scope. Authorization is required before repairs.\n",
"Authorization: is required before repairs.\n",
"Authorization=is required before repairs.\n",
]) {
test(`ambiguous or encoded permission value remains private ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "unavailable",
reason: "review-document-not-safe-to-retain",
});
});
}

for (const lineEnding of ["\n", "\r\n", "\r"]) {
for (const spaces of [0, 1, 2, 3]) {
for (const content of [
`Repair scope${lineEnding}${" ".repeat(spaces)}\tAuthorization is required before repairs.${lineEnding}`,
`Authorization${lineEnding}${" ".repeat(spaces)}\tis required before repairs.${lineEnding}`,
]) {
test(`tab-indented permission context remains private ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "unavailable",
reason: "review-document-not-safe-to-retain",
});
});
}
for (const content of [
`Repair scope${lineEnding}${" ".repeat(spaces)}Authorization is required before repairs.${lineEnding}`,
`Authorization${lineEnding}${" ".repeat(spaces)}is required before repairs.${lineEnding}`,
]) {
test(`ordinary prose indentation retains permission context ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "observed",
content,
sha256: `sha256:${createHash("sha256").update(content).digest("hex")}`,
});
});
}
}
for (const fence of ["```", "~~~"]) {
const encoded = `${fence}text${lineEnding}Authorization is required before repairs.${lineEnding}${fence}${lineEnding}`;
test(`active fence permission remains private ${encoded}`, async () => {
expect(await observe(encoded)).toEqual({
kind: "unavailable",
reason: "review-document-not-safe-to-retain",
});
});
const prose = `${fence}text${lineEnding}Ordinary example.${lineEnding}${fence}${lineEnding}Authorization is required before repairs.${lineEnding}`;
test(`closed fence permits ordinary permission prose ${prose}`, async () => {
expect(await observe(prose)).toEqual({
kind: "observed",
content: prose,
sha256: `sha256:${createHash("sha256").update(prose).digest("hex")}`,
});
});
}
const content = `Authorization is required before repairs${lineEnding}Next action.${lineEnding}`;
test(`physical line ends the permission predicate ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "observed",
content,
sha256: `sha256:${createHash("sha256").update(content).digest("hex")}`,
});
});
for (const content of [
`Authorization${lineEnding} is required before repairs.${lineEnding}`,
`Repair scope${lineEnding} Authorization is required before repairs.${lineEnding}`,
]) {
test(`space-indented permission context remains private ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "unavailable",
reason: "review-document-not-safe-to-retain",
});
});
}
}

for (const lineEnding of ["\n", "\r\n", "\r"]) {
const markers = [
...([0, 1, 2, 3] as const).map((spaces) => `${" ".repeat(spaces)}> `),
">> ",
"> > ",
"- > ",
"1. > ",
"1. - > ",
];
for (const marker of markers) {
for (const content of [
`${marker}Status. Authorization is required before repairs.${lineEnding}`,
`${marker}Status.${lineEnding}Authorization is required before repairs.${lineEnding}`,
]) {
test(`quoted permission paragraph remains private ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "unavailable",
reason: "review-document-not-safe-to-retain",
});
});
}
}
for (const separator of [
lineEnding,
`>${lineEnding}`,
`## Repair scope${lineEnding}`,
]) {
const content = `> Status.${lineEnding}${separator}Authorization is required before repairs.${lineEnding}`;
test(`permission prose outside a quoted paragraph is retained ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "observed",
content,
sha256: `sha256:${createHash("sha256").update(content).digest("hex")}`,
});
});
}
for (const spaces of [0, 1, 2, 3]) {
const content = `> Status.${lineEnding}${" ".repeat(spaces)}Authorization is required before repairs.${lineEnding}`;
test(`partial plain prefix remains in its lazy quote ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "unavailable",
reason: "review-document-not-safe-to-retain",
});
});
}
const quoted = `> Authorization is required before repairs.${lineEnding}`;
test(`partial quote marker remains private ${quoted}`, async () => {
expect(await observe(quoted)).toEqual({
kind: "unavailable",
reason: "review-document-not-safe-to-retain",
});
});
const ordinary = `> Status.${lineEnding}- Authorization is required before repairs.${lineEnding}`;
test(`new list interrupts the quoted paragraph ${ordinary}`, async () => {
expect(await observe(ordinary)).toEqual({
kind: "observed",
content: ordinary,
sha256: `sha256:${createHash("sha256").update(ordinary).digest("hex")}`,
});
});
}

for (const lineEnding of ["\n", "\r\n", "\r"]) {
for (const fence of ["```", "~~~"]) {
const closed = `${fence}text${lineEnding}> shell output${lineEnding}${fence}${lineEnding}Authorization is required before repairs.${lineEnding}`;
test(`quote-like code cannot taint prose after its closed fence ${closed}`, async () => {
expect(await observe(closed)).toEqual({
kind: "observed",
content: closed,
sha256: `sha256:${createHash("sha256").update(closed).digest("hex")}`,
});
});
for (const content of [
`${fence}text${lineEnding}> shell output${lineEnding}Authorization is required before repairs.${lineEnding}`,
`> Status.${lineEnding}> ${fence}${lineEnding}Authorization is required before repairs.${lineEnding}`,
]) {
test(`active code or quoted fence content remains private ${content}`, async () => {
expect(await observe(content)).toEqual({
kind: "unavailable",
reason: "review-document-not-safe-to-retain",
});
});
}
}
}
Loading