diff --git a/evals/harness.ts b/evals/harness.ts index f1c9fcc8..ff8851a2 100644 --- a/evals/harness.ts +++ b/evals/harness.ts @@ -381,7 +381,7 @@ const SENSITIVE_DOCUMENT_ASSIGNMENT = const SENSITIVE_DOCUMENT_FIELD_NAME = /\b(?:[A-Za-z_][A-Za-z0-9_-]*?)?(?:token|password|passwd|secret|key|authorization|credential)[A-Za-z0-9_-]*\b/i; const SENSITIVE_DOCUMENT_DISCLOSURE = - /(?:^|[\s"'`{,])(?:[A-Za-z_][A-Za-z0-9_-]*?)?(?:token|password|passwd|secret|key|authorization|credential)[A-Za-z0-9_-]*\s*["'`]?\s+(?:is|was|are|were|equals?|contains?)\s+["'`]?[\S]+/im; + /(?:^|[\s"'`{,])((?:[A-Za-z_][A-Za-z0-9_-]*?)?(?:token|password|passwd|secret|key|authorization|credential)[A-Za-z0-9_-]*)\s*["'`]?\s+((?:is|was|are|were|equals?|contains?)\s+["'`]?[\S]+)/gim; const SENSITIVE_INLINE_ASSIGNMENT = /(?:token|password|passwd|secret|key|authorization)\s*=/i; @@ -403,8 +403,82 @@ function documentPrefixHasCodeFence(prefix: string): boolean { return fence !== null; } +function documentPrefixHasBlockQuote(prefix: string): boolean { + const lines = prefix.split(/\r\n?|\n/); + const current = lines.pop() ?? ""; + const quotePrefix = /^ {0,3}(?:(?:[-*+]|\d{1,9}[.)])[ \t]+)*(?:>[ \t]*)+/; + let quotedParagraph = false; + for (const line of lines) { + const quote = quotePrefix.exec(line); + if (quote) { + quotedParagraph = line.slice(quote[0].length).trim() !== ""; + } else if ( + !line.trim() || + /^ {0,3}(?:`{3,}|~{3,})/.test(line) || + /^ {0,3}(?:#{1,6}(?:[ \t]|$)|(?:[-*+]|1[.)])[ \t]+\S)/.test(line) + ) { + quotedParagraph = false; + } + } + if (quotePrefix.test(current)) return true; + return ( + quotedParagraph && + !/^ {0,3}(?:#{1,6}[ \t]|(?:[-*+]|1[.)])[ \t]+)/.test(current) + ); +} + +function isPermissionStatusClause( + content: string, + fieldStart: number, + predicateStart: number, +): boolean { + const lineStart = + Math.max( + content.lastIndexOf("\n", fieldStart), + content.lastIndexOf("\r", fieldStart), + ) + 1; + const linePrefix = content.slice(lineStart, fieldStart); + const predicateLineStart = + Math.max( + content.lastIndexOf("\n", predicateStart), + content.lastIndexOf("\r", predicateStart), + ) + 1; + const predicateLinePrefix = content.slice(predicateLineStart, predicateStart); + const prefix = linePrefix.split(/[.;!?](?=\s|$)/).at(-1) ?? ""; + if ( + documentPrefixHasBlockQuote(content.slice(0, fieldStart)) || + /[`"']/.test(linePrefix) || + /^(?: {4}| {0,3}\t)/.test(linePrefix) || + /^(?: {4}| {0,3}\t)/.test(predicateLinePrefix) || + /^ {0,3}#{1,6}\s/.test(linePrefix) || + documentPrefixHasCodeFence( + content.slice(0, fieldStart).replace(/\r\n?/g, "\n"), + ) || + !/^\s*(?:[-*+]\s+)?(?:(?:the|this|separate|explicit|prior|additional|user|human)\s+)*$/i.test( + prefix, + ) + ) + return false; + const remainder = content.slice(predicateStart); + const boundary = remainder.search(/[;.!?](?=\s|$)|[\r\n]/); + const predicate = ( + boundary < 0 ? remainder : remainder.slice(0, boundary) + ).trim(); + return /^(?:is|was|are|were)\s+(?:not\s+)?(?:required|needed|necessary|pending|requested|granted|denied|obtained)\s+(?:before|for|to)\s+(?:(?:(?:the|this|product|code|test|gate)\s+)*(?:repairs?|inspection|review|modification|deployment)(?:\s+(?:work|task))?|(?:repair(?:ing)?|inspect(?:ing)?|review(?:ing)?|modify|modifying|change|changing|deploy(?:ing)?)\s+(?:(?:the|this|product|code|test|gate)\s+)*(?:files|code|product|tests|gate))$/i.test( + predicate, + ); +} + function hasSensitiveDocumentAssignment(content: string): boolean { - if (SENSITIVE_DOCUMENT_DISCLOSURE.test(content)) return true; + for (const match of content.matchAll(SENSITIVE_DOCUMENT_DISCLOSURE)) { + const fieldStart = match.index + match[0].indexOf(match[1] ?? ""); + const predicateStart = match.index + match[0].lastIndexOf(match[2] ?? ""); + if ( + match[1]?.toLowerCase() !== "authorization" || + !isPermissionStatusClause(content, fieldStart, predicateStart) + ) + return true; + } for (const match of content.matchAll(SENSITIVE_DOCUMENT_ASSIGNMENT)) { const keyStart = match.index + match[0].indexOf(match[1] ?? ""); const prefix = content.slice( diff --git a/tests/inspection-capture.test.ts b/tests/inspection-capture.test.ts index 9b294a3d..79c81122 100644 --- a/tests/inspection-capture.test.ts +++ b/tests/inspection-capture.test.ts @@ -291,3 +291,216 @@ for (const suffix of [ }); }); } + +for (const content of [ + "Separate authorization is required before product repairs.\n", + "Authorization\nis required before repairs.\n", + "Authorization is needed before changing product files.\n", + "Explicit authorization was requested before repair work.\n", + "Authorization is pending for repairs.\n", + "Authorization is granted for the repair task.\n", + "Authorization is denied for deployment.\n", + "Prior authorization was obtained to change product files.\n", + "The authorization is necessary before code modification.\n", + "Authorization is not required before this inspection.\n", + "Authorization was not granted for repairs.\n", + "- Separate authorization is required before product repairs.\n", + "## Repair scope\nSeparate authorization is required before product repairs.\n", + "Separate authorization is required before product repairs.\n\n## Key findings\nThe parser has a defect.\n", + "Separate authorization is required before repairs; authorization is needed to modify code.\n", +]) { + test(`complete permission status is retained ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "observed", + content, + sha256: `sha256:${createHash("sha256").update(content).digest("hex")}`, + }); + }); +} + +for (const content of [ + "Authorization is required.\n", + "Authorization is opaque-short.\n", + "Authorization is required opaque-short.\n", + "Authorization is required before repairs opaque-short.\n", + "Authorization is required before repairs.opaque-short\n", + "Authorization is required before\nopaque-short.\n", + "Authorization is required before repairs; the token is opaque-short.\n", + "Authorization is required before repairs. The authorization is required.\n", + "Authorization is required before repairs.\n\nThe authorization is opaque-short.\n", + "Authorization is required before repairs.\n\nThe token is opaque-short.\n", + "API_AUTHORIZATION is required before repairs.\n", + "authorizationToken is required before repairs.\n", + "The credential is required before repairs.\n", + '"authorization" is required before repairs.\n', + "`authorization` is required before repairs.\n", + "`Authorization is required before repairs.`\n", + "```text\nAuthorization is required before repairs.\n```\n", + "~~~text\nAuthorization is required before repairs.\n~~~\n", + "````text\n```\nAuthorization is required before repairs.\n````\n", + " Authorization is required before repairs.\n", + "\tAuthorization is required before repairs.\n", + "Authorization\n is required before repairs.\n", + "Authorization\n\tis required before repairs.\n", + "## Authorization is required before repairs.\n", + "## Repair scope. Authorization is required before repairs.\n", + "Authorization: is required before repairs.\n", + "Authorization=is required before repairs.\n", +]) { + test(`ambiguous or encoded permission value remains private ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "unavailable", + reason: "review-document-not-safe-to-retain", + }); + }); +} + +for (const lineEnding of ["\n", "\r\n", "\r"]) { + for (const spaces of [0, 1, 2, 3]) { + for (const content of [ + `Repair scope${lineEnding}${" ".repeat(spaces)}\tAuthorization is required before repairs.${lineEnding}`, + `Authorization${lineEnding}${" ".repeat(spaces)}\tis required before repairs.${lineEnding}`, + ]) { + test(`tab-indented permission context remains private ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "unavailable", + reason: "review-document-not-safe-to-retain", + }); + }); + } + for (const content of [ + `Repair scope${lineEnding}${" ".repeat(spaces)}Authorization is required before repairs.${lineEnding}`, + `Authorization${lineEnding}${" ".repeat(spaces)}is required before repairs.${lineEnding}`, + ]) { + test(`ordinary prose indentation retains permission context ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "observed", + content, + sha256: `sha256:${createHash("sha256").update(content).digest("hex")}`, + }); + }); + } + } + for (const fence of ["```", "~~~"]) { + const encoded = `${fence}text${lineEnding}Authorization is required before repairs.${lineEnding}${fence}${lineEnding}`; + test(`active fence permission remains private ${encoded}`, async () => { + expect(await observe(encoded)).toEqual({ + kind: "unavailable", + reason: "review-document-not-safe-to-retain", + }); + }); + const prose = `${fence}text${lineEnding}Ordinary example.${lineEnding}${fence}${lineEnding}Authorization is required before repairs.${lineEnding}`; + test(`closed fence permits ordinary permission prose ${prose}`, async () => { + expect(await observe(prose)).toEqual({ + kind: "observed", + content: prose, + sha256: `sha256:${createHash("sha256").update(prose).digest("hex")}`, + }); + }); + } + const content = `Authorization is required before repairs${lineEnding}Next action.${lineEnding}`; + test(`physical line ends the permission predicate ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "observed", + content, + sha256: `sha256:${createHash("sha256").update(content).digest("hex")}`, + }); + }); + for (const content of [ + `Authorization${lineEnding} is required before repairs.${lineEnding}`, + `Repair scope${lineEnding} Authorization is required before repairs.${lineEnding}`, + ]) { + test(`space-indented permission context remains private ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "unavailable", + reason: "review-document-not-safe-to-retain", + }); + }); + } +} + +for (const lineEnding of ["\n", "\r\n", "\r"]) { + const markers = [ + ...([0, 1, 2, 3] as const).map((spaces) => `${" ".repeat(spaces)}> `), + ">> ", + "> > ", + "- > ", + "1. > ", + "1. - > ", + ]; + for (const marker of markers) { + for (const content of [ + `${marker}Status. Authorization is required before repairs.${lineEnding}`, + `${marker}Status.${lineEnding}Authorization is required before repairs.${lineEnding}`, + ]) { + test(`quoted permission paragraph remains private ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "unavailable", + reason: "review-document-not-safe-to-retain", + }); + }); + } + } + for (const separator of [ + lineEnding, + `>${lineEnding}`, + `## Repair scope${lineEnding}`, + ]) { + const content = `> Status.${lineEnding}${separator}Authorization is required before repairs.${lineEnding}`; + test(`permission prose outside a quoted paragraph is retained ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "observed", + content, + sha256: `sha256:${createHash("sha256").update(content).digest("hex")}`, + }); + }); + } + for (const spaces of [0, 1, 2, 3]) { + const content = `> Status.${lineEnding}${" ".repeat(spaces)}Authorization is required before repairs.${lineEnding}`; + test(`partial plain prefix remains in its lazy quote ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "unavailable", + reason: "review-document-not-safe-to-retain", + }); + }); + } + const quoted = `> Authorization is required before repairs.${lineEnding}`; + test(`partial quote marker remains private ${quoted}`, async () => { + expect(await observe(quoted)).toEqual({ + kind: "unavailable", + reason: "review-document-not-safe-to-retain", + }); + }); + const ordinary = `> Status.${lineEnding}- Authorization is required before repairs.${lineEnding}`; + test(`new list interrupts the quoted paragraph ${ordinary}`, async () => { + expect(await observe(ordinary)).toEqual({ + kind: "observed", + content: ordinary, + sha256: `sha256:${createHash("sha256").update(ordinary).digest("hex")}`, + }); + }); +} + +for (const lineEnding of ["\n", "\r\n", "\r"]) { + for (const fence of ["```", "~~~"]) { + const closed = `${fence}text${lineEnding}> shell output${lineEnding}${fence}${lineEnding}Authorization is required before repairs.${lineEnding}`; + test(`quote-like code cannot taint prose after its closed fence ${closed}`, async () => { + expect(await observe(closed)).toEqual({ + kind: "observed", + content: closed, + sha256: `sha256:${createHash("sha256").update(closed).digest("hex")}`, + }); + }); + for (const content of [ + `${fence}text${lineEnding}> shell output${lineEnding}Authorization is required before repairs.${lineEnding}`, + `> Status.${lineEnding}> ${fence}${lineEnding}Authorization is required before repairs.${lineEnding}`, + ]) { + test(`active code or quoted fence content remains private ${content}`, async () => { + expect(await observe(content)).toEqual({ + kind: "unavailable", + reason: "review-document-not-safe-to-retain", + }); + }); + } + } +}