From 2b4865cb5dd9253700654185ff25c77226b99011 Mon Sep 17 00:00:00 2001 From: andres Date: Sat, 12 Sep 2026 22:14:15 +0200 Subject: [PATCH 1/2] fix(vba): read the Access ERD export with Access encoding rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #53 gave the VBA family an encoding-robust read because Access writes Windows-1252 and accented identifiers are everywhere in Spanish-language projects, but it scoped that to extensions. The Access structure export is ERD/*.md, so it stayed on the plain utf-8 path. The damage was not cosmetic. A .sql naming a table DID get the CP1252 fallback and emitted a placeholder named TbSituación; the ERD declaring that same table did NOT and declared a replacement-character spelling. The two never matched, so the declaration never joined the placeholder and the table's field list hung off an orphan node — for exactly the tables whose names carry accents. Widen isVbaFamilyFile with the ERD path shape, reusing isAccessErdFile from grammars so the shape has one owner. Matching on shape rather than extension is load-bearing: .md alone would pull every markdown file in a repo through the CP1252 fallback. Unlike the JSON artifacts in #316, the fallback is correct here — the ERD document comes out of the same Access tooling as the rest of the export and shares its encoding provenance, where JSON is UTF-8 by specification and must fail loudly instead. Refs #322 --- CHANGELOG.md | 1 + __tests__/vba-erd-encoding.test.ts | 142 +++++++++++++++++++++++++++++ __tests__/vba-source.test.ts | 9 ++ src/extraction/vba-source.ts | 20 +++- 4 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 __tests__/vba-erd-encoding.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e1d152..3d2e6224 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - A test manifest or test sequence saved with a UTF-8 byte-order mark is read correctly again. Until now such a file was rejected outright, so every test it registered quietly vanished from the graph with nothing but a warning to show for it — and on Windows that mark is what PowerShell writes by default. (#316) - Saved Access queries now appear in the graph no matter what order Dysflow writes them. The `queries.json` manifest is what marks the `.sql` files beside it as saved queries, and a manifest arriving after its queries left them unindexed until the next manual sync. Editing only the manifest — renaming a query, adding an entry — now re-indexes too. (#318) - Agents are now told about three Access surfaces the index already carried but never advertised: saved queries and the tables they touch, Dysflow test sequences alongside the manifests, and the table-and-column structure from the Access ERD export. An agent that was not told these exist fell back to reading the files by hand. (#320) +- Table and column names from an Access structure export keep their accents. The export is read with the same encoding rules as the rest of an Access project, so a table like `TbSituación` is no longer recorded under a corrupted name — which had also stopped it from lining up with the queries and code that reference it, leaving its column list attached to nothing. (#322) ## [1.17.1] - 2026-09-12 diff --git a/__tests__/vba-erd-encoding.test.ts b/__tests__/vba-erd-encoding.test.ts new file mode 100644 index 00000000..27b0c18a --- /dev/null +++ b/__tests__/vba-erd-encoding.test.ts @@ -0,0 +1,142 @@ +/** + * Issue #322: a CP1252 Access ERD export must decode like the rest of the + * export, so its table names still join the graph. + * + * #53 gave `.bas` / `.cls` / `.form.txt` / `.report.txt` / `.sql` an + * encoding-robust read because Access writes Windows-1252 and accented + * identifiers are everywhere in Spanish-language Access projects. The Access + * structure export (`ERD/*.md`) was left on the plain UTF-8 path. + * + * The damage was not cosmetic. The `.sql` naming a table DID get the CP1252 + * fallback, so it emitted a placeholder named `TbSituación`; the ERD declaring + * that same table did NOT, so it declared `TbSituacin`. The two never + * matched: the declaration never joined the placeholder, and the table's + * fields hung off an orphan node. This suite pins both halves — the decode, + * and the join it exists to make possible. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { readVbaSource } from '../src/extraction/vba-source'; +import CodeGraph from '../src/index'; + +/** The generated export's real byte shape: BOM, CRLF, `# Estructura de Datos:`. */ +const ERD_DOC = [ + '# Estructura de Datos: Datos.accdb', + '', + '## Tabla: TbSituación', + '| Campo | Tipo | Longitud |', + '| :--- | :--- | :--- |', + '| Código | 4 | 4 |', + '| Descripción | 10 | 255 |', + '', +].join('\r\n'); + +/** Same document, CP1252 on disk — one byte per accented character. */ +const cp1252 = (text: string) => Buffer.from(text, 'latin1'); +const utf8 = (text: string) => Buffer.from(text, 'utf8'); + +const openProjects: Array<{ cg: CodeGraph; dir: string }> = []; + +afterEach(async () => { + while (openProjects.length > 0) { + const { cg, dir } = openProjects.pop()!; + try { + await cg.close(); + } catch { + // ignore close errors + } + await new Promise((r) => setTimeout(r, 50)); + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // ignore cleanup EPERM on Windows + } + } +}); + +describe('Access ERD export — encoding (#322)', () => { + it('decodes a CP1252 export to the same names as its UTF-8 twin', () => { + const asUtf8 = readVbaSource('ERD/Estructura_Datos.md', { + readFile: () => utf8(ERD_DOC), + }); + let fallbacks = 0; + const asCp1252 = readVbaSource('ERD/Estructura_Datos.md', { + readFile: () => cp1252(ERD_DOC), + onFallback: () => { fallbacks++; }, + }); + + expect(asCp1252.text).toBe(asUtf8.text); + expect(asCp1252.text).toContain('TbSituación'); + expect(asCp1252.text).toContain('Código'); + // The fallback is what did the work; a silent pass would mean the bytes + // happened to be valid UTF-8 and the test proves nothing. + expect(fallbacks).toBe(1); + }); + + it('leaves a UTF-8 export with a BOM byte-identical', () => { + const withBom = Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + utf8(ERD_DOC), + ]); + let fallbacks = 0; + const r = readVbaSource('ERD/Estructura_Datos.md', { + readFile: () => withBom, + onFallback: () => { fallbacks++; }, + }); + + expect(r.text).toBe(ERD_DOC); + expect(r.bomStripped).toBe(true); + expect(fallbacks).toBe(0); + }); + + it('joins a CP1252 ERD table to the query that references it', async () => { + // The failure this issue is really about: both files are CP1252 on disk, + // but only one of them used to be decoded as such, so the two spellings of + // the same table name never met. + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-erd-enc-')); + fs.mkdirSync(path.join(dir, 'ERD'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'queries'), { recursive: true }); + + fs.writeFileSync(path.join(dir, 'ERD', 'Estructura_Datos.md'), cp1252(ERD_DOC)); + fs.writeFileSync( + path.join(dir, 'queries', 'Q_Situaciones.sql'), + cp1252('SELECT * FROM TbSituación;\r\n'), + ); + fs.writeFileSync( + path.join(dir, 'queries', 'queries.json'), + '[{ "name": "Q_Situaciones", "file": "Q_Situaciones.sql" }]', + ); + + const cg = await CodeGraph.init(dir, { index: false }); + openProjects.push({ cg, dir }); + await cg.indexAll(); + + // The ERD declared the table and its fields under their real names. A + // replacement character anywhere here is the exact signature of the bug. + const erdNodes = cg.getNodesInFile('ERD/Estructura_Datos.md'); + const names = erdNodes.map((n) => n.name); + expect(names).toContain('TbSituación'); + expect(names).toContain('Código'); + expect(names).toContain('Descripción'); + expect(names.filter((n) => (n ?? '').includes('�'))).toEqual([]); + + // And the two sides meet: the saved query's table reference resolves onto + // the ERD's own declaration rather than a second, differently-spelled node. + const table = erdNodes.find( + (n) => n.kind === 'class' && n.name === 'TbSituación', + ); + expect(table).toBeDefined(); + + const query = cg + .getNodesInFile('queries/Q_Situaciones.sql') + .find((n) => n.kind === 'query'); + expect(query).toBeDefined(); + + const toTable = cg + .getOutgoingEdges(query!.id) + .filter((e) => e.target === table!.id); + expect(toTable.length).toBeGreaterThan(0); + }); +}); diff --git a/__tests__/vba-source.test.ts b/__tests__/vba-source.test.ts index b3d9c787..4ecefd63 100644 --- a/__tests__/vba-source.test.ts +++ b/__tests__/vba-source.test.ts @@ -120,6 +120,11 @@ describe('isVbaFamilyFile - extension routing (Issue #53)', () => { 'MiClase.CLS', 'Form_Login.FORM.TXT', 'qryGetRiesgos.SQL', + // The Access structure export is matched by path shape, not extension + // (#322) — it carries the same encoding as the rest of the export. + 'ERD/Estructura_Datos.md', + 'src/backend/erd/Estructura_Datos.md', + 'C:\\proj\\ERD\\Estructura_Datos.md', ]) { expect(isVbaFamilyFile(p)).toBe(true); } @@ -134,6 +139,10 @@ describe('isVbaFamilyFile - extension routing (Issue #53)', () => { 'Weird.clsss', 'MyModule.txt', // NOT `.form.txt` / `.report.txt` 'notes.form.md', + // `.md` alone must never route markdown through the CP1252 fallback; + // only a document directly inside an `ERD/` directory qualifies (#322). + 'docs/architecture.md', + 'ERD/notes/deep.md', ]) { expect(isVbaFamilyFile(p)).toBe(false); } diff --git a/src/extraction/vba-source.ts b/src/extraction/vba-source.ts index 2dba0008..5c77cd51 100644 --- a/src/extraction/vba-source.ts +++ b/src/extraction/vba-source.ts @@ -13,11 +13,13 @@ * `Attribute VB_Name` detection → wrong module name). * * Scope (do NOT change decoding for tree-sitter languages): only - * `.bas`/`.cls`/`.frm`/`.dsr`/`.form.txt`/`.report.txt`/`.sql` files get - * the BOM strip + CP1252 fallback; everything else stays byte-identical to - * the existing `fs.readFile(path, 'utf-8')` path. + * `.bas`/`.cls`/`.frm`/`.dsr`/`.form.txt`/`.report.txt`/`.sql` files and the + * Access structure export (`ERD/*.md`, matched by path shape) get the BOM + * strip + CP1252 fallback; everything else stays byte-identical to the + * existing `fs.readFile(path, 'utf-8')` path. */ import * as fs from 'fs'; +import { isAccessErdFile } from './grammars'; /** * Family of extensions routed through Dysflow's VBA pipeline (or the @@ -28,7 +30,17 @@ import * as fs from 'fs'; const VBA_FAMILY_RE = /\.(?:bas|cls|frm|dsr|form\.txt|report\.txt|sql)$/i; export function isVbaFamilyFile(filePath: string): boolean { - return VBA_FAMILY_RE.test(filePath); + // The Access structure export (`ERD/*.md`) comes out of the same Access + // tooling as everything else here and carries the same encoding, but it is + // matched by path shape rather than extension — `.md` on its own must NOT + // pull every markdown file in a repo through the CP1252 fallback. The shape + // itself is owned by `grammars.ts`, so the two stay in step (issue #322). + // + // Without this, an ERD naming a table with accents decoded to replacement + // characters while the `.sql` and `.bas` referencing that same table decoded + // it correctly, so the ERD declaration never joined the placeholder those + // scanners emit and the table's field list hung off an orphan node. + return VBA_FAMILY_RE.test(filePath) || isAccessErdFile(filePath); } /** From 031b8522c4e5212d50bc81fd4c44a92c1f7126b2 Mon Sep 17 00:00:00 2001 From: andres Date: Sat, 12 Sep 2026 22:26:42 +0200 Subject: [PATCH 2/2] fix(vba): keep vba-source a leaf and route the ERD read at the call site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first attempt put the ERD path shape inside isVbaFamilyFile by importing isAccessErdFile into vba-source.ts. That turned a leaf module into a grammars dependant and broke index-orphan-watchdog on CI: that suite mocks grammars with a from-scratch object, so the newly imported helper was undefined and the orchestrator's own index reported success: false — a failure that looked unrelated to encoding. Combine the two predicates at the read site instead, as usesAccessEncoding in extraction/index.ts, the only place that needs both. isVbaFamilyFile goes back to owning extensions only, grammars keeps owning the ERD path shape, and vba-source.ts stays a leaf. Also spread the real module in that suite's grammars mock instead of rebuilding it. A from-scratch mock silently returns undefined for every helper it forgot, so any future import from grammars fails there as an unrelated-looking index error; the intentional stubs still override. Refs #322 --- __tests__/index-orphan-watchdog.test.ts | 7 +++++- __tests__/vba-erd-encoding.test.ts | 24 ++++++++++++++++++++ __tests__/vba-source.test.ts | 12 ++++------ src/extraction/index.ts | 29 ++++++++++++++++++++++--- src/extraction/vba-source.ts | 23 ++++++-------------- 5 files changed, 67 insertions(+), 28 deletions(-) diff --git a/__tests__/index-orphan-watchdog.test.ts b/__tests__/index-orphan-watchdog.test.ts index a3d49e94..f7283101 100644 --- a/__tests__/index-orphan-watchdog.test.ts +++ b/__tests__/index-orphan-watchdog.test.ts @@ -56,7 +56,12 @@ vi.mock('../src/project-config', () => ({ vi.mock('../src/resolution/frameworks', () => ({ detectFrameworks: () => [] })); vi.mock('../src/extraction/tree-sitter', () => ({ extractFromSource: vi.fn() })); -vi.mock('../src/extraction/grammars', () => ({ +// Spread the real module and override only what this test needs to control. +// A from-scratch object silently returns `undefined` for every grammars helper +// it forgot, so the orchestrator importing one more of them (e.g. +// `isAccessErdFile`, #322) failed here as an unrelated-looking index error. +vi.mock('../src/extraction/grammars', async (importOriginal) => ({ + ...(await importOriginal()), detectLanguage: () => 'typescript', isSourceFile: (file: string) => file.endsWith('.ts'), isLanguageSupported: () => true, diff --git a/__tests__/vba-erd-encoding.test.ts b/__tests__/vba-erd-encoding.test.ts index 27b0c18a..63115259 100644 --- a/__tests__/vba-erd-encoding.test.ts +++ b/__tests__/vba-erd-encoding.test.ts @@ -19,6 +19,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import * as os from 'node:os'; import { readVbaSource } from '../src/extraction/vba-source'; +import { usesAccessEncoding } from '../src/extraction'; import CodeGraph from '../src/index'; /** The generated export's real byte shape: BOM, CRLF, `# Estructura de Datos:`. */ @@ -57,6 +58,29 @@ afterEach(async () => { }); describe('Access ERD export — encoding (#322)', () => { + it('routes the ERD export by path shape, and nothing else markdown', () => { + // `.md` must never become a family EXTENSION — that would pull every + // markdown file in every repo through the CP1252 fallback. Only a document + // directly inside an `ERD/` directory qualifies. + for (const p of [ + 'ERD/Estructura_Datos.md', + 'src/backend/erd/Estructura_Datos.md', + 'C:\\proj\\ERD\\Estructura_Datos.md', + ]) { + expect(usesAccessEncoding(p), p).toBe(true); + } + for (const p of [ + 'docs/architecture.md', + 'README.md', + 'ERD/notes/deep.md', + 'index.ts', + ]) { + expect(usesAccessEncoding(p), p).toBe(false); + } + // The extension half still routes on its own. + expect(usesAccessEncoding('src/modules/constantes.bas')).toBe(true); + }); + it('decodes a CP1252 export to the same names as its UTF-8 twin', () => { const asUtf8 = readVbaSource('ERD/Estructura_Datos.md', { readFile: () => utf8(ERD_DOC), diff --git a/__tests__/vba-source.test.ts b/__tests__/vba-source.test.ts index 4ecefd63..4e03ccb6 100644 --- a/__tests__/vba-source.test.ts +++ b/__tests__/vba-source.test.ts @@ -120,11 +120,6 @@ describe('isVbaFamilyFile - extension routing (Issue #53)', () => { 'MiClase.CLS', 'Form_Login.FORM.TXT', 'qryGetRiesgos.SQL', - // The Access structure export is matched by path shape, not extension - // (#322) — it carries the same encoding as the rest of the export. - 'ERD/Estructura_Datos.md', - 'src/backend/erd/Estructura_Datos.md', - 'C:\\proj\\ERD\\Estructura_Datos.md', ]) { expect(isVbaFamilyFile(p)).toBe(true); } @@ -139,10 +134,11 @@ describe('isVbaFamilyFile - extension routing (Issue #53)', () => { 'Weird.clsss', 'MyModule.txt', // NOT `.form.txt` / `.report.txt` 'notes.form.md', - // `.md` alone must never route markdown through the CP1252 fallback; - // only a document directly inside an `ERD/` directory qualifies (#322). + // `.md` is never a family EXTENSION. The Access structure export shares + // the decoding but is matched by path shape at the read site instead — + // see `usesAccessEncoding` and its tests (#322). + 'ERD/Estructura_Datos.md', 'docs/architecture.md', - 'ERD/notes/deep.md', ]) { expect(isVbaFamilyFile(p)).toBe(false); } diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 8d76dab0..af799d15 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -24,7 +24,7 @@ import { extractFromSource } from './tree-sitter'; import type { VbaExtractionOptions } from './vba/options'; import { readVbaSource, isVbaFamilyFile } from './vba-source'; import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs } from './parse-pool'; -import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes } from './grammars'; +import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes, isAccessErdFile } from './grammars'; import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadVbaConfig, loadIncludePatterns, loadDysflowExportConfig } from '../project-config'; import { isCodeGraphDataDir } from '../directory'; import { logDebug, logWarn } from '../errors'; @@ -130,15 +130,38 @@ export function hashContent(content: string): string { * change-detection hashing so a BOM-carrying or CP1252 VBA file doesn't hash * differently at detect-time than what was stored (perpetual "modified"). */ +/** + * Files that must be decoded with Access encoding rules (BOM strip + CP1252 + * fallback) rather than as plain UTF-8. + * + * Two predicates because the family is matched two ways. `isVbaFamilyFile` + * owns the extensions (`.bas`, `.cls`, `.form.txt`, `.sql`, …). The Access + * structure export is `ERD/*.md`, so it is matched by PATH SHAPE — `.md` + * alone must never route every markdown file in a repo through the CP1252 + * fallback — and that shape is owned by `grammars.ts` (issue #322). + * + * Combining them here rather than inside `isVbaFamilyFile` keeps + * `vba-source.ts` a leaf module: this is the only place that needs both. + * + * Without the ERD half, an export naming a table with accents decoded to + * replacement characters while the `.sql` and `.bas` referencing that same + * table decoded it correctly, so the ERD declaration never joined the + * placeholder those scanners emit and the table's field list hung off an + * orphan node. + */ +export function usesAccessEncoding(filePath: string): boolean { + return isVbaFamilyFile(filePath) || isAccessErdFile(filePath); +} + async function readForExtraction(fullPath: string): Promise { - return isVbaFamilyFile(fullPath) + return usesAccessEncoding(fullPath) ? readVbaSource(fullPath).text : await fsp.readFile(fullPath, 'utf-8'); } /** Synchronous counterpart of {@link readForExtraction} for the sync read sites. */ function readForExtractionSync(fullPath: string): string { - return isVbaFamilyFile(fullPath) + return usesAccessEncoding(fullPath) ? readVbaSource(fullPath).text : fs.readFileSync(fullPath, 'utf-8'); } diff --git a/src/extraction/vba-source.ts b/src/extraction/vba-source.ts index 5c77cd51..ac366335 100644 --- a/src/extraction/vba-source.ts +++ b/src/extraction/vba-source.ts @@ -13,13 +13,14 @@ * `Attribute VB_Name` detection → wrong module name). * * Scope (do NOT change decoding for tree-sitter languages): only - * `.bas`/`.cls`/`.frm`/`.dsr`/`.form.txt`/`.report.txt`/`.sql` files and the - * Access structure export (`ERD/*.md`, matched by path shape) get the BOM - * strip + CP1252 fallback; everything else stays byte-identical to the - * existing `fs.readFile(path, 'utf-8')` path. + * `.bas`/`.cls`/`.frm`/`.dsr`/`.form.txt`/`.report.txt`/`.sql` files get + * the BOM strip + CP1252 fallback; everything else stays byte-identical to + * the existing `fs.readFile(path, 'utf-8')` path. The Access structure export + * (`ERD/*.md`) shares that decoding but is matched by path shape, so the + * combined predicate lives at the read site — see `usesAccessEncoding` in + * `./index.ts` (issue #322). */ import * as fs from 'fs'; -import { isAccessErdFile } from './grammars'; /** * Family of extensions routed through Dysflow's VBA pipeline (or the @@ -30,17 +31,7 @@ import { isAccessErdFile } from './grammars'; const VBA_FAMILY_RE = /\.(?:bas|cls|frm|dsr|form\.txt|report\.txt|sql)$/i; export function isVbaFamilyFile(filePath: string): boolean { - // The Access structure export (`ERD/*.md`) comes out of the same Access - // tooling as everything else here and carries the same encoding, but it is - // matched by path shape rather than extension — `.md` on its own must NOT - // pull every markdown file in a repo through the CP1252 fallback. The shape - // itself is owned by `grammars.ts`, so the two stay in step (issue #322). - // - // Without this, an ERD naming a table with accents decoded to replacement - // characters while the `.sql` and `.bas` referencing that same table decoded - // it correctly, so the ERD declaration never joined the placeholder those - // scanners emit and the table's field list hung off an orphan node. - return VBA_FAMILY_RE.test(filePath) || isAccessErdFile(filePath); + return VBA_FAMILY_RE.test(filePath); } /**