diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e1d15..3d2e622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - A test manifest or test sequence saved with a UTF-8 byte-order mark is read correctly again. Until now such a file was rejected outright, so every test it registered quietly vanished from the graph with nothing but a warning to show for it — and on Windows that mark is what PowerShell writes by default. (#316) - Saved Access queries now appear in the graph no matter what order Dysflow writes them. The `queries.json` manifest is what marks the `.sql` files beside it as saved queries, and a manifest arriving after its queries left them unindexed until the next manual sync. Editing only the manifest — renaming a query, adding an entry — now re-indexes too. (#318) - Agents are now told about three Access surfaces the index already carried but never advertised: saved queries and the tables they touch, Dysflow test sequences alongside the manifests, and the table-and-column structure from the Access ERD export. An agent that was not told these exist fell back to reading the files by hand. (#320) +- Table and column names from an Access structure export keep their accents. The export is read with the same encoding rules as the rest of an Access project, so a table like `TbSituación` is no longer recorded under a corrupted name — which had also stopped it from lining up with the queries and code that reference it, leaving its column list attached to nothing. (#322) ## [1.17.1] - 2026-09-12 diff --git a/__tests__/index-orphan-watchdog.test.ts b/__tests__/index-orphan-watchdog.test.ts index a3d49e9..f728310 100644 --- a/__tests__/index-orphan-watchdog.test.ts +++ b/__tests__/index-orphan-watchdog.test.ts @@ -56,7 +56,12 @@ vi.mock('../src/project-config', () => ({ vi.mock('../src/resolution/frameworks', () => ({ detectFrameworks: () => [] })); vi.mock('../src/extraction/tree-sitter', () => ({ extractFromSource: vi.fn() })); -vi.mock('../src/extraction/grammars', () => ({ +// Spread the real module and override only what this test needs to control. +// A from-scratch object silently returns `undefined` for every grammars helper +// it forgot, so the orchestrator importing one more of them (e.g. +// `isAccessErdFile`, #322) failed here as an unrelated-looking index error. +vi.mock('../src/extraction/grammars', async (importOriginal) => ({ + ...(await importOriginal()), detectLanguage: () => 'typescript', isSourceFile: (file: string) => file.endsWith('.ts'), isLanguageSupported: () => true, diff --git a/__tests__/vba-erd-encoding.test.ts b/__tests__/vba-erd-encoding.test.ts new file mode 100644 index 0000000..6311525 --- /dev/null +++ b/__tests__/vba-erd-encoding.test.ts @@ -0,0 +1,166 @@ +/** + * Issue #322: a CP1252 Access ERD export must decode like the rest of the + * export, so its table names still join the graph. + * + * #53 gave `.bas` / `.cls` / `.form.txt` / `.report.txt` / `.sql` an + * encoding-robust read because Access writes Windows-1252 and accented + * identifiers are everywhere in Spanish-language Access projects. The Access + * structure export (`ERD/*.md`) was left on the plain UTF-8 path. + * + * The damage was not cosmetic. The `.sql` naming a table DID get the CP1252 + * fallback, so it emitted a placeholder named `TbSituación`; the ERD declaring + * that same table did NOT, so it declared `TbSituacin`. The two never + * matched: the declaration never joined the placeholder, and the table's + * fields hung off an orphan node. This suite pins both halves — the decode, + * and the join it exists to make possible. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { readVbaSource } from '../src/extraction/vba-source'; +import { usesAccessEncoding } from '../src/extraction'; +import CodeGraph from '../src/index'; + +/** The generated export's real byte shape: BOM, CRLF, `# Estructura de Datos:`. */ +const ERD_DOC = [ + '# Estructura de Datos: Datos.accdb', + '', + '## Tabla: TbSituación', + '| Campo | Tipo | Longitud |', + '| :--- | :--- | :--- |', + '| Código | 4 | 4 |', + '| Descripción | 10 | 255 |', + '', +].join('\r\n'); + +/** Same document, CP1252 on disk — one byte per accented character. */ +const cp1252 = (text: string) => Buffer.from(text, 'latin1'); +const utf8 = (text: string) => Buffer.from(text, 'utf8'); + +const openProjects: Array<{ cg: CodeGraph; dir: string }> = []; + +afterEach(async () => { + while (openProjects.length > 0) { + const { cg, dir } = openProjects.pop()!; + try { + await cg.close(); + } catch { + // ignore close errors + } + await new Promise((r) => setTimeout(r, 50)); + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // ignore cleanup EPERM on Windows + } + } +}); + +describe('Access ERD export — encoding (#322)', () => { + it('routes the ERD export by path shape, and nothing else markdown', () => { + // `.md` must never become a family EXTENSION — that would pull every + // markdown file in every repo through the CP1252 fallback. Only a document + // directly inside an `ERD/` directory qualifies. + for (const p of [ + 'ERD/Estructura_Datos.md', + 'src/backend/erd/Estructura_Datos.md', + 'C:\\proj\\ERD\\Estructura_Datos.md', + ]) { + expect(usesAccessEncoding(p), p).toBe(true); + } + for (const p of [ + 'docs/architecture.md', + 'README.md', + 'ERD/notes/deep.md', + 'index.ts', + ]) { + expect(usesAccessEncoding(p), p).toBe(false); + } + // The extension half still routes on its own. + expect(usesAccessEncoding('src/modules/constantes.bas')).toBe(true); + }); + + it('decodes a CP1252 export to the same names as its UTF-8 twin', () => { + const asUtf8 = readVbaSource('ERD/Estructura_Datos.md', { + readFile: () => utf8(ERD_DOC), + }); + let fallbacks = 0; + const asCp1252 = readVbaSource('ERD/Estructura_Datos.md', { + readFile: () => cp1252(ERD_DOC), + onFallback: () => { fallbacks++; }, + }); + + expect(asCp1252.text).toBe(asUtf8.text); + expect(asCp1252.text).toContain('TbSituación'); + expect(asCp1252.text).toContain('Código'); + // The fallback is what did the work; a silent pass would mean the bytes + // happened to be valid UTF-8 and the test proves nothing. + expect(fallbacks).toBe(1); + }); + + it('leaves a UTF-8 export with a BOM byte-identical', () => { + const withBom = Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + utf8(ERD_DOC), + ]); + let fallbacks = 0; + const r = readVbaSource('ERD/Estructura_Datos.md', { + readFile: () => withBom, + onFallback: () => { fallbacks++; }, + }); + + expect(r.text).toBe(ERD_DOC); + expect(r.bomStripped).toBe(true); + expect(fallbacks).toBe(0); + }); + + it('joins a CP1252 ERD table to the query that references it', async () => { + // The failure this issue is really about: both files are CP1252 on disk, + // but only one of them used to be decoded as such, so the two spellings of + // the same table name never met. + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-erd-enc-')); + fs.mkdirSync(path.join(dir, 'ERD'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'queries'), { recursive: true }); + + fs.writeFileSync(path.join(dir, 'ERD', 'Estructura_Datos.md'), cp1252(ERD_DOC)); + fs.writeFileSync( + path.join(dir, 'queries', 'Q_Situaciones.sql'), + cp1252('SELECT * FROM TbSituación;\r\n'), + ); + fs.writeFileSync( + path.join(dir, 'queries', 'queries.json'), + '[{ "name": "Q_Situaciones", "file": "Q_Situaciones.sql" }]', + ); + + const cg = await CodeGraph.init(dir, { index: false }); + openProjects.push({ cg, dir }); + await cg.indexAll(); + + // The ERD declared the table and its fields under their real names. A + // replacement character anywhere here is the exact signature of the bug. + const erdNodes = cg.getNodesInFile('ERD/Estructura_Datos.md'); + const names = erdNodes.map((n) => n.name); + expect(names).toContain('TbSituación'); + expect(names).toContain('Código'); + expect(names).toContain('Descripción'); + expect(names.filter((n) => (n ?? '').includes('�'))).toEqual([]); + + // And the two sides meet: the saved query's table reference resolves onto + // the ERD's own declaration rather than a second, differently-spelled node. + const table = erdNodes.find( + (n) => n.kind === 'class' && n.name === 'TbSituación', + ); + expect(table).toBeDefined(); + + const query = cg + .getNodesInFile('queries/Q_Situaciones.sql') + .find((n) => n.kind === 'query'); + expect(query).toBeDefined(); + + const toTable = cg + .getOutgoingEdges(query!.id) + .filter((e) => e.target === table!.id); + expect(toTable.length).toBeGreaterThan(0); + }); +}); diff --git a/__tests__/vba-source.test.ts b/__tests__/vba-source.test.ts index b3d9c78..4e03ccb 100644 --- a/__tests__/vba-source.test.ts +++ b/__tests__/vba-source.test.ts @@ -134,6 +134,11 @@ describe('isVbaFamilyFile - extension routing (Issue #53)', () => { 'Weird.clsss', 'MyModule.txt', // NOT `.form.txt` / `.report.txt` 'notes.form.md', + // `.md` is never a family EXTENSION. The Access structure export shares + // the decoding but is matched by path shape at the read site instead — + // see `usesAccessEncoding` and its tests (#322). + 'ERD/Estructura_Datos.md', + 'docs/architecture.md', ]) { expect(isVbaFamilyFile(p)).toBe(false); } diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 8d76dab..af799d1 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -24,7 +24,7 @@ import { extractFromSource } from './tree-sitter'; import type { VbaExtractionOptions } from './vba/options'; import { readVbaSource, isVbaFamilyFile } from './vba-source'; import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs } from './parse-pool'; -import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes } from './grammars'; +import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes, isAccessErdFile } from './grammars'; import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadVbaConfig, loadIncludePatterns, loadDysflowExportConfig } from '../project-config'; import { isCodeGraphDataDir } from '../directory'; import { logDebug, logWarn } from '../errors'; @@ -130,15 +130,38 @@ export function hashContent(content: string): string { * change-detection hashing so a BOM-carrying or CP1252 VBA file doesn't hash * differently at detect-time than what was stored (perpetual "modified"). */ +/** + * Files that must be decoded with Access encoding rules (BOM strip + CP1252 + * fallback) rather than as plain UTF-8. + * + * Two predicates because the family is matched two ways. `isVbaFamilyFile` + * owns the extensions (`.bas`, `.cls`, `.form.txt`, `.sql`, …). The Access + * structure export is `ERD/*.md`, so it is matched by PATH SHAPE — `.md` + * alone must never route every markdown file in a repo through the CP1252 + * fallback — and that shape is owned by `grammars.ts` (issue #322). + * + * Combining them here rather than inside `isVbaFamilyFile` keeps + * `vba-source.ts` a leaf module: this is the only place that needs both. + * + * Without the ERD half, an export naming a table with accents decoded to + * replacement characters while the `.sql` and `.bas` referencing that same + * table decoded it correctly, so the ERD declaration never joined the + * placeholder those scanners emit and the table's field list hung off an + * orphan node. + */ +export function usesAccessEncoding(filePath: string): boolean { + return isVbaFamilyFile(filePath) || isAccessErdFile(filePath); +} + async function readForExtraction(fullPath: string): Promise { - return isVbaFamilyFile(fullPath) + return usesAccessEncoding(fullPath) ? readVbaSource(fullPath).text : await fsp.readFile(fullPath, 'utf-8'); } /** Synchronous counterpart of {@link readForExtraction} for the sync read sites. */ function readForExtractionSync(fullPath: string): string { - return isVbaFamilyFile(fullPath) + return usesAccessEncoding(fullPath) ? readVbaSource(fullPath).text : fs.readFileSync(fullPath, 'utf-8'); } diff --git a/src/extraction/vba-source.ts b/src/extraction/vba-source.ts index 2dba000..ac36633 100644 --- a/src/extraction/vba-source.ts +++ b/src/extraction/vba-source.ts @@ -15,7 +15,10 @@ * Scope (do NOT change decoding for tree-sitter languages): only * `.bas`/`.cls`/`.frm`/`.dsr`/`.form.txt`/`.report.txt`/`.sql` files get * the BOM strip + CP1252 fallback; everything else stays byte-identical to - * the existing `fs.readFile(path, 'utf-8')` path. + * the existing `fs.readFile(path, 'utf-8')` path. The Access structure export + * (`ERD/*.md`) shares that decoding but is matched by path shape, so the + * combined predicate lives at the read site — see `usesAccessEncoding` in + * `./index.ts` (issue #322). */ import * as fs from 'fs';