fix(sync): record only indexed files as synthesis inputs

CHANGELOG.md

@@ -14,6 +14,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixes
 
+- `codegraph sync` no longer fails with "FOREIGN KEY constraint failed" in a project whose code lazy-loads a workspace package, like Supabase Studio's `dynamic(() => import('icons').then((mod) => mod.EdgeFunctions))` or vben's `() => import('@vben/layouts').then((m) => m.IFrameView)`. Before, every sync after an edit in such a project rolled back, so the index stopped following your changes until a full re-index. Thanks @bompus for the report and @inth3shadows. (#2565)
+
 - In VB.NET, every member of a `Structure` is now indexed, including its fields, properties, methods, constructors and nested enums. Before, only the first member was, so the rest could not be found and their callers looked empty.
 - In VB.NET and C#, what a property's `Get` and `Set` code calls, creates and reads now belongs to that property, as do C#'s `get => …` accessors and `=> …` property bodies. Before, it was dropped, so a method used only from a property looked unused.
 - In VB.NET, a field or property initializer like `= Compute()` or `As New List(Of Order)` now links what it calls and creates, and so do a `Custom Event`'s `AddHandler`, `RemoveHandler` and `RaiseEvent` blocks. Re-index VB.NET and C# projects after upgrading.

__tests__/sync-synthesis-inputs.test.ts

@@ -0,0 +1,124 @@
+/**
+ * A sync records only indexed files as synthesis inputs (#2565).
+ *
+ * A lazy value that loads a workspace package, `dynamic(() =>
+ * import('icons').then((m) => m.EdgeFunctions))`, names the package's
+ * directory as the module it reads. Synthesis recorded that directory as an
+ * input, and `synthesis_inputs.file_path` references `files(path)`, so the
+ * commit failed with "FOREIGN KEY constraint failed" and the whole sync rolled
+ * back. Every later sync failed the same way. A full index of the same tree
+ * succeeded.
+ */
+import { describe, it, expect, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import CodeGraph from '../src/index';
+import { DatabaseConnection } from '../src/db';
+import { QueryBuilder } from '../src/db/queries';
+import { SynthesisStage } from '../src/db/synthesis-stage';
+
+type Files = Record<string, string>;
+
+/** A pnpm workspace whose app lazy-loads a component from a workspace package (supabase studio's shape). */
+const WORKSPACE: Files = {
+  'pnpm-workspace.yaml': "packages:\n  - 'packages/*'\n",
+  'package.json': JSON.stringify({ name: 'app', private: true, workspaces: ['packages/*'], dependencies: { react: '^18.0.0', next: '^14.0.0' } }),
+  'packages/icons/package.json': JSON.stringify({ name: 'icons', main: './src/icons/index.ts' }),
+  'packages/icons/src/icons/index.ts': 'export function EdgeFunctions() {\n  return null;\n}\n',
+  'src/App.tsx':
+    "import dynamic from 'next/dynamic';\n" +
+    "const Chart = dynamic(() => import('icons').then((mod) => mod.EdgeFunctions));\n" +
+    'export function App() {\n  return <Chart />;\n}\n',
+};
+
+let roots: string[] = [];
+let graphs: CodeGraph[] = [];
+
+afterEach(() => {
+  for (const graph of graphs) graph.close();
+  graphs = [];
+  for (const root of roots) fs.rmSync(root, { recursive: true, force: true });
+  roots = [];
+});
+
+function write(root: string, files: Files): void {
+  for (const [file, text] of Object.entries(files)) {
+    fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
+    fs.writeFileSync(path.join(root, file), text);
+  }
+}
+
+async function indexed(files: Files): Promise<{ root: string; graph: CodeGraph }> {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-synth-inputs-'));
+  roots.push(root);
+  write(root, files);
+  const graph = await CodeGraph.init(root, { index: true });
+  graphs.push(graph);
+  return { root, graph };
+}
+
+/** Every edge, by its ends' natural keys. */
+function edgesOf(graph: CodeGraph): string[] {
+  const keys = new Map(
+    graph.getFiles()
+      .flatMap((file) => graph.getNodesInFile(file.path))
+      .map((n) => [n.id, `${n.kind} ${n.filePath}:${n.qualifiedName}:${n.startLine}`] as const)
+  );
+  return graph.getOutgoingEdgesFrom([...keys.keys()])
+    .map((e) => `${e.kind} ${keys.get(e.source)} -> ${keys.get(e.target) ?? e.target} @${e.line}:${e.column} ${JSON.stringify(e.metadata ?? {})}`)
+    .sort();
+}
+
+describe('a sync whose lazy value loads a workspace package', () => {
+  it('commits instead of failing on the package directory, and matches a fresh index', async () => {
+    const { root, graph } = await indexed(WORKSPACE);
+    const appended = { 'src/App.tsx': `${WORKSPACE['src/App.tsx']}// touched\n` };
+    write(root, appended);
+    const result = await graph.sync();
+    expect(result.filesModified).toBe(1);
+
+    // A second sync after another edit still works: nothing was left half-done.
+    write(root, { 'src/App.tsx': `${appended['src/App.tsx']}// touched again\n` });
+    expect((await graph.sync()).filesModified).toBe(1);
+
+    const fresh = await indexed({ ...WORKSPACE, 'src/App.tsx': `${appended['src/App.tsx']}// touched again\n` });
+    expect(edgesOf(graph)).toEqual(edgesOf(fresh.graph));
+  }, 120_000);
+});
+
+describe('recording synthesis inputs', () => {
+  let connection: DatabaseConnection;
+  let queries: QueryBuilder;
+
+  /** A fresh database holding one indexed file, `src/App.tsx`. */
+  function freshDb(): void {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-synth-inputs-db-'));
+    roots.push(dir);
+    connection = DatabaseConnection.initialize(path.join(dir, 'test.db'));
+    queries = new QueryBuilder(connection.getDb());
+    queries.upsertFile({ path: 'src/App.tsx', contentHash: 'h', language: 'tsx', size: 1, modifiedAt: 1, indexedAt: 1, nodeCount: 0, errors: [], generated: false } as never);
+  }
+  const recorded = (): string[] =>
+    (connection.getDb().prepare('SELECT file_path FROM synthesis_inputs ORDER BY file_path').all() as Array<{ file_path: string }>).map((r) => r.file_path);
+
+  afterEach(() => connection?.close());
+
+  it('keeps only indexed files when written straight to the index', () => {
+    freshDb();
+    queries.replaceSynthesisInputs(['packages/icons', 'src/App.tsx', 'src/deleted.ts']);
+    expect(recorded()).toEqual(['src/App.tsx']);
+  });
+
+  it('keeps only indexed files when a stage publishes them, whatever the stage was handed', async () => {
+    freshDb();
+    const stage = new SynthesisStage(connection.getPath());
+    try {
+      // A file that leaves the index while the passes run reaches the stage's
+      // own list unchecked; publishing must still only keep indexed files.
+      stage.db.exec("INSERT INTO temp.synthesis_inputs (file_path) VALUES ('packages/icons'), ('src/App.tsx'), ('src/deleted.ts')");
+      await stage.publish();
+    } finally { stage.close(); }
+    expect(recorded()).toEqual(['src/App.tsx']);
+  });
+});

src/db/queries.ts

@@ -2126,10 +2126,11 @@ export class QueryBuilder {
     return !!this.db.prepare('SELECT 1 FROM synthesis_inputs WHERE file_path = ?').get(filePath);
   }
 
+  /** Record the files synthesis read; a path the index doesn't hold as a file (a package directory) is not one (#2565). */
   replaceSynthesisInputs(files: string[]): void {
     this.db.transaction(() => {
       this.db.exec('DELETE FROM synthesis_inputs');
-      const insert = this.db.prepare('INSERT INTO synthesis_inputs(file_path) VALUES (?)');
+      const insert = this.db.prepare('INSERT OR IGNORE INTO synthesis_inputs(file_path) SELECT path FROM files WHERE path = ?');
       for (const file of files) insert.run(file);
     })();
   }

src/db/synthesis-stage.ts

@@ -70,8 +70,14 @@ export class SynthesisStage {
         insert.run(i, i + 2000);
         await yieldToLoop();
       }
+      // Only indexed files: an input names a file the passes read, but a lazy
+      // value's module can resolve to a workspace package's directory
+      // (`import('icons')`), and a file can leave the index while the passes
+      // run. Either broke the foreign key into files(path) and rolled the whole
+      // sync back (#2565).
       this.db.exec(`DELETE FROM main.synthesis_inputs;
-        INSERT INTO main.synthesis_inputs SELECT * FROM temp.synthesis_inputs`);
+        INSERT INTO main.synthesis_inputs (file_path)
+          SELECT file_path FROM temp.synthesis_inputs WHERE file_path IN (SELECT path FROM main.files)`);
       this.db.exec('COMMIT');
     } catch (error) {
       this.db.exec('ROLLBACK');