From 7ba8eee4a78920e9f637624d351e042f0fc98772 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Mon, 12 Oct 2026 07:10:08 +0000 Subject: [PATCH] fix(sync): imports through a re-exporting module follow it when it is deleted or added (#2512) Deleting a module that re-exports what it imports (a package's `__init__.py`, an `index.ts` barrel) left every file importing through it linked to the symbols it re-exported: those edges point into files that stay, so the removal cascaded none of them. When such a module was added or came back, its importers kept the edges a name lookup had made in its absence, or stayed failed; and in a large Python project the failed import of the module itself stayed failed, because the failed-import retry skips a stem with more than 500 imports waiting (Django's `test`, `models`). Sync now re-opens, as pending references for its sweep, what the files importing a removed or added module bind through those imports: before the removed files are deleted, and after the retries for the added ones (with their failed references and the module's own failed import). Importers are found by the import edge into the file, by Python import nodes naming the module or its package, and through what the module passes on (an import inside a function has no node); in Python, a package or star import re-exporting a changed module leads on to its own importers, as deep as the resolver chases a name. All changed files are handled in one pass, with importer edges and failed references filtered by the bound names in SQL. Python and the ES-module family only; no schema or extraction change. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + __tests__/sync-module-importers.test.ts | 154 ++++++++++ src/db/queries.ts | 138 +++++++++ src/extraction/index.ts | 27 +- src/index.ts | 19 ++ src/resolution/import-resolver.ts | 2 +- src/resolution/module-importers.ts | 362 ++++++++++++++++++++++++ 7 files changed, 700 insertions(+), 3 deletions(-) create mode 100644 __tests__/sync-module-importers.test.ts create mode 100644 src/resolution/module-importers.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 31ab29da8..c750ed86e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph sync` and the file watcher now link a file to a module it imports under a name of its own when the module is added after the file was indexed, comes back after being deleted, or gets the export in a later edit: a default import like `import tagsController from './tag/tag.controller'`, a namespace import like `import * as api from './api'`, or an aliased one like `import { Component as GroupsPageWrapper } from './groups_page'`, in JavaScript, TypeScript, Vue, Svelte and Astro. Before, the import and every use of the name, such as a call, `new`, a base class, a JSX or template tag or an Express `app.use` mount, stayed unlinked until the importing file changed or the project was re-indexed, so the module looked unused and `codegraph affected` missed the files that use it. Re-index after upgrading to pick up links an earlier version missed. - `codegraph sync` and the file watcher now do the same for a Python import under a name of its own, like `from .models import User as UserModel`, `from . import helpers as h` or `from app.pkg import sub as subpkg`: when the module or package is added after the importing file was indexed, comes back after being deleted, or gets the imported name in a later edit, the import and every use of the alias, such as a call, a base class or the class passed as a value, now link to it. Before, they stayed unlinked until the importing file changed or the project was re-indexed. Re-index Python projects after upgrading to pick up links an earlier version missed. - `codegraph sync` and the file watcher now also link a Python import of a name a package re-exports, like `from fastapi import FastAPI` through `fastapi/__init__.py`, when that `__init__.py` is added after the importing files were indexed or gains the re-export in a later edit. Before, calls through the name stayed unlinked until each importing file changed or the project was re-indexed. Large projects gain the most: sync also used to skip, in any language, a module or package that more than 500 references were waiting for, and now restoring Django's `django/urls/__init__.py` relinks every file that imports `path` from it. Re-index Python projects after upgrading to pick up links an earlier version missed. +- `codegraph sync` and the file watcher now update the files that import through a module that re-exports what it imports, like a package's `__init__.py` or an `index.ts` barrel, when that module is deleted or added, and in Python also the files that reach it through another package re-exporting it. Before, deleting `shop/__init__.py` left every `from shop import Order` linked to the class it re-exported, and deleting `components/index.ts` left every `import { Button } from './components'` linked, until each importing file changed or the project was re-indexed. When such a module came back, the imports stayed linked the way they were while it was missing, and in a large Python project an import of a module whose name is as common as `models` or `test`, like `from django.test import TestCase`, stayed unlinked. Python, JavaScript, TypeScript, Vue, Svelte and Astro. - In C, C++ and Objective-C, a struct or class member declared as a pointer, reference, array or function, like `SharedState* shared;`, `jv elements[];` or `virtual Status Put(…) = 0;`, no longer makes the struct look like it inherits from the member's type. Before, `codegraph_explore` and impact listed those types as base classes, every struct holding a pointer to a type as one of its subclasses, and a method of that struct as an override of the type's method with the same name. Real base classes like `class Derived : public Base`, Objective-C superclasses and Go's embedded fields are unchanged. Re-index C, C++ and Objective-C projects after upgrading. - In C++, a call on a variable, parameter or member declared through a `typedef` or `using` alias now links to the method of the type the alias names, with the alias looked up the way C++ does it: in the calling function, its class and the classes that class inherits from, then the namespaces around it. In google/leveldb, `table_.Insert(buf)` on a `Table table_;` declared next to `typedef SkipList Table;` now reaches `SkipList::Insert` instead of the unrelated `HandleTable::Insert`, and `Table::Iterator iter(&table_);` reaches `SkipList::Iterator`'s constructor. A class that only shares the alias's name is no longer taken for it, and when the aliased type has no such method, as with an alias of `std::vector` or of a template parameter, no method is guessed from the receiver's name. Re-index C++ projects after upgrading. - In C++, a call on a variable, parameter or member declared with template arguments nested inside its template arguments, like `autovector>` or `Striped>`, now links to the method of the class it is declared as. Before, the nested arguments hid that class: in RocksDB, `files_marked_for_compaction_.clear()` on such an `autovector` linked to an unrelated `CompactionInputFiles::clear` that only shares words with the variable's name, and `mutex_.Get(key)` on a `Striped` linked to nothing. Re-index C++ projects after upgrading. diff --git a/__tests__/sync-module-importers.test.ts b/__tests__/sync-module-importers.test.ts new file mode 100644 index 000000000..2a8c27236 --- /dev/null +++ b/__tests__/sync-module-importers.test.ts @@ -0,0 +1,154 @@ +/** + * A sync re-resolves what an import statement binds when the module it names + * appears or goes away. + * + * `from shop import Order` reaches `shop/models.py` through `shop/__init__.py`, + * which re-exports `Order`; `import { Button } from './components'` reaches + * `components/Button.ts` through `components/index.ts`. Deleting the + * re-exporting file left every importer linked through it: those edges point + * into files that stay, so nothing removed them, while a fresh index of the + * tree without the file links them by name or not at all. And a module that + * appeared after its importers were bound by name in its absence left those + * edges as the name lookup made them, while a fresh index binds them through + * the module. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; + +type Files = Record; + +/** + * Python: an absolute import, imports from inside and beside the package, a + * renamed re-export, a plain module, a subpackage imported from the package + * around it (`from app import orm`, then `orm.Model`), and an import inside a + * function, which has no import node. Two packages re-export the shop package + * in turn, one with a star import and one by name, and are imported in its + * stead. `vendored.py` makes `shop` as common a name among failed imports as + * `models` or `test` is in a large project: more than sync's failed-import + * retry looks up by name. + */ +const PYTHON: Files = { + 'vendored.py': Array.from({ length: 501 }, (_, i) => `import vendor${i}.shop\n`).join(''), + 'app/store/__init__.py': 'from app.shop import *\n', + 'report.py': 'from app.store import Order\n\n\ndef report():\n return Order()\n', + 'app/front/__init__.py': 'from ..shop import Cart as Basket\n', + 'kiosk.py': 'from app.front import Basket\n\n\ndef sell():\n return Basket()\n', + 'app/__init__.py': '', + 'app/shop/models.py': 'class _Order:\n pass\n\n\nclass Cart:\n pass\n', + 'app/shop/__init__.py': 'from .models import _Order as Order\nfrom .models import Cart\n', + 'app/shop/api/views.py': 'from .. import Order, Cart\n\n\ndef view():\n return Order(), Cart()\n', + 'app/checkout.py': 'from .shop import Cart\n\n\ndef run():\n return Cart()\n', + 'main.py': 'from app.shop import Order\n\n\ndef go():\n return Order()\n', + 'lib/__init__.py': '', + 'lib/impl.py': 'def helper():\n return 1\n', + 'lib/api.py': 'from .impl import helper as public_helper\n', + 'tool.py': 'from lib.api import public_helper\n\n\ndef use():\n return public_helper()\n', + 'app/orm/base.py': 'class _Model:\n pass\n', + 'app/orm/__init__.py': 'from .base import _Model as Model\n', + 'app/blog.py': 'from app import orm\n\n\nclass Post(orm.Model):\n pass\n', + 'test_orders.py': 'def test_order():\n from app.shop import Order\n return Order()\n', +}; +const PYTHON_BARRELS = ['app/shop/__init__.py', 'lib/api.py', 'app/orm/__init__.py']; + +/** ES modules: an `index.ts` re-exporting under the same name and under another. */ +const TS: Files = { + 'package.json': '{"name":"app"}', + 'src/components/Button.ts': 'export function Button() { return 1; }\n', + 'src/components/Icon.ts': 'export function BaseIcon() { return 2; }\n', + 'src/components/index.ts': "export { Button } from './Button';\nexport { BaseIcon as Icon } from './Icon';\n", + 'src/app.ts': "import { Button, Icon } from './components';\nexport function render() { return [Button(), Icon()]; }\n", +}; +const TS_BARRELS = ['src/components/index.ts']; + +let roots: string[] = []; +let graphs: CodeGraph[] = []; + +afterEach(() => { + for (const graph of graphs) graph.close(); + graphs = []; + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); + roots = []; +}); + +function tempRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-importers-')); + roots.push(root); + return root; +} + +function write(root: string, files: Files): void { + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } +} + +async function index(root: string): Promise { + const graph = await CodeGraph.init(root, { index: true }); + graphs.push(graph); + return graph; +} + +/** Every edge, by its ends' natural keys, with where it is written and how it resolved. */ +function edgesOf(graph: CodeGraph): string[] { + const keys = new Map( + graph.getFiles() + .flatMap((file) => graph.getNodesInFile(file.path)) + .map((n) => [n.id, `${n.kind} ${n.filePath}:${n.qualifiedName}:${n.startLine}`] as const) + ); + return graph.getOutgoingEdgesFrom([...keys.keys()]) + .map((e) => `${e.kind} ${keys.get(e.source)} -> ${keys.get(e.target) ?? e.target} @${e.line}:${e.column} ${JSON.stringify(e.metadata ?? {})}`) + .sort(); +} + +/** The tree, indexed from scratch in a folder of its own: what the sync has to match. */ +async function fullIndexOf(files: Files): Promise { + const root = tempRoot(); + write(root, files); + return edgesOf(await index(root)); +} + +const without = (files: Files, gone: string[]): Files => + Object.fromEntries(Object.entries(files).filter(([file]) => !gone.includes(file))); +const only = (files: Files, kept: string[]): Files => + Object.fromEntries(Object.entries(files).filter(([file]) => kept.includes(file))); + +describe.each([ + ['Python package and module', PYTHON, PYTHON_BARRELS], + ['ES module barrel', TS, TS_BARRELS], +])('sync re-resolves the imports through a re-exporting file: %s', (_label, files, barrels) => { + it('when it is deleted', async () => { + const root = tempRoot(); + write(root, files); + const graph = await index(root); + + for (const file of barrels) fs.rmSync(path.join(root, file)); + expect((await graph.sync()).filesRemoved).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(without(files, barrels))); + }, 120_000); + + it('when it comes back after it was deleted', async () => { + const root = tempRoot(); + write(root, files); + const graph = await index(root); + + for (const file of barrels) fs.rmSync(path.join(root, file)); + await graph.sync(); + write(root, only(files, barrels)); + expect((await graph.sync()).filesAdded).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(files)); + }, 120_000); + + it('when it appears after its importers were bound without it', async () => { + const root = tempRoot(); + write(root, without(files, barrels)); + const graph = await index(root); + + write(root, only(files, barrels)); + expect((await graph.sync()).filesAdded).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(files)); + }, 120_000); +}); diff --git a/src/db/queries.ts b/src/db/queries.ts index e5a2cccf0..9700e3d63 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -74,6 +74,16 @@ const IS_INTERFACE_MEMBER = (alias: string): string => `EXISTS ( WHERE ce.target = ${alias}.id AND ce.kind = 'contains' AND owner.kind = 'interface' )`; +/** + * A SQL predicate: is the reference name in `column` written through one of + * the names in the JSON array bound to the next two parameters, as the name + * itself or as a member read through it (`Order.objects` through `Order`)? + */ +const THROUGH_NAMES = (column: string): string => `( + ${column} IN (SELECT value FROM json_each(?)) + OR (instr(${column}, '.') > 1 AND substr(${column}, 1, instr(${column}, '.') - 1) IN (SELECT value FROM json_each(?))) +)`; + /** * How much of the exact-name bonus a `deprioritize`d path keeps (#982). Damped * rather than zeroed: a query that genuinely targets that tree must still rank @@ -3086,6 +3096,134 @@ export class QueryBuilder { })); } + /** + * Resolution edges out of the nodes of `filePath` whose reference was + * written through one of `names` (see {@link THROUGH_NAMES}), with the source + * file and language a resurrection needs and the file of each target. + * Synthesized (`provenance='heuristic'`) and `contains` edges are left out. + */ + getResolutionEdgesThrough( + filePath: string, + names: readonly string[] + ): Array { + const rows = this.db + .prepare( + `SELECT * FROM ( + SELECT e.*, src.file_path AS source_file_path, src.language AS source_language, tgt.file_path AS target_file_path, + CASE WHEN json_valid(e.metadata) THEN json_extract(e.metadata, '$.refName') END AS ref_name + FROM nodes src + JOIN edges e ON e.source = src.id + JOIN nodes tgt ON tgt.id = e.target + WHERE src.file_path = ? + AND e.kind != 'contains' + AND (e.provenance IS NULL OR e.provenance != 'heuristic') + ) WHERE ${THROUGH_NAMES('ref_name')}` + ) + .all(filePath, JSON.stringify(names), JSON.stringify(names)) as Array< + EdgeRow & { source_file_path: string; source_language: Language; target_file_path: string } + >; + return rows.map((row) => ({ + ...rowToEdge(row), + edgeId: row.id, + sourceFilePath: row.source_file_path, + sourceLanguage: row.source_language, + targetFilePath: row.target_file_path, + })); + } + + /** + * The row ids of the failed references of `filePath` that import one of + * `modules` or are written through one of `names` (see {@link THROUGH_NAMES}). + */ + getFailedReferencesThrough(filePath: string, modules: readonly string[], names: readonly string[]): number[] { + return (this.db + .prepare( + `SELECT id FROM unresolved_refs + WHERE file_path = ? AND status = 'failed' + AND ((reference_kind = 'imports' AND reference_name IN (SELECT value FROM json_each(?))) + OR ${THROUGH_NAMES('reference_name')})` + ) + .all(filePath, JSON.stringify(modules), JSON.stringify(names), JSON.stringify(names)) as Array<{ id: number }>) + .map((row) => row.id); + } + + /** The targets, in other files, of the non-synthesized `imports` edges out of the nodes of `filePath`. */ + getImportTargetsFromFile(filePath: string): string[] { + return (this.db + .prepare( + `SELECT DISTINCT e.target AS target + FROM nodes src + JOIN edges e ON e.source = src.id + JOIN nodes tgt ON tgt.id = e.target + WHERE src.file_path = ? + AND e.kind = 'imports' + AND (e.provenance IS NULL OR e.provenance != 'heuristic') + AND tgt.file_path != ?` + ) + .all(filePath, filePath) as Array<{ target: string }>) + .map((row) => row.target); + } + + /** + * The `imports` edges into the node `targetId` from files other than + * `exceptFile`: each importing file, its language, and the module specifier + * it wrote (the edge's `refName`; null when it carries none). + */ + getImportsInto( + targetId: string, + exceptFile: string + ): Array<{ filePath: string; language: Language; refName: string | null }> { + return (this.db + .prepare( + `SELECT src.file_path AS file_path, src.language AS language, + CASE WHEN json_valid(e.metadata) THEN json_extract(e.metadata, '$.refName') END AS ref_name + FROM edges e + JOIN nodes src ON src.id = e.source + WHERE e.target = ? AND e.kind = 'imports' AND src.file_path != ?` + ) + .all(targetId, exceptFile) as Array<{ file_path: string; language: Language; ref_name: unknown }>) + .map((row) => ({ + filePath: row.file_path, + language: row.language, + refName: typeof row.ref_name === 'string' ? row.ref_name : null, + })); + } + + /** The files with a resolution edge into one of `targetIds`. */ + getFilesWithEdgesInto(targetIds: string[]): string[] { + const files = new Set(); + for (let i = 0; i < targetIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = targetIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare( + `SELECT DISTINCT src.file_path AS file_path + FROM edges e + JOIN nodes src ON src.id = e.source + WHERE e.target IN (${placeholders}) + AND e.kind != 'contains' + AND (e.provenance IS NULL OR e.provenance != 'heuristic')` + ) + .all(...chunk) as Array<{ file_path: string }>; + for (const row of rows) files.add(row.file_path); + } + return [...files]; + } + + /** The import nodes named one of `names` (an import node is named by the module path it imports). */ + getImportNodesByNames(names: string[]): Array<{ filePath: string; name: string; language: Language }> { + const out: Array<{ filePath: string; name: string; language: Language }> = []; + for (let i = 0; i < names.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = names.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare(`SELECT file_path, name, language FROM nodes WHERE name IN (${placeholders}) AND kind = 'import'`) + .all(...chunk) as Array<{ file_path: string; name: string; language: Language }>; + for (const row of rows) out.push({ filePath: row.file_path, name: row.name, language: row.language }); + } + return out; + } + // =========================================================================== // File Operations // =========================================================================== diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 42ff58f04..8623d7203 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -35,6 +35,7 @@ import { declaredDependencies } from '../resolution/frameworks/package-deps'; import type { ResolutionContext } from '../resolution/types'; import { createYielder, type MaybeYield } from '../resolution/cooperative-yield'; import { resurrectRefFromDroppedEdge } from '../resolution/resurrect-ref'; +import { createImportReader, reopenReferencesThroughRemovedFiles } from '../resolution/module-importers'; import { MAX_SOURCE_FILE_SIZE_BYTES, oversizeStamp, readBoundedSource, readBoundedSourceSync } from '../file-limits'; export { oversizeStamp }; @@ -3280,6 +3281,8 @@ export class ExtractionOrchestrator { // BEFORE their nodes are replaced/deleted. Compared against the post-store // pairs below to derive `definitionDelta` (CG-33). const pairsBefore = new Set(); + // Importers' import statements, read once per sync for the files it removes. + const importReader = createImportReader(this.rootDir); onProgress?.({ phase: 'scanning', @@ -3385,14 +3388,31 @@ export class ExtractionOrchestrator { this.queries.deleteFile(tracked.path); filesRemoved++; }; + const removed: FileRecord[] = []; for (const tracked of trackedFiles) { if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) { - removeTracked(tracked); + removed.push(tracked); } if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) { await new Promise((resolve) => setImmediate(resolve)); } } + // The references other files make THROUGH the removed ones to files that + // stay (a package `__init__.py` or an `index.ts` re-exporting what it + // imports) go back to pending too: the cascade keeps their edges, and a + // fresh index of the tree without these files binds them differently or + // not at all. Before any of them is deleted, and for all of them at once. + if (removed.length > 0) { + const tImporters = Date.now(); + const reopened = reopenReferencesThroughRemovedFiles(this.queries, importReader, removed); + if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-removed-module-importers: ${Date.now() - tImporters}ms (${reopened} refs re-opened)`); + } + for (const tracked of removed) { + removeTracked(tracked); + if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) { + await new Promise((resolve) => setImmediate(resolve)); + } + } // Adds / modifications. for (const filePath of currentFiles) { @@ -3441,7 +3461,10 @@ export class ExtractionOrchestrator { // Not source after all — an MPEG transport stream named `.ts` (#1910): // a new one is ignored, a tracked file that became one is removed. if (content === null) { - if (tracked) removeTracked(tracked); + if (tracked) { + reopenReferencesThroughRemovedFiles(this.queries, importReader, [tracked]); + removeTracked(tracked); + } continue; } const contentHash = hashContent(content); diff --git a/src/index.ts b/src/index.ts index 75e32ad28..2e777dbf5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -53,6 +53,7 @@ import { ResolutionResult, } from './resolution'; import { hasSynthesisPattern, isJsxChildCandidate } from './resolution/callback-synthesizer'; +import { createImportReader, reopenReferencesThroughAddedFiles } from './resolution/module-importers'; import { GraphTraverser, GraphQueryManager } from './graph'; import { findNamedCopybooks, type NamedCopybook } from './graph/cobol-copybooks'; import { ContextBuilder, createContextBuilder } from './context'; @@ -1002,6 +1003,9 @@ export class CodeGraph { // per-name ceiling counts only edges up to it (see // resurrectStaleResolutionEdges). let edgesBeforeRetry: number | undefined; + // And the last edge before this sync resolved anything: an importer's + // edge up to it was bound before a module this sync adds was there. + const edgesBeforeResolution = this.queries.getMaxEdgeId(); if (filesChanged) { if (result.changedFilePaths) { // Scope resolution to changed files (git fast path — bounded set) @@ -1079,6 +1083,21 @@ export class CodeGraph { }); } if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-failed-ref-retry: ${Date.now() - tRetry}ms (${retryable.length} refs)`); + + // A module that appeared, such as a package `__init__.py` or an + // `index.ts` re-exporting what it imports: what the files importing + // it bound by name in its absence, or failed, goes back to pending + // for the sweep below, which binds it through the module as a fresh + // index does. (The removal half runs before removed files are deleted.) + if (result.addedFilePaths && result.addedFilePaths.length > 0) { + const tAdded = Date.now(); + const added = result.addedFilePaths.flatMap((filePath) => { + const file = this.queries.getFileByPath(filePath); + return file ? [{ path: filePath, language: file.language }] : []; + }); + const reopened = reopenReferencesThroughAddedFiles(this.queries, createImportReader(this.projectRoot), added, edgesBeforeResolution); + if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-added-module-importers: ${Date.now() - tAdded}ms (${reopened} refs re-opened)`); + } } else { // No git info — use batched resolution to avoid OOM const unresolvedCount = this.queries.getUnresolvedReferencesCount(); diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index c3b66202b..f6fa14041 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -3227,7 +3227,7 @@ function projectFolderSuffixes(context: ResolutionContext): Set { * from `app/sub/views.py`, `.models` → 'app/sub/models', `..` → 'app' and * `..forms` → 'app/forms'. Null when its dots climb above the project. */ -function pythonModulePath(source: string, fromFile: string): string | null { +export function pythonModulePath(source: string, fromFile: string): string | null { const name = source.replace(/^\.+/, ''); const dots = source.length - name.length; if (dots === 0) return name.replace(/\./g, '/'); diff --git a/src/resolution/module-importers.ts b/src/resolution/module-importers.ts new file mode 100644 index 000000000..2f8a1c575 --- /dev/null +++ b/src/resolution/module-importers.ts @@ -0,0 +1,362 @@ +/** + * The references an import statement makes through a module that appears or + * goes away. + * + * `from shop import Order` reaches `shop/models.py` through `shop/__init__.py`, + * which re-exports `Order`; `import { Button } from './components'` reaches + * `components/Button.ts` through `components/index.ts`. When the re-exporting + * file goes away, their edges point into files that stay, so the cascade keeps + * them; when it comes back, the edges a name lookup made in its absence stay. + * Nothing else revisits either: the importers did not change. A fresh index + * binds them through the module when it is there, and another way (by name) + * or not at all when it is not. + * + * A Python module that re-exports another one (`from .shop import *`, or a + * package's `__init__.py` importing names from it) is the way in for its own + * importers, so theirs are followed too, as deep as the resolver chases a + * name through such modules. An ES module's `export … from` leaves no import + * edge or node to find it by, so an `index.ts` re-exporting another is not + * followed. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { QueryBuilder } from '../db/queries'; +import type { Language, UnresolvedReference } from '../types'; +import type { ImportMapping } from './types'; +import { extractImportMappings, pythonModulePath } from './import-resolver'; +import { resurrectRefFromDroppedEdge } from './resurrect-ref'; + +/** + * Languages whose import bindings name what their references are written as: + * Python's `from x import Y` and the ES-module family's `import { Y } from`. + * An `#include` or a Go package import binds no reference name of its own. + */ +const BINDING_LANGUAGES = new Set([ + 'python', 'typescript', 'tsx', 'javascript', 'jsx', 'arkts', 'vue', 'svelte', 'astro', +]); + +/** + * How many re-exporting Python modules deep the importers of a file are + * followed: as deep as the resolver chases a name through star imports and + * re-exports (`pythonModuleSymbol`). + */ +const PYTHON_REEXPORT_DEPTH = 3; + +/** A Python star import, read the way the resolver reads one. */ +const PYTHON_STAR_IMPORT = /^\s*from\s+([\w.]+)\s+import\s+\*/gm; + +/** + * A file this sync adds or removes, or a module re-exporting from one, with + * the names it passes on from them (null: whatever it has). + */ +interface Module { + path: string; + language: Language; + passes: Set | null; +} + +/** + * Each importer's language and, per module specifier of a statement naming + * one of the modules, which of its bindings go through it: all of them + * (null), or those importing one of these names — what a re-exporting module + * passes on, or the module the package around a file imports as a name + * (`from django.db import models`, whose `models` is the file's module). + */ +type Importers = Map | null> }>; + +/** A statement naming a module: whole, or as `onlyName` of the package around it. */ +type AddImporter = (importer: string, language: Language, source: string, module: Module, onlyName?: string) => void; + +/** What a file's import statements say, each file read once per sync. */ +export interface ImportReader { + /** Its import mappings. */ + mappings(file: string, language: Language): ImportMapping[]; + /** The modules a Python file star-imports, which have no import mapping. */ + starImports(file: string, language: Language): string[]; +} + +export function createImportReader(rootDir: string): ImportReader { + const memo = new Map(); + const read = (file: string, language: Language) => { + let entry = memo.get(file); + if (!entry) { + let content = ''; + try { + content = fs.readFileSync(path.join(rootDir, file), 'utf8'); + } catch { + // Gone or unreadable: it imports nothing. + } + entry = { + mappings: content ? extractImportMappings(file, content, language) : [], + starImports: language === 'python' ? [...content.matchAll(PYTHON_STAR_IMPORT)].map((m) => m[1]!) : [], + }; + memo.set(file, entry); + } + return entry; + }; + return { + mappings: (file, language) => read(file, language).mappings, + starImports: (file, language) => read(file, language).starImports, + }; +} + +/** + * Re-open, as pending references for the sync's sweep, the references the + * files importing any of `removed` make through those imports. The sweep binds + * them against the post-sync graph, or parks them where a sync that brings a + * file back finds them. Called before the files' rows are deleted, for all of + * them at once, so an importer of several is visited once; an edge into a + * removed file is left to the resurrection its removal does. Returns the + * number of references re-opened. + */ +export function reopenReferencesThroughRemovedFiles( + queries: QueryBuilder, + reader: ImportReader, + removed: ReadonlyArray<{ path: string; language: Language }>, +): number { + const gone = new Set(removed.map((file) => file.path)); + const bindings = bindingsOf(importersOf(queries, removed, reader), reader); + return reopenEdges(queries, bindings, (e) => !gone.has(e.targetFilePath)); +} + +/** + * The other half: after a sync's retries, re-open the references the files + * importing any of `added` make through those imports — the edges a name + * lookup made while a module was missing (only those up to + * `resolvedUpToEdgeId`: later ones were just resolved with the modules there) + * and the references that failed, the import of the module itself among + * them. (The failed-import retry looks those up by the module's name, and + * skips a name as common as `models` or `test`.) Returns the number re-opened. + */ +export function reopenReferencesThroughAddedFiles( + queries: QueryBuilder, + reader: ImportReader, + added: ReadonlyArray<{ path: string; language: Language }>, + resolvedUpToEdgeId: number, +): number { + const bindings = bindingsOf(importersOf(queries, added, reader), reader); + const failed = [...bindings].flatMap(([file, { modules, names }]) => queries.getFailedReferencesThrough(file, modules, names)); + return reopenEdges(queries, bindings, (e) => e.edgeId <= resolvedUpToEdgeId) + queries.reopenFailedReferences(failed); +} + +/** + * The files with an import statement naming one of `files`, or one of the + * modules re-exporting from them, other than `files` themselves (going away, + * or just resolved with the others in place). + */ +function importersOf( + queries: QueryBuilder, + files: ReadonlyArray<{ path: string; language: Language }>, + reader: ImportReader, +): Importers { + const importers: Importers = new Map(); + const changed = new Set(files.map((file) => file.path)); + const seen = new Set(changed); + let level: Module[] = files.filter((file) => BINDING_LANGUAGES.has(file.language)).map((file) => ({ ...file, passes: null })); + for (let depth = 1; level.length > 0; depth++) { + // Per importer found here, its statements naming each module of the level. + const naming = new Map }>(); + const add: AddImporter = (importer, language, source, module, onlyName) => { + if (!BINDING_LANGUAGES.has(language) || changed.has(importer)) return; + const only = onlyName !== undefined ? new Set([onlyName]) : module.passes; + const entry = importers.get(importer) ?? { language, sources: new Map | null>() }; + const names = entry.sources.get(source); + entry.sources.set(source, names === null || only === null ? null : new Set([...(names ?? []), ...only])); + importers.set(importer, entry); + const found = naming.get(importer) ?? { language, statements: [] }; + found.statements.push({ source, module, onlyName }); + naming.set(importer, found); + }; + for (const module of level) { + // A module-path import that resolved to the module: an ES module + // specifier, a dotted Python import. + for (const i of queries.getImportsInto(`file:${module.path}`, module.path)) { + if (i.refName !== null) add(i.filePath, i.language, i.refName, module); + } + } + const python = level.filter((module) => module.language === 'python'); + if (python.length > 0) addPythonImporters(queries, python, reader, add); + // The next level: the Python importers that pass on what they import from these. + if (depth > PYTHON_REEXPORT_DEPTH) break; + const next: Module[] = []; + for (const [importer, { language, statements }] of naming) { + if (language !== 'python' || seen.has(importer)) continue; + const passes = pythonPassedOn(importer, statements, reader); + if (passes === undefined) continue; + seen.add(importer); + next.push({ path: importer, language, passes }); + } + level = next; + } + return importers; +} + +/** + * What the Python module `importer` passes on of what its `statements` + * import, or undefined when the resolver reads no name through it: what it + * star-imports, and for a package's `__init__.py`, the names it imports too. + */ +function pythonPassedOn( + importer: string, + statements: ReadonlyArray<{ source: string; module: Module; onlyName?: string }>, + reader: ImportReader, +): Set | null | undefined { + let passes: Set | null | undefined; + const pass = (names: Set | null): void => { + passes = passes === null || names === null ? null : new Set([...(passes ?? []), ...names]); + }; + const stars = new Set(reader.starImports(importer, 'python')); + const isPackage = importer === '__init__.py' || importer.endsWith('/__init__.py'); + for (const { source, module, onlyName } of statements) { + if (onlyName === undefined && stars.has(source)) pass(module.passes); + if (!isPackage) continue; + const names = reader.mappings(importer, 'python') + .filter((m) => m.source === source && (onlyName !== undefined + ? m.exportedName === onlyName + : module.passes === null || module.passes.has(m.exportedName))) + .map((m) => m.localName); + if (names.length > 0) pass(new Set(names)); + } + return passes; +} + +/** + * Python's other ways to name the modules: an import node whose module path + * denotes one, relative to the importer's own package or absolutely (`from . + * import Order`, `from .. import Order`, `from shop import Order`), which + * never resolve to a file, or the package around one with its module among + * the names imported (`from django.db import models`). An import inside a + * function has no import node, so a file that reaches what the modules pass + * on — the targets of their own import edges — counts too when its imports + * name one. + */ +function addPythonImporters( + queries: QueryBuilder, + modules: readonly Module[], + reader: ImportReader, + add: AddImporter, +): void { + // What a module path a statement names stands for: a module whole, or a + // package with the module of `onlyName` among the names imported. + type Naming = { module: Module; onlyName?: string }; + const byPath = new Map(); + const bySuffix = new Map(); + const importNames = new Set(); + const push = (map: Map, key: string, naming: Naming): void => { + const list = map.get(key); + if (list) list.push(naming); + else map.set(key, [naming]); + }; + const index = (modulePath: string, naming: Naming): void => { + push(byPath, modulePath, naming); + const segments = modulePath.split('/'); + for (let i = 0; i < segments.length; i++) push(bySuffix, segments.slice(i).join('/'), naming); + for (const name of pythonImportNamesFor(modulePath)) importNames.add(name); + }; + for (const module of modules) { + const modulePath = pythonModuleOfFile(module.path); + if (!modulePath) continue; + index(modulePath, { module }); + const slash = modulePath.lastIndexOf('/'); + if (slash > 0) index(modulePath.slice(0, slash), { module, onlyName: modulePath.slice(slash + 1) }); + } + if (importNames.size === 0) return; + // A relative import names a module by where the importer sits, an absolute + // one by the tail of its path (the project's import root is not known). + const named = (source: string, importer: string): Naming[] => + source.startsWith('.') + ? byPath.get(pythonModulePath(source, importer)?.replace(/\/+$/, '') ?? '') ?? [] + : bySuffix.get(source.replace(/\./g, '/')) ?? []; + + for (const node of queries.getImportNodesByNames([...importNames])) { + if (node.language !== 'python') continue; + for (const { module, onlyName } of named(node.name, node.filePath)) add(node.filePath, 'python', node.name, module, onlyName); + } + const paths = new Set(modules.map((module) => module.path)); + const passedOnTargets = new Set([...paths].flatMap((filePath) => queries.getImportTargetsFromFile(filePath))); + if (passedOnTargets.size === 0) return; + for (const candidate of queries.getFilesWithEdgesInto([...passedOnTargets])) { + if (!candidate.endsWith('.py') || paths.has(candidate)) continue; + for (const m of reader.mappings(candidate, 'python')) { + for (const { module, onlyName } of named(m.source, candidate)) { + if (onlyName === undefined || onlyName === m.exportedName) add(candidate, 'python', m.source, module, onlyName); + } + } + } +} + +/** + * Per importer, what its statements naming the modules import: the modules + * named whole (not the package around one), and the local names bound. + */ +function bindingsOf(importers: Importers, reader: ImportReader): Map { + const bindings = new Map(); + for (const [file, { language, sources }] of importers) { + const names = new Set( + reader.mappings(file, language) + .filter((m) => { + const only = sources.get(m.source); + return only === null || (only !== undefined && only.has(m.exportedName)); + }) + .map((m) => m.localName) + ); + const modules = [...sources].filter(([, only]) => only === null).map(([source]) => source); + if (modules.length > 0 || names.size > 0) bindings.set(file, { modules, names: [...names] }); + } + return bindings; +} + +/** + * Swap for their references the edges of each importer's references through + * its bound names — the binding itself or a member read through it + * (`Order.objects`, `ns.fn`) — that pass `keep`. Returns how many. + */ +function reopenEdges( + queries: QueryBuilder, + bindings: Map, + keep: (e: { edgeId: number; targetFilePath: string }) => boolean, +): number { + const edgeIds: number[] = []; + const refs: UnresolvedReference[] = []; + for (const [file, { names }] of bindings) { + if (names.length === 0) continue; + for (const e of queries.getResolutionEdgesThrough(file, names)) { + if (!keep(e)) continue; + const ref = resurrectRefFromDroppedEdge(e); + if (!ref) continue; + edgeIds.push(e.edgeId); + refs.push(ref); + } + } + if (refs.length > 0) queries.replaceResolutionEdgesWithUnresolvedRefs(edgeIds, refs); + return refs.length; +} + +/** A Python file's module as a path: `shop/models.py` → 'shop/models', `shop/__init__.py` → 'shop'. Null for a top-level `__init__.py`. */ +function pythonModuleOfFile(filePath: string): string | null { + if (!filePath.endsWith('.py') || filePath === '__init__.py') return null; + if (filePath.endsWith('/__init__.py')) return filePath.slice(0, -'/__init__.py'.length); + return filePath.slice(0, -'.py'.length); +} + +/** Deepest relative import a lookup spells out: `from ........ import x`. */ +const MAX_RELATIVE_DOTS = 8; + +/** + * The import names that can denote the module at `modulePath`: its trailing + * dotted paths (`models`, `shop.models`), each also behind one dot or more + * (`.models`, `..shop.models`), and the dots alone (`.`, `..`), which name a + * package by where the importer sits. + */ +function pythonImportNamesFor(modulePath: string): string[] { + const segments = modulePath.split('/'); + const absolute = segments.map((_, i) => segments.slice(i).join('.')); + const names = new Set(absolute); + for (let dots = 1; dots <= MAX_RELATIVE_DOTS; dots++) { + const prefix = '.'.repeat(dots); + names.add(prefix); + for (const name of absolute) names.add(prefix + name); + } + return [...names]; +}