fix(c): a macro alone below an #include <…> no longer hides the next declaration

CHANGELOG.md

@@ -195,6 +195,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - In Objective-C, the doc comment above a method, function or static written inside an `@implementation` is now read as its description, as the one above an `@interface` property already was. Before, the parser's wrapper around each item of an `@implementation` hid the comment, so search and `codegraph_explore` showed these methods without their documentation. Re-index Objective-C projects after upgrading.
 - In Objective-C, a file-scope constant or global, like `NSString *const SDWebImageErrorDomain = @"SDWebImageErrorDomain";` or `static const NSInteger kMaxRetries = 3;`, is now indexed with its value, and so is one written inside an `@implementation`. Before, only a global declared with a bare name, like `int count;`, was, so error domains, notification names, keys and shared queues were missing from search and `codegraph_explore`. A global that a `.m` file defines is reached from other files through the header that declares it, and a call of a block parameter, like `completionHandler(nil, image)`, no longer links to a namesake that another `.m` file keeps. Re-index Objective-C projects after upgrading.
 - In Objective-C, a message to an instance variable or a property now reaches a method of the class it is declared as, as a message to a local variable already did. That covers an instance variable a header, an `@implementation` or a superclass declares, the one a property synthesizes (`_name`, or the one `@synthesize` names), `self.name`, and a local or parameter whose type ObjFW writes with `OF_GENERIC(…)`. Before, such a message was matched by the receiver's name: in ObjFW's archive tool, `[_archive nextEntry]` on a tar archive reached the LHA archive's method, and in React Native, `[_displayLink invalidate]` on a `CADisplayLink` reached React Native's own `RCTDisplayLink`. A message to a variable whose class comes from outside your project, like `NSMapTable` or `CADisplayLink`, no longer links to a project method of the same name. Re-index Objective-C projects after upgrading.
+- In C and C++, a macro alone on its line right below an `#include <…>`, like GLib's `G_BEGIN_DECLS` below `#include <glib/gtypes.h>`, no longer hides the declaration that follows it. Before, the first declaration in nearly every GLib header was missing, such as the `GBytes`, `GError`, `GHashTable`, `GList` and `GQuark` types, as were enums like `GIOCondition` with their values, and a few headers showed made-up functions in their place. Re-index C and C++ projects after upgrading.
 - In TypeScript React code, a styled-components or emotion component declared with a type argument, like `const Wrapper = styled.div<WrapperProps>` followed by its CSS, or `styled(Base)<Props>`, is now indexed as a component, as one written without a type argument already was. Before, it was indexed as a plain constant, so `codegraph callers` and impact found none of the code that renders it, and its tags could link to another file's component of the same name instead: in outline/outline, `NudeButton` and `HStack` had no callers, and `<Text>` tags linked to a different `Text` than the one each file imports. Re-index TypeScript React projects after upgrading.
 - `codegraph sync` and the file watcher now move a React Router route to its page when the router file loads that page lazily into a value of its own, like `const Docs = lazy(() => import('./pages/Docs'))` rendered by `<Route path="/docs" element={<Docs />} />`, and the page's file is added after the router was indexed, comes back after being deleted, or gets or changes its default export in a later edit. Before, the route kept pointing at the `Docs` declaration, or at the page's old component, until the router file itself changed or the project was re-indexed, unless the page's component happened to share the declaration's name. The same now holds for a `lazy: () => import('./pages/Docs')` route whose page switches its default export to another component. Re-index React projects after upgrading to correct routes an earlier version left behind.
 - `codegraph sync` and the file watcher now link a component to the JSX that renders it, like `<Team />`, when the component is added in a file with no JSX of its own, as in `export default function Team() { return null; }`. Before, the link waited for an unrelated change to a file with JSX, or for a re-index, so the new component looked unused.

__tests__/c-include-lone-macros.test.ts

@@ -0,0 +1,111 @@
+/**
+ * A macro alone on its line right below an `#include <…>`, like GLib's
+ *
+ *   #include <glib/gtypes.h>
+ *
+ *   G_BEGIN_DECLS
+ *
+ *   typedef struct _GBytes GBytes;
+ *
+ * stayed in C and C++, while the same macro below any other line was blanked
+ * before parsing: the directive ends in `>`, which read as an operator going
+ * on into the macro. The parser then took `G_BEGIN_DECLS typedef struct
+ * _GBytes GBytes;` for one broken declaration, so the first declaration of
+ * nearly every GLib header was missing, and the error kept the file off the
+ * native kernel. A preprocessor directive ends at its own line, as
+ * Objective-C already read it. Runs against the native kernel (when built)
+ * and the wasm extractor, which must agree.
+ */
+import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import { extractFromSource } from '../src/extraction';
+import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
+import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel';
+import { blankLoneMacroLines } from '../src/extraction/languages/c-cpp';
+import type { ExtractionResult } from '../src/types';
+
+const KERNEL_PATH = path.join(
+  __dirname,
+  '..',
+  'codegraph-kernel',
+  'prebuilds',
+  `${process.platform}-${process.arch}`,
+  'codegraph-kernel.node'
+);
+const kernelAvailable = fs.existsSync(KERNEL_PATH) || process.env.CODEGRAPH_KERNEL_EXPECT === '1';
+
+const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const;
+
+describe('a macro alone on its line below an #include <…>', () => {
+  let savedEnv: Record<string, string | undefined> = {};
+
+  beforeAll(async () => {
+    await initGrammars();
+    await loadGrammarsForLanguages(['c', 'cpp']);
+  });
+
+  beforeEach(() => {
+    savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
+    resetKernelForTests();
+  });
+
+  afterEach(() => {
+    for (const k of ENV_KEYS) {
+      if (savedEnv[k] === undefined) delete process.env[k];
+      else process.env[k] = savedEnv[k];
+    }
+    resetKernelForTests();
+  });
+
+  it('is blanked, as below any other line, unless the directive goes on with a backslash', () => {
+    const blanked = (source: string, macro: string) => !blankLoneMacroLines(source).includes(macro);
+    expect(blanked(['#include <glib/gtypes.h>', '', 'G_BEGIN_DECLS', '', 'typedef struct _GBytes GBytes;', ''].join('\n'), 'G_BEGIN_DECLS')).toBe(true);
+    expect(blanked(['#include <type_traits>', 'FMT_BEGIN_NAMESPACE', 'class buffer;', ''].join('\r\n'), 'FMT_BEGIN_NAMESPACE')).toBe(true);
+    expect(blanked(['#define G_BEGIN_DECLS \\', '  G_EXTERN_BEGIN', 'int x;', ''].join('\n'), 'G_EXTERN_BEGIN')).toBe(false);
+  });
+
+  function extract(backend: 'kernel' | 'wasm', file: string, source: string): ExtractionResult {
+    if (backend === 'wasm') {
+      process.env.CODEGRAPH_KERNEL = '0';
+      return extractFromSource(file, source, 'c');
+    }
+    delete process.env.CODEGRAPH_KERNEL;
+    process.env.CODEGRAPH_KERNEL_LANGS = 'all';
+    const result = tryKernelExtract(file, source, 'c');
+    expect(result, `kernel extraction of ${file}`).not.toBeNull();
+    return result!;
+  }
+
+  const backends = kernelAvailable ? (['kernel', 'wasm'] as const) : (['wasm'] as const);
+
+  for (const backend of backends) {
+    for (const crlf of [false, true]) {
+      const eol = (s: string) => (crlf ? s.replace(/\n/g, '\r\n') : s);
+      it(`leaves the header's first declaration in place on ${backend}${crlf ? ' (CRLF)' : ''}`, () => {
+        const source = [
+          '#include <glib/gtypes.h>',
+          '',
+          'G_BEGIN_DECLS',
+          '',
+          'typedef struct _GBytes GBytes;',
+          'typedef struct _GArray GArray;',
+          '',
+          'struct _GArray',
+          '{',
+          '  gchar *data;',
+          '  guint len;',
+          '};',
+          '',
+          'G_END_DECLS',
+          '',
+        ].join('\n');
+        const result = extract(backend, 'glib/garray.h', eol(source));
+        const symbols = result.nodes
+          .filter((n) => n.kind !== 'file' && n.kind !== 'import')
+          .map((n) => `${n.kind} ${n.qualifiedName}`);
+        expect(symbols).toEqual(['type_alias GBytes', 'type_alias GArray', 'struct _GArray']);
+      });
+    }
+  }
+});

src/extraction/languages/c-cpp.ts

@@ -1213,7 +1213,10 @@ export function blankCppAnnotationMacroCalls(source: string): string {
  *    (`=`, an operator, `,`, `(`, `?`, `:`, or a `\` macro-definition
  *    continuation) — so an ALL-CAPS operand split onto its own line inside a
  *    multi-line expression (`int x =\n  SOME_CONST\n  | OTHER;`) is left
- *    alone; and
+ *    alone. A preprocessor directive ends at its own line, even one that
+ *    ends in `>`: GLib's `G_BEGIN_DECLS` below `#include <glib.h>`, and
+ *    Objective-C's `NS_ASSUME_NONNULL_BEGIN` below `#import
+ *    <Foundation/Foundation.h>`, are blanked; and
  *  - the NEXT line of code starts like a declaration/scope token
  *    (letter, `_`, `#`, `{`, `}`, or `~`) or the file ends — an operator,
  *    string literal, or `;` continuation rejects the match. Comments in
@@ -1229,29 +1232,23 @@ const LONE_MACRO_CONTINUATION_END_RE = /[=+\-*/%&|^<>?:,(\\]$/;
 interface LoneMacroRules {
   /** What the code after the macro may start with. */
   nextCode: RegExp;
-  /** Whether a preprocessor directive above the macro ends at its own line. */
-  directivesEnd: boolean;
   /** Whether a macro that is a body's only statement, between `{` and `}`, stays. */
   keepsSoleStatement: boolean;
 }
 
-const C_LONE_MACROS: LoneMacroRules = { nextCode: /[A-Za-z_#{}~]/, directivesEnd: false, keepsSoleStatement: false };
+const C_LONE_MACROS: LoneMacroRules = { nextCode: /[A-Za-z_#{}~]/, keepsSoleStatement: false };
 
 /**
  * Objective-C's: the code after the macro may be a directive, like the
  * `@interface` below `NS_ASSUME_NONNULL_BEGIN` or the `@end` below
- * `OF_SINGLETON_METHODS`. A preprocessor line above it continues nothing,
- * even one that ends in `>`, like the `#import <Foundation/Foundation.h>`
- * above most headers' `NS_ASSUME_NONNULL_BEGIN`. Blanking that macro but not
- * another before the same `@interface` (`OF_SUBCLASSING_RESTRICTED`) left
- * `OF_ASSUME_NONNULL_BEGIN @interface …`, which reads as a function.
+ * `OF_SINGLETON_METHODS`.
  *
  * A macro that is a method's whole body, like ObjFW's `{
  * OF_UNRECOGNIZED_SELECTOR }`, stays: the parser's error there stays inside
  * that body, but blanking it changed how the parser recovered from an
  * earlier misread, and OFMutableString.m lost 11 calls in the methods below.
  */
-export const OBJC_LONE_MACROS: LoneMacroRules = { nextCode: /[A-Za-z_#{}~@]/, directivesEnd: true, keepsSoleStatement: true };
+export const OBJC_LONE_MACROS: LoneMacroRules = { nextCode: /[A-Za-z_#{}~@]/, keepsSoleStatement: true };
 
 export function blankLoneMacroLines(source: string, rules: LoneMacroRules = C_LONE_MACROS): string {
   if (!/^[ \t]*[A-Z][A-Z0-9_]{3,}[ \t]*\r?$/m.test(source)) return source;
@@ -1274,7 +1271,7 @@ export function blankLoneMacroLines(source: string, rules: LoneMacroRules = C_LO
     // continues nothing either.
     const prevText = prev >= 0 ? content(lines[prev] as string) : '';
     const prevCode = /^(?:\/\/|\/\*|\*)/.test(prevText) ? '' : prevText.replace(/\/\*.*?\*\/|\/\/.*$/g, '').trim();
-    const directiveEnds = rules.directivesEnd && prevCode.startsWith('#') && !prevCode.endsWith('\\');
+    const directiveEnds = prevCode.startsWith('#') && !prevCode.endsWith('\\');
     if (prevCode !== '' && !directiveEnds && !/^template\s*</.test(prevCode) && LONE_MACRO_CONTINUATION_END_RE.test(prevCode)) continue;
     const first = firstCodeCharAfter(lines, i + 1);
     if (first !== undefined && !rules.nextCode.test(first)) continue;