Copy in Aether's parser as the new air_r_parser

.config/nextest.toml

@@ -1,11 +1,14 @@
-[profile.ci]
+[profile.default]
+fail-fast = false
+# Try up to 3 times with 10 second timeout each time.
+# Helps detect deadlocks without hanging infinitely.
+slow-timeout = { period = "10s", terminate-after = 3 }
 # Print out output for failing tests as soon as they fail, and also at the end
 # of the run (for easy scrollability).
 failure-output = "immediate-final"
+status-level = "fail"
 
-# Do not cancel the test run on the first failure.
-fail-fast = false
-
-# Mark tests that take longer than 60s as slow.
-# Terminate after 60s as a stop-gap measure to terminate on deadlock.
-slow-timeout = { period = "60s", terminate-after = 1 }
+[profile.ci]
+inherits = "default"
+retries = 0
+final-status-level = "slow"

.gitattributes

@@ -2,4 +2,3 @@
 
 # Windows specific test files where we need CRLF endings
 crates/air/fixtures/crlf/*.R text eol=crlf
-crates/air_r_parser/tests/snapshots/ok/crlf/*.R text eol=crlf

.github/workflows/test-health.yml

@@ -20,8 +20,19 @@ jobs:
       - name: Rust cache
         uses: Swatinem/rust-cache@v2
 
+      - name: Install just
+        uses: taiki-e/install-action@v2
+        with:
+          tool: just
+
       - name: Check formatting
         run: cargo fmt --all --check
 
       - name: Clippy
         run: cargo clippy --workspace --all-targets -- -D warnings
+
+      - name: Codegen is up to date
+        run: just gen-grammar && git diff --exit-code
+
+      - name: Parser markdown snapshots are up to date
+        run: cargo run -p xtask_accept && git diff --exit-code

CHANGELOG.md

@@ -2,7 +2,35 @@
 
 # Development version
 
-- `for` loops with `...` or `..1` as the loop variable, like `for (... in x)`, are now supported.
+- Air no longer utilizes [tree-sitter-r](https://github.com/r-lib/tree-sitter-r) for parsing R files. Instead, Air now ships with a completely custom R parser written entirely in Rust. This comes with a number of benefits!
+
+  - Air is now roughly 2x faster. Formatting all of dplyr has gone from 300ms to 150ms on an M2 Mac. The actual parsing stage is around 7x faster.
+
+  - Air now emits precise error messages when a file fails to parse due to syntax errors.
+
+  - Air no longer has C in its dependency chain. This means that we'll be able to add WASM support for Air much easier now.
+
+  If you are building a Rust based project that needs to parse R files, we encourage you to use our `air_r_parser` crate (pinned to a git SHA since we haven't put it on crates.io yet).
+
+- We've fixed a number of minor issues through the new parser:
+
+  - Air more accurately accepts certain syntax:
+
+    - `r"|...|"`, which are raw strings using the undocumented `|` delimiter
+
+    - `%a\b%` where previously the `\` failed to parse
+
+    - `for` loops with `...` or `..1` as the loop variable, like `for (... in x)`
+
+  - Air more accurately rejects certain syntax errors:
+
+    - `1 2`, which are two expressions side by side without a semicolon or newline
+
+    - Stray top-level `;`
+
+    - `pkg::` without a right-hand side
+
+    - `x$1` where the right-hand side is not an identifier
 
 # 0.12.0
 

CLAUDE.md

@@ -37,8 +37,11 @@ Air is an R language server and formatter written in Rust. Workspace crate layou
 ### Running Tests
 
 - Use `just test` to run tests via nextest, NOT `cargo test`
-- Example: `just test -p workspace --lib discovery::test`
-- Passes arguments through to `cargo nextest run`
+    - Example: `just test -p workspace --lib discovery::test`
+    - Passes arguments through to `cargo nextest run`
+- `just test-snapshot-accept` regenerates the parser's markdown snapshots
+- `just test-insta` regenerates the formatter's insta snapshots, `cargo insta review` to accept them
+- `just test-quick-parser` and `just test-quick-formatter` run the `quick_test.rs` scratchpads
 
 ### Unit Tests
 

CONTRIBUTING.md

@@ -187,6 +187,6 @@ We use [nextest](https://nexte.st/) for testing rather than a standard `cargo te
 
 Install the nextest cli tool using a [prebuilt binary](https://nexte.st/docs/installation/pre-built-binaries/).
 
-Run tests locally with `just test`, which calls `cargo nextest run`. Run insta snapshot tests in "update" mode with `just test-insta`.
+Run tests locally with `just test`, which calls `cargo nextest run`. Run insta snapshot tests in "update" mode with `just test-insta`. Regenerate the parser's markdown snapshots with `just test-snapshot-accept`.
 
-On CI we use the nextest profile found in `.config/nextest.toml`.
+The nextest profiles are in `.config/nextest.toml`. Local runs use `default`, and CI uses `ci`.

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["crates/*", "editors/zed", "xtask/codegen", "xtask/json_schema"]
+members = ["crates/*", "editors/zed", "xtask/accept", "xtask/codegen", "xtask/json_schema"]
 
 [workspace.package]
 authors = ["Posit Software, PBC"]
@@ -18,17 +18,21 @@ air_formatter_test = { path = "./crates/air_formatter_test" }
 air_r_factory = { path = "./crates/air_r_factory" }
 air_r_formatter = { path = "./crates/air_r_formatter" }
 air_r_parser = { path = "./crates/air_r_parser" }
+air_r_parser_tests = { path = "./crates/air_r_parser_tests" }
 air_r_syntax = { path = "./crates/air_r_syntax" }
+air_r_test_macros = { path = "./crates/air_r_test_macros" }
 anyhow = "1.0.89"
 assert_matches = "1.5.0"
+biome_console = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
+biome_diagnostics = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
 biome_formatter = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
 biome_line_index = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
 biome_parser = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
 biome_rowan = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
 biome_string_case = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
 biome_text_size = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
 biome_ungrammar = { path = "./crates/biome_ungrammar" }
-biome_unicode_table = { git = "https://github.com/lionel-/biome", rev = "a1296ea6ba363d8b8d8f02181b2a4ce9315c5ef9" }
+bpaf = { version = "0.9.15", features = ["derive"] }
 bytes = "1.8.0"
 cargo_metadata = "0.20.0"
 case = "1.0.0"
@@ -42,6 +46,7 @@ etcetera = "0.11.0"
 fs = { path = "./crates/fs" }
 futures = "0.3.31"
 futures-util = "0.3.31"
+git2 = { version = "0.21.0", default-features = false }
 globwalk = "0.9.1"
 httparse = "1.9.5"
 ignore = "0.4.23"
@@ -60,6 +65,7 @@ schemars = "0.9.0"
 serde = "1.0.215"
 serde_json = "1.0.132"
 settings = { path = "./crates/settings" }
+similar = "2.6"
 similar-asserts = "1.6.0"
 struct-field-names-as-array = "0.3.0"
 strum = "0.26"
@@ -74,8 +80,7 @@ toml = "0.8.19"
 tower-lsp = { branch = "bugfix/patches", git = "https://github.com/lionel-/tower-lsp" }
 tracing = { version = "0.1.40", default-features = false, features = ["std"] }
 tracing-subscriber = "0.3.19"
-tree-sitter = "0.24.7"
-tree-sitter-r = { git = "https://github.com/r-lib/tree-sitter-r", rev = "d459b97a9a11dd81643ab674938bede2874663b0" }
+unicode-ident = "1.0"
 url = "2.5.3"
 uuid = { version = "1.11.0", features = ["v4"] }
 workspace = { path = "./crates/workspace" }

crates/air_r_formatter/Cargo.toml

@@ -24,9 +24,7 @@ tracing = { workspace = true }
 
 [dev-dependencies]
 air_formatter_test = { workspace = true }
-air_r_factory = { workspace = true }
 air_r_parser = { workspace = true }
-anyhow = { workspace = true }
 biome_parser = { workspace = true }
 tests_macros = { workspace = true }
 workspace = { workspace = true }