From 245122c7569bd13abe37d90c92445baa311135c6 Mon Sep 17 00:00:00 2001 From: ascandone Date: Fri, 16 Oct 2026 17:03:35 +0200 Subject: [PATCH 1/2] feat(parser): replace the ANTLR parser with a hand-written, fault-tolerant one The ANTLR parser threw away whole subtrees on common in-progress edits (e.g. a bare `@`, `$`, or `allowing` with nothing after it), and its lexer modes could even garble the rest of the statement. The LSP needs a meaningful AST for programs that are being typed. The new recursive descent parser produces the same AST types, never gives up (missing tokens are reported without being consumed, unexpected tokens are skipped up to a token an enclosing construct can handle), and reports clearer errors (e.g. "expected `destination`, found `destinaton`"). It can also report what it expected at a given cursor position (ParseForCompletion), which is the basis for LSP completion. The ANTLR parser is kept as a test oracle (internal/parser/internal/legacy): on ~45k programs (the repository corpus plus mutations), both parsers accept the same programs and build the same ASTs. The only intended difference is that a line comment at the end of the file no longer needs a trailing newline. Claude-Session: https://claude.ai/code/session_013WheaNzvuvbKPqmRbyREjs --- CONTRIBUTING.md | 4 +- Justfile | 4 +- codecov.yml | 3 +- go.mod | 2 +- internal/analysis/check.go | 4 + internal/mcp_impl/handlers_test.go | 2 +- .../parser_fault_tolerance_test.snap | 665 ++++++- .../parser/__snapshots__/parser_test.snap | 24 +- internal/parser/completion.go | 127 ++ internal/parser/differential_test.go | 332 ++++ .../legacy}/antlrParser/Lexer.interp | 0 .../legacy}/antlrParser/Lexer.tokens | 0 .../legacy}/antlrParser/Numscript.interp | 0 .../legacy}/antlrParser/Numscript.tokens | 0 .../legacy}/antlrParser/lexer.go | 0 .../antlrParser/numscript_base_listener.go | 0 .../legacy}/antlrParser/numscript_listener.go | 0 .../legacy}/antlrParser/numscript_parser.go | 0 internal/parser/internal/legacy/legacy.go | 678 +++++++ internal/parser/lexer.go | 495 ++++++ internal/parser/parser.go | 1568 ++++++++++++----- .../parser/parser_fault_tolerance_test.go | 62 + internal/parser/parser_internal_test.go | 32 - internal/parser/parser_test.go | 2 +- .../__snapshots__/runner_test.snap | 11 +- 25 files changed, 3498 insertions(+), 517 deletions(-) create mode 100644 internal/parser/completion.go create mode 100644 internal/parser/differential_test.go rename internal/parser/{ => internal/legacy}/antlrParser/Lexer.interp (100%) rename internal/parser/{ => internal/legacy}/antlrParser/Lexer.tokens (100%) rename internal/parser/{ => internal/legacy}/antlrParser/Numscript.interp (100%) rename internal/parser/{ => internal/legacy}/antlrParser/Numscript.tokens (100%) rename internal/parser/{ => internal/legacy}/antlrParser/lexer.go (100%) rename internal/parser/{ => internal/legacy}/antlrParser/numscript_base_listener.go (100%) rename internal/parser/{ => internal/legacy}/antlrParser/numscript_listener.go (100%) rename internal/parser/{ => internal/legacy}/antlrParser/numscript_parser.go (100%) create mode 100644 internal/parser/internal/legacy/legacy.go create mode 100644 internal/parser/lexer.go delete mode 100644 internal/parser/parser_internal_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 50e4da95..b42fa389 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,4 +3,6 @@ You can update snaphshots with the `UPDATE_SNAPS=true` variable while running the tests -If you need to update the grammar, you can generate the parser by running `just generate` (you'll need to install the antlr4 command first) +The Numscript parser (`internal/parser`) is hand-written, so that it can recover from syntax errors: the LSP needs a meaningful AST for programs that are being typed. The [Numscript.g4](Numscript.g4) and [Lexer.g4](Lexer.g4) grammars are still the reference for the syntax, and they generate the previous, ANTLR-based parser (`internal/parser/internal/legacy`). The tests check that both parsers agree on every program of the repository, so if you change the syntax, update both the hand-written parser and the grammar. + +You can generate the ANTLR parsers by running `just generate` (you'll need to install the antlr4 command first) diff --git a/Justfile b/Justfile index b586574f..2aee7996 100644 --- a/Justfile +++ b/Justfile @@ -13,8 +13,8 @@ tidy: @go mod tidy generate: - @antlr4 -Dlanguage=Go Lexer.g4 Numscript.g4 -o internal/parser/antlrParser -package antlrParser - @mv internal/parser/antlrParser/_lexer.go internal/parser/antlrParser/lexer.go + @antlr4 -Dlanguage=Go Lexer.g4 Numscript.g4 -o internal/parser/internal/legacy/antlrParser -package antlrParser + @mv internal/parser/internal/legacy/antlrParser/_lexer.go internal/parser/internal/legacy/antlrParser/lexer.go @antlr4 -Dlanguage=Go IR.g4 -o internal/ir/internal/syntax/antlrParser -package antlrParser tests: diff --git a/codecov.yml b/codecov.yml index 78226af2..548f700b 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,2 +1,3 @@ ignore: - - "internal/parser/antlrParser" + - "internal/parser/internal/legacy" + - "internal/ir/internal/syntax/antlrParser" diff --git a/go.mod b/go.mod index 1309376c..ffa61b68 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/kr/pretty v0.3.1 github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.9.2 // indirect github.com/maruel/natural v1.3.0 // indirect diff --git a/internal/analysis/check.go b/internal/analysis/check.go index 428111e7..d07e4864 100644 --- a/internal/analysis/check.go +++ b/internal/analysis/check.go @@ -178,6 +178,10 @@ func (r *CheckResult) GetVarDeclType(decl parser.VarDeclaration) Type { } func (r *CheckResult) unifyNodeWith(expr parser.ValueExpr, t Type) { + if expr == nil { + // a missing node of an incomplete program + return + } exprT := r.GetExprType(expr) r.unify(expr.GetRange(), exprT, t) } diff --git a/internal/mcp_impl/handlers_test.go b/internal/mcp_impl/handlers_test.go index 36c00e1d..9560d5de 100644 --- a/internal/mcp_impl/handlers_test.go +++ b/internal/mcp_impl/handlers_test.go @@ -30,7 +30,7 @@ func TestHandleEvalToolRejectsParseErrors(t *testing.T) { require.NotEmpty(t, result.Content) text, ok := result.Content[0].(mcp.TextContent) require.True(t, ok) - require.Contains(t, text.Text, "mismatched input") + require.Contains(t, text.Text, "expected `source`") } func TestHandleEvalToolRejectsDuplicateBalancesWithScope(t *testing.T) { diff --git a/internal/parser/__snapshots__/parser_fault_tolerance_test.snap b/internal/parser/__snapshots__/parser_fault_tolerance_test.snap index a31860ea..44ac6c24 100755 --- a/internal/parser/__snapshots__/parser_fault_tolerance_test.snap +++ b/internal/parser/__snapshots__/parser_fault_tolerance_test.snap @@ -11,7 +11,7 @@ parser.Program{ { Range: parser.Range{ Start: parser.Position{Character:7, Line:0}, - End: parser.Position{Character:18, Line:0}, + End: parser.Position{Character:15, Line:0}, }, Name: (*parser.Variable)(nil), Type: &parser.TypeDecl{ @@ -59,7 +59,26 @@ parser.Program{ Start: parser.Position{}, End: parser.Position{Character:11, Line:0}, }, - SentValue: nil, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:11, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:11, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: nil, + }, + }, Source: nil, Destination: nil, }, @@ -76,16 +95,16 @@ parser.Program{ &parser.SendStatement{ Range: parser.Range{ Start: parser.Position{}, - End: parser.Position{Character:17, Line:4}, + End: parser.Position{Character:1, Line:5}, }, - SentValue: &parser.SentValueLiteral{ + SentValue: nil, + Source: &parser.SourceInorder{ Range: parser.Range{ - Start: parser.Position{Character:6, Line:0}, - End: parser.Position{Character:17, Line:4}, + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:3, Line:3}, }, - Monetary: nil, + Sources: nil, }, - Source: nil, Destination: nil, }, }, @@ -247,7 +266,6 @@ parser.Program{ abs: {0x1}, }, }, - nil, }, }, }, @@ -319,7 +337,7 @@ parser.Program{ { Range: parser.Range{ Start: parser.Position{Character:3, Line:3}, - End: parser.Position{Character:3, Line:4}, + End: parser.Position{Character:16, Line:3}, }, Cap: &parser.MonetaryLiteral{ Range: parser.Range{ @@ -501,3 +519,630 @@ parser.Program{ Comments: nil, } --- + +[TestFaultToleranceBareAt - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:1, Line:3}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:0}, + End: parser.Position{Character:13, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceInorder{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:19, Line:1}, + }, + Sources: { + &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:13, Line:1}, + End: parser.Position{Character:15, Line:1}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:13, Line:1}, + End: parser.Position{Character:15, Line:1}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:16, Line:2}, + End: parser.Position{Character:18, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceBareAt - 2] +Got errors while parsing: +expected an account name, found `}` + 0 | send [COIN 10] ( + 1 | source = { @a @ } + | ~ + 2 | destination = @b + + +--- + +[TestFaultToleranceIncompleteOverdraft - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:1, Line:3}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:0}, + End: parser.Position{Character:13, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:22, Line:1}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:16, Line:2}, + End: parser.Position{Character:18, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceIncompleteOverdraft - 2] +Got errors while parsing: +expected `unbounded` or `overdraft`, found `destination` + 0 | send [COIN 10] ( + 1 | source = @a allowing + | ~ + 2 | destination = @b + + +--- + +[TestFaultToleranceIncompleteInfix - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:1, Line:3}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:11, Line:0}, + }, + Monetary: &parser.BinaryInfix{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:11, Line:0}, + }, + Operator: "+", + Left: &parser.Variable{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:7, Line:0}, + }, + Name: "m", + }, + Right: nil, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:16, Line:2}, + End: parser.Position{Character:18, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceIncompleteInfix - 2] +Got errors while parsing: +expected an expression, found `source` + 0 | send $m + ( + | ~ + 1 | source = @a + + +--- + +[TestFaultToleranceBareDollar - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:12, Line:0}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:12, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:12, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: nil, + }, + }, + Source: nil, + Destination: nil, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceBareDollar - 2] +Got errors while parsing: +invalid variable name: expected a lowercase letter or `_` after `$` + 0 | send [COIN $ + | ~ +expected `]`, found end of file + 0 | send [COIN $ + | ~ + +--- + +[TestFaultToleranceMisspelledKeyword - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:1, Line:3}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:0}, + End: parser.Position{Character:13, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:15, Line:2}, + End: parser.Position{Character:17, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceMisspelledKeyword - 2] +Got errors while parsing: +expected `destination`, found `destinaton` + 1 | source = @a + 2 | destinaton = @b + | ~~~~~~~~~~ + 3 | ) + + +--- + +[TestFaultToleranceMissingParens - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.FnCall{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:19, Line:0}, + }, + Caller: &parser.FnCallIdentifier{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:11, Line:0}, + }, + Name: "set_tx_meta", + }, + Args: { + &parser.StringLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:12, Line:0}, + End: parser.Position{Character:15, Line:0}, + }, + String: "k", + }, + &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:17, Line:0}, + End: parser.Position{Character:19, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0x2a}, + }, + }, + }, + }, + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{Character:0, Line:1}, + End: parser.Position{Character:1, Line:4}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:1}, + End: parser.Position{Character:14, Line:1}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:1}, + End: parser.Position{Character:14, Line:1}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:1}, + End: parser.Position{Character:10, Line:1}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:2}, + End: parser.Position{Character:13, Line:2}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:2}, + End: parser.Position{Character:13, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:16, Line:3}, + End: parser.Position{Character:18, Line:3}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceMissingParens - 2] +Got errors while parsing: +expected `,` or `)`, found `send` + 0 | set_tx_meta("k", 42 + | ~ + 1 | send [COIN 10] ( + + +--- + +[TestFaultToleranceGarbageBetweenStatements - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:47, Line:0}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:0}, + End: parser.Position{Character:13, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:26, Line:0}, + End: parser.Position{Character:28, Line:0}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:26, Line:0}, + End: parser.Position{Character:28, Line:0}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:43, Line:0}, + End: parser.Position{Character:45, Line:0}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + &parser.SaveStatement{ + Range: parser.Range{ + Start: parser.Position{Character:0, Line:2}, + End: parser.Position{Character:21, Line:2}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:2}, + End: parser.Position{Character:13, Line:2}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:2}, + End: parser.Position{Character:13, Line:2}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:2}, + End: parser.Position{Character:10, Line:2}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:2}, + End: parser.Position{Character:12, Line:2}, + }, + Number: &big.Int{ + neg: false, + abs: {0x1}, + }, + }, + }, + }, + Account: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:19, Line:2}, + End: parser.Position{Character:21, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"c"}, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceGarbageBetweenStatements - 2] +Got errors while parsing: +invalid character: `!!` + 0 | send [COIN 10] ( source = @a destination = @b ) + 1 | !! this is not numscript ]] + | ~~ + 2 | save [COIN 1] from @c + +expected a statement, found `this` + 0 | send [COIN 10] ( source = @a destination = @b ) + 1 | !! this is not numscript ]] + | ~~~~ + 2 | save [COIN 1] from @c + + +--- diff --git a/internal/parser/__snapshots__/parser_test.snap b/internal/parser/__snapshots__/parser_test.snap index f4c50c03..6b9a21d1 100755 --- a/internal/parser/__snapshots__/parser_test.snap +++ b/internal/parser/__snapshots__/parser_test.snap @@ -2359,20 +2359,18 @@ parser.Program{ [TestShowErrorLines - 1] Got errors while parsing: -no viable alternative at input 'errdestination' +expected an expression, found `err` + 0 | send [EUR/2 100] ( 1 | source = err + | ~~~ 2 | destination = ee - | ~~~~~~~~~~ - 3 | ) -missing '(' at ')' - 2 | destination = ee - 3 | ) - | -missing ')' at '' +expected an expression, found `ee` + 1 | source = err 2 | destination = ee + | ~~ 3 | ) - | ~~~~ + --- @@ -3423,16 +3421,16 @@ parser.Program{ { Range: parser.Range{ Start: parser.Position{Character:2, Line:1}, - End: parser.Position{Character:19, Line:1}, + End: parser.Position{Character:18, Line:1}, }, - Content: " first comment\n", + Content: " first comment", }, { Range: parser.Range{ Start: parser.Position{Character:2, Line:4}, - End: parser.Position{Character:20, Line:4}, + End: parser.Position{Character:19, Line:4}, }, - Content: " second comment\n", + Content: " second comment", }, }, } diff --git a/internal/parser/completion.go b/internal/parser/completion.go new file mode 100644 index 00000000..0384b64d --- /dev/null +++ b/internal/parser/completion.go @@ -0,0 +1,127 @@ +package parser + +import "unicode/utf8" + +// A syntactic position where the parser was expecting something when it reached the cursor +type CompletionSlot interface{ completionSlot() } + +// An expression is expected. Types contains the types the expression is expected to have, +// when they are known from the syntactic context (e.g. "account" for a source). +type ExprSlot struct{ Types []string } + +// The type of a variable declaration is expected +type VarTypeSlot struct{} + +// A statement is expected +type StatementSlot struct{} + +// A part of an account literal (e.g. `@users:$id`) is expected +type AccountPartSlot struct{ LiteralStart Position } + +func (*ExprSlot) completionSlot() {} +func (*VarTypeSlot) completionSlot() {} +func (*StatementSlot) completionSlot() {} +func (*AccountPartSlot) completionSlot() {} + +// A function call enclosing the cursor +type FnCallFrame struct { + Name string + // The index of the argument the cursor is on + ArgIndex int +} + +// What the parser expected at the cursor position +type CompletionContext struct { + // The keywords that are valid at the cursor + Keywords []string + + Slots []CompletionSlot + + // The function calls enclosing the cursor (innermost last) + FnCalls []FnCallFrame + + // The range of the word under the cursor, that a completion would replace. + // It's an empty range at the cursor when no word is being typed. + Range Range + + // The part of the word under the cursor that precedes the cursor + Prefix string +} + +// Tokens that can be extended by typing more characters: when the cursor is right after +// one of them, that token is the one being completed +func (k tokenKind) isWordLike() bool { + switch k { + case tkIdentifier, tkVariable, tkAccountText, tkAsset, tkNumber: + return true + default: + return k.isKeyword() + } +} + +// Parses the input and returns what the parser expected at the cursor position. +// +// Returns a nil context when the cursor is inside a comment or a string literal. +func ParseForCompletion(input string, cursor Position) (ParseResult, *CompletionContext) { + tokens, comments, lexErrors := lex(input) + + for _, comment := range comments { + if posLt(comment.Start, cursor) && !posLt(comment.End, cursor) { + return Parse(input), nil + } + } + + cursorTok := -1 + completionRange := Range{Start: cursor, End: cursor} + prefix := "" + + for index, tk := range tokens { + if tk.Kind == tkEOF || !posLt(tk.Start, cursor) { + cursorTok = index + break + } + + containsCursor := posLt(cursor, tk.End) + touchesCursor := tk.End == cursor && tk.Kind.isWordLike() + if !containsCursor && !touchesCursor { + continue + } + + if tk.Kind == tkString { + return Parse(input), nil + } + + cursorTok = index + completionRange = tk.Range + // tokens never span multiple lines + prefix = runePrefix(tk.Text, cursor.Character-tk.Start.Character) + break + } + + p := newParser(tokens, cursorTok) + result := p.finish(input, comments, lexErrors) + + ctx := p.completion + ctx.Range = completionRange + ctx.Prefix = prefix + return result, &ctx +} + +func posLt(a, b Position) bool { + if a.Line != b.Line { + return a.Line < b.Line + } + return a.Character < b.Character +} + +func runePrefix(s string, n int) string { + offset := 0 + for range n { + if offset >= len(s) { + break + } + _, size := utf8.DecodeRuneInString(s[offset:]) + offset += size + } + return s[:offset] +} diff --git a/internal/parser/differential_test.go b/internal/parser/differential_test.go new file mode 100644 index 00000000..aff4f60b --- /dev/null +++ b/internal/parser/differential_test.go @@ -0,0 +1,332 @@ +package parser_test + +import ( + "go/ast" + goparser "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + + "github.com/formancehq/numscript/internal/parser" + "github.com/formancehq/numscript/internal/parser/internal/legacy" + "github.com/kr/pretty" +) + +// The hand-written parser replaced the ANTLR one. These tests check that both +// parsers agree on the corpus of numscript programs found in the repository: +// +// - when the ANTLR parser accepts a program, the new parser must accept it and produce the same AST +// - when the ANTLR parser rejects a program, the new parser must reject it too +// +// TODO remove these tests (and the legacy package) once the new parser has been released for a while. + +// Programs the ANTLR parser rejects, but are now accepted on purpose +func isIntentionalDivergence(src string) bool { + // a line comment at the end of the file without a trailing newline used to be a parsing error + lastLine := src[strings.LastIndex(src, "\n")+1:] + return strings.Contains(lastLine, "//") +} + +var handwrittenCorpus = []string{ + // operators precedence and associativity + `set_tx_meta("k", 1 + 2 + 3)`, + `set_tx_meta("k", 1 - 2 - 3)`, + `set_tx_meta("k", 1 / 2 / 3)`, + `set_tx_meta("k", 1 + 2 / 3 - 4)`, + `set_tx_meta("k", -1 / 2)`, + `set_tx_meta("k", - -1)`, + `set_tx_meta("k", -$x + 1)`, + `set_tx_meta("k", (1 + 2) / 3)`, + `set_tx_meta("k", -(1 + 2))`, + + // monetary literals, where the asset and amount expressions are juxtaposed + `send [COIN -100] ( source = @a destination = @b )`, + `send [$asset -100] ( source = @a destination = @b )`, + `send [$a - 1 2] ( source = @a destination = @b )`, + `send [$a - 1 - 2] ( source = @a destination = @b )`, + `send [$a - 1 + 2] ( source = @a destination = @b )`, + `send [$a - 1 - 2 - 3] ( source = @a destination = @b )`, + `send [($a - 1) -2] ( source = @a destination = @b )`, + `send [$a $b - $c] ( source = @a destination = @b )`, + `send [$a -$b - $c] ( source = @a destination = @b )`, + `send [COIN 10] + [COIN 20] ( source = @a destination = @b )`, + `send [COIN 10] - $x ( source = @a destination = @b )`, + `send [[COIN 1] -1] ( source = @a destination = @b )`, + `send [COIN *] ( source = @a destination = @b )`, + `send [COIN - 1 *] ( source = @a destination = @b )`, + `send [COIN] ( source = @a destination = @b )`, + `send [COIN 1 2] ( source = @a destination = @b )`, + + // account literals + `set_tx_meta("k", @a:b:c)`, + `set_tx_meta("k", @a:$b:c)`, + `set_tx_meta("k", @a-b_C:0)`, + `set_tx_meta("k", @a :b)`, + `set_tx_meta("k", @a: b)`, + `set_tx_meta("k", @ a)`, + `set_tx_meta("k", @$a)`, + `set_tx_meta("k", @a:)`, + + // sources and destinations + `send $x ( source = { } destination = @b )`, + `send $x ( source = { @a -@b } destination = @b )`, + `send $x ( source = { @a @b \ "c" } destination = @b )`, + `send $x ( source = { $p from @a remaining from @b } destination = @b )`, + `send $x ( source = { remaining from @a } destination = @b )`, + `send $x ( source = { 1/2 from @a @b } destination = @b )`, + `send $x ( source = { @a 1/2 from @b } destination = @b )`, + `send $x ( source = { max $m from @a @b } destination = @b )`, + `send $x ( source = oneof { @a @b } destination = @b )`, + `send $x ( source = oneof { } destination = @b )`, + `send $x ( source = @a \ "c" with scaling through @s destination = @b )`, + `send $x ( source = @a with scaling through @s \ "c" destination = @b )`, + `send $x ( source = @a allowing overdraft up to [COIN 10] destination = @b )`, + `send $x ( source = @a \ $c allowing unbounded overdraft destination = @b )`, + `send $x ( source = @a destination = { remaining kept } )`, + `send $x ( source = @a destination = { remaining to @b } )`, + `send $x ( source = @a destination = { remaining kept remaining to @c } )`, + `send $x ( source = @a destination = { max $m to @c remaining kept } )`, + `send $x ( source = @a destination = { max $m kept } )`, + `send $x ( source = @a destination = { } )`, + `send $x ( source = @a destination = { 1/2 to @a 1/2 kept } )`, + `send $x ( source = @a destination = { 1/2 to @a remaining kept } )`, + `send $x ( source = @a destination = { 1/2 to { max $m to @b remaining kept } remaining kept } )`, + `send $x ( source = @a destination = oneof { max $m to @c remaining kept } )`, + `send $x ( source = @a destination = oneof { remaining kept } )`, + `send $x ( destination = @b source = @a )`, + + // other statements + `save [COIN 10] from @a`, + `save [COIN *] from @a`, + `save $m from @a`, + `overdraft(@a, COIN)`, + `f()`, + `f(,)`, + `f(1,)`, + `f(1 2)`, + + // vars and flags + `vars { }`, + `vars { monetary $m = balance(@a, COIN) account $a }`, + `vars { monetary $m = [COIN -1] }`, + `vars { max $m }`, + "#![feature(\"a\", \"b\")]\nsend $x ( source = @a destination = @b )", + "#![feature()]", + "#![feature(\"a\",)]", + "vars { }\nvars { }", + "set_tx_meta(\"k\", 1)\nvars { }", + + // lexing + `set_tx_meta("k", 1_000)`, + `set_tx_meta("k", 12.5%)`, + `set_tx_meta("k", 12%)`, + `set_tx_meta("k", 12.5)`, + `set_tx_meta("k", 1.%)`, + `set_tx_meta("k", "a\"b")`, + `set_tx_meta("k", "a\"")`, + `set_tx_meta("k", "a\\"b")`, + `set_tx_meta("k", "a\\")`, + `set_tx_meta("k", "a)`, + `set_tx_meta("k", USD/2)`, + `set_tx_meta("k", USD/ 2)`, + `set_tx_meta("k", A1/02)`, + `set_tx_meta("k", $_x)`, + `set_tx_meta("k", $x1)`, + `set_tx_meta("k", $1)`, + `set_tx_meta("k", $X)`, + "set_tx_meta(\"k\", /* a /* nested */ comment */ 1)", + "set_tx_meta(\"k\", /* unterminated 1)", + "set_tx_meta(\"k\", 1) // comment\n", + "set_tx_meta(\"k\", 1)\r\n", + "sendx", + "send_", + "#", + "!", + "Ω", + "", +} + +func TestDifferentialAgainstAntlrParser(t *testing.T) { + corpus := collectCorpus(t) + if len(corpus) < 300 { + t.Fatalf("the corpus is suspiciously small: %d programs", len(corpus)) + } + + mutate := !testing.Short() + checked := 0 + for name, src := range corpus { + checkAgreement(t, name, src) + checked++ + + if mutate { + // remove one word at a time, to get many slightly broken (and some still valid) programs. + // Words are joined by newlines, so that line comments keep spanning a single word + words := strings.Fields(src) + if len(words) > 120 { + continue + } + for i := range words { + removed := slices.Delete(slices.Clone(words), i, i+1) + checkAgreement(t, name+"#remove"+strconv.Itoa(i), strings.Join(removed, "\n")) + + duplicated := slices.Insert(slices.Clone(words), i, words[i]) + checkAgreement(t, name+"#duplicate"+strconv.Itoa(i), strings.Join(duplicated, "\n")) + + if i+1 < len(words) { + swapped := slices.Clone(words) + swapped[i], swapped[i+1] = swapped[i+1], swapped[i] + checkAgreement(t, name+"#swap"+strconv.Itoa(i), strings.Join(swapped, "\n")) + } + checked += 3 + } + } + } + t.Logf("checked %d programs", checked) +} + +func checkAgreement(t *testing.T, name string, src string) { + t.Helper() + + expected := legacy.Parse(src) + got := parser.Parse(src) + + legacyOk := len(expected.Errors) == 0 + newOk := len(got.Errors) == 0 + + switch { + case legacyOk && !newOk: + t.Errorf("%s: the new parser rejects a valid program:\n%s\n\n%s", name, src, parser.ParseErrorsToString(got.Errors, src)) + + case !legacyOk && newOk: + if isIntentionalDivergence(src) { + return + } + t.Errorf("%s: the new parser accepts an invalid program:\n%s\n\nANTLR errors: %v", name, src, expected.Errors) + + case legacyOk && newOk: + expectedProgram, gotProgram := expected.Value, got.Value + expectedProgram.Comments, gotProgram.Comments = nil, nil + if diff := pretty.Diff(expectedProgram, gotProgram); len(diff) != 0 { + t.Errorf("%s: the ASTs differ for:\n%s\n\n%s", name, src, strings.Join(diff, "\n")) + } + } +} + +func collectCorpus(t *testing.T) map[string]string { + t.Helper() + + root := "../.." + corpus := map[string]string{} + for i, src := range handwrittenCorpus { + corpus["handwritten#"+strconv.Itoa(i)] = src + } + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // the oracle package contains the grammar of a different language (the machine numscript) + if d.Name() == "oracle" || d.Name() == "legacy" { + return filepath.SkipDir + } + return nil + } + + switch { + case strings.HasSuffix(path, ".num"): + content, err := os.ReadFile(path) + if err != nil { + return err + } + corpus[path] = string(content) + + case strings.HasSuffix(path, "_test.go"): + // Numscript programs embedded in go tests + for i, lit := range stringLiteralsOf(t, path) { + if looksLikeNumscript(lit) { + corpus[path+"#"+strconv.Itoa(i)] = lit + } + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + return corpus +} + +func stringLiteralsOf(t *testing.T, path string) []string { + fset := token.NewFileSet() + file, err := goparser.ParseFile(fset, path, nil, goparser.SkipObjectResolution) + if err != nil { + t.Fatal(err) + } + + var literals []string + ast.Inspect(file, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + value, err := strconv.Unquote(lit.Value) + if err == nil { + literals = append(literals, value) + } + return true + }) + return literals +} + +func looksLikeNumscript(s string) bool { + for _, keyword := range []string{"send ", "save ", "vars ", "set_tx_meta(", "set_account_meta("} { + if strings.Contains(s, keyword) { + return true + } + } + return false +} + +const benchmarkProgram = `vars { + monetary $amount + account $user + portion $fee = 2.5% +} + +send $amount ( + source = { + $user allowing overdraft up to [USD/2 100] + @users:$user:savings + @world + } + destination = { + $fee to @platform:fees + remaining to { + max [USD/2 1000] to @merchant + remaining kept + } + } +) + +set_tx_meta("processed", true_value(1 + 2 / 3)) +` + +func BenchmarkParse(b *testing.B) { + b.Run("handwritten", func(b *testing.B) { + for b.Loop() { + parser.Parse(benchmarkProgram) + } + }) + b.Run("antlr", func(b *testing.B) { + for b.Loop() { + legacy.Parse(benchmarkProgram) + } + }) +} diff --git a/internal/parser/antlrParser/Lexer.interp b/internal/parser/internal/legacy/antlrParser/Lexer.interp similarity index 100% rename from internal/parser/antlrParser/Lexer.interp rename to internal/parser/internal/legacy/antlrParser/Lexer.interp diff --git a/internal/parser/antlrParser/Lexer.tokens b/internal/parser/internal/legacy/antlrParser/Lexer.tokens similarity index 100% rename from internal/parser/antlrParser/Lexer.tokens rename to internal/parser/internal/legacy/antlrParser/Lexer.tokens diff --git a/internal/parser/antlrParser/Numscript.interp b/internal/parser/internal/legacy/antlrParser/Numscript.interp similarity index 100% rename from internal/parser/antlrParser/Numscript.interp rename to internal/parser/internal/legacy/antlrParser/Numscript.interp diff --git a/internal/parser/antlrParser/Numscript.tokens b/internal/parser/internal/legacy/antlrParser/Numscript.tokens similarity index 100% rename from internal/parser/antlrParser/Numscript.tokens rename to internal/parser/internal/legacy/antlrParser/Numscript.tokens diff --git a/internal/parser/antlrParser/lexer.go b/internal/parser/internal/legacy/antlrParser/lexer.go similarity index 100% rename from internal/parser/antlrParser/lexer.go rename to internal/parser/internal/legacy/antlrParser/lexer.go diff --git a/internal/parser/antlrParser/numscript_base_listener.go b/internal/parser/internal/legacy/antlrParser/numscript_base_listener.go similarity index 100% rename from internal/parser/antlrParser/numscript_base_listener.go rename to internal/parser/internal/legacy/antlrParser/numscript_base_listener.go diff --git a/internal/parser/antlrParser/numscript_listener.go b/internal/parser/internal/legacy/antlrParser/numscript_listener.go similarity index 100% rename from internal/parser/antlrParser/numscript_listener.go rename to internal/parser/internal/legacy/antlrParser/numscript_listener.go diff --git a/internal/parser/antlrParser/numscript_parser.go b/internal/parser/internal/legacy/antlrParser/numscript_parser.go similarity index 100% rename from internal/parser/antlrParser/numscript_parser.go rename to internal/parser/internal/legacy/antlrParser/numscript_parser.go diff --git a/internal/parser/internal/legacy/legacy.go b/internal/parser/internal/legacy/legacy.go new file mode 100644 index 00000000..6a3cc431 --- /dev/null +++ b/internal/parser/internal/legacy/legacy.go @@ -0,0 +1,678 @@ +package legacy + +import ( + "math/big" + "strings" + + "github.com/formancehq/numscript/internal/parser" + antlrParser "github.com/formancehq/numscript/internal/parser/internal/legacy/antlrParser" + "github.com/formancehq/numscript/internal/utils" + + "github.com/antlr4-go/antlr/v4" +) + +type ParserError struct { + parser.Range + Msg string +} + +func (e ParserError) Error() string { + return e.Msg +} + +type ParseResult struct { + Source string + Value parser.Program + Errors []ParserError +} + +type ErrorListener struct { + antlr.DefaultErrorListener + Errors []ParserError +} + +func (l *ErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, startL, startC int, msg string, e antlr.RecognitionException) { + length := 1 + if token, ok := offendingSymbol.(antlr.Token); ok { + length = len(token.GetText()) + } + endL := startL + endC := startC + length - 1 // -1 so that end character is inside the offending token + l.Errors = append(l.Errors, ParserError{ + Msg: msg, + Range: parser.Range{ + Start: parser.Position{Character: startC, Line: startL - 1}, + End: parser.Position{Character: endC, Line: endL - 1}, + }, + }) +} + +func Parse(input string) ParseResult { + // TODO handle lexer errors + listener := &ErrorListener{} + + is := antlr.NewInputStream(input) + lexer := antlrParser.NewLexer(is) + lexer.RemoveErrorListeners() + lexer.AddErrorListener(listener) + + stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel) + + p := antlrParser.NewNumscriptParser(stream) + p.RemoveErrorListeners() + p.AddErrorListener(listener) + + parsed := parseProgram(p.Program()) + + for _, tk := range stream.GetAllTokens() { + if tk.GetChannel() == antlr.TokenHiddenChannel { + parsed.Comments = append(parsed.Comments, parser.Comment{ + Content: tk.GetText()[2:], + Range: tokenToRange(tk), + }) + } + } + + return ParseResult{ + Source: input, + Value: parsed, + Errors: listener.Errors, + } +} + +func ParseErrorsToString(errors []ParserError, source string) string { + buf := "Got errors while parsing:\n" + for _, err := range errors { + buf += err.Msg + "\n" + err.ShowOnSource(source) + "\n" + } + return buf +} + +func parseVarsDeclaration(varsCtx antlrParser.IVarsDeclarationContext) *parser.VarDeclarations { + + if varsCtx == nil { + return nil + } + + varBlock := parser.VarDeclarations{ + Range: ctxToRange(varsCtx), + } + + for _, varDecl := range varsCtx.AllVarDeclaration() { + decl := parseVarDeclaration(varDecl) + if decl != nil { + varBlock.Declarations = append(varBlock.Declarations, *decl) + } + } + return &varBlock +} + +func parseProgram(programCtx antlrParser.IProgramContext) parser.Program { + vars := parseVarsDeclaration(programCtx.VarsDeclaration()) + + featureCtx := programCtx.FeatureDecl() + + var flagIdents []parser.StringLiteral + if featureCtx != nil { + for _, flagTk := range featureCtx.GetFlag() { + flagIdents = append(flagIdents, *parseStringTk(flagTk)) + } + } + + var statements []parser.Statement + for _, statementCtx := range programCtx.AllStatement() { + statements = append(statements, parseStatement(statementCtx)) + } + + return parser.Program{ + Flags: flagIdents, + Statements: statements, + Vars: vars, + } +} + +func parseVarDeclaration(varDecl antlrParser.IVarDeclarationContext) *parser.VarDeclaration { + if varDecl == nil { + return nil + } + + var origin *parser.ValueExpr + if varDecl.VarOrigin() != nil { + expr := parseValueExpr(varDecl.VarOrigin().ValueExpr()) + // when the origin expression is missing or malformed, + // parseValueExpr returns nil; do not wrap it in a non-nil pointer + if expr != nil { + origin = &expr + } + } + + return &parser.VarDeclaration{ + Range: ctxToRange(varDecl), + Type: parseVarType(varDecl.GetType_()), + Name: parseVarLiteral(varDecl.GetName()), + Origin: origin, + } +} + +func parseVarLiteral(tk antlr.Token) *parser.Variable { + if tk == nil || tk.GetTokenIndex() == -1 { + return nil + } + + name := tk.GetText()[1:] + + return &parser.Variable{ + Range: tokenToRange(tk), + Name: name, + } +} + +func parseVarType(tk antlr.Token) *parser.TypeDecl { + if tk == nil || tk.GetTokenIndex() == -1 { + return nil + } + + return &parser.TypeDecl{ + Range: tokenToRange(tk), + Name: tk.GetText(), + } +} + +func parseColorConstraint(colorConstraintCtx antlrParser.IColorConstraintContext) parser.ValueExpr { + if colorConstraintCtx == nil { + return nil + } + + return parseValueExpr( + colorConstraintCtx.ValueExpr(), + ) +} + +func parseSource(sourceCtx antlrParser.ISourceContext) parser.Source { + if sourceCtx == nil { + return nil + } + + range_ := ctxToRange(sourceCtx) + + switch sourceCtx := sourceCtx.(type) { + case *antlrParser.SrcAccountContext: + return &parser.SourceAccount{ + Range: range_, + Color: parseColorConstraint(sourceCtx.ColorConstraint()), + ValueExpr: parseValueExpr(sourceCtx.ValueExpr()), + } + + case *antlrParser.SrcInorderContext: + var sources []parser.Source + for _, sourceCtx := range sourceCtx.AllSource() { + sources = append(sources, parseSource(sourceCtx)) + } + return &parser.SourceInorder{ + Range: range_, + Sources: sources, + } + + case *antlrParser.SrcOneofContext: + var sources []parser.Source + for _, sourceCtx := range sourceCtx.AllSource() { + sources = append(sources, parseSource(sourceCtx)) + } + return &parser.SourceOneof{ + Range: range_, + Sources: sources, + } + + case *antlrParser.SrcAllotmentContext: + var items []parser.SourceAllotmentItem + for _, itemCtx := range sourceCtx.AllAllotmentClauseSrc() { + item := parser.SourceAllotmentItem{ + Range: ctxToRange(itemCtx), + Allotment: parseAllotment(itemCtx.Allotment()), + From: parseSource(itemCtx.Source()), + } + items = append(items, item) + } + return &parser.SourceAllotment{ + Range: range_, + Items: items, + } + + case *antlrParser.SrcCappedContext: + return &parser.SourceCapped{ + Range: range_, + From: parseSource(sourceCtx.Source()), + Cap: parseValueExpr(sourceCtx.GetCap_()), + } + + case *antlrParser.SrcAccountUnboundedOverdraftContext: + return &parser.SourceOverdraft{ + Range: ctxToRange(sourceCtx), + Color: parseColorConstraint(sourceCtx.ColorConstraint()), + Address: parseValueExpr(sourceCtx.GetAddress()), + } + + case *antlrParser.SrcAccountWithScalingContext: + return &parser.SourceWithScaling{ + Range: ctxToRange(sourceCtx), + Address: parseValueExpr(sourceCtx.GetAddress()), + Through: parseValueExpr(sourceCtx.GetSwap()), + } + + case *antlrParser.SrcAccountBoundedOverdraftContext: + varMon := parseValueExpr(sourceCtx.GetMaxOvedraft()) + + return &parser.SourceOverdraft{ + Range: ctxToRange(sourceCtx), + Color: parseColorConstraint(sourceCtx.ColorConstraint()), + Address: parseValueExpr(sourceCtx.GetAddress()), + Bounded: &varMon, + } + + case *antlrParser.SourceContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.Source](sourceCtx.GetText()) + } +} + +func parsePercentageRatio(source string, range_ parser.Range) *parser.PercentageLiteral { + num, floatingDigits, err := parser.ParsePercentageRatio(source) + if err != nil { + // This should be unreachable for tokens emitted by the lexer. + // Degrade gracefully to a zero-value literal instead of panicking, + // so that a malformed token cannot crash the process (e.g. the LSP server). + return &parser.PercentageLiteral{ + Range: range_, + Amount: big.NewInt(0), + } + } + + return &parser.PercentageLiteral{ + Range: range_, + Amount: num, + FloatingDigits: floatingDigits, + } +} + +func parseAllotment(allotmentCtx antlrParser.IAllotmentContext) parser.AllotmentValue { + switch allotmentCtx := allotmentCtx.(type) { + case *antlrParser.PortionedAllotmentContext: + expr := parseValueExpr(allotmentCtx.ValueExpr()) + return &parser.ValueExprAllotment{Value: expr} + + case *antlrParser.RemainingAllotmentContext: + return &parser.RemainingAllotment{ + Range: ctxToRange(allotmentCtx), + } + + case *antlrParser.AllotmentContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.AllotmentValue](allotmentCtx.GetText()) + } +} + +func parseStringTk(stringCtx antlr.Token) *parser.StringLiteral { + rawStr := stringCtx.GetText() + // Remove leading and trailing '"' + innerStr := rawStr[1 : len(rawStr)-1] + return &parser.StringLiteral{ + Range: tokenToRange(stringCtx), + String: innerStr, + } +} + +func parseValueExpr(valueExprCtx antlrParser.IValueExprContext) parser.ValueExpr { + switch valueExprCtx := valueExprCtx.(type) { + case *antlrParser.ParenthesizedExprContext: + return parseValueExpr(valueExprCtx.ValueExpr()) + + case *antlrParser.AccountLiteralContext: + litRng := ctxToRange(valueExprCtx) + + var parts []parser.AccountNamePart + for _, accLit := range valueExprCtx.GetChildren() { + switch accLit := accLit.(type) { + case *antlrParser.AccountTextPartContext: + varPartText := accLit.GetText() + parts = append(parts, parser.AccountTextPart{Name: varPartText}) + case *antlrParser.AccountVarPartContext: + v := parseVarLiteral(accLit.VARIABLE_NAME_ACC().GetSymbol()) + parts = append(parts, v) + + case antlr.TerminalNode: + if accLit.GetSymbol().GetTokenType() == antlrParser.NumscriptParserCOLON { + parts = append(parts, parser.AccountTextPart{Name: ":"}) + } + } + + } + + return &parser.AccountInterpLiteral{ + Range: litRng, + Parts: parts, + } + + case *antlrParser.MonetaryLiteralContext: + return parseMonetaryLit(valueExprCtx.MonetaryLit()) + + case *antlrParser.AssetLiteralContext: + return &parser.AssetLiteral{ + Range: ctxToRange(valueExprCtx), + Asset: valueExprCtx.GetText(), + } + + case *antlrParser.NumberLiteralContext: + return parseNumberLiteral(valueExprCtx.NUMBER()) + + case *antlrParser.PercentagePortionLiteralContext: + return parsePercentageRatio(valueExprCtx.GetText(), ctxToRange(valueExprCtx)) + + case *antlrParser.VariableExprContext: + return variableLiteralFromCtx(valueExprCtx) + + case *antlrParser.StringLiteralContext: + return parseStringTk(valueExprCtx.STRING().GetSymbol()) + + case *antlrParser.PrefixExprContext: + return &parser.Prefix{ + Range: ctxToRange(valueExprCtx), + Operator: parser.PrefixOperator(valueExprCtx.GetOp().GetText()), + Expr: parseValueExpr(valueExprCtx.ValueExpr()), + } + + case *antlrParser.InfixExprContext: + return &parser.BinaryInfix{ + Range: ctxToRange(valueExprCtx), + Operator: parser.InfixOperator(valueExprCtx.GetOp().GetText()), + Left: parseValueExpr(valueExprCtx.GetLeft()), + Right: parseValueExpr(valueExprCtx.GetRight()), + } + + case nil, *antlrParser.ValueExprContext: + return nil + + case *antlrParser.ApplicationContext: + return parseFnCall(valueExprCtx.FunctionCall()) + + default: + return utils.NonExhaustiveMatchPanic[parser.ValueExpr](valueExprCtx.GetText()) + } +} + +func variableLiteralFromCtx(ctx antlr.ParserRuleContext) *parser.Variable { + // Discard the '$' + name := ctx.GetText()[1:] + + return &parser.Variable{ + Range: ctxToRange(ctx), + Name: name, + } +} + +func parseDestination(destCtx antlrParser.IDestinationContext) parser.Destination { + if destCtx == nil { + return nil + } + + range_ := ctxToRange(destCtx) + + switch destCtx := destCtx.(type) { + case *antlrParser.DestAccountContext: + return &parser.DestinationAccount{ + ValueExpr: parseValueExpr(destCtx.ValueExpr()), + } + + case *antlrParser.DestInorderContext: + var inorderClauses []parser.CappedKeptOrDestination + for _, destInorderClause := range destCtx.AllDestinationInOrderClause() { + inorderClauses = append(inorderClauses, parseDestinationInorderClause(destInorderClause)) + } + + return &parser.DestinationInorder{ + Range: range_, + Clauses: inorderClauses, + Remaining: parseKeptOrDestination(destCtx.KeptOrDestination()), + } + + case *antlrParser.DestOneofContext: + var inorderClauses []parser.CappedKeptOrDestination + for _, destInorderClause := range destCtx.AllDestinationInOrderClause() { + inorderClauses = append(inorderClauses, parseDestinationInorderClause(destInorderClause)) + } + + return &parser.DestinationOneof{ + Range: range_, + Clauses: inorderClauses, + Remaining: parseKeptOrDestination(destCtx.KeptOrDestination()), + } + + case *antlrParser.DestAllotmentContext: + var items []parser.DestinationAllotmentItem + for _, itemCtx := range destCtx.AllAllotmentClauseDest() { + item := parser.DestinationAllotmentItem{ + Range: ctxToRange(itemCtx), + Allotment: parseDestinationAllotment(itemCtx.Allotment()), + To: parseKeptOrDestination(itemCtx.KeptOrDestination()), + } + items = append(items, item) + } + return &parser.DestinationAllotment{ + Range: range_, + Items: items, + } + + case *antlrParser.DestinationContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.Destination](destCtx.GetText()) + } + +} + +func parseDestinationInorderClause(clauseCtx antlrParser.IDestinationInOrderClauseContext) parser.CappedKeptOrDestination { + return parser.CappedKeptOrDestination{ + Range: ctxToRange(clauseCtx), + Cap: parseValueExpr(clauseCtx.ValueExpr()), + To: parseKeptOrDestination(clauseCtx.KeptOrDestination()), + } +} + +func parseKeptOrDestination(clauseCtx antlrParser.IKeptOrDestinationContext) parser.KeptOrDestination { + if clauseCtx == nil { + return nil + } + + switch clauseCtx := clauseCtx.(type) { + case *antlrParser.DestinationToContext: + return &parser.DestinationTo{ + Destination: parseDestination(clauseCtx.Destination()), + } + case *antlrParser.DestinationKeptContext: + return &parser.DestinationKept{ + Range: ctxToRange(clauseCtx), + } + case *antlrParser.KeptOrDestinationContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.KeptOrDestination](clauseCtx.GetText()) + } + +} + +func parseDestinationAllotment(allotmentCtx antlrParser.IAllotmentContext) parser.AllotmentValue { + switch allotmentCtx := allotmentCtx.(type) { + case *antlrParser.RemainingAllotmentContext: + return &parser.RemainingAllotment{ + Range: ctxToRange(allotmentCtx), + } + + case *antlrParser.PortionedAllotmentContext: + expr := parseValueExpr(allotmentCtx.ValueExpr()) + return &parser.ValueExprAllotment{Value: expr} + + case *antlrParser.AllotmentContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.AllotmentValue](allotmentCtx.GetText()) + } +} + +func parseFnArgs(fnCallArgCtx antlrParser.IFunctionCallArgsContext) []parser.ValueExpr { + if fnCallArgCtx == nil { + return nil + } + + var args []parser.ValueExpr + for _, valueExpr := range fnCallArgCtx.AllValueExpr() { + args = append(args, parseValueExpr(valueExpr)) + } + return args +} + +func parseFnCall(fnCallCtx antlrParser.IFunctionCallContext) *parser.FnCall { + if fnCallCtx == nil { + return nil + } + + ident := fnCallCtx.GetFnName() + if ident == nil { + return nil + } + + allArgs := fnCallCtx.FunctionCallArgs() + + return &parser.FnCall{ + Range: ctxToRange(fnCallCtx), + Caller: &parser.FnCallIdentifier{ + Range: tokenToRange(ident), + Name: ident.GetText(), + }, + Args: parseFnArgs(allArgs), + } +} + +func parseSaveStatement(saveCtx *antlrParser.SaveStatementContext) *parser.SaveStatement { + return &parser.SaveStatement{ + Range: ctxToRange(saveCtx), + SentValue: parseSentValue(saveCtx.SentValue()), + Account: parseValueExpr(saveCtx.ValueExpr()), + } +} + +func parseStatement(statementCtx antlrParser.IStatementContext) parser.Statement { + switch statementCtx := statementCtx.(type) { + case *antlrParser.SendStatementContext: + return parseSendStatement(statementCtx) + + case *antlrParser.SaveStatementContext: + return parseSaveStatement(statementCtx) + + case *antlrParser.FnCallStatementContext: + return parseFnCall(statementCtx.FunctionCall()) + + case *antlrParser.StatementContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.Statement](statementCtx.GetText()) + } +} + +func parseSentValue(statementCtx antlrParser.ISentValueContext) parser.SentValue { + switch statementCtx := statementCtx.(type) { + case *antlrParser.SentLiteralContext: + return &parser.SentValueLiteral{ + Range: ctxToRange(statementCtx), + Monetary: parseValueExpr(statementCtx.ValueExpr()), + } + case *antlrParser.SentAllContext: + return &parser.SentValueAll{ + Range: ctxToRange(statementCtx), + Asset: parseValueExpr(statementCtx.SentAllLit().GetAsset()), + } + + case *antlrParser.SentValueContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.SentValue](statementCtx.GetText()) + } + +} + +func parseSendStatement(statementCtx *antlrParser.SendStatementContext) *parser.SendStatement { + return &parser.SendStatement{ + Source: parseSource(statementCtx.Source()), + Destination: parseDestination(statementCtx.Destination()), + Range: ctxToRange(statementCtx), + SentValue: parseSentValue(statementCtx.SentValue()), + } +} + +func parseNumberLiteral(numNode antlr.TerminalNode) *parser.NumberLiteral { + amtStr := numNode.GetText() + amtStr = strings.ReplaceAll(amtStr, "_", "") + + amt, ok := new(big.Int).SetString(amtStr, 10) + if !ok { + // This should be unreachable for tokens emitted by the lexer. + // Degrade gracefully to a zero-value literal instead of panicking, + // so that a malformed token cannot crash the process (e.g. the LSP server). + amt = big.NewInt(0) + } + + return &parser.NumberLiteral{ + Range: tokenToRange(numNode.GetSymbol()), + Number: amt, + } +} + +func parseMonetaryLit(monetaryLitCtx antlrParser.IMonetaryLitContext) *parser.MonetaryLiteral { + if monetaryLitCtx.GetAmt() == nil { + return nil + } + + return &parser.MonetaryLiteral{ + Range: ctxToRange(monetaryLitCtx), + Asset: parseValueExpr(monetaryLitCtx.GetAsset()), + Amount: parseValueExpr(monetaryLitCtx.GetAmt()), + } +} + +func ctxToRange(ctx antlr.ParserRuleContext) parser.Range { + startTk := ctx.GetStart() + endTk := ctx.GetStop() + + return parser.Range{ + Start: parser.Position{ + Line: startTk.GetLine() - 1, + Character: startTk.GetColumn(), + }, + End: parser.Position{ + Line: endTk.GetLine() - 1, + + // this is based on the assumption that a token cannot span multiple lines + Character: endTk.GetColumn() + len(endTk.GetText()), + }, + } +} + +func tokenToRange(tk antlr.Token) parser.Range { + return parser.Range{ + Start: parser.Position{ + Line: tk.GetLine() - 1, + Character: tk.GetColumn(), + }, + End: parser.Position{ + Line: tk.GetLine() - 1, + Character: tk.GetColumn() + len(tk.GetText()), + }, + } +} diff --git a/internal/parser/lexer.go b/internal/parser/lexer.go new file mode 100644 index 00000000..4e60a91e --- /dev/null +++ b/internal/parser/lexer.go @@ -0,0 +1,495 @@ +package parser + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +type tokenKind uint8 + +const ( + tkEOF tokenKind = iota + + // Keywords + tkVars + tkMax + tkSource + tkDestination + tkSend + tkFrom + tkUp + tkTo + tkRemaining + tkAllowing + tkUnbounded + tkOverdraft + tkOneof + tkKept + tkSave + tkWith + tkScaling + tkThrough + tkFeature + + // Punctuation + tkLParen + tkRParen + tkLBracket + tkRBracket + tkLBrace + tkRBrace + tkComma + tkEq + tkStar + tkPlus + tkMinus + tkDiv + tkRestrict + tkHashBang + tkColon + tkAt + + // Literals + tkPercentage + tkString + tkIdentifier + tkNumber + tkAsset + tkVariable + tkAccountText +) + +var keywords = map[string]tokenKind{ + "vars": tkVars, + "max": tkMax, + "source": tkSource, + "destination": tkDestination, + "send": tkSend, + "from": tkFrom, + "up": tkUp, + "to": tkTo, + "remaining": tkRemaining, + "allowing": tkAllowing, + "unbounded": tkUnbounded, + "overdraft": tkOverdraft, + "oneof": tkOneof, + "kept": tkKept, + "save": tkSave, + "with": tkWith, + "scaling": tkScaling, + "through": tkThrough, + "feature": tkFeature, +} + +var punctuation = map[tokenKind]string{ + tkLParen: "(", + tkRParen: ")", + tkLBracket: "[", + tkRBracket: "]", + tkLBrace: "{", + tkRBrace: "}", + tkComma: ",", + tkEq: "=", + tkStar: "*", + tkPlus: "+", + tkMinus: "-", + tkDiv: "/", + tkRestrict: "\\", + tkHashBang: "#!", + tkColon: ":", + tkAt: "@", +} + +func (k tokenKind) isKeyword() bool { + return k >= tkVars && k <= tkFeature +} + +// A short human readable description, used in error messages +func (k tokenKind) String() string { + if k.isKeyword() { + for text, kind := range keywords { + if kind == k { + return "`" + text + "`" + } + } + } + if text, ok := punctuation[k]; ok { + return "`" + text + "`" + } + + switch k { + case tkEOF: + return "end of file" + case tkPercentage: + return "percentage" + case tkString: + return "string" + case tkIdentifier: + return "identifier" + case tkNumber: + return "number" + case tkAsset: + return "asset" + case tkVariable: + return "variable" + case tkAccountText: + return "account name" + default: + return "token" + } +} + +type token struct { + Kind tokenKind + Text string + Range + // byte offsets in the source + StartOffset, EndOffset int +} + +func (t token) describe() string { + switch t.Kind { + case tkEOF: + return "end of file" + case tkString: + return "string " + t.Text + default: + return "`" + t.Text + "`" + } +} + +type lexer struct { + src string + offset int + pos Position + + // true right after '@' or ':', where account name parts are lexed + accountMode bool + + tokens []token + comments []Comment + errors []ParserError +} + +func lex(src string) ([]token, []Comment, []ParserError) { + l := lexer{src: src} + l.run() + return l.tokens, l.comments, l.errors +} + +func (l *lexer) peekAt(n int) byte { + if l.offset+n >= len(l.src) { + return 0 + } + return l.src[l.offset+n] +} + +func (l *lexer) advance() { + r, size := utf8.DecodeRuneInString(l.src[l.offset:]) + l.offset += size + if r == '\n' { + l.pos.Line++ + l.pos.Character = 0 + } else { + l.pos.Character++ + } +} + +func (l *lexer) advanceWhile(pred func(byte) bool) { + for l.offset < len(l.src) && pred(l.src[l.offset]) { + l.advance() + } +} + +func (l *lexer) emit(kind tokenKind, startOffset int, startPos Position) { + l.tokens = append(l.tokens, token{ + Kind: kind, + Text: l.src[startOffset:l.offset], + Range: Range{Start: startPos, End: l.pos}, + StartOffset: startOffset, + EndOffset: l.offset, + }) +} + +func (l *lexer) errorFrom(startPos Position, msg string) { + l.errors = append(l.errors, ParserError{ + Range: Range{Start: startPos, End: l.pos}, + Msg: msg, + }) +} + +func isLower(c byte) bool { return c >= 'a' && c <= 'z' } +func isUpper(c byte) bool { return c >= 'A' && c <= 'Z' } +func isDigit(c byte) bool { return c >= '0' && c <= '9' } +func isIdentTail(c byte) bool { return isLower(c) || c == '_' } +func isVarNameStart(c byte) bool { return isLower(c) || c == '_' } +func isVarNameTail(c byte) bool { return isLower(c) || isDigit(c) || c == '_' } +func isAssetTail(c byte) bool { return isUpper(c) || isDigit(c) } +func isAccountText(c byte) bool { + return isLower(c) || isUpper(c) || isDigit(c) || c == '_' || c == '-' +} +func isWhitespace(c byte) bool { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' +} + +// Characters that cannot start any token. Consecutive invalid chars are reported as a single error. +func (l *lexer) isInvalidStart() bool { + c := l.src[l.offset] + if isWhitespace(c) || isLower(c) || isUpper(c) || isDigit(c) { + return false + } + switch c { + case '(', ')', '[', ']', '{', '}', ',', '=', '*', '+', '-', '/', '\\', ':', '@', '"', '$': + return false + case '#': + return l.peekAt(1) != '!' + default: + return true + } +} + +func (l *lexer) run() { + for { + if l.accountMode { + l.accountMode = false + l.lexAccountPart() + } + + l.advanceWhile(isWhitespace) + + startOffset, startPos := l.offset, l.pos + if l.offset >= len(l.src) { + l.emit(tkEOF, startOffset, startPos) + return + } + + c := l.src[l.offset] + switch { + case c == '/' && l.peekAt(1) == '/': + l.advanceWhile(func(c byte) bool { return c != '\n' && c != '\r' }) + l.comments = append(l.comments, Comment{ + Content: l.src[startOffset+2 : l.offset], + Range: Range{Start: startPos, End: l.pos}, + }) + + case c == '/' && l.peekAt(1) == '*': + l.lexMultilineComment(startPos) + + case isLower(c): + l.advance() + l.advanceWhile(isIdentTail) + kind, ok := keywords[l.src[startOffset:l.offset]] + if !ok { + kind = tkIdentifier + } + l.emit(kind, startOffset, startPos) + + case isUpper(c): + l.advance() + l.advanceWhile(isAssetTail) + if l.peekAt(0) == '/' && isDigit(l.peekAt(1)) { + l.advance() + l.advanceWhile(isDigit) + } + l.emit(tkAsset, startOffset, startPos) + + case isDigit(c): + l.lexNumber(startOffset, startPos) + + case c == '"': + l.lexString(startOffset, startPos) + + case c == '$': + l.lexVariable(startOffset, startPos) + + case c == '@': + l.advance() + l.emit(tkAt, startOffset, startPos) + l.accountMode = true + + case c == ':': + l.advance() + l.emit(tkColon, startOffset, startPos) + l.accountMode = true + + case c == '#' && l.peekAt(1) == '!': + l.advance() + l.advance() + l.emit(tkHashBang, startOffset, startPos) + + default: + kind, ok := singleCharTokens[c] + if ok { + l.advance() + l.emit(kind, startOffset, startPos) + continue + } + + for l.offset < len(l.src) && l.isInvalidStart() { + l.advance() + } + l.errorFrom(startPos, fmt.Sprintf("invalid character: `%s`", l.src[startOffset:l.offset])) + } + } +} + +var singleCharTokens = map[byte]tokenKind{ + '(': tkLParen, + ')': tkRParen, + '[': tkLBracket, + ']': tkRBracket, + '{': tkLBrace, + '}': tkRBrace, + ',': tkComma, + '=': tkEq, + '*': tkStar, + '+': tkPlus, + '-': tkMinus, + '/': tkDiv, + '\\': tkRestrict, +} + +// Lexes the part of an account literal right after '@' or ':'. +// No whitespace is allowed there: when no part is found, no token is emitted +// and the parser reports the missing part. +func (l *lexer) lexAccountPart() { + if l.offset >= len(l.src) { + return + } + + startOffset, startPos := l.offset, l.pos + c := l.src[l.offset] + switch { + case c == '$' && isVarNameStart(l.peekAt(1)): + l.lexVariable(startOffset, startPos) + case isAccountText(c): + l.advanceWhile(isAccountText) + l.emit(tkAccountText, startOffset, startPos) + } +} + +func (l *lexer) lexVariable(startOffset int, startPos Position) { + l.advance() // '$' + if !isVarNameStart(l.peekAt(0)) { + // Emit an (invalid) variable anyway, so that the parser knows a variable is being typed + l.emit(tkVariable, startOffset, startPos) + l.errorFrom(startPos, "invalid variable name: expected a lowercase letter or `_` after `$`") + return + } + l.advanceWhile(isVarNameTail) + l.emit(tkVariable, startOffset, startPos) +} + +func (l *lexer) lexNumber(startOffset int, startPos Position) { + l.advanceWhile(isDigit) + + // percentage: [0-9]+ ('.' [0-9]+)? '%' + if l.peekAt(0) == '%' { + l.advance() + l.emit(tkPercentage, startOffset, startPos) + return + } + if l.peekAt(0) == '.' && isDigit(l.peekAt(1)) { + i := 1 + for isDigit(l.peekAt(i)) { + i++ + } + if l.peekAt(i) == '%' { + for range i + 1 { + l.advance() + } + l.emit(tkPercentage, startOffset, startPos) + return + } + } + + // number: [0-9]+ ('_' [0-9]+)* + for l.peekAt(0) == '_' && isDigit(l.peekAt(1)) { + l.advance() + l.advanceWhile(isDigit) + } + l.emit(tkNumber, startOffset, startPos) +} + +// STRING: '"' ('\\"' | ~[\r\n"])* '"' +// +// We emulate the longest-match semantics: an escaped quote may also be the closing one. +func (l *lexer) lexString(startOffset int, startPos Position) { + end := -1 // offset right after the closing quote + i := l.offset + 1 + for i < len(l.src) { + c := l.src[i] + if c == '\n' || c == '\r' { + break + } + if c == '"' { + end = i + 1 + if l.src[i-1] != '\\' || i-1 == l.offset { + break + } + } + i++ + } + + if end == -1 { + // Unterminated string: take the rest of the line, so that the following lines are not affected + for l.offset < i { + l.advance() + } + l.emit(tkString, startOffset, startPos) + l.errorFrom(startPos, "unterminated string literal") + return + } + + for l.offset < end { + l.advance() + } + l.emit(tkString, startOffset, startPos) +} + +// MULTILINE_COMMENT: '/*' (MULTILINE_COMMENT | .)*? '*/' +// +// A nested '/*' opens a nested comment when that leads to a terminated comment, +// otherwise it's treated as plain text (this mimics the ANTLR lexer). +func (l *lexer) lexMultilineComment(startPos Position) { + memo := map[int]int{} + + // returns the offset right after the '*/' closing the comment whose content starts at i, or -1 + var matchFrom func(i int) int + matchFrom = func(start int) int { + if end, ok := memo[start]; ok { + return end + } + i := start + + end := -1 + for i < len(l.src) { + if strings.HasPrefix(l.src[i:], "*/") { + end = i + 2 + break + } + if strings.HasPrefix(l.src[i:], "/*") { + if nestedEnd := matchFrom(i + 2); nestedEnd != -1 { + if outerEnd := matchFrom(nestedEnd); outerEnd != -1 { + end = outerEnd + break + } + } + } + i++ + } + + memo[start] = end + return end + } + + end := matchFrom(l.offset + 2) + if end == -1 { + end = len(l.src) + defer l.errorFrom(startPos, "unterminated comment") + } + for l.offset < end { + l.advance() + } +} diff --git a/internal/parser/parser.go b/internal/parser/parser.go index deb00eeb..4ba7f036 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -3,14 +3,21 @@ package parser import ( "fmt" "math/big" + "slices" "strings" - - antlrParser "github.com/formancehq/numscript/internal/parser/antlrParser" - "github.com/formancehq/numscript/internal/utils" - - "github.com/antlr4-go/antlr/v4" ) +// A hand-written, error-tolerant recursive descent parser. +// +// The parser never gives up: when an expected token is missing, it reports an +// error and keeps going without consuming anything; when an unexpected token +// is found, it skips tokens until it reaches a token that some enclosing +// construct knows how to handle (see the "stop" token sets). +// This way, the AST of an incomplete program (e.g. while it's being typed) +// keeps as much information as possible. +// +// Missing nodes are represented as nil. + type ParserError struct { Range Msg string @@ -26,671 +33,1330 @@ type ParseResult struct { Errors []ParserError } -type ErrorListener struct { - antlr.DefaultErrorListener - Errors []ParserError +func Parse(input string) ParseResult { + tokens, comments, lexErrors := lex(input) + p := newParser(tokens, -1) + return p.finish(input, comments, lexErrors) } -func (l *ErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, startL, startC int, msg string, e antlr.RecognitionException) { - length := 1 - if token, ok := offendingSymbol.(antlr.Token); ok { - length = len(token.GetText()) +func ParseErrorsToString(errors []ParserError, source string) string { + buf := "Got errors while parsing:\n" + for _, err := range errors { + buf += err.Msg + "\n" + err.ShowOnSource(source) + "\n" } - endL := startL - endC := startC + length - 1 // -1 so that end character is inside the offending token - l.Errors = append(l.Errors, ParserError{ - Msg: msg, - Range: Range{ - Start: Position{Character: startC, Line: startL - 1}, - End: Position{Character: endC, Line: endL - 1}, - }, - }) + return buf } -func Parse(input string) ParseResult { - // TODO handle lexer errors - listener := &ErrorListener{} +// Returns (a, b) representing a/(10^b) +func ParsePercentageRatio(source string) (*big.Int, int, error) { + source = strings.TrimSuffix(source, "%") + + scale := 0 + if i := strings.Index(source, "."); i != -1 { + scale = len(source) - i - 1 + } + + intPart := strings.ReplaceAll(source, ".", "") + num, ok := new(big.Int).SetString(intPart, 10) + if !ok { + return nil, 0, fmt.Errorf("unexpected invalid string literal: %s", source) + } + + return num, scale, nil +} - is := antlr.NewInputStream(input) - lexer := antlrParser.NewLexer(is) - lexer.RemoveErrorListeners() - lexer.AddErrorListener(listener) +type tokenSet uint64 - stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel) +func setOf(kinds ...tokenKind) tokenSet { + var s tokenSet + for _, k := range kinds { + s |= 1 << k + } + return s +} + +func (s tokenSet) has(k tokenKind) bool { + return s&(1<= 0, infix operators at or after this token index are not consumed (used for backtracking) + stopAt int + + // --- completion + + // index of the token under the cursor, or -1 when not computing completions + cursorTok int + frozen bool + recorded bool + completion CompletionContext + fnFrames []FnCallFrame +} + +func newParser(tokens []token, cursorTok int) *parser { + return &parser{ + tokens: tokens, + lastErrorTok: -1, + stopAt: -1, + cursorTok: cursorTok, } +} + +func (p *parser) finish(input string, comments []Comment, lexErrors []ParserError) ParseResult { + program := p.parseProgram() + program.Comments = comments + + errors := append(lexErrors, p.errors...) + slices.SortStableFunc(errors, func(a, b ParserError) int { + if a.Start.Line != b.Start.Line { + return a.Start.Line - b.Start.Line + } + return a.Start.Character - b.Start.Character + }) return ParseResult{ Source: input, - Value: parsed, - Errors: listener.Errors, + Value: program, + Errors: errors, } } -func ParseErrorsToString(errors []ParserError, source string) string { - buf := "Got errors while parsing:\n" - for _, err := range errors { - buf += err.Msg + "\n" + err.ShowOnSource(source) + "\n" - } - return buf +// --- Token stream utilities + +func (p *parser) cur() token { + return p.tokens[p.pos] } -func parseVarsDeclaration(varsCtx antlrParser.IVarsDeclarationContext) *VarDeclarations { +func (p *parser) peek(n int) token { + i := min(p.pos+n, len(p.tokens)-1) + return p.tokens[i] +} - if varsCtx == nil { - return nil - } +func (p *parser) atCursor() bool { + return p.pos == p.cursorTok && !p.frozen +} - varBlock := VarDeclarations{ - Range: ctxToRange(varsCtx), +// Checks the current token kind, and records it as a possible completion +func (p *parser) at(kind tokenKind) bool { + if p.atCursor() { + p.recordCompletion() } - - for _, varDecl := range varsCtx.AllVarDeclaration() { - decl := parseVarDeclaration(varDecl) - if decl != nil { - varBlock.Declarations = append(varBlock.Declarations, *decl) + if p.atCursor() && kind.isKeyword() { + for text, k := range keywords { + if k == kind && !slices.Contains(p.completion.Keywords, text) { + p.completion.Keywords = append(p.completion.Keywords, text) + } } } - return &varBlock + return p.cur().Kind == kind +} + +func (p *parser) atExprStart(hints []string) bool { + p.recordSlot(&ExprSlot{Types: hints}) + return exprStart.has(p.cur().Kind) } -func parseProgram(programCtx antlrParser.IProgramContext) Program { - vars := parseVarsDeclaration(programCtx.VarsDeclaration()) +func (p *parser) recordSlot(slot CompletionSlot) { + if p.atCursor() { + p.recordCompletion() + p.completion.Slots = append(p.completion.Slots, slot) + } +} - featureCtx := programCtx.FeatureDecl() +func (p *parser) recordCompletion() { + if !p.recorded { + p.recorded = true + p.completion.FnCalls = slices.Clone(p.fnFrames) + } +} - var flagIdents []StringLiteral - if featureCtx != nil { - for _, flagTk := range featureCtx.GetFlag() { - flagIdents = append(flagIdents, *parseStringTk(flagTk)) +func (p *parser) bump() token { + t := p.cur() + if t.Kind == tkEOF { + return t + } + if p.cursorTok >= 0 && p.pos >= p.cursorTok { + if p.atCursor() { + p.recordCompletion() } + p.frozen = true } + p.pos++ + p.prevEnd = t.End + return t +} - var statements []Statement - for _, statementCtx := range programCtx.AllStatement() { - statements = append(statements, parseStatement(statementCtx)) +func (p *parser) eat(kind tokenKind) bool { + if p.at(kind) { + p.bump() + return true } + return false +} - return Program{ - Flags: flagIdents, - Statements: statements, - Vars: vars, +func (p *parser) expect(kind tokenKind) bool { + if p.eat(kind) { + return true } + p.errExpected(kind.String()) + return false } -func parseVarDeclaration(varDecl antlrParser.IVarDeclarationContext) *VarDeclaration { - if varDecl == nil { - return nil +// Like expect, but when the token is missing it skips tokens until either the +// expected token or a token in the stop set is found +func (p *parser) expectRecover(kind tokenKind, stop tokenSet) bool { + if p.eat(kind) { + return true } + p.errExpected(kind.String()) + p.skipUntil(stop | setOf(kind)) + return p.eat(kind) +} - var origin *ValueExpr - if varDecl.VarOrigin() != nil { - expr := parseValueExpr(varDecl.VarOrigin().ValueExpr()) - // when the origin expression is missing or malformed, - // parseValueExpr returns nil; do not wrap it in a non-nil pointer - if expr != nil { - origin = &expr +// Skip tokens until one in the given set is found. Balanced groups are skipped as a whole. +func (p *parser) skipUntil(stop tokenSet) { + depth := 0 + for { + kind := p.cur().Kind + if kind == tkEOF || (depth == 0 && stop.has(kind)) || stmtStop.has(kind) { + return + } + switch kind { + case tkLParen, tkLBrace, tkLBracket: + depth++ + case tkRParen, tkRBrace, tkRBracket: + if depth > 0 { + depth-- + } } + p.bump() } +} - return &VarDeclaration{ - Range: ctxToRange(varDecl), - Type: parseVarType(varDecl.GetType_()), - Name: parseVarLiteral(varDecl.GetName()), - Origin: origin, - } +func (p *parser) sameLine() bool { + return p.cur().Start.Line == p.prevEnd.Line } -func parseVarLiteral(tk antlr.Token) *Variable { - if tk == nil || tk.GetTokenIndex() == -1 { - return nil +func (p *parser) rangeFrom(start Position) Range { + end := p.prevEnd + if end.Line < start.Line || (end.Line == start.Line && end.Character < start.Character) { + end = start } + return Range{Start: start, End: end} +} - name := tk.GetText()[1:] +// --- Errors - return &Variable{ - Range: tokenToRange(tk), - Name: name, +func (p *parser) report(rng Range, msg string) { + if p.cursorTok >= 0 && p.pos >= p.cursorTok { + p.frozen = true } -} + if p.lastErrorTok == p.pos { + return + } + p.lastErrorTok = p.pos -func parseVarType(tk antlr.Token) *TypeDecl { - if tk == nil || tk.GetTokenIndex() == -1 { - return nil + if rng.Start == rng.End { + rng.End.Character++ } + p.errors = append(p.errors, ParserError{Range: rng, Msg: msg}) +} + +func (p *parser) errExpected(what string) { + cur := p.cur() + msg := fmt.Sprintf("expected %s, found %s", what, cur.describe()) - return &TypeDecl{ - Range: tokenToRange(tk), - Name: tk.GetText(), + // When the unexpected token is on a different line, the error is likely + // about something missing at the end of the previous line (unless the token + // is an identifier, which is likely a typo) + rng := cur.Range + if p.pos > 0 && !p.sameLine() && cur.Kind != tkIdentifier { + rng = Range{Start: p.prevEnd, End: p.prevEnd} } + p.report(rng, msg) } -func parseColorConstraint(colorConstraintCtx antlrParser.IColorConstraintContext) ValueExpr { - if colorConstraintCtx == nil { - return nil +// Reports an error on the current token, which is about to be skipped +func (p *parser) errUnexpected(what string) { + cur := p.cur() + p.report(cur.Range, fmt.Sprintf("expected %s, found %s", what, cur.describe())) +} + +// --- Backtracking + +type checkpoint struct { + pos int + nErrors int + lastErrorTok int + prevEnd Position + frozen bool + recorded bool + completion CompletionContext +} + +func (p *parser) checkpoint() checkpoint { + return checkpoint{ + pos: p.pos, + nErrors: len(p.errors), + lastErrorTok: p.lastErrorTok, + prevEnd: p.prevEnd, + frozen: p.frozen, + recorded: p.recorded, + completion: p.completion, } +} - return parseValueExpr( - colorConstraintCtx.ValueExpr(), - ) +func (p *parser) restore(c checkpoint) { + p.pos = c.pos + p.errors = p.errors[:c.nErrors] + p.lastErrorTok = c.lastErrorTok + p.prevEnd = c.prevEnd + p.frozen = c.frozen + p.recorded = c.recorded + p.completion = c.completion } -func parseSource(sourceCtx antlrParser.ISourceContext) Source { - if sourceCtx == nil { - return nil +// --- Program + +func (p *parser) parseProgram() Program { + var program Program + + if p.at(tkHashBang) { + program.Flags = p.parseFeatureDecl() } - range_ := ctxToRange(sourceCtx) + if p.at(tkVars) { + program.Vars = p.parseVars() + } - switch sourceCtx := sourceCtx.(type) { - case *antlrParser.SrcAccountContext: - return &SourceAccount{ - Range: range_, - Color: parseColorConstraint(sourceCtx.ColorConstraint()), - ValueExpr: parseValueExpr(sourceCtx.ValueExpr()), - } + for { + p.recordSlot(&StatementSlot{}) + isSend := p.at(tkSend) + isSave := p.at(tkSave) + isEOF := p.at(tkEOF) - case *antlrParser.SrcInorderContext: - var sources []Source - for _, sourceCtx := range sourceCtx.AllSource() { - sources = append(sources, parseSource(sourceCtx)) - } - return &SourceInorder{ - Range: range_, - Sources: sources, - } + switch { + case isSend: + program.Statements = append(program.Statements, p.parseSend()) - case *antlrParser.SrcOneofContext: - var sources []Source - for _, sourceCtx := range sourceCtx.AllSource() { - sources = append(sources, parseSource(sourceCtx)) - } - return &SourceOneof{ - Range: range_, - Sources: sources, - } + case isSave: + program.Statements = append(program.Statements, p.parseSave()) - case *antlrParser.SrcAllotmentContext: - var items []SourceAllotmentItem - for _, itemCtx := range sourceCtx.AllAllotmentClauseSrc() { - item := SourceAllotmentItem{ - Range: ctxToRange(itemCtx), - Allotment: parseAllotment(itemCtx.Allotment()), - From: parseSource(itemCtx.Source()), + case p.atFnCall(): + program.Statements = append(program.Statements, p.parseFnCall()) + + case isEOF: + return program + + default: + if p.cur().Kind == tkVars { + p.report(p.cur().Range, "the vars block must be declared before any statement") + } else { + p.errUnexpected("a statement") } - items = append(items, item) + p.bump() + p.skipUntilStatement() } - return &SourceAllotment{ - Range: range_, - Items: items, + } +} + +func (p *parser) atFnCall() bool { + kind := p.cur().Kind + return (kind == tkIdentifier || kind == tkOverdraft) && p.peek(1).Kind == tkLParen +} + +func (p *parser) skipUntilStatement() { + for !stmtStop.has(p.cur().Kind) && !p.atFnCall() { + p.bump() + } +} + +// '#!' '[' 'feature' '(' (STRING (',' STRING)*)? ')' ']' +func (p *parser) parseFeatureDecl() []StringLiteral { + p.bump() + + var flags []StringLiteral + if !p.expect(tkLBracket) || !p.expect(tkFeature) || !p.expect(tkLParen) { + p.skipUntil(setOf(tkRBracket)) + p.eat(tkRBracket) + return flags + } + + if p.cur().Kind == tkString { + for { + if !p.at(tkString) { + p.errExpected("a string") + break + } + flags = append(flags, *parseString(p.bump())) + if !p.eat(tkComma) { + break + } } + } + + p.expect(tkRParen) + p.expectRecover(tkRBracket, stmtStop) + return flags +} + +// 'vars' '{' varDeclaration* '}' +func (p *parser) parseVars() *VarDeclarations { + start := p.bump().Start + vars := &VarDeclarations{} + + if p.expect(tkLBrace) { + for { + p.recordSlot(&VarTypeSlot{}) + if p.at(tkIdentifier) && p.peek(1).Kind != tkLParen { + vars.Declarations = append(vars.Declarations, p.parseVarDeclaration()) + continue + } + + if p.eat(tkRBrace) { + break + } + + if stmtStop.has(p.cur().Kind) || p.atFnCall() { + p.errExpected("`}`") + break + } - case *antlrParser.SrcCappedContext: - return &SourceCapped{ - Range: range_, - From: parseSource(sourceCtx.Source()), - Cap: parseValueExpr(sourceCtx.GetCap_()), + p.errUnexpected("a variable declaration or `}`") + p.bump() } + } + + vars.Range = p.rangeFrom(start) + return vars +} - case *antlrParser.SrcAccountUnboundedOverdraftContext: - return &SourceOverdraft{ - Range: ctxToRange(sourceCtx), - Color: parseColorConstraint(sourceCtx.ColorConstraint()), - Address: parseValueExpr(sourceCtx.GetAddress()), +// IDENTIFIER VARIABLE_NAME ('=' valueExpr)? +func (p *parser) parseVarDeclaration() VarDeclaration { + typeTk := p.bump() + decl := VarDeclaration{ + Type: &TypeDecl{Range: typeTk.Range, Name: typeTk.Text}, + } + + if p.at(tkVariable) { + decl.Name = parseVariable(p.bump()) + } else { + p.errExpected("a variable name") + } + + if p.eat(tkEq) { + var hints []string + if typeTk.Text != "" { + hints = []string{typeTk.Text} } - case *antlrParser.SrcAccountWithScalingContext: - return &SourceWithScaling{ - Range: ctxToRange(sourceCtx), - Address: parseValueExpr(sourceCtx.GetAddress()), - Through: parseValueExpr(sourceCtx.GetSwap()), + origin := p.parseExpr(hints) + if origin != nil { + decl.Origin = &origin } + } - case *antlrParser.SrcAccountBoundedOverdraftContext: - varMon := parseValueExpr(sourceCtx.GetMaxOvedraft()) + decl.Range = p.rangeFrom(typeTk.Start) + return decl +} - return &SourceOverdraft{ - Range: ctxToRange(sourceCtx), - Color: parseColorConstraint(sourceCtx.ColorConstraint()), - Address: parseValueExpr(sourceCtx.GetAddress()), - Bounded: &varMon, - } +// --- Statements - case *antlrParser.SourceContext: - return nil +// 'send' sentValue '(' 'source' '=' source 'destination' '=' destination ')' +func (p *parser) parseSend() *SendStatement { + start := p.bump().Start + stmt := &SendStatement{ + SentValue: p.parseSentValue(), + } - default: - return utils.NonExhaustiveMatchPanic[Source](sourceCtx.GetText()) + p.expectRecover(tkLParen, setOf(tkSource, tkDestination, tkRParen)) + + if p.expectKeywordClause(tkSource) { + stmt.Source = p.parseSource(setOf(tkDestination, tkRParen), 0) + } + + if !p.at(tkDestination) { + p.errExpected(tkDestination.String()) + p.skipUntil(setOf(tkDestination, tkRParen) | p.misspelledKeywordSet()) } + + if p.expectKeywordClause(tkDestination) { + stmt.Destination = p.parseDestination(setOf(tkRParen), 0) + } + + p.expectRecover(tkRParen, 0) + + stmt.Range = p.rangeFrom(start) + return stmt } -// Returns (a, b) representing a/(10^b) -func ParsePercentageRatio(source string) (*big.Int, int, error) { - source = strings.TrimSuffix(source, "%") +func (p *parser) misspelledKeywordSet() tokenSet { + if p.cur().Kind == tkIdentifier && p.peek(1).Kind == tkEq { + return setOf(tkIdentifier) + } + return 0 +} - scale := 0 - if i := strings.Index(source, "."); i != -1 { - scale = len(source) - i - 1 +// Parses ` =`. A misspelled keyword (an identifier followed by '=') is +// reported but accepted, so that the rest of the clause can still be parsed. +func (p *parser) expectKeywordClause(keyword tokenKind) bool { + if !p.at(keyword) { + if p.cur().Kind == tkIdentifier && p.peek(1).Kind == tkEq { + p.errExpected(keyword.String()) + p.bump() + } else if p.cur().Kind == tkEq { + p.errExpected(keyword.String()) + } else { + p.errExpected(keyword.String()) + return false + } + } else { + p.bump() } - intPart := strings.ReplaceAll(source, ".", "") - num, ok := new(big.Int).SetString(intPart, 10) - if !ok { - return nil, 0, fmt.Errorf("unexpected invalid string literal: %s", source) + p.expect(tkEq) + return true +} + +// 'save' sentValue 'from' valueExpr +func (p *parser) parseSave() *SaveStatement { + start := p.bump().Start + stmt := &SaveStatement{ + SentValue: p.parseSentValue(), } - return num, scale, nil + if p.expect(tkFrom) || (p.sameLine() && exprStart.has(p.cur().Kind)) { + stmt.Account = p.parseExpr(hintAccount) + } + + stmt.Range = p.rangeFrom(start) + return stmt } -func parsePercentageRatio(source string, range_ Range) *PercentageLiteral { - num, floatingDigits, err := ParsePercentageRatio(source) - if err != nil { - // This should be unreachable for tokens emitted by the lexer. - // Degrade gracefully to a zero-value literal instead of panicking, - // so that a malformed token cannot crash the process (e.g. the LSP server). - return &PercentageLiteral{ - Range: range_, - Amount: big.NewInt(0), +// valueExpr | '[' valueExpr '*' ']' +func (p *parser) parseSentValue() SentValue { + isBracket := p.at(tkLBracket) + isExpr := p.atExprStart(hintMonetary) + + if isBracket { + start := p.cur().Start + lit, sendAll := p.parseBracketLiteral(true) + if sendAll != nil { + return sendAll } + expr := p.parseBinaryRest(start, lit, 0, hintMonetary) + return &SentValueLiteral{Range: expr.GetRange(), Monetary: expr} } - return &PercentageLiteral{ - Range: range_, - Amount: num, - FloatingDigits: floatingDigits, + if isExpr { + expr := p.parseExpr(hintMonetary) + if expr == nil { + return nil + } + return &SentValueLiteral{Range: expr.GetRange(), Monetary: expr} } + + p.errExpected("a monetary value") + return nil +} + +// --- Sources + +func (p *parser) atSourceStart() bool { + isBrace := p.at(tkLBrace) + isOneof := p.at(tkOneof) + isMax := p.at(tkMax) + isExpr := p.atExprStart(hintAccount) + return isBrace || isOneof || isMax || isExpr } -func parseAllotment(allotmentCtx antlrParser.IAllotmentContext) AllotmentValue { - switch allotmentCtx := allotmentCtx.(type) { - case *antlrParser.PortionedAllotmentContext: - expr := parseValueExpr(allotmentCtx.ValueExpr()) - return &ValueExprAllotment{expr} +func (p *parser) parseSource(stop tokenSet, follow tokenSet) Source { + start := p.cur().Start - case *antlrParser.RemainingAllotmentContext: - return &RemainingAllotment{ - Range: ctxToRange(allotmentCtx), + isBrace := p.at(tkLBrace) + isOneof := p.at(tkOneof) + isMax := p.at(tkMax) + isExpr := p.atExprStart(hintAccount) + + switch { + case isBrace: + return p.parseBraceSource(stop) + + case isOneof: + p.bump() + src := &SourceOneof{} + if p.expect(tkLBrace) { + if p.at(tkRBrace) { + p.errExpected("a source") + } + src.Sources = p.parseInorderSources(nil, stop) } + src.Range = p.rangeFrom(start) + return src - case *antlrParser.AllotmentContext: - return nil + case isMax: + p.bump() + src := &SourceCapped{ + Cap: p.parseExpr(hintMonetary), + } + if p.expect(tkFrom) || p.sameLine() { + if p.atSourceStart() { + src.From = p.parseSource(stop, follow) + } else { + p.errExpected("a source") + } + } + src.Range = p.rangeFrom(start) + return src + + case isExpr: + addr := p.parseExprBeforeAllotment(hintAccount, follow) + return p.parseSourceAccountSuffix(start, addr, follow) default: - return utils.NonExhaustiveMatchPanic[AllotmentValue](allotmentCtx.GetText()) + p.errExpected("a source") + return nil } } -func parseStringTk(stringCtx antlr.Token) *StringLiteral { - rawStr := stringCtx.GetText() - // Remove leading and trailing '"' - innerStr := rawStr[1 : len(rawStr)-1] - return &StringLiteral{ - Range: tokenToRange(stringCtx), - String: innerStr, +// Parses what follows the address of a source account: +// +// ('\' valueExpr)? ('allowing' 'unbounded' 'overdraft' | 'allowing' 'overdraft' 'up' 'to' valueExpr)? +// | 'with' 'scaling' 'through' valueExpr +func (p *parser) parseSourceAccountSuffix(start Position, addr ValueExpr, follow tokenSet) Source { + var color ValueExpr + if p.eat(tkRestrict) { + color = p.parseExprBeforeAllotment(hintString, follow) } -} -func parseValueExpr(valueExprCtx antlrParser.IValueExprContext) ValueExpr { - switch valueExprCtx := valueExprCtx.(type) { - case *antlrParser.ParenthesizedExprContext: - return parseValueExpr(valueExprCtx.ValueExpr()) + if p.eat(tkAllowing) { + if p.eat(tkUnbounded) { + p.expect(tkOverdraft) + return &SourceOverdraft{ + Range: p.rangeFrom(start), + Color: color, + Address: addr, + } + } + + if p.eat(tkOverdraft) { + p.expect(tkUp) + p.expect(tkTo) + bound := p.parseExprBeforeAllotment(hintMonetary, follow) + return &SourceOverdraft{ + Range: p.rangeFrom(start), + Color: color, + Address: addr, + Bounded: &bound, + } + } - case *antlrParser.AccountLiteralContext: - litRng := ctxToRange(valueExprCtx) + p.errExpected("`unbounded` or `overdraft`") + } else if p.at(tkWith) { + withTk := p.bump() + if color != nil { + p.report(withTk.Range, "a color restriction cannot be used together with scaling") + } + p.expect(tkScaling) + p.expect(tkThrough) + through := p.parseExprBeforeAllotment(hintAccount, follow) + return &SourceWithScaling{ + Range: p.rangeFrom(start), + Address: addr, + Through: through, + } + } - var parts []AccountNamePart - for _, accLit := range valueExprCtx.GetChildren() { - switch accLit := accLit.(type) { - case *antlrParser.AccountTextPartContext: - varPartText := accLit.GetText() - parts = append(parts, AccountTextPart{Name: varPartText}) - case *antlrParser.AccountVarPartContext: - v := parseVarLiteral(accLit.VARIABLE_NAME_ACC().GetSymbol()) - parts = append(parts, v) + if addr == nil && color == nil { + return nil + } - case antlr.TerminalNode: - if accLit.GetSymbol().GetTokenType() == antlrParser.NumscriptParserCOLON { - parts = append(parts, AccountTextPart{Name: ":"}) - } - } + return &SourceAccount{ + Range: p.rangeFrom(start), + Color: color, + ValueExpr: addr, + } +} +// Parses either an allotment or an inorder source, starting from the '{' +func (p *parser) parseBraceSource(stop tokenSet) Source { + start := p.bump().Start + + isRemaining := p.at(tkRemaining) + isExpr := p.atExprStart([]string{"portion", "account"}) + + if isRemaining { + return p.parseSourceAllotment(start, nil, Position{}, stop) + } + + if isExpr { + exprStart := p.cur().Start + expr := p.parseExpr([]string{"portion", "account"}) + if p.at(tkFrom) { + return p.parseSourceAllotment(start, expr, exprStart, stop) } - return &AccountInterpLiteral{ - Range: litRng, - Parts: parts, + var sources []Source + first := p.parseSourceAccountSuffix(exprStart, expr, 0) + if first != nil { + sources = append(sources, first) + } + return &SourceInorder{ + Sources: p.parseInorderSources(sources, stop), + Range: p.rangeFrom(start), } + } - case *antlrParser.MonetaryLiteralContext: - return parseMonetaryLit(valueExprCtx.MonetaryLit()) + return &SourceInorder{ + Sources: p.parseInorderSources(nil, stop), + Range: p.rangeFrom(start), + } +} - case *antlrParser.AssetLiteralContext: - return &AssetLiteral{ - Range: ctxToRange(valueExprCtx), - Asset: valueExprCtx.GetText(), +// source* '}' +func (p *parser) parseInorderSources(sources []Source, stop tokenSet) []Source { + for { + if p.atSourceStart() { + src := p.parseSource(stop|setOf(tkRBrace), 0) + if src != nil { + sources = append(sources, src) + } + continue } - case *antlrParser.NumberLiteralContext: - return parseNumberLiteral(valueExprCtx.NUMBER()) - - case *antlrParser.PercentagePortionLiteralContext: - return parsePercentageRatio(valueExprCtx.GetText(), ctxToRange(valueExprCtx)) + if p.eat(tkRBrace) { + return sources + } - case *antlrParser.VariableExprContext: - return variableLiteralFromCtx(valueExprCtx) + if stop.has(p.cur().Kind) || stmtStop.has(p.cur().Kind) { + p.errExpected("`}`") + return sources + } - case *antlrParser.StringLiteralContext: - return parseStringTk(valueExprCtx.STRING().GetSymbol()) + p.errUnexpected("a source or `}`") + p.bump() + } +} - case *antlrParser.PrefixExprContext: - return &Prefix{ - Range: ctxToRange(valueExprCtx), - Operator: PrefixOperator(valueExprCtx.GetOp().GetText()), - Expr: parseValueExpr(valueExprCtx.ValueExpr()), +// (allotment 'from' source)+ '}' +func (p *parser) parseSourceAllotment(start Position, first ValueExpr, firstStart Position, stop tokenSet) *SourceAllotment { + src := &SourceAllotment{} + inner := stop | setOf(tkRBrace) + + parseItem := func(itemStart Position, allotment AllotmentValue) { + item := SourceAllotmentItem{Allotment: allotment} + if p.expect(tkFrom) || p.sameLine() { + if p.atSourceStart() { + item.From = p.parseSource(inner, setOf(tkFrom)) + } else { + p.errExpected("a source") + } } + item.Range = p.rangeFrom(itemStart) + src.Items = append(src.Items, item) + } - case *antlrParser.InfixExprContext: - return &BinaryInfix{ - Range: ctxToRange(valueExprCtx), - Operator: InfixOperator(valueExprCtx.GetOp().GetText()), - Left: parseValueExpr(valueExprCtx.GetLeft()), - Right: parseValueExpr(valueExprCtx.GetRight()), + if first != nil { + parseItem(firstStart, &ValueExprAllotment{Value: first}) + } + + for { + isRemaining := p.at(tkRemaining) + isExpr := p.atExprStart(hintPortion) + if isRemaining || isExpr { + itemStart := p.cur().Start + parseItem(itemStart, p.parseAllotment()) + continue } - case nil, *antlrParser.ValueExprContext: - return nil + if p.eat(tkRBrace) { + break + } - case *antlrParser.ApplicationContext: - return parseFnCall(valueExprCtx.FunctionCall()) + if stop.has(p.cur().Kind) || stmtStop.has(p.cur().Kind) { + p.errExpected("`}`") + break + } - default: - return utils.NonExhaustiveMatchPanic[ValueExpr](valueExprCtx.GetText()) + p.errUnexpected("an allotment or `}`") + p.bump() } + + src.Range = p.rangeFrom(start) + return src } -func variableLiteralFromCtx(ctx antlr.ParserRuleContext) *Variable { - // Discard the '$' - name := ctx.GetText()[1:] +// Pre: at 'remaining' or at the beginning of an expression +func (p *parser) parseAllotment() AllotmentValue { + if p.at(tkRemaining) { + return &RemainingAllotment{Range: p.bump().Range} + } - return &Variable{ - Range: ctxToRange(ctx), - Name: name, + expr := p.parseExpr(hintPortion) + if expr == nil { + return nil } + return &ValueExprAllotment{Value: expr} } -func parseDestination(destCtx antlrParser.IDestinationContext) Destination { - if destCtx == nil { +// --- Destinations + +func (p *parser) parseDestination(stop tokenSet, follow tokenSet) Destination { + start := p.cur().Start + + isBrace := p.at(tkLBrace) + isOneof := p.at(tkOneof) + isExpr := p.atExprStart(hintAccount) + + switch { + case isBrace: + p.bump() + isMax := p.at(tkMax) + isRemaining := p.at(tkRemaining) + isExpr := p.atExprStart(hintPortion) + + if isMax || (!isRemaining && !isExpr) { + clauses, remaining := p.parseInorderClauses(stop) + return &DestinationInorder{ + Clauses: clauses, + Remaining: remaining, + Range: p.rangeFrom(start), + } + } + + return p.parseDestinationAllotment(start, stop) + + case isOneof: + p.bump() + dest := &DestinationOneof{} + if p.expect(tkLBrace) { + dest.Clauses, dest.Remaining = p.parseInorderClauses(stop) + } + dest.Range = p.rangeFrom(start) + return dest + + case isExpr: + expr := p.parseExprBeforeAllotment(hintAccount, follow) + if expr == nil { + return nil + } + return &DestinationAccount{ValueExpr: expr} + + default: + p.errExpected("a destination") return nil } +} - range_ := ctxToRange(destCtx) +// ('max' valueExpr keptOrDestination)* 'remaining' keptOrDestination '}' +func (p *parser) parseInorderClauses(stop tokenSet) ([]CappedKeptOrDestination, KeptOrDestination) { + var clauses []CappedKeptOrDestination + inner := stop | setOf(tkRBrace, tkMax, tkRemaining) - switch destCtx := destCtx.(type) { - case *antlrParser.DestAccountContext: - return &DestinationAccount{ - ValueExpr: parseValueExpr(destCtx.ValueExpr()), + for { + if p.at(tkMax) { + start := p.bump().Start + clause := CappedKeptOrDestination{ + Cap: p.parseExpr(hintMonetary), + } + clause.To = p.parseKeptOrDestination(inner, 0) + clause.Range = p.rangeFrom(start) + clauses = append(clauses, clause) + continue } - case *antlrParser.DestInorderContext: - var inorderClauses []CappedKeptOrDestination - for _, destInorderClause := range destCtx.AllDestinationInOrderClause() { - inorderClauses = append(inorderClauses, parseDestinationInorderClause(destInorderClause)) + if p.eat(tkRemaining) { + remaining := p.parseKeptOrDestination(stop|setOf(tkRBrace), 0) + p.expectRecover(tkRBrace, stop) + return clauses, remaining } - return &DestinationInorder{ - Range: range_, - Clauses: inorderClauses, - Remaining: parseKeptOrDestination(destCtx.KeptOrDestination()), + if p.at(tkRBrace) { + p.errExpected("`max` or `remaining`") + p.bump() + return clauses, nil } - case *antlrParser.DestOneofContext: - var inorderClauses []CappedKeptOrDestination - for _, destInorderClause := range destCtx.AllDestinationInOrderClause() { - inorderClauses = append(inorderClauses, parseDestinationInorderClause(destInorderClause)) + if stop.has(p.cur().Kind) || stmtStop.has(p.cur().Kind) { + p.errExpected("`remaining`") + return clauses, nil } - return &DestinationOneof{ - Range: range_, - Clauses: inorderClauses, - Remaining: parseKeptOrDestination(destCtx.KeptOrDestination()), - } + p.errUnexpected("`max` or `remaining`") + p.bump() + } +} - case *antlrParser.DestAllotmentContext: - var items []DestinationAllotmentItem - for _, itemCtx := range destCtx.AllAllotmentClauseDest() { +// (allotment keptOrDestination)+ '}' +func (p *parser) parseDestinationAllotment(start Position, stop tokenSet) *DestinationAllotment { + dest := &DestinationAllotment{} + inner := stop | setOf(tkRBrace) + + for { + isRemaining := p.at(tkRemaining) + isExpr := p.atExprStart(hintPortion) + if isRemaining || isExpr { + itemStart := p.cur().Start item := DestinationAllotmentItem{ - Range: ctxToRange(itemCtx), - Allotment: parseDestinationAllotment(itemCtx.Allotment()), - To: parseKeptOrDestination(itemCtx.KeptOrDestination()), + Allotment: p.parseAllotment(), } - items = append(items, item) + item.To = p.parseKeptOrDestination(inner, setOf(tkTo, tkKept)) + item.Range = p.rangeFrom(itemStart) + dest.Items = append(dest.Items, item) + continue } - return &DestinationAllotment{ - Range: range_, - Items: items, + + if p.eat(tkRBrace) { + break } - case *antlrParser.DestinationContext: - return nil + if stop.has(p.cur().Kind) || stmtStop.has(p.cur().Kind) { + p.errExpected("`}`") + break + } - default: - return utils.NonExhaustiveMatchPanic[Destination](destCtx.GetText()) + p.errUnexpected("an allotment or `}`") + p.bump() } + dest.Range = p.rangeFrom(start) + return dest } -func parseDestinationInorderClause(clauseCtx antlrParser.IDestinationInOrderClauseContext) CappedKeptOrDestination { - return CappedKeptOrDestination{ - Range: ctxToRange(clauseCtx), - Cap: parseValueExpr(clauseCtx.ValueExpr()), - To: parseKeptOrDestination(clauseCtx.KeptOrDestination()), +// 'to' destination | 'kept' +func (p *parser) parseKeptOrDestination(stop tokenSet, follow tokenSet) KeptOrDestination { + isTo := p.at(tkTo) + isKept := p.at(tkKept) + + switch { + case isTo: + p.bump() + return &DestinationTo{ + Destination: p.parseDestination(stop, follow), + } + + case isKept: + return &DestinationKept{Range: p.bump().Range} + + default: + p.errExpected("`to` or `kept`") + return nil } } -func parseKeptOrDestination(clauseCtx antlrParser.IKeptOrDestinationContext) KeptOrDestination { - if clauseCtx == nil { +// --- Expressions + +func infixPrecedence(kind tokenKind) int { + switch kind { + case tkPlus, tkMinus: + return 1 + case tkDiv: + return 2 + default: + return 0 + } +} + +// Parses an expression, reporting an error (and returning nil) when no expression is found +func (p *parser) parseExpr(hints []string) ValueExpr { + return p.parseBinary(1, hints) +} + +func (p *parser) parseBinary(minPrecedence int, hints []string) ValueExpr { + if !p.atExprStart(hints) { + p.errExpected("an expression") return nil } - switch clauseCtx := clauseCtx.(type) { - case *antlrParser.DestinationToContext: - return &DestinationTo{ - Destination: parseDestination(clauseCtx.Destination()), + start := p.cur().Start + left := p.parseUnary(hints) + return p.parseBinaryRest(start, left, minPrecedence, hints) +} + +// Precedence climbing: all the operators are left-associative +func (p *parser) parseBinaryRest(start Position, left ValueExpr, minPrecedence int, hints []string) ValueExpr { + for { + if p.stopAt >= 0 && p.pos >= p.stopAt { + return left } - case *antlrParser.DestinationKeptContext: - return &DestinationKept{ - Range: ctxToRange(clauseCtx), + + opTk := p.cur() + precedence := infixPrecedence(opTk.Kind) + if precedence == 0 || precedence < minPrecedence { + return left } - case *antlrParser.KeptOrDestinationContext: - return nil + p.bump() - default: - return utils.NonExhaustiveMatchPanic[KeptOrDestination](clauseCtx.GetText()) + rightHints := hints + if opTk.Kind == tkDiv { + rightHints = hintNumber + } + right := p.parseBinary(precedence+1, rightHints) + + left = &BinaryInfix{ + Range: p.rangeFrom(start), + Operator: InfixOperator(opTk.Text), + Left: left, + Right: right, + } + } +} + +// '-' valueExpr | primary +func (p *parser) parseUnary(hints []string) ValueExpr { + if p.cur().Kind != tkMinus { + return p.parsePrimary(hints) + } + + opTk := p.bump() + var expr ValueExpr + if p.atExprStart(hints) { + expr = p.parseUnary(hints) + } else { + p.errExpected("an expression") } + return &Prefix{ + Range: p.rangeFrom(opTk.Start), + Operator: PrefixOperator(opTk.Text), + Expr: expr, + } } -func parseDestinationAllotment(allotmentCtx antlrParser.IAllotmentContext) AllotmentValue { - switch allotmentCtx := allotmentCtx.(type) { - case *antlrParser.RemainingAllotmentContext: - return &RemainingAllotment{ - Range: ctxToRange(allotmentCtx), +// Pre: the current token is in the exprStart set +func (p *parser) parsePrimary(hints []string) ValueExpr { + tk := p.cur() + switch tk.Kind { + case tkVariable: + v := parseVariable(p.bump()) + if v == nil { + return nil } + return v + + case tkAsset: + p.bump() + return &AssetLiteral{Range: tk.Range, Asset: tk.Text} + + case tkString: + return parseString(p.bump()) + + case tkNumber: + return parseNumberLiteral(p.bump()) + + case tkPercentage: + p.bump() + return parsePercentageRatio(tk.Text, tk.Range) - case *antlrParser.PortionedAllotmentContext: - expr := parseValueExpr(allotmentCtx.ValueExpr()) - return &ValueExprAllotment{Value: expr} + case tkAt: + return p.parseAccountLiteral() - case *antlrParser.AllotmentContext: + case tkLBracket: + lit, _ := p.parseBracketLiteral(false) + return lit + + case tkLParen: + p.bump() + expr := p.parseExpr(hints) + p.expect(tkRParen) + return expr + + case tkIdentifier, tkOverdraft: + if p.peek(1).Kind == tkLParen { + return p.parseFnCall() + } + p.errUnexpected("an expression") + p.bump() return nil default: - return utils.NonExhaustiveMatchPanic[AllotmentValue](allotmentCtx.GetText()) + p.errExpected("an expression") + return nil } } -func parseFnArgs(fnCallArgCtx antlrParser.IFunctionCallArgsContext) []ValueExpr { - if fnCallArgCtx == nil { +// '@' accountPart (':' accountPart)* +func (p *parser) parseAccountLiteral() ValueExpr { + atTk := p.bump() + slot := &AccountPartSlot{LiteralStart: atTk.Start} + + var parts []AccountNamePart + p.parseAccountPart(&parts, slot) + for p.cur().Kind == tkColon { + p.bump() + parts = append(parts, AccountTextPart{Name: ":"}) + p.parseAccountPart(&parts, slot) + } + + if len(parts) == 0 { return nil } - var args []ValueExpr - for _, valueExpr := range fnCallArgCtx.AllValueExpr() { - args = append(args, parseValueExpr(valueExpr)) + return &AccountInterpLiteral{ + Range: p.rangeFrom(atTk.Start), + Parts: parts, } - return args } -func parseFnCall(fnCallCtx antlrParser.IFunctionCallContext) *FnCall { - if fnCallCtx == nil { - return nil - } +func (p *parser) parseAccountPart(parts *[]AccountNamePart, slot *AccountPartSlot) { + p.recordSlot(slot) - ident := fnCallCtx.GetFnName() - if ident == nil { - return nil + tk := p.cur() + // Account parts must not be separated by whitespace from the preceding '@' or ':' + adjacent := p.pos > 0 && p.tokens[p.pos-1].EndOffset == tk.StartOffset + + switch { + case tk.Kind == tkAccountText: + p.bump() + *parts = append(*parts, AccountTextPart{Name: tk.Text}) + + case tk.Kind == tkVariable && adjacent: + v := parseVariable(p.bump()) + if v != nil { + *parts = append(*parts, v) + } + + default: + p.errExpected("an account name") } +} - allArgs := fnCallCtx.FunctionCallArgs() +// Parses either a monetary literal ('[' valueExpr valueExpr ']') +// or, when allowSendAll is true, a sendAll literal ('[' valueExpr '*' ']') +func (p *parser) parseBracketLiteral(allowSendAll bool) (ValueExpr, *SentValueAll) { + openTk := p.bump() + start := openTk.Start - return &FnCall{ - Range: ctxToRange(fnCallCtx), - Caller: &FnCallIdentifier{ - Range: tokenToRange(ident), - Name: ident.GetText(), + asset := p.parseJuxtaposedExpr(hintAsset, + func() bool { + kind := p.cur().Kind + return !exprStart.has(kind) && (!allowSendAll || kind != tkStar) }, - Args: parseFnArgs(allArgs), + func() bool { + return p.parseExpr(hintNumber) != nil && p.cur().Kind == tkRBracket + }, + ) + + if allowSendAll && p.eat(tkStar) { + p.expectRecover(tkRBracket, setOf(tkLParen)) + return nil, &SentValueAll{ + Range: p.rangeFrom(start), + Asset: asset, + } } -} -func parseSaveStatement(saveCtx *antlrParser.SaveStatementContext) *SaveStatement { - return &SaveStatement{ - Range: ctxToRange(saveCtx), - SentValue: parseSentValue(saveCtx.SentValue()), - Account: parseValueExpr(saveCtx.ValueExpr()), + var amount ValueExpr + if p.atExprStart(hintNumber) { + amount = p.parseExpr(hintNumber) + } else { + p.errExpected("an amount") } -} -func parseStatement(statementCtx antlrParser.IStatementContext) Statement { - switch statementCtx := statementCtx.(type) { - case *antlrParser.SendStatementContext: - return parseSendStatement(statementCtx) + p.expectRecover(tkRBracket, setOf(tkLParen)) - case *antlrParser.SaveStatementContext: - return parseSaveStatement(statementCtx) + return &MonetaryLiteral{ + Range: p.rangeFrom(start), + Asset: asset, + Amount: amount, + }, nil +} - case *antlrParser.FnCallStatementContext: - return parseFnCall(statementCtx.FunctionCall()) +// Parses an expression that may be directly followed by an allotment, e.g. the +// `@a` in `{ 1/2 to @a -1/2 to @b }`, where the greedy parse would read `@a - 1/2`. +// follow is the set of tokens that follows an allotment in that position. +func (p *parser) parseExprBeforeAllotment(hints []string, follow tokenSet) ValueExpr { + if follow == 0 { + return p.parseExpr(hints) + } - case *antlrParser.StatementContext: - return nil + return p.parseJuxtaposedExpr(hints, + func() bool { return follow.has(p.cur().Kind) }, + func() bool { return p.parseExpr(hintPortion) != nil && follow.has(p.cur().Kind) }, + ) +} - default: - return utils.NonExhaustiveMatchPanic[Statement](statementCtx.GetText()) +// In some places of the grammar, two expressions are not separated by any token, +// (e.g. in the `[$asset -1]` monetary literal). When the greedy parse of the first +// expression consumes the beginning of the second one (needsSplit returns true after it), +// we look for a binary '-' that can be reinterpreted as the prefix operator of the +// second expression. The rightmost split for which isViable succeeds wins (that's the +// choice the ANTLR parser used to make). +// +// isViable is called after the first expression of the split, and it's allowed to consume tokens: +// the parser state is restored afterwards. +func (p *parser) parseJuxtaposedExpr(hints []string, needsSplit func() bool, isViable func() bool) ValueExpr { + before := p.checkpoint() + expr := p.parseExpr(hints) + if !needsSplit() { + return expr } -} -func parseSentValue(statementCtx antlrParser.ISentValueContext) SentValue { - switch statementCtx := statementCtx.(type) { - case *antlrParser.SentLiteralContext: - return &SentValueLiteral{ - Range: ctxToRange(statementCtx), - Monetary: parseValueExpr(statementCtx.ValueExpr()), - } - case *antlrParser.SentAllContext: - return &SentValueAll{ - Range: ctxToRange(statementCtx), - Asset: parseValueExpr(statementCtx.SentAllLit().GetAsset()), + greedyEnd := p.pos + + var candidates []int + depth := 0 + for i := before.pos; i < greedyEnd; i++ { + switch p.tokens[i].Kind { + case tkLParen, tkLBracket: + depth++ + case tkRParen, tkRBracket: + depth-- + case tkMinus: + if depth == 0 && i > before.pos && !prefixPosition.has(p.tokens[i-1].Kind) { + candidates = append(candidates, i) + } } + } + if len(candidates) == 0 { + return expr + } - case *antlrParser.SentValueContext: - return nil + prevStopAt := p.stopAt + defer func() { p.stopAt = prevStopAt }() - default: - return utils.NonExhaustiveMatchPanic[SentValue](statementCtx.GetText()) + for _, splitAt := range slices.Backward(candidates) { + p.restore(before) + p.lastErrorTok = -1 + + p.stopAt = splitAt + splitExpr := p.parseExpr(hints) + p.stopAt = prevStopAt + + if p.pos != splitAt || len(p.errors) != before.nErrors { + continue + } + + afterSplit := p.checkpoint() + viable := isViable() && len(p.errors) == before.nErrors + p.restore(afterSplit) + if viable { + return splitExpr + } } + // No viable split: parse the expression greedily again + p.restore(before) + expr = p.parseExpr(hints) + if p.pos != greedyEnd { + panic("unreachable: the re-parse of an expression diverged") + } + return expr } -func parseSendStatement(statementCtx *antlrParser.SendStatementContext) *SendStatement { - return &SendStatement{ - Source: parseSource(statementCtx.Source()), - Destination: parseDestination(statementCtx.Destination()), - Range: ctxToRange(statementCtx), - SentValue: parseSentValue(statementCtx.SentValue()), +// (IDENTIFIER | 'overdraft') '(' (valueExpr (',' valueExpr)*)? ')' +func (p *parser) parseFnCall() *FnCall { + nameTk := p.bump() + p.bump() // '(' + + p.fnFrames = append(p.fnFrames, FnCallFrame{Name: nameTk.Text}) + defer func() { p.fnFrames = p.fnFrames[:len(p.fnFrames)-1] }() + + call := &FnCall{ + Caller: &FnCallIdentifier{Range: nameTk.Range, Name: nameTk.Text}, } -} -func parseNumberLiteral(numNode antlr.TerminalNode) *NumberLiteral { - amtStr := numNode.GetText() - amtStr = strings.ReplaceAll(amtStr, "_", "") + isExpr := p.atExprStart(nil) + if isExpr || !p.at(tkRParen) { + for { + arg := p.parseExpr(nil) + if arg != nil { + call.Args = append(call.Args, arg) + } - amt, ok := new(big.Int).SetString(amtStr, 10) - if !ok { - // This should be unreachable for tokens emitted by the lexer. - // Degrade gracefully to a zero-value literal instead of panicking, - // so that a malformed token cannot crash the process (e.g. the LSP server). - amt = big.NewInt(0) + if !p.eat(tkComma) { + break + } + p.fnFrames[len(p.fnFrames)-1].ArgIndex++ + } } - return &NumberLiteral{ - Range: tokenToRange(numNode.GetSymbol()), - Number: amt, + if !p.eat(tkRParen) { + p.errExpected("`,` or `)`") } + + call.Range = p.rangeFrom(nameTk.Start) + return call } -func parseMonetaryLit(monetaryLitCtx antlrParser.IMonetaryLitContext) *MonetaryLiteral { - if monetaryLitCtx.GetAmt() == nil { +// --- Literals + +func parseVariable(tk token) *Variable { + if len(tk.Text) <= 1 { + // an invalid `$`: the error was already reported by the lexer return nil } + return &Variable{Range: tk.Range, Name: tk.Text[1:]} +} - return &MonetaryLiteral{ - Range: ctxToRange(monetaryLitCtx), - Asset: parseValueExpr(monetaryLitCtx.GetAsset()), - Amount: parseValueExpr(monetaryLitCtx.GetAmt()), +func parseString(tk token) *StringLiteral { + inner := tk.Text[1:] + // an unterminated string literal has no closing quote (the error is reported by the lexer) + if len(tk.Text) >= 2 && strings.HasSuffix(tk.Text, `"`) { + inner = inner[:len(inner)-1] } + return &StringLiteral{Range: tk.Range, String: inner} } -func ctxToRange(ctx antlr.ParserRuleContext) Range { - startTk := ctx.GetStart() - endTk := ctx.GetStop() +func parseNumberLiteral(tk token) *NumberLiteral { + amt, ok := new(big.Int).SetString(strings.ReplaceAll(tk.Text, "_", ""), 10) + if !ok { + // This should be unreachable for tokens emitted by the lexer. + // Degrade gracefully to a zero-value literal instead of panicking, + // so that a malformed token cannot crash the process (e.g. the LSP server). + amt = big.NewInt(0) + } - return Range{ - Start: Position{ - Line: startTk.GetLine() - 1, - Character: startTk.GetColumn(), - }, - End: Position{ - Line: endTk.GetLine() - 1, + return &NumberLiteral{Range: tk.Range, Number: amt} +} - // this is based on the assumption that a token cannot span multiple lines - Character: endTk.GetColumn() + len(endTk.GetText()), - }, +func parsePercentageRatio(source string, range_ Range) *PercentageLiteral { + num, floatingDigits, err := ParsePercentageRatio(source) + if err != nil { + // This should be unreachable for tokens emitted by the lexer. + // Degrade gracefully to a zero-value literal instead of panicking, + // so that a malformed token cannot crash the process (e.g. the LSP server). + return &PercentageLiteral{ + Range: range_, + Amount: big.NewInt(0), + } } -} -func tokenToRange(tk antlr.Token) Range { - return Range{ - Start: Position{ - Line: tk.GetLine() - 1, - Character: tk.GetColumn(), - }, - End: Position{ - Line: tk.GetLine() - 1, - Character: tk.GetColumn() + len(tk.GetText()), - }, + return &PercentageLiteral{ + Range: range_, + Amount: num, + FloatingDigits: floatingDigits, } } diff --git a/internal/parser/parser_fault_tolerance_test.go b/internal/parser/parser_fault_tolerance_test.go index bbb218d3..b8201b6a 100644 --- a/internal/parser/parser_fault_tolerance_test.go +++ b/internal/parser/parser_fault_tolerance_test.go @@ -96,3 +96,65 @@ save [COIN 10] from `) snaps.MatchSnapshot(t, p.Value) } + +// The cases below used to lose (or garble) most of the statement with the ANTLR parser + +func TestFaultToleranceBareAt(t *testing.T) { + p := parser.Parse(`send [COIN 10] ( + source = { @a @ } + destination = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceIncompleteOverdraft(t *testing.T) { + p := parser.Parse(`send [COIN 10] ( + source = @a allowing + destination = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceIncompleteInfix(t *testing.T) { + p := parser.Parse(`send $m + ( + source = @a + destination = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceBareDollar(t *testing.T) { + p := parser.Parse(`send [COIN $`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceMisspelledKeyword(t *testing.T) { + p := parser.Parse(`send [COIN 10] ( + source = @a + destinaton = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceMissingParens(t *testing.T) { + p := parser.Parse(`set_tx_meta("k", 42 +send [COIN 10] ( + source = @a + destination = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceGarbageBetweenStatements(t *testing.T) { + p := parser.Parse(`send [COIN 10] ( source = @a destination = @b ) +!! this is not numscript ]] +save [COIN 1] from @c`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} diff --git a/internal/parser/parser_internal_test.go b/internal/parser/parser_internal_test.go deleted file mode 100644 index c4b6d90d..00000000 --- a/internal/parser/parser_internal_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package parser - -import ( - "math/big" - "testing" - - "github.com/antlr4-go/antlr/v4" - "github.com/stretchr/testify/require" -) - -// These inputs should be unreachable through Parse() (the lexer only emits -// well-formed tokens), but the parsing helpers must degrade gracefully to a -// zero-value literal instead of panicking and killing the process. - -func TestParsePercentageRatioInvalidSourceDoesNotPanic(t *testing.T) { - require.NotPanics(t, func() { - lit := parsePercentageRatio("not a number%", Range{}) - require.Equal(t, big.NewInt(0), lit.Amount) - require.Equal(t, 0, lit.FloatingDigits) - }) -} - -func TestParseNumberLiteralInvalidNumberDoesNotPanic(t *testing.T) { - tk := antlr.NewCommonToken(&antlr.TokenSourceCharStreamPair{}, 0, antlr.TokenDefaultChannel, 0, 0) - tk.SetText("_") - node := antlr.NewTerminalNodeImpl(tk) - - require.NotPanics(t, func() { - lit := parseNumberLiteral(node) - require.Equal(t, big.NewInt(0), lit.Number) - }) -} diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index c477ad58..951e3a22 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -65,7 +65,7 @@ func TestForbidInvalidNumberUnderscores(t *testing.T) { require.Len(t, parser.Parse(` set_tx_meta("k1", 1__23) - `).Errors, 3) + `).Errors, 2) } func TestVariableMonetary(t *testing.T) { diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index 19a6406a..2acb5acb 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -131,16 +131,19 @@ Error: example.num.specs.json  Error: example.num:1:1 -token recognition error at: '!' +invalid character: `!` + + 0 | !err + | ~  -Error: example.num:1:5 +Error: example.num:1:2 -mismatched input '' expecting '(' +expected a statement, found `err` 0 | !err - | ~~~~ + | ~~~ --- From bc5461677d5c3026d8cfaf76fe785495bfae4556 Mon Sep 17 00:00:00 2001 From: ascandone Date: Fri, 16 Oct 2026 17:03:35 +0200 Subject: [PATCH 2/2] feat(lsp): add completion and signature help Built on top of the parser's completion context: - keywords valid at the cursor (e.g. `allowing`, `with`, `destination` after a source account) - variables, filtered by the type expected at the cursor (account in a source, portion in an allotment, function parameter types, ...) - builtin functions (as snippets), filtered by return type - account and asset literals already used in the document, including `@world` - variable interpolation in account literals (`@users:$id`) - variable types in the vars block, and statement snippets - signature help for builtin function calls, including incomplete ones Also tests that analysing every prefix of the corpus programs (i.e. typing them one char at a time) never panics, plus a fuzz test. Claude-Session: https://claude.ai/code/session_013WheaNzvuvbKPqmRbyREjs --- internal/analysis/completion.go | 520 ++++++++++++++++++ internal/analysis/completion_test.go | 186 +++++++ internal/analysis/robustness_test.go | 80 +++ internal/lsp/__snapshots__/handlers_test.snap | 114 ++++ internal/lsp/handlers.go | 112 +++- internal/lsp/handlers_test.go | 69 +++ 6 files changed, 1080 insertions(+), 1 deletion(-) create mode 100644 internal/analysis/completion.go create mode 100644 internal/analysis/completion_test.go create mode 100644 internal/analysis/robustness_test.go diff --git a/internal/analysis/completion.go b/internal/analysis/completion.go new file mode 100644 index 00000000..24805164 --- /dev/null +++ b/internal/analysis/completion.go @@ -0,0 +1,520 @@ +package analysis + +import ( + "fmt" + "slices" + "strings" + + "github.com/formancehq/numscript/internal/parser" +) + +type CompletionItemKind uint8 + +const ( + CompletionKeyword CompletionItemKind = iota + CompletionVariable + CompletionFunction + CompletionType + CompletionAccount + CompletionAsset + CompletionSnippet +) + +type CompletionItem struct { + Label string + Kind CompletionItemKind + Detail string + Docs string + + // The range of text that the completion replaces + Range parser.Range + + // The text inserted when snippets aren't supported + InsertText string + // The text inserted when snippets are supported. Empty when the item isn't a snippet. + SnippetText string + + // Items with a lower priority are displayed first + Priority int +} + +const ( + priorityVariable = iota + priorityLiteral + priorityFunction + priorityKeyword + prioritySnippet +) + +// Returns the completions that are valid at the given position. +// The program does not need to be valid: in fact it's usually being typed. +func Complete(source string, position parser.Position) []CompletionItem { + parsed, ctx := parser.ParseForCompletion(source, position) + if ctx == nil { + return nil + } + + c := completer{ + source: source, + position: position, + ctx: ctx, + program: parsed.Value, + seen: map[string]struct{}{}, + } + return c.complete() +} + +type completer struct { + source string + position parser.Position + ctx *parser.CompletionContext + program parser.Program + items []CompletionItem + seen map[string]struct{} +} + +func (c *completer) add(item CompletionItem) { + key := fmt.Sprintf("%d:%s", item.Kind, item.Label) + if _, ok := c.seen[key]; ok { + return + } + c.seen[key] = struct{}{} + + if item.Range == (parser.Range{}) { + item.Range = c.ctx.Range + } + if item.InsertText == "" { + item.InsertText = item.Label + } + c.items = append(c.items, item) +} + +func (c *completer) complete() []CompletionItem { + for _, slot := range c.ctx.Slots { + switch slot := slot.(type) { + case *parser.ExprSlot: + c.completeExpr(c.expectedTypes(slot)) + case *parser.AccountPartSlot: + c.completeAccountPart(slot) + case *parser.VarTypeSlot: + c.completeVarType() + case *parser.StatementSlot: + c.completeStatement() + } + } + + for _, keyword := range c.ctx.Keywords { + c.add(CompletionItem{ + Label: keyword, + Kind: CompletionKeyword, + Priority: priorityKeyword, + }) + } + + // When a variable or an account is being typed, other kinds of completions are just noise + if strings.HasPrefix(c.ctx.Prefix, "$") || strings.HasPrefix(c.ctx.Prefix, "@") { + prefix := c.ctx.Prefix[:1] + c.items = slices.DeleteFunc(c.items, func(item CompletionItem) bool { + return !strings.HasPrefix(item.Label, prefix) + }) + } + + slices.SortStableFunc(c.items, func(a, b CompletionItem) int { + return a.Priority - b.Priority + }) + return c.items +} + +// The types the expression at the cursor is expected to have. An empty slice means any type. +func (c *completer) expectedTypes(slot *parser.ExprSlot) []string { + if len(slot.Types) != 0 { + return slot.Types + } + + if len(c.ctx.FnCalls) == 0 { + return nil + } + + call := c.ctx.FnCalls[len(c.ctx.FnCalls)-1] + resolution, ok := Builtins[call.Name] + if !ok { + return nil + } + + params := resolution.GetParams() + if call.ArgIndex >= len(params) || params[call.ArgIndex] == TypeAny { + return nil + } + return []string{params[call.ArgIndex]} +} + +func typeAllowed(t string, expected []string) bool { + return len(expected) == 0 || slices.Contains(expected, t) +} + +func (c *completer) completeExpr(expected []string) { + if c.program.Vars != nil { + for _, decl := range c.program.Vars.Declarations { + if decl.Name == nil || decl.Type == nil || !typeAllowed(decl.Type.Name, expected) { + continue + } + c.add(CompletionItem{ + Label: "$" + decl.Name.Name, + Kind: CompletionVariable, + Detail: decl.Type.Name, + Priority: priorityVariable, + }) + } + } + + if typeAllowed(TypeAccount, expected) { + for _, account := range c.knownAccounts() { + c.add(CompletionItem{ + Label: account, + Kind: CompletionAccount, + Detail: TypeAccount, + Priority: priorityLiteral, + }) + } + } + + if typeAllowed(TypeAsset, expected) && len(expected) != 0 { + for _, asset := range c.knownAssets() { + c.add(CompletionItem{ + Label: asset, + Kind: CompletionAsset, + Detail: TypeAsset, + Priority: priorityLiteral, + }) + } + } + + for _, name := range sortedBuiltins() { + resolution, ok := Builtins[name].(VarOriginFnCallResolution) + if !ok || (resolution.Return != TypeAny && !typeAllowed(resolution.Return, expected)) { + continue + } + c.addFunction(name, resolution) + } +} + +func (c *completer) addFunction(name string, resolution FnCallResolution) { + var placeholders []string + for index, param := range resolution.GetParams() { + placeholders = append(placeholders, fmt.Sprintf("${%d:%s}", index+1, param)) + } + + c.add(CompletionItem{ + Label: name, + Kind: CompletionFunction, + Detail: FnSignature(name, resolution), + Docs: fnDocs(resolution), + InsertText: name + "(", + SnippetText: name + "(" + strings.Join(placeholders, ", ") + ")", + Priority: priorityFunction, + }) +} + +func (c *completer) completeAccountPart(slot *parser.AccountPartSlot) { + // The completion replaces the whole account literal typed so far + end := c.ctx.Range.End + if c.position.GtEq(end) { + end = c.position + } + rng := parser.Range{Start: slot.LiteralStart, End: end} + + for _, account := range c.knownAccounts() { + c.add(CompletionItem{ + Label: account, + Kind: CompletionAccount, + Detail: TypeAccount, + Range: rng, + Priority: priorityLiteral, + }) + } + + // After a ':', suggest interpolating a variable (e.g. `@users:$id`) + typed := textInRange(c.source, parser.Range{Start: slot.LiteralStart, End: c.position}) + lastColon := strings.LastIndex(typed, ":") + if lastColon == -1 || c.program.Vars == nil { + return + } + for _, decl := range c.program.Vars.Declarations { + if decl.Name == nil || decl.Type == nil { + continue + } + c.add(CompletionItem{ + Label: typed[:lastColon+1] + "$" + decl.Name.Name, + Kind: CompletionVariable, + Detail: decl.Type.Name, + Range: rng, + Priority: priorityVariable, + }) + } +} + +func (c *completer) completeVarType() { + for _, t := range AllowedTypes { + c.add(CompletionItem{ + Label: t, + Kind: CompletionType, + Priority: priorityVariable, + }) + } +} + +func (c *completer) completeStatement() { + for _, name := range sortedBuiltins() { + if resolution, ok := Builtins[name].(StatementFnCallResolution); ok { + c.addFunction(name, resolution) + } + } + + if slices.Contains(c.ctx.Keywords, "send") { + c.add(CompletionItem{ + Label: "send", + Kind: CompletionSnippet, + Detail: "send statement", + InsertText: "send ", + SnippetText: "send ${1:[ASSET amount]} (\n\tsource = ${2:@source}\n\tdestination = ${3:@destination}\n)", + Priority: prioritySnippet, + }) + } + + if slices.Contains(c.ctx.Keywords, "vars") { + c.add(CompletionItem{ + Label: "vars", + Kind: CompletionSnippet, + Detail: "variables declaration", + InsertText: "vars ", + SnippetText: "vars {\n\t${1:type} \\$${2:name}\n}", + Priority: prioritySnippet, + }) + } +} + +// The account literals (without interpolated variables) of the program, and @world +func (c *completer) knownAccounts() []string { + accounts := []string{"@world"} + walkExprs(c.program, func(expr parser.ValueExpr) { + account, ok := expr.(*parser.AccountInterpLiteral) + if !ok || account.Contains(c.position) { + return + } + for _, part := range account.Parts { + if _, isVar := part.(*parser.Variable); isVar { + return + } + } + name := "@" + account.String() + if !slices.Contains(accounts, name) { + accounts = append(accounts, name) + } + }) + return accounts +} + +func (c *completer) knownAssets() []string { + var assets []string + walkExprs(c.program, func(expr parser.ValueExpr) { + asset, ok := expr.(*parser.AssetLiteral) + if ok && !asset.Contains(c.position) && !slices.Contains(assets, asset.Asset) { + assets = append(assets, asset.Asset) + } + }) + return assets +} + +type SignatureHelp struct { + Label string + Params []string + ActiveParam int + Docs string +} + +// Returns the signature of the builtin function whose arguments enclose the given position +func SignatureHelpAt(source string, position parser.Position) *SignatureHelp { + _, ctx := parser.ParseForCompletion(source, position) + if ctx == nil || len(ctx.FnCalls) == 0 { + return nil + } + + call := ctx.FnCalls[len(ctx.FnCalls)-1] + resolution, ok := Builtins[call.Name] + if !ok { + return nil + } + + return &SignatureHelp{ + Label: FnSignature(call.Name, resolution), + Params: resolution.GetParams(), + ActiveParam: call.ArgIndex, + Docs: fnDocs(resolution), + } +} + +func FnSignature(name string, resolution FnCallResolution) string { + signature := name + "(" + strings.Join(resolution.GetParams(), ", ") + ")" + if resolution, ok := resolution.(VarOriginFnCallResolution); ok { + signature += " -> " + resolution.Return + } + return signature +} + +func fnDocs(resolution FnCallResolution) string { + switch resolution := resolution.(type) { + case VarOriginFnCallResolution: + return resolution.Docs + case StatementFnCallResolution: + return resolution.Docs + default: + return "" + } +} + +func sortedBuiltins() []string { + var names []string + for name := range Builtins { + names = append(names, name) + } + slices.Sort(names) + return names +} + +// Pre: the range is valid for the source +func textInRange(source string, rng parser.Range) string { + lines := strings.Split(source, "\n") + if rng.Start.Line != rng.End.Line || rng.Start.Line >= len(lines) { + return "" + } + line := []rune(lines[rng.Start.Line]) + start := min(rng.Start.Character, len(line)) + end := min(max(rng.End.Character, start), len(line)) + return string(line[start:end]) +} + +// Calls fn on every expression of the program (including nested ones) +func walkExprs(program parser.Program, fn func(parser.ValueExpr)) { + var walkExpr func(expr parser.ValueExpr) + walkExpr = func(expr parser.ValueExpr) { + if expr == nil { + return + } + fn(expr) + switch expr := expr.(type) { + case *parser.MonetaryLiteral: + walkExpr(expr.Asset) + walkExpr(expr.Amount) + case *parser.BinaryInfix: + walkExpr(expr.Left) + walkExpr(expr.Right) + case *parser.Prefix: + walkExpr(expr.Expr) + case *parser.FnCall: + for _, arg := range expr.Args { + walkExpr(arg) + } + } + } + + walkAllotment := func(allotment parser.AllotmentValue) { + if allotment, ok := allotment.(*parser.ValueExprAllotment); ok { + walkExpr(allotment.Value) + } + } + + var walkSource func(source parser.Source) + walkSource = func(source parser.Source) { + switch source := source.(type) { + case *parser.SourceAccount: + walkExpr(source.ValueExpr) + walkExpr(source.Color) + case *parser.SourceOverdraft: + walkExpr(source.Address) + walkExpr(source.Color) + if source.Bounded != nil { + walkExpr(*source.Bounded) + } + case *parser.SourceWithScaling: + walkExpr(source.Address) + walkExpr(source.Through) + case *parser.SourceCapped: + walkExpr(source.Cap) + walkSource(source.From) + case *parser.SourceInorder: + for _, s := range source.Sources { + walkSource(s) + } + case *parser.SourceOneof: + for _, s := range source.Sources { + walkSource(s) + } + case *parser.SourceAllotment: + for _, item := range source.Items { + walkAllotment(item.Allotment) + walkSource(item.From) + } + } + } + + var walkDestination func(destination parser.Destination) + walkKeptOrDestination := func(target parser.KeptOrDestination) { + if to, ok := target.(*parser.DestinationTo); ok { + walkDestination(to.Destination) + } + } + walkDestination = func(destination parser.Destination) { + switch destination := destination.(type) { + case *parser.DestinationAccount: + walkExpr(destination.ValueExpr) + case *parser.DestinationInorder: + for _, clause := range destination.Clauses { + walkExpr(clause.Cap) + walkKeptOrDestination(clause.To) + } + walkKeptOrDestination(destination.Remaining) + case *parser.DestinationOneof: + for _, clause := range destination.Clauses { + walkExpr(clause.Cap) + walkKeptOrDestination(clause.To) + } + walkKeptOrDestination(destination.Remaining) + case *parser.DestinationAllotment: + for _, item := range destination.Items { + walkAllotment(item.Allotment) + walkKeptOrDestination(item.To) + } + } + } + + walkSentValue := func(sentValue parser.SentValue) { + switch sentValue := sentValue.(type) { + case *parser.SentValueLiteral: + walkExpr(sentValue.Monetary) + case *parser.SentValueAll: + walkExpr(sentValue.Asset) + } + } + + if program.Vars != nil { + for _, decl := range program.Vars.Declarations { + if decl.Origin != nil { + walkExpr(*decl.Origin) + } + } + } + + for _, statement := range program.Statements { + switch statement := statement.(type) { + case *parser.SendStatement: + walkSentValue(statement.SentValue) + walkSource(statement.Source) + walkDestination(statement.Destination) + case *parser.SaveStatement: + walkSentValue(statement.SentValue) + walkExpr(statement.Account) + case *parser.FnCall: + walkExpr(statement) + } + } +} diff --git a/internal/analysis/completion_test.go b/internal/analysis/completion_test.go new file mode 100644 index 00000000..f86c5531 --- /dev/null +++ b/internal/analysis/completion_test.go @@ -0,0 +1,186 @@ +package analysis_test + +import ( + "strings" + "testing" + + "github.com/formancehq/numscript/internal/analysis" + "github.com/formancehq/numscript/internal/parser" + "github.com/stretchr/testify/require" +) + +// The cursor position is marked with `|` +func completeAtCursor(t *testing.T, src string) []analysis.CompletionItem { + t.Helper() + pos := parser.PositionOf(src, "|") + require.NotNil(t, pos, "missing cursor marker") + return analysis.Complete(strings.Replace(src, "|", "", 1), *pos) +} + +func labels(items []analysis.CompletionItem) []string { + var out []string + for _, item := range items { + out = append(out, item.Label) + } + return out +} + +const varsBlock = `vars { + monetary $mon + account $acc + portion $por + asset $ast +} +` + +func TestCompleteEmptyProgram(t *testing.T) { + got := labels(completeAtCursor(t, "|")) + require.Subset(t, got, []string{"vars", "send", "save", "set_tx_meta", "set_account_meta"}) +} + +func TestCompleteNoVarsAfterVarsBlock(t *testing.T) { + got := labels(completeAtCursor(t, varsBlock+"|")) + require.Subset(t, got, []string{"send", "save", "set_tx_meta"}) + require.NotContains(t, got, "vars") +} + +func TestCompletePartialKeyword(t *testing.T) { + items := completeAtCursor(t, "send [COIN 10] (\n sou|") + require.Equal(t, []string{"source"}, labels(items)) + // the partial word is replaced + require.Equal(t, parser.Range{ + Start: parser.Position{Line: 1, Character: 2}, + End: parser.Position{Line: 1, Character: 5}, + }, items[0].Range) +} + +func TestCompleteKeywordsAfterSourceAccount(t *testing.T) { + got := labels(completeAtCursor(t, "send [COIN 10] (\n source = @a |\n destination = @b\n)")) + require.ElementsMatch(t, []string{"allowing", "with", "destination"}, got) +} + +func TestCompleteOverdraftKeywords(t *testing.T) { + got := labels(completeAtCursor(t, "send [COIN 10] ( source = @a allowing | destination = @b )")) + require.ElementsMatch(t, []string{"unbounded", "overdraft"}, got) +} + +func TestCompleteVariablesByExpectedType(t *testing.T) { + cases := map[string]string{ + "send $|": "$mon", + "send $mon ( source = $|": "$acc", + "send $mon ( source = @a destination = $|": "$acc", + "send [$|": "$ast", + "send $mon ( source = @a destination = { $| to @b } )": "$por", + "save $mon from $|": "$acc", + `set_tx_meta("k", balance($|`: "$acc", + `set_tx_meta("k", balance(@a, $|`: "$ast", + } + for src, expected := range cases { + t.Run(src, func(t *testing.T) { + got := labels(completeAtCursor(t, varsBlock+src)) + require.Equal(t, []string{expected}, got) + }) + } +} + +func TestCompleteAnyVariableInUntypedPosition(t *testing.T) { + got := labels(completeAtCursor(t, varsBlock+`set_tx_meta("k", $|`)) + require.ElementsMatch(t, []string{"$mon", "$acc", "$por", "$ast"}, got) +} + +func TestCompleteFunctionsByReturnType(t *testing.T) { + got := labels(completeAtCursor(t, "vars { monetary $m = | }")) + require.Subset(t, got, []string{"balance", "overdraft", "meta"}) + require.NotContains(t, got, "get_amount") + require.NotContains(t, got, "set_tx_meta") +} + +func TestCompleteFunctionSnippet(t *testing.T) { + items := completeAtCursor(t, "vars { monetary $m = bal| }") + var balance *analysis.CompletionItem + for _, item := range items { + if item.Label == "balance" { + balance = &item + } + } + require.NotNil(t, balance) + require.Equal(t, "balance(${1:account}, ${2:asset})", balance.SnippetText) + require.Equal(t, "balance(account, asset) -> monetary", balance.Detail) +} + +func TestCompleteVarTypes(t *testing.T) { + got := labels(completeAtCursor(t, "vars {\n mon|")) + require.ElementsMatch(t, analysis.AllowedTypes, got) +} + +func TestCompleteAccounts(t *testing.T) { + items := completeAtCursor(t, "send [C 1] ( source = @users:alice destination = @us| )") + require.ElementsMatch(t, []string{"@world", "@users:alice"}, labels(items)) + // the whole account literal is replaced + require.Equal(t, parser.Range{ + Start: parser.Position{Character: 49}, + End: parser.Position{Character: 52}, + }, items[0].Range) +} + +func TestCompleteAccountAfterAt(t *testing.T) { + got := labels(completeAtCursor(t, "send [C 1] ( source = @a destination = @| )")) + require.ElementsMatch(t, []string{"@world", "@a"}, got) +} + +func TestCompleteAccountInterpolation(t *testing.T) { + got := labels(completeAtCursor(t, varsBlock+"send [C 1] ( source = @users:| )")) + require.Contains(t, got, "@users:$acc") +} + +func TestCompleteAssets(t *testing.T) { + got := labels(completeAtCursor(t, "send [USD/2 1] ( source = @a destination = @b )\nsend [|")) + require.Contains(t, got, "USD/2") +} + +func TestCompleteSourcesInBraces(t *testing.T) { + got := labels(completeAtCursor(t, "send [C 1] ( source = { @a\n | } destination = @b )")) + require.Subset(t, got, []string{"max", "oneof", "@world", "@a"}) +} + +func TestNoCompletionInStringsAndComments(t *testing.T) { + require.Empty(t, completeAtCursor(t, `set_tx_meta("a|")`)) + require.Empty(t, completeAtCursor(t, "// a comment|\n")) +} + +func signatureAtCursor(t *testing.T, src string) *analysis.SignatureHelp { + t.Helper() + pos := parser.PositionOf(src, "|") + require.NotNil(t, pos, "missing cursor marker") + return analysis.SignatureHelpAt(strings.Replace(src, "|", "", 1), *pos) +} + +func TestSignatureHelp(t *testing.T) { + help := signatureAtCursor(t, `set_tx_meta("k", balance(@a, |))`) + require.Equal(t, &analysis.SignatureHelp{ + Label: "balance(account, asset) -> monetary", + Params: []string{"account", "asset"}, + ActiveParam: 1, + Docs: "fetch account balance", + }, help) +} + +func TestSignatureHelpInnermostCall(t *testing.T) { + help := signatureAtCursor(t, `set_tx_meta("k", balance(@a, meta(@b, |)))`) + require.Equal(t, "meta(account, string) -> any", help.Label) + require.Equal(t, 1, help.ActiveParam) +} + +func TestSignatureHelpIncompleteCall(t *testing.T) { + help := signatureAtCursor(t, "vars { monetary $m = balance(@a, | }") + require.Equal(t, 1, help.ActiveParam) + + help = signatureAtCursor(t, "set_tx_meta(|") + require.Equal(t, "set_tx_meta(string, any)", help.Label) + require.Equal(t, 0, help.ActiveParam) +} + +func TestNoSignatureHelpOutsideCalls(t *testing.T) { + require.Nil(t, signatureAtCursor(t, `set_tx_meta("k", 1)|`)) + require.Nil(t, signatureAtCursor(t, `unknown_fn(|`)) +} diff --git a/internal/analysis/robustness_test.go b/internal/analysis/robustness_test.go new file mode 100644 index 00000000..a9a91adb --- /dev/null +++ b/internal/analysis/robustness_test.go @@ -0,0 +1,80 @@ +package analysis_test + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/formancehq/numscript/internal/analysis" + "github.com/formancehq/numscript/internal/parser" +) + +// Simulates a user typing each program of the corpus, one character at a time: +// the analysis of every intermediate (and mostly invalid) program must not panic. +func TestAnalysisOfIncompleteProgramsDoesNotPanic(t *testing.T) { + if testing.Short() { + t.Skip() + } + + for path, src := range numscriptCorpus(t) { + for end := range len(src) + 1 { + partial := src[:end] + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("panic analysing a prefix of %s: %v\n---\n%s", path, r, partial) + } + }() + analyseEverywhere(partial) + }() + } + } +} + +func FuzzAnalysis(f *testing.F) { + for _, src := range numscriptCorpus(f) { + f.Add(src) + } + f.Fuzz(func(t *testing.T, src string) { + analyseEverywhere(src) + }) +} + +// Runs the whole analysis, and the position-based queries on each line of the program +func analyseEverywhere(src string) { + res := analysis.CheckSource(src) + res.GetSymbols() + + lines := strings.Split(src, "\n") + for line, text := range lines { + for _, char := range []int{0, len(text) / 2, len(text)} { + pos := parser.Position{Line: line, Character: char} + analysis.HoverOn(res.Program, pos) + analysis.GotoDefinition(res.Program, pos, res) + analysis.Complete(src, pos) + analysis.SignatureHelpAt(src, pos) + } + } +} + +func numscriptCorpus(t testing.TB) map[string]string { + t.Helper() + corpus := map[string]string{} + err := filepath.WalkDir("../interpreter/testdata", func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".num") { + return err + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + corpus[path] = string(content) + return nil + }) + if err != nil { + t.Fatal(err) + } + return corpus +} diff --git a/internal/lsp/__snapshots__/handlers_test.snap b/internal/lsp/__snapshots__/handlers_test.snap index 6904d465..0b7a10e3 100755 --- a/internal/lsp/__snapshots__/handlers_test.snap +++ b/internal/lsp/__snapshots__/handlers_test.snap @@ -149,3 +149,117 @@ "uri": "example.num" } --- + + +[TestCompletionKeyword - 1] +{ + "isIncomplete": false, + "items": [ + { + "insertTextFormat": 1, + "kind": 14, + "label": "allowing", + "sortText": "0000", + "textEdit": { + "newText": "allowing", + "range": { + "end": { + "character": 19, + "line": 1 + }, + "start": { + "character": 15, + "line": 1 + } + } + } + }, + { + "insertTextFormat": 1, + "kind": 14, + "label": "with", + "sortText": "0001", + "textEdit": { + "newText": "with", + "range": { + "end": { + "character": 19, + "line": 1 + }, + "start": { + "character": 15, + "line": 1 + } + } + } + }, + { + "insertTextFormat": 1, + "kind": 14, + "label": "destination", + "sortText": "0002", + "textEdit": { + "newText": "destination", + "range": { + "end": { + "character": 19, + "line": 1 + }, + "start": { + "character": 15, + "line": 1 + } + } + } + } + ] +} +--- + +[TestSignatureHelp - 1] +{ + "activeParameter": 1, + "signatures": [ + { + "documentation": "fetch account balance", + "label": "balance(account, asset) -\u003e monetary", + "parameters": [ + { + "label": "account" + }, + { + "label": "asset" + } + ] + } + ] +} +--- + +[TestCompletion - 1] +{ + "isIncomplete": false, + "items": [ + { + "detail": "account", + "insertTextFormat": 1, + "kind": 6, + "label": "$acc", + "sortText": "0000", + "textEdit": { + "newText": "$acc", + "range": { + "end": { + "character": 11, + "line": 6 + }, + "start": { + "character": 10, + "line": 6 + } + } + } + } + ] +} +--- diff --git a/internal/lsp/handlers.go b/internal/lsp/handlers.go index 1f4ff58a..5ca5147a 100644 --- a/internal/lsp/handlers.go +++ b/internal/lsp/handlers.go @@ -5,6 +5,7 @@ import ( "log" "os" "slices" + "sync/atomic" "github.com/formancehq/numscript/internal/analysis" "github.com/formancehq/numscript/internal/jsonrpc2" @@ -20,6 +21,9 @@ type InMemoryDocument struct { type State struct { documents documentStore[InMemoryDocument] + + // whether the client supports snippets in completion items + snippetSupport atomic.Bool } func (state *State) updateDocument(conn *jsonrpc2.Conn, uri protocol.DocumentURI, text string) { @@ -136,6 +140,96 @@ func (state *State) handleGotoDefinition(params protocol.DefinitionParams) *prot } } +func (state *State) handleCompletion(params protocol.CompletionParams) *protocol.CompletionList { + doc, ok := state.documents.Get(params.TextDocument.URI) + if !ok { + return nil + } + + snippetSupport := state.snippetSupport.Load() + items := analysis.Complete(doc.Text, fromLspPosition(params.Position)) + + lspItems := make([]protocol.CompletionItem, 0, len(items)) + for index, item := range items { + newText := item.InsertText + format := protocol.InsertTextFormatPlainText + if item.SnippetText != "" { + if snippetSupport { + newText = item.SnippetText + format = protocol.InsertTextFormatSnippet + } else if item.Kind == analysis.CompletionSnippet { + continue + } + } + + var docs any + if item.Docs != "" { + docs = item.Docs + } + + lspItems = append(lspItems, protocol.CompletionItem{ + Label: item.Label, + Kind: toLspCompletionKind(item.Kind), + Detail: item.Detail, + Documentation: docs, + InsertTextFormat: format, + // keep the order computed by the analysis + SortText: fmt.Sprintf("%04d", index), + TextEdit: &protocol.TextEdit{ + Range: *toLspRange(item.Range), + NewText: newText, + }, + }) + } + + return &protocol.CompletionList{Items: lspItems} +} + +func toLspCompletionKind(kind analysis.CompletionItemKind) protocol.CompletionItemKind { + switch kind { + case analysis.CompletionKeyword: + return protocol.CompletionItemKindKeyword + case analysis.CompletionVariable: + return protocol.CompletionItemKindVariable + case analysis.CompletionFunction: + return protocol.CompletionItemKindFunction + case analysis.CompletionType: + return protocol.CompletionItemKindTypeParameter + case analysis.CompletionAccount, analysis.CompletionAsset: + return protocol.CompletionItemKindValue + case analysis.CompletionSnippet: + return protocol.CompletionItemKindSnippet + default: + return protocol.CompletionItemKindText + } +} + +func (state *State) handleSignatureHelp(params protocol.SignatureHelpParams) *protocol.SignatureHelp { + doc, ok := state.documents.Get(params.TextDocument.URI) + if !ok { + return nil + } + + help := analysis.SignatureHelpAt(doc.Text, fromLspPosition(params.Position)) + if help == nil { + return nil + } + + params_ := make([]protocol.ParameterInformation, len(help.Params)) + for index, param := range help.Params { + params_[index] = protocol.ParameterInformation{Label: param} + } + + return &protocol.SignatureHelp{ + Signatures: []protocol.SignatureInformation{{ + Label: help.Label, + Documentation: help.Docs, + Parameters: params_, + }}, + ActiveParameter: uint32(help.ActiveParam), + } +} + func (state *State) handleGetSymbols(params protocol.DocumentSymbolParams) []protocol.DocumentSymbol { doc, ok := state.documents.Get(params.TextDocument.URI) if !ok { @@ -235,6 +329,13 @@ var initializeResult protocol.InitializeResult = protocol.InitializeResult{ DefinitionProvider: true, DocumentSymbolProvider: true, CodeActionProvider: true, + CompletionProvider: &protocol.CompletionOptions{ + TriggerCharacters: []string{"$", "@", ":"}, + }, + SignatureHelpProvider: &protocol.SignatureHelpOptions{ + TriggerCharacters: []string{"(", ","}, + RetriggerCharacters: []string{","}, + }, }, ServerInfo: &protocol.ServerInfo{ Name: "numscript-ls", @@ -253,7 +354,10 @@ func NewConn(objStream jsonrpc2.MessageStream) *jsonrpc2.Conn { } return jsonrpc2.NewConn(objStream, - jsonrpc2.NewRequestHandler("initialize", jsonrpc2.SyncHandling, func(_ any, conn *jsonrpc2.Conn) any { + jsonrpc2.NewRequestHandler("initialize", jsonrpc2.SyncHandling, func(p protocol.InitializeParams, conn *jsonrpc2.Conn) any { + if completion := p.Capabilities.TextDocument; completion != nil && completion.Completion != nil && completion.Completion.CompletionItem != nil { + state.snippetSupport.Store(completion.Completion.CompletionItem.SnippetSupport) + } return initializeResult }), jsonrpc2.NewNotificationHandler("textDocument/didOpen", jsonrpc2.SyncHandling, func(p protocol.DidOpenTextDocumentParams, conn *jsonrpc2.Conn) { @@ -270,6 +374,12 @@ func NewConn(objStream jsonrpc2.MessageStream) *jsonrpc2.Conn { jsonrpc2.NewRequestHandler("textDocument/hover", jsonrpc2.AsyncHandling, func(p protocol.HoverParams, conn *jsonrpc2.Conn) any { return state.handleHover(p) }), + jsonrpc2.NewRequestHandler("textDocument/completion", jsonrpc2.AsyncHandling, func(p protocol.CompletionParams, _ *jsonrpc2.Conn) any { + return state.handleCompletion(p) + }), + jsonrpc2.NewRequestHandler("textDocument/signatureHelp", jsonrpc2.AsyncHandling, func(p protocol.SignatureHelpParams, _ *jsonrpc2.Conn) any { + return state.handleSignatureHelp(p) + }), jsonrpc2.NewRequestHandler("textDocument/codeAction", jsonrpc2.AsyncHandling, func(p protocol.CodeActionParams, _ *jsonrpc2.Conn) any { return state.handleCodeAction(p) }), diff --git a/internal/lsp/handlers_test.go b/internal/lsp/handlers_test.go index 91fb3c57..5547511c 100644 --- a/internal/lsp/handlers_test.go +++ b/internal/lsp/handlers_test.go @@ -104,6 +104,57 @@ func TestGotoDef(t *testing.T) { snaps.MatchJSON(t, raw) } +func TestCompletion(t *testing.T) { + client := newTestClient() + + doc, input, _ := client.OpenFile("example.num", `vars { + monetary $mon + account $acc +} + +send $mon ( + source = $ + destination = @dest +) +`) + + // right after the `$` of the source + pos := *parser.PositionOfIndexed(input, "$", 3) + pos.Character++ + raw, err := client.Completion(doc, pos) + require.Nil(t, err) + snaps.MatchJSON(t, raw) +} + +func TestCompletionKeyword(t *testing.T) { + client := newTestClient() + + doc, input, _ := client.OpenFile("example.num", `send [COIN 100] ( + source = @src allo + destination = @dest +) +`) + + pos := *parser.PositionOf(input, "allo") + pos.Character += len("allo") + raw, err := client.Completion(doc, pos) + require.Nil(t, err) + snaps.MatchJSON(t, raw) +} + +func TestSignatureHelp(t *testing.T) { + client := newTestClient() + + doc, input, _ := client.OpenFile("example.num", `vars { + monetary $mon = balance(@acc, ) +} +`) + + raw, err := client.SignatureHelp(doc, *parser.PositionOf(input, ")")) + require.Nil(t, err) + snaps.MatchJSON(t, raw) +} + // Testing utilities type TestClient struct { conn *jsonrpc2.Conn @@ -138,6 +189,24 @@ func (c *TestClient) Hover(doc protocol.TextDocumentIdentifier, position parser. }) } +func (c *TestClient) Completion(doc protocol.TextDocumentIdentifier, position parser.Position) (json.RawMessage, *jsonrpc2.ResponseError) { + return c.conn.SendRequest("textDocument/completion", protocol.CompletionParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: doc, + Position: lsp.ParserToLspPosition(position), + }, + }) +} + +func (c *TestClient) SignatureHelp(doc protocol.TextDocumentIdentifier, position parser.Position) (json.RawMessage, *jsonrpc2.ResponseError) { + return c.conn.SendRequest("textDocument/signatureHelp", protocol.SignatureHelpParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: doc, + Position: lsp.ParserToLspPosition(position), + }, + }) +} + func (c *TestClient) GetSymbols(doc protocol.TextDocumentIdentifier) (json.RawMessage, *jsonrpc2.ResponseError) { return c.conn.SendRequest("textDocument/documentSymbol", protocol.DocumentSymbolParams{ TextDocument: doc,