diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 50e4da95..b42fa389 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,4 +3,6 @@ You can update snaphshots with the `UPDATE_SNAPS=true` variable while running the tests -If you need to update the grammar, you can generate the parser by running `just generate` (you'll need to install the antlr4 command first) +The Numscript parser (`internal/parser`) is hand-written, so that it can recover from syntax errors: the LSP needs a meaningful AST for programs that are being typed. The [Numscript.g4](Numscript.g4) and [Lexer.g4](Lexer.g4) grammars are still the reference for the syntax, and they generate the previous, ANTLR-based parser (`internal/parser/internal/legacy`). The tests check that both parsers agree on every program of the repository, so if you change the syntax, update both the hand-written parser and the grammar. + +You can generate the ANTLR parsers by running `just generate` (you'll need to install the antlr4 command first) diff --git a/Justfile b/Justfile index b586574f..2aee7996 100644 --- a/Justfile +++ b/Justfile @@ -13,8 +13,8 @@ tidy: @go mod tidy generate: - @antlr4 -Dlanguage=Go Lexer.g4 Numscript.g4 -o internal/parser/antlrParser -package antlrParser - @mv internal/parser/antlrParser/_lexer.go internal/parser/antlrParser/lexer.go + @antlr4 -Dlanguage=Go Lexer.g4 Numscript.g4 -o internal/parser/internal/legacy/antlrParser -package antlrParser + @mv internal/parser/internal/legacy/antlrParser/_lexer.go internal/parser/internal/legacy/antlrParser/lexer.go @antlr4 -Dlanguage=Go IR.g4 -o internal/ir/internal/syntax/antlrParser -package antlrParser tests: diff --git a/codecov.yml b/codecov.yml index 78226af2..548f700b 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,2 +1,3 @@ ignore: - - "internal/parser/antlrParser" + - "internal/parser/internal/legacy" + - "internal/ir/internal/syntax/antlrParser" diff --git a/go.mod b/go.mod index 1309376c..ffa61b68 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/kr/pretty v0.3.1 github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.9.2 // indirect github.com/maruel/natural v1.3.0 // indirect diff --git a/internal/analysis/check.go b/internal/analysis/check.go index 428111e7..d07e4864 100644 --- a/internal/analysis/check.go +++ b/internal/analysis/check.go @@ -178,6 +178,10 @@ func (r *CheckResult) GetVarDeclType(decl parser.VarDeclaration) Type { } func (r *CheckResult) unifyNodeWith(expr parser.ValueExpr, t Type) { + if expr == nil { + // a missing node of an incomplete program + return + } exprT := r.GetExprType(expr) r.unify(expr.GetRange(), exprT, t) } diff --git a/internal/analysis/completion.go b/internal/analysis/completion.go new file mode 100644 index 00000000..24805164 --- /dev/null +++ b/internal/analysis/completion.go @@ -0,0 +1,520 @@ +package analysis + +import ( + "fmt" + "slices" + "strings" + + "github.com/formancehq/numscript/internal/parser" +) + +type CompletionItemKind uint8 + +const ( + CompletionKeyword CompletionItemKind = iota + CompletionVariable + CompletionFunction + CompletionType + CompletionAccount + CompletionAsset + CompletionSnippet +) + +type CompletionItem struct { + Label string + Kind CompletionItemKind + Detail string + Docs string + + // The range of text that the completion replaces + Range parser.Range + + // The text inserted when snippets aren't supported + InsertText string + // The text inserted when snippets are supported. Empty when the item isn't a snippet. + SnippetText string + + // Items with a lower priority are displayed first + Priority int +} + +const ( + priorityVariable = iota + priorityLiteral + priorityFunction + priorityKeyword + prioritySnippet +) + +// Returns the completions that are valid at the given position. +// The program does not need to be valid: in fact it's usually being typed. +func Complete(source string, position parser.Position) []CompletionItem { + parsed, ctx := parser.ParseForCompletion(source, position) + if ctx == nil { + return nil + } + + c := completer{ + source: source, + position: position, + ctx: ctx, + program: parsed.Value, + seen: map[string]struct{}{}, + } + return c.complete() +} + +type completer struct { + source string + position parser.Position + ctx *parser.CompletionContext + program parser.Program + items []CompletionItem + seen map[string]struct{} +} + +func (c *completer) add(item CompletionItem) { + key := fmt.Sprintf("%d:%s", item.Kind, item.Label) + if _, ok := c.seen[key]; ok { + return + } + c.seen[key] = struct{}{} + + if item.Range == (parser.Range{}) { + item.Range = c.ctx.Range + } + if item.InsertText == "" { + item.InsertText = item.Label + } + c.items = append(c.items, item) +} + +func (c *completer) complete() []CompletionItem { + for _, slot := range c.ctx.Slots { + switch slot := slot.(type) { + case *parser.ExprSlot: + c.completeExpr(c.expectedTypes(slot)) + case *parser.AccountPartSlot: + c.completeAccountPart(slot) + case *parser.VarTypeSlot: + c.completeVarType() + case *parser.StatementSlot: + c.completeStatement() + } + } + + for _, keyword := range c.ctx.Keywords { + c.add(CompletionItem{ + Label: keyword, + Kind: CompletionKeyword, + Priority: priorityKeyword, + }) + } + + // When a variable or an account is being typed, other kinds of completions are just noise + if strings.HasPrefix(c.ctx.Prefix, "$") || strings.HasPrefix(c.ctx.Prefix, "@") { + prefix := c.ctx.Prefix[:1] + c.items = slices.DeleteFunc(c.items, func(item CompletionItem) bool { + return !strings.HasPrefix(item.Label, prefix) + }) + } + + slices.SortStableFunc(c.items, func(a, b CompletionItem) int { + return a.Priority - b.Priority + }) + return c.items +} + +// The types the expression at the cursor is expected to have. An empty slice means any type. +func (c *completer) expectedTypes(slot *parser.ExprSlot) []string { + if len(slot.Types) != 0 { + return slot.Types + } + + if len(c.ctx.FnCalls) == 0 { + return nil + } + + call := c.ctx.FnCalls[len(c.ctx.FnCalls)-1] + resolution, ok := Builtins[call.Name] + if !ok { + return nil + } + + params := resolution.GetParams() + if call.ArgIndex >= len(params) || params[call.ArgIndex] == TypeAny { + return nil + } + return []string{params[call.ArgIndex]} +} + +func typeAllowed(t string, expected []string) bool { + return len(expected) == 0 || slices.Contains(expected, t) +} + +func (c *completer) completeExpr(expected []string) { + if c.program.Vars != nil { + for _, decl := range c.program.Vars.Declarations { + if decl.Name == nil || decl.Type == nil || !typeAllowed(decl.Type.Name, expected) { + continue + } + c.add(CompletionItem{ + Label: "$" + decl.Name.Name, + Kind: CompletionVariable, + Detail: decl.Type.Name, + Priority: priorityVariable, + }) + } + } + + if typeAllowed(TypeAccount, expected) { + for _, account := range c.knownAccounts() { + c.add(CompletionItem{ + Label: account, + Kind: CompletionAccount, + Detail: TypeAccount, + Priority: priorityLiteral, + }) + } + } + + if typeAllowed(TypeAsset, expected) && len(expected) != 0 { + for _, asset := range c.knownAssets() { + c.add(CompletionItem{ + Label: asset, + Kind: CompletionAsset, + Detail: TypeAsset, + Priority: priorityLiteral, + }) + } + } + + for _, name := range sortedBuiltins() { + resolution, ok := Builtins[name].(VarOriginFnCallResolution) + if !ok || (resolution.Return != TypeAny && !typeAllowed(resolution.Return, expected)) { + continue + } + c.addFunction(name, resolution) + } +} + +func (c *completer) addFunction(name string, resolution FnCallResolution) { + var placeholders []string + for index, param := range resolution.GetParams() { + placeholders = append(placeholders, fmt.Sprintf("${%d:%s}", index+1, param)) + } + + c.add(CompletionItem{ + Label: name, + Kind: CompletionFunction, + Detail: FnSignature(name, resolution), + Docs: fnDocs(resolution), + InsertText: name + "(", + SnippetText: name + "(" + strings.Join(placeholders, ", ") + ")", + Priority: priorityFunction, + }) +} + +func (c *completer) completeAccountPart(slot *parser.AccountPartSlot) { + // The completion replaces the whole account literal typed so far + end := c.ctx.Range.End + if c.position.GtEq(end) { + end = c.position + } + rng := parser.Range{Start: slot.LiteralStart, End: end} + + for _, account := range c.knownAccounts() { + c.add(CompletionItem{ + Label: account, + Kind: CompletionAccount, + Detail: TypeAccount, + Range: rng, + Priority: priorityLiteral, + }) + } + + // After a ':', suggest interpolating a variable (e.g. `@users:$id`) + typed := textInRange(c.source, parser.Range{Start: slot.LiteralStart, End: c.position}) + lastColon := strings.LastIndex(typed, ":") + if lastColon == -1 || c.program.Vars == nil { + return + } + for _, decl := range c.program.Vars.Declarations { + if decl.Name == nil || decl.Type == nil { + continue + } + c.add(CompletionItem{ + Label: typed[:lastColon+1] + "$" + decl.Name.Name, + Kind: CompletionVariable, + Detail: decl.Type.Name, + Range: rng, + Priority: priorityVariable, + }) + } +} + +func (c *completer) completeVarType() { + for _, t := range AllowedTypes { + c.add(CompletionItem{ + Label: t, + Kind: CompletionType, + Priority: priorityVariable, + }) + } +} + +func (c *completer) completeStatement() { + for _, name := range sortedBuiltins() { + if resolution, ok := Builtins[name].(StatementFnCallResolution); ok { + c.addFunction(name, resolution) + } + } + + if slices.Contains(c.ctx.Keywords, "send") { + c.add(CompletionItem{ + Label: "send", + Kind: CompletionSnippet, + Detail: "send statement", + InsertText: "send ", + SnippetText: "send ${1:[ASSET amount]} (\n\tsource = ${2:@source}\n\tdestination = ${3:@destination}\n)", + Priority: prioritySnippet, + }) + } + + if slices.Contains(c.ctx.Keywords, "vars") { + c.add(CompletionItem{ + Label: "vars", + Kind: CompletionSnippet, + Detail: "variables declaration", + InsertText: "vars ", + SnippetText: "vars {\n\t${1:type} \\$${2:name}\n}", + Priority: prioritySnippet, + }) + } +} + +// The account literals (without interpolated variables) of the program, and @world +func (c *completer) knownAccounts() []string { + accounts := []string{"@world"} + walkExprs(c.program, func(expr parser.ValueExpr) { + account, ok := expr.(*parser.AccountInterpLiteral) + if !ok || account.Contains(c.position) { + return + } + for _, part := range account.Parts { + if _, isVar := part.(*parser.Variable); isVar { + return + } + } + name := "@" + account.String() + if !slices.Contains(accounts, name) { + accounts = append(accounts, name) + } + }) + return accounts +} + +func (c *completer) knownAssets() []string { + var assets []string + walkExprs(c.program, func(expr parser.ValueExpr) { + asset, ok := expr.(*parser.AssetLiteral) + if ok && !asset.Contains(c.position) && !slices.Contains(assets, asset.Asset) { + assets = append(assets, asset.Asset) + } + }) + return assets +} + +type SignatureHelp struct { + Label string + Params []string + ActiveParam int + Docs string +} + +// Returns the signature of the builtin function whose arguments enclose the given position +func SignatureHelpAt(source string, position parser.Position) *SignatureHelp { + _, ctx := parser.ParseForCompletion(source, position) + if ctx == nil || len(ctx.FnCalls) == 0 { + return nil + } + + call := ctx.FnCalls[len(ctx.FnCalls)-1] + resolution, ok := Builtins[call.Name] + if !ok { + return nil + } + + return &SignatureHelp{ + Label: FnSignature(call.Name, resolution), + Params: resolution.GetParams(), + ActiveParam: call.ArgIndex, + Docs: fnDocs(resolution), + } +} + +func FnSignature(name string, resolution FnCallResolution) string { + signature := name + "(" + strings.Join(resolution.GetParams(), ", ") + ")" + if resolution, ok := resolution.(VarOriginFnCallResolution); ok { + signature += " -> " + resolution.Return + } + return signature +} + +func fnDocs(resolution FnCallResolution) string { + switch resolution := resolution.(type) { + case VarOriginFnCallResolution: + return resolution.Docs + case StatementFnCallResolution: + return resolution.Docs + default: + return "" + } +} + +func sortedBuiltins() []string { + var names []string + for name := range Builtins { + names = append(names, name) + } + slices.Sort(names) + return names +} + +// Pre: the range is valid for the source +func textInRange(source string, rng parser.Range) string { + lines := strings.Split(source, "\n") + if rng.Start.Line != rng.End.Line || rng.Start.Line >= len(lines) { + return "" + } + line := []rune(lines[rng.Start.Line]) + start := min(rng.Start.Character, len(line)) + end := min(max(rng.End.Character, start), len(line)) + return string(line[start:end]) +} + +// Calls fn on every expression of the program (including nested ones) +func walkExprs(program parser.Program, fn func(parser.ValueExpr)) { + var walkExpr func(expr parser.ValueExpr) + walkExpr = func(expr parser.ValueExpr) { + if expr == nil { + return + } + fn(expr) + switch expr := expr.(type) { + case *parser.MonetaryLiteral: + walkExpr(expr.Asset) + walkExpr(expr.Amount) + case *parser.BinaryInfix: + walkExpr(expr.Left) + walkExpr(expr.Right) + case *parser.Prefix: + walkExpr(expr.Expr) + case *parser.FnCall: + for _, arg := range expr.Args { + walkExpr(arg) + } + } + } + + walkAllotment := func(allotment parser.AllotmentValue) { + if allotment, ok := allotment.(*parser.ValueExprAllotment); ok { + walkExpr(allotment.Value) + } + } + + var walkSource func(source parser.Source) + walkSource = func(source parser.Source) { + switch source := source.(type) { + case *parser.SourceAccount: + walkExpr(source.ValueExpr) + walkExpr(source.Color) + case *parser.SourceOverdraft: + walkExpr(source.Address) + walkExpr(source.Color) + if source.Bounded != nil { + walkExpr(*source.Bounded) + } + case *parser.SourceWithScaling: + walkExpr(source.Address) + walkExpr(source.Through) + case *parser.SourceCapped: + walkExpr(source.Cap) + walkSource(source.From) + case *parser.SourceInorder: + for _, s := range source.Sources { + walkSource(s) + } + case *parser.SourceOneof: + for _, s := range source.Sources { + walkSource(s) + } + case *parser.SourceAllotment: + for _, item := range source.Items { + walkAllotment(item.Allotment) + walkSource(item.From) + } + } + } + + var walkDestination func(destination parser.Destination) + walkKeptOrDestination := func(target parser.KeptOrDestination) { + if to, ok := target.(*parser.DestinationTo); ok { + walkDestination(to.Destination) + } + } + walkDestination = func(destination parser.Destination) { + switch destination := destination.(type) { + case *parser.DestinationAccount: + walkExpr(destination.ValueExpr) + case *parser.DestinationInorder: + for _, clause := range destination.Clauses { + walkExpr(clause.Cap) + walkKeptOrDestination(clause.To) + } + walkKeptOrDestination(destination.Remaining) + case *parser.DestinationOneof: + for _, clause := range destination.Clauses { + walkExpr(clause.Cap) + walkKeptOrDestination(clause.To) + } + walkKeptOrDestination(destination.Remaining) + case *parser.DestinationAllotment: + for _, item := range destination.Items { + walkAllotment(item.Allotment) + walkKeptOrDestination(item.To) + } + } + } + + walkSentValue := func(sentValue parser.SentValue) { + switch sentValue := sentValue.(type) { + case *parser.SentValueLiteral: + walkExpr(sentValue.Monetary) + case *parser.SentValueAll: + walkExpr(sentValue.Asset) + } + } + + if program.Vars != nil { + for _, decl := range program.Vars.Declarations { + if decl.Origin != nil { + walkExpr(*decl.Origin) + } + } + } + + for _, statement := range program.Statements { + switch statement := statement.(type) { + case *parser.SendStatement: + walkSentValue(statement.SentValue) + walkSource(statement.Source) + walkDestination(statement.Destination) + case *parser.SaveStatement: + walkSentValue(statement.SentValue) + walkExpr(statement.Account) + case *parser.FnCall: + walkExpr(statement) + } + } +} diff --git a/internal/analysis/completion_test.go b/internal/analysis/completion_test.go new file mode 100644 index 00000000..f86c5531 --- /dev/null +++ b/internal/analysis/completion_test.go @@ -0,0 +1,186 @@ +package analysis_test + +import ( + "strings" + "testing" + + "github.com/formancehq/numscript/internal/analysis" + "github.com/formancehq/numscript/internal/parser" + "github.com/stretchr/testify/require" +) + +// The cursor position is marked with `|` +func completeAtCursor(t *testing.T, src string) []analysis.CompletionItem { + t.Helper() + pos := parser.PositionOf(src, "|") + require.NotNil(t, pos, "missing cursor marker") + return analysis.Complete(strings.Replace(src, "|", "", 1), *pos) +} + +func labels(items []analysis.CompletionItem) []string { + var out []string + for _, item := range items { + out = append(out, item.Label) + } + return out +} + +const varsBlock = `vars { + monetary $mon + account $acc + portion $por + asset $ast +} +` + +func TestCompleteEmptyProgram(t *testing.T) { + got := labels(completeAtCursor(t, "|")) + require.Subset(t, got, []string{"vars", "send", "save", "set_tx_meta", "set_account_meta"}) +} + +func TestCompleteNoVarsAfterVarsBlock(t *testing.T) { + got := labels(completeAtCursor(t, varsBlock+"|")) + require.Subset(t, got, []string{"send", "save", "set_tx_meta"}) + require.NotContains(t, got, "vars") +} + +func TestCompletePartialKeyword(t *testing.T) { + items := completeAtCursor(t, "send [COIN 10] (\n sou|") + require.Equal(t, []string{"source"}, labels(items)) + // the partial word is replaced + require.Equal(t, parser.Range{ + Start: parser.Position{Line: 1, Character: 2}, + End: parser.Position{Line: 1, Character: 5}, + }, items[0].Range) +} + +func TestCompleteKeywordsAfterSourceAccount(t *testing.T) { + got := labels(completeAtCursor(t, "send [COIN 10] (\n source = @a |\n destination = @b\n)")) + require.ElementsMatch(t, []string{"allowing", "with", "destination"}, got) +} + +func TestCompleteOverdraftKeywords(t *testing.T) { + got := labels(completeAtCursor(t, "send [COIN 10] ( source = @a allowing | destination = @b )")) + require.ElementsMatch(t, []string{"unbounded", "overdraft"}, got) +} + +func TestCompleteVariablesByExpectedType(t *testing.T) { + cases := map[string]string{ + "send $|": "$mon", + "send $mon ( source = $|": "$acc", + "send $mon ( source = @a destination = $|": "$acc", + "send [$|": "$ast", + "send $mon ( source = @a destination = { $| to @b } )": "$por", + "save $mon from $|": "$acc", + `set_tx_meta("k", balance($|`: "$acc", + `set_tx_meta("k", balance(@a, $|`: "$ast", + } + for src, expected := range cases { + t.Run(src, func(t *testing.T) { + got := labels(completeAtCursor(t, varsBlock+src)) + require.Equal(t, []string{expected}, got) + }) + } +} + +func TestCompleteAnyVariableInUntypedPosition(t *testing.T) { + got := labels(completeAtCursor(t, varsBlock+`set_tx_meta("k", $|`)) + require.ElementsMatch(t, []string{"$mon", "$acc", "$por", "$ast"}, got) +} + +func TestCompleteFunctionsByReturnType(t *testing.T) { + got := labels(completeAtCursor(t, "vars { monetary $m = | }")) + require.Subset(t, got, []string{"balance", "overdraft", "meta"}) + require.NotContains(t, got, "get_amount") + require.NotContains(t, got, "set_tx_meta") +} + +func TestCompleteFunctionSnippet(t *testing.T) { + items := completeAtCursor(t, "vars { monetary $m = bal| }") + var balance *analysis.CompletionItem + for _, item := range items { + if item.Label == "balance" { + balance = &item + } + } + require.NotNil(t, balance) + require.Equal(t, "balance(${1:account}, ${2:asset})", balance.SnippetText) + require.Equal(t, "balance(account, asset) -> monetary", balance.Detail) +} + +func TestCompleteVarTypes(t *testing.T) { + got := labels(completeAtCursor(t, "vars {\n mon|")) + require.ElementsMatch(t, analysis.AllowedTypes, got) +} + +func TestCompleteAccounts(t *testing.T) { + items := completeAtCursor(t, "send [C 1] ( source = @users:alice destination = @us| )") + require.ElementsMatch(t, []string{"@world", "@users:alice"}, labels(items)) + // the whole account literal is replaced + require.Equal(t, parser.Range{ + Start: parser.Position{Character: 49}, + End: parser.Position{Character: 52}, + }, items[0].Range) +} + +func TestCompleteAccountAfterAt(t *testing.T) { + got := labels(completeAtCursor(t, "send [C 1] ( source = @a destination = @| )")) + require.ElementsMatch(t, []string{"@world", "@a"}, got) +} + +func TestCompleteAccountInterpolation(t *testing.T) { + got := labels(completeAtCursor(t, varsBlock+"send [C 1] ( source = @users:| )")) + require.Contains(t, got, "@users:$acc") +} + +func TestCompleteAssets(t *testing.T) { + got := labels(completeAtCursor(t, "send [USD/2 1] ( source = @a destination = @b )\nsend [|")) + require.Contains(t, got, "USD/2") +} + +func TestCompleteSourcesInBraces(t *testing.T) { + got := labels(completeAtCursor(t, "send [C 1] ( source = { @a\n | } destination = @b )")) + require.Subset(t, got, []string{"max", "oneof", "@world", "@a"}) +} + +func TestNoCompletionInStringsAndComments(t *testing.T) { + require.Empty(t, completeAtCursor(t, `set_tx_meta("a|")`)) + require.Empty(t, completeAtCursor(t, "// a comment|\n")) +} + +func signatureAtCursor(t *testing.T, src string) *analysis.SignatureHelp { + t.Helper() + pos := parser.PositionOf(src, "|") + require.NotNil(t, pos, "missing cursor marker") + return analysis.SignatureHelpAt(strings.Replace(src, "|", "", 1), *pos) +} + +func TestSignatureHelp(t *testing.T) { + help := signatureAtCursor(t, `set_tx_meta("k", balance(@a, |))`) + require.Equal(t, &analysis.SignatureHelp{ + Label: "balance(account, asset) -> monetary", + Params: []string{"account", "asset"}, + ActiveParam: 1, + Docs: "fetch account balance", + }, help) +} + +func TestSignatureHelpInnermostCall(t *testing.T) { + help := signatureAtCursor(t, `set_tx_meta("k", balance(@a, meta(@b, |)))`) + require.Equal(t, "meta(account, string) -> any", help.Label) + require.Equal(t, 1, help.ActiveParam) +} + +func TestSignatureHelpIncompleteCall(t *testing.T) { + help := signatureAtCursor(t, "vars { monetary $m = balance(@a, | }") + require.Equal(t, 1, help.ActiveParam) + + help = signatureAtCursor(t, "set_tx_meta(|") + require.Equal(t, "set_tx_meta(string, any)", help.Label) + require.Equal(t, 0, help.ActiveParam) +} + +func TestNoSignatureHelpOutsideCalls(t *testing.T) { + require.Nil(t, signatureAtCursor(t, `set_tx_meta("k", 1)|`)) + require.Nil(t, signatureAtCursor(t, `unknown_fn(|`)) +} diff --git a/internal/analysis/robustness_test.go b/internal/analysis/robustness_test.go new file mode 100644 index 00000000..a9a91adb --- /dev/null +++ b/internal/analysis/robustness_test.go @@ -0,0 +1,80 @@ +package analysis_test + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/formancehq/numscript/internal/analysis" + "github.com/formancehq/numscript/internal/parser" +) + +// Simulates a user typing each program of the corpus, one character at a time: +// the analysis of every intermediate (and mostly invalid) program must not panic. +func TestAnalysisOfIncompleteProgramsDoesNotPanic(t *testing.T) { + if testing.Short() { + t.Skip() + } + + for path, src := range numscriptCorpus(t) { + for end := range len(src) + 1 { + partial := src[:end] + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("panic analysing a prefix of %s: %v\n---\n%s", path, r, partial) + } + }() + analyseEverywhere(partial) + }() + } + } +} + +func FuzzAnalysis(f *testing.F) { + for _, src := range numscriptCorpus(f) { + f.Add(src) + } + f.Fuzz(func(t *testing.T, src string) { + analyseEverywhere(src) + }) +} + +// Runs the whole analysis, and the position-based queries on each line of the program +func analyseEverywhere(src string) { + res := analysis.CheckSource(src) + res.GetSymbols() + + lines := strings.Split(src, "\n") + for line, text := range lines { + for _, char := range []int{0, len(text) / 2, len(text)} { + pos := parser.Position{Line: line, Character: char} + analysis.HoverOn(res.Program, pos) + analysis.GotoDefinition(res.Program, pos, res) + analysis.Complete(src, pos) + analysis.SignatureHelpAt(src, pos) + } + } +} + +func numscriptCorpus(t testing.TB) map[string]string { + t.Helper() + corpus := map[string]string{} + err := filepath.WalkDir("../interpreter/testdata", func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".num") { + return err + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + corpus[path] = string(content) + return nil + }) + if err != nil { + t.Fatal(err) + } + return corpus +} diff --git a/internal/lsp/__snapshots__/handlers_test.snap b/internal/lsp/__snapshots__/handlers_test.snap index 6904d465..0b7a10e3 100755 --- a/internal/lsp/__snapshots__/handlers_test.snap +++ b/internal/lsp/__snapshots__/handlers_test.snap @@ -149,3 +149,117 @@ "uri": "example.num" } --- + + +[TestCompletionKeyword - 1] +{ + "isIncomplete": false, + "items": [ + { + "insertTextFormat": 1, + "kind": 14, + "label": "allowing", + "sortText": "0000", + "textEdit": { + "newText": "allowing", + "range": { + "end": { + "character": 19, + "line": 1 + }, + "start": { + "character": 15, + "line": 1 + } + } + } + }, + { + "insertTextFormat": 1, + "kind": 14, + "label": "with", + "sortText": "0001", + "textEdit": { + "newText": "with", + "range": { + "end": { + "character": 19, + "line": 1 + }, + "start": { + "character": 15, + "line": 1 + } + } + } + }, + { + "insertTextFormat": 1, + "kind": 14, + "label": "destination", + "sortText": "0002", + "textEdit": { + "newText": "destination", + "range": { + "end": { + "character": 19, + "line": 1 + }, + "start": { + "character": 15, + "line": 1 + } + } + } + } + ] +} +--- + +[TestSignatureHelp - 1] +{ + "activeParameter": 1, + "signatures": [ + { + "documentation": "fetch account balance", + "label": "balance(account, asset) -\u003e monetary", + "parameters": [ + { + "label": "account" + }, + { + "label": "asset" + } + ] + } + ] +} +--- + +[TestCompletion - 1] +{ + "isIncomplete": false, + "items": [ + { + "detail": "account", + "insertTextFormat": 1, + "kind": 6, + "label": "$acc", + "sortText": "0000", + "textEdit": { + "newText": "$acc", + "range": { + "end": { + "character": 11, + "line": 6 + }, + "start": { + "character": 10, + "line": 6 + } + } + } + } + ] +} +--- diff --git a/internal/lsp/handlers.go b/internal/lsp/handlers.go index 1f4ff58a..5ca5147a 100644 --- a/internal/lsp/handlers.go +++ b/internal/lsp/handlers.go @@ -5,6 +5,7 @@ import ( "log" "os" "slices" + "sync/atomic" "github.com/formancehq/numscript/internal/analysis" "github.com/formancehq/numscript/internal/jsonrpc2" @@ -20,6 +21,9 @@ type InMemoryDocument struct { type State struct { documents documentStore[InMemoryDocument] + + // whether the client supports snippets in completion items + snippetSupport atomic.Bool } func (state *State) updateDocument(conn *jsonrpc2.Conn, uri protocol.DocumentURI, text string) { @@ -136,6 +140,96 @@ func (state *State) handleGotoDefinition(params protocol.DefinitionParams) *prot } } +func (state *State) handleCompletion(params protocol.CompletionParams) *protocol.CompletionList { + doc, ok := state.documents.Get(params.TextDocument.URI) + if !ok { + return nil + } + + snippetSupport := state.snippetSupport.Load() + items := analysis.Complete(doc.Text, fromLspPosition(params.Position)) + + lspItems := make([]protocol.CompletionItem, 0, len(items)) + for index, item := range items { + newText := item.InsertText + format := protocol.InsertTextFormatPlainText + if item.SnippetText != "" { + if snippetSupport { + newText = item.SnippetText + format = protocol.InsertTextFormatSnippet + } else if item.Kind == analysis.CompletionSnippet { + continue + } + } + + var docs any + if item.Docs != "" { + docs = item.Docs + } + + lspItems = append(lspItems, protocol.CompletionItem{ + Label: item.Label, + Kind: toLspCompletionKind(item.Kind), + Detail: item.Detail, + Documentation: docs, + InsertTextFormat: format, + // keep the order computed by the analysis + SortText: fmt.Sprintf("%04d", index), + TextEdit: &protocol.TextEdit{ + Range: *toLspRange(item.Range), + NewText: newText, + }, + }) + } + + return &protocol.CompletionList{Items: lspItems} +} + +func toLspCompletionKind(kind analysis.CompletionItemKind) protocol.CompletionItemKind { + switch kind { + case analysis.CompletionKeyword: + return protocol.CompletionItemKindKeyword + case analysis.CompletionVariable: + return protocol.CompletionItemKindVariable + case analysis.CompletionFunction: + return protocol.CompletionItemKindFunction + case analysis.CompletionType: + return protocol.CompletionItemKindTypeParameter + case analysis.CompletionAccount, analysis.CompletionAsset: + return protocol.CompletionItemKindValue + case analysis.CompletionSnippet: + return protocol.CompletionItemKindSnippet + default: + return protocol.CompletionItemKindText + } +} + +func (state *State) handleSignatureHelp(params protocol.SignatureHelpParams) *protocol.SignatureHelp { + doc, ok := state.documents.Get(params.TextDocument.URI) + if !ok { + return nil + } + + help := analysis.SignatureHelpAt(doc.Text, fromLspPosition(params.Position)) + if help == nil { + return nil + } + + params_ := make([]protocol.ParameterInformation, len(help.Params)) + for index, param := range help.Params { + params_[index] = protocol.ParameterInformation{Label: param} + } + + return &protocol.SignatureHelp{ + Signatures: []protocol.SignatureInformation{{ + Label: help.Label, + Documentation: help.Docs, + Parameters: params_, + }}, + ActiveParameter: uint32(help.ActiveParam), + } +} + func (state *State) handleGetSymbols(params protocol.DocumentSymbolParams) []protocol.DocumentSymbol { doc, ok := state.documents.Get(params.TextDocument.URI) if !ok { @@ -235,6 +329,13 @@ var initializeResult protocol.InitializeResult = protocol.InitializeResult{ DefinitionProvider: true, DocumentSymbolProvider: true, CodeActionProvider: true, + CompletionProvider: &protocol.CompletionOptions{ + TriggerCharacters: []string{"$", "@", ":"}, + }, + SignatureHelpProvider: &protocol.SignatureHelpOptions{ + TriggerCharacters: []string{"(", ","}, + RetriggerCharacters: []string{","}, + }, }, ServerInfo: &protocol.ServerInfo{ Name: "numscript-ls", @@ -253,7 +354,10 @@ func NewConn(objStream jsonrpc2.MessageStream) *jsonrpc2.Conn { } return jsonrpc2.NewConn(objStream, - jsonrpc2.NewRequestHandler("initialize", jsonrpc2.SyncHandling, func(_ any, conn *jsonrpc2.Conn) any { + jsonrpc2.NewRequestHandler("initialize", jsonrpc2.SyncHandling, func(p protocol.InitializeParams, conn *jsonrpc2.Conn) any { + if completion := p.Capabilities.TextDocument; completion != nil && completion.Completion != nil && completion.Completion.CompletionItem != nil { + state.snippetSupport.Store(completion.Completion.CompletionItem.SnippetSupport) + } return initializeResult }), jsonrpc2.NewNotificationHandler("textDocument/didOpen", jsonrpc2.SyncHandling, func(p protocol.DidOpenTextDocumentParams, conn *jsonrpc2.Conn) { @@ -270,6 +374,12 @@ func NewConn(objStream jsonrpc2.MessageStream) *jsonrpc2.Conn { jsonrpc2.NewRequestHandler("textDocument/hover", jsonrpc2.AsyncHandling, func(p protocol.HoverParams, conn *jsonrpc2.Conn) any { return state.handleHover(p) }), + jsonrpc2.NewRequestHandler("textDocument/completion", jsonrpc2.AsyncHandling, func(p protocol.CompletionParams, _ *jsonrpc2.Conn) any { + return state.handleCompletion(p) + }), + jsonrpc2.NewRequestHandler("textDocument/signatureHelp", jsonrpc2.AsyncHandling, func(p protocol.SignatureHelpParams, _ *jsonrpc2.Conn) any { + return state.handleSignatureHelp(p) + }), jsonrpc2.NewRequestHandler("textDocument/codeAction", jsonrpc2.AsyncHandling, func(p protocol.CodeActionParams, _ *jsonrpc2.Conn) any { return state.handleCodeAction(p) }), diff --git a/internal/lsp/handlers_test.go b/internal/lsp/handlers_test.go index 91fb3c57..5547511c 100644 --- a/internal/lsp/handlers_test.go +++ b/internal/lsp/handlers_test.go @@ -104,6 +104,57 @@ func TestGotoDef(t *testing.T) { snaps.MatchJSON(t, raw) } +func TestCompletion(t *testing.T) { + client := newTestClient() + + doc, input, _ := client.OpenFile("example.num", `vars { + monetary $mon + account $acc +} + +send $mon ( + source = $ + destination = @dest +) +`) + + // right after the `$` of the source + pos := *parser.PositionOfIndexed(input, "$", 3) + pos.Character++ + raw, err := client.Completion(doc, pos) + require.Nil(t, err) + snaps.MatchJSON(t, raw) +} + +func TestCompletionKeyword(t *testing.T) { + client := newTestClient() + + doc, input, _ := client.OpenFile("example.num", `send [COIN 100] ( + source = @src allo + destination = @dest +) +`) + + pos := *parser.PositionOf(input, "allo") + pos.Character += len("allo") + raw, err := client.Completion(doc, pos) + require.Nil(t, err) + snaps.MatchJSON(t, raw) +} + +func TestSignatureHelp(t *testing.T) { + client := newTestClient() + + doc, input, _ := client.OpenFile("example.num", `vars { + monetary $mon = balance(@acc, ) +} +`) + + raw, err := client.SignatureHelp(doc, *parser.PositionOf(input, ")")) + require.Nil(t, err) + snaps.MatchJSON(t, raw) +} + // Testing utilities type TestClient struct { conn *jsonrpc2.Conn @@ -138,6 +189,24 @@ func (c *TestClient) Hover(doc protocol.TextDocumentIdentifier, position parser. }) } +func (c *TestClient) Completion(doc protocol.TextDocumentIdentifier, position parser.Position) (json.RawMessage, *jsonrpc2.ResponseError) { + return c.conn.SendRequest("textDocument/completion", protocol.CompletionParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: doc, + Position: lsp.ParserToLspPosition(position), + }, + }) +} + +func (c *TestClient) SignatureHelp(doc protocol.TextDocumentIdentifier, position parser.Position) (json.RawMessage, *jsonrpc2.ResponseError) { + return c.conn.SendRequest("textDocument/signatureHelp", protocol.SignatureHelpParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: doc, + Position: lsp.ParserToLspPosition(position), + }, + }) +} + func (c *TestClient) GetSymbols(doc protocol.TextDocumentIdentifier) (json.RawMessage, *jsonrpc2.ResponseError) { return c.conn.SendRequest("textDocument/documentSymbol", protocol.DocumentSymbolParams{ TextDocument: doc, diff --git a/internal/mcp_impl/handlers_test.go b/internal/mcp_impl/handlers_test.go index 36c00e1d..9560d5de 100644 --- a/internal/mcp_impl/handlers_test.go +++ b/internal/mcp_impl/handlers_test.go @@ -30,7 +30,7 @@ func TestHandleEvalToolRejectsParseErrors(t *testing.T) { require.NotEmpty(t, result.Content) text, ok := result.Content[0].(mcp.TextContent) require.True(t, ok) - require.Contains(t, text.Text, "mismatched input") + require.Contains(t, text.Text, "expected `source`") } func TestHandleEvalToolRejectsDuplicateBalancesWithScope(t *testing.T) { diff --git a/internal/parser/__snapshots__/parser_fault_tolerance_test.snap b/internal/parser/__snapshots__/parser_fault_tolerance_test.snap index a31860ea..44ac6c24 100755 --- a/internal/parser/__snapshots__/parser_fault_tolerance_test.snap +++ b/internal/parser/__snapshots__/parser_fault_tolerance_test.snap @@ -11,7 +11,7 @@ parser.Program{ { Range: parser.Range{ Start: parser.Position{Character:7, Line:0}, - End: parser.Position{Character:18, Line:0}, + End: parser.Position{Character:15, Line:0}, }, Name: (*parser.Variable)(nil), Type: &parser.TypeDecl{ @@ -59,7 +59,26 @@ parser.Program{ Start: parser.Position{}, End: parser.Position{Character:11, Line:0}, }, - SentValue: nil, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:11, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:11, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: nil, + }, + }, Source: nil, Destination: nil, }, @@ -76,16 +95,16 @@ parser.Program{ &parser.SendStatement{ Range: parser.Range{ Start: parser.Position{}, - End: parser.Position{Character:17, Line:4}, + End: parser.Position{Character:1, Line:5}, }, - SentValue: &parser.SentValueLiteral{ + SentValue: nil, + Source: &parser.SourceInorder{ Range: parser.Range{ - Start: parser.Position{Character:6, Line:0}, - End: parser.Position{Character:17, Line:4}, + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:3, Line:3}, }, - Monetary: nil, + Sources: nil, }, - Source: nil, Destination: nil, }, }, @@ -247,7 +266,6 @@ parser.Program{ abs: {0x1}, }, }, - nil, }, }, }, @@ -319,7 +337,7 @@ parser.Program{ { Range: parser.Range{ Start: parser.Position{Character:3, Line:3}, - End: parser.Position{Character:3, Line:4}, + End: parser.Position{Character:16, Line:3}, }, Cap: &parser.MonetaryLiteral{ Range: parser.Range{ @@ -501,3 +519,630 @@ parser.Program{ Comments: nil, } --- + +[TestFaultToleranceBareAt - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:1, Line:3}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:0}, + End: parser.Position{Character:13, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceInorder{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:19, Line:1}, + }, + Sources: { + &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:13, Line:1}, + End: parser.Position{Character:15, Line:1}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:13, Line:1}, + End: parser.Position{Character:15, Line:1}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:16, Line:2}, + End: parser.Position{Character:18, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceBareAt - 2] +Got errors while parsing: +expected an account name, found `}` + 0 | send [COIN 10] ( + 1 | source = { @a @ } + | ~ + 2 | destination = @b + + +--- + +[TestFaultToleranceIncompleteOverdraft - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:1, Line:3}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:0}, + End: parser.Position{Character:13, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:22, Line:1}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:16, Line:2}, + End: parser.Position{Character:18, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceIncompleteOverdraft - 2] +Got errors while parsing: +expected `unbounded` or `overdraft`, found `destination` + 0 | send [COIN 10] ( + 1 | source = @a allowing + | ~ + 2 | destination = @b + + +--- + +[TestFaultToleranceIncompleteInfix - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:1, Line:3}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:11, Line:0}, + }, + Monetary: &parser.BinaryInfix{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:11, Line:0}, + }, + Operator: "+", + Left: &parser.Variable{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:7, Line:0}, + }, + Name: "m", + }, + Right: nil, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:16, Line:2}, + End: parser.Position{Character:18, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceIncompleteInfix - 2] +Got errors while parsing: +expected an expression, found `source` + 0 | send $m + ( + | ~ + 1 | source = @a + + +--- + +[TestFaultToleranceBareDollar - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:12, Line:0}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:12, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:12, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: nil, + }, + }, + Source: nil, + Destination: nil, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceBareDollar - 2] +Got errors while parsing: +invalid variable name: expected a lowercase letter or `_` after `$` + 0 | send [COIN $ + | ~ +expected `]`, found end of file + 0 | send [COIN $ + | ~ + +--- + +[TestFaultToleranceMisspelledKeyword - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:1, Line:3}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:0}, + End: parser.Position{Character:13, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:15, Line:2}, + End: parser.Position{Character:17, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceMisspelledKeyword - 2] +Got errors while parsing: +expected `destination`, found `destinaton` + 1 | source = @a + 2 | destinaton = @b + | ~~~~~~~~~~ + 3 | ) + + +--- + +[TestFaultToleranceMissingParens - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.FnCall{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:19, Line:0}, + }, + Caller: &parser.FnCallIdentifier{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:11, Line:0}, + }, + Name: "set_tx_meta", + }, + Args: { + &parser.StringLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:12, Line:0}, + End: parser.Position{Character:15, Line:0}, + }, + String: "k", + }, + &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:17, Line:0}, + End: parser.Position{Character:19, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0x2a}, + }, + }, + }, + }, + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{Character:0, Line:1}, + End: parser.Position{Character:1, Line:4}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:1}, + End: parser.Position{Character:14, Line:1}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:1}, + End: parser.Position{Character:14, Line:1}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:1}, + End: parser.Position{Character:10, Line:1}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:1}, + End: parser.Position{Character:13, Line:1}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:2}, + End: parser.Position{Character:13, Line:2}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:2}, + End: parser.Position{Character:13, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:16, Line:3}, + End: parser.Position{Character:18, Line:3}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceMissingParens - 2] +Got errors while parsing: +expected `,` or `)`, found `send` + 0 | set_tx_meta("k", 42 + | ~ + 1 | send [COIN 10] ( + + +--- + +[TestFaultToleranceGarbageBetweenStatements - 1] +parser.Program{ + Flags: nil, + Vars: (*parser.VarDeclarations)(nil), + Statements: { + &parser.SendStatement{ + Range: parser.Range{ + Start: parser.Position{}, + End: parser.Position{Character:47, Line:0}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:0}, + End: parser.Position{Character:14, Line:0}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:0}, + End: parser.Position{Character:10, Line:0}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:0}, + End: parser.Position{Character:13, Line:0}, + }, + Number: &big.Int{ + neg: false, + abs: {0xa}, + }, + }, + }, + }, + Source: &parser.SourceAccount{ + Range: parser.Range{ + Start: parser.Position{Character:26, Line:0}, + End: parser.Position{Character:28, Line:0}, + }, + Color: nil, + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:26, Line:0}, + End: parser.Position{Character:28, Line:0}, + }, + Parts: { + parser.AccountTextPart{Name:"a"}, + }, + }, + }, + Destination: &parser.DestinationAccount{ + ValueExpr: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:43, Line:0}, + End: parser.Position{Character:45, Line:0}, + }, + Parts: { + parser.AccountTextPart{Name:"b"}, + }, + }, + }, + }, + &parser.SaveStatement{ + Range: parser.Range{ + Start: parser.Position{Character:0, Line:2}, + End: parser.Position{Character:21, Line:2}, + }, + SentValue: &parser.SentValueLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:2}, + End: parser.Position{Character:13, Line:2}, + }, + Monetary: &parser.MonetaryLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:5, Line:2}, + End: parser.Position{Character:13, Line:2}, + }, + Asset: &parser.AssetLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:6, Line:2}, + End: parser.Position{Character:10, Line:2}, + }, + Asset: "COIN", + }, + Amount: &parser.NumberLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:11, Line:2}, + End: parser.Position{Character:12, Line:2}, + }, + Number: &big.Int{ + neg: false, + abs: {0x1}, + }, + }, + }, + }, + Account: &parser.AccountInterpLiteral{ + Range: parser.Range{ + Start: parser.Position{Character:19, Line:2}, + End: parser.Position{Character:21, Line:2}, + }, + Parts: { + parser.AccountTextPart{Name:"c"}, + }, + }, + }, + }, + Comments: nil, +} +--- + +[TestFaultToleranceGarbageBetweenStatements - 2] +Got errors while parsing: +invalid character: `!!` + 0 | send [COIN 10] ( source = @a destination = @b ) + 1 | !! this is not numscript ]] + | ~~ + 2 | save [COIN 1] from @c + +expected a statement, found `this` + 0 | send [COIN 10] ( source = @a destination = @b ) + 1 | !! this is not numscript ]] + | ~~~~ + 2 | save [COIN 1] from @c + + +--- diff --git a/internal/parser/__snapshots__/parser_test.snap b/internal/parser/__snapshots__/parser_test.snap index f4c50c03..6b9a21d1 100755 --- a/internal/parser/__snapshots__/parser_test.snap +++ b/internal/parser/__snapshots__/parser_test.snap @@ -2359,20 +2359,18 @@ parser.Program{ [TestShowErrorLines - 1] Got errors while parsing: -no viable alternative at input 'errdestination' +expected an expression, found `err` + 0 | send [EUR/2 100] ( 1 | source = err + | ~~~ 2 | destination = ee - | ~~~~~~~~~~ - 3 | ) -missing '(' at ')' - 2 | destination = ee - 3 | ) - | -missing ')' at '' +expected an expression, found `ee` + 1 | source = err 2 | destination = ee + | ~~ 3 | ) - | ~~~~ + --- @@ -3423,16 +3421,16 @@ parser.Program{ { Range: parser.Range{ Start: parser.Position{Character:2, Line:1}, - End: parser.Position{Character:19, Line:1}, + End: parser.Position{Character:18, Line:1}, }, - Content: " first comment\n", + Content: " first comment", }, { Range: parser.Range{ Start: parser.Position{Character:2, Line:4}, - End: parser.Position{Character:20, Line:4}, + End: parser.Position{Character:19, Line:4}, }, - Content: " second comment\n", + Content: " second comment", }, }, } diff --git a/internal/parser/completion.go b/internal/parser/completion.go new file mode 100644 index 00000000..0384b64d --- /dev/null +++ b/internal/parser/completion.go @@ -0,0 +1,127 @@ +package parser + +import "unicode/utf8" + +// A syntactic position where the parser was expecting something when it reached the cursor +type CompletionSlot interface{ completionSlot() } + +// An expression is expected. Types contains the types the expression is expected to have, +// when they are known from the syntactic context (e.g. "account" for a source). +type ExprSlot struct{ Types []string } + +// The type of a variable declaration is expected +type VarTypeSlot struct{} + +// A statement is expected +type StatementSlot struct{} + +// A part of an account literal (e.g. `@users:$id`) is expected +type AccountPartSlot struct{ LiteralStart Position } + +func (*ExprSlot) completionSlot() {} +func (*VarTypeSlot) completionSlot() {} +func (*StatementSlot) completionSlot() {} +func (*AccountPartSlot) completionSlot() {} + +// A function call enclosing the cursor +type FnCallFrame struct { + Name string + // The index of the argument the cursor is on + ArgIndex int +} + +// What the parser expected at the cursor position +type CompletionContext struct { + // The keywords that are valid at the cursor + Keywords []string + + Slots []CompletionSlot + + // The function calls enclosing the cursor (innermost last) + FnCalls []FnCallFrame + + // The range of the word under the cursor, that a completion would replace. + // It's an empty range at the cursor when no word is being typed. + Range Range + + // The part of the word under the cursor that precedes the cursor + Prefix string +} + +// Tokens that can be extended by typing more characters: when the cursor is right after +// one of them, that token is the one being completed +func (k tokenKind) isWordLike() bool { + switch k { + case tkIdentifier, tkVariable, tkAccountText, tkAsset, tkNumber: + return true + default: + return k.isKeyword() + } +} + +// Parses the input and returns what the parser expected at the cursor position. +// +// Returns a nil context when the cursor is inside a comment or a string literal. +func ParseForCompletion(input string, cursor Position) (ParseResult, *CompletionContext) { + tokens, comments, lexErrors := lex(input) + + for _, comment := range comments { + if posLt(comment.Start, cursor) && !posLt(comment.End, cursor) { + return Parse(input), nil + } + } + + cursorTok := -1 + completionRange := Range{Start: cursor, End: cursor} + prefix := "" + + for index, tk := range tokens { + if tk.Kind == tkEOF || !posLt(tk.Start, cursor) { + cursorTok = index + break + } + + containsCursor := posLt(cursor, tk.End) + touchesCursor := tk.End == cursor && tk.Kind.isWordLike() + if !containsCursor && !touchesCursor { + continue + } + + if tk.Kind == tkString { + return Parse(input), nil + } + + cursorTok = index + completionRange = tk.Range + // tokens never span multiple lines + prefix = runePrefix(tk.Text, cursor.Character-tk.Start.Character) + break + } + + p := newParser(tokens, cursorTok) + result := p.finish(input, comments, lexErrors) + + ctx := p.completion + ctx.Range = completionRange + ctx.Prefix = prefix + return result, &ctx +} + +func posLt(a, b Position) bool { + if a.Line != b.Line { + return a.Line < b.Line + } + return a.Character < b.Character +} + +func runePrefix(s string, n int) string { + offset := 0 + for range n { + if offset >= len(s) { + break + } + _, size := utf8.DecodeRuneInString(s[offset:]) + offset += size + } + return s[:offset] +} diff --git a/internal/parser/differential_test.go b/internal/parser/differential_test.go new file mode 100644 index 00000000..aff4f60b --- /dev/null +++ b/internal/parser/differential_test.go @@ -0,0 +1,332 @@ +package parser_test + +import ( + "go/ast" + goparser "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + + "github.com/formancehq/numscript/internal/parser" + "github.com/formancehq/numscript/internal/parser/internal/legacy" + "github.com/kr/pretty" +) + +// The hand-written parser replaced the ANTLR one. These tests check that both +// parsers agree on the corpus of numscript programs found in the repository: +// +// - when the ANTLR parser accepts a program, the new parser must accept it and produce the same AST +// - when the ANTLR parser rejects a program, the new parser must reject it too +// +// TODO remove these tests (and the legacy package) once the new parser has been released for a while. + +// Programs the ANTLR parser rejects, but are now accepted on purpose +func isIntentionalDivergence(src string) bool { + // a line comment at the end of the file without a trailing newline used to be a parsing error + lastLine := src[strings.LastIndex(src, "\n")+1:] + return strings.Contains(lastLine, "//") +} + +var handwrittenCorpus = []string{ + // operators precedence and associativity + `set_tx_meta("k", 1 + 2 + 3)`, + `set_tx_meta("k", 1 - 2 - 3)`, + `set_tx_meta("k", 1 / 2 / 3)`, + `set_tx_meta("k", 1 + 2 / 3 - 4)`, + `set_tx_meta("k", -1 / 2)`, + `set_tx_meta("k", - -1)`, + `set_tx_meta("k", -$x + 1)`, + `set_tx_meta("k", (1 + 2) / 3)`, + `set_tx_meta("k", -(1 + 2))`, + + // monetary literals, where the asset and amount expressions are juxtaposed + `send [COIN -100] ( source = @a destination = @b )`, + `send [$asset -100] ( source = @a destination = @b )`, + `send [$a - 1 2] ( source = @a destination = @b )`, + `send [$a - 1 - 2] ( source = @a destination = @b )`, + `send [$a - 1 + 2] ( source = @a destination = @b )`, + `send [$a - 1 - 2 - 3] ( source = @a destination = @b )`, + `send [($a - 1) -2] ( source = @a destination = @b )`, + `send [$a $b - $c] ( source = @a destination = @b )`, + `send [$a -$b - $c] ( source = @a destination = @b )`, + `send [COIN 10] + [COIN 20] ( source = @a destination = @b )`, + `send [COIN 10] - $x ( source = @a destination = @b )`, + `send [[COIN 1] -1] ( source = @a destination = @b )`, + `send [COIN *] ( source = @a destination = @b )`, + `send [COIN - 1 *] ( source = @a destination = @b )`, + `send [COIN] ( source = @a destination = @b )`, + `send [COIN 1 2] ( source = @a destination = @b )`, + + // account literals + `set_tx_meta("k", @a:b:c)`, + `set_tx_meta("k", @a:$b:c)`, + `set_tx_meta("k", @a-b_C:0)`, + `set_tx_meta("k", @a :b)`, + `set_tx_meta("k", @a: b)`, + `set_tx_meta("k", @ a)`, + `set_tx_meta("k", @$a)`, + `set_tx_meta("k", @a:)`, + + // sources and destinations + `send $x ( source = { } destination = @b )`, + `send $x ( source = { @a -@b } destination = @b )`, + `send $x ( source = { @a @b \ "c" } destination = @b )`, + `send $x ( source = { $p from @a remaining from @b } destination = @b )`, + `send $x ( source = { remaining from @a } destination = @b )`, + `send $x ( source = { 1/2 from @a @b } destination = @b )`, + `send $x ( source = { @a 1/2 from @b } destination = @b )`, + `send $x ( source = { max $m from @a @b } destination = @b )`, + `send $x ( source = oneof { @a @b } destination = @b )`, + `send $x ( source = oneof { } destination = @b )`, + `send $x ( source = @a \ "c" with scaling through @s destination = @b )`, + `send $x ( source = @a with scaling through @s \ "c" destination = @b )`, + `send $x ( source = @a allowing overdraft up to [COIN 10] destination = @b )`, + `send $x ( source = @a \ $c allowing unbounded overdraft destination = @b )`, + `send $x ( source = @a destination = { remaining kept } )`, + `send $x ( source = @a destination = { remaining to @b } )`, + `send $x ( source = @a destination = { remaining kept remaining to @c } )`, + `send $x ( source = @a destination = { max $m to @c remaining kept } )`, + `send $x ( source = @a destination = { max $m kept } )`, + `send $x ( source = @a destination = { } )`, + `send $x ( source = @a destination = { 1/2 to @a 1/2 kept } )`, + `send $x ( source = @a destination = { 1/2 to @a remaining kept } )`, + `send $x ( source = @a destination = { 1/2 to { max $m to @b remaining kept } remaining kept } )`, + `send $x ( source = @a destination = oneof { max $m to @c remaining kept } )`, + `send $x ( source = @a destination = oneof { remaining kept } )`, + `send $x ( destination = @b source = @a )`, + + // other statements + `save [COIN 10] from @a`, + `save [COIN *] from @a`, + `save $m from @a`, + `overdraft(@a, COIN)`, + `f()`, + `f(,)`, + `f(1,)`, + `f(1 2)`, + + // vars and flags + `vars { }`, + `vars { monetary $m = balance(@a, COIN) account $a }`, + `vars { monetary $m = [COIN -1] }`, + `vars { max $m }`, + "#![feature(\"a\", \"b\")]\nsend $x ( source = @a destination = @b )", + "#![feature()]", + "#![feature(\"a\",)]", + "vars { }\nvars { }", + "set_tx_meta(\"k\", 1)\nvars { }", + + // lexing + `set_tx_meta("k", 1_000)`, + `set_tx_meta("k", 12.5%)`, + `set_tx_meta("k", 12%)`, + `set_tx_meta("k", 12.5)`, + `set_tx_meta("k", 1.%)`, + `set_tx_meta("k", "a\"b")`, + `set_tx_meta("k", "a\"")`, + `set_tx_meta("k", "a\\"b")`, + `set_tx_meta("k", "a\\")`, + `set_tx_meta("k", "a)`, + `set_tx_meta("k", USD/2)`, + `set_tx_meta("k", USD/ 2)`, + `set_tx_meta("k", A1/02)`, + `set_tx_meta("k", $_x)`, + `set_tx_meta("k", $x1)`, + `set_tx_meta("k", $1)`, + `set_tx_meta("k", $X)`, + "set_tx_meta(\"k\", /* a /* nested */ comment */ 1)", + "set_tx_meta(\"k\", /* unterminated 1)", + "set_tx_meta(\"k\", 1) // comment\n", + "set_tx_meta(\"k\", 1)\r\n", + "sendx", + "send_", + "#", + "!", + "Ω", + "", +} + +func TestDifferentialAgainstAntlrParser(t *testing.T) { + corpus := collectCorpus(t) + if len(corpus) < 300 { + t.Fatalf("the corpus is suspiciously small: %d programs", len(corpus)) + } + + mutate := !testing.Short() + checked := 0 + for name, src := range corpus { + checkAgreement(t, name, src) + checked++ + + if mutate { + // remove one word at a time, to get many slightly broken (and some still valid) programs. + // Words are joined by newlines, so that line comments keep spanning a single word + words := strings.Fields(src) + if len(words) > 120 { + continue + } + for i := range words { + removed := slices.Delete(slices.Clone(words), i, i+1) + checkAgreement(t, name+"#remove"+strconv.Itoa(i), strings.Join(removed, "\n")) + + duplicated := slices.Insert(slices.Clone(words), i, words[i]) + checkAgreement(t, name+"#duplicate"+strconv.Itoa(i), strings.Join(duplicated, "\n")) + + if i+1 < len(words) { + swapped := slices.Clone(words) + swapped[i], swapped[i+1] = swapped[i+1], swapped[i] + checkAgreement(t, name+"#swap"+strconv.Itoa(i), strings.Join(swapped, "\n")) + } + checked += 3 + } + } + } + t.Logf("checked %d programs", checked) +} + +func checkAgreement(t *testing.T, name string, src string) { + t.Helper() + + expected := legacy.Parse(src) + got := parser.Parse(src) + + legacyOk := len(expected.Errors) == 0 + newOk := len(got.Errors) == 0 + + switch { + case legacyOk && !newOk: + t.Errorf("%s: the new parser rejects a valid program:\n%s\n\n%s", name, src, parser.ParseErrorsToString(got.Errors, src)) + + case !legacyOk && newOk: + if isIntentionalDivergence(src) { + return + } + t.Errorf("%s: the new parser accepts an invalid program:\n%s\n\nANTLR errors: %v", name, src, expected.Errors) + + case legacyOk && newOk: + expectedProgram, gotProgram := expected.Value, got.Value + expectedProgram.Comments, gotProgram.Comments = nil, nil + if diff := pretty.Diff(expectedProgram, gotProgram); len(diff) != 0 { + t.Errorf("%s: the ASTs differ for:\n%s\n\n%s", name, src, strings.Join(diff, "\n")) + } + } +} + +func collectCorpus(t *testing.T) map[string]string { + t.Helper() + + root := "../.." + corpus := map[string]string{} + for i, src := range handwrittenCorpus { + corpus["handwritten#"+strconv.Itoa(i)] = src + } + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // the oracle package contains the grammar of a different language (the machine numscript) + if d.Name() == "oracle" || d.Name() == "legacy" { + return filepath.SkipDir + } + return nil + } + + switch { + case strings.HasSuffix(path, ".num"): + content, err := os.ReadFile(path) + if err != nil { + return err + } + corpus[path] = string(content) + + case strings.HasSuffix(path, "_test.go"): + // Numscript programs embedded in go tests + for i, lit := range stringLiteralsOf(t, path) { + if looksLikeNumscript(lit) { + corpus[path+"#"+strconv.Itoa(i)] = lit + } + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + return corpus +} + +func stringLiteralsOf(t *testing.T, path string) []string { + fset := token.NewFileSet() + file, err := goparser.ParseFile(fset, path, nil, goparser.SkipObjectResolution) + if err != nil { + t.Fatal(err) + } + + var literals []string + ast.Inspect(file, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + value, err := strconv.Unquote(lit.Value) + if err == nil { + literals = append(literals, value) + } + return true + }) + return literals +} + +func looksLikeNumscript(s string) bool { + for _, keyword := range []string{"send ", "save ", "vars ", "set_tx_meta(", "set_account_meta("} { + if strings.Contains(s, keyword) { + return true + } + } + return false +} + +const benchmarkProgram = `vars { + monetary $amount + account $user + portion $fee = 2.5% +} + +send $amount ( + source = { + $user allowing overdraft up to [USD/2 100] + @users:$user:savings + @world + } + destination = { + $fee to @platform:fees + remaining to { + max [USD/2 1000] to @merchant + remaining kept + } + } +) + +set_tx_meta("processed", true_value(1 + 2 / 3)) +` + +func BenchmarkParse(b *testing.B) { + b.Run("handwritten", func(b *testing.B) { + for b.Loop() { + parser.Parse(benchmarkProgram) + } + }) + b.Run("antlr", func(b *testing.B) { + for b.Loop() { + legacy.Parse(benchmarkProgram) + } + }) +} diff --git a/internal/parser/antlrParser/Lexer.interp b/internal/parser/internal/legacy/antlrParser/Lexer.interp similarity index 100% rename from internal/parser/antlrParser/Lexer.interp rename to internal/parser/internal/legacy/antlrParser/Lexer.interp diff --git a/internal/parser/antlrParser/Lexer.tokens b/internal/parser/internal/legacy/antlrParser/Lexer.tokens similarity index 100% rename from internal/parser/antlrParser/Lexer.tokens rename to internal/parser/internal/legacy/antlrParser/Lexer.tokens diff --git a/internal/parser/antlrParser/Numscript.interp b/internal/parser/internal/legacy/antlrParser/Numscript.interp similarity index 100% rename from internal/parser/antlrParser/Numscript.interp rename to internal/parser/internal/legacy/antlrParser/Numscript.interp diff --git a/internal/parser/antlrParser/Numscript.tokens b/internal/parser/internal/legacy/antlrParser/Numscript.tokens similarity index 100% rename from internal/parser/antlrParser/Numscript.tokens rename to internal/parser/internal/legacy/antlrParser/Numscript.tokens diff --git a/internal/parser/antlrParser/lexer.go b/internal/parser/internal/legacy/antlrParser/lexer.go similarity index 100% rename from internal/parser/antlrParser/lexer.go rename to internal/parser/internal/legacy/antlrParser/lexer.go diff --git a/internal/parser/antlrParser/numscript_base_listener.go b/internal/parser/internal/legacy/antlrParser/numscript_base_listener.go similarity index 100% rename from internal/parser/antlrParser/numscript_base_listener.go rename to internal/parser/internal/legacy/antlrParser/numscript_base_listener.go diff --git a/internal/parser/antlrParser/numscript_listener.go b/internal/parser/internal/legacy/antlrParser/numscript_listener.go similarity index 100% rename from internal/parser/antlrParser/numscript_listener.go rename to internal/parser/internal/legacy/antlrParser/numscript_listener.go diff --git a/internal/parser/antlrParser/numscript_parser.go b/internal/parser/internal/legacy/antlrParser/numscript_parser.go similarity index 100% rename from internal/parser/antlrParser/numscript_parser.go rename to internal/parser/internal/legacy/antlrParser/numscript_parser.go diff --git a/internal/parser/internal/legacy/legacy.go b/internal/parser/internal/legacy/legacy.go new file mode 100644 index 00000000..6a3cc431 --- /dev/null +++ b/internal/parser/internal/legacy/legacy.go @@ -0,0 +1,678 @@ +package legacy + +import ( + "math/big" + "strings" + + "github.com/formancehq/numscript/internal/parser" + antlrParser "github.com/formancehq/numscript/internal/parser/internal/legacy/antlrParser" + "github.com/formancehq/numscript/internal/utils" + + "github.com/antlr4-go/antlr/v4" +) + +type ParserError struct { + parser.Range + Msg string +} + +func (e ParserError) Error() string { + return e.Msg +} + +type ParseResult struct { + Source string + Value parser.Program + Errors []ParserError +} + +type ErrorListener struct { + antlr.DefaultErrorListener + Errors []ParserError +} + +func (l *ErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, startL, startC int, msg string, e antlr.RecognitionException) { + length := 1 + if token, ok := offendingSymbol.(antlr.Token); ok { + length = len(token.GetText()) + } + endL := startL + endC := startC + length - 1 // -1 so that end character is inside the offending token + l.Errors = append(l.Errors, ParserError{ + Msg: msg, + Range: parser.Range{ + Start: parser.Position{Character: startC, Line: startL - 1}, + End: parser.Position{Character: endC, Line: endL - 1}, + }, + }) +} + +func Parse(input string) ParseResult { + // TODO handle lexer errors + listener := &ErrorListener{} + + is := antlr.NewInputStream(input) + lexer := antlrParser.NewLexer(is) + lexer.RemoveErrorListeners() + lexer.AddErrorListener(listener) + + stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel) + + p := antlrParser.NewNumscriptParser(stream) + p.RemoveErrorListeners() + p.AddErrorListener(listener) + + parsed := parseProgram(p.Program()) + + for _, tk := range stream.GetAllTokens() { + if tk.GetChannel() == antlr.TokenHiddenChannel { + parsed.Comments = append(parsed.Comments, parser.Comment{ + Content: tk.GetText()[2:], + Range: tokenToRange(tk), + }) + } + } + + return ParseResult{ + Source: input, + Value: parsed, + Errors: listener.Errors, + } +} + +func ParseErrorsToString(errors []ParserError, source string) string { + buf := "Got errors while parsing:\n" + for _, err := range errors { + buf += err.Msg + "\n" + err.ShowOnSource(source) + "\n" + } + return buf +} + +func parseVarsDeclaration(varsCtx antlrParser.IVarsDeclarationContext) *parser.VarDeclarations { + + if varsCtx == nil { + return nil + } + + varBlock := parser.VarDeclarations{ + Range: ctxToRange(varsCtx), + } + + for _, varDecl := range varsCtx.AllVarDeclaration() { + decl := parseVarDeclaration(varDecl) + if decl != nil { + varBlock.Declarations = append(varBlock.Declarations, *decl) + } + } + return &varBlock +} + +func parseProgram(programCtx antlrParser.IProgramContext) parser.Program { + vars := parseVarsDeclaration(programCtx.VarsDeclaration()) + + featureCtx := programCtx.FeatureDecl() + + var flagIdents []parser.StringLiteral + if featureCtx != nil { + for _, flagTk := range featureCtx.GetFlag() { + flagIdents = append(flagIdents, *parseStringTk(flagTk)) + } + } + + var statements []parser.Statement + for _, statementCtx := range programCtx.AllStatement() { + statements = append(statements, parseStatement(statementCtx)) + } + + return parser.Program{ + Flags: flagIdents, + Statements: statements, + Vars: vars, + } +} + +func parseVarDeclaration(varDecl antlrParser.IVarDeclarationContext) *parser.VarDeclaration { + if varDecl == nil { + return nil + } + + var origin *parser.ValueExpr + if varDecl.VarOrigin() != nil { + expr := parseValueExpr(varDecl.VarOrigin().ValueExpr()) + // when the origin expression is missing or malformed, + // parseValueExpr returns nil; do not wrap it in a non-nil pointer + if expr != nil { + origin = &expr + } + } + + return &parser.VarDeclaration{ + Range: ctxToRange(varDecl), + Type: parseVarType(varDecl.GetType_()), + Name: parseVarLiteral(varDecl.GetName()), + Origin: origin, + } +} + +func parseVarLiteral(tk antlr.Token) *parser.Variable { + if tk == nil || tk.GetTokenIndex() == -1 { + return nil + } + + name := tk.GetText()[1:] + + return &parser.Variable{ + Range: tokenToRange(tk), + Name: name, + } +} + +func parseVarType(tk antlr.Token) *parser.TypeDecl { + if tk == nil || tk.GetTokenIndex() == -1 { + return nil + } + + return &parser.TypeDecl{ + Range: tokenToRange(tk), + Name: tk.GetText(), + } +} + +func parseColorConstraint(colorConstraintCtx antlrParser.IColorConstraintContext) parser.ValueExpr { + if colorConstraintCtx == nil { + return nil + } + + return parseValueExpr( + colorConstraintCtx.ValueExpr(), + ) +} + +func parseSource(sourceCtx antlrParser.ISourceContext) parser.Source { + if sourceCtx == nil { + return nil + } + + range_ := ctxToRange(sourceCtx) + + switch sourceCtx := sourceCtx.(type) { + case *antlrParser.SrcAccountContext: + return &parser.SourceAccount{ + Range: range_, + Color: parseColorConstraint(sourceCtx.ColorConstraint()), + ValueExpr: parseValueExpr(sourceCtx.ValueExpr()), + } + + case *antlrParser.SrcInorderContext: + var sources []parser.Source + for _, sourceCtx := range sourceCtx.AllSource() { + sources = append(sources, parseSource(sourceCtx)) + } + return &parser.SourceInorder{ + Range: range_, + Sources: sources, + } + + case *antlrParser.SrcOneofContext: + var sources []parser.Source + for _, sourceCtx := range sourceCtx.AllSource() { + sources = append(sources, parseSource(sourceCtx)) + } + return &parser.SourceOneof{ + Range: range_, + Sources: sources, + } + + case *antlrParser.SrcAllotmentContext: + var items []parser.SourceAllotmentItem + for _, itemCtx := range sourceCtx.AllAllotmentClauseSrc() { + item := parser.SourceAllotmentItem{ + Range: ctxToRange(itemCtx), + Allotment: parseAllotment(itemCtx.Allotment()), + From: parseSource(itemCtx.Source()), + } + items = append(items, item) + } + return &parser.SourceAllotment{ + Range: range_, + Items: items, + } + + case *antlrParser.SrcCappedContext: + return &parser.SourceCapped{ + Range: range_, + From: parseSource(sourceCtx.Source()), + Cap: parseValueExpr(sourceCtx.GetCap_()), + } + + case *antlrParser.SrcAccountUnboundedOverdraftContext: + return &parser.SourceOverdraft{ + Range: ctxToRange(sourceCtx), + Color: parseColorConstraint(sourceCtx.ColorConstraint()), + Address: parseValueExpr(sourceCtx.GetAddress()), + } + + case *antlrParser.SrcAccountWithScalingContext: + return &parser.SourceWithScaling{ + Range: ctxToRange(sourceCtx), + Address: parseValueExpr(sourceCtx.GetAddress()), + Through: parseValueExpr(sourceCtx.GetSwap()), + } + + case *antlrParser.SrcAccountBoundedOverdraftContext: + varMon := parseValueExpr(sourceCtx.GetMaxOvedraft()) + + return &parser.SourceOverdraft{ + Range: ctxToRange(sourceCtx), + Color: parseColorConstraint(sourceCtx.ColorConstraint()), + Address: parseValueExpr(sourceCtx.GetAddress()), + Bounded: &varMon, + } + + case *antlrParser.SourceContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.Source](sourceCtx.GetText()) + } +} + +func parsePercentageRatio(source string, range_ parser.Range) *parser.PercentageLiteral { + num, floatingDigits, err := parser.ParsePercentageRatio(source) + if err != nil { + // This should be unreachable for tokens emitted by the lexer. + // Degrade gracefully to a zero-value literal instead of panicking, + // so that a malformed token cannot crash the process (e.g. the LSP server). + return &parser.PercentageLiteral{ + Range: range_, + Amount: big.NewInt(0), + } + } + + return &parser.PercentageLiteral{ + Range: range_, + Amount: num, + FloatingDigits: floatingDigits, + } +} + +func parseAllotment(allotmentCtx antlrParser.IAllotmentContext) parser.AllotmentValue { + switch allotmentCtx := allotmentCtx.(type) { + case *antlrParser.PortionedAllotmentContext: + expr := parseValueExpr(allotmentCtx.ValueExpr()) + return &parser.ValueExprAllotment{Value: expr} + + case *antlrParser.RemainingAllotmentContext: + return &parser.RemainingAllotment{ + Range: ctxToRange(allotmentCtx), + } + + case *antlrParser.AllotmentContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.AllotmentValue](allotmentCtx.GetText()) + } +} + +func parseStringTk(stringCtx antlr.Token) *parser.StringLiteral { + rawStr := stringCtx.GetText() + // Remove leading and trailing '"' + innerStr := rawStr[1 : len(rawStr)-1] + return &parser.StringLiteral{ + Range: tokenToRange(stringCtx), + String: innerStr, + } +} + +func parseValueExpr(valueExprCtx antlrParser.IValueExprContext) parser.ValueExpr { + switch valueExprCtx := valueExprCtx.(type) { + case *antlrParser.ParenthesizedExprContext: + return parseValueExpr(valueExprCtx.ValueExpr()) + + case *antlrParser.AccountLiteralContext: + litRng := ctxToRange(valueExprCtx) + + var parts []parser.AccountNamePart + for _, accLit := range valueExprCtx.GetChildren() { + switch accLit := accLit.(type) { + case *antlrParser.AccountTextPartContext: + varPartText := accLit.GetText() + parts = append(parts, parser.AccountTextPart{Name: varPartText}) + case *antlrParser.AccountVarPartContext: + v := parseVarLiteral(accLit.VARIABLE_NAME_ACC().GetSymbol()) + parts = append(parts, v) + + case antlr.TerminalNode: + if accLit.GetSymbol().GetTokenType() == antlrParser.NumscriptParserCOLON { + parts = append(parts, parser.AccountTextPart{Name: ":"}) + } + } + + } + + return &parser.AccountInterpLiteral{ + Range: litRng, + Parts: parts, + } + + case *antlrParser.MonetaryLiteralContext: + return parseMonetaryLit(valueExprCtx.MonetaryLit()) + + case *antlrParser.AssetLiteralContext: + return &parser.AssetLiteral{ + Range: ctxToRange(valueExprCtx), + Asset: valueExprCtx.GetText(), + } + + case *antlrParser.NumberLiteralContext: + return parseNumberLiteral(valueExprCtx.NUMBER()) + + case *antlrParser.PercentagePortionLiteralContext: + return parsePercentageRatio(valueExprCtx.GetText(), ctxToRange(valueExprCtx)) + + case *antlrParser.VariableExprContext: + return variableLiteralFromCtx(valueExprCtx) + + case *antlrParser.StringLiteralContext: + return parseStringTk(valueExprCtx.STRING().GetSymbol()) + + case *antlrParser.PrefixExprContext: + return &parser.Prefix{ + Range: ctxToRange(valueExprCtx), + Operator: parser.PrefixOperator(valueExprCtx.GetOp().GetText()), + Expr: parseValueExpr(valueExprCtx.ValueExpr()), + } + + case *antlrParser.InfixExprContext: + return &parser.BinaryInfix{ + Range: ctxToRange(valueExprCtx), + Operator: parser.InfixOperator(valueExprCtx.GetOp().GetText()), + Left: parseValueExpr(valueExprCtx.GetLeft()), + Right: parseValueExpr(valueExprCtx.GetRight()), + } + + case nil, *antlrParser.ValueExprContext: + return nil + + case *antlrParser.ApplicationContext: + return parseFnCall(valueExprCtx.FunctionCall()) + + default: + return utils.NonExhaustiveMatchPanic[parser.ValueExpr](valueExprCtx.GetText()) + } +} + +func variableLiteralFromCtx(ctx antlr.ParserRuleContext) *parser.Variable { + // Discard the '$' + name := ctx.GetText()[1:] + + return &parser.Variable{ + Range: ctxToRange(ctx), + Name: name, + } +} + +func parseDestination(destCtx antlrParser.IDestinationContext) parser.Destination { + if destCtx == nil { + return nil + } + + range_ := ctxToRange(destCtx) + + switch destCtx := destCtx.(type) { + case *antlrParser.DestAccountContext: + return &parser.DestinationAccount{ + ValueExpr: parseValueExpr(destCtx.ValueExpr()), + } + + case *antlrParser.DestInorderContext: + var inorderClauses []parser.CappedKeptOrDestination + for _, destInorderClause := range destCtx.AllDestinationInOrderClause() { + inorderClauses = append(inorderClauses, parseDestinationInorderClause(destInorderClause)) + } + + return &parser.DestinationInorder{ + Range: range_, + Clauses: inorderClauses, + Remaining: parseKeptOrDestination(destCtx.KeptOrDestination()), + } + + case *antlrParser.DestOneofContext: + var inorderClauses []parser.CappedKeptOrDestination + for _, destInorderClause := range destCtx.AllDestinationInOrderClause() { + inorderClauses = append(inorderClauses, parseDestinationInorderClause(destInorderClause)) + } + + return &parser.DestinationOneof{ + Range: range_, + Clauses: inorderClauses, + Remaining: parseKeptOrDestination(destCtx.KeptOrDestination()), + } + + case *antlrParser.DestAllotmentContext: + var items []parser.DestinationAllotmentItem + for _, itemCtx := range destCtx.AllAllotmentClauseDest() { + item := parser.DestinationAllotmentItem{ + Range: ctxToRange(itemCtx), + Allotment: parseDestinationAllotment(itemCtx.Allotment()), + To: parseKeptOrDestination(itemCtx.KeptOrDestination()), + } + items = append(items, item) + } + return &parser.DestinationAllotment{ + Range: range_, + Items: items, + } + + case *antlrParser.DestinationContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.Destination](destCtx.GetText()) + } + +} + +func parseDestinationInorderClause(clauseCtx antlrParser.IDestinationInOrderClauseContext) parser.CappedKeptOrDestination { + return parser.CappedKeptOrDestination{ + Range: ctxToRange(clauseCtx), + Cap: parseValueExpr(clauseCtx.ValueExpr()), + To: parseKeptOrDestination(clauseCtx.KeptOrDestination()), + } +} + +func parseKeptOrDestination(clauseCtx antlrParser.IKeptOrDestinationContext) parser.KeptOrDestination { + if clauseCtx == nil { + return nil + } + + switch clauseCtx := clauseCtx.(type) { + case *antlrParser.DestinationToContext: + return &parser.DestinationTo{ + Destination: parseDestination(clauseCtx.Destination()), + } + case *antlrParser.DestinationKeptContext: + return &parser.DestinationKept{ + Range: ctxToRange(clauseCtx), + } + case *antlrParser.KeptOrDestinationContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.KeptOrDestination](clauseCtx.GetText()) + } + +} + +func parseDestinationAllotment(allotmentCtx antlrParser.IAllotmentContext) parser.AllotmentValue { + switch allotmentCtx := allotmentCtx.(type) { + case *antlrParser.RemainingAllotmentContext: + return &parser.RemainingAllotment{ + Range: ctxToRange(allotmentCtx), + } + + case *antlrParser.PortionedAllotmentContext: + expr := parseValueExpr(allotmentCtx.ValueExpr()) + return &parser.ValueExprAllotment{Value: expr} + + case *antlrParser.AllotmentContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.AllotmentValue](allotmentCtx.GetText()) + } +} + +func parseFnArgs(fnCallArgCtx antlrParser.IFunctionCallArgsContext) []parser.ValueExpr { + if fnCallArgCtx == nil { + return nil + } + + var args []parser.ValueExpr + for _, valueExpr := range fnCallArgCtx.AllValueExpr() { + args = append(args, parseValueExpr(valueExpr)) + } + return args +} + +func parseFnCall(fnCallCtx antlrParser.IFunctionCallContext) *parser.FnCall { + if fnCallCtx == nil { + return nil + } + + ident := fnCallCtx.GetFnName() + if ident == nil { + return nil + } + + allArgs := fnCallCtx.FunctionCallArgs() + + return &parser.FnCall{ + Range: ctxToRange(fnCallCtx), + Caller: &parser.FnCallIdentifier{ + Range: tokenToRange(ident), + Name: ident.GetText(), + }, + Args: parseFnArgs(allArgs), + } +} + +func parseSaveStatement(saveCtx *antlrParser.SaveStatementContext) *parser.SaveStatement { + return &parser.SaveStatement{ + Range: ctxToRange(saveCtx), + SentValue: parseSentValue(saveCtx.SentValue()), + Account: parseValueExpr(saveCtx.ValueExpr()), + } +} + +func parseStatement(statementCtx antlrParser.IStatementContext) parser.Statement { + switch statementCtx := statementCtx.(type) { + case *antlrParser.SendStatementContext: + return parseSendStatement(statementCtx) + + case *antlrParser.SaveStatementContext: + return parseSaveStatement(statementCtx) + + case *antlrParser.FnCallStatementContext: + return parseFnCall(statementCtx.FunctionCall()) + + case *antlrParser.StatementContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.Statement](statementCtx.GetText()) + } +} + +func parseSentValue(statementCtx antlrParser.ISentValueContext) parser.SentValue { + switch statementCtx := statementCtx.(type) { + case *antlrParser.SentLiteralContext: + return &parser.SentValueLiteral{ + Range: ctxToRange(statementCtx), + Monetary: parseValueExpr(statementCtx.ValueExpr()), + } + case *antlrParser.SentAllContext: + return &parser.SentValueAll{ + Range: ctxToRange(statementCtx), + Asset: parseValueExpr(statementCtx.SentAllLit().GetAsset()), + } + + case *antlrParser.SentValueContext: + return nil + + default: + return utils.NonExhaustiveMatchPanic[parser.SentValue](statementCtx.GetText()) + } + +} + +func parseSendStatement(statementCtx *antlrParser.SendStatementContext) *parser.SendStatement { + return &parser.SendStatement{ + Source: parseSource(statementCtx.Source()), + Destination: parseDestination(statementCtx.Destination()), + Range: ctxToRange(statementCtx), + SentValue: parseSentValue(statementCtx.SentValue()), + } +} + +func parseNumberLiteral(numNode antlr.TerminalNode) *parser.NumberLiteral { + amtStr := numNode.GetText() + amtStr = strings.ReplaceAll(amtStr, "_", "") + + amt, ok := new(big.Int).SetString(amtStr, 10) + if !ok { + // This should be unreachable for tokens emitted by the lexer. + // Degrade gracefully to a zero-value literal instead of panicking, + // so that a malformed token cannot crash the process (e.g. the LSP server). + amt = big.NewInt(0) + } + + return &parser.NumberLiteral{ + Range: tokenToRange(numNode.GetSymbol()), + Number: amt, + } +} + +func parseMonetaryLit(monetaryLitCtx antlrParser.IMonetaryLitContext) *parser.MonetaryLiteral { + if monetaryLitCtx.GetAmt() == nil { + return nil + } + + return &parser.MonetaryLiteral{ + Range: ctxToRange(monetaryLitCtx), + Asset: parseValueExpr(monetaryLitCtx.GetAsset()), + Amount: parseValueExpr(monetaryLitCtx.GetAmt()), + } +} + +func ctxToRange(ctx antlr.ParserRuleContext) parser.Range { + startTk := ctx.GetStart() + endTk := ctx.GetStop() + + return parser.Range{ + Start: parser.Position{ + Line: startTk.GetLine() - 1, + Character: startTk.GetColumn(), + }, + End: parser.Position{ + Line: endTk.GetLine() - 1, + + // this is based on the assumption that a token cannot span multiple lines + Character: endTk.GetColumn() + len(endTk.GetText()), + }, + } +} + +func tokenToRange(tk antlr.Token) parser.Range { + return parser.Range{ + Start: parser.Position{ + Line: tk.GetLine() - 1, + Character: tk.GetColumn(), + }, + End: parser.Position{ + Line: tk.GetLine() - 1, + Character: tk.GetColumn() + len(tk.GetText()), + }, + } +} diff --git a/internal/parser/lexer.go b/internal/parser/lexer.go new file mode 100644 index 00000000..4e60a91e --- /dev/null +++ b/internal/parser/lexer.go @@ -0,0 +1,495 @@ +package parser + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +type tokenKind uint8 + +const ( + tkEOF tokenKind = iota + + // Keywords + tkVars + tkMax + tkSource + tkDestination + tkSend + tkFrom + tkUp + tkTo + tkRemaining + tkAllowing + tkUnbounded + tkOverdraft + tkOneof + tkKept + tkSave + tkWith + tkScaling + tkThrough + tkFeature + + // Punctuation + tkLParen + tkRParen + tkLBracket + tkRBracket + tkLBrace + tkRBrace + tkComma + tkEq + tkStar + tkPlus + tkMinus + tkDiv + tkRestrict + tkHashBang + tkColon + tkAt + + // Literals + tkPercentage + tkString + tkIdentifier + tkNumber + tkAsset + tkVariable + tkAccountText +) + +var keywords = map[string]tokenKind{ + "vars": tkVars, + "max": tkMax, + "source": tkSource, + "destination": tkDestination, + "send": tkSend, + "from": tkFrom, + "up": tkUp, + "to": tkTo, + "remaining": tkRemaining, + "allowing": tkAllowing, + "unbounded": tkUnbounded, + "overdraft": tkOverdraft, + "oneof": tkOneof, + "kept": tkKept, + "save": tkSave, + "with": tkWith, + "scaling": tkScaling, + "through": tkThrough, + "feature": tkFeature, +} + +var punctuation = map[tokenKind]string{ + tkLParen: "(", + tkRParen: ")", + tkLBracket: "[", + tkRBracket: "]", + tkLBrace: "{", + tkRBrace: "}", + tkComma: ",", + tkEq: "=", + tkStar: "*", + tkPlus: "+", + tkMinus: "-", + tkDiv: "/", + tkRestrict: "\\", + tkHashBang: "#!", + tkColon: ":", + tkAt: "@", +} + +func (k tokenKind) isKeyword() bool { + return k >= tkVars && k <= tkFeature +} + +// A short human readable description, used in error messages +func (k tokenKind) String() string { + if k.isKeyword() { + for text, kind := range keywords { + if kind == k { + return "`" + text + "`" + } + } + } + if text, ok := punctuation[k]; ok { + return "`" + text + "`" + } + + switch k { + case tkEOF: + return "end of file" + case tkPercentage: + return "percentage" + case tkString: + return "string" + case tkIdentifier: + return "identifier" + case tkNumber: + return "number" + case tkAsset: + return "asset" + case tkVariable: + return "variable" + case tkAccountText: + return "account name" + default: + return "token" + } +} + +type token struct { + Kind tokenKind + Text string + Range + // byte offsets in the source + StartOffset, EndOffset int +} + +func (t token) describe() string { + switch t.Kind { + case tkEOF: + return "end of file" + case tkString: + return "string " + t.Text + default: + return "`" + t.Text + "`" + } +} + +type lexer struct { + src string + offset int + pos Position + + // true right after '@' or ':', where account name parts are lexed + accountMode bool + + tokens []token + comments []Comment + errors []ParserError +} + +func lex(src string) ([]token, []Comment, []ParserError) { + l := lexer{src: src} + l.run() + return l.tokens, l.comments, l.errors +} + +func (l *lexer) peekAt(n int) byte { + if l.offset+n >= len(l.src) { + return 0 + } + return l.src[l.offset+n] +} + +func (l *lexer) advance() { + r, size := utf8.DecodeRuneInString(l.src[l.offset:]) + l.offset += size + if r == '\n' { + l.pos.Line++ + l.pos.Character = 0 + } else { + l.pos.Character++ + } +} + +func (l *lexer) advanceWhile(pred func(byte) bool) { + for l.offset < len(l.src) && pred(l.src[l.offset]) { + l.advance() + } +} + +func (l *lexer) emit(kind tokenKind, startOffset int, startPos Position) { + l.tokens = append(l.tokens, token{ + Kind: kind, + Text: l.src[startOffset:l.offset], + Range: Range{Start: startPos, End: l.pos}, + StartOffset: startOffset, + EndOffset: l.offset, + }) +} + +func (l *lexer) errorFrom(startPos Position, msg string) { + l.errors = append(l.errors, ParserError{ + Range: Range{Start: startPos, End: l.pos}, + Msg: msg, + }) +} + +func isLower(c byte) bool { return c >= 'a' && c <= 'z' } +func isUpper(c byte) bool { return c >= 'A' && c <= 'Z' } +func isDigit(c byte) bool { return c >= '0' && c <= '9' } +func isIdentTail(c byte) bool { return isLower(c) || c == '_' } +func isVarNameStart(c byte) bool { return isLower(c) || c == '_' } +func isVarNameTail(c byte) bool { return isLower(c) || isDigit(c) || c == '_' } +func isAssetTail(c byte) bool { return isUpper(c) || isDigit(c) } +func isAccountText(c byte) bool { + return isLower(c) || isUpper(c) || isDigit(c) || c == '_' || c == '-' +} +func isWhitespace(c byte) bool { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' +} + +// Characters that cannot start any token. Consecutive invalid chars are reported as a single error. +func (l *lexer) isInvalidStart() bool { + c := l.src[l.offset] + if isWhitespace(c) || isLower(c) || isUpper(c) || isDigit(c) { + return false + } + switch c { + case '(', ')', '[', ']', '{', '}', ',', '=', '*', '+', '-', '/', '\\', ':', '@', '"', '$': + return false + case '#': + return l.peekAt(1) != '!' + default: + return true + } +} + +func (l *lexer) run() { + for { + if l.accountMode { + l.accountMode = false + l.lexAccountPart() + } + + l.advanceWhile(isWhitespace) + + startOffset, startPos := l.offset, l.pos + if l.offset >= len(l.src) { + l.emit(tkEOF, startOffset, startPos) + return + } + + c := l.src[l.offset] + switch { + case c == '/' && l.peekAt(1) == '/': + l.advanceWhile(func(c byte) bool { return c != '\n' && c != '\r' }) + l.comments = append(l.comments, Comment{ + Content: l.src[startOffset+2 : l.offset], + Range: Range{Start: startPos, End: l.pos}, + }) + + case c == '/' && l.peekAt(1) == '*': + l.lexMultilineComment(startPos) + + case isLower(c): + l.advance() + l.advanceWhile(isIdentTail) + kind, ok := keywords[l.src[startOffset:l.offset]] + if !ok { + kind = tkIdentifier + } + l.emit(kind, startOffset, startPos) + + case isUpper(c): + l.advance() + l.advanceWhile(isAssetTail) + if l.peekAt(0) == '/' && isDigit(l.peekAt(1)) { + l.advance() + l.advanceWhile(isDigit) + } + l.emit(tkAsset, startOffset, startPos) + + case isDigit(c): + l.lexNumber(startOffset, startPos) + + case c == '"': + l.lexString(startOffset, startPos) + + case c == '$': + l.lexVariable(startOffset, startPos) + + case c == '@': + l.advance() + l.emit(tkAt, startOffset, startPos) + l.accountMode = true + + case c == ':': + l.advance() + l.emit(tkColon, startOffset, startPos) + l.accountMode = true + + case c == '#' && l.peekAt(1) == '!': + l.advance() + l.advance() + l.emit(tkHashBang, startOffset, startPos) + + default: + kind, ok := singleCharTokens[c] + if ok { + l.advance() + l.emit(kind, startOffset, startPos) + continue + } + + for l.offset < len(l.src) && l.isInvalidStart() { + l.advance() + } + l.errorFrom(startPos, fmt.Sprintf("invalid character: `%s`", l.src[startOffset:l.offset])) + } + } +} + +var singleCharTokens = map[byte]tokenKind{ + '(': tkLParen, + ')': tkRParen, + '[': tkLBracket, + ']': tkRBracket, + '{': tkLBrace, + '}': tkRBrace, + ',': tkComma, + '=': tkEq, + '*': tkStar, + '+': tkPlus, + '-': tkMinus, + '/': tkDiv, + '\\': tkRestrict, +} + +// Lexes the part of an account literal right after '@' or ':'. +// No whitespace is allowed there: when no part is found, no token is emitted +// and the parser reports the missing part. +func (l *lexer) lexAccountPart() { + if l.offset >= len(l.src) { + return + } + + startOffset, startPos := l.offset, l.pos + c := l.src[l.offset] + switch { + case c == '$' && isVarNameStart(l.peekAt(1)): + l.lexVariable(startOffset, startPos) + case isAccountText(c): + l.advanceWhile(isAccountText) + l.emit(tkAccountText, startOffset, startPos) + } +} + +func (l *lexer) lexVariable(startOffset int, startPos Position) { + l.advance() // '$' + if !isVarNameStart(l.peekAt(0)) { + // Emit an (invalid) variable anyway, so that the parser knows a variable is being typed + l.emit(tkVariable, startOffset, startPos) + l.errorFrom(startPos, "invalid variable name: expected a lowercase letter or `_` after `$`") + return + } + l.advanceWhile(isVarNameTail) + l.emit(tkVariable, startOffset, startPos) +} + +func (l *lexer) lexNumber(startOffset int, startPos Position) { + l.advanceWhile(isDigit) + + // percentage: [0-9]+ ('.' [0-9]+)? '%' + if l.peekAt(0) == '%' { + l.advance() + l.emit(tkPercentage, startOffset, startPos) + return + } + if l.peekAt(0) == '.' && isDigit(l.peekAt(1)) { + i := 1 + for isDigit(l.peekAt(i)) { + i++ + } + if l.peekAt(i) == '%' { + for range i + 1 { + l.advance() + } + l.emit(tkPercentage, startOffset, startPos) + return + } + } + + // number: [0-9]+ ('_' [0-9]+)* + for l.peekAt(0) == '_' && isDigit(l.peekAt(1)) { + l.advance() + l.advanceWhile(isDigit) + } + l.emit(tkNumber, startOffset, startPos) +} + +// STRING: '"' ('\\"' | ~[\r\n"])* '"' +// +// We emulate the longest-match semantics: an escaped quote may also be the closing one. +func (l *lexer) lexString(startOffset int, startPos Position) { + end := -1 // offset right after the closing quote + i := l.offset + 1 + for i < len(l.src) { + c := l.src[i] + if c == '\n' || c == '\r' { + break + } + if c == '"' { + end = i + 1 + if l.src[i-1] != '\\' || i-1 == l.offset { + break + } + } + i++ + } + + if end == -1 { + // Unterminated string: take the rest of the line, so that the following lines are not affected + for l.offset < i { + l.advance() + } + l.emit(tkString, startOffset, startPos) + l.errorFrom(startPos, "unterminated string literal") + return + } + + for l.offset < end { + l.advance() + } + l.emit(tkString, startOffset, startPos) +} + +// MULTILINE_COMMENT: '/*' (MULTILINE_COMMENT | .)*? '*/' +// +// A nested '/*' opens a nested comment when that leads to a terminated comment, +// otherwise it's treated as plain text (this mimics the ANTLR lexer). +func (l *lexer) lexMultilineComment(startPos Position) { + memo := map[int]int{} + + // returns the offset right after the '*/' closing the comment whose content starts at i, or -1 + var matchFrom func(i int) int + matchFrom = func(start int) int { + if end, ok := memo[start]; ok { + return end + } + i := start + + end := -1 + for i < len(l.src) { + if strings.HasPrefix(l.src[i:], "*/") { + end = i + 2 + break + } + if strings.HasPrefix(l.src[i:], "/*") { + if nestedEnd := matchFrom(i + 2); nestedEnd != -1 { + if outerEnd := matchFrom(nestedEnd); outerEnd != -1 { + end = outerEnd + break + } + } + } + i++ + } + + memo[start] = end + return end + } + + end := matchFrom(l.offset + 2) + if end == -1 { + end = len(l.src) + defer l.errorFrom(startPos, "unterminated comment") + } + for l.offset < end { + l.advance() + } +} diff --git a/internal/parser/parser.go b/internal/parser/parser.go index deb00eeb..4ba7f036 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -3,14 +3,21 @@ package parser import ( "fmt" "math/big" + "slices" "strings" - - antlrParser "github.com/formancehq/numscript/internal/parser/antlrParser" - "github.com/formancehq/numscript/internal/utils" - - "github.com/antlr4-go/antlr/v4" ) +// A hand-written, error-tolerant recursive descent parser. +// +// The parser never gives up: when an expected token is missing, it reports an +// error and keeps going without consuming anything; when an unexpected token +// is found, it skips tokens until it reaches a token that some enclosing +// construct knows how to handle (see the "stop" token sets). +// This way, the AST of an incomplete program (e.g. while it's being typed) +// keeps as much information as possible. +// +// Missing nodes are represented as nil. + type ParserError struct { Range Msg string @@ -26,671 +33,1330 @@ type ParseResult struct { Errors []ParserError } -type ErrorListener struct { - antlr.DefaultErrorListener - Errors []ParserError +func Parse(input string) ParseResult { + tokens, comments, lexErrors := lex(input) + p := newParser(tokens, -1) + return p.finish(input, comments, lexErrors) } -func (l *ErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, startL, startC int, msg string, e antlr.RecognitionException) { - length := 1 - if token, ok := offendingSymbol.(antlr.Token); ok { - length = len(token.GetText()) +func ParseErrorsToString(errors []ParserError, source string) string { + buf := "Got errors while parsing:\n" + for _, err := range errors { + buf += err.Msg + "\n" + err.ShowOnSource(source) + "\n" } - endL := startL - endC := startC + length - 1 // -1 so that end character is inside the offending token - l.Errors = append(l.Errors, ParserError{ - Msg: msg, - Range: Range{ - Start: Position{Character: startC, Line: startL - 1}, - End: Position{Character: endC, Line: endL - 1}, - }, - }) + return buf } -func Parse(input string) ParseResult { - // TODO handle lexer errors - listener := &ErrorListener{} +// Returns (a, b) representing a/(10^b) +func ParsePercentageRatio(source string) (*big.Int, int, error) { + source = strings.TrimSuffix(source, "%") + + scale := 0 + if i := strings.Index(source, "."); i != -1 { + scale = len(source) - i - 1 + } + + intPart := strings.ReplaceAll(source, ".", "") + num, ok := new(big.Int).SetString(intPart, 10) + if !ok { + return nil, 0, fmt.Errorf("unexpected invalid string literal: %s", source) + } + + return num, scale, nil +} - is := antlr.NewInputStream(input) - lexer := antlrParser.NewLexer(is) - lexer.RemoveErrorListeners() - lexer.AddErrorListener(listener) +type tokenSet uint64 - stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel) +func setOf(kinds ...tokenKind) tokenSet { + var s tokenSet + for _, k := range kinds { + s |= 1 << k + } + return s +} + +func (s tokenSet) has(k tokenKind) bool { + return s&(1<= 0, infix operators at or after this token index are not consumed (used for backtracking) + stopAt int + + // --- completion + + // index of the token under the cursor, or -1 when not computing completions + cursorTok int + frozen bool + recorded bool + completion CompletionContext + fnFrames []FnCallFrame +} + +func newParser(tokens []token, cursorTok int) *parser { + return &parser{ + tokens: tokens, + lastErrorTok: -1, + stopAt: -1, + cursorTok: cursorTok, } +} + +func (p *parser) finish(input string, comments []Comment, lexErrors []ParserError) ParseResult { + program := p.parseProgram() + program.Comments = comments + + errors := append(lexErrors, p.errors...) + slices.SortStableFunc(errors, func(a, b ParserError) int { + if a.Start.Line != b.Start.Line { + return a.Start.Line - b.Start.Line + } + return a.Start.Character - b.Start.Character + }) return ParseResult{ Source: input, - Value: parsed, - Errors: listener.Errors, + Value: program, + Errors: errors, } } -func ParseErrorsToString(errors []ParserError, source string) string { - buf := "Got errors while parsing:\n" - for _, err := range errors { - buf += err.Msg + "\n" + err.ShowOnSource(source) + "\n" - } - return buf +// --- Token stream utilities + +func (p *parser) cur() token { + return p.tokens[p.pos] } -func parseVarsDeclaration(varsCtx antlrParser.IVarsDeclarationContext) *VarDeclarations { +func (p *parser) peek(n int) token { + i := min(p.pos+n, len(p.tokens)-1) + return p.tokens[i] +} - if varsCtx == nil { - return nil - } +func (p *parser) atCursor() bool { + return p.pos == p.cursorTok && !p.frozen +} - varBlock := VarDeclarations{ - Range: ctxToRange(varsCtx), +// Checks the current token kind, and records it as a possible completion +func (p *parser) at(kind tokenKind) bool { + if p.atCursor() { + p.recordCompletion() } - - for _, varDecl := range varsCtx.AllVarDeclaration() { - decl := parseVarDeclaration(varDecl) - if decl != nil { - varBlock.Declarations = append(varBlock.Declarations, *decl) + if p.atCursor() && kind.isKeyword() { + for text, k := range keywords { + if k == kind && !slices.Contains(p.completion.Keywords, text) { + p.completion.Keywords = append(p.completion.Keywords, text) + } } } - return &varBlock + return p.cur().Kind == kind +} + +func (p *parser) atExprStart(hints []string) bool { + p.recordSlot(&ExprSlot{Types: hints}) + return exprStart.has(p.cur().Kind) } -func parseProgram(programCtx antlrParser.IProgramContext) Program { - vars := parseVarsDeclaration(programCtx.VarsDeclaration()) +func (p *parser) recordSlot(slot CompletionSlot) { + if p.atCursor() { + p.recordCompletion() + p.completion.Slots = append(p.completion.Slots, slot) + } +} - featureCtx := programCtx.FeatureDecl() +func (p *parser) recordCompletion() { + if !p.recorded { + p.recorded = true + p.completion.FnCalls = slices.Clone(p.fnFrames) + } +} - var flagIdents []StringLiteral - if featureCtx != nil { - for _, flagTk := range featureCtx.GetFlag() { - flagIdents = append(flagIdents, *parseStringTk(flagTk)) +func (p *parser) bump() token { + t := p.cur() + if t.Kind == tkEOF { + return t + } + if p.cursorTok >= 0 && p.pos >= p.cursorTok { + if p.atCursor() { + p.recordCompletion() } + p.frozen = true } + p.pos++ + p.prevEnd = t.End + return t +} - var statements []Statement - for _, statementCtx := range programCtx.AllStatement() { - statements = append(statements, parseStatement(statementCtx)) +func (p *parser) eat(kind tokenKind) bool { + if p.at(kind) { + p.bump() + return true } + return false +} - return Program{ - Flags: flagIdents, - Statements: statements, - Vars: vars, +func (p *parser) expect(kind tokenKind) bool { + if p.eat(kind) { + return true } + p.errExpected(kind.String()) + return false } -func parseVarDeclaration(varDecl antlrParser.IVarDeclarationContext) *VarDeclaration { - if varDecl == nil { - return nil +// Like expect, but when the token is missing it skips tokens until either the +// expected token or a token in the stop set is found +func (p *parser) expectRecover(kind tokenKind, stop tokenSet) bool { + if p.eat(kind) { + return true } + p.errExpected(kind.String()) + p.skipUntil(stop | setOf(kind)) + return p.eat(kind) +} - var origin *ValueExpr - if varDecl.VarOrigin() != nil { - expr := parseValueExpr(varDecl.VarOrigin().ValueExpr()) - // when the origin expression is missing or malformed, - // parseValueExpr returns nil; do not wrap it in a non-nil pointer - if expr != nil { - origin = &expr +// Skip tokens until one in the given set is found. Balanced groups are skipped as a whole. +func (p *parser) skipUntil(stop tokenSet) { + depth := 0 + for { + kind := p.cur().Kind + if kind == tkEOF || (depth == 0 && stop.has(kind)) || stmtStop.has(kind) { + return + } + switch kind { + case tkLParen, tkLBrace, tkLBracket: + depth++ + case tkRParen, tkRBrace, tkRBracket: + if depth > 0 { + depth-- + } } + p.bump() } +} - return &VarDeclaration{ - Range: ctxToRange(varDecl), - Type: parseVarType(varDecl.GetType_()), - Name: parseVarLiteral(varDecl.GetName()), - Origin: origin, - } +func (p *parser) sameLine() bool { + return p.cur().Start.Line == p.prevEnd.Line } -func parseVarLiteral(tk antlr.Token) *Variable { - if tk == nil || tk.GetTokenIndex() == -1 { - return nil +func (p *parser) rangeFrom(start Position) Range { + end := p.prevEnd + if end.Line < start.Line || (end.Line == start.Line && end.Character < start.Character) { + end = start } + return Range{Start: start, End: end} +} - name := tk.GetText()[1:] +// --- Errors - return &Variable{ - Range: tokenToRange(tk), - Name: name, +func (p *parser) report(rng Range, msg string) { + if p.cursorTok >= 0 && p.pos >= p.cursorTok { + p.frozen = true } -} + if p.lastErrorTok == p.pos { + return + } + p.lastErrorTok = p.pos -func parseVarType(tk antlr.Token) *TypeDecl { - if tk == nil || tk.GetTokenIndex() == -1 { - return nil + if rng.Start == rng.End { + rng.End.Character++ } + p.errors = append(p.errors, ParserError{Range: rng, Msg: msg}) +} + +func (p *parser) errExpected(what string) { + cur := p.cur() + msg := fmt.Sprintf("expected %s, found %s", what, cur.describe()) - return &TypeDecl{ - Range: tokenToRange(tk), - Name: tk.GetText(), + // When the unexpected token is on a different line, the error is likely + // about something missing at the end of the previous line (unless the token + // is an identifier, which is likely a typo) + rng := cur.Range + if p.pos > 0 && !p.sameLine() && cur.Kind != tkIdentifier { + rng = Range{Start: p.prevEnd, End: p.prevEnd} } + p.report(rng, msg) } -func parseColorConstraint(colorConstraintCtx antlrParser.IColorConstraintContext) ValueExpr { - if colorConstraintCtx == nil { - return nil +// Reports an error on the current token, which is about to be skipped +func (p *parser) errUnexpected(what string) { + cur := p.cur() + p.report(cur.Range, fmt.Sprintf("expected %s, found %s", what, cur.describe())) +} + +// --- Backtracking + +type checkpoint struct { + pos int + nErrors int + lastErrorTok int + prevEnd Position + frozen bool + recorded bool + completion CompletionContext +} + +func (p *parser) checkpoint() checkpoint { + return checkpoint{ + pos: p.pos, + nErrors: len(p.errors), + lastErrorTok: p.lastErrorTok, + prevEnd: p.prevEnd, + frozen: p.frozen, + recorded: p.recorded, + completion: p.completion, } +} - return parseValueExpr( - colorConstraintCtx.ValueExpr(), - ) +func (p *parser) restore(c checkpoint) { + p.pos = c.pos + p.errors = p.errors[:c.nErrors] + p.lastErrorTok = c.lastErrorTok + p.prevEnd = c.prevEnd + p.frozen = c.frozen + p.recorded = c.recorded + p.completion = c.completion } -func parseSource(sourceCtx antlrParser.ISourceContext) Source { - if sourceCtx == nil { - return nil +// --- Program + +func (p *parser) parseProgram() Program { + var program Program + + if p.at(tkHashBang) { + program.Flags = p.parseFeatureDecl() } - range_ := ctxToRange(sourceCtx) + if p.at(tkVars) { + program.Vars = p.parseVars() + } - switch sourceCtx := sourceCtx.(type) { - case *antlrParser.SrcAccountContext: - return &SourceAccount{ - Range: range_, - Color: parseColorConstraint(sourceCtx.ColorConstraint()), - ValueExpr: parseValueExpr(sourceCtx.ValueExpr()), - } + for { + p.recordSlot(&StatementSlot{}) + isSend := p.at(tkSend) + isSave := p.at(tkSave) + isEOF := p.at(tkEOF) - case *antlrParser.SrcInorderContext: - var sources []Source - for _, sourceCtx := range sourceCtx.AllSource() { - sources = append(sources, parseSource(sourceCtx)) - } - return &SourceInorder{ - Range: range_, - Sources: sources, - } + switch { + case isSend: + program.Statements = append(program.Statements, p.parseSend()) - case *antlrParser.SrcOneofContext: - var sources []Source - for _, sourceCtx := range sourceCtx.AllSource() { - sources = append(sources, parseSource(sourceCtx)) - } - return &SourceOneof{ - Range: range_, - Sources: sources, - } + case isSave: + program.Statements = append(program.Statements, p.parseSave()) - case *antlrParser.SrcAllotmentContext: - var items []SourceAllotmentItem - for _, itemCtx := range sourceCtx.AllAllotmentClauseSrc() { - item := SourceAllotmentItem{ - Range: ctxToRange(itemCtx), - Allotment: parseAllotment(itemCtx.Allotment()), - From: parseSource(itemCtx.Source()), + case p.atFnCall(): + program.Statements = append(program.Statements, p.parseFnCall()) + + case isEOF: + return program + + default: + if p.cur().Kind == tkVars { + p.report(p.cur().Range, "the vars block must be declared before any statement") + } else { + p.errUnexpected("a statement") } - items = append(items, item) + p.bump() + p.skipUntilStatement() } - return &SourceAllotment{ - Range: range_, - Items: items, + } +} + +func (p *parser) atFnCall() bool { + kind := p.cur().Kind + return (kind == tkIdentifier || kind == tkOverdraft) && p.peek(1).Kind == tkLParen +} + +func (p *parser) skipUntilStatement() { + for !stmtStop.has(p.cur().Kind) && !p.atFnCall() { + p.bump() + } +} + +// '#!' '[' 'feature' '(' (STRING (',' STRING)*)? ')' ']' +func (p *parser) parseFeatureDecl() []StringLiteral { + p.bump() + + var flags []StringLiteral + if !p.expect(tkLBracket) || !p.expect(tkFeature) || !p.expect(tkLParen) { + p.skipUntil(setOf(tkRBracket)) + p.eat(tkRBracket) + return flags + } + + if p.cur().Kind == tkString { + for { + if !p.at(tkString) { + p.errExpected("a string") + break + } + flags = append(flags, *parseString(p.bump())) + if !p.eat(tkComma) { + break + } } + } + + p.expect(tkRParen) + p.expectRecover(tkRBracket, stmtStop) + return flags +} + +// 'vars' '{' varDeclaration* '}' +func (p *parser) parseVars() *VarDeclarations { + start := p.bump().Start + vars := &VarDeclarations{} + + if p.expect(tkLBrace) { + for { + p.recordSlot(&VarTypeSlot{}) + if p.at(tkIdentifier) && p.peek(1).Kind != tkLParen { + vars.Declarations = append(vars.Declarations, p.parseVarDeclaration()) + continue + } + + if p.eat(tkRBrace) { + break + } + + if stmtStop.has(p.cur().Kind) || p.atFnCall() { + p.errExpected("`}`") + break + } - case *antlrParser.SrcCappedContext: - return &SourceCapped{ - Range: range_, - From: parseSource(sourceCtx.Source()), - Cap: parseValueExpr(sourceCtx.GetCap_()), + p.errUnexpected("a variable declaration or `}`") + p.bump() } + } + + vars.Range = p.rangeFrom(start) + return vars +} - case *antlrParser.SrcAccountUnboundedOverdraftContext: - return &SourceOverdraft{ - Range: ctxToRange(sourceCtx), - Color: parseColorConstraint(sourceCtx.ColorConstraint()), - Address: parseValueExpr(sourceCtx.GetAddress()), +// IDENTIFIER VARIABLE_NAME ('=' valueExpr)? +func (p *parser) parseVarDeclaration() VarDeclaration { + typeTk := p.bump() + decl := VarDeclaration{ + Type: &TypeDecl{Range: typeTk.Range, Name: typeTk.Text}, + } + + if p.at(tkVariable) { + decl.Name = parseVariable(p.bump()) + } else { + p.errExpected("a variable name") + } + + if p.eat(tkEq) { + var hints []string + if typeTk.Text != "" { + hints = []string{typeTk.Text} } - case *antlrParser.SrcAccountWithScalingContext: - return &SourceWithScaling{ - Range: ctxToRange(sourceCtx), - Address: parseValueExpr(sourceCtx.GetAddress()), - Through: parseValueExpr(sourceCtx.GetSwap()), + origin := p.parseExpr(hints) + if origin != nil { + decl.Origin = &origin } + } - case *antlrParser.SrcAccountBoundedOverdraftContext: - varMon := parseValueExpr(sourceCtx.GetMaxOvedraft()) + decl.Range = p.rangeFrom(typeTk.Start) + return decl +} - return &SourceOverdraft{ - Range: ctxToRange(sourceCtx), - Color: parseColorConstraint(sourceCtx.ColorConstraint()), - Address: parseValueExpr(sourceCtx.GetAddress()), - Bounded: &varMon, - } +// --- Statements - case *antlrParser.SourceContext: - return nil +// 'send' sentValue '(' 'source' '=' source 'destination' '=' destination ')' +func (p *parser) parseSend() *SendStatement { + start := p.bump().Start + stmt := &SendStatement{ + SentValue: p.parseSentValue(), + } - default: - return utils.NonExhaustiveMatchPanic[Source](sourceCtx.GetText()) + p.expectRecover(tkLParen, setOf(tkSource, tkDestination, tkRParen)) + + if p.expectKeywordClause(tkSource) { + stmt.Source = p.parseSource(setOf(tkDestination, tkRParen), 0) + } + + if !p.at(tkDestination) { + p.errExpected(tkDestination.String()) + p.skipUntil(setOf(tkDestination, tkRParen) | p.misspelledKeywordSet()) } + + if p.expectKeywordClause(tkDestination) { + stmt.Destination = p.parseDestination(setOf(tkRParen), 0) + } + + p.expectRecover(tkRParen, 0) + + stmt.Range = p.rangeFrom(start) + return stmt } -// Returns (a, b) representing a/(10^b) -func ParsePercentageRatio(source string) (*big.Int, int, error) { - source = strings.TrimSuffix(source, "%") +func (p *parser) misspelledKeywordSet() tokenSet { + if p.cur().Kind == tkIdentifier && p.peek(1).Kind == tkEq { + return setOf(tkIdentifier) + } + return 0 +} - scale := 0 - if i := strings.Index(source, "."); i != -1 { - scale = len(source) - i - 1 +// Parses ` =`. A misspelled keyword (an identifier followed by '=') is +// reported but accepted, so that the rest of the clause can still be parsed. +func (p *parser) expectKeywordClause(keyword tokenKind) bool { + if !p.at(keyword) { + if p.cur().Kind == tkIdentifier && p.peek(1).Kind == tkEq { + p.errExpected(keyword.String()) + p.bump() + } else if p.cur().Kind == tkEq { + p.errExpected(keyword.String()) + } else { + p.errExpected(keyword.String()) + return false + } + } else { + p.bump() } - intPart := strings.ReplaceAll(source, ".", "") - num, ok := new(big.Int).SetString(intPart, 10) - if !ok { - return nil, 0, fmt.Errorf("unexpected invalid string literal: %s", source) + p.expect(tkEq) + return true +} + +// 'save' sentValue 'from' valueExpr +func (p *parser) parseSave() *SaveStatement { + start := p.bump().Start + stmt := &SaveStatement{ + SentValue: p.parseSentValue(), } - return num, scale, nil + if p.expect(tkFrom) || (p.sameLine() && exprStart.has(p.cur().Kind)) { + stmt.Account = p.parseExpr(hintAccount) + } + + stmt.Range = p.rangeFrom(start) + return stmt } -func parsePercentageRatio(source string, range_ Range) *PercentageLiteral { - num, floatingDigits, err := ParsePercentageRatio(source) - if err != nil { - // This should be unreachable for tokens emitted by the lexer. - // Degrade gracefully to a zero-value literal instead of panicking, - // so that a malformed token cannot crash the process (e.g. the LSP server). - return &PercentageLiteral{ - Range: range_, - Amount: big.NewInt(0), +// valueExpr | '[' valueExpr '*' ']' +func (p *parser) parseSentValue() SentValue { + isBracket := p.at(tkLBracket) + isExpr := p.atExprStart(hintMonetary) + + if isBracket { + start := p.cur().Start + lit, sendAll := p.parseBracketLiteral(true) + if sendAll != nil { + return sendAll } + expr := p.parseBinaryRest(start, lit, 0, hintMonetary) + return &SentValueLiteral{Range: expr.GetRange(), Monetary: expr} } - return &PercentageLiteral{ - Range: range_, - Amount: num, - FloatingDigits: floatingDigits, + if isExpr { + expr := p.parseExpr(hintMonetary) + if expr == nil { + return nil + } + return &SentValueLiteral{Range: expr.GetRange(), Monetary: expr} } + + p.errExpected("a monetary value") + return nil +} + +// --- Sources + +func (p *parser) atSourceStart() bool { + isBrace := p.at(tkLBrace) + isOneof := p.at(tkOneof) + isMax := p.at(tkMax) + isExpr := p.atExprStart(hintAccount) + return isBrace || isOneof || isMax || isExpr } -func parseAllotment(allotmentCtx antlrParser.IAllotmentContext) AllotmentValue { - switch allotmentCtx := allotmentCtx.(type) { - case *antlrParser.PortionedAllotmentContext: - expr := parseValueExpr(allotmentCtx.ValueExpr()) - return &ValueExprAllotment{expr} +func (p *parser) parseSource(stop tokenSet, follow tokenSet) Source { + start := p.cur().Start - case *antlrParser.RemainingAllotmentContext: - return &RemainingAllotment{ - Range: ctxToRange(allotmentCtx), + isBrace := p.at(tkLBrace) + isOneof := p.at(tkOneof) + isMax := p.at(tkMax) + isExpr := p.atExprStart(hintAccount) + + switch { + case isBrace: + return p.parseBraceSource(stop) + + case isOneof: + p.bump() + src := &SourceOneof{} + if p.expect(tkLBrace) { + if p.at(tkRBrace) { + p.errExpected("a source") + } + src.Sources = p.parseInorderSources(nil, stop) } + src.Range = p.rangeFrom(start) + return src - case *antlrParser.AllotmentContext: - return nil + case isMax: + p.bump() + src := &SourceCapped{ + Cap: p.parseExpr(hintMonetary), + } + if p.expect(tkFrom) || p.sameLine() { + if p.atSourceStart() { + src.From = p.parseSource(stop, follow) + } else { + p.errExpected("a source") + } + } + src.Range = p.rangeFrom(start) + return src + + case isExpr: + addr := p.parseExprBeforeAllotment(hintAccount, follow) + return p.parseSourceAccountSuffix(start, addr, follow) default: - return utils.NonExhaustiveMatchPanic[AllotmentValue](allotmentCtx.GetText()) + p.errExpected("a source") + return nil } } -func parseStringTk(stringCtx antlr.Token) *StringLiteral { - rawStr := stringCtx.GetText() - // Remove leading and trailing '"' - innerStr := rawStr[1 : len(rawStr)-1] - return &StringLiteral{ - Range: tokenToRange(stringCtx), - String: innerStr, +// Parses what follows the address of a source account: +// +// ('\' valueExpr)? ('allowing' 'unbounded' 'overdraft' | 'allowing' 'overdraft' 'up' 'to' valueExpr)? +// | 'with' 'scaling' 'through' valueExpr +func (p *parser) parseSourceAccountSuffix(start Position, addr ValueExpr, follow tokenSet) Source { + var color ValueExpr + if p.eat(tkRestrict) { + color = p.parseExprBeforeAllotment(hintString, follow) } -} -func parseValueExpr(valueExprCtx antlrParser.IValueExprContext) ValueExpr { - switch valueExprCtx := valueExprCtx.(type) { - case *antlrParser.ParenthesizedExprContext: - return parseValueExpr(valueExprCtx.ValueExpr()) + if p.eat(tkAllowing) { + if p.eat(tkUnbounded) { + p.expect(tkOverdraft) + return &SourceOverdraft{ + Range: p.rangeFrom(start), + Color: color, + Address: addr, + } + } + + if p.eat(tkOverdraft) { + p.expect(tkUp) + p.expect(tkTo) + bound := p.parseExprBeforeAllotment(hintMonetary, follow) + return &SourceOverdraft{ + Range: p.rangeFrom(start), + Color: color, + Address: addr, + Bounded: &bound, + } + } - case *antlrParser.AccountLiteralContext: - litRng := ctxToRange(valueExprCtx) + p.errExpected("`unbounded` or `overdraft`") + } else if p.at(tkWith) { + withTk := p.bump() + if color != nil { + p.report(withTk.Range, "a color restriction cannot be used together with scaling") + } + p.expect(tkScaling) + p.expect(tkThrough) + through := p.parseExprBeforeAllotment(hintAccount, follow) + return &SourceWithScaling{ + Range: p.rangeFrom(start), + Address: addr, + Through: through, + } + } - var parts []AccountNamePart - for _, accLit := range valueExprCtx.GetChildren() { - switch accLit := accLit.(type) { - case *antlrParser.AccountTextPartContext: - varPartText := accLit.GetText() - parts = append(parts, AccountTextPart{Name: varPartText}) - case *antlrParser.AccountVarPartContext: - v := parseVarLiteral(accLit.VARIABLE_NAME_ACC().GetSymbol()) - parts = append(parts, v) + if addr == nil && color == nil { + return nil + } - case antlr.TerminalNode: - if accLit.GetSymbol().GetTokenType() == antlrParser.NumscriptParserCOLON { - parts = append(parts, AccountTextPart{Name: ":"}) - } - } + return &SourceAccount{ + Range: p.rangeFrom(start), + Color: color, + ValueExpr: addr, + } +} +// Parses either an allotment or an inorder source, starting from the '{' +func (p *parser) parseBraceSource(stop tokenSet) Source { + start := p.bump().Start + + isRemaining := p.at(tkRemaining) + isExpr := p.atExprStart([]string{"portion", "account"}) + + if isRemaining { + return p.parseSourceAllotment(start, nil, Position{}, stop) + } + + if isExpr { + exprStart := p.cur().Start + expr := p.parseExpr([]string{"portion", "account"}) + if p.at(tkFrom) { + return p.parseSourceAllotment(start, expr, exprStart, stop) } - return &AccountInterpLiteral{ - Range: litRng, - Parts: parts, + var sources []Source + first := p.parseSourceAccountSuffix(exprStart, expr, 0) + if first != nil { + sources = append(sources, first) + } + return &SourceInorder{ + Sources: p.parseInorderSources(sources, stop), + Range: p.rangeFrom(start), } + } - case *antlrParser.MonetaryLiteralContext: - return parseMonetaryLit(valueExprCtx.MonetaryLit()) + return &SourceInorder{ + Sources: p.parseInorderSources(nil, stop), + Range: p.rangeFrom(start), + } +} - case *antlrParser.AssetLiteralContext: - return &AssetLiteral{ - Range: ctxToRange(valueExprCtx), - Asset: valueExprCtx.GetText(), +// source* '}' +func (p *parser) parseInorderSources(sources []Source, stop tokenSet) []Source { + for { + if p.atSourceStart() { + src := p.parseSource(stop|setOf(tkRBrace), 0) + if src != nil { + sources = append(sources, src) + } + continue } - case *antlrParser.NumberLiteralContext: - return parseNumberLiteral(valueExprCtx.NUMBER()) - - case *antlrParser.PercentagePortionLiteralContext: - return parsePercentageRatio(valueExprCtx.GetText(), ctxToRange(valueExprCtx)) + if p.eat(tkRBrace) { + return sources + } - case *antlrParser.VariableExprContext: - return variableLiteralFromCtx(valueExprCtx) + if stop.has(p.cur().Kind) || stmtStop.has(p.cur().Kind) { + p.errExpected("`}`") + return sources + } - case *antlrParser.StringLiteralContext: - return parseStringTk(valueExprCtx.STRING().GetSymbol()) + p.errUnexpected("a source or `}`") + p.bump() + } +} - case *antlrParser.PrefixExprContext: - return &Prefix{ - Range: ctxToRange(valueExprCtx), - Operator: PrefixOperator(valueExprCtx.GetOp().GetText()), - Expr: parseValueExpr(valueExprCtx.ValueExpr()), +// (allotment 'from' source)+ '}' +func (p *parser) parseSourceAllotment(start Position, first ValueExpr, firstStart Position, stop tokenSet) *SourceAllotment { + src := &SourceAllotment{} + inner := stop | setOf(tkRBrace) + + parseItem := func(itemStart Position, allotment AllotmentValue) { + item := SourceAllotmentItem{Allotment: allotment} + if p.expect(tkFrom) || p.sameLine() { + if p.atSourceStart() { + item.From = p.parseSource(inner, setOf(tkFrom)) + } else { + p.errExpected("a source") + } } + item.Range = p.rangeFrom(itemStart) + src.Items = append(src.Items, item) + } - case *antlrParser.InfixExprContext: - return &BinaryInfix{ - Range: ctxToRange(valueExprCtx), - Operator: InfixOperator(valueExprCtx.GetOp().GetText()), - Left: parseValueExpr(valueExprCtx.GetLeft()), - Right: parseValueExpr(valueExprCtx.GetRight()), + if first != nil { + parseItem(firstStart, &ValueExprAllotment{Value: first}) + } + + for { + isRemaining := p.at(tkRemaining) + isExpr := p.atExprStart(hintPortion) + if isRemaining || isExpr { + itemStart := p.cur().Start + parseItem(itemStart, p.parseAllotment()) + continue } - case nil, *antlrParser.ValueExprContext: - return nil + if p.eat(tkRBrace) { + break + } - case *antlrParser.ApplicationContext: - return parseFnCall(valueExprCtx.FunctionCall()) + if stop.has(p.cur().Kind) || stmtStop.has(p.cur().Kind) { + p.errExpected("`}`") + break + } - default: - return utils.NonExhaustiveMatchPanic[ValueExpr](valueExprCtx.GetText()) + p.errUnexpected("an allotment or `}`") + p.bump() } + + src.Range = p.rangeFrom(start) + return src } -func variableLiteralFromCtx(ctx antlr.ParserRuleContext) *Variable { - // Discard the '$' - name := ctx.GetText()[1:] +// Pre: at 'remaining' or at the beginning of an expression +func (p *parser) parseAllotment() AllotmentValue { + if p.at(tkRemaining) { + return &RemainingAllotment{Range: p.bump().Range} + } - return &Variable{ - Range: ctxToRange(ctx), - Name: name, + expr := p.parseExpr(hintPortion) + if expr == nil { + return nil } + return &ValueExprAllotment{Value: expr} } -func parseDestination(destCtx antlrParser.IDestinationContext) Destination { - if destCtx == nil { +// --- Destinations + +func (p *parser) parseDestination(stop tokenSet, follow tokenSet) Destination { + start := p.cur().Start + + isBrace := p.at(tkLBrace) + isOneof := p.at(tkOneof) + isExpr := p.atExprStart(hintAccount) + + switch { + case isBrace: + p.bump() + isMax := p.at(tkMax) + isRemaining := p.at(tkRemaining) + isExpr := p.atExprStart(hintPortion) + + if isMax || (!isRemaining && !isExpr) { + clauses, remaining := p.parseInorderClauses(stop) + return &DestinationInorder{ + Clauses: clauses, + Remaining: remaining, + Range: p.rangeFrom(start), + } + } + + return p.parseDestinationAllotment(start, stop) + + case isOneof: + p.bump() + dest := &DestinationOneof{} + if p.expect(tkLBrace) { + dest.Clauses, dest.Remaining = p.parseInorderClauses(stop) + } + dest.Range = p.rangeFrom(start) + return dest + + case isExpr: + expr := p.parseExprBeforeAllotment(hintAccount, follow) + if expr == nil { + return nil + } + return &DestinationAccount{ValueExpr: expr} + + default: + p.errExpected("a destination") return nil } +} - range_ := ctxToRange(destCtx) +// ('max' valueExpr keptOrDestination)* 'remaining' keptOrDestination '}' +func (p *parser) parseInorderClauses(stop tokenSet) ([]CappedKeptOrDestination, KeptOrDestination) { + var clauses []CappedKeptOrDestination + inner := stop | setOf(tkRBrace, tkMax, tkRemaining) - switch destCtx := destCtx.(type) { - case *antlrParser.DestAccountContext: - return &DestinationAccount{ - ValueExpr: parseValueExpr(destCtx.ValueExpr()), + for { + if p.at(tkMax) { + start := p.bump().Start + clause := CappedKeptOrDestination{ + Cap: p.parseExpr(hintMonetary), + } + clause.To = p.parseKeptOrDestination(inner, 0) + clause.Range = p.rangeFrom(start) + clauses = append(clauses, clause) + continue } - case *antlrParser.DestInorderContext: - var inorderClauses []CappedKeptOrDestination - for _, destInorderClause := range destCtx.AllDestinationInOrderClause() { - inorderClauses = append(inorderClauses, parseDestinationInorderClause(destInorderClause)) + if p.eat(tkRemaining) { + remaining := p.parseKeptOrDestination(stop|setOf(tkRBrace), 0) + p.expectRecover(tkRBrace, stop) + return clauses, remaining } - return &DestinationInorder{ - Range: range_, - Clauses: inorderClauses, - Remaining: parseKeptOrDestination(destCtx.KeptOrDestination()), + if p.at(tkRBrace) { + p.errExpected("`max` or `remaining`") + p.bump() + return clauses, nil } - case *antlrParser.DestOneofContext: - var inorderClauses []CappedKeptOrDestination - for _, destInorderClause := range destCtx.AllDestinationInOrderClause() { - inorderClauses = append(inorderClauses, parseDestinationInorderClause(destInorderClause)) + if stop.has(p.cur().Kind) || stmtStop.has(p.cur().Kind) { + p.errExpected("`remaining`") + return clauses, nil } - return &DestinationOneof{ - Range: range_, - Clauses: inorderClauses, - Remaining: parseKeptOrDestination(destCtx.KeptOrDestination()), - } + p.errUnexpected("`max` or `remaining`") + p.bump() + } +} - case *antlrParser.DestAllotmentContext: - var items []DestinationAllotmentItem - for _, itemCtx := range destCtx.AllAllotmentClauseDest() { +// (allotment keptOrDestination)+ '}' +func (p *parser) parseDestinationAllotment(start Position, stop tokenSet) *DestinationAllotment { + dest := &DestinationAllotment{} + inner := stop | setOf(tkRBrace) + + for { + isRemaining := p.at(tkRemaining) + isExpr := p.atExprStart(hintPortion) + if isRemaining || isExpr { + itemStart := p.cur().Start item := DestinationAllotmentItem{ - Range: ctxToRange(itemCtx), - Allotment: parseDestinationAllotment(itemCtx.Allotment()), - To: parseKeptOrDestination(itemCtx.KeptOrDestination()), + Allotment: p.parseAllotment(), } - items = append(items, item) + item.To = p.parseKeptOrDestination(inner, setOf(tkTo, tkKept)) + item.Range = p.rangeFrom(itemStart) + dest.Items = append(dest.Items, item) + continue } - return &DestinationAllotment{ - Range: range_, - Items: items, + + if p.eat(tkRBrace) { + break } - case *antlrParser.DestinationContext: - return nil + if stop.has(p.cur().Kind) || stmtStop.has(p.cur().Kind) { + p.errExpected("`}`") + break + } - default: - return utils.NonExhaustiveMatchPanic[Destination](destCtx.GetText()) + p.errUnexpected("an allotment or `}`") + p.bump() } + dest.Range = p.rangeFrom(start) + return dest } -func parseDestinationInorderClause(clauseCtx antlrParser.IDestinationInOrderClauseContext) CappedKeptOrDestination { - return CappedKeptOrDestination{ - Range: ctxToRange(clauseCtx), - Cap: parseValueExpr(clauseCtx.ValueExpr()), - To: parseKeptOrDestination(clauseCtx.KeptOrDestination()), +// 'to' destination | 'kept' +func (p *parser) parseKeptOrDestination(stop tokenSet, follow tokenSet) KeptOrDestination { + isTo := p.at(tkTo) + isKept := p.at(tkKept) + + switch { + case isTo: + p.bump() + return &DestinationTo{ + Destination: p.parseDestination(stop, follow), + } + + case isKept: + return &DestinationKept{Range: p.bump().Range} + + default: + p.errExpected("`to` or `kept`") + return nil } } -func parseKeptOrDestination(clauseCtx antlrParser.IKeptOrDestinationContext) KeptOrDestination { - if clauseCtx == nil { +// --- Expressions + +func infixPrecedence(kind tokenKind) int { + switch kind { + case tkPlus, tkMinus: + return 1 + case tkDiv: + return 2 + default: + return 0 + } +} + +// Parses an expression, reporting an error (and returning nil) when no expression is found +func (p *parser) parseExpr(hints []string) ValueExpr { + return p.parseBinary(1, hints) +} + +func (p *parser) parseBinary(minPrecedence int, hints []string) ValueExpr { + if !p.atExprStart(hints) { + p.errExpected("an expression") return nil } - switch clauseCtx := clauseCtx.(type) { - case *antlrParser.DestinationToContext: - return &DestinationTo{ - Destination: parseDestination(clauseCtx.Destination()), + start := p.cur().Start + left := p.parseUnary(hints) + return p.parseBinaryRest(start, left, minPrecedence, hints) +} + +// Precedence climbing: all the operators are left-associative +func (p *parser) parseBinaryRest(start Position, left ValueExpr, minPrecedence int, hints []string) ValueExpr { + for { + if p.stopAt >= 0 && p.pos >= p.stopAt { + return left } - case *antlrParser.DestinationKeptContext: - return &DestinationKept{ - Range: ctxToRange(clauseCtx), + + opTk := p.cur() + precedence := infixPrecedence(opTk.Kind) + if precedence == 0 || precedence < minPrecedence { + return left } - case *antlrParser.KeptOrDestinationContext: - return nil + p.bump() - default: - return utils.NonExhaustiveMatchPanic[KeptOrDestination](clauseCtx.GetText()) + rightHints := hints + if opTk.Kind == tkDiv { + rightHints = hintNumber + } + right := p.parseBinary(precedence+1, rightHints) + + left = &BinaryInfix{ + Range: p.rangeFrom(start), + Operator: InfixOperator(opTk.Text), + Left: left, + Right: right, + } + } +} + +// '-' valueExpr | primary +func (p *parser) parseUnary(hints []string) ValueExpr { + if p.cur().Kind != tkMinus { + return p.parsePrimary(hints) + } + + opTk := p.bump() + var expr ValueExpr + if p.atExprStart(hints) { + expr = p.parseUnary(hints) + } else { + p.errExpected("an expression") } + return &Prefix{ + Range: p.rangeFrom(opTk.Start), + Operator: PrefixOperator(opTk.Text), + Expr: expr, + } } -func parseDestinationAllotment(allotmentCtx antlrParser.IAllotmentContext) AllotmentValue { - switch allotmentCtx := allotmentCtx.(type) { - case *antlrParser.RemainingAllotmentContext: - return &RemainingAllotment{ - Range: ctxToRange(allotmentCtx), +// Pre: the current token is in the exprStart set +func (p *parser) parsePrimary(hints []string) ValueExpr { + tk := p.cur() + switch tk.Kind { + case tkVariable: + v := parseVariable(p.bump()) + if v == nil { + return nil } + return v + + case tkAsset: + p.bump() + return &AssetLiteral{Range: tk.Range, Asset: tk.Text} + + case tkString: + return parseString(p.bump()) + + case tkNumber: + return parseNumberLiteral(p.bump()) + + case tkPercentage: + p.bump() + return parsePercentageRatio(tk.Text, tk.Range) - case *antlrParser.PortionedAllotmentContext: - expr := parseValueExpr(allotmentCtx.ValueExpr()) - return &ValueExprAllotment{Value: expr} + case tkAt: + return p.parseAccountLiteral() - case *antlrParser.AllotmentContext: + case tkLBracket: + lit, _ := p.parseBracketLiteral(false) + return lit + + case tkLParen: + p.bump() + expr := p.parseExpr(hints) + p.expect(tkRParen) + return expr + + case tkIdentifier, tkOverdraft: + if p.peek(1).Kind == tkLParen { + return p.parseFnCall() + } + p.errUnexpected("an expression") + p.bump() return nil default: - return utils.NonExhaustiveMatchPanic[AllotmentValue](allotmentCtx.GetText()) + p.errExpected("an expression") + return nil } } -func parseFnArgs(fnCallArgCtx antlrParser.IFunctionCallArgsContext) []ValueExpr { - if fnCallArgCtx == nil { +// '@' accountPart (':' accountPart)* +func (p *parser) parseAccountLiteral() ValueExpr { + atTk := p.bump() + slot := &AccountPartSlot{LiteralStart: atTk.Start} + + var parts []AccountNamePart + p.parseAccountPart(&parts, slot) + for p.cur().Kind == tkColon { + p.bump() + parts = append(parts, AccountTextPart{Name: ":"}) + p.parseAccountPart(&parts, slot) + } + + if len(parts) == 0 { return nil } - var args []ValueExpr - for _, valueExpr := range fnCallArgCtx.AllValueExpr() { - args = append(args, parseValueExpr(valueExpr)) + return &AccountInterpLiteral{ + Range: p.rangeFrom(atTk.Start), + Parts: parts, } - return args } -func parseFnCall(fnCallCtx antlrParser.IFunctionCallContext) *FnCall { - if fnCallCtx == nil { - return nil - } +func (p *parser) parseAccountPart(parts *[]AccountNamePart, slot *AccountPartSlot) { + p.recordSlot(slot) - ident := fnCallCtx.GetFnName() - if ident == nil { - return nil + tk := p.cur() + // Account parts must not be separated by whitespace from the preceding '@' or ':' + adjacent := p.pos > 0 && p.tokens[p.pos-1].EndOffset == tk.StartOffset + + switch { + case tk.Kind == tkAccountText: + p.bump() + *parts = append(*parts, AccountTextPart{Name: tk.Text}) + + case tk.Kind == tkVariable && adjacent: + v := parseVariable(p.bump()) + if v != nil { + *parts = append(*parts, v) + } + + default: + p.errExpected("an account name") } +} - allArgs := fnCallCtx.FunctionCallArgs() +// Parses either a monetary literal ('[' valueExpr valueExpr ']') +// or, when allowSendAll is true, a sendAll literal ('[' valueExpr '*' ']') +func (p *parser) parseBracketLiteral(allowSendAll bool) (ValueExpr, *SentValueAll) { + openTk := p.bump() + start := openTk.Start - return &FnCall{ - Range: ctxToRange(fnCallCtx), - Caller: &FnCallIdentifier{ - Range: tokenToRange(ident), - Name: ident.GetText(), + asset := p.parseJuxtaposedExpr(hintAsset, + func() bool { + kind := p.cur().Kind + return !exprStart.has(kind) && (!allowSendAll || kind != tkStar) }, - Args: parseFnArgs(allArgs), + func() bool { + return p.parseExpr(hintNumber) != nil && p.cur().Kind == tkRBracket + }, + ) + + if allowSendAll && p.eat(tkStar) { + p.expectRecover(tkRBracket, setOf(tkLParen)) + return nil, &SentValueAll{ + Range: p.rangeFrom(start), + Asset: asset, + } } -} -func parseSaveStatement(saveCtx *antlrParser.SaveStatementContext) *SaveStatement { - return &SaveStatement{ - Range: ctxToRange(saveCtx), - SentValue: parseSentValue(saveCtx.SentValue()), - Account: parseValueExpr(saveCtx.ValueExpr()), + var amount ValueExpr + if p.atExprStart(hintNumber) { + amount = p.parseExpr(hintNumber) + } else { + p.errExpected("an amount") } -} -func parseStatement(statementCtx antlrParser.IStatementContext) Statement { - switch statementCtx := statementCtx.(type) { - case *antlrParser.SendStatementContext: - return parseSendStatement(statementCtx) + p.expectRecover(tkRBracket, setOf(tkLParen)) - case *antlrParser.SaveStatementContext: - return parseSaveStatement(statementCtx) + return &MonetaryLiteral{ + Range: p.rangeFrom(start), + Asset: asset, + Amount: amount, + }, nil +} - case *antlrParser.FnCallStatementContext: - return parseFnCall(statementCtx.FunctionCall()) +// Parses an expression that may be directly followed by an allotment, e.g. the +// `@a` in `{ 1/2 to @a -1/2 to @b }`, where the greedy parse would read `@a - 1/2`. +// follow is the set of tokens that follows an allotment in that position. +func (p *parser) parseExprBeforeAllotment(hints []string, follow tokenSet) ValueExpr { + if follow == 0 { + return p.parseExpr(hints) + } - case *antlrParser.StatementContext: - return nil + return p.parseJuxtaposedExpr(hints, + func() bool { return follow.has(p.cur().Kind) }, + func() bool { return p.parseExpr(hintPortion) != nil && follow.has(p.cur().Kind) }, + ) +} - default: - return utils.NonExhaustiveMatchPanic[Statement](statementCtx.GetText()) +// In some places of the grammar, two expressions are not separated by any token, +// (e.g. in the `[$asset -1]` monetary literal). When the greedy parse of the first +// expression consumes the beginning of the second one (needsSplit returns true after it), +// we look for a binary '-' that can be reinterpreted as the prefix operator of the +// second expression. The rightmost split for which isViable succeeds wins (that's the +// choice the ANTLR parser used to make). +// +// isViable is called after the first expression of the split, and it's allowed to consume tokens: +// the parser state is restored afterwards. +func (p *parser) parseJuxtaposedExpr(hints []string, needsSplit func() bool, isViable func() bool) ValueExpr { + before := p.checkpoint() + expr := p.parseExpr(hints) + if !needsSplit() { + return expr } -} -func parseSentValue(statementCtx antlrParser.ISentValueContext) SentValue { - switch statementCtx := statementCtx.(type) { - case *antlrParser.SentLiteralContext: - return &SentValueLiteral{ - Range: ctxToRange(statementCtx), - Monetary: parseValueExpr(statementCtx.ValueExpr()), - } - case *antlrParser.SentAllContext: - return &SentValueAll{ - Range: ctxToRange(statementCtx), - Asset: parseValueExpr(statementCtx.SentAllLit().GetAsset()), + greedyEnd := p.pos + + var candidates []int + depth := 0 + for i := before.pos; i < greedyEnd; i++ { + switch p.tokens[i].Kind { + case tkLParen, tkLBracket: + depth++ + case tkRParen, tkRBracket: + depth-- + case tkMinus: + if depth == 0 && i > before.pos && !prefixPosition.has(p.tokens[i-1].Kind) { + candidates = append(candidates, i) + } } + } + if len(candidates) == 0 { + return expr + } - case *antlrParser.SentValueContext: - return nil + prevStopAt := p.stopAt + defer func() { p.stopAt = prevStopAt }() - default: - return utils.NonExhaustiveMatchPanic[SentValue](statementCtx.GetText()) + for _, splitAt := range slices.Backward(candidates) { + p.restore(before) + p.lastErrorTok = -1 + + p.stopAt = splitAt + splitExpr := p.parseExpr(hints) + p.stopAt = prevStopAt + + if p.pos != splitAt || len(p.errors) != before.nErrors { + continue + } + + afterSplit := p.checkpoint() + viable := isViable() && len(p.errors) == before.nErrors + p.restore(afterSplit) + if viable { + return splitExpr + } } + // No viable split: parse the expression greedily again + p.restore(before) + expr = p.parseExpr(hints) + if p.pos != greedyEnd { + panic("unreachable: the re-parse of an expression diverged") + } + return expr } -func parseSendStatement(statementCtx *antlrParser.SendStatementContext) *SendStatement { - return &SendStatement{ - Source: parseSource(statementCtx.Source()), - Destination: parseDestination(statementCtx.Destination()), - Range: ctxToRange(statementCtx), - SentValue: parseSentValue(statementCtx.SentValue()), +// (IDENTIFIER | 'overdraft') '(' (valueExpr (',' valueExpr)*)? ')' +func (p *parser) parseFnCall() *FnCall { + nameTk := p.bump() + p.bump() // '(' + + p.fnFrames = append(p.fnFrames, FnCallFrame{Name: nameTk.Text}) + defer func() { p.fnFrames = p.fnFrames[:len(p.fnFrames)-1] }() + + call := &FnCall{ + Caller: &FnCallIdentifier{Range: nameTk.Range, Name: nameTk.Text}, } -} -func parseNumberLiteral(numNode antlr.TerminalNode) *NumberLiteral { - amtStr := numNode.GetText() - amtStr = strings.ReplaceAll(amtStr, "_", "") + isExpr := p.atExprStart(nil) + if isExpr || !p.at(tkRParen) { + for { + arg := p.parseExpr(nil) + if arg != nil { + call.Args = append(call.Args, arg) + } - amt, ok := new(big.Int).SetString(amtStr, 10) - if !ok { - // This should be unreachable for tokens emitted by the lexer. - // Degrade gracefully to a zero-value literal instead of panicking, - // so that a malformed token cannot crash the process (e.g. the LSP server). - amt = big.NewInt(0) + if !p.eat(tkComma) { + break + } + p.fnFrames[len(p.fnFrames)-1].ArgIndex++ + } } - return &NumberLiteral{ - Range: tokenToRange(numNode.GetSymbol()), - Number: amt, + if !p.eat(tkRParen) { + p.errExpected("`,` or `)`") } + + call.Range = p.rangeFrom(nameTk.Start) + return call } -func parseMonetaryLit(monetaryLitCtx antlrParser.IMonetaryLitContext) *MonetaryLiteral { - if monetaryLitCtx.GetAmt() == nil { +// --- Literals + +func parseVariable(tk token) *Variable { + if len(tk.Text) <= 1 { + // an invalid `$`: the error was already reported by the lexer return nil } + return &Variable{Range: tk.Range, Name: tk.Text[1:]} +} - return &MonetaryLiteral{ - Range: ctxToRange(monetaryLitCtx), - Asset: parseValueExpr(monetaryLitCtx.GetAsset()), - Amount: parseValueExpr(monetaryLitCtx.GetAmt()), +func parseString(tk token) *StringLiteral { + inner := tk.Text[1:] + // an unterminated string literal has no closing quote (the error is reported by the lexer) + if len(tk.Text) >= 2 && strings.HasSuffix(tk.Text, `"`) { + inner = inner[:len(inner)-1] } + return &StringLiteral{Range: tk.Range, String: inner} } -func ctxToRange(ctx antlr.ParserRuleContext) Range { - startTk := ctx.GetStart() - endTk := ctx.GetStop() +func parseNumberLiteral(tk token) *NumberLiteral { + amt, ok := new(big.Int).SetString(strings.ReplaceAll(tk.Text, "_", ""), 10) + if !ok { + // This should be unreachable for tokens emitted by the lexer. + // Degrade gracefully to a zero-value literal instead of panicking, + // so that a malformed token cannot crash the process (e.g. the LSP server). + amt = big.NewInt(0) + } - return Range{ - Start: Position{ - Line: startTk.GetLine() - 1, - Character: startTk.GetColumn(), - }, - End: Position{ - Line: endTk.GetLine() - 1, + return &NumberLiteral{Range: tk.Range, Number: amt} +} - // this is based on the assumption that a token cannot span multiple lines - Character: endTk.GetColumn() + len(endTk.GetText()), - }, +func parsePercentageRatio(source string, range_ Range) *PercentageLiteral { + num, floatingDigits, err := ParsePercentageRatio(source) + if err != nil { + // This should be unreachable for tokens emitted by the lexer. + // Degrade gracefully to a zero-value literal instead of panicking, + // so that a malformed token cannot crash the process (e.g. the LSP server). + return &PercentageLiteral{ + Range: range_, + Amount: big.NewInt(0), + } } -} -func tokenToRange(tk antlr.Token) Range { - return Range{ - Start: Position{ - Line: tk.GetLine() - 1, - Character: tk.GetColumn(), - }, - End: Position{ - Line: tk.GetLine() - 1, - Character: tk.GetColumn() + len(tk.GetText()), - }, + return &PercentageLiteral{ + Range: range_, + Amount: num, + FloatingDigits: floatingDigits, } } diff --git a/internal/parser/parser_fault_tolerance_test.go b/internal/parser/parser_fault_tolerance_test.go index bbb218d3..b8201b6a 100644 --- a/internal/parser/parser_fault_tolerance_test.go +++ b/internal/parser/parser_fault_tolerance_test.go @@ -96,3 +96,65 @@ save [COIN 10] from `) snaps.MatchSnapshot(t, p.Value) } + +// The cases below used to lose (or garble) most of the statement with the ANTLR parser + +func TestFaultToleranceBareAt(t *testing.T) { + p := parser.Parse(`send [COIN 10] ( + source = { @a @ } + destination = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceIncompleteOverdraft(t *testing.T) { + p := parser.Parse(`send [COIN 10] ( + source = @a allowing + destination = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceIncompleteInfix(t *testing.T) { + p := parser.Parse(`send $m + ( + source = @a + destination = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceBareDollar(t *testing.T) { + p := parser.Parse(`send [COIN $`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceMisspelledKeyword(t *testing.T) { + p := parser.Parse(`send [COIN 10] ( + source = @a + destinaton = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceMissingParens(t *testing.T) { + p := parser.Parse(`set_tx_meta("k", 42 +send [COIN 10] ( + source = @a + destination = @b +)`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} + +func TestFaultToleranceGarbageBetweenStatements(t *testing.T) { + p := parser.Parse(`send [COIN 10] ( source = @a destination = @b ) +!! this is not numscript ]] +save [COIN 1] from @c`) + snaps.MatchSnapshot(t, p.Value) + snaps.MatchSnapshot(t, parser.ParseErrorsToString(p.Errors, p.Source)) +} diff --git a/internal/parser/parser_internal_test.go b/internal/parser/parser_internal_test.go deleted file mode 100644 index c4b6d90d..00000000 --- a/internal/parser/parser_internal_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package parser - -import ( - "math/big" - "testing" - - "github.com/antlr4-go/antlr/v4" - "github.com/stretchr/testify/require" -) - -// These inputs should be unreachable through Parse() (the lexer only emits -// well-formed tokens), but the parsing helpers must degrade gracefully to a -// zero-value literal instead of panicking and killing the process. - -func TestParsePercentageRatioInvalidSourceDoesNotPanic(t *testing.T) { - require.NotPanics(t, func() { - lit := parsePercentageRatio("not a number%", Range{}) - require.Equal(t, big.NewInt(0), lit.Amount) - require.Equal(t, 0, lit.FloatingDigits) - }) -} - -func TestParseNumberLiteralInvalidNumberDoesNotPanic(t *testing.T) { - tk := antlr.NewCommonToken(&antlr.TokenSourceCharStreamPair{}, 0, antlr.TokenDefaultChannel, 0, 0) - tk.SetText("_") - node := antlr.NewTerminalNodeImpl(tk) - - require.NotPanics(t, func() { - lit := parseNumberLiteral(node) - require.Equal(t, big.NewInt(0), lit.Number) - }) -} diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index c477ad58..951e3a22 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -65,7 +65,7 @@ func TestForbidInvalidNumberUnderscores(t *testing.T) { require.Len(t, parser.Parse(` set_tx_meta("k1", 1__23) - `).Errors, 3) + `).Errors, 2) } func TestVariableMonetary(t *testing.T) { diff --git a/internal/specs_format/__snapshots__/runner_test.snap b/internal/specs_format/__snapshots__/runner_test.snap index 19a6406a..2acb5acb 100755 --- a/internal/specs_format/__snapshots__/runner_test.snap +++ b/internal/specs_format/__snapshots__/runner_test.snap @@ -131,16 +131,19 @@ Error: example.num.specs.json  Error: example.num:1:1 -token recognition error at: '!' +invalid character: `!` + + 0 | !err + | ~  -Error: example.num:1:5 +Error: example.num:1:2 -mismatched input '' expecting '(' +expected a statement, found `err` 0 | !err - | ~~~~ + | ~~~ ---