diff --git a/CLAUDE.md b/CLAUDE.md index 4ba96ca..38bd174 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,9 @@ parameter interpolation. | ------------------- | ------------------------------------------------------------------ | | `engine.go` | `Compile`, `Option`, `map`/`if` and keyword field preprocessing | | `program.go` | AST walker — the evaluator | -| `reflect.go` + `prepared.go` | Env lookup, function dispatch, cached signatures | +| `access.go` | The one implementation of `x.y`, `x[i]`, `?.`, `?[` and env lookup | +| `reflect.go` + `prepared.go` | Conversions, equality, function dispatch, cached `callSig`s | +| `format.go` | Bounded, cycle-safe rendering for `string`, `sprintf`, templates | | `builtins.go` | Default function set exposed by `Builtins()` / `WithBuiltins()` | | `builtin_groups.go` | Opt-in `MathFuncs()` / `StringFuncs()` / `CollectionFuncs()` sets | | `higher_order.go` | `map`, `filter`, `any`, `all`, `find`, `count`, `try`, `if` forms | @@ -30,7 +32,9 @@ parameter interpolation. | `template.go` | `${...}` interpolation via `NewTemplate` / `Render` | | `truthy.go` | `IsTruthy` rules used by `!`, `&&`, `\|\|`, `bool(v)` | | `suggest.go` | "Did you mean…" for unknown identifiers | +| `internal/srcedit/` | Shared scanner, bracket matcher and splicer for the source rewrites | | `internal/jsonlit/` | Pre-parse rewrite of bare `[...]` / `{...}` JSON literals | +| `internal/optaccess/` | Pre-parse rewrite of `?.` / `?[` into sentinel calls | | `cmd/expr/` | Small CLI for one-off evaluation | | `docs/reference/spec.md` | Authoritative language spec | | `docs/guides/` | User-facing guides (examples, functions, env, sandboxing, …) | @@ -51,6 +55,14 @@ parameter interpolation. value (`[upper, 1]`), which `Run` still evaluates. - When changing language behavior, update `docs/reference/spec.md` in the same change; if the change affects how expressions *look*, update `docs/guides/examples.md` and `llms.txt` too. +- Every optimization must be unobservable. A fast path or fold needs an + equivalence test in `optimize_equiv_test.go` (compiled with and without + `withoutOptimizations()`) next to its benchmark. +- Value-shape rules live in one place (`access.go`, `mapKey`, `convertValue`, + `canCompare`, `callSig`). `value_matrix_test.go`, `call_matrix_test.go` and + `hostgen_test.go` hold every path to them; keep new paths on the shared code. +- A test that could overflow the stack or hang belongs in + `isolated_test.go`, which runs each case in a child process. - Guide pages under `docs/guides/` have runnable companions under `examples/` and tests in `docs_guides_test.go` / `docs_examples_test.go`. When you change a guide's code snippets, update the matching test so the docs stay honest. diff --git a/access.go b/access.go new file mode 100644 index 0000000..1f84ca1 --- /dev/null +++ b/access.go @@ -0,0 +1,346 @@ +package expr + +import ( + "fmt" + "reflect" +) + +// This file holds the one implementation of field selection (`x.y`) and +// indexing (`x[i]`) on Go values. Every evaluation path goes through it: +// the general evaluator (selectField, indexValue), the reflect-space +// chain fast paths (selectFieldRV, indexValueRV), and optional access +// (trySelectName, tryIndexValue). The rules, also stated in the spec +// under "Selectors" and "Optional access": +// +// - The receiver is unwrapped through interfaces and pointers, at any +// depth. A nil met on the way is a nil receiver. +// - A nil receiver is an error, or nil under optional access. +// - A missing field or key, or an out-of-range index, is an error, or +// nil under optional access. +// - Any other failure (selecting on a non-struct, non-map value; an +// index of the wrong kind; an unhashable key) is an error in both +// modes. +// +// The wrappers keep a type-asserted map[string]any branch in front of +// the reflect code because it is the hot case; it gives the same +// results as the reflect branch. + +// accessMode selects how a missing value is reported. +type accessMode bool + +const ( + strictAccess accessMode = false // missing → ErrEvaluate + optionalAccess accessMode = true // missing → nil (`?.`, `?[`) +) + +// maxPointerHops bounds unwrapReceiver so a self-referential pointer +// type (`type P *P`) cannot loop forever. +const maxPointerHops = 64 + +// unwrapReceiver follows interfaces and pointers from rv to the value +// they lead to. It returns an invalid Value when the chain ends at nil; +// nilPtr then reports whether that nil was a pointer (for the error +// message). A chain longer than maxPointerHops is returned as is, and +// its pointer kind then fails as an unsupported receiver. +func unwrapReceiver(rv reflect.Value) (out reflect.Value, nilPtr bool) { + for hops := 0; rv.IsValid() && hops < maxPointerHops; hops++ { + switch rv.Kind() { + case reflect.Interface: + if rv.IsNil() { + return reflect.Value{}, false + } + rv = rv.Elem() + case reflect.Pointer: + if rv.IsNil() { + return reflect.Value{}, true + } + rv = rv.Elem() + default: + return rv, false + } + } + return rv, false +} + +// boxValue materializes rv as an any; an invalid Value is nil. +func boxValue(rv reflect.Value) any { + if !rv.IsValid() { + return nil + } + return rv.Interface() +} + +// selectRV looks up the field or string key name on rv. Under +// optionalAccess an invalid Value with a nil error means nil. +func selectRV(rv reflect.Value, name string, fieldTags *structTagConfig, mode accessMode) (reflect.Value, error) { + rv, nilPtr := unwrapReceiver(rv) + if !rv.IsValid() { + if mode == optionalAccess { + return reflect.Value{}, nil + } + if nilPtr { + return reflect.Value{}, fmt.Errorf("%w: cannot access %q on nil pointer", ErrEvaluate, name) + } + return reflect.Value{}, fmt.Errorf("%w: cannot access %q on nil", ErrEvaluate, name) + } + switch rv.Kind() { + case reflect.Struct: + fv, ok, err := structFieldByName(rv, name, fieldTags) + if err != nil { + return reflect.Value{}, err + } + // An unexported field is treated as missing rather than + // panicking in Interface(). + if ok && fv.IsValid() && fv.CanInterface() { + return fv, nil + } + if mode == optionalAccess { + return reflect.Value{}, nil + } + return reflect.Value{}, fmt.Errorf("%w: field %q not found on %v%s", + ErrEvaluate, name, rv.Type(), fieldHint(rv.Interface(), name, fieldTags)) + case reflect.Map: + if rv.Type().Key().Kind() != reflect.String { + return reflect.Value{}, fmt.Errorf("%w: cannot select %q on map with non-string keys", ErrEvaluate, name) + } + mv := rv.MapIndex(mapStringKey(rv.Type().Key(), name)) + if mv.IsValid() { + return mv, nil + } + if mode == optionalAccess { + return reflect.Value{}, nil + } + return reflect.Value{}, fmt.Errorf("%w: key %q not found%s", + ErrEvaluate, name, fieldHint(rv.Interface(), name, fieldTags)) + } + return reflect.Value{}, fmt.Errorf("%w: cannot select %q on %v", ErrEvaluate, name, rv.Type()) +} + +// indexRV indexes rv by idx. Under optionalAccess an invalid Value with +// a nil error means nil. +func indexRV(rv reflect.Value, idx any, mode accessMode) (reflect.Value, error) { + rv, _ = unwrapReceiver(rv) + if !rv.IsValid() { + if mode == optionalAccess { + return reflect.Value{}, nil + } + return reflect.Value{}, fmt.Errorf("%w: cannot index nil", ErrEvaluate) + } + if rv.Type() == mapStringAnyType { + key, ok := idx.(string) + if !ok { + return reflect.Value{}, fmt.Errorf("%w: map index must be string, got %T", ErrEvaluate, idx) + } + mv := rv.MapIndex(reflect.ValueOf(key)) + if mv.IsValid() { + return mv, nil + } + if mode == optionalAccess { + return reflect.Value{}, nil + } + return reflect.Value{}, fmt.Errorf("%w: key %q not found%s", + ErrEvaluate, key, fieldHint(rv.Interface(), key, nil)) + } + switch rv.Kind() { + case reflect.Slice, reflect.Array: + i, err := toIndexInt(idx) + if err != nil { + return reflect.Value{}, err + } + if i >= 0 && i < int64(rv.Len()) { + return rv.Index(int(i)), nil + } + if mode == optionalAccess { + return reflect.Value{}, nil + } + return reflect.Value{}, fmt.Errorf("%w: index %d out of range [0, %d)", ErrEvaluate, i, rv.Len()) + case reflect.String: + i, err := toIndexInt(idx) + if err != nil { + return reflect.Value{}, err + } + runes := []rune(rv.String()) + if i >= 0 && i < int64(len(runes)) { + return reflect.ValueOf(string(runes[i])), nil + } + if mode == optionalAccess { + return reflect.Value{}, nil + } + return reflect.Value{}, fmt.Errorf("%w: index %d out of range [0, %d)", ErrEvaluate, i, len(runes)) + case reflect.Map: + kv, err := mapKey(rv.Type().Key(), idx) + if err != nil { + return reflect.Value{}, err + } + mv := rv.MapIndex(kv) + if mv.IsValid() { + return mv, nil + } + if mode == optionalAccess { + return reflect.Value{}, nil + } + return reflect.Value{}, fmt.Errorf("%w: key %v not found", ErrEvaluate, idx) + } + return reflect.Value{}, fmt.Errorf("%w: cannot index %v", ErrEvaluate, rv.Type()) +} + +// mapStringAnyType is the hot map type, given its own branch in indexRV +// so its errors name the key the way the type-asserted wrappers do. +var mapStringAnyType = reflect.TypeOf(map[string]any(nil)) + +// selectField is `recv.name`. +func selectField(recv any, name string, fieldTags *structTagConfig) (any, error) { + // Translate rewritten identifiers (currently only `map`) back to + // their user-visible form so field/key/error lookups match what + // the user actually typed. + name = displayIdent(name) + if m, ok := recv.(map[string]any); ok { + v, ok := m[name] + if !ok { + return nil, fmt.Errorf("%w: key %q not found%s", + ErrEvaluate, name, fieldHint(recv, name, fieldTags)) + } + return v, nil + } + v, err := selectRV(reflect.ValueOf(recv), name, fieldTags, strictAccess) + if err != nil { + return nil, err + } + return boxValue(v), nil +} + +// selectFieldRV is selectField for a receiver already in reflect space, +// so a chain of hops never boxes an intermediate struct. +func selectFieldRV(rv reflect.Value, name string, fieldTags *structTagConfig) (reflect.Value, error) { + return selectRV(rv, name, fieldTags, strictAccess) +} + +// trySelectName is `recv?.name` once recv is known to be non-nil. +func trySelectName(recv any, name string, fieldTags *structTagConfig) (any, error) { + if m, ok := recv.(map[string]any); ok { + return m[name], nil + } + v, err := selectRV(reflect.ValueOf(recv), name, fieldTags, optionalAccess) + if err != nil { + return nil, err + } + return boxValue(v), nil +} + +// indexValue is `recv[idx]`. +func indexValue(recv, idx any) (any, error) { + if m, ok := recv.(map[string]any); ok { + key, ok := idx.(string) + if !ok { + return nil, fmt.Errorf("%w: map index must be string, got %T", ErrEvaluate, idx) + } + v, ok := m[key] + if !ok { + return nil, fmt.Errorf("%w: key %q not found%s", + ErrEvaluate, key, fieldHint(recv, key, nil)) + } + return v, nil + } + v, err := indexRV(reflect.ValueOf(recv), idx, strictAccess) + if err != nil { + return nil, err + } + return boxValue(v), nil +} + +// indexValueRV is indexValue for a receiver already in reflect space. +func indexValueRV(rv reflect.Value, idx any) (reflect.Value, error) { + return indexRV(rv, idx, strictAccess) +} + +// tryIndexValue is `recv?[idx]` once recv is known to be non-nil. +func tryIndexValue(recv, idx any) (any, error) { + if m, ok := recv.(map[string]any); ok { + key, ok := idx.(string) + if !ok { + return nil, fmt.Errorf("%w: map index must be string, got %T", ErrEvaluate, idx) + } + return m[key], nil + } + v, err := indexRV(reflect.ValueOf(recv), idx, optionalAccess) + if err != nil { + return nil, err + } + return boxValue(v), nil +} + +// lookupEnv resolves a top-level identifier against env. env may be a +// map[string]any, any map with string keys, a struct, a pointer to a +// struct, or an *itEnv wrapping one of the above. For structs, configured +// field tags participate in field lookup, and fields are preferred over +// methods when both match. Methods are returned as bound function values +// so they can be invoked by a CallExpr node. An itEnv binds its element +// name (`it`, or the named binding of a three-arg form) and `index` +// ahead of anything in its parent. +func lookupEnv(env any, name string, fieldTags *structTagConfig) (any, bool, error) { + if m, ok := env.(map[string]any); ok { + v, ok := m[name] + return v, ok, nil + } + rv, ok, err := lookupEnvRV(env, name, fieldTags) + if err != nil || !ok { + return nil, ok, err + } + return boxValue(rv), true, nil +} + +// lookupEnvRV is lookupEnv returning a reflect.Value, so callers can +// chain field accesses without boxing intermediate struct values. +func lookupEnvRV(env any, name string, fieldTags *structTagConfig) (reflect.Value, bool, error) { + if env == nil { + return reflect.Value{}, false, nil + } + if it, ok := env.(*itEnv); ok { + switch name { + case it.elementName(): + return reflect.ValueOf(it.it), true, nil + case "index": + return reflect.ValueOf(it.index), true, nil + } + return lookupEnvRV(it.parent, name, fieldTags) + } + if m, ok := env.(map[string]any); ok { + v, ok := m[name] + if !ok { + return reflect.Value{}, false, nil + } + return reflect.ValueOf(v), true, nil + } + rv := reflect.ValueOf(env) + // Methods are looked up on the original (possibly pointer) value so + // pointer-receiver methods are visible, but a struct field of the + // same name wins. + orig := rv + if rv.Kind() == reflect.Pointer { + if rv.IsNil() { + return reflect.Value{}, false, nil + } + rv = rv.Elem() + } + switch rv.Kind() { + case reflect.Struct: + fv, ok, err := structFieldByName(rv, name, fieldTags) + if err != nil { + return reflect.Value{}, false, err + } + if ok && fv.CanInterface() { + return fv, true, nil + } + if mv := orig.MethodByName(name); mv.IsValid() { + return reflect.ValueOf(methodCallable(name, mv)), true, nil + } + case reflect.Map: + if rv.Type().Key().Kind() == reflect.String { + mv := rv.MapIndex(mapStringKey(rv.Type().Key(), name)) + if mv.IsValid() { + return mv, true, nil + } + } + } + return reflect.Value{}, false, nil +} diff --git a/analysis_growth_test.go b/analysis_growth_test.go new file mode 100644 index 0000000..74e8fa9 --- /dev/null +++ b/analysis_growth_test.go @@ -0,0 +1,97 @@ +package expr + +import ( + "strings" + "testing" + "time" +) + +// The static passes over a compiled program (Check, Identifiers, Calls) +// walk the syntax tree, and a walk that rescans its enclosing scopes or +// reprints subtrees is quadratic or worse in the nesting, without +// allocating anything TestCompileGrowthIsLinear would see. This test +// times the whole static pipeline on inputs four times apart: linear +// growth is about 4x, quadratic about 16x. Taking the fastest of several +// runs keeps machine load from failing it. + +var analysisShapes = []growthShape{ + {name: "nested_map_form", build: func(n int) string { return nest(n, "map(x, ", "it", ")") }}, + {name: "nested_named_filter", build: func(n int) string { return nest(n, "filter(x, e, ", "e", ")") }}, + {name: "nested_if", build: func(n int) string { return nest(n, "if(c, ", "1", ", 2)") }}, + {name: "nested_calls", build: func(n int) string { return nest(n, "f(", "x", ")") }}, + {name: "pipe_chain", build: func(n int) string { return "x" + strings.Repeat(" | f()", n) }}, + {name: "nested_try", build: func(n int) string { return nest(n, "try(", "x.a", ", 1)") }}, + {name: "optional_chain", build: func(n int) string { return "x" + strings.Repeat("?.a", n) }}, + {name: "binary_chain", build: func(n int) string { return "x" + strings.Repeat(" + x", n) }}, + {name: "nested_objects", build: func(n int) string { return nest(n, `{"a": `, "x", "}") }}, +} + +// narrowingShapes run against nullableChain, where x and every .a below +// it may be nil, so narrowing applies at every node. +var narrowingShapes = []growthShape{ + {name: "narrowing_chain", build: func(n int) string { return "x != nil" + strings.Repeat(" && x.a != nil", n) }}, + {name: "narrowing_or_chain", build: func(n int) string { return "x == nil" + strings.Repeat(" || x.a == nil", n) }}, + {name: "narrowed_deep_optional", build: func(n int) string { return "x != nil && x" + strings.Repeat("?.a", n) }}, + {name: "narrowed_nested_if", build: func(n int) string { return nest(n, "if(x != nil && x.a != nil, x.a.a == nil && ", "true", ", false)") }}, +} + +// nullableChain is {a: {a: ... }?}? nested depth levels. +func nullableChain(depth int) *Type { + t := Number().OrNil() + for i := 0; i < depth; i++ { + t = ObjectOf(map[string]*Type{"a": t}).OrNil() + } + return t +} + +func staticPipeline(src string, x *Type) { + p, err := Compile(src, WithFunctions(map[string]any{"f": func(v any) any { return v }})) + if err != nil { + return + } + _, _ = p.Check(map[string]*Type{"x": x, "c": Bool()}) + _ = p.Identifiers() + _ = p.Calls("f") +} + +func fastest(runs int, f func()) time.Duration { + best := time.Duration(1<<63 - 1) + for i := 0; i < runs; i++ { + start := time.Now() + f() + if d := time.Since(start); d < best { + best = d + } + } + return best +} + +func TestStaticAnalysisGrowthIsLinear(t *testing.T) { + if testing.Short() { + t.Skip("times the static passes up to MaxSourceLength per shape") + } + deep := nullableChain(MaxSourceLength / 3) + shapes := append(append([]growthShape{}, analysisShapes...), narrowingShapes...) + for i, s := range shapes { + x := Any + if i >= len(analysisShapes) { + x = deep + } + t.Run(s.name, func(t *testing.T) { + big := sizeFor(s, MaxSourceLength) + srcSmall, srcBig := s.build(big/4), s.build(big) + staticPipeline(srcSmall, x) // warm up + a := fastest(5, func() { staticPipeline(srcSmall, x) }) + b := fastest(5, func() { staticPipeline(srcBig, x) }) + ratio := float64(b) / float64(a) + lenRatio := float64(len(srcBig)) / float64(len(srcSmall)) + t.Logf("%d -> %d bytes: %v -> %v (x%.1f)", len(srcSmall), len(srcBig), a, b, ratio) + if ratio > 2.5*lenRatio { + t.Errorf("static passes took x%.1f longer for x%.1f input: superlinear", ratio, lenRatio) + } + if b > 2*time.Second { + t.Errorf("static passes took %v on a %d-byte source", b, len(srcBig)) + } + }) + } +} diff --git a/bench_test.go b/bench_test.go index 80e77e0..992f03b 100644 --- a/bench_test.go +++ b/bench_test.go @@ -20,6 +20,8 @@ import ( // into steady-state measurements. var engOpts = []expr.Option{expr.WithBuiltins()} +// Exercises the typed binary-operator fast path; equivalence: +// TestApplyBinaryFastMatchesGeneral. func Benchmark_expr(b *testing.B) { params := map[string]any{ "Origin": "MOW", @@ -50,6 +52,8 @@ func Benchmark_expr(b *testing.B) { } } +// Exercises the prepared-call cache; equivalence: +// TestEquivalencePreparedCallCache. func Benchmark_len(b *testing.B) { env := map[string]any{ "arr": make([]int, 100), @@ -132,6 +136,8 @@ func Benchmark_filterLen(b *testing.B) { } } +// Exercises filter(xs, p)[k] streaming; equivalence: +// TestEquivalenceFilterIndex. func Benchmark_filterFirst(b *testing.B) { ints := make([]int, 1000) for i := 1; i <= len(ints); i++ { @@ -224,6 +230,8 @@ type priceEnv struct { Price Price } +// Exercises the reflect selector chain; equivalence: +// TestEquivalenceSelectorChain. func Benchmark_envStruct(b *testing.B) { program, err := expr.Compile(`Price.Value > 0`, engOpts...) if err != nil { @@ -362,6 +370,8 @@ func Benchmark_structTagsWide(b *testing.B) { } } +// Exercises the map[string]any access branches; equivalence: +// TestValueMatrixPathsAgree (map env vs struct env). func Benchmark_envMap(b *testing.B) { env := map[string]any{ "price": Price{Value: 1}, @@ -410,6 +420,8 @@ func (CallFoo) Method() string { return "method" } +// Exercises calls through a struct env; equivalence: +// TestEquivalencePreparedCallCache and TestCallMatrixPathsAgree. func Benchmark_callFunc(b *testing.B) { program, err := expr.Compile(`Func()`, engOpts...) if err != nil { @@ -513,6 +525,8 @@ func Benchmark_largeStructAccess(b *testing.B) { } } +// Exercises the reflect selector chain; equivalence: +// TestEquivalenceSelectorChain. func Benchmark_largeNestedStructAccess(b *testing.B) { type Env struct { Inner struct { @@ -545,6 +559,8 @@ func Benchmark_largeNestedStructAccess(b *testing.B) { } } +// Exercises the reflect index chain; equivalence: +// TestEquivalenceIndexChain. func Benchmark_largeNestedArrayAccess(b *testing.B) { type Env struct { Data [1][1024 * 1024 * 10]byte diff --git a/builtin_groups.go b/builtin_groups.go index 6808534..ad46fee 100644 --- a/builtin_groups.go +++ b/builtin_groups.go @@ -314,32 +314,38 @@ func nativeSum(_ context.Context, args []any) (any, error) { if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { return nil, fmt.Errorf("%w: sum: expected list, got %T", ErrEvaluate, args[0]) } + // Integers accumulate exactly; floats separately. A list with any + // float sums to a float, so an integer overflow is only an error + // when every element is an integer — otherwise the overflowing + // partial sum moves into the float accumulator, whatever the order + // of the elements. var intSum int64 var floatSum float64 - floating := false + floating, overflowed := false, false for i := 0; i < rv.Len(); i++ { elem := rv.Index(i).Interface() - if !floating { - if v, ok := toInt64(elem); ok { - next, ok := checkedAddInt64(intSum, v) - if !ok { - return nil, fmt.Errorf("%w: integer overflow", ErrEvaluate) - } - intSum = next - continue + if v, ok := toInt64(elem); ok { + next, ok := checkedAddInt64(intSum, v) + if !ok { + overflowed = true + floatSum += float64(intSum) + next = v } - // First non-integral element: switch to float accumulation. - floating = true - floatSum = float64(intSum) + intSum = next + continue } v, ok := toFloat64(elem) if !ok { return nil, fmt.Errorf("%w: sum: element %d is %T, not a number", ErrEvaluate, i, elem) } + floating = true floatSum += v } if floating { - return floatSum, nil + return floatSum + float64(intSum), nil + } + if overflowed { + return nil, fmt.Errorf("%w: integer overflow", ErrEvaluate) } return intSum, nil } diff --git a/call_matrix_test.go b/call_matrix_test.go new file mode 100644 index 0000000..cb3c8c8 --- /dev/null +++ b/call_matrix_test.go @@ -0,0 +1,262 @@ +package expr + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// The call matrix invokes the same host functions with the same +// arguments through every way a host can supply one: registered with +// WithFunctions (the prepared path, with and without the compile-time +// call cache), stored in the env map, and stored in a struct field. The +// paths must agree on the result, the error category and message, and +// on whether the function body ran. + +type callProbe struct { + name string + // fn builds the function, reporting each run through called. + fn func(called *int) any +} + +type mxStringer struct{} + +func (mxStringer) String() string { return "stringer" } + +func callProbes() []callProbe { + return []callProbe{ + {"int", func(c *int) any { return func(x int) int { *c++; return x + 1 } }}, + {"int8", func(c *int) any { return func(x int8) int8 { *c++; return x } }}, + {"uint8", func(c *int) any { return func(x uint8) uint8 { *c++; return x } }}, + {"float32", func(c *int) any { return func(x float32) float32 { *c++; return x } }}, + {"string", func(c *int) any { return func(s string) string { *c++; return s + "!" } }}, + {"named string", func(c *int) any { return func(s mxName) mxName { *c++; return s } }}, + {"any", func(c *int) any { return func(v any) any { *c++; return v } }}, + {"[]int", func(c *int) any { return func(xs []int) int { *c++; return len(xs) } }}, + {"[2]int", func(c *int) any { return func(xs [2]int) int { *c++; return xs[0] } }}, + {"*[2]int", func(c *int) any { return func(xs *[2]int) bool { *c++; return xs == nil } }}, + {"map[string]int", func(c *int) any { return func(m map[string]int) int { *c++; return len(m) } }}, + {"error param", func(c *int) any { return func(e error) bool { *c++; return e == nil } }}, + {"Stringer param", func(c *int) any { return func(s fmt.Stringer) string { *c++; return s.String() } }}, + {"variadic", func(c *int) any { return func(p string, xs ...int) int { *c++; return len(p) + len(xs) } }}, + {"ctx", func(c *int) any { + return func(ctx context.Context, s string) string { *c++; return fmt.Sprint(ctx != nil, s) } + }}, + {"no result", func(c *int) any { return func(any) { *c++ } }}, + {"(T, nil error)", func(c *int) any { return func(any) (int, error) { *c++; return 1, nil } }}, + {"(T, error)", func(c *int) any { return func(any) (int, error) { *c++; return 0, errors.New("host failed") } }}, + {"(T, ErrEvaluate)", func(c *int) any { + return func(any) (int, error) { *c++; return 0, fmt.Errorf("%w: already wrapped", ErrEvaluate) } + }}, + {"(T, concrete error)", func(c *int) any { return func(any) (int, mxErr) { *c++; return 0, mxErr{"concrete"} } }}, + {"(T, nil *ptr error)", func(c *int) any { return func(any) (int, *mxPtrErr) { *c++; return 2, nil } }}, + {"(T, typed nil in error)", func(c *int) any { + // The Go gotcha: a nil *mxPtrErr boxed in error is a non-nil + // error, and expr follows Go. + return func(any) (int, error) { *c++; var e *mxPtrErr; return 3, e } + }}, + {"runtime panic", func(c *int) any { return func(xs []int) int { *c++; return xs[5] } }}, + {"two results", func(c *int) any { return func() (int, int) { *c++; return 1, 2 } }}, + {"error first", func(c *int) any { return func() (error, int) { *c++; return nil, 1 } }}, + {"three results", func(c *int) any { return func() (int, int, error) { *c++; return 1, 2, nil } }}, + {"Func", func(c *int) any { + return Func(func(_ context.Context, args []any) (any, error) { *c++; return len(args), nil }) + }}, + {"Func error", func(c *int) any { + return Func(func(context.Context, []any) (any, error) { *c++; return nil, mxErr{"native"} }) + }}, + } +} + +// callArgs are argument lists, written as expr source. +var callArgs = []string{ + "", "1", "300", "-1", "1.0", "1.5", `"s"`, "nil", "[1, 2]", "[1]", "A", "S", "E", "N", "M", `"p", 1, 2`, `"p"`, +} + +// callEnv supplies the host-typed arguments: A an []int too short for +// [2]int, S a Stringer, E a concrete error, N a named string, M a +// map[string]any (not convertible to map[string]int). +func callArgEnv() map[string]any { + return map[string]any{ + "A": []int{1}, "S": mxStringer{}, "E": mxErr{"e"}, "N": mxName("n"), + "M": map[string]any{"a": 1}, + } +} + +type callFieldEnv struct { + F any + A any + S any + E any + N any + M any +} + +type callOutcome struct { + mxOutcome + called int +} + +func (o callOutcome) String() string { return fmt.Sprintf("%v (ran %d)", o.mxOutcome, o.called) } + +// errorsAsMx reports whether err carries the mxErr a host returned, so +// the original cause stays reachable through the wrapping. +func errorsAsMx(err error) bool { + var target mxErr + return errors.As(err, &target) +} + +func runCall(probe callProbe, args, how string) (out callOutcome, causeKept bool) { + called := 0 + fn := probe.fn(&called) + argEnv := callArgEnv() + src := "F(" + args + ")" + var ( + p *Program + err error + env any + ) + defer func() { + if r := recover(); r != nil { + out = callOutcome{mxOutcome{panicked: r}, called} + } + }() + switch how { + case "registered", "registered/noopt": + opts := []Option{WithFunctions(map[string]any{"F": fn})} + if how == "registered/noopt" { + opts = append(opts, withoutOptimizations()) + } + p, err = Compile(src, opts...) + env = argEnv + case "env": + p, err = Compile(src) + argEnv["F"] = fn + env = argEnv + case "field": + p, err = Compile(src) + env = callFieldEnv{F: fn, A: argEnv["A"], S: argEnv["S"], E: argEnv["E"], N: argEnv["N"], M: argEnv["M"]} + } + if err != nil { + return callOutcome{mxOutcome{category: errCategory(err), msg: err.Error()}, called}, true + } + v, err := p.Run(context.Background(), env) + if err != nil { + returnsMx := probe.name == "(T, concrete error)" || probe.name == "Func error" + return callOutcome{mxOutcome{category: errCategory(err), msg: err.Error()}, called}, !returnsMx || called == 0 || errorsAsMx(err) + } + return callOutcome{mxOutcome{value: v}, called}, true +} + +func TestCallMatrixPathsAgree(t *testing.T) { + for _, probe := range callProbes() { + t.Run(probe.name, func(t *testing.T) { + for _, args := range callArgs { + reg, _ := runCall(probe, args, "registered") + if reg.category == "compile" { + // Registration rejected the signature; the other paths + // must reject it at run time without calling it. + for _, how := range []string{"env", "field"} { + o, _ := runCall(probe, args, how) + if o.category != "evaluate" || o.called != 0 { + t.Errorf("F(%s) via %s = %v; registration rejects this signature, so Run must fail before calling", args, how, o) + } + } + continue + } + for _, how := range []string{"registered/noopt", "env", "field"} { + o, kept := runCall(probe, args, how) + if !sameOutcome(reg.mxOutcome, o.mxOutcome) || reg.called != o.called { + t.Errorf("F(%s):\n registered: %v\n %-11s %v", args, reg, how+":", o) + } + if !kept { + t.Errorf("F(%s) via %s lost the host error's cause: %v", args, how, o) + } + } + if reg.panicked != nil { + t.Errorf("F(%s) panicked: %v", args, reg.panicked) + } + if reg.category == "UNWRAPPED" { + t.Errorf("F(%s): error does not wrap ErrEvaluate: %s", args, reg.msg) + } + // A conversion failure must stop the call before the body runs. + if reg.category != "" && reg.called != 0 && !isHostFailure(probe.name) && !strings.Contains(reg.msg, "panicked during call") { + t.Errorf("F(%s) failed (%s) after running the function", args, reg.msg) + } + } + }) + } +} + +// isHostFailure reports whether the probe's own body returns an error +// or panics, so a failed call that ran it is expected. +func isHostFailure(name string) bool { + switch name { + case "(T, error)", "(T, ErrEvaluate)", "(T, concrete error)", "(T, typed nil in error)", "runtime panic", "Func error": + return true + } + return false +} + +// TestCallMatrixConversions pins the argument conversions themselves, +// beyond agreement between paths. +func TestCallMatrixConversions(t *testing.T) { + probes := map[string]callProbe{} + for _, p := range callProbes() { + probes[p.name] = p + } + for _, tc := range []struct { + probe, args string + want any // a value, or the error category + }{ + {"int", "1", int(2)}, + {"int", "1.0", int(2)}, + {"int", "1.5", int(2)}, // float → int truncates, as Go's conversion does + {"int8", "300", "evaluate"}, + {"uint8", "-1", "evaluate"}, + {"float32", "1.5", float32(1.5)}, + {"string", "1", "evaluate"}, // no integer → string conversion + {"named string", `"s"`, mxName("s")}, + {"string", "N", "n!"}, + {"[]int", "[1, 2]", "evaluate"}, // []any does not convert to []int + {"[]int", "A", 1}, + {"[2]int", "A", "evaluate"}, // a slice shorter than the array + {"*[2]int", "A", "evaluate"}, + {"*[2]int", "nil", true}, + {"map[string]int", "M", "evaluate"}, + {"error param", "E", false}, + {"error param", "nil", true}, + {"Stringer param", "S", "stringer"}, + {"variadic", `"p", 1, 2`, 3}, + {"variadic", `"p"`, 1}, + {"variadic", "", "evaluate"}, + {"ctx", `"s"`, "trues"}, + {"no result", "1", nil}, + {"(T, nil error)", "1", 1}, + {"(T, error)", "1", "evaluate"}, + {"(T, concrete error)", "1", "evaluate"}, + {"(T, nil *ptr error)", "1", 2}, + {"(T, typed nil in error)", "1", "evaluate"}, + {"runtime panic", "A", "evaluate"}, + {"two results", "", "compile"}, + {"error first", "", "compile"}, + {"three results", "", "compile"}, + {"Func", `"p", 1`, 2}, + {"Func error", "", "evaluate"}, + } { + got, _ := runCall(probes[tc.probe], tc.args, "registered") + want := mxOutcome{value: tc.want} + if cat, isCat := tc.want.(string); isCat && (cat == "evaluate" || cat == "compile") { + if got.category != cat { + t.Errorf("%s: F(%s) = %v, want %s error", tc.probe, tc.args, got, cat) + } + continue + } + if !reflect.DeepEqual(got.value, want.value) || got.category != "" { + t.Errorf("%s: F(%s) = %v, want %#v", tc.probe, tc.args, got, tc.want) + } + } +} diff --git a/calls.go b/calls.go index 08b2f08..5bf9a21 100644 --- a/calls.go +++ b/calls.go @@ -1,6 +1,9 @@ package expr -import "go/ast" +import ( + "go/ast" + "sync" +) // Call is one call of a registered function, as Program.Calls reports it. type Call struct { @@ -97,18 +100,45 @@ func (p *Program) walkCalls(node ast.Expr, bound *boundIdents, want map[string]b // sub returns the subtree node as a Program of its own. It shares the // parent's caches, which are read-only after compile, and its Source is // the subtree printed back in display form. +// +// Source and Identifiers are computed on first use: each costs time in +// the size of the subtree, and Calls makes a sub for every argument of +// every matching call, so computing them eagerly is quadratic in the +// nesting of calls (`f(f(f(...)))`, or a long pipe chain). func (p *Program) sub(node ast.Expr) *Program { return &Program{ - source: exprDisplayString(node), - root: node, - funcs: p.funcs, - prepared: p.prepared, - fieldTags: p.fieldTags, - callCache: p.callCache, - litCache: p.litCache, - identifiers: collectIdentifiers(node, p.funcs), - evalBudget: p.evalBudget, + root: node, + funcs: p.funcs, + prepared: p.prepared, + fieldTags: p.fieldTags, + callCache: p.callCache, + litCache: p.litCache, + evalBudget: p.evalBudget, + noOptimize: p.noOptimize, + deferred: &deferredInfo{}, + } +} + +// deferredInfo holds the Source and Identifiers of a Program made by +// sub, computed once when first asked for. +type deferredInfo struct { + once sync.Once + source string + identifiers []string +} + +// resolveDeferred fills in p's source and identifiers if they were +// deferred. It is safe for concurrent use. +func (p *Program) resolveDeferred() (source string, identifiers []string) { + if p.deferred == nil { + return p.source, p.identifiers } + d := p.deferred + d.once.Do(func() { + d.source = exprDisplayString(p.root) + d.identifiers = collectIdentifiers(p.root, p.funcs) + }) + return d.source, d.identifiers } // Literal returns the program's value when the whole expression is a diff --git a/check.go b/check.go index d3002cf..8b26b1c 100644 --- a/check.go +++ b/check.go @@ -104,6 +104,9 @@ type checker struct { // here: the right side of `x != nil && ...`, the then-branch of // `if(x != nil, ...)`. narrowed map[string]int + // narrowedMaxLen is the length of the longest text ever added to + // narrowed in this check. + narrowedMaxLen int } // narrowings returns the expressions that cond being truthy (or falsy) @@ -142,17 +145,77 @@ func isNilIdent(e ast.Expr) bool { } // narrow checks node with the given expressions known not to be nil. -func (c *checker) narrow(names []string, node ast.Expr, sc *scope) *Type { +// logicalChain checks a run of one logical operator, `a && b && c` +// (which parses as `(a && b) && c`). `x && y` is x when x is falsy, else +// y, checked knowing x is truthy; `x || y` is x when x is truthy (so +// never nil), else y, checked knowing x is falsy. What the left side +// proves is everything its operands prove, so the chain is checked as a +// list: each operand's narrowings are added once and stay in force for +// the operands after it. Checking it as nested binaries instead +// recollects the whole left side at every level, quadratic in the +// chain's length. +func (c *checker) logicalChain(n *ast.BinaryExpr, sc *scope) *Type { + op := n.Op + operands := []ast.Expr{n.Y} + x := n.X + for { + b, ok := x.(*ast.BinaryExpr) + if !ok || b.Op != op { + break + } + operands = append(operands, b.Y) + x = b.X + } + operands = append(operands, x) + // operands is right to left; walk it left to right. + truthy := op == token.LAND + var applied []string + var acc *Type + for i := len(operands) - 1; i >= 0; i-- { + e := operands[i] + t := c.check(e, sc) + switch { + case acc == nil: + acc = t + case op == token.LAND: + acc = Join(acc, t) + default: + acc = Join(acc.NonNil(), t) + } + if i > 0 { + names := narrowings(e, truthy) + c.addNarrowings(names) + applied = append(applied, names...) + } + } + c.dropNarrowings(applied) + return acc +} + +func (c *checker) addNarrowings(names []string) { if c.narrowed == nil { c.narrowed = map[string]int{} } for _, n := range names { c.narrowed[n]++ + // applyNarrowing skips nodes too big to print as any + // narrowed text. + if size := len(n); size > c.narrowedMaxLen { + c.narrowedMaxLen = size + } } - t := c.check(node, sc) +} + +func (c *checker) dropNarrowings(names []string) { for _, n := range names { c.narrowed[n]-- } +} + +func (c *checker) narrow(names []string, node ast.Expr, sc *scope) *Type { + c.addNarrowings(names) + t := c.check(node, sc) + c.dropNarrowings(names) return t } @@ -161,6 +224,14 @@ func (c *checker) applyNarrowing(node ast.Expr, t *Type) *Type { if len(c.narrowed) == 0 || t == nil || !t.Optional { return t } + // A node prints as at least half a byte per syntax node (the + // smallest ratio is a displayed literal: `[]any{}`, three nodes, + // prints `[]`), so a node with more than twice as many nodes as the + // longest narrowed text has bytes cannot match it. Skipping those + // keeps this from printing every enclosing node of a deep expression. + if !astWithin(node, 2*c.narrowedMaxLen+2) { + return t + } if c.narrowed[exprDisplayString(node)] > 0 { return t.NonNil() } @@ -171,15 +242,34 @@ func (c *checker) applyNarrowing(node ast.Expr, t *Type) *Type { type scope struct { names map[string]*Type parent *scope + // memo caches the answers lookup got from parent. Nested forms make + // a chain as long as their nesting, and each level looks names up; + // without the memo every lookup walks the whole chain, which is + // quadratic in the nesting. + memo map[string]scopeHit +} + +type scopeHit struct { + t *Type + ok bool } func (s *scope) lookup(name string) (*Type, bool) { - for ; s != nil; s = s.parent { - if t, ok := s.names[name]; ok { - return t, true - } + if s == nil { + return nil, false + } + if t, ok := s.names[name]; ok { + return t, true + } + if h, ok := s.memo[name]; ok { + return h.t, h.ok + } + t, ok := s.parent.lookup(name) + if s.memo == nil { + s.memo = map[string]scopeHit{} } - return nil, false + s.memo[name] = scopeHit{t, ok} + return t, ok } func (c *checker) report(node ast.Expr, format string, args ...any) { @@ -312,16 +402,8 @@ func plain(t *Type) *Type { func (c *checker) binary(n *ast.BinaryExpr, sc *scope) *Type { switch n.Op { - case token.LAND: - // `a && b` is a when a is falsy, else b. - l := c.check(n.X, sc) - r := c.narrow(narrowings(n.X, true), n.Y, sc) - return Join(l, r) - case token.LOR: - // `a || b` is a when a is truthy (so never nil), else b. - l := c.check(n.X, sc) - r := c.narrow(narrowings(n.X, false), n.Y, sc) - return Join(l.NonNil(), r) + case token.LAND, token.LOR: + return c.logicalChain(n, sc) } l, r := c.check(n.X, sc), c.check(n.Y, sc) switch n.Op { @@ -385,6 +467,11 @@ func (c *checker) comparable(n *ast.BinaryExpr, l, r *Type) { return } } + // Two values that may both be nil can be equal whatever their + // non-nil types are. + if l.Optional && r.Optional { + return + } shared := false for _, a := range lk { for _, b := range rk { @@ -424,10 +511,13 @@ func (c *checker) field(n, recvNode ast.Expr, recv *Type, name string, optional case recv.isAny(): return Any case recv.Kind == KindNull: - if !optional && c.lenient == 0 { + if optional { + return Null() + } + if c.lenient == 0 { c.report(n, "the value is always nil") } - return Null() + return never } if !optional { c.nilCheck(recvNode, recv) @@ -489,20 +579,30 @@ func (c *checker) index(n, recvNode ast.Expr, recv *Type, idxNode ast.Expr, idx return Any } if recv.Kind == KindNull { - if !optional && c.lenient == 0 { + if optional { + return Null() + } + if c.lenient == 0 { c.report(n, "the value is always nil") } - return Null() + return never } if !optional { c.nilCheck(recvNode, recv) } // A nil index fails even after ?[. if idx.Kind == KindNull { - if c.lenient == 0 { + switch { + case recv.Kind != KindList && recv.Kind != KindString && recv.Kind != KindObject: + c.report(n, "cannot index %s", recv) + case c.lenient == 0: c.report(idxNode, "the index is always nil") } - return Any + if optional && recv.Optional { + // ?[ gives nil for a nil receiver without reading the index. + return Null() + } + return never } c.nilCheck(idxNode, idx) if !c.oneKind(idxNode, idx) { @@ -513,6 +613,14 @@ func (c *checker) index(n, recvNode ast.Expr, recv *Type, idxNode ast.Expr, idx case KindList, KindString: what := "list" out = recv.Elem + if out == empty { + // The list is always empty: [i] is out of range and ?[i] + // is nil. + out = never + if optional { + out = Null() + } + } if recv.Kind == KindString { what, out = "string", String() } @@ -551,6 +659,13 @@ func (c *checker) index(n, recvNode ast.Expr, recv *Type, idxNode ast.Expr, idx if recv.Open { out = Join(out, recv.Elem) } + if out == nil { + // A closed object with no fields: every key is absent. + if optional { + return Null() + } + return never + } default: c.report(n, "cannot index %s", recv) return Any @@ -723,16 +838,23 @@ func (c *checker) iterating(n *ast.CallExpr, form string, sc *scope) *Type { for k, v := range hidden { c.narrowed[k] = v } + if ct.Kind == KindNull || ct.Kind == KindList && ct.Elem == empty { + // The body never runs: the list is empty, or nil, which + // iterates as an empty list. + switch form { + case "find": + return Null() + case "map", "filter", "sortBy", "flatMap": + return ListOf(empty) + } + } switch form { case "map": return ListOf(bt) case "filter", "sortBy": return ListOf(elem) case "flatMap": - if bt.Kind == KindList { - return ListOf(bt.Elem) - } - return ListOf(Any) + return ListOf(flatElem(bt)) case "any", "all": return Bool() case "find": @@ -743,6 +865,30 @@ func (c *checker) iterating(n *ast.CallExpr, form string, sc *scope) *Type { return Any } +// flatElem is the element type of flatMap's result for a body of type bt: +// a list body adds its elements, nil adds nothing, and any other value is +// added as one element. +func flatElem(bt *Type) *Type { + if bt.isAny() { + return Any + } + var out *Type + for _, v := range variants(bt.NonNil()) { + switch { + case v.Kind == KindNull: + case v.Kind == KindList && v.Elem == empty: + case v.Kind == KindList: + out = Join(out, orAny(v.Elem)) + default: + out = Join(out, v) + } + } + if out == nil { + return empty + } + return out +} + // mentions reports whether the source text src refers to the name as an // identifier, not as a field (`x.name`) or inside a string. func mentions(src, name string) bool { @@ -783,11 +929,13 @@ func (c *checker) composite(n *ast.CompositeLit, sc *scope) *Type { } kt := c.check(kv.Key, sc) vt := c.check(kv.Value, sc) - if key, ok := literalString(kt); ok { + // A later key replaces an earlier one, so a literal key's + // field has exactly the value's type. + if key, ok := literalString(kt); ok && !kt.Optional { out.Fields[key] = vt + delete(out.Absent, key) continue } - out.Open = true switch { case kt.Kind == KindNull: c.report(kv.Key, "an object key is always nil") @@ -797,12 +945,54 @@ func (c *checker) composite(n *ast.CompositeLit, sc *scope) *Type { default: c.nilCheck(kv.Key, kt) } + c.computedKey(out, kt, orAny(vt)) } return out } return Any } +// computedKey adds to out, an object literal's type so far, an entry +// whose key is not a literal. The key may replace any field listed so far +// (only those in its enum, if it has one), so each such field may hold +// either type. A key from an enum adds each value of the enum as a field +// that may be absent; any other key may name any field, so the object is +// open and its unlisted fields, which may be absent, have the value's +// type. +func (c *checker) computedKey(out *Type, kt, vt *Type) { + enum := kt.Kind == KindString && len(kt.Enum) > 0 + for name, ft := range out.Fields { + if enum && !kt.hasEnumValue(name) { + continue + } + out.Fields[name] = Join(orAny(ft), vt) + } + if !enum { + if out.Open { + out.Elem = Join(orAny(out.Elem), vt) + } else { + out.Open, out.Elem = true, vt + } + return + } + for _, v := range kt.Enum { + name, _ := v.(string) + if _, ok := out.Fields[name]; ok { + continue + } + field := vt + if out.Open { + // The key may also hold an earlier computed key's value. + field = Join(orAny(out.Elem), vt) + } + out.Fields[name] = field + if out.Absent == nil { + out.Absent = map[string]bool{} + } + out.Absent[name] = true + } +} + func literalString(t *Type) (string, bool) { if t.Kind != KindString || len(t.Enum) != 1 { return "", false @@ -1005,31 +1195,54 @@ func entriesType(a []*Type) *Type { return ListOf(ObjectOf(map[string]*Type{"key": String(), "value": value})) } +// elemOrNil is the result of first and last: an element, or nil for an +// empty or nil list. func elemOrNil(a []*Type) *Type { - if len(a) == 1 && a[0].Kind == KindList { + switch { + case len(a) != 1: + return Any + case a[0].Kind == KindList && a[0].Elem == empty: + return Null() // the list is always empty + case a[0].Kind == KindList: return orAny(a[0].Elem).OrNil() + case a[0].Kind == KindNull: + return Null() } return Any } // listType is the result of sort and reverse: a new list, empty for nil. func listType(a []*Type) *Type { - if len(a) == 1 && a[0].Kind == KindList { + switch { + case len(a) == 1 && a[0].Kind == KindList: return ListOf(orAny(a[0].Elem)) + case len(a) == 1 && a[0].Kind == KindNull: + return ListOf(empty) } return ListOf(Any) } -// sliceType is a list for a list or nil, and a string for a string. +// sliceType is a list for a list or nil, and a string for a string. A +// union argument gives the union of those results. func sliceType(a []*Type) *Type { if len(a) == 0 || a[0].isAny() { return Any } - if a[0].Kind == KindString { - if a[0].Optional { - return Join(String(), ListOf(Any)) + var out, list *Type + for _, v := range variants(a[0]) { + switch v.Kind { + case KindString: + out = Join(out, String()) + case KindList: + list = Join(list, listType([]*Type{v})) } - return String() } - return listType(a[:1]) + // nil slices to an empty list, which fits any list element type. + if list == nil && (a[0].Optional || a[0].Kind == KindNull) { + list = ListOf(Any) + } + if out = Join(out, list); out == nil { + return ListOf(Any) + } + return out } diff --git a/check_regression_test.go b/check_regression_test.go new file mode 100644 index 0000000..c14aad2 --- /dev/null +++ b/check_regression_test.go @@ -0,0 +1,175 @@ +package expr + +import ( + "context" + "testing" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +func TestCheckRegressionNullableEquality(t *testing.T) { + for _, tc := range []struct { + name string + a, b *Type + }{ + {"different kinds", String().OrNil(), Number().OrNil()}, + {"disjoint enums", EnumOf("a").OrNil(), EnumOf("b").OrNil()}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, comparison := range []struct { + source string + want bool + }{ + {"a == b", true}, + {"a != b", false}, + } { + t.Run(comparison.source, func(t *testing.T) { + p, err := Compile(comparison.source) + require.NoError(t, err) + got, err := p.Run(context.Background(), map[string]any{"a": nil, "b": nil}) + require.NoError(t, err) + require.Equal(t, comparison.want, got) + + // Disjoint non-nil values still share nil, so comparing + // these nullable types is valid. + typ, err := p.Check(map[string]*Type{"a": tc.a, "b": tc.b}) + require.NoError(t, err) + require.Equal(t, KindBool, typ.Kind) + }) + } + }) + } +} + +func TestCheckRegressionDynamicKeyOverwrite(t *testing.T) { + p, err := Compile(`{"x": 1, k: "s"}.x + 1`) + require.NoError(t, err) + _, err = p.Run(context.Background(), map[string]any{"k": "x"}) + require.ErrorIs(t, err, ErrEvaluate) + + // The later dynamic key can replace x with a string. Its original + // numeric field type must not make the addition appear safe. + _, err = p.Check(map[string]*Type{"k": String()}) + require.ErrorIs(t, err, ErrType) +} + +func TestCheckRegressionNullableEnumObjectKey(t *testing.T) { + p, err := Compile(`{k: 1}`) + require.NoError(t, err) + _, err = p.Run(context.Background(), map[string]any{"k": nil}) + require.ErrorIs(t, err, ErrEvaluate) + + // Knowing the sole non-nil key does not make a nullable key safe. + _, err = p.Check(map[string]*Type{"k": EnumOf("x").OrNil()}) + require.ErrorIs(t, err, ErrType) +} + +func TestCheckRegressionSliceUnion(t *testing.T) { + const source = `slice(if(c, "abc", [1]), 0, 1)` + env := map[string]*Type{"c": Bool()} + + t.Run("result retains both kinds", func(t *testing.T) { + p, err := Compile(source, WithFunctions(CollectionFuncs())) + require.NoError(t, err) + for _, tc := range []struct { + condition bool + want any + }{ + {true, "a"}, + {false, []any{int64(1)}}, + } { + got, err := p.Run(context.Background(), map[string]any{"c": tc.condition}) + require.NoError(t, err) + require.Equal(t, tc.want, got) + } + + typ, err := p.Check(env) + require.NoError(t, err) + require.True(t, typ.IsUnion(), "slice must preserve its string and list result possibilities: "+typ.String()) + kinds := map[Kind]bool{} + for _, variant := range typ.Variants { + kinds[variant.Kind] = true + } + require.Equal(t, map[Kind]bool{KindString: true, KindList: true}, kinds) + }) + + t.Run("list-only consumer is rejected", func(t *testing.T) { + p, err := Compile("sum("+source+")", WithFunctions(CollectionFuncs())) + require.NoError(t, err) + _, err = p.Run(context.Background(), map[string]any{"c": true}) + require.ErrorIs(t, err, ErrEvaluate) + + _, err = p.Check(env) + require.ErrorIs(t, err, ErrType) + }) +} + +// Cases the soundness harness (check_soundness_test.go) found. Each pairs +// a schema with a value Run accepts and the type Check must now infer +// for it, or with a value on which Run fails and the problem Check must +// now report. +func TestCheckRegressionSoundness(t *testing.T) { + funcs := []Option{WithBuiltins(), WithFunctions(StringFuncs()), WithFunctions(MathFuncs()), WithFunctions(CollectionFuncs())} + for _, tc := range []struct { + name string + src string + schema map[string]*Type + value map[string]any + // want is the inferred type when Run succeeds, or a substring of + // the Check problem when Run fails. + want string + }{ + {"computed key types unlisted fields", `{"a": 2, k: c}?.b`, + map[string]*Type{"k": String(), "c": Bool()}, map[string]any{"k": "b", "c": true}, "bool?"}, + {"computed key field may be absent", `{"a": 2, k: c}.b`, + map[string]*Type{"k": String(), "c": Bool()}, map[string]any{"k": "a", "c": true}, `field "b" may be absent`}, + {"enum key adds absent fields", `{k: 1}?.b`, + map[string]*Type{"k": EnumOf("a", "b")}, map[string]any{"k": "b"}, "1?"}, + {"enum key field may be absent", `{k: 1}.b`, + map[string]*Type{"k": EnumOf("a", "b")}, map[string]any{"k": "a"}, `field "b" may be absent`}, + {"literal key after computed key is present", `{k: 1, "b": "s"}.b`, + map[string]*Type{"k": EnumOf("a", "b")}, map[string]any{"k": "b"}, `"s"`}, + {"computed key indexed by any string", `{"a": c, k: y}[trim(k)].type`, + map[string]*Type{"k": String(), "c": Bool(), "y": String()}, map[string]any{"k": "a", "c": true, "y": ""}, "the key may be absent"}, + {"first of nil", `first(x).a`, + map[string]*Type{"x": Null()}, map[string]any{"x": nil}, "the value is always nil"}, + {"find over nil", `-find(x, it)`, + map[string]*Type{"x": Null()}, map[string]any{"x": nil}, "nil is not a number"}, + {"find over a nil form result", `find(map(x, it), true).type`, + map[string]*Type{"x": Null()}, map[string]any{"x": nil}, "the value is always nil"}, + {"last of an empty list", `round(last(filter(nil, it)))`, + nil, nil, "argument 1 is always nil"}, + {"sort of nil", `-sort(nil)?[0]`, + nil, nil, "nil is not a number"}, + {"try of a nil index", `-try(x[nil], y)`, + map[string]*Type{"x": ListOf(Number()), "y": String()}, map[string]any{"x": []any{1}, "y": "s"}, "is not a number"}, + {"try of a nil index types as its default", `try(x[nil], y)`, + map[string]*Type{"x": ListOf(Number()), "y": String()}, map[string]any{"x": []any{1}, "y": "s"}, "string"}, + {"?[nil] of a nil receiver is nil", `try(x?[nil], 1)`, + map[string]*Type{"x": ListOf(Number()).OrNil()}, map[string]any{"x": nil}, "1?"}, + {"nil index of a number", `2.5?[z]`, + map[string]*Type{"z": Null(), "k": String()}, map[string]any{"z": nil, "k": "a"}, "cannot index 2.5"}, + {"flatMap of a non-list body", `first(flatMap(x, 0)).b`, + map[string]*Type{"x": ListOf(Number())}, map[string]any{"x": []any{1}}, `cannot read field "b"`}, + {"flatMap of a union body", `sum(flatMap(z, it))`, + map[string]*Type{"z": ListOf(Join(Number(), ObjectOf(map[string]*Type{"a": String()})))}, + map[string]any{"z": []any{1, map[string]any{"a": ""}}}, "elements that may be"}, + {"index of an empty object", `lower(o?[k])`, + map[string]*Type{"o": ObjectOf(map[string]*Type{}), "k": String()}, map[string]any{"o": map[string]any{}, "k": "a"}, "argument 1 is always nil"}, + } { + t.Run(tc.name, func(t *testing.T) { + p, err := Compile(tc.src, funcs...) + require.NoError(t, err) + got, runErr := p.Run(context.Background(), tc.value) + typ, checkErr := p.Check(tc.schema) + if runErr != nil { + require.ErrorIs(t, checkErr, ErrType, "Run failed, so Check must report it") + require.Contains(t, checkErr.Error(), tc.want) + return + } + require.NoError(t, checkErr) + require.Equal(t, tc.want, typ.String()) + require.True(t, fits(got, typ), "Run returned a value outside the inferred type") + }) + } +} diff --git a/check_soundness_test.go b/check_soundness_test.go new file mode 100644 index 0000000..3c121f0 --- /dev/null +++ b/check_soundness_test.go @@ -0,0 +1,742 @@ +package expr + +import ( + "context" + "errors" + "fmt" + "math/rand" + "os" + "reflect" + "sort" + "strconv" + "strings" + "testing" +) + +// The soundness harness checks the rule Check is built on: the type it +// infers includes every value Run can produce from inputs the schema +// allows, and an expression Check accepts does not fail at run time on +// the wrong kind of value, a nil receiver or a missing field. +// +// It pairs schemas with representative values (valuesFor), runs each +// expression Check accepts on every combination, and asserts each result +// fits the inferred type (fits). Seeds are fixed so the default run is +// deterministic; EXPR_SOUNDNESS_ITERS raises the number of random cases +// and EXPR_SOUNDNESS_SEED picks another seed. + +// fits reports whether the runtime value v belongs to t, following expr's +// runtime representation: integers and floats of any width are numbers, +// slices and arrays are lists, and string-keyed maps and structs are +// objects. +func fits(v any, t *Type) bool { + if t == nil || t == empty || (t.Kind == KindAny && len(t.Variants) == 0) { + return true + } + if v == nil { + return t.Optional || t.Kind == KindNull + } + if t.IsUnion() { + for _, variant := range t.Variants { + if fits(v, variant) { + return true + } + } + return false + } + rv := reflect.ValueOf(v) + for rv.Kind() == reflect.Pointer || rv.Kind() == reflect.Interface { + if rv.IsNil() { + return t.Optional || t.Kind == KindNull + } + rv = rv.Elem() + } + var value any + switch t.Kind { + case KindNull: + return false + case KindBool: + if rv.Kind() != reflect.Bool { + return false + } + value = rv.Bool() + case KindNumber: + switch { + case rv.CanInt(): + value = rv.Int() + case rv.CanUint(): + value = int64(rv.Uint()) + case rv.CanFloat(): + value = rv.Float() + default: + return false + } + case KindString: + if rv.Kind() != reflect.String { + return false + } + value = rv.String() + case KindList: + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return false + } + for i := 0; i < rv.Len(); i++ { + if !fits(rv.Index(i).Interface(), t.Elem) { + return false + } + } + return true + case KindObject: + return fitsObject(rv, t) + } + return len(t.Enum) == 0 || t.hasEnumValue(value) +} + +func fitsObject(rv reflect.Value, t *Type) bool { + fields := map[string]any{} + switch rv.Kind() { + case reflect.Map: + if rv.Type().Key().Kind() != reflect.String { + return false + } + iter := rv.MapRange() + for iter.Next() { + fields[iter.Key().String()] = iter.Value().Interface() + } + case reflect.Struct: + for i := 0; i < rv.NumField(); i++ { + name := rv.Type().Field(i).Tag.Get("json") + if name == "" { + name = rv.Type().Field(i).Name + } + fields[name] = rv.Field(i).Interface() + } + default: + return false + } + for name, ft := range t.Fields { + v, ok := fields[name] + if !ok { + if !t.Absent[name] { + return false + } + continue + } + if !fits(v, ft) { + return false + } + } + for name, v := range fields { + if _, listed := t.Fields[name]; listed { + continue + } + if !t.Open || !fits(v, t.Elem) { + return false + } + } + return true +} + +// valuesFor returns representative runtime values of t: nil when t may be +// nil, a value of every union variant, every enum value, empty and +// non-empty lists, objects with every field (and without each absent +// one), and extra keys in open objects. Some objects are structs. +func valuesFor(t *Type, rng *rand.Rand, depth int) []any { + var out []any + if t != nil && (t.Optional || t.Kind == KindNull) { + out = append(out, nil) + } + if t == nil || t.Kind == KindAny && len(t.Variants) == 0 { + return append(out, nil, int64(1), "s", true, []any{int64(1)}, map[string]any{"a": int64(1)}) + } + if t.IsUnion() { + for _, v := range t.Variants { + out = append(out, valuesFor(v, rng, depth)...) + } + return out + } + switch t.Kind { + case KindBool: + if len(t.Enum) > 0 { + return append(out, t.Enum...) + } + return append(out, true, false) + case KindNumber: + if len(t.Enum) > 0 { + return append(out, t.Enum...) + } + return append(out, int64(0), int64(3), 2.5, int(-1), int32(7), float32(0.5)) + case KindString: + if len(t.Enum) > 0 { + return append(out, t.Enum...) + } + return append(out, "", "a", "Hello") + case KindList: + out = append(out, []any{}) + if depth <= 0 { + return out + } + elems := valuesFor(t.Elem, rng, depth-1) + if len(elems) > 3 { + rng.Shuffle(len(elems), func(i, j int) { elems[i], elems[j] = elems[j], elems[i] }) + elems = elems[:3] + } + out = append(out, append([]any(nil), elems...)) + if len(elems) > 0 { + out = append(out, []any{elems[0]}) + } + if typed := typedSlice(t.Elem, elems); typed != nil { + out = append(out, typed) + } + return out + case KindObject: + return append(out, objectValues(t, rng, depth)...) + } + return out +} + +// typedSlice returns elems as a []string or []int64 when the element type +// allows it, so lists also arrive as typed Go slices. +func typedSlice(elem *Type, elems []any) any { + if elem == nil || elem.Optional || len(elems) == 0 { + return nil + } + switch elem.Kind { + case KindString: + out := make([]string, 0, len(elems)) + for _, e := range elems { + s, ok := e.(string) + if !ok { + return nil + } + out = append(out, s) + } + return out + case KindNumber: + out := make([]int64, 0, len(elems)) + for _, e := range elems { + n, ok := e.(int64) + if !ok { + return nil + } + out = append(out, n) + } + return out + } + return nil +} + +func objectValues(t *Type, rng *rand.Rand, depth int) []any { + names := make([]string, 0, len(t.Fields)) + for name := range t.Fields { + names = append(names, name) + } + sort.Strings(names) + choices := map[string][]any{} + most := 1 + for _, name := range names { + if depth <= 0 { + // Keep the values small: the first value of each field. + vs := valuesFor(t.Fields[name], rng, 0) + choices[name] = vs[:min(1, len(vs))] + } else { + choices[name] = valuesFor(t.Fields[name], rng, depth-1) + } + most = max(most, len(choices[name])) + } + most = min(most, 4) + var out []any + for i := 0; i < most; i++ { + m := map[string]any{} + for _, name := range names { + vs := choices[name] + if len(vs) == 0 { + continue + } + // The i-th variant omits each absent field in turn. + if t.Absent[name] && i%2 == 1 { + continue + } + m[name] = vs[(i+rng.Intn(len(vs)))%len(vs)] + } + if t.Open && i == most-1 { + extra := valuesFor(t.Elem, rng, 0) + m["extra"] = extra[rng.Intn(len(extra))] + } + out = append(out, m) + } + if soundStructs && !t.Open && len(t.Absent) == 0 && len(names) > 0 { + if s, ok := structValue(names, out[0].(map[string]any)); ok { + out = append(out, s) + } + } + return out +} + +// soundStructs adds struct values for closed objects. It is off in the +// random runs: Run reads struct fields with . and ?. but len, has, keys, +// entries and o["k"] reject structs, which Check types as objects. +// TestCheckSoundnessStructFields covers field reads. +var soundStructs = false + +// structValue builds a struct with a json tag per field, matching m. +func structValue(names []string, m map[string]any) (any, bool) { + fields := make([]reflect.StructField, len(names)) + for i, name := range names { + v, ok := m[name] + if !ok || v == nil { + return nil, false + } + fields[i] = reflect.StructField{ + Name: "F" + strconv.Itoa(i), + Type: reflect.TypeOf(v), + Tag: reflect.StructTag(`json:"` + name + `"`), + } + } + sv := reflect.New(reflect.StructOf(fields)).Elem() + for i, name := range names { + sv.Field(i).Set(reflect.ValueOf(m[name])) + } + return sv.Interface(), true +} + +// schemaGen draws random schemas of bounded depth. Field names come from +// a small pool so random expressions often name a real field. +type schemaGen struct{ rng *rand.Rand } + +var soundFieldNames = []string{"a", "b", "type"} + +func (g schemaGen) schema(depth int) *Type { + t := g.base(depth) + switch g.rng.Intn(6) { + case 0: + t = t.OrNil() + case 1: + if depth > 0 { + t = Join(t, g.base(depth-1)) + } + } + return t +} + +func (g schemaGen) base(depth int) *Type { + n := 6 + if depth <= 0 { + n = 4 + } + switch g.rng.Intn(n) { + case 0: + if g.rng.Intn(3) == 0 { + return &Type{Kind: KindBool, Enum: []any{true}} + } + return Bool() + case 1: + if g.rng.Intn(3) == 0 { + return &Type{Kind: KindNumber, Enum: []any{int64(1), int64(2)}} + } + return Number() + case 2: + if g.rng.Intn(3) == 0 { + return EnumOf("a", "b") + } + return String() + case 3: + if g.rng.Intn(8) == 0 { + return Null() + } + return Number() + case 4: + return ListOf(g.schema(depth - 1)) + default: + t := ObjectOf(map[string]*Type{}) + for _, name := range soundFieldNames { + if g.rng.Intn(3) > 0 { + t.Fields[name] = g.schema(depth - 1) + if g.rng.Intn(5) == 0 { + if t.Absent == nil { + t.Absent = map[string]bool{} + } + t.Absent[name] = true + } + } + } + if g.rng.Intn(4) == 0 { + t.Open = true + if g.rng.Intn(2) == 0 { + t.Elem = g.schema(depth - 1) + } + } + return t + } +} + +// typeHasAny reports whether t leaves anything unchecked. +func typeHasAny(t *Type) bool { + if t == nil || t.isAny() || t == empty { + return true + } + for _, v := range t.Variants { + if typeHasAny(v) { + return true + } + } + if t.Kind == KindList && typeHasAny(t.Elem) { + return true + } + if t.Kind == KindObject { + if t.Open && typeHasAny(t.Elem) { + return true + } + for _, f := range t.Fields { + if typeHasAny(f) { + return true + } + } + } + return false +} + +// exprGen draws random expressions over the soundness env's names. +type exprGen struct{ rng *rand.Rand } + +var soundVars = []string{"x", "y", "z", "k", "c"} + +func (g exprGen) pick(xs ...string) string { return xs[g.rng.Intn(len(xs))] } + +func (g exprGen) expr(depth int, bound string) string { + leaf := func() string { + vars := soundVars + if bound != "" { + vars = append([]string{bound, bound}, vars...) + } + switch g.rng.Intn(6) { + case 0: + return g.pick(`"a"`, `"b"`, `""`, `1`, `2`, `0`, `2.5`, `true`, `nil`, `[]`, `{}`) + default: + return vars[g.rng.Intn(len(vars))] + } + } + if depth <= 0 { + return leaf() + } + sub := func() string { return g.expr(depth-1, bound) } + field := func() string { return g.pick(soundFieldNames...) } + switch g.rng.Intn(19) { + case 0: + return leaf() + case 1: + return sub() + "." + field() + case 2: + return sub() + "?." + field() + case 3: + return sub() + "[" + sub() + "]" + case 4: + return sub() + "?[" + sub() + "]" + case 5: + return "(" + sub() + " " + g.pick("+", "-", "*", "/", "%", "==", "!=", "<", "<=", ">", ">=", "&&", "||") + " " + sub() + ")" + case 6: + return g.pick("!", "-") + sub() + case 7: + return "[" + sub() + ", " + sub() + "]" + case 8: + if g.rng.Intn(2) == 0 { + return `{"` + field() + `": ` + sub() + `, ` + sub() + `: ` + sub() + `}` + } + return `{"` + field() + `": ` + sub() + `}` + case 9: + return "if(" + sub() + ", " + sub() + ", " + sub() + ")" + case 10: + return "try(" + sub() + ", " + sub() + ")" + case 11: + form := g.pick("map", "filter", "any", "all", "find", "count", "sortBy", "flatMap") + if g.rng.Intn(3) == 0 { + return form + "(" + sub() + ", v, " + g.expr(depth-1, "v") + ")" + } + return form + "(" + sub() + ", " + g.expr(depth-1, "it") + ")" + case 12: + v := leaf() + return "(" + v + " != nil && " + v + "." + field() + ")" + case 13: + v := leaf() + return "if(" + v + " != nil, " + v + "." + field() + ", " + sub() + ")" + case 14: + return "(" + sub() + " || " + g.pick(`0`, `""`, `[]`, `{}`, `nil`, `false`) + ")" + case 15: + return "[" + sub() + "]" + default: + name := g.pick(soundBuiltinNames...) + n := len(builtinSigs[name].params) + if builtinSigs[name].variadic && g.rng.Intn(2) == 0 { + n++ + } + args := make([]string, n) + for i := range args { + args[i] = sub() + } + return name + "(" + strings.Join(args, ", ") + ")" + } +} + +var soundBuiltinNames = func() []string { + var names []string + for name := range builtinSigs { + names = append(names, name) + } + sort.Strings(names) + return names +}() + +// soundCorpus are hand-picked expressions that reach each part of the +// checker; random expressions add combinations. +var soundCorpus = []string{ + `x`, `x.a`, `x?.a`, `x?.a?.b`, `x.a.b`, `x[0]`, `x?[0]`, `x[k]`, `x?[k]`, `x["a"]`, `x?["type"]`, + `x.type`, `x?.type`, `x[0].a`, `x?[0]?.a`, + `x + y`, `x - y`, `x * y`, `x / y`, `x % y`, `-x`, `!x`, + `x == y`, `x != y`, `x < y`, `x >= y`, `x && y`, `x || y`, `x || 0`, `x || ""`, `x || []`, `x || {}`, + `x?.a || y`, `x != nil && x.a`, `x == nil || x.a`, `if(x != nil, x.a, nil)`, `if(x, y, z)`, + `if(c, x, y).a`, `if(c, x, y)?.a`, `try(x.a, y)`, `try(x[0], nil)`, + `[x, y]`, `[x, y][0]`, `[]`, `{}`, `{"a": x}`, `{"a": x, k: y}`, `{"a": x, k: y}.a`, + `{"a": x, k: y}?.a`, `{k: x}`, `{"a": x}.a`, `{"a": x, "a": y}.a`, `{x: y}`, `{"a": 1, x: 2}.a`, + `map(x, it.a)`, `map(x, v, v.a)`, `filter(x, it)`, `filter(x, it.a)`, `any(x, it)`, `all(x, it == y)`, + `find(x, it.a == y)`, `find(x, it)`, `count(x, it)`, `sortBy(x, it)`, `sortBy(x, it.a)`, + `flatMap(x, it)`, `flatMap(x, it.a)`, `flatMap(x, [it])`, `map(x, index)`, + `len(x)`, `string(x)`, `int(x)`, `float(x)`, `bool(x)`, `contains(x, y)`, `has(x, k)`, `keys(x)`, + `entries(x)`, `entries(x)[0].value`, `lower(x)`, `upper(x)`, `sprintf(x, y)`, `min(x, y)`, `max(x, y, z)`, + `abs(x)`, `floor(x)`, `ceil(x)`, `round(x)`, `trim(x)`, `split(x, y)`, `join(x, y)`, `replace(x, y, z)`, + `startsWith(x, y)`, `endsWith(x, y)`, `first(x)`, `last(x)`, `sum(x)`, `slice(x, 0, 1)`, `slice(x, y, z)`, + `sort(x)`, `reverse(x)`, `first(x).a`, `first(x)?.a`, `slice(if(c, x, y), 0, 1)`, + `x[y]`, `x?[y]`, `x[y]?.a`, `y[x]`, `x.a[k]`, `x?.a?[k]`, +} + +type soundCase struct { + src string + schema map[string]*Type +} + +// soundCheck runs one expression against a schema and every combination +// of values for it (capped), reporting violations of the soundness rule. +func soundCheck(t *testing.T, sc soundCase, rng *rand.Rand, maxCombos int) { + t.Helper() + p, err := Compile(sc.src, WithBuiltins(), WithFunctions(StringFuncs()), WithFunctions(MathFuncs()), + WithFunctions(CollectionFuncs()), WithFieldTags("json")) + if err != nil { + return + } + typ, err := p.Check(sc.schema) + if err != nil { + return + } + soundStats.accepted++ + names := make([]string, 0, len(sc.schema)) + for name := range sc.schema { + names = append(names, name) + } + sort.Strings(names) + values := map[string][]any{} + unchecked := false + for _, name := range names { + values[name] = valuesFor(sc.schema[name], rng, 2) + unchecked = unchecked || typeHasAny(sc.schema[name]) + } + // Walk the cartesian product, sampling when it is large. + total := 1 + for _, name := range names { + total *= len(values[name]) + } + for i := 0; i < min(total, maxCombos); i++ { + combo := i + if total > maxCombos { + combo = rng.Intn(total) + } + env := map[string]any{} + for _, name := range names { + vs := values[name] + env[name] = vs[combo%len(vs)] + combo /= len(vs) + } + got, err := runSafely(p, env) + soundStats.runs++ + if err == nil { + soundStats.ok++ + } + if err != nil { + if !unchecked && typeShapedErr(err) { + t.Errorf("%s: Check accepted %s but Run failed on %s: %v", sc.src, formatSchema(sc.schema), formatEnv(env), err) + return + } + continue + } + if !fits(got, typ) { + t.Errorf("%s: Check inferred %s, but Run returned %#v on %s (schema %s)", sc.src, typ, got, formatEnv(env), formatSchema(sc.schema)) + return + } + } +} + +// soundStats counts what the harness exercised, so a change that makes +// Check reject everything shows up in the log rather than as a pass. +var soundStats struct{ accepted, runs, ok int } + +func runSafely(p *Program, env map[string]any) (v any, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + return p.Run(context.Background(), env) +} + +// valueDependent are run-time errors Check cannot see: they depend on the +// value, not its kind, and the spec documents them as Run's to report. +var valueDependent = []string{ + "out of range", "division by zero", "modulo by zero", "integer overflow", + "cannot parse", "invalid syntax", "NaN", "Inf", "not a whole number", + "budget", "too deeply", "too large", "sprintf", + // A number index or slice bound may be fractional. + "must be integer", +} + +// typeShapedErr reports whether err is the kind of failure Check is meant +// to rule out: a wrong-kind operation, a nil receiver or a missing field. +func typeShapedErr(err error) bool { + if !errors.Is(err, ErrEvaluate) { + return true // panics and unwrapped errors are always bugs + } + msg := err.Error() + for _, s := range valueDependent { + if strings.Contains(msg, s) { + return false + } + } + return true +} + +func formatSchema(schema map[string]*Type) string { + names := make([]string, 0, len(schema)) + for name := range schema { + names = append(names, name) + } + sort.Strings(names) + parts := make([]string, len(names)) + for i, name := range names { + parts[i] = name + ": " + schema[name].String() + } + return "{" + strings.Join(parts, ", ") + "}" +} + +func formatEnv(env map[string]any) string { + names := make([]string, 0, len(env)) + for name := range env { + names = append(names, name) + } + sort.Strings(names) + parts := make([]string, len(names)) + for i, name := range names { + parts[i] = fmt.Sprintf("%s=%#v", name, env[name]) + } + return strings.Join(parts, " ") +} + +func soundEnvInt(name string, def int) int { + if s := os.Getenv(name); s != "" { + if n, err := strconv.Atoi(s); err == nil { + return n + } + } + return def +} + +func randomSchema(rng *rand.Rand) map[string]*Type { + g := schemaGen{rng} + schema := map[string]*Type{} + for _, name := range soundVars { + schema[name] = g.schema(2) + } + schema["c"] = Bool() + if rng.Intn(2) == 0 { + schema["k"] = String() + } + return schema +} + +// TestCheckSoundnessCorpus runs each corpus expression against random +// schemas. +func TestCheckSoundnessCorpus(t *testing.T) { + seed := int64(soundEnvInt("EXPR_SOUNDNESS_SEED", 1)) + schemas := soundEnvInt("EXPR_SOUNDNESS_ITERS", 60) + rng := rand.New(rand.NewSource(seed)) + for i := 0; i < schemas; i++ { + schema := randomSchema(rng) + for _, src := range soundCorpus { + soundCheck(t, soundCase{src, schema}, rng, 24) + } + } +} + +// TestCheckSoundnessRandom runs random expressions against random +// schemas. +func TestCheckSoundnessRandom(t *testing.T) { + seed := int64(soundEnvInt("EXPR_SOUNDNESS_SEED", 1)) + iters := soundEnvInt("EXPR_SOUNDNESS_ITERS", 60) * 150 + rng := rand.New(rand.NewSource(seed)) + g := exprGen{rng} + soundStats = struct{ accepted, runs, ok int }{} + for i := 0; i < iters; i++ { + soundCheck(t, soundCase{g.expr(3, ""), randomSchema(rng)}, rng, 16) + } + t.Logf("%d expressions: %d accepted by Check, %d runs, %d succeeded", iters, soundStats.accepted, soundStats.runs, soundStats.ok) + if soundStats.accepted < iters/20 { + t.Errorf("only %d of %d random expressions passed Check; the harness is not exercising the checker", soundStats.accepted, iters) + } +} + +// TestCheckSoundnessHarness pins the harness itself: values of a schema +// fit it, and fits rejects values outside it. +func TestCheckSoundnessHarness(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + g := schemaGen{rng} + for i := 0; i < 500; i++ { + s := g.schema(3) + for _, v := range valuesFor(s, rng, 3) { + if !fits(v, s) { + t.Fatalf("valuesFor(%s) produced %#v, which does not fit", s, v) + } + } + } + for _, tc := range []struct { + v any + t *Type + ok bool + }{ + {nil, String(), false}, + {nil, String().OrNil(), true}, + {"c", EnumOf("a", "b"), false}, + {int32(2), &Type{Kind: KindNumber, Enum: []any{int64(2)}}, true}, + {[]string{"a"}, ListOf(String()), true}, + {[]any{1}, ListOf(String()), false}, + {map[string]any{}, ObjectOf(map[string]*Type{"a": Number()}), false}, + {map[string]any{}, &Type{Kind: KindObject, Fields: map[string]*Type{"a": Number()}, Absent: map[string]bool{"a": true}}, true}, + {map[string]any{"z": 1}, ObjectOf(nil), false}, + {map[string]any{"z": 1}, OpenObject(nil), true}, + {"s", Join(Number(), String()), true}, + {true, Join(Number(), String()), false}, + } { + if got := fits(tc.v, tc.t); got != tc.ok { + t.Errorf("fits(%#v, %s) = %v, want %v", tc.v, tc.t, got, tc.ok) + } + } +} + +// TestCheckSoundnessStructFields reads fields of struct values, which Run +// supports for every closed object Check accepts. +func TestCheckSoundnessStructFields(t *testing.T) { + soundStructs = true + defer func() { soundStructs = false }() + rng := rand.New(rand.NewSource(1)) + g := schemaGen{rng} + for i := 0; i < 200; i++ { + schema := map[string]*Type{"x": g.schema(2), "y": g.schema(2)} + for _, src := range []string{`x.a`, `x?.a`, `x.a.b`, `x?.a?.type`, `x.type == y`, `[x.a, y.b]`, `x != nil && x.a`} { + soundCheck(t, soundCase{src, schema}, rng, 16) + } + } +} diff --git a/check_test.go b/check_test.go index ef2b140..bc227be 100644 --- a/check_test.go +++ b/check_test.go @@ -280,12 +280,13 @@ func TestCheckMatchesRun(t *testing.T) { "bx": ObjectOf(map[string]*Type{"x": String(), "y": Number()}), "nb": ObjectOf(map[string]*Type{"x": String(), "y": Number()}).OrNil(), "c": Bool(), + "e": EnumOf("x").OrNil(), // A host may mark a field absent itself: draft's title is // missing at run time. "draft": &Type{Kind: KindObject, Fields: map[string]*Type{"title": String()}, Absent: map[string]bool{"title": true}}, } values := map[string]any{ - "b": map[string]any{"y": "s"}, "bx": map[string]any{"x": "v", "y": 1}, "nb": nil, "c": false, "draft": map[string]any{}, + "b": map[string]any{"y": "s"}, "bx": map[string]any{"x": "v", "y": 1}, "nb": nil, "c": false, "e": nil, "draft": map[string]any{}, "s": nil, "w": "x", "xs": []any{1, 2}, "ys": nil, "a": map[string]any{"t": []any{"q"}, "n": nil}, "obj": map[string]any{"k": "v"}, "scores": map[string]any{"alice": 1}, "k": "bob", @@ -375,6 +376,23 @@ func TestCheckMatchesRun(t *testing.T) { `{"o": if(c, bx, b)}.o.x`: `field "x" may be absent; use ?.`, `draft.title`: `field "title" may be absent; use ?.`, `if(c, {"x": 1}, {}).x`: `field "x" may be absent; use ?.`, + // A computed key may replace an earlier field. + `{"bob": 1, k: "s"}.bob + 1`: "the operation needs one kind", + // A nullable key is not its sole non-nil value. + `{e: 1}`: "`e`: may be nil", + // slice keeps both kinds of a union argument. + `sum(slice(if(c, [1], "abc"), 0, 1))`: "may be string, and must be a list", + // Soundness harness finds: a computed key may name any field, + // and the value of an operation that always fails, or of a list + // that is always empty, is not unknown. + `{"bob": 1, k: c}.x`: `field "x" may be absent; use ?.`, + `first(nil).a`: "the value is always nil", + `-find(nil, it)`: "nil is not a number", + `-try(xs[nil], "s")`: `"s" is not a number`, + `first(flatMap(xs, 0)).b`: `cannot read field "b"`, + `round(last(filter(nil, it)))`: "argument 1 is always nil", + `lower({}?[k])`: "argument 1 is always nil", + `2.5?[nil]`: "cannot index 2.5", // The items of a list joined with [] are still checked. `any(xn || [], it.nope == 1)`: `no field "nope"`, `any(coalesce(xn, []), it.nope == 1)`: `no field "nope"`, @@ -395,6 +413,9 @@ func TestCheckMatchesRun(t *testing.T) { `"a" == 1`: "is never true", `"a" == "b"`: `never "b"`, `1 != "1"`: "is never true", + // first([]) and find([], ...) are always nil. + `if(c, first([]), "s") == 1`: "is never true", + `if(c, "s", find([], it > 1)) == 1`: "is never true", } { _, err := compile(src).Check(env) require.Error(t, err, src) @@ -440,10 +461,18 @@ func TestCheckMatchesRun(t *testing.T) { `filter(coalesce(xn, []), it.a > 0)`: "list<{a: number}>", `if(c, [], x)`: "list<{a: number}>", `[]`: "list", - // The element of an empty list is unknown, not nil-free: first([]) - // is nil at run time, so this comparison can be true. - `if(c, first([]), "s") == 1`: "bool", - `if(c, "s", find([], it > 1)) == 1`: "bool", + // first([]) and find([], ...) are nil at run time, so comparing + // them with nil can be true. + `if(c, first([]), "s") == nil`: "bool", + `if(c, "s", find([], it > 1)) == nil`: "bool", + // A computed key's value is the type of the fields it may name. + `{"bob": 1, k: "s"}?.x`: `"s"?`, + `{"bob": 1, k: "s"}.bob`: `1 | "s"`, + `{"bob": 1, k: "s"}?[k]`: `(1 | "s")?`, + `try(xs[nil], "s")`: `"s"`, + `flatMap(xs, [it, "s"])`: "list", + `flatMap(nil, it)`: "list", + `first(filter(nil, it))`: "nil", } { p := compile(src) got, err := p.Check(env, coalesceType) diff --git a/docs/reference/spec.md b/docs/reference/spec.md index fb90555..c18a176 100644 --- a/docs/reference/spec.md +++ b/docs/reference/spec.md @@ -90,7 +90,10 @@ Compile time with `ErrCompile`. - String vs string uses `strings` comparison. - Numeric comparisons work across any combination of integer kinds and - floats (see [Equality](#equality)). + floats (see [Equality](#equality)). Two integers compare exactly, + whatever their width or signedness, so `uint64` values above + `MaxInt64` order correctly; an integer and a float compare as + `float64`. - Other comparable Go types use native equality when both sides are the same type. Mismatched-but-comparable types yield `false` without an error. Uncomparable types (slices, maps, funcs) return `ErrEvaluate`. @@ -204,12 +207,17 @@ accessing a field on a piped result is written 2. `X == nil` is `true` if `X` is a typed nil value of a nilable kind (chan, func, interface, map, pointer, slice). This means a `(*T)(nil)` or `[]any(nil)` stored in `any` compares equal to the literal `nil`. -3. If both sides are any combination of integer or float kinds, they - convert to `float64` and compare. `int32(7) == int64(7) == float64(7)` - is `true`. -4. Otherwise: if both runtime types are comparable, Go's `==` is used. - Different types compare as `false` (no error). Uncomparable types - return `ErrEvaluate`. +3. If both sides are integers of any width or signedness (including + named integer types), they compare exactly: a value above 2^53 is + not rounded, and `uint64(18446744073709551615)` equals nothing but + itself. If either side is a float, both convert to `float64` and + compare. `int32(7) == int64(7) == float64(7)` is `true`. +4. Otherwise: if both values are comparable, Go's `==` is used. + Different types compare as `false` (no error). Uncomparable values + return `ErrEvaluate` — including a struct or array whose type is + comparable but which holds a slice, map or func in an interface + field (Go's `==` would panic on it). `contains` skips elements it + cannot compare. `!=` is exactly `!(==)`. @@ -282,16 +290,22 @@ rather than choosing one by declaration order. ## Selectors (`x.y`) -`x.y` evaluates `x`, then looks up `y` on the result: +`x.y` evaluates `x`, then looks up `y` on the result. The rules below +are shared by `x.y`, `x[i]` and the optional forms `x?.y` / `x?[i]`, on +every evaluation path: +- The receiver is unwrapped through pointers and interfaces at any + depth (`*T`, `**T`, an interface field holding a pointer). A nil met + along the way is a nil receiver. - Nil receiver → `ErrEvaluate`. - `map[string]any` or any map with string keys → `y` is a key. Missing keys return an error (not a zero value). - Map with non-string keys → `ErrEvaluate`. - Struct → the exported field `y`, using configured struct tags if any, or `ErrEvaluate` if missing. -- Pointer to struct → dereferenced and re-tried. Nil pointer → - `ErrEvaluate`. +- An unexported field, or a field promoted through a nil embedded + pointer, is not readable: the first is "not found", the second an + `ErrEvaluate` naming the nil pointer. - Anything else → `ErrEvaluate`. Selectors chain left-to-right: `a.b.c` is `(a.b).c`. Selector chains are @@ -306,9 +320,17 @@ internal use. This applies to field names as well. ## Index expressions (`x[i]`) +The receiver is unwrapped through pointers and interfaces as for +[selectors](#selectors-xy), so a `*[3]int` indexes like a `[3]int`. A +nil receiver → `ErrEvaluate`. + - `map[string]any`: `i` must be a string. Missing key → `ErrEvaluate`. - Other maps: `i` is converted to the map's key type if assignable or - convertible. A nil index on a typed map is an error (not a panic). + convertible. A nil index on a typed map is an error (not a panic), as + is a key that cannot be hashed (`m[[1]]` on a `map[any]any`) or a + slice too short to convert to an array key type. An integer key type + takes a float only when it is integral, as slices do: `m[1.0]` reads + `m[1]`, `m[1.5]` is an error. - Slice, array: `i` must be an integer or an integer-valued float (`xs[1.0]` works, `xs[1.5]` is an error). Negative indices and indices `>= len(x)` return `ErrEvaluate` (expr does not support @@ -363,7 +385,9 @@ before any reflect call that would panic. **range-checked** conversion to the declared parameter type. For example, `int64(10)` → `int8` succeeds; `int64(300)` → `int8` is an error (not a silent wraparound). Negative → unsigned fails. - `float64`↔`float32` is allowed and may lose precision. + `float64`↔`float32` is allowed and may lose precision. A float passed + to an integer parameter truncates toward zero, as Go's conversion + does, and must fit the type. - Nil may be passed for any nilable-kind parameter (interface, pointer, map, slice, chan, func). Passing nil to a non-nilable parameter is an error. @@ -381,8 +405,15 @@ Supported: replaces the normal result. Anything else (two non-error returns, three returns, `(error, T)`) -returns `ErrEvaluate`. Errors from functions propagate wrapped inside -an `ErrEvaluate` chain via `errors.Is`. +returns `ErrEvaluate` before the function is called. Errors from +functions propagate wrapped inside an `ErrEvaluate` chain via +`errors.Is`. + +Registered functions, functions stored in env or in a struct field, and +bound methods share one call path: the same argument checks and +conversions, the same result rules, and the same recovery of a runtime +panic in the callee as `ErrEvaluate`. A conversion failure is reported +before the function runs. ## Builtins @@ -412,7 +443,7 @@ The standard set is: | Name | Signature | Notes | | --------------- | ------------------------------- | ----- | | `len(v)` | `(any) -> int, error` | Rune count for strings, element count for slice/array/map/chan, `0` for nil, error otherwise. | -| `string(v)` | `(any) -> string` | Passthrough for strings, `fmt.Sprintf("%v", v)` otherwise, `""` for nil. | +| `string(v)` | `(any) -> string` | Passthrough for strings, `fmt.Sprintf("%v", v)` otherwise, `""` for nil. Map entries are ordered by rendered key, then key type, then rendered value, so the result is the same on every run. Cyclic, very deep and very large values render with markers (see [Limits and safety](#limits-and-safety)). | | `int(v)` | `(any) -> int64, error` | Numeric values convert (float truncates toward zero). Strings are parsed strictly with `strconv.ParseInt` base-10 (trimmed whitespace, no `0x`, no trailing garbage). | | `float(v)` | `(any) -> float64, error` | Like `int`, but `strconv.ParseFloat` 64-bit. | | `bool(v)` | `(any) -> bool` | Same semantics as [truthiness](#truthiness). | @@ -431,7 +462,7 @@ The standard set is: ### Registration validation `WithFunctions` entries are validated when `Compile` applies its -options. A nil entry, a value that is not a Go function, or a function +options. A nil entry (including a nil `Func`), a value that is not a Go function, or a function with an unsupported signature (more than two return values, or a second return that is not `error`) fails `Compile` with `ErrCompile` — the mistake surfaces at load time rather than on the expression's @@ -480,7 +511,7 @@ p, err := expr.Compile(src, | ----------------- | ------------------------------- | ----- | | `first(xs)` | `(list) -> any, error` | First element; `nil` for nil or empty lists. | | `last(xs)` | `(list) -> any, error` | Last element; `nil` for nil or empty lists. | -| `sum(xs)` | `(list) -> num, error` | Numeric sum. `int64` (overflow-checked) when every element is integral, `float64` otherwise. Nil/empty → `0`. | +| `sum(xs)` | `(list) -> num, error` | Numeric sum. `int64` (overflow-checked) when every element is integral, `float64` otherwise (in any element order, without integer overflow). Nil/empty → `0`. | | `slice(xs, i, j)` | `(list\|string, int, int) -> list\|string, error` | Half-open range `[i, j)`. Lists yield `[]any`; strings slice by rune. Negative indices count from the end; out-of-range bounds clamp; `i > j` → empty. Covers the rejected `xs[i:j]` syntax. | | `sort(xs)` | `(list) -> []any, error` | Ascending stable sort. All elements must be numbers (any int/float mix, compared numerically) or all strings (lexicographic). Elements are not converted: ints stay ints. Nil/empty → `[]any{}`. Mixed or non-comparable types → error. Never mutates the input. | | `reverse(xs)` | `(list) -> []any, error` | Reversed copy. Never mutates the input. Nil/empty → `[]any{}`. | @@ -693,8 +724,10 @@ config?.feature?.enabled The semantics: -- If the receiver is `nil`, the result is `nil` and the right-hand - side is not consulted. +- If the receiver is `nil` — including a typed nil that `== nil` + reports as nil, and a nil met while unwrapping pointers — the result + is `nil` and the right-hand side (the index expression) is not + evaluated. - For `?.`, a missing struct field or absent map key resolves to `nil`. - For `?[`, a missing map key or an out-of-range slice/string index @@ -706,8 +739,9 @@ The semantics: The receiver can be any primary expression: an identifier, a selector or index chain, a call (including `map(...)` and `if(...)`), a -parenthesized group, or an array/object literal (`[1, 2]?[0]`, -`{"a": 1}?.a`). `?.map` and `?.if` work the same way `.map` and +parenthesized group, an array/object literal (`[1, 2]?[0]`, +`{"a": 1}?.a`), or a typed literal (`[]any{1}?[0]`, +`map[string]any{}?.a`). `?.map` and `?.if` work the same way `.map` and `.if` do. Calling the result of an optional access (`a?.b()`) is not supported and is rejected at Compile time. @@ -757,10 +791,12 @@ Check reports, as a `*TypeError` wrapping `ErrType`, every: with a typed `Elem`, a field in `Absent` (which Join sets for a field only one side has), or an index by a string that is not a literal (`scores[name]`), unless read with `?.`/`?[` or inside `try`; -- object literal key that is not a string or may be nil; +- object literal key that is not a string or may be nil (a key typed + `"x"?` may be nil too); - `==`/`!=` that can never be true: different kinds, or a literal outside the other side's enum (`kind == "bugfix"` when kind is - `"bug_fix" | "feature"`). + `"bug_fix" | "feature"`). Two sides that may both be nil can be equal, + so they are not reported whatever their non-nil types. - registered function name used as a value instead of called (`first.id`, `[upper, 1]`). An env entry or bound name with the same name wins. @@ -781,7 +817,27 @@ an open object does not list may be absent, so it is read with `?.`. Joining two objects (through `if`, `||`, a list literal or a function typed with `Join`) keeps the fields both have, as present; a field only one has is in `Absent`, so `if(c, {"x": 1}, {}).x` is reported and -`if(c, {"x": 1}, {})?.x` types as `1?`. +`if(c, {"x": 1}, {})?.x` types as `1?`. In an object literal, a key that +is not a literal may replace any field listed before it, so in +`{"x": 1, k: "s"}` the field `x` types as `1 | "s"`; a literal key after it +sets the field's type outright. Such a key may also name a field not +listed, so the object is open with the key's value as the type of its +unlisted fields, which may be absent: `{"x": 1, k: "s"}?.y` types as +`"s"?`, and `.y` is reported. A key typed as an enum instead adds each of +its values as a field that may be absent. `slice` of a union of a string +and a list types as the union of a string and a list. + +The rule behind these types: the type Check infers includes every value +Run can produce from inputs the env types allow — each value that may be +nil, each member of a union, and each field a computed key may replace. A +value that is always nil is typed as nil, not left unchecked: `first`, +`last` and `find` of a nil or empty list (`find(nil, ...)`, +`first(filter(nil, ...))`), and `o?[k]` on an object with no fields. An +operation that always fails (indexing with nil) produces no value, so in +`try(xs[nil], "s")` the result types as `"s"`. `flatMap` adds a list +body's elements and any other non-nil body as one element, so +`flatMap(xs, [it, "s"])` types as `list`. `check_soundness_test.go` +checks the rule against generated schemas, values and expressions. `==` and `!=` on lists or objects are reported, since Run rejects them. `Any` disables checking for the value and everything derived from it, so a @@ -836,6 +892,35 @@ The following are hard limits: do not stop nested higher-order forms from multiplying work (`map(xs, map(xs, map(xs, it)))` is `len(xs)³` predicate evaluations from a ~30-byte expression). The default is unlimited. +- **Compile cost**: `Compile` and `NewTemplate` run in time and memory + about linear in the source length, whatever its shape (deeply nested + literals, long `?.` chains, many template segments). The source + rewrites expand their input by at most a constant factor. + +Compilation, evaluation and formatting are bounded separately: + +- **Compile** (`Compile`, `Check`, `Identifiers`, `Calls`): the source + length bounds the parser, the rewrites and every walk of the syntax + tree. Each of them takes time and memory near-linear in the source, + so a maximal source compiles in well under a second. Error messages + quote a higher-order predicate only when it is small (64 syntax + nodes or fewer), so an error passing through deeply nested forms + does not print the expression again at each level. `Calls` prints an + argument's `Source` only when it is asked for. +- **Run**: `MaxEvalDepth` bounds recursion and `WithEvalBudget` bounds + work. Unwrapping pointers stops after 64 levels, so a + self-referential pointer type fails as an unsupported receiver + instead of looping. +- **Formatting** (`string(v)`, `sprintf`, template rendering): host + values can be cyclic, deeply nested or share structure, and + formatting cannot fail, so the bounds show up in the text. A + reference already being rendered higher up prints `` + (including through map keys); nesting deeper than 1000 levels prints + `<...>`; after 2^20 values the rest prints ``. Structure + shared without a cycle (a list holding the same list twice, nested + 64 times) would otherwise take time exponential in the value's size. + Templates render composites as JSON only within the same bounds and + fall back to this formatting outside them. Under adversarial input, expr must never: @@ -845,6 +930,11 @@ Under adversarial input, expr must never: - Expose unexported struct fields. See `FuzzCompile` and `FuzzEval` for the enforcing test targets. +`TestIsolatedCrashRegressions` runs each crash-shaped case (cyclic, +deep and shared values through every formatter and evaluator path; +maximal sources through Compile, Check and Run) in a child process +with a 64 MiB stack, so a regression into unbounded recursion fails +one case instead of killing the test run. ## Cancellation and termination guarantees @@ -1147,7 +1237,13 @@ for _, c := range p.Calls("ask") { ## Error model All runtime failures wrap `ErrEvaluate`; all parse failures wrap -`ErrCompile`. Callers check with `errors.Is(err, ErrEvaluate)` / -`errors.Is(err, ErrCompile)`. User function errors returned from -`(T, error)` signatures are wrapped such that both `errors.Is` and -`errors.As` still find the original cause. +`ErrCompile`. A compile error names the 1-based `line:column` of the +problem in the source as written: expr rewrites JSON literals, `?.`/`?[`, +keyword fields and `map`/`if` before parsing, but positions are mapped back +through those rewrites. In a template, the position is within the +`${...}` body. Callers check with `errors.Is(err, ErrEvaluate)` / +`errors.Is(err, ErrCompile)`. Errors returned by host functions — the +error of a `(T, error)` signature, including a non-pointer error value, +and the error of a `Func` — are wrapped such that both `errors.Is` and +`errors.As` still find the original cause. An error that already wraps +`ErrEvaluate`, or a context cancellation error, is returned as is. diff --git a/engine.go b/engine.go index 8989696..44c9957 100644 --- a/engine.go +++ b/engine.go @@ -50,10 +50,12 @@ import ( "go/parser" "go/scanner" "go/token" + "sort" "strings" "github.com/deepnoodle-ai/expr/internal/jsonlit" "github.com/deepnoodle-ai/expr/internal/optaccess" + "github.com/deepnoodle-ai/expr/internal/srcedit" ) // ErrCompile wraps parse failures so callers can match with errors.Is. @@ -90,6 +92,9 @@ type compileConfig struct { fieldTags *structTagConfig evalBudget int errs []error + // noOptimize turns off constant folding and every evaluator fast + // path, so tests can compare optimized and general evaluation. + noOptimize bool tmplOpen string tmplClose string @@ -144,6 +149,15 @@ func WithFunctions(funcs map[string]any) Option { } } +// withoutOptimizations compiles a program that skips constant folding +// and every evaluator fast path (prepared-call cache, reflect-space +// selector and index chains, filter(...)[k] streaming, typed binary +// operators). Results must not depend on it; the differential tests in +// optimize_equiv_test.go hold that. +func withoutOptimizations() Option { + return func(c *compileConfig) { c.noOptimize = true } +} + // WithEvalBudget bounds the total work a single Run may perform. Each // AST node evaluated — including every per-element re-evaluation of a // higher-order form's predicate — consumes one unit; when the budget @@ -209,6 +223,42 @@ func Compile(code string, opts ...Option) (*Program, error) { // set. NewTemplate uses it to compile every `${...}` segment without // re-applying the option closures per segment (and without tripping // the template-only option check that guards Compile). +// +// # Compile pipeline +// +// Source passes through four token-level rewrites before go/parser sees +// it, then three tree passes. Each rewrite scans with go/scanner (so the +// contents of strings, runes and comments are never touched), plans +// splices with internal/srcedit, and returns a map from its output +// offsets back to its input, so errors report the position the user +// wrote. The stages, in order: +// +// 1. rewriteKeywordFields — input: user source. Renames a Go keyword +// that follows a `.` (`x.type`, `x.map`) to __expr_kw_. It +// skips the `.` of `?.`, whose field optaccess quotes instead. +// Output: no keyword follows a plain `.`. +// 2. optaccess.RewriteMap — input may contain `?.f` / `?[i]` (f may be +// a keyword) and __expr_kw_ fields. Emits __try_select__(lhs, "f") +// and __try_index__(lhs, i), finding lhs by walking back over +// identifiers, selector chains, balanced brackets, calls (including +// `map(`/`if(`, still keywords here), bare [..] / {..} literals and +// typed []T{} / [N]T{} / map[K]V{} literals. Output: no `?.`/`?[` +// left except ones it declined (which the parser rejects). +// 3. jsonlit.RewriteMap — input: may contain __try_* calls and the +// `map`/`if` keywords. Rewrites bare `[..]` to []any{..} and bare +// `{..}` to map[string]any{..}, leaving index brackets, typed +// literals and type expressions alone. It relies on stage 2 having +// removed `?`, so a `[` after `?` never reaches it. +// 4. preprocessSource — input: may contain map[string]any{ from stage +// 3. Renames the `map` keyword (except `map[`, a map type) and `if` +// to the sentinels __expr_map__ and __expr_if__. Output: valid Go +// expression syntax for every accepted form. +// +// After parsing: restoreKeywordFields gives __expr_kw_ selectors their +// real names; desugarPipes turns `a | f(x)` into f(a, x), so later +// passes see only calls; validate rejects Go syntax expr does not +// support. The evaluator dispatches the sentinels as special forms and +// displayIdent shows their user-visible names in errors. func compileWithConfig(code string, cfg *compileConfig) (*Program, error) { if len(code) > MaxSourceLength { return nil, fmt.Errorf("%w: source length %d exceeds maximum %d", @@ -217,27 +267,23 @@ func compileWithConfig(code string, cfg *compileConfig) (*Program, error) { if len(cfg.errs) > 0 { return nil, fmt.Errorf("%w: %w", ErrCompile, errors.Join(cfg.errs...)) } - // Pipeline order: rewriteKeywordFields renames keyword field names - // (`x.type`) first, so the later rewrites see an identifier there; - // optaccess turns `?.`/`?[` into sentinel calls while the source - // still uses raw operator syntax; jsonlit then rewrites bare - // composite literals; preprocessSource handles the keyword rewrites - // (`map`) so the parser accepts them as identifiers. - parsed := preprocessSource(jsonlit.Rewrite(optaccess.Rewrite(rewriteKeywordFields(code)))) + parsed, srcMap := rewriteSource(code) fset := token.NewFileSet() node, err := parser.ParseExprFrom(fset, "", parsed, 0) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrCompile, err) + return nil, fmt.Errorf("%w: %v", ErrCompile, sourceErrors(err, code, srcMap)) } - restoreKeywordFields(node) - // Desugar pipe expressions (`a | f(x)` → `f(a, x)`) before - // validation so validate and the evaluator only ever see ordinary - // call nodes. See pipe.go. - node, err = desugarPipes(fset, node) - if err != nil { - return nil, err + node, err = checkTree(fset, node) + if err != nil && !srcMap.Identity() { + // The passes report positions through fset. Point them at code + // and run them again on a fresh tree, so a successful compile + // pays nothing for the translation. + fset = token.NewFileSet() + node, _ = parser.ParseExprFrom(fset, "", parsed, 0) + mapPositions(fset.File(node.Pos()), code, srcMap) + _, err = checkTree(fset, node) } - if err := validate(fset, node); err != nil { + if err != nil { return nil, err } p := &Program{ @@ -247,6 +293,7 @@ func compileWithConfig(code string, cfg *compileConfig) (*Program, error) { prepared: cfg.prepared, fieldTags: cfg.fieldTags, evalBudget: cfg.evalBudget, + noOptimize: cfg.noOptimize, } p.compile() return p, nil @@ -280,17 +327,86 @@ const ( tryIndexFormName = "__try_index__" ) +// checkTree runs the tree passes of the compile pipeline on a freshly +// parsed node: keyword field names, pipes, validation. +func checkTree(fset *token.FileSet, node ast.Expr) (ast.Expr, error) { + restoreKeywordFields(node) + // Desugar pipe expressions (`a | f(x)` → `f(a, x)`) before + // validation so validate and the evaluator only ever see ordinary + // call nodes. See pipe.go. + node, err := desugarPipes(fset, node) + if err != nil { + return nil, err + } + if err := validate(fset, node); err != nil { + return nil, err + } + return node, nil +} + +// rewriteSource runs the token-level rewrites of the compile pipeline +// (see compileWithConfig) and returns the parser's input with the map +// from its offsets back to code. +func rewriteSource(code string) (string, srcedit.Map) { + s1, m1 := rewriteKeywordFields(code) + s2, m2 := optaccess.RewriteMap(s1) + s3, m3 := jsonlit.RewriteMap(s2) + s4, m4 := preprocessSource(s3) + return s4, m4.Then(m3).Then(m2).Then(m1) +} + +// sourceErrors moves the positions in a parse error from the rewritten +// source to code. +func sourceErrors(err error, code string, m srcedit.Map) error { + list, ok := err.(scanner.ErrorList) + if !ok { + return err + } + lines := lineStarts(code) + out := make(scanner.ErrorList, len(list)) + for i, e := range list { + c := *e + c.Pos.Offset, c.Pos.Line, c.Pos.Column = sourcePosition(lines, len(code), m.Orig(e.Pos.Offset)) + out[i] = &c + } + return out +} + +// mapPositions records on file, at the start of each rewritten segment, +// the line and column in code that it came from, so fset.Position +// reports positions in code. +func mapPositions(file *token.File, code string, m srcedit.Map) { + lines := lineStarts(code) + m.Segments(func(out, in int, _ bool) { + _, line, col := sourcePosition(lines, len(code), in) + file.AddLineColumnInfo(out, "", line, col) + }) +} + +// lineStarts returns the offset of the start of each line of code. +func lineStarts(code string) []int { + lines := []int{0} + for i := 0; i < len(code); i++ { + if code[i] == '\n' { + lines = append(lines, i+1) + } + } + return lines +} + +// sourcePosition is the offset (clamped to the source), 1-based line and +// 1-based byte column of off. +func sourcePosition(lines []int, size, off int) (offset, line, col int) { + off = max(0, min(off, size)) + i := sort.SearchInts(lines, off+1) - 1 + return off, i + 1, off - lines[i] + 1 +} + // keywordRewrites lists the Go keyword tokens that expr accepts as -// ordinary identifiers. Each entry maps the source spelling to its -// internal sentinel; preprocessSource walks tokens and substitutes -// occurrences in place. -var keywordRewrites = []struct { - tok token.Token - src string - internal string -}{ - {token.MAP, "map", mapFormName}, - {token.IF, "if", ifFuncName}, +// ordinary identifiers, with the internal sentinel each becomes. +var keywordRewrites = map[token.Token]string{ + token.MAP: mapFormName, + token.IF: ifFuncName, } // preprocessSource rewrites Go keyword tokens that expr wants to @@ -307,54 +423,28 @@ var keywordRewrites = []struct { // silently turned into a meaningless call. `if` has no analogous // safe context inside expr expressions, so every `if` token is // rewritten unconditionally. -func preprocessSource(src string) string { +func preprocessSource(src string) (string, srcedit.Map) { // Fast path: skip the scanner unless the source mentions one of // the rewritable keywords. Substring checks on short inputs are // much cheaper than spinning up go/scanner. - if !containsAnyRewriteKeyword(src) { - return src - } - fs := token.NewFileSet() - file := fs.AddFile("expr", fs.Base(), len(src)) - var s scanner.Scanner - s.Init(file, []byte(src), nil, 0) - - type tokInfo struct { - pos token.Pos - tok token.Token - } - var toks []tokInfo - for { - pos, tok, _ := s.Scan() - if tok == token.EOF { - break - } - toks = append(toks, tokInfo{pos, tok}) + if !strings.Contains(src, "map") && !strings.Contains(src, "if") { + return src, srcedit.Map{} } - - var out strings.Builder - out.Grow(len(src) + 16) - last := 0 - for i := 0; i < len(toks); i++ { - rule, ok := matchRewrite(toks[i].tok) + toks := srcedit.Scan(src) + var edits []srcedit.Edit + for i, t := range toks { + internal, ok := keywordRewrites[t.Kind] if !ok { continue } // Leave `map[...]` alone so Go map type literals continue to // produce a normal "unsupported syntax" error at eval time. - if rule.tok == token.MAP && i+1 < len(toks) && toks[i+1].tok == token.LBRACK { + if t.Kind == token.MAP && i+1 < len(toks) && toks[i+1].Kind == token.LBRACK { continue } - off := file.Offset(toks[i].pos) - out.WriteString(src[last:off]) - out.WriteString(rule.internal) - last = off + len(rule.src) - } - if last == 0 { - return src + edits = append(edits, srcedit.Edit{Pos: t.Pos, End: t.End, Str: internal}) } - out.WriteString(src[last:]) - return out.String() + return srcedit.Apply(src, edits) } // keywordFieldPrefix marks a field name that is a Go keyword. Go's parser @@ -366,38 +456,56 @@ const keywordFieldPrefix = "__expr_kw_" // rewriteKeywordFields renames every Go keyword that directly follows a // `.` (`event.type`, `item.range`, `x.map`) so the expression parses. It // leaves `?.type` alone: optaccess puts that name in a string literal. -func rewriteKeywordFields(src string) string { - if !strings.Contains(src, ".") { - return src +func rewriteKeywordFields(src string) (string, srcedit.Map) { + if !mayHaveKeywordField(src) { + return src, srcedit.Map{} } - fs := token.NewFileSet() - file := fs.AddFile("expr", fs.Base(), len(src)) - var s scanner.Scanner - s.Init(file, []byte(src), nil, 0) - var out strings.Builder - last := 0 - // period is set when the previous token is a `.` that is not part - // of `?.`. - period, question := false, false - for { - pos, tok, lit := s.Scan() - if tok == token.EOF { - break + toks := srcedit.Scan(src) + var edits []srcedit.Edit + for i := 1; i < len(toks); i++ { + t, prev := toks[i], toks[i-1] + if !t.Kind.IsKeyword() || prev.Kind != token.PERIOD { + continue } - if period && tok.IsKeyword() { - off := file.Offset(pos) - out.WriteString(src[last:off]) - out.WriteString(keywordFieldPrefix + lit) - last = off + len(lit) + if i >= 2 && toks[i-2].Kind == token.ILLEGAL && toks[i-2].Lit == "?" { + continue } - period = tok == token.PERIOD && !question - question = tok == token.ILLEGAL && lit == "?" + edits = append(edits, srcedit.Edit{Pos: t.Pos, End: t.End, Str: keywordFieldPrefix + t.Lit}) } - if last == 0 { - return src + return srcedit.Apply(src, edits) +} + +// mayHaveKeywordField is a cheap pre-check for rewriteKeywordFields: it +// reports whether some `.` is followed, after spaces, by a word that is a +// Go keyword, or whether src has a comment (which may sit between the two). +// A false positive only costs a scan. +func mayHaveKeywordField(src string) bool { + if strings.Contains(src, "/") { + return strings.Contains(src, ".") + } + for i := strings.IndexByte(src, '.'); i >= 0; { + j := i + 1 + for j < len(src) && (src[j] == ' ' || src[j] == '\t' || src[j] == '\n' || src[j] == '\r') { + j++ + } + k := j + for k < len(src) && 'a' <= src[k] && src[k] <= 'z' { + k++ + } + if k > j && token.Lookup(src[j:k]).IsKeyword() && (k == len(src) || !isIdentByte(src[k])) { + return true + } + next := strings.IndexByte(src[i+1:], '.') + if next < 0 { + break + } + i += 1 + next } - out.WriteString(src[last:]) - return out.String() + return false +} + +func isIdentByte(c byte) bool { + return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c >= 0x80 } // restoreKeywordFields undoes rewriteKeywordFields on the parsed tree. @@ -413,32 +521,6 @@ func restoreKeywordFields(node ast.Node) { }) } -func containsAnyRewriteKeyword(src string) bool { - for _, r := range keywordRewrites { - if strings.Contains(src, r.src) { - return true - } - } - return false -} - -func matchRewrite(tok token.Token) (struct { - tok token.Token - src string - internal string -}, bool) { - for _, r := range keywordRewrites { - if r.tok == tok { - return r, true - } - } - return struct { - tok token.Token - src string - internal string - }{}, false -} - // displayIdent converts an internal rewritten identifier back to the // name the user originally typed, for use in error messages and // field/method lookups. diff --git a/format.go b/format.go index f92cc52..2918962 100644 --- a/format.go +++ b/format.go @@ -1,6 +1,8 @@ package expr import ( + "encoding" + "encoding/json" "fmt" "reflect" "sort" @@ -8,21 +10,70 @@ import ( "strings" ) +// Formatting renders host values that may be cyclic, deeply nested, or +// share structure, so it carries three bounds through every recursive +// step, including the rendering of map keys: +// +// - cycles: a pointer, map or slice already being rendered higher up +// renders as ; +// - depth: a value nested more than maxFormatDepth levels renders as +// <...> below that level, so the recursion's stack use is bounded; +// - work: after maxFormatNodes values the rest renders as . +// Structure shared without a cycle (a list holding the same list +// twice, nested 64 times) is otherwise rendered in full, which is +// exponential in the input's size. +// +// Formatting cannot fail, so the bounds show up as markers in the text. +const ( + maxFormatDepth = 1000 + maxFormatNodes = 1 << 20 +) + type visit struct { typ reflect.Type ptr uintptr } +// formatState is the state one top-level formatting call threads through +// its recursion. +type formatState struct { + seen map[visit]bool + depth int + nodes int +} + +func newFormatState() *formatState { return &formatState{seen: map[visit]bool{}} } + func safeFormatValue(v any) string { if v == nil { return "" } var b strings.Builder - formatValue(&b, reflect.ValueOf(v), map[visit]bool{}) + newFormatState().format(&b, reflect.ValueOf(v)) return b.String() } -func formatValue(b *strings.Builder, rv reflect.Value, seen map[visit]bool) { +// format renders rv. Every recursive call goes through here, so the +// depth and work bounds apply to all of them. +func (st *formatState) format(b *strings.Builder, rv reflect.Value) { + if st.nodes >= maxFormatNodes { + if st.nodes == maxFormatNodes { + b.WriteString("") + st.nodes++ + } + return + } + st.nodes++ + if st.depth >= maxFormatDepth { + b.WriteString("<...>") + return + } + st.depth++ + defer func() { st.depth-- }() + st.formatValue(b, rv) +} + +func (st *formatState) formatValue(b *strings.Builder, rv reflect.Value) { if !rv.IsValid() { b.WriteString("") return @@ -55,37 +106,56 @@ func formatValue(b *strings.Builder, rv reflect.Value, seen map[visit]bool) { b.WriteString("") return } - v, ok := enterVisit(rv, seen) + v, ok := st.enter(rv) if !ok { b.WriteString("") return } - defer delete(seen, v) + defer delete(st.seen, v) b.WriteByte('&') - formatValue(b, rv.Elem(), seen) + st.format(b, rv.Elem()) case reflect.Map: if rv.IsNil() { b.WriteString("map[]") return } - v, ok := enterVisit(rv, seen) + v, ok := st.enter(rv) if !ok { b.WriteString("") return } - defer delete(seen, v) + defer delete(st.seen, v) b.WriteString("map[") + // Format each entry once, with this map already marked as seen: + // a key that leads back to the map renders instead of + // recursing. Entries sort by rendered key, then key type, then + // rendered value, so keys that render alike (int 1 and uint16 1 + // in a map[any]V, or NaNs) do not fall back to map order. keys := rv.MapKeys() - sort.Slice(keys, func(i, j int) bool { - return safeFormatValue(keys[i].Interface()) < safeFormatValue(keys[j].Interface()) - }) + entries := make([]mapEntry, len(keys)) for i, k := range keys { + var kb, vb strings.Builder + st.format(&kb, k) + st.format(&vb, rv.MapIndex(k)) + entries[i] = mapEntry{key: kb.String(), keyType: dynamicTypeName(k), value: vb.String()} + } + sort.Slice(entries, func(i, j int) bool { + a, b := entries[i], entries[j] + if a.key != b.key { + return a.key < b.key + } + if a.keyType != b.keyType { + return a.keyType < b.keyType + } + return a.value < b.value + }) + for i, e := range entries { if i > 0 { b.WriteByte(' ') } - formatValue(b, k, seen) + b.WriteString(e.key) b.WriteByte(':') - formatValue(b, rv.MapIndex(k), seen) + b.WriteString(e.value) } b.WriteByte(']') case reflect.Slice: @@ -93,15 +163,15 @@ func formatValue(b *strings.Builder, rv reflect.Value, seen map[visit]bool) { b.WriteString("[]") return } - v, ok := enterVisit(rv, seen) + v, ok := st.enter(rv) if !ok { b.WriteString("") return } - defer delete(seen, v) - formatList(b, rv, seen) + defer delete(st.seen, v) + st.formatList(b, rv) case reflect.Array: - formatList(b, rv, seen) + st.formatList(b, rv) case reflect.Struct: b.WriteByte('{') for i := 0; i < rv.NumField(); i++ { @@ -113,7 +183,7 @@ func formatValue(b *strings.Builder, rv reflect.Value, seen map[visit]bool) { b.WriteString("") continue } - formatValue(b, f, seen) + st.format(b, f) } b.WriteByte('}') case reflect.Chan, reflect.Func, reflect.UnsafePointer: @@ -127,30 +197,125 @@ func formatValue(b *strings.Builder, rv reflect.Value, seen map[visit]bool) { } } -func formatList(b *strings.Builder, rv reflect.Value, seen map[visit]bool) { +type mapEntry struct { + key, keyType, value string +} + +// dynamicTypeName names the type of the value in k, looking through an +// interface. +func dynamicTypeName(k reflect.Value) string { + for k.Kind() == reflect.Interface && !k.IsNil() { + k = k.Elem() + } + if !k.IsValid() || k.Kind() == reflect.Interface { + return "" + } + return k.Type().String() +} + +func (st *formatState) formatList(b *strings.Builder, rv reflect.Value) { b.WriteByte('[') for i := 0; i < rv.Len(); i++ { if i > 0 { b.WriteByte(' ') } - formatValue(b, rv.Index(i), seen) + st.format(b, rv.Index(i)) } b.WriteByte(']') } -func enterVisit(rv reflect.Value, seen map[visit]bool) (visit, bool) { +// enter marks the reference rv as being rendered. ok is false when it +// already is: a cycle. +func (st *formatState) enter(rv reflect.Value) (visit, bool) { ptr := rv.Pointer() if ptr == 0 { return visit{}, true } v := visit{typ: rv.Type(), ptr: ptr} - if seen[v] { + if st.seen[v] { return v, false } - seen[v] = true + st.seen[v] = true return v, true } +var ( + jsonMarshalerType = reflect.TypeOf((*json.Marshaler)(nil)).Elem() + textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() +) + +// jsonWithinBounds reports whether encoding/json can render v within +// the formatting bounds: no cycle, at most maxFormatDepth levels, and +// at most maxFormatNodes values. encoding/json has no work bound and +// recurses as deep as the value goes, so templates check this before +// marshaling and otherwise fall back to safeFormatValue, which applies +// the bounds itself. The walk is a superset of what json visits (it +// ignores `json:"-"`), and stops at a value with its own marshaler. +func jsonWithinBounds(v any) bool { + st := newFormatState() + return st.jsonWalk(reflect.ValueOf(v)) +} + +func (st *formatState) jsonWalk(rv reflect.Value) bool { + st.nodes++ + if st.nodes > maxFormatNodes || st.depth >= maxFormatDepth { + return false + } + if !rv.IsValid() { + return true + } + t := rv.Type() + if t.Implements(jsonMarshalerType) || t.Implements(textMarshalerType) { + return true + } + st.depth++ + defer func() { st.depth-- }() + switch rv.Kind() { + case reflect.Interface: + return rv.IsNil() || st.jsonWalk(rv.Elem()) + case reflect.Pointer, reflect.Map, reflect.Slice: + if rv.IsNil() { + return true + } + v, ok := st.enter(rv) + if !ok { + return false + } + defer delete(st.seen, v) + switch rv.Kind() { + case reflect.Pointer: + return st.jsonWalk(rv.Elem()) + case reflect.Map: + iter := rv.MapRange() + for iter.Next() { + if !st.jsonWalk(iter.Key()) || !st.jsonWalk(iter.Value()) { + return false + } + } + return true + } + return st.jsonWalkList(rv) + case reflect.Array: + return st.jsonWalkList(rv) + case reflect.Struct: + for i := 0; i < rv.NumField(); i++ { + if f := rv.Field(i); f.CanInterface() && !st.jsonWalk(f) { + return false + } + } + } + return true +} + +func (st *formatState) jsonWalkList(rv reflect.Value) bool { + for i := 0; i < rv.Len(); i++ { + if !st.jsonWalk(rv.Index(i)) { + return false + } + } + return true +} + type safeFormatArg struct { v any } diff --git a/higher_order.go b/higher_order.go index 9746c17..fa6eb74 100644 --- a/higher_order.go +++ b/higher_order.go @@ -271,6 +271,12 @@ func wrapPredicateErr(name string, predicate ast.Expr, index int, err error) err // a backtick (which would clash with the surrounding error format), // letting wrapPredicateErr fall back to a position-only message. func formatPredicate(node ast.Expr) string { + // A large predicate is left out of the message: printing it costs + // time in its size at every enclosing form an error passes through, + // which is quadratic for deeply nested forms. + if !astWithin(node, maxPredicateDisplayNodes) { + return "" + } out := exprDisplayString(node) if out == "" || strings.ContainsRune(out, '`') { return "" @@ -280,6 +286,24 @@ func formatPredicate(node ast.Expr) string { // exprDisplayString prints node via go/printer after rewriting it // into user-facing display form. Returns "" when printing fails. +// maxPredicateDisplayNodes is the largest predicate, in AST nodes, that +// an error message quotes. +const maxPredicateDisplayNodes = 64 + +// astWithin reports whether node has at most limit AST nodes, visiting +// at most about limit of them. +func astWithin(node ast.Node, limit int) bool { + n := 0 + ast.Inspect(node, func(x ast.Node) bool { + if x == nil || n > limit { + return false + } + n++ + return true + }) + return n <= limit +} + func exprDisplayString(node ast.Expr) string { var buf bytes.Buffer if err := printer.Fprint(&buf, token.NewFileSet(), displayExpr(node)); err != nil { @@ -659,7 +683,7 @@ func formTrySelect(p *Program, ctx context.Context, n *ast.CallExpr, env any, de if err != nil { return nil, err } - if recv == nil { + if isNilValue(recv) { return nil, nil } name, err := p.eval(ctx, n.Args[1], env, depth) @@ -687,7 +711,7 @@ func formTryIndex(p *Program, ctx context.Context, n *ast.CallExpr, env any, dep if err != nil { return nil, err } - if recv == nil { + if isNilValue(recv) { return nil, nil } idx, err := p.eval(ctx, n.Args[1], env, depth) @@ -697,116 +721,6 @@ func formTryIndex(p *Program, ctx context.Context, n *ast.CallExpr, env any, dep return tryIndexValue(recv, idx) } -// trySelectName mirrors selectField but returns nil for missing -// fields or keys. Method values are not surfaced; selectField does -// not surface them either, so the two paths agree on what counts as -// a "field" lookup. -func trySelectName(recv any, name string, fieldTags *structTagConfig) (any, error) { - if recv == nil { - return nil, nil - } - if m, ok := recv.(map[string]any); ok { - v := m[name] - return v, nil - } - rv := reflect.ValueOf(recv) - if rv.Kind() == reflect.Pointer { - if rv.IsNil() { - return nil, nil - } - rv = rv.Elem() - } - switch rv.Kind() { - case reflect.Struct: - fv, ok, err := structFieldByName(rv, name, fieldTags) - if err != nil { - return nil, err - } - if !ok || !fv.IsValid() || !fv.CanInterface() { - return nil, nil - } - return fv.Interface(), nil - case reflect.Map: - if rv.Type().Key().Kind() != reflect.String { - return nil, fmt.Errorf("%w: cannot select %q on map with non-string keys", - ErrEvaluate, name) - } - mv := rv.MapIndex(mapStringKey(rv.Type().Key(), name)) - if !mv.IsValid() { - return nil, nil - } - return mv.Interface(), nil - } - return nil, fmt.Errorf("%w: cannot select %q on %T", ErrEvaluate, name, recv) -} - -// tryIndexValue mirrors indexValue but returns nil for missing -// keys and out-of-range slice/string indices. Wrong-kind errors -// (e.g., string index into a slice, non-integer slice index) still -// propagate. -func tryIndexValue(recv, idx any) (any, error) { - if recv == nil { - return nil, nil - } - if m, ok := recv.(map[string]any); ok { - key, ok := idx.(string) - if !ok { - return nil, fmt.Errorf("%w: map index must be string, got %T", - ErrEvaluate, idx) - } - v := m[key] - return v, nil - } - rv := reflect.ValueOf(recv) - switch rv.Kind() { - case reflect.Slice, reflect.Array: - i, err := toIndexInt(idx) - if err != nil { - return nil, err - } - if i < 0 || i >= int64(rv.Len()) { - return nil, nil - } - return rv.Index(int(i)).Interface(), nil - case reflect.String: - i, err := toIndexInt(idx) - if err != nil { - return nil, err - } - runes := []rune(rv.String()) - if i < 0 || i >= int64(len(runes)) { - return nil, nil - } - return string(runes[i]), nil - case reflect.Map: - keyType := rv.Type().Key() - if idx == nil { - return nil, fmt.Errorf("%w: cannot use nil as map key %v", ErrEvaluate, keyType) - } - kv := reflect.ValueOf(idx) - if !kv.Type().AssignableTo(keyType) { - if isNumericKind(keyType.Kind()) && isNumericKind(kv.Kind()) { - converted, err := safeNumericConvert(kv, keyType) - if err != nil { - return nil, fmt.Errorf("%w: map key conversion: %v", ErrEvaluate, err) - } - kv = converted - } else if kv.Type().ConvertibleTo(keyType) { - kv = kv.Convert(keyType) - } else { - return nil, fmt.Errorf("%w: cannot use %T as map key %v", - ErrEvaluate, idx, keyType) - } - } - mv := rv.MapIndex(kv) - if !mv.IsValid() { - return nil, nil - } - return mv.Interface(), nil - } - return nil, fmt.Errorf("%w: cannot index %T", ErrEvaluate, recv) -} - // formIf implements `if(cond, then, else)` as a lazy special form: // only the branch selected by the condition's truthiness is // evaluated, so the guard idiom `if(n != 0, total/n, 0)` works diff --git a/hostgen_test.go b/hostgen_test.go new file mode 100644 index 0000000..110f013 --- /dev/null +++ b/hostgen_test.go @@ -0,0 +1,527 @@ +package expr + +import ( + "context" + "fmt" + "math" + "math/rand/v2" + "os" + "reflect" + "strconv" + "strings" + "testing" +) + +// hostGen draws bounded random Go types, values and function signatures, +// to test evaluation against host data shapes no hand-written table +// lists. Everything is derived from one seed, so a failure reproduces +// from the seed the test prints. +type hostGen struct { + r *rand.Rand +} + +func newHostGen(seed uint64) *hostGen { + return &hostGen{r: rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))} +} + +func (g *hostGen) n(k int) int { return g.r.IntN(k) } + +var ( + leafTypes = []reflect.Type{ + reflect.TypeFor[int](), reflect.TypeFor[int8](), reflect.TypeFor[uint16](), reflect.TypeFor[uint64](), + reflect.TypeFor[int64](), reflect.TypeFor[float32](), reflect.TypeFor[float64](), reflect.TypeFor[string](), + reflect.TypeFor[bool](), reflect.TypeFor[mxID](), reflect.TypeFor[mxName](), reflect.TypeFor[mxFlag](), + reflect.TypeFor[any](), reflect.TypeFor[error](), + } + mapKeyTypes = []reflect.Type{ + reflect.TypeFor[string](), reflect.TypeFor[string](), reflect.TypeFor[int](), reflect.TypeFor[mxName](), + reflect.TypeFor[mxID](), reflect.TypeFor[[2]int](), reflect.TypeFor[any](), reflect.TypeFor[bool](), + } + fieldNames = []string{"A", "B", "C", "Type", "Map"} +) + +// typ returns a random type at most depth levels deep. +func (g *hostGen) typ(depth int) reflect.Type { + if depth <= 0 || g.n(3) == 0 { + return leafTypes[g.n(len(leafTypes))] + } + switch g.n(6) { + case 0: + return reflect.SliceOf(g.typ(depth - 1)) + case 1: + return reflect.ArrayOf(g.n(4), g.typ(depth-1)) + case 2: + return reflect.MapOf(mapKeyTypes[g.n(len(mapKeyTypes))], g.typ(depth-1)) + case 3: + return reflect.PointerTo(g.typ(depth - 1)) + default: + nf := 1 + g.n(3) + fields := make([]reflect.StructField, nf) + for i := range fields { + fields[i] = reflect.StructField{Name: fieldNames[i], Type: g.typ(depth - 1)} + if g.n(3) == 0 { + fields[i].Tag = reflect.StructTag(`json:"` + strings.ToLower(fieldNames[i]) + `"`) + } + } + // An unexported field, which expr must never expose. + if g.n(4) == 0 { + fields = append(fields, reflect.StructField{Name: "hidden", PkgPath: "expr", Type: reflect.TypeFor[int]()}) + } + return reflect.StructOf(fields) + } +} + +// value returns a random value of type t, at most depth levels deep. +// Nilable kinds are sometimes typed nils; interfaces hold values of +// random types, sometimes interfaces again. +func (g *hostGen) value(t reflect.Type, depth int) reflect.Value { + v := reflect.New(t).Elem() + switch t.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + vals := []int64{0, 1, -1, 7, math.MaxInt8, math.MaxInt64, math.MinInt64, 1<<53 + 1} + x := vals[g.n(len(vals))] + if v.OverflowInt(x) { + x = int64(g.n(100)) + } + v.SetInt(x) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + vals := []uint64{0, 1, 7, math.MaxUint16, math.MaxUint64} + x := vals[g.n(len(vals))] + if v.OverflowUint(x) { + x = uint64(g.n(100)) + } + v.SetUint(x) + case reflect.Float32, reflect.Float64: + vals := []float64{0, 1, -1.5, 2.5, 1e300} + x := vals[g.n(len(vals))] + if v.OverflowFloat(x) { + x = 1.5 + } + v.SetFloat(x) + case reflect.String: + vals := []string{"", "a", "A", "héllo", "type", "missing"} + v.SetString(vals[g.n(len(vals))]) + case reflect.Bool: + v.SetBool(g.n(2) == 0) + case reflect.Interface: + if depth <= 0 || g.n(4) == 0 { + return v // nil interface + } + if t == reflect.TypeFor[error]() { + errs := []error{mxErr{"e"}, &mxPtrErr{"p"}, (*mxPtrErr)(nil)} + v.Set(reflect.ValueOf(errs[g.n(len(errs))])) + return v + } + inner := g.typ(depth - 1) + v.Set(g.value(inner, depth-1)) + case reflect.Pointer: + if depth <= 0 || g.n(4) == 0 { + return v // typed nil pointer + } + p := reflect.New(t.Elem()) + p.Elem().Set(g.value(t.Elem(), depth-1)) + v.Set(p) + case reflect.Slice: + if g.n(5) == 0 { + return v // typed nil slice + } + n := g.n(4) + s := reflect.MakeSlice(t, n, n) + for i := 0; i < n; i++ { + s.Index(i).Set(g.value(t.Elem(), depth-1)) + } + v.Set(s) + case reflect.Array: + for i := 0; i < t.Len(); i++ { + v.Index(i).Set(g.value(t.Elem(), depth-1)) + } + case reflect.Map: + if g.n(5) == 0 { + return v // typed nil map + } + m := reflect.MakeMap(t) + for i, n := 0, g.n(4); i < n; i++ { + k := g.value(t.Key(), 1) + if !k.Comparable() { + continue // an interface key holding a slice + } + m.SetMapIndex(k, g.value(t.Elem(), depth-1)) + } + v.Set(m) + case reflect.Struct: + for i := 0; i < t.NumField(); i++ { + if t.Field(i).IsExported() { + v.Field(i).Set(g.value(t.Field(i).Type, depth-1)) + } + } + } + return v +} + +// pathStep is one hop of a generated access path, with what the hop is +// known to produce. +type pathStep struct { + src string // ".A", "[0]", `["a"]` + opt string // the same hop with ?. or ?[ +} + +// accessOutcome is what reading a path is known to give, from walking +// the value with reflect. +type accessOutcome int + +const ( + accessValue accessOutcome = iota // a value: want + accessMissing // a missing field or key, or an index out of range + accessNilRecv // a nil receiver along the way + accessUnknown // the walk stopped at something it does not model +) + +// path walks v for up to hops steps, choosing existing and missing +// fields, keys and indexes, and returns the steps with the outcome a +// strict read of them must have. +func (g *hostGen) path(v reflect.Value, hops int) ([]pathStep, accessOutcome, any) { + var steps []pathStep + for h := 0; h < hops; h++ { + u, _ := unwrapReceiver(v) + if !u.IsValid() { + return append(steps, pathStep{".A", "?.A"}), accessNilRecv, nil + } + switch u.Kind() { + case reflect.Struct: + t := u.Type() + if g.n(5) == 0 { + return append(steps, pathStep{".Zz", "?.Zz"}), accessMissing, nil + } + i := g.n(t.NumField()) + f := t.Field(i) + if !f.IsExported() { + return append(steps, pathStep{"." + f.Name, "?." + f.Name}), accessMissing, nil + } + name := displayFieldName(f.Name) + steps = append(steps, pathStep{"." + name, "?." + name}) + v = u.Field(i) + case reflect.Map: + if u.Type().Key().Kind() != reflect.String { + return steps, accessUnknown, nil + } + keys := u.MapKeys() + if len(keys) == 0 || g.n(4) == 0 { + return append(steps, pathStep{`["zz"]`, `?["zz"]`}), accessMissing, nil + } + k := keys[g.n(len(keys))] + q := strconv.Quote(k.String()) + steps = append(steps, pathStep{"[" + q + "]", "?[" + q + "]"}) + v = u.MapIndex(k) + case reflect.Slice, reflect.Array: + i := g.n(u.Len() + 1) // sometimes one past the end + steps = append(steps, pathStep{"[" + strconv.Itoa(i) + "]", "?[" + strconv.Itoa(i) + "]"}) + if i >= u.Len() { + return steps, accessMissing, nil + } + v = u.Index(i) + default: + return steps, accessValue, boxValue(v) + } + } + return steps, accessValue, boxValue(v) +} + +// displayFieldName writes a field name as expr source: keyword names +// like `Type` and `Map` are not keywords (Go keywords are lower case), +// so names pass through. +func displayFieldName(name string) string { return name } + +func joinSteps(steps []pathStep, optionalFrom int) string { + var b strings.Builder + b.WriteString("V") + for i, s := range steps { + if i >= optionalFrom { + b.WriteString(s.opt) + } else { + b.WriteString(s.src) + } + } + return b.String() +} + +// checkHostValue runs the value matrix expressions and generated access +// paths over one generated value, through every path, and checks the +// read oracle and the strict/optional relations. +func checkHostValue(t *testing.T, label string, g *hostGen, v any) { + t.Helper() + got := map[string]mxOutcome{} + for _, src := range mxExprs { + got[src] = mxEval(t, label, mxCase{src: src}, v) + } + checkMatrixRelations(t, namedValue{label, v}, got) + + for i := 0; i < 6; i++ { + steps, outcome, want := g.path(reflect.ValueOf(v), 1+g.n(4)) + if len(steps) == 0 || outcome == accessUnknown { + continue + } + strictSrc := joinSteps(steps, len(steps)) + optSrc := joinSteps(steps, 0) + strict := mxEval(t, label, mxCase{src: strictSrc}, v) + opt := mxEval(t, label, mxCase{src: optSrc}, v) + switch outcome { + case accessValue: + if !sameOutcome(strict, mxOutcome{value: want}) { + t.Errorf("%s: %s = %v, want %#v", label, strictSrc, strict, want) + } + if !sameOutcome(opt, mxOutcome{value: want}) { + t.Errorf("%s: %s = %v, want %#v", label, optSrc, opt, want) + } + case accessMissing, accessNilRecv: + if strict.category != "evaluate" { + t.Errorf("%s: %s = %v, want ErrEvaluate (missing or nil)", label, strictSrc, strict) + } + if !sameOutcome(opt, mxOutcome{}) { + t.Errorf("%s: %s = %v, want nil", label, optSrc, opt) + } + } + // try of a strict read is the read, or the default. + tried := mxEval(t, label, mxCase{src: "try(" + strictSrc + `, "d")`}, v) + wantTry := strict + if strict.category != "" { + wantTry = mxOutcome{value: "d"} + } + if !sameOutcome(tried, wantTry) { + t.Errorf("%s: try(%s, \"d\") = %v, want %v", label, strictSrc, tried, wantTry) + } + } +} + +// hostSeeds is how many generated values TestHostValuesRandom checks; +// EXPR_HOSTGEN_SEEDS raises it for a longer search. +func hostSeeds() int { + if s, err := strconv.Atoi(os.Getenv("EXPR_HOSTGEN_SEEDS")); err == nil && s > 0 { + return s + } + if testing.Short() { + return 50 + } + return 400 +} + +// hostRegressionSeeds are seeds that once found a bug, kept beyond the +// default range so they always run. +var hostRegressionSeeds = []uint64{ + 90, // [0]map[any]int8 compared with ==: panicked + 2623, // map[any]uint16 with keys 1 and uint16(1): nondeterministic string(V) +} + +func TestHostValuesRandom(t *testing.T) { + seeds := hostRegressionSeeds + for seed := uint64(1); seed <= uint64(hostSeeds()); seed++ { + seeds = append(seeds, seed) + } + for _, seed := range seeds { + g := newHostGen(seed) + typ := g.typ(4) + v := g.value(typ, 5) + label := fmt.Sprintf("seed %d (%v)", seed, typ) + checkHostValue(t, label, g, boxValue(v)) + if t.Failed() { + t.Fatalf("first failing seed: %d; rerun with EXPR_HOSTGEN_SEEDS or add it to hostRegressionSeeds", seed) + } + } +} + +// FuzzHostValues explores generated host values from fuzzed seeds: +// go test -fuzz=FuzzHostValues. +func FuzzHostValues(f *testing.F) { + for _, s := range []uint64{1, 2, 3, 42} { + f.Add(s) + } + f.Fuzz(func(t *testing.T, seed uint64) { + g := newHostGen(seed) + typ := g.typ(4) + checkHostValue(t, fmt.Sprintf("seed %d (%v)", seed, typ), g, boxValue(g.value(typ, 5))) + }) +} + +// funcSig draws a function type: parameters of generated types +// (sometimes a leading context.Context, sometimes variadic) and one of +// the result shapes expr supports or rejects. +func (g *hostGen) funcSig() reflect.Type { + var in []reflect.Type + if g.n(4) == 0 { + in = append(in, ctxType) + } + for i, n := 0, g.n(3); i < n; i++ { + in = append(in, g.typ(2)) + } + variadic := false + if g.n(4) == 0 { + in = append(in, reflect.SliceOf(g.typ(1))) + variadic = true + } + var out []reflect.Type + switch g.n(6) { + case 0: + case 1, 2: + out = []reflect.Type{g.typ(2)} + case 3: + out = []reflect.Type{g.typ(1), reflect.TypeFor[error]()} + case 4: + out = []reflect.Type{g.typ(1), reflect.TypeFor[mxErr]()} + default: + out = []reflect.Type{g.typ(1), g.typ(1)} // unsupported unless the second is an error + } + return reflect.FuncOf(in, out, variadic) +} + +type hostCall struct { + ran int + args []any +} + +// makeHostFunc builds a function of type ft that records its arguments +// and returns generated results, failing on every third call shape. +func (g *hostGen) makeHostFunc(ft reflect.Type, rec *hostCall, results []reflect.Value) any { + return reflect.MakeFunc(ft, func(in []reflect.Value) []reflect.Value { + rec.ran++ + for i, a := range in { + switch { + case a.Type() == ctxType: + case ft.IsVariadic() && i == len(in)-1: + // MakeFunc passes the variadic arguments as one slice. + for j := 0; j < a.Len(); j++ { + rec.args = append(rec.args, boxValue(a.Index(j))) + } + default: + rec.args = append(rec.args, boxValue(a)) + } + } + return results + }).Interface() +} + +func TestHostFuncsRandom(t *testing.T) { + n := hostSeeds() + for seed := uint64(1); seed <= uint64(n); seed++ { + g := newHostGen(seed) + ft := g.funcSig() + results := make([]reflect.Value, ft.NumOut()) + for i := range results { + results[i] = g.value(ft.Out(i), 2) + } + // Arguments: values of the parameter types (which must reach the + // body unchanged), of other types, or nil. + var args []any + var exact []bool + params := ft.NumIn() + for i := 0; i < params; i++ { + pt := ft.In(i) + if pt == ctxType { + continue + } + if ft.IsVariadic() && i == params-1 { + pt = pt.Elem() + } + switch g.n(4) { + case 0: + args = append(args, nil) + exact = append(exact, false) + case 1: + args = append(args, boxValue(g.value(g.typ(2), 2))) + exact = append(exact, false) + default: + args = append(args, boxValue(g.value(pt, 2))) + exact = append(exact, true) + } + } + if ft.IsVariadic() && g.n(2) == 0 && len(args) > 0 { + args = args[:len(args)-1] // no variadic arguments + exact = exact[:len(exact)-1] + } + names := make([]string, len(args)) + env := map[string]any{} + for i, a := range args { + names[i] = fmt.Sprintf("A%d", i) + env[names[i]] = a + } + src := "F(" + strings.Join(names, ", ") + ")" + label := fmt.Sprintf("seed %d: %v called as %s with %#v", seed, ft, src, args) + + outcomes := map[string]callOutcome{} + var recs []*hostCall + for _, how := range []string{"registered", "registered/noopt", "env"} { + rec := &hostCall{} + recs = append(recs, rec) + fn := g.makeHostFunc(ft, rec, results) + outcomes[how] = runHostCall(fn, how, src, env) + outcomes[how] = callOutcome{outcomes[how].mxOutcome, rec.ran} + } + reg := outcomes["registered"] + for how, o := range outcomes { + if o.panicked != nil { + t.Errorf("%s via %s panicked: %v", label, how, o.panicked) + } + if o.category == "UNWRAPPED" { + t.Errorf("%s via %s: unwrapped error %s", label, how, o.msg) + } + if how == "registered" || reg.category == "compile" { + continue + } + if !sameOutcome(reg.mxOutcome, o.mxOutcome) || reg.called != o.called { + t.Errorf("%s:\n registered: %v\n %s: %v", label, reg, how, o) + } + } + if reg.category == "compile" { + if o := outcomes["env"]; o.category != "evaluate" || o.called != 0 { + t.Errorf("%s: registration rejects the signature, so env call must fail unrun: %v", label, o) + } + continue + } + // An argument whose type is the parameter type reaches the body + // unchanged. + if rec := recs[0]; rec.ran == 1 { + for i, a := range args { + if exact[i] && i < len(rec.args) && !reflect.DeepEqual(rec.args[i], a) && !hasNaN(a) { + t.Errorf("%s: argument %d reached the body as %#v, want %#v", label, i, rec.args[i], a) + } + } + } + if t.Failed() { + t.Fatalf("first failing seed: %d", seed) + } + } +} + +func hasNaN(v any) bool { + f, ok := v.(float64) + return ok && math.IsNaN(f) +} + +func runHostCall(fn any, how, src string, argEnv map[string]any) (out callOutcome) { + defer func() { + if r := recover(); r != nil { + out = callOutcome{mxOutcome{panicked: r}, 0} + } + }() + env := map[string]any{} + for k, v := range argEnv { + env[k] = v + } + var opts []Option + switch how { + case "registered", "registered/noopt": + opts = append(opts, WithFunctions(map[string]any{"F": fn})) + if how == "registered/noopt" { + opts = append(opts, withoutOptimizations()) + } + case "env": + env["F"] = fn + } + p, err := Compile(src, opts...) + if err != nil { + return callOutcome{mxOutcome{category: errCategory(err), msg: err.Error()}, 0} + } + v, err := p.Run(context.Background(), env) + if err != nil { + return callOutcome{mxOutcome{category: errCategory(err), msg: err.Error()}, 0} + } + return callOutcome{mxOutcome{value: v}, 0} +} diff --git a/identifiers.go b/identifiers.go index 2ced9a7..130394a 100644 --- a/identifiers.go +++ b/identifiers.go @@ -32,8 +32,9 @@ import ( // a host deliberately shadows it. Every name that can only resolve // through the env is always included. func (p *Program) Identifiers() []string { - out := make([]string, len(p.identifiers)) - copy(out, p.identifiers) + _, identifiers := p.resolveDeferred() + out := make([]string, len(identifiers)) + copy(out, identifiers) return out } @@ -57,18 +58,28 @@ func collectIdentifiers(root ast.Expr, funcs map[string]any) []string { type boundIdents struct { element string parent *boundIdents + // memo caches has's answers from parent, so names in deeply nested + // forms resolve in amortized constant time instead of walking every + // enclosing binding. + memo map[string]bool } func (b *boundIdents) has(name string) bool { - if b != nil && name == "index" { + if b == nil { + return false + } + if name == "index" || name == b.element { return true } - for s := b; s != nil; s = s.parent { - if name == s.element { - return true - } + if v, ok := b.memo[name]; ok { + return v + } + v := b.parent.has(name) + if b.memo == nil { + b.memo = map[string]bool{} } - return false + b.memo[name] = v + return v } // walkIdentifiers visits node and records env-resolved identifier diff --git a/internal/jsonlit/jsonlit.go b/internal/jsonlit/jsonlit.go index 62247be..f8a40e9 100644 --- a/internal/jsonlit/jsonlit.go +++ b/internal/jsonlit/jsonlit.go @@ -22,9 +22,10 @@ package jsonlit import ( - "go/scanner" "go/token" "strings" + + "github.com/deepnoodle-ai/expr/internal/srcedit" ) // Rewrite returns src with bare JSON-style bracket and brace literals @@ -32,72 +33,21 @@ import ( // src contains neither '[' nor '{', Rewrite returns src unchanged // without invoking the scanner. func Rewrite(src string) string { - if !strings.ContainsAny(src, "[{") { - return src - } - toks := scanTokens(src) - if len(toks) == 0 { - return src - } - edits := plan(src, toks) - if len(edits) == 0 { - return src - } - return apply(src, edits) -} - -// tokenInfo records one scanned token's byte range and kind. We keep -// byte offsets rather than token.Pos values so apply can splice the -// original source without consulting the FileSet. -type tokenInfo struct { - pos int - end int - kind token.Token -} - -// scanTokens runs go/scanner over src and returns every significant -// token. Comments and auto-inserted semicolons are filtered so that -// "previous token" tracking in plan sees only meaningful tokens. -func scanTokens(src string) []tokenInfo { - fs := token.NewFileSet() - file := fs.AddFile("", fs.Base(), len(src)) - var s scanner.Scanner - s.Init(file, []byte(src), nil, scanner.ScanComments) - - var out []tokenInfo - for { - pos, t, lit := s.Scan() - if t == token.EOF { - break - } - if t == token.COMMENT || t == token.SEMICOLON { - continue - } - off := file.Offset(pos) - out = append(out, tokenInfo{pos: off, end: off + tokLen(t, lit), kind: t}) - } + out, _ := RewriteMap(src) return out } -// tokLen reports the byte length of a scanned token in the original -// source. For tokens with a non-empty literal (identifiers, numbers, -// strings, runes) we use the literal length directly. For operators -// and keywords the scanner leaves lit empty, but token.Token.String() -// returns the exact source form for those kinds, so its length is -// the right answer. -func tokLen(t token.Token, lit string) int { - if lit != "" { - return len(lit) +// RewriteMap is Rewrite that also returns the map from offsets in the +// result back to offsets in src. +func RewriteMap(src string) (string, srcedit.Map) { + if !strings.ContainsAny(src, "[{") { + return src, srcedit.Map{} } - return len(t.String()) -} - -// edit is a single splice into the source. [pos,end) is replaced -// with str. Insertions use pos == end. -type edit struct { - pos int - end int - str string + toks := srcedit.Scan(src) + if len(toks) == 0 { + return src, srcedit.Map{} + } + return srcedit.Apply(src, plan(src, toks)) } // Stack markers for bracket pairing. We push on '[' and pop on ']'. @@ -124,38 +74,39 @@ const ( // classify each '[' by context (prev token, content, following // token) and pair it with its close via a small stack. Each '{' is // classified by its preceding token and the brace stack. -func plan(src string, toks []tokenInfo) []edit { - matches := matchBrackets(toks) - var edits []edit +func plan(src string, toks []srcedit.Token) []srcedit.Edit { + matches := srcedit.Brackets(toks) + commas := topLevelCommas(toks, matches) + var edits []srcedit.Edit var stack []byte var braces []byte prev := token.ILLEGAL for i, t := range toks { - switch t.kind { + switch t.Kind { case token.LBRACK: // '[' immediately followed by ']' is either a slice-type // prefix like []T / [][]T / []map[K]V, or an empty array // literal. We disambiguate by looking at the token just // past the ']': if it could start a type, we treat the // pair as a type prefix and leave it alone. - if i+1 < len(toks) && toks[i+1].kind == token.RBRACK { + if i+1 < len(toks) && toks[i+1].Kind == token.RBRACK { if isSliceTypeFollow(toks, i+2) { stack = append(stack, frameSlice) - prev = t.kind + prev = t.Kind continue } // Emit two edits rather than a single splice over the // whole pair so that any comments between '[' and ']' // are preserved verbatim in the output. - edits = append(edits, edit{ - pos: t.pos, end: t.end, str: "[]any{", + edits = append(edits, srcedit.Edit{ + Pos: t.Pos, End: t.End, Str: "[]any{", }) - edits = append(edits, edit{ - pos: toks[i+1].pos, end: toks[i+1].end, str: "}", + edits = append(edits, srcedit.Edit{ + Pos: toks[i+1].Pos, End: toks[i+1].End, Str: "}", }) stack = append(stack, frameEmpty) - prev = t.kind + prev = t.Kind continue } // Non-empty '['. Leave it alone if the previous token @@ -171,14 +122,14 @@ func plan(src string, toks []tokenInfo) []edit { // as an array type only when it is unambiguously one: no // top-level comma in the brackets, and the token after // the matching ']' starts a type expression. - if close, ok := matches[i]; ok && - !hasTopLevelComma(toks, i, close) && + if close := matches[i]; close >= 0 && + !commas[i] && isArrayTypeFollow(toks, close+1, matches) { stack = append(stack, frameIndex) break } - edits = append(edits, edit{ - pos: t.pos, end: t.end, str: "[]any{", + edits = append(edits, srcedit.Edit{ + Pos: t.Pos, End: t.End, Str: "[]any{", }) stack = append(stack, frameArray) @@ -190,8 +141,8 @@ func plan(src string, toks []tokenInfo) []edit { top := stack[len(stack)-1] stack = stack[:len(stack)-1] if top == frameArray { - edits = append(edits, edit{ - pos: t.pos, end: t.end, str: "}", + edits = append(edits, srcedit.Edit{ + Pos: t.Pos, End: t.End, Str: "}", }) } // frameIndex / frameSlice / frameEmpty: nothing to do. @@ -210,8 +161,8 @@ func plan(src string, toks []tokenInfo) []edit { leave = true } if !leave { - edits = append(edits, edit{ - pos: t.pos, end: t.pos, str: "map[string]any", + edits = append(edits, srcedit.Edit{ + Pos: t.Pos, End: t.Pos, Str: "map[string]any", }) braces = append(braces, braceObject) break @@ -232,7 +183,7 @@ func plan(src string, toks []tokenInfo) []edit { } } - prev = t.kind + prev = t.Kind } return edits } @@ -250,15 +201,53 @@ func plan(src string, toks []tokenInfo) []edit { // receive-only channel types ([]<-chan T). If i runs off the end, // we conservatively say no — bare "[]" at end of input is an empty // array literal. -func isSliceTypeFollow(toks []tokenInfo, i int) bool { +func isSliceTypeFollow(toks []srcedit.Token, i int) bool { if i >= len(toks) { return false } - switch toks[i].kind { + switch toks[i].Kind { case token.IDENT, token.LBRACK, token.MAP, token.INTERFACE, token.STRUCT, - token.CHAN, token.FUNC, token.MUL, token.ARROW: + token.CHAN, token.FUNC, token.ARROW: return true + case token.MUL: + return isPointerTypeFollow(toks, i, nil) + } + return false +} + +// isPointerTypeFollow reports whether the '*' at toks[i] starts a pointer +// element type ([]*T{...}, [N]*T{...}) rather than a multiplication +// ([6][0] * 2, [] * n). After a value, '*' multiplies, so it is a type +// only when no operand can follow it — a type keyword or another '*' — +// or when a named type is followed by the literal's '{'. matches is nil +// for the "[]" prefix, whose following '[' is already a type. +func isPointerTypeFollow(toks []srcedit.Token, i int, matches []int) bool { + j := i + 1 + if j >= len(toks) { + return false + } + switch toks[j].Kind { + case token.MAP, token.INTERFACE, token.STRUCT, token.CHAN, token.FUNC, token.ARROW: + return true + case token.MUL: + return isPointerTypeFollow(toks, j, matches) + case token.LBRACK: + if matches == nil { + return isSliceTypeFollow(toks, j) + } + if j+1 < len(toks) && toks[j+1].Kind == token.RBRACK { + return isSliceTypeFollow(toks, j+2) + } + close := matches[j] + return close >= 0 && isArrayTypeFollow(toks, close+1, matches) + case token.IDENT: + // T{ or pkg.T{ + k := j + 1 + if k+1 < len(toks) && toks[k].Kind == token.PERIOD && toks[k+1].Kind == token.IDENT { + k += 2 + } + return k < len(toks) && toks[k].Kind == token.LBRACE } return false } @@ -269,20 +258,22 @@ func isSliceTypeFollow(toks []tokenInfo, i int) bool { // literal-then-index, an inner '[' is only accepted when it is // itself part of an array/slice type expression — i.e., its own // matching ']' is followed by something that begins a type. -func isArrayTypeFollow(toks []tokenInfo, i int, matches map[int]int) bool { +func isArrayTypeFollow(toks []srcedit.Token, i int, matches []int) bool { if i >= len(toks) { return false } - switch toks[i].kind { + switch toks[i].Kind { case token.IDENT, token.MAP, token.INTERFACE, token.STRUCT, - token.CHAN, token.FUNC, token.MUL, token.ARROW: + token.CHAN, token.FUNC, token.ARROW: return true + case token.MUL: + return isPointerTypeFollow(toks, i, matches) case token.LBRACK: // "[N][...]": classify the outer "[N]" as an array type only // when the inner bracket pair is itself a type prefix. - close, ok := matches[i] - if !ok { + close := matches[i] + if close < 0 { return false } return isArrayTypeFollow(toks, close+1, matches) @@ -290,47 +281,28 @@ func isArrayTypeFollow(toks []tokenInfo, i int, matches map[int]int) bool { return false } -// hasTopLevelComma reports whether the tokens strictly between open -// and close (both LBRACK/RBRACK indices) contain a COMMA at bracket -// depth 0. A top-level comma inside "[...]" rules out the -// array-type-length interpretation: [1, 2]int is never valid Go, -// but [1, 2] is a perfectly good array literal. -func hasTopLevelComma(toks []tokenInfo, open, close int) bool { - depth := 0 - for j := open + 1; j < close; j++ { - switch toks[j].kind { - case token.LPAREN, token.LBRACK, token.LBRACE: - depth++ - case token.RPAREN, token.RBRACK, token.RBRACE: - depth-- - case token.COMMA: - if depth == 0 { - return true - } - } - } - return false -} - -// matchBrackets precomputes, for every LBRACK token in toks, the -// index of its matching RBRACK. Unbalanced opens are silently -// omitted from the result — the main plan loop will then fall -// through and let the parser surface the error. -func matchBrackets(toks []tokenInfo) map[int]int { - m := make(map[int]int) +// topLevelCommas reports, for each opening bracket with a match, whether +// a COMMA sits directly inside its pair (not nested in another bracket). +// A top-level comma inside "[...]" rules out the array-type-length +// interpretation: [1, 2]int is never valid Go, but [1, 2] is a perfectly +// good array literal. One pass keeps nested literals linear. +func topLevelCommas(toks []srcedit.Token, matches []int) []bool { + out := make([]bool, len(toks)) var stack []int for i, t := range toks { - switch t.kind { - case token.LBRACK: + n := len(stack) + switch { + case matches[i] >= 0: stack = append(stack, i) - case token.RBRACK: - if n := len(stack); n > 0 { - m[stack[n-1]] = i - stack = stack[:n-1] + case t.Kind == token.COMMA: + if n > 0 { + out[stack[n-1]] = true } + case n > 0 && matches[stack[n-1]] == i: + stack = stack[:n-1] } } - return m + return out } // leaveBracketAlone reports whether a '[' following this token @@ -380,13 +352,13 @@ func followsTypeName(prev token.Token) bool { // other typed outer (struct slices, slice-of-slice, simple typed // slices, etc.) Go accepts type omission, and we must leave nested // {...} alone. -func elementIsAnyOrInterface(src string, toks []tokenInfo, idx int) bool { +func elementIsAnyOrInterface(src string, toks []srcedit.Token, idx int) bool { if idx < 1 { return false } j := idx - 1 var bracketEnd int - switch toks[j].kind { + switch toks[j].Kind { case token.IDENT: if literalAt(src, toks[j]) != "any" { return false @@ -395,21 +367,21 @@ func elementIsAnyOrInterface(src string, toks []tokenInfo, idx int) bool { case token.RBRACE: // `interface{}` body: ... interface { } { if j < 2 || - toks[j-1].kind != token.LBRACE || - toks[j-2].kind != token.INTERFACE { + toks[j-1].Kind != token.LBRACE || + toks[j-2].Kind != token.INTERFACE { return false } bracketEnd = j - 3 default: return false } - if bracketEnd < 0 || toks[bracketEnd].kind != token.RBRACK { + if bracketEnd < 0 || toks[bracketEnd].Kind != token.RBRACK { return false } // Find the matching '['. depth := 1 for k := bracketEnd - 1; k >= 0; k-- { - switch toks[k].kind { + switch toks[k].Kind { case token.RBRACK: depth++ case token.LBRACK: @@ -423,9 +395,9 @@ func elementIsAnyOrInterface(src string, toks []tokenInfo, idx int) bool { } // Map prefix: `map[string]any` / `map[string]interface{}`. if bracketEnd-k == 2 && - toks[k+1].kind == token.IDENT && + toks[k+1].Kind == token.IDENT && literalAt(src, toks[k+1]) == "string" && - k > 0 && toks[k-1].kind == token.MAP { + k > 0 && toks[k-1].Kind == token.MAP { return !typeWraps(toks, k-1) } return false @@ -441,11 +413,11 @@ func elementIsAnyOrInterface(src string, toks []tokenInfo, idx int) bool { // (e.g. it's `[]any` inside `[][]any` or `map[string]any` inside // `[]map[string]any`), and Go's type-omission rules govern nested // composite literals — leave them alone. -func typeWraps(toks []tokenInfo, k int) bool { +func typeWraps(toks []srcedit.Token, k int) bool { if k == 0 { return false } - switch toks[k-1].kind { + switch toks[k-1].Kind { case token.RBRACK, token.RBRACE: return true } @@ -455,24 +427,9 @@ func typeWraps(toks []tokenInfo, k int) bool { // literalAt returns the source text for the given token. Identifier // and string tokens are stored with positions only, so we read them // out of the original source. -func literalAt(src string, t tokenInfo) string { - if t.pos < 0 || t.end > len(src) || t.pos > t.end { +func literalAt(src string, t srcedit.Token) string { + if t.Pos < 0 || t.End > len(src) || t.Pos > t.End { return "" } - return src[t.pos:t.end] -} - -// apply splices edits into src. Edits are produced by plan in -// source order and never overlap, so a single pass suffices. -func apply(src string, edits []edit) string { - var b strings.Builder - b.Grow(len(src) + 16*len(edits)) - last := 0 - for _, e := range edits { - b.WriteString(src[last:e.pos]) - b.WriteString(e.str) - last = e.end - } - b.WriteString(src[last:]) - return b.String() + return src[t.Pos:t.End] } diff --git a/internal/jsonlit/jsonlit_test.go b/internal/jsonlit/jsonlit_test.go index a727c21..3e774e3 100644 --- a/internal/jsonlit/jsonlit_test.go +++ b/internal/jsonlit/jsonlit_test.go @@ -151,6 +151,12 @@ func TestRewrite(t *testing.T) { "[]struct{X int}{{1},{2}}", "[]struct{X int}{{1},{2}}"}, {"slice_of_pointer_omitted", "[]*T{{}, {}}", "[]*T{{}, {}}"}, + {"array_of_pointer", "[2]*T{}", "[2]*T{}"}, + {"array_of_qualified_pointer", "[2]*pkg.T{}", "[2]*pkg.T{}"}, + {"literal_index_mul_int", "[6][0] * 2", "[]any{6}[0] * 2"}, + {"literal_index_mul_ident", "[6][0] * factor", "[]any{6}[0] * factor"}, + {"nested_literal_index_mul", "[[6]][0][0] * 2", "[]any{[]any{6}}[0][0] * 2"}, + {"empty_list_mul", "[] * n", "[]any{} * n"}, {"slice_of_map_omitted", `[]map[string]int{{"a":1}}`, `[]map[string]int{{"a":1}}`}, diff --git a/internal/optaccess/optaccess.go b/internal/optaccess/optaccess.go index c95bc7d..f20848f 100644 --- a/internal/optaccess/optaccess.go +++ b/internal/optaccess/optaccess.go @@ -13,8 +13,8 @@ // // The LHS of `?.` / `?[` is the primary expression that ends just // before the `?`. The walker matches balanced parens, brackets, and -// braces so `f(a)?.b`, `a[0]?.b`, `(a + b)?.c`, `[1, 2]?[0]`, and -// `{"k": 1}?.k` all rewrite with the expected LHS, as do calls on +// braces so `f(a)?.b`, `a[0]?.b`, `(a + b)?.c`, `[1, 2]?[0]`, +// `{"k": 1}?.k` and `[]any{1}?[0]` all rewrite with the expected LHS, as do calls on // the `map` and `if` keywords that expr accepts as call targets. // Chained optional access (`a?.b?.c`) is processed by // repeated single-rewrite passes until the source no longer contains @@ -26,134 +26,138 @@ package optaccess import ( - "go/scanner" "go/token" "strings" + + "github.com/deepnoodle-ai/expr/internal/srcedit" ) // Rewrite returns src with `?.field` and `?[idx]` rewritten to calls // on the sentinel functions __try_select__ and __try_index__. // // When src contains no `?`, Rewrite returns src unchanged without -// invoking the scanner. Chained optional access is handled by -// iteratively rewriting the leftmost remaining `?.` or `?[` until -// the source contains none, so each rewrite sees the previous one -// already rewritten as its LHS. +// invoking the scanner. Otherwise it scans once and decides every +// rewrite from the original token stream: the LHS of a `?` is found by +// walking back from it, and a chain (`a?.b?.c`) gives later operators a +// LHS that starts where the earlier one's does, so their calls nest: +// `__try_select__(__try_select__(a, "b"), "c")`. The text is then +// spliced in a single forward pass, keeping compile time linear in the +// length of a chain. func Rewrite(src string) string { + out, _ := RewriteMap(src) + return out +} + +// RewriteMap is Rewrite that also returns the map from offsets in the +// result back to offsets in src. +func RewriteMap(src string) (string, srcedit.Map) { if !strings.Contains(src, "?") { - return src + return src, srcedit.Map{} + } + toks := srcedit.Scan(src) + if len(toks) == 0 { + return src, srcedit.Map{} + } + closes := srcedit.Brackets(toks) + // opens[c] is the index of the bracket that c closes, or -1. + scratch := make([]int, 2*len(toks)) + opens, lhsOf := scratch[:len(toks)], scratch[len(toks):] + for i := range scratch { + scratch[i] = -1 } - for { - next, changed := rewriteOnce(src) - if !changed { - return src + for i, c := range closes { + if c >= 0 { + opens[c] = i } - src = next } -} - -// rewriteOnce rewrites a single leftmost `?.` or `?[` occurrence in -// src and returns the new source. The boolean is false when no -// rewritable occurrence was found, in which case src is returned -// unchanged. -func rewriteOnce(src string) (string, bool) { - toks := scanTokens(src) - if len(toks) == 0 { - return src, false + // lhsOf[e] is the LHS start of the rewrite whose last token is e + // (the field of `?.f`, the `]` of `?[i]`), or -1. + // prefixes[i] are the calls that open before token i, innermost + // first; tails[i] replaces tokens i..tails[i].last. Both are indexed + // by token, and read back in token order so the edits come out in + // source order. + type tail struct { + str string + last int } + prefixes := make(map[int][]string) + tails := make(map[int]tail) + found := false for i := 0; i < len(toks); i++ { - if !isQuestion(src, toks[i]) { + if !isQuestion(src, toks[i]) || i+1 >= len(toks) { continue } - if i+1 >= len(toks) { - continue - } - next := toks[i+1] - switch next.kind { + switch toks[i+1].Kind { case token.PERIOD: - if i+2 >= len(toks) || !isFieldToken(toks[i+2].kind) { + if i+2 >= len(toks) || !isFieldToken(toks[i+2].Kind) { continue } - lhsStart := lhsStartIdx(toks, src, i) - if lhsStart < 0 || lhsStart >= i { + lhs := lhsStartIdx(toks, src, i, lhsOf, opens) + if lhs < 0 || lhs >= i { continue } - field := toks[i+2].lit + field := toks[i+2].Lit if field == "" { - field = src[toks[i+2].pos:toks[i+2].end] + field = src[toks[i+2].Pos:toks[i+2].End] } - lhsBytes := src[toks[lhsStart].pos:toks[i].pos] - rep := "__try_select__(" + lhsBytes + ", " + quoteString(field) + ")" - return splice(src, toks[lhsStart].pos, toks[i+2].end, rep), true + lhsOf[i+2] = lhs + prefixes[lhs] = append(prefixes[lhs], "__try_select__(") + tails[i] = tail{str: ", " + quoteString(field) + ")", last: i + 2} + found = true + i += 2 case token.LBRACK: - closeIdx := matchBracketForward(toks, i+1) + closeIdx := closes[i+1] if closeIdx < 0 { continue } - lhsStart := lhsStartIdx(toks, src, i) - if lhsStart < 0 || lhsStart >= i { + lhs := lhsStartIdx(toks, src, i, lhsOf, opens) + if lhs < 0 || lhs >= i { continue } - lhsBytes := src[toks[lhsStart].pos:toks[i].pos] - idxBytes := src[toks[i+1].end:toks[closeIdx].pos] - rep := "__try_index__(" + lhsBytes + ", " + idxBytes + ")" - return splice(src, toks[lhsStart].pos, toks[closeIdx].end, rep), true + lhsOf[closeIdx] = lhs + prefixes[lhs] = append(prefixes[lhs], "__try_index__(") + tails[i] = tail{str: ", ", last: i + 1} + tails[closeIdx] = tail{str: ")", last: closeIdx} + found = true + // The index expression may hold optional access of its own; + // the loop continues into it. + i++ } } - return src, false -} - -// tokenInfo records one scanned token's byte range and kind. lit is -// retained for IDENT tokens so we can splice the field name into the -// rewrite without re-reading the source. -type tokenInfo struct { - pos int - end int - kind token.Token - lit string -} - -// scanTokens runs go/scanner over src and returns every significant -// token. Comments and auto-inserted semicolons are filtered so -// "previous token" tracking sees only meaningful tokens. -func scanTokens(src string) []tokenInfo { - fs := token.NewFileSet() - file := fs.AddFile("", fs.Base(), len(src)) - var s scanner.Scanner - s.Init(file, []byte(src), nil, scanner.ScanComments) - - var out []tokenInfo - for { - pos, t, lit := s.Scan() - if t == token.EOF { - break + if !found { + return src, srcedit.Map{} + } + var edits []srcedit.Edit + for i, t := range toks { + if ps := prefixes[i]; len(ps) > 0 { + var b strings.Builder + // Keep the call name from fusing with a preceding word in + // malformed input such as `A.0?.b`. + if t.Pos > 0 && isWordByte(src[t.Pos-1]) { + b.WriteByte(' ') + } + // A later operator wraps an earlier one: outermost first. + for k := len(ps) - 1; k >= 0; k-- { + b.WriteString(ps[k]) + } + edits = append(edits, srcedit.Edit{Pos: t.Pos, End: t.Pos, Str: b.String()}) } - if t == token.COMMENT || t == token.SEMICOLON { - continue + if tl, ok := tails[i]; ok { + edits = append(edits, srcedit.Edit{Pos: t.Pos, End: toks[tl.last].End, Str: tl.str}) } - off := file.Offset(pos) - out = append(out, tokenInfo{ - pos: off, - end: off + tokLen(t, lit), - kind: t, - lit: lit, - }) } - return out + return srcedit.Apply(src, edits) } -func tokLen(t token.Token, lit string) int { - if lit != "" { - return len(lit) - } - return len(t.String()) +func isWordByte(c byte) bool { + return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c >= 0x80 } // isQuestion reports whether t is a single `?` byte. The scanner // flags `?` as ILLEGAL but other bytes (e.g. `@`, `#`) end up there // too, so the byte check is needed to avoid rewriting around them. -func isQuestion(src string, t tokenInfo) bool { - return t.kind == token.ILLEGAL && t.end-t.pos == 1 && src[t.pos] == '?' +func isQuestion(src string, t srcedit.Token) bool { + return t.Kind == token.ILLEGAL && t.End-t.Pos == 1 && src[t.Pos] == '?' } // isFieldToken reports whether t can be the field name after `?.`. @@ -165,24 +169,6 @@ func isFieldToken(t token.Token) bool { return t == token.IDENT || t.IsKeyword() } -// matchBracketForward finds the index of the `]` that closes the `[` -// at lbrack. Returns -1 if no matching close exists. -func matchBracketForward(toks []tokenInfo, lbrack int) int { - depth := 1 - for j := lbrack + 1; j < len(toks); j++ { - switch toks[j].kind { - case token.LBRACK: - depth++ - case token.RBRACK: - depth-- - if depth == 0 { - return j - } - } - } - return -1 -} - // lhsStartIdx returns the token index where the LHS of the `?` at // qIdx starts. The walk extends backwards through identifiers, // selector chains (`.IDENT`), balanced index brackets, balanced @@ -191,25 +177,35 @@ func matchBracketForward(toks []tokenInfo, lbrack int) int { // Returns -1 if the LHS would extend past the start of the input // without ever closing — that is, when the source is unbalanced or // the `?` has nothing to the left of it. -func lhsStartIdx(toks []tokenInfo, src string, qIdx int) int { +// +// opens maps each closing bracket to its opener (-1 if unpaired). +// +// lhsOf maps the last token of each earlier rewrite to its LHS start. +// That rewrite becomes one call, a complete primary, so a walk reaching +// its last token ends at its start — which also keeps `a?.type?.b`, +// whose keyword field the walk could not step over, working. +func lhsStartIdx(toks []srcedit.Token, src string, qIdx int, lhsOf, opens []int) int { if qIdx == 0 { return -1 } i := qIdx - 1 for i >= 0 { + if lhsOf[i] >= 0 { + return lhsOf[i] + } t := toks[i] - switch t.kind { + switch t.Kind { case token.IDENT, token.INT, token.FLOAT, token.STRING, token.CHAR, token.MAP, token.IF: // `map` and `if` walk like identifiers: expr accepts them // as ordinary names in selector chains (`a.map?.x`). - if i-1 >= 0 && toks[i-1].kind == token.PERIOD { + if i-1 >= 0 && toks[i-1].Kind == token.PERIOD { i -= 2 continue } return i case token.RBRACK: - j := matchBracketBack(toks, i) + j := opens[i] if j < 0 { return -1 } @@ -221,28 +217,33 @@ func lhsStartIdx(toks []tokenInfo, src string, qIdx int) int { i = j - 1 continue case token.RBRACE: - // Brace-closed primary: a bare object literal `{...}?.x`. - // Typed composite literals (`[]any{...}`, `T{...}`) would - // need a backwards type-expression walk; decline those and - // let the parser report the stray `?`. - j := matchBraceBack(toks, i) + // Brace-closed primary: a bare object literal `{...}?.x`, + // or a typed composite literal (`[]any{...}?[0]`) whose + // type the walk steps back over. Other typed forms decline + // and leave the parser to report the stray `?`. + j := opens[i] if j < 0 { return -1 } if j == 0 { return 0 } - if startsTypedComposite(toks[j-1].kind) { - return -1 + if startsTypedComposite(toks[j-1].Kind) { + return compositeTypeStart(toks, j, opens) } return j case token.RPAREN: - j := matchParenBack(toks, i) + j := opens[i] if j < 0 { return -1 } prev := j - 1 - if prev >= 0 && extendsPrimary(toks[prev].kind) { + if prev >= 0 && toks[prev].Kind == token.PERIOD { + // A type assertion `x.(T)`, which expr rejects; splicing + // inside it would only garble the parser's error. + return -1 + } + if prev >= 0 && extendsPrimary(toks[prev].Kind) { i = prev continue } @@ -264,60 +265,6 @@ func lhsStartIdx(toks []tokenInfo, src string, qIdx int) int { return -1 } -// matchBracketBack returns the index of the `[` that opens the `]` -// at rbrack. Negative on imbalance. -func matchBracketBack(toks []tokenInfo, rbrack int) int { - depth := 1 - for j := rbrack - 1; j >= 0; j-- { - switch toks[j].kind { - case token.RBRACK: - depth++ - case token.LBRACK: - depth-- - if depth == 0 { - return j - } - } - } - return -1 -} - -// matchParenBack returns the index of the `(` that opens the `)` -// at rparen. Negative on imbalance. -func matchParenBack(toks []tokenInfo, rparen int) int { - depth := 1 - for j := rparen - 1; j >= 0; j-- { - switch toks[j].kind { - case token.RPAREN: - depth++ - case token.LPAREN: - depth-- - if depth == 0 { - return j - } - } - } - return -1 -} - -// matchBraceBack returns the index of the `{` that opens the `}` -// at rbrace. Negative on imbalance. -func matchBraceBack(toks []tokenInfo, rbrace int) int { - depth := 1 - for j := rbrace - 1; j >= 0; j-- { - switch toks[j].kind { - case token.RBRACE: - depth++ - case token.LBRACE: - depth-- - if depth == 0 { - return j - } - } - } - return -1 -} - // startsTypedComposite reports whether a `{` directly preceded by t // belongs to a typed composite literal (`[]any{...}`, `T{...}`, // `map[string]any{...}`) rather than a bare object literal. @@ -330,6 +277,50 @@ func startsTypedComposite(t token.Token) bool { return false } +// compositeTypeStart returns the index of the first token of the type +// of the composite literal whose body opens at toks[lbrace], or -1 when +// that type is not a slice, array or map type. The element (or value) +// type is a single name (`any`, `T`, `pkg.T`) or `interface{}`; anything +// fancier is not a literal expr accepts, so declining it costs nothing. +func compositeTypeStart(toks []srcedit.Token, lbrace int, opens []int) int { + elem := typeNameStart(toks, lbrace-1) + if elem < 1 || toks[elem-1].Kind != token.RBRACK { + return -1 + } + open := opens[elem-1] + if open < 0 { + return -1 + } + if open > 0 && toks[open-1].Kind == token.MAP { + // map[K]V: K is a single name too. + if typeNameStart(toks, elem-2) != open+1 { + return -1 + } + return open - 1 + } + return open +} + +// typeNameStart returns the index of the first token of a type name that +// ends at toks[end]: `T`, `pkg.T` or `interface{}`; -1 otherwise. +func typeNameStart(toks []srcedit.Token, end int) int { + if end < 0 { + return -1 + } + switch toks[end].Kind { + case token.IDENT: + if end >= 2 && toks[end-1].Kind == token.PERIOD && toks[end-2].Kind == token.IDENT { + return end - 2 + } + return end + case token.RBRACE: + if end >= 2 && toks[end-1].Kind == token.LBRACE && toks[end-2].Kind == token.INTERFACE { + return end - 2 + } + } + return -1 +} + // extendsPrimary reports whether t can directly precede a `(` that // is part of a call expression. A `(` after one of these tokens is // always a call; after anything else it begins a paren group. `map` @@ -356,13 +347,3 @@ func quoteString(s string) string { b.WriteByte('"') return b.String() } - -// splice replaces src[pos:end] with str. -func splice(src string, pos, end int, str string) string { - var b strings.Builder - b.Grow(len(src) + len(str)) - b.WriteString(src[:pos]) - b.WriteString(str) - b.WriteString(src[end:]) - return b.String() -} diff --git a/internal/optaccess/optaccess_test.go b/internal/optaccess/optaccess_test.go index 83b8986..43fe5c8 100644 --- a/internal/optaccess/optaccess_test.go +++ b/internal/optaccess/optaccess_test.go @@ -63,10 +63,18 @@ func TestRewrite(t *testing.T) { {`a?.map`, `__try_select__(a, "map")`}, {`a?.if`, `__try_select__(a, "if")`}, - // typed composite literals are declined (no backwards type - // walk); the parser reports the stray `?` - {`[]any{1}?.x`, `[]any{1}?.x`}, - {`map[string]any{"a": 1}?.a`, `map[string]any{"a": 1}?.a`}, + // typed slice, array and map literals include their type + {`[]any{1}?.x`, `__try_select__([]any{1}, "x")`}, + {`x * []any{1}?[0]`, `x * __try_index__([]any{1}, 0)`}, + {`[]interface{}{1}?[0]`, `__try_index__([]interface{}{1}, 0)`}, + {`[2]int{1, 2}?[0]`, `__try_index__([2]int{1, 2}, 0)`}, + {`map[string]any{"a": 1}?.a`, `__try_select__(map[string]any{"a": 1}, "a")`}, + {`map[string]any{"a": {"b": 1}}?.a?.b`, `__try_select__(__try_select__(map[string]any{"a": {"b": 1}}, "a"), "b")`}, + + // other typed composites are declined; the parser reports the + // stray `?` + {`T{1}?.x`, `T{1}?.x`}, + {`struct{}{}?.x`, `struct{}{}?.x`}, // `?` not followed by `.` or `[` is left alone {`a ? b : c`, `a ? b : c`}, diff --git a/internal/srcedit/srcedit.go b/internal/srcedit/srcedit.go new file mode 100644 index 0000000..1101142 --- /dev/null +++ b/internal/srcedit/srcedit.go @@ -0,0 +1,244 @@ +// Package srcedit holds the token scanning and splicing shared by expr's +// pre-parse source rewrites (keyword fields, optional access, JSON +// literals, the map/if keywords). +// +// Each rewrite scans the source once with [Scan], plans a list of +// [Edit]s, and splices them with [Apply]. Apply also returns a [Map] +// from offsets in its output back to offsets in its input, so the +// compiler can report parse and validation errors at the position the +// user wrote, whatever the rewrites inserted before it. +package srcedit + +import ( + "go/scanner" + "go/token" + "sort" + "strings" +) + +// Token is one significant token of a source string: [Pos, End) are +// byte offsets into it. +type Token struct { + Pos, End int + Kind token.Token + Lit string +} + +// Scan returns the significant tokens of src. Comments and the +// semicolons Go inserts at newlines are dropped, so "previous token" +// logic sees only what the user wrote; an explicit `;` is kept. The +// scanner reports `?` (and other stray bytes) as token.ILLEGAL. +func Scan(src string) []Token { + fs := token.NewFileSet() + file := fs.AddFile("", fs.Base(), len(src)) + var s scanner.Scanner + // A no-op error handler keeps the scanner going through stray + // bytes such as `?`; the parser reports real errors later. + s.Init(file, []byte(src), func(token.Position, string) {}, 0) + out := make([]Token, 0, len(src)/3+4) + for { + pos, tok, lit := s.Scan() + if tok == token.EOF { + break + } + if tok == token.SEMICOLON && lit != ";" { + continue + } + off := file.Offset(pos) + out = append(out, Token{Pos: off, End: tokenEnd(src, off, tok, lit), Kind: tok, Lit: lit}) + } + return out +} + +// tokenEnd is the offset just past the token at off. The scanner's lit +// is not always the source text: it drops carriage returns from raw +// strings, so the end of a raw string is found in src. +func tokenEnd(src string, off int, tok token.Token, lit string) int { + if tok == token.STRING && strings.HasPrefix(src[off:], "`") { + if i := strings.IndexByte(src[off+1:], '`'); i >= 0 { + return off + i + 2 + } + return len(src) + } + if lit != "" { + return off + len(lit) + } + return off + len(tok.String()) +} + +// Brackets returns, for each token index, the index of the token that +// closes it when it is `(`, `[` or `{`, and -1 otherwise (or when it is +// unclosed). Mismatched closers are skipped, so malformed input still +// pairs what it can and the parser reports the error. +func Brackets(toks []Token) []int { + out := make([]int, len(toks)) + var stack []int + for i, t := range toks { + out[i] = -1 + switch t.Kind { + case token.LPAREN, token.LBRACK, token.LBRACE: + stack = append(stack, i) + case token.RPAREN, token.RBRACK, token.RBRACE: + n := len(stack) + if n > 0 && closer(toks[stack[n-1]].Kind) == t.Kind { + out[stack[n-1]] = i + stack = stack[:n-1] + } + } + } + return out +} + +func closer(open token.Token) token.Token { + switch open { + case token.LPAREN: + return token.RPAREN + case token.LBRACK: + return token.RBRACK + } + return token.RBRACE +} + +// Edit replaces src[Pos:End] with Str; Pos == End inserts. +type Edit struct { + Pos, End int + Str string +} + +// Apply splices edits into src. Edits must not overlap; they are applied +// in order of position, and insertions at the same offset keep their +// order. It returns the result and the map from its offsets back to +// src's. +func Apply(src string, edits []Edit) (string, Map) { + if len(edits) == 0 { + return src, Map{} + } + less := func(i, j int) bool { return edits[i].Pos < edits[j].Pos } + if !sort.SliceIsSorted(edits, less) { + sort.SliceStable(edits, less) + } + var b strings.Builder + n := len(src) + for _, e := range edits { + n += len(e.Str) - (e.End - e.Pos) + } + b.Grow(n) + m := Map{segs: make([]segment, 0, 2*len(edits)+1)} + last := 0 + for _, e := range edits { + m.add(b.Len(), last, false) + b.WriteString(src[last:e.Pos]) + m.add(b.Len(), e.Pos, true) + b.WriteString(e.Str) + last = e.End + } + m.add(b.Len(), last, false) + b.WriteString(src[last:]) + return b.String(), m +} + +// Map translates offsets in a rewrite's output back to its input. The +// zero Map is the identity. +type Map struct { + segs []segment +} + +// segment starts at output offset out. A copied segment continues the +// input at in byte for byte; an inserted one (replacement text) maps +// every offset in it to in, where the replaced text began. +type segment struct { + out, in int + inserted bool +} + +func (m *Map) add(out, in int, inserted bool) { + if k := len(m.segs); k > 0 && m.segs[k-1].out == out { + m.segs[k-1] = segment{out, in, inserted} + return + } + m.segs = append(m.segs, segment{out, in, inserted}) +} + +// Identity reports whether m maps every offset to itself because no +// edit was made. +func (m Map) Identity() bool { return len(m.segs) == 0 } + +// Orig returns the input offset that output offset off came from. +func (m Map) Orig(off int) int { + if len(m.segs) == 0 { + return off + } + i := sort.Search(len(m.segs), func(i int) bool { return m.segs[i].out > off }) - 1 + if i < 0 { + return off + } + s := m.segs[i] + if s.inserted { + return s.in + } + return s.in + off - s.out +} + +// Then returns the map that applies m and then next: m translates the +// last rewrite's output to its input, next translates that input to an +// earlier one. +func (m Map) Then(next Map) Map { + if len(next.segs) == 0 { + return m + } + if len(m.segs) == 0 { + return next + } + // Every boundary of either map is a boundary of the composition: + // m's own, and the output offsets where m's copied segments cross + // one of next's. + var out Map + for i, s := range m.segs { + end := -1 + if i+1 < len(m.segs) { + end = m.segs[i+1].out + } + if s.inserted { + out.add(s.out, next.Orig(s.in), true) + continue + } + inEnd := -1 + if end >= 0 { + inEnd = s.in + end - s.out + } + // next's segment containing s.in, and every later one that + // starts inside this copied range. + j := sort.Search(len(next.segs), func(j int) bool { return next.segs[j].out > s.in }) - 1 + if j < 0 { + out.add(s.out, s.in, false) + j = 0 + } + for ; j < len(next.segs); j++ { + ns := next.segs[j] + start := ns.out + if start < s.in { + start = s.in + } + if inEnd >= 0 && start >= inEnd { + break + } + o := s.out + start - s.in + if ns.inserted { + out.add(o, ns.in, true) + } else { + out.add(o, ns.in+start-ns.out, false) + } + } + } + return out +} + +// Segments calls fn with the start of each segment of m, in output +// order, with the input offset it maps to. Offsets after a copied +// segment's start advance with the input; an inserted segment's all map +// to in. +func (m Map) Segments(fn func(out, in int, inserted bool)) { + for _, s := range m.segs { + fn(s.out, s.in, s.inserted) + } +} diff --git a/internal/srcedit/srcedit_test.go b/internal/srcedit/srcedit_test.go new file mode 100644 index 0000000..0f7c32d --- /dev/null +++ b/internal/srcedit/srcedit_test.go @@ -0,0 +1,152 @@ +package srcedit + +import ( + "go/token" + "math/rand" + "strings" + "testing" +) + +func TestScanEnds(t *testing.T) { + // The scanner drops '\r' from a raw string's literal; End must still + // be the end of the token in the source. + src := "`a\r\nb` + x /* c\r\n */ ?.y" + toks := Scan(src) + var kinds []token.Token + for _, tk := range toks { + kinds = append(kinds, tk.Kind) + if tk.Pos < 0 || tk.End > len(src) || tk.Pos >= tk.End { + t.Fatalf("bad range %+v", tk) + } + } + if got := src[toks[0].Pos:toks[0].End]; got != "`a\r\nb`" { + t.Fatalf("raw string token is %q", got) + } + want := []token.Token{token.STRING, token.ADD, token.IDENT, token.ILLEGAL, token.PERIOD, token.IDENT} + if len(kinds) != len(want) { + t.Fatalf("kinds %v, want %v", kinds, want) + } + for i := range want { + if kinds[i] != want[i] { + t.Fatalf("kinds %v, want %v", kinds, want) + } + } +} + +func TestScanKeepsExplicitSemicolon(t *testing.T) { + toks := Scan("a\nb;c") + if len(toks) != 4 || toks[2].Kind != token.SEMICOLON { + t.Fatalf("got %+v", toks) + } +} + +func TestBrackets(t *testing.T) { + // The `[` before `0` is unclosed, and the `)` in "[)" closes nothing. + toks := Scan("f([a, {b: (c)}])[0 + [)") + got := Brackets(toks) + want := map[int]int{1: 13, 2: 12, 5: 11, 8: 10} + for i, c := range got { + if w, ok := want[i]; ok { + if c != w { + t.Errorf("token %d (%v) closes at %d, want %d", i, toks[i].Kind, c, w) + } + } else if c != -1 { + t.Errorf("token %d (%v) closes at %d, want unpaired", i, toks[i].Kind, c) + } + } +} + +// randomEdits returns non-overlapping edits of src. +func randomEdits(r *rand.Rand, src string) []Edit { + var edits []Edit + pos := 0 + for pos < len(src) { + pos += r.Intn(4) + if pos > len(src) { + break + } + end := pos + r.Intn(3) + if end > len(src) { + end = len(src) + } + edits = append(edits, Edit{Pos: pos, End: end, Str: strings.Repeat("#", r.Intn(4))}) + pos = end + 1 + } + return edits +} + +func randomSource(r *rand.Rand) string { + const alphabet = "abcdefg \n" + b := make([]byte, r.Intn(30)) + for i := range b { + b[i] = alphabet[r.Intn(len(alphabet))] + } + return string(b) +} + +// copiedOffsets reports, for each output offset, whether Apply copied it +// from the input (rather than inserting it). +func checkMap(t *testing.T, src string, edits []Edit, out string, m Map) { + t.Helper() + inserted := make([]bool, len(out)+1) + o := 0 + last := 0 + for _, e := range edits { + o += e.Pos - last + for k := 0; k < len(e.Str); k++ { + inserted[o+k] = true + } + o += len(e.Str) + last = e.End + } + for off := 0; off < len(out); off++ { + in := m.Orig(off) + if in < 0 || in > len(src) { + t.Fatalf("%q -> %q: Orig(%d) = %d out of range", src, out, off, in) + } + if !inserted[off] && src[in] != out[off] { + t.Fatalf("%q -> %q: Orig(%d) = %d maps %q to %q", src, out, off, in, out[off], src[in]) + } + } + if got := m.Orig(len(out)); got != len(src) { + t.Fatalf("%q -> %q: Orig(end) = %d, want %d", src, out, got, len(src)) + } +} + +func TestApplyMap(t *testing.T) { + r := rand.New(rand.NewSource(1)) + for i := 0; i < 5000; i++ { + src := randomSource(r) + edits := randomEdits(r, src) + out, m := Apply(src, append([]Edit(nil), edits...)) + checkMap(t, src, edits, out, m) + } +} + +func TestThenComposes(t *testing.T) { + r := rand.New(rand.NewSource(2)) + for i := 0; i < 5000; i++ { + s0 := randomSource(r) + s1, m1 := Apply(s0, randomEdits(r, s0)) + s2, m2 := Apply(s1, randomEdits(r, s1)) + s3, m3 := Apply(s2, randomEdits(r, s2)) + all := m3.Then(m2).Then(m1) + for off := 0; off <= len(s3); off++ { + want := m1.Orig(m2.Orig(m3.Orig(off))) + if got := all.Orig(off); got != want { + t.Fatalf("%q -> %q -> %q -> %q: composed Orig(%d) = %d, want %d", s0, s1, s2, s3, off, got, want) + } + } + } +} + +func TestZeroMapIsIdentity(t *testing.T) { + var m Map + if m.Orig(7) != 7 || m.Then(Map{}).Orig(3) != 3 { + t.Fatal("zero map is not the identity") + } + out, m := Apply("abc", nil) + if out != "abc" || m.Orig(2) != 2 { + t.Fatal("no edits is not the identity") + } +} diff --git a/isolated_test.go b/isolated_test.go new file mode 100644 index 0000000..c387c07 --- /dev/null +++ b/isolated_test.go @@ -0,0 +1,280 @@ +package expr + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "reflect" + "runtime/debug" + "sort" + "strings" + "testing" + "time" +) + +// Crash regressions run in child processes. A stack overflow or an +// out-of-memory abort cannot be recovered, so a test that provokes one +// in-process takes the whole suite down with it and hides every other +// result. Each case below runs in its own child, with a small stack +// limit so a regression fails fast, and a timeout so a hang fails too. + +const isolatedEnvVar = "EXPR_ISOLATED_CASE" + +// isolatedCase returns nil when the guarantee holds. +type isolatedCase func() error + +func isolatedCases() map[string]isolatedCase { + cases := map[string]isolatedCase{} + // Formatting: every formatter on every awkward value shape. + for name, build := range map[string]func() (any, string){ + "cyclic-map-keys": func() (any, string) { + type node struct{ M map[*node]int } + m := map[*node]int{} + m[&node{M: m}] = 1 + m[&node{M: m}] = 2 + return m, "" + }, + "cyclic-slice": func() (any, string) { + s := []any{1, nil} + s[1] = s + return s, "" + }, + "cyclic-map": func() (any, string) { + m := map[string]any{"a": 1} + m["self"] = m + return m, "" + }, + "pointer-cycle": func() (any, string) { + type node struct { + V int + Next *node + } + n := &node{V: 1} + n.Next = &node{V: 2, Next: n} + return n, "" + }, + "self-pointer": func() (any, string) { + var p mxSelf + p = mxSelf(&p) + return p, "" + }, + "deep-list": func() (any, string) { + var v any = "leaf" + for i := 0; i < 200_000; i++ { + v = []any{v} + } + return v, "<...>" + }, + "deep-interface-struct": func() (any, string) { + var v any = "leaf" + for i := 0; i < 200_000; i++ { + v = mxHolder{V: v} + } + return v, "<...>" + }, + "shared-dag": func() (any, string) { + // Not a cycle: each level holds the level below twice, so a + // full rendering has 2^64 leaves. + var v any = []any{1} + for i := 0; i < 64; i++ { + v = []any{v, v} + } + return v, "" + }, + "shared-dag-map": func() (any, string) { + var v any = map[string]any{"x": 1} + for i := 0; i < 64; i++ { + v = map[string]any{"a": v, "b": v} + } + return v, "" + }, + } { + for _, how := range []string{"string", "sprintf", "template"} { + cases["format/"+name+"/"+how] = func() error { + v, marker := build() + out, err := formatWith(how, v) + if err != nil { + return err + } + if !strings.Contains(out, marker) { + return fmt.Errorf("output does not mark the bound with %s: %.200q", marker, out) + } + if len(out) > 64<<20 { + return fmt.Errorf("output is %d bytes", len(out)) + } + return nil + } + } + } + + // Evaluation over cyclic and deep values must terminate with a value + // or an ErrEvaluate. + cyclicSlice := func() any { s := []any{1, nil}; s[1] = s; return s } + cyclicMap := func() any { m := map[string]any{"a": 1}; m["self"] = m; return m } + selfPtr := func() any { var p mxSelf; p = mxSelf(&p); return p } + deepPtr := func() any { + // A 10,000-level pointer chain to a struct. + v := reflect.ValueOf(mxInner{B: 1}) + for i := 0; i < 10_000; i++ { + p := reflect.New(v.Type()) + p.Elem().Set(v) + v = p + } + return v.Interface() + } + for name, tc := range map[string]struct { + v func() any + src []string + }{ + "cyclic-slice": {cyclicSlice, []string{"V == V", "contains(V, V)", "len(V)", "map(V, it)[1][1][1][0]", "sort(V)", "V[1][1][1][1][0]", "flatMap(V, it)", "reverse(V)", "V | string() | len() > 0"}}, + "cyclic-map": {cyclicMap, []string{"V == V", "keys(V)", "entries(V)", "V.self.self.self.a", "V?.self?.self?.a", "has(V, \"self\")", "contains(V, \"self\")"}}, + "self-pointer": {selfPtr, []string{"V.x", "V?.x", "V[0]", "V?[0]", "V == nil", "!V", "V == V"}}, + "deep-pointer": {deepPtr, []string{"V.B", "V?.B", "V == nil", "string(V) != \"\""}}, + } { + for _, src := range tc.src { + cases["eval/"+name+"/"+src] = func() error { + p, err := Compile(src, WithBuiltins(), WithFunctions(CollectionFuncs())) + if err != nil { + return err + } + _, err = p.Run(context.Background(), map[string]any{"V": tc.v()}) + if err != nil && !errors.Is(err, ErrEvaluate) { + return fmt.Errorf("error does not wrap ErrEvaluate: %v", err) + } + return nil + } + } + } + + // Compile, Check and Run each bound their own recursion on maximal + // sources: MaxSourceLength bounds the parser and the AST walkers, + // MaxEvalDepth bounds Run. + half := MaxSourceLength/2 - 2 + for name, src := range map[string]string{ + "parens": strings.Repeat("(", half) + "1" + strings.Repeat(")", half), + "unary": strings.Repeat("!", MaxSourceLength-1) + "x", + "index": "x" + strings.Repeat("[0]", (MaxSourceLength-1)/3), + "selector": "x" + strings.Repeat(".a", (MaxSourceLength-1)/2), + "optional": "x" + strings.Repeat("?.a", (MaxSourceLength-1)/3), + "json-list": strings.Repeat("[", half) + strings.Repeat("]", half), + "json-object": strings.Repeat(`{"a":`, (MaxSourceLength-2)/6) + "1" + strings.Repeat("}", (MaxSourceLength-2)/6), + "calls": strings.Repeat("f(", (MaxSourceLength-1)/3) + "x" + strings.Repeat(")", (MaxSourceLength-1)/3), + "pipes": "x" + strings.Repeat(" | f()", (MaxSourceLength-1)/6), + "binary": "1" + strings.Repeat(" + 1", (MaxSourceLength-1)/4), + "binary-nested": strings.Repeat("1 + (", (MaxSourceLength-1)/6) + "1" + strings.Repeat(")", (MaxSourceLength-1)/6), + "if": strings.Repeat("if(c, ", (MaxSourceLength-1)/10) + "1" + strings.Repeat(", 2)", (MaxSourceLength-1)/10), + "map-form": strings.Repeat("map(x, ", (MaxSourceLength-1)/10) + "it" + strings.Repeat(")", (MaxSourceLength-1)/10), + } { + cases["compile/"+name] = func() error { + if len(src) > MaxSourceLength { + return fmt.Errorf("case source is %d bytes, over MaxSourceLength", len(src)) + } + p, err := Compile(src, WithFunctions(map[string]any{"f": func(v any) any { return v }})) + if err != nil { + if !errors.Is(err, ErrCompile) { + return fmt.Errorf("compile error does not wrap ErrCompile: %v", err) + } + return nil + } + if _, err := p.Check(map[string]*Type{"x": Any, "c": Bool()}); err != nil && !errors.Is(err, ErrType) { + return fmt.Errorf("check error does not wrap ErrType: %v", err) + } + _ = p.Identifiers() + _ = p.Calls("f") + _, err = p.Run(context.Background(), map[string]any{"x": []any{1}, "c": true}) + if err != nil && !errors.Is(err, ErrEvaluate) { + return fmt.Errorf("run error does not wrap ErrEvaluate: %v", err) + } + return nil + } + } + cases["compile/template-segments"] = func() error { + src := strings.Repeat("${x}", MaxSourceLength/4) + tmpl, err := NewTemplate(src) + if err != nil { + return err + } + _, err = tmpl.Render(context.Background(), map[string]any{"x": 1}) + return err + } + return cases +} + +// formatWith renders v through one of the three formatters. +func formatWith(how string, v any) (string, error) { + env := map[string]any{"V": v} + if how == "template" { + tmpl, err := NewTemplate(`${V}`) + if err != nil { + return "", err + } + return tmpl.Render(context.Background(), env) + } + src := `string(V)` + if how == "sprintf" { + src = `sprintf("%v", V)` + } + p, err := Compile(src, WithBuiltins()) + if err != nil { + return "", err + } + out, err := p.Run(context.Background(), env) + if err != nil { + return "", err + } + s, ok := out.(string) + if !ok { + return "", fmt.Errorf("formatting returned %T", out) + } + return s, nil +} + +func TestIsolatedCrashRegressions(t *testing.T) { + cases := isolatedCases() + if name := os.Getenv(isolatedEnvVar); name != "" { + // Child: run one case. 64 MiB of stack is ample for every bound + // expr promises and small enough that a regression into unbounded + // recursion dies in well under a second. + debug.SetMaxStack(64 << 20) + c, ok := cases[name] + if !ok { + t.Fatalf("unknown isolated case %q", name) + } + if err := c(); err != nil { + t.Fatal(err) + } + return + } + if testing.Short() { + t.Skip("spawns a child process per case") + } + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + names := make([]string, 0, len(cases)) + for name := range cases { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + t.Run(name, func(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, executable, "-test.run=^TestIsolatedCrashRegressions$", "-test.count=1") + cmd.Env = append(os.Environ(), isolatedEnvVar+"="+name) + start := time.Now() + out, err := cmd.CombinedOutput() + if err != nil { + // A fatal stack dump is large; keep its cause and first frames. + if len(out) > 2000 { + out = append(out[:2000], "\n[output truncated]"...) + } + t.Fatalf("child failed after %v: %v\n%s", time.Since(start).Round(time.Millisecond), err, out) + } + }) + } +} diff --git a/optimize_equiv_test.go b/optimize_equiv_test.go new file mode 100644 index 0000000..02c76fd --- /dev/null +++ b/optimize_equiv_test.go @@ -0,0 +1,344 @@ +package expr + +import ( + "context" + "errors" + "fmt" + "go/token" + "math" + "strings" + "testing" +) + +// Every optimization must be unobservable. Each one below has an +// equivalence test here, comparing a program compiled normally with the +// same program compiled withoutOptimizations, and a benchmark measuring +// what it buys: +// +// optimization equivalence test benchmark +// constant folding (prewalk) TestEquivalenceConstantFolding BenchmarkConstantFolding +// prepared-call cache (callCache) TestEquivalencePreparedCallCache Benchmark_len, Benchmark_callFunc +// reflect selector chain TestEquivalenceSelectorChain Benchmark_envStruct, Benchmark_largeNestedStructAccess +// reflect index chain TestEquivalenceIndexChain Benchmark_largeNestedArrayAccess +// filter(xs, p)[k] streaming TestEquivalenceFilterIndex Benchmark_filterFirst +// typed binary operators TestApplyBinaryFastMatchesGeneral Benchmark_expr +// map[string]any access branches TestValueMatrixPathsAgree Benchmark_envMap +// +// "Equivalent" means the same returned value, the same error category +// and message, and the same host-function calls in the same order. +// Check is not compared: it reads the folded AST, so a constant +// subexpression types as its literal (`1 + 2` as `3`, not number), a +// narrower type that every run still fits. + +// hostLog records calls to the host functions the corpus uses: rec(x) +// logs x and returns it, boom() logs and fails. +type hostLog struct{ calls []string } + +func (h *hostLog) funcs() map[string]any { + return map[string]any{ + "rec": Func(func(_ context.Context, args []any) (any, error) { + h.calls = append(h.calls, fmt.Sprint(args...)) + if len(args) == 0 { + return nil, nil + } + return args[0], nil + }), + "boom": Func(func(context.Context, []any) (any, error) { + h.calls = append(h.calls, "boom") + return nil, errors.New("boom") + }), + "inc": func(x int) int { h.calls = append(h.calls, fmt.Sprint("inc ", x)); return x + 1 }, + } +} + +type equivOutcome struct { + mxOutcome + calls []string +} + +func (o equivOutcome) String() string { + return fmt.Sprintf("%v calls=%q", o.mxOutcome, o.calls) +} + +func runEquiv(src string, env any, noopt bool, opts ...Option) (out equivOutcome) { + h := &hostLog{} + defer func() { + if r := recover(); r != nil { + out = equivOutcome{mxOutcome: mxOutcome{panicked: r}, calls: h.calls} + } + }() + all := append([]Option{WithBuiltins(), WithFunctions(h.funcs())}, opts...) + if noopt { + all = append(all, withoutOptimizations()) + } + p, err := Compile(src, all...) + if err != nil { + return equivOutcome{mxOutcome: mxOutcome{category: errCategory(err), msg: err.Error()}} + } + v, err := p.Run(context.Background(), env) + if err != nil { + return equivOutcome{mxOutcome: mxOutcome{category: errCategory(err), msg: err.Error()}, calls: h.calls} + } + return equivOutcome{mxOutcome: mxOutcome{value: v}, calls: h.calls} +} + +func sameEquiv(a, b equivOutcome) bool { + return sameOutcome(a.mxOutcome, b.mxOutcome) && fmt.Sprint(a.calls) == fmt.Sprint(b.calls) +} + +// assertEquivalent compiles src with and without optimizations, runs +// both against each env, and reports any observable difference. +func assertEquivalent(t *testing.T, src string, envs ...any) { + t.Helper() + for i, env := range envs { + opt, gen := runEquiv(src, env, false), runEquiv(src, env, true) + if opt.panicked != nil || gen.panicked != nil { + t.Errorf("%s (env %d) panicked:\n optimized: %v\n general: %v", src, i, opt, gen) + continue + } + if !sameEquiv(opt, gen) { + t.Errorf("%s (env %d):\n optimized: %v\n general: %v", src, i, opt, gen) + } + } +} + +func TestEquivalenceConstantFolding(t *testing.T) { + env := map[string]any{"V": 2} + for _, src := range []string{ + "1 + 2 * 3", "-(1)", "- -1", "+1", `"a" + "b"`, "1 / 0", "1 % 0", "1.0 / 0", "7 / 2", "7.0 / 2", + "9223372036854775807 + 1", "-9223372036854775808", "-(-9223372036854775808)", + "-9223372036854775807 - 1", "-9223372036854775808 / -1", "-9223372036854775808 % -1", + "2.0 * 3", "(1)", "((1 + 2))", "!0", `!""`, "!nil", "1 + rec(2)", "rec(1) + 2 * 3", + "1.0 / 3", "1e308 * 10", "-1e308 * 10", `+"a"`, `-"a"`, `"a" * 2`, "1 + nil", "nil + nil", + "(1 + 2) == 3", "[1 + 1][0]", `{"k": 2 * 3}.k`, "1 < 2", `"a" < "b"`, "2 > 1.5", + "1 + 2 | string()", "try(1 / 0, \"d\")", "if(1 + 1 == 2, \"y\", 1 / 0)", "V * (2 + 3)", + "0x10 + 0o7 + 0b1", "1_000 * 2", "'a' + 1", "1 + 2 + V", "V + 1 + 2", `"a" + 1`, + "true && 1 / 0", "false || 2 * 3", "(1 + 2) * (3 + 4)", "1 << 2", "5 & 3", "1 + 2.5", + "9007199254740993 == 9007199254740992", "9007199254740993 + 0 == 9007199254740992 + 0", + } { + assertEquivalent(t, src, env) + } +} + +// BenchmarkConstantFolding measures a constant-heavy expression with and +// without folding. +func BenchmarkConstantFolding(b *testing.B) { + env := map[string]any{"x": 3} + for _, noopt := range []bool{false, true} { + name := "folded" + opts := []Option{} + if noopt { + name = "unfolded" + opts = append(opts, withoutOptimizations()) + } + p, err := Compile(`x * (60 * 60 * 24) + (1024 * 1024) - (2 + 3) * 4`, opts...) + if err != nil { + b.Fatal(err) + } + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := p.Run(context.Background(), env); err != nil { + b.Fatal(err) + } + } + }) + } +} + +type equivCallEnv struct { + V any + Len any `expr:"len"` +} + +func TestEquivalencePreparedCallCache(t *testing.T) { + envs := []any{ + map[string]any{"V": []any{1, 2}}, + map[string]any{"V": "héllo"}, + map[string]any{"V": nil}, + // An env entry shadows the registered function: the cache must + // not bypass it. + map[string]any{"V": []any{1}, "len": func(any) int { return 99 }}, + map[string]any{"V": []any{1}, "len": 5}, + map[string]any{"V": []any{1}, "rec": func(x any) string { return "env rec" }}, + equivCallEnv{V: []int{1, 2, 3}}, + } + for _, src := range []string{ + "len(V)", "len(rec(V))", "rec(len(V))", "len(V) + len(V)", "rec(1, 2)", "rec()", + "boom()", "try(boom(), 1)", "rec(boom())", "inc(1)", "inc(1.5)", "inc(\"x\")", "inc()", + "inc(9223372036854775807)", "contains(rec(V), 1)", "V | len()", "V | rec() | len()", + "len(V) > 1 && rec(true)", "upper(\"a\")", "sprintf(\"%v-%v\", rec(1), rec(2))", + } { + assertEquivalent(t, src, envs...) + } + // A struct env with a tagged field named like the function. + for _, src := range []string{"len(V)"} { + for _, noopt := range []bool{false, true} { + o := runEquiv(src, equivCallEnv{V: []int{1}, Len: func(any) int { return 7 }}, noopt, WithFieldTags("expr")) + if o.value != 7 { + t.Errorf("%s with a tagged env field shadowing len (noopt=%v) = %v, want 7", src, noopt, o) + } + } + } +} + +type eqInner struct { + B any + C *eqInner + M map[string]any +} + +func (e eqInner) Method() string { return "m" } +func (e *eqInner) PMethod() string { return "pm" } + +type eqEnv struct { + A eqInner + P *eqInner + N *eqInner + I any + L []eqInner + D [2][3]int + M map[string]any + S map[string]eqInner + K any + PP **eqInner +} + +func equivStructEnvs() []any { + inner := &eqInner{B: 1, C: &eqInner{B: "deep"}, M: map[string]any{"x": []any{1, 2}}} + env := eqEnv{ + A: *inner, P: inner, I: inner, L: []eqInner{*inner, {}}, + D: [2][3]int{{1, 2, 3}, {4, 5, 6}}, M: map[string]any{"a": map[string]any{"b": 1}, "l": []any{1}}, + S: map[string]eqInner{"k": *inner}, K: "a", PP: &inner, + } + return []any{env, &env, eqEnv{}, map[string]any{"A": env.A, "P": env.P, "N": nil, "I": env.I, "L": env.L, "D": env.D, "M": env.M, "S": env.S, "K": "a", "PP": env.PP}} +} + +func TestEquivalenceSelectorChain(t *testing.T) { + for _, src := range []string{ + "A.B", "A.C.B", "A.C.C.B", "P.B", "P.C.B", "N.B", "I.B", "I.C.B", "A.M.x", "M.a.b", "M.a.c", + "S.k.B", "S.missing.B", "A.missing", "A.b", "P.missing", "N.C", "PP.B", "PP.C.B", + "A.Method", "P.PMethod", "A.Method()", "P.PMethod()", "P.Method()", "I.Method()", + "A.B.x", "D.x", "L.x", "K.x", "A?.C?.B", "N?.C?.B", + } { + assertEquivalent(t, src, equivStructEnvs()...) + } +} + +func TestEquivalenceIndexChain(t *testing.T) { + for _, src := range []string{ + "D[0][1]", "D[rec(1)][rec(2)]", "D[1][3]", "D[2][0]", "D[-1][0]", "D[0][1.0]", "D[0][1.5]", + "L[0].B", "L[rec(0)].C.B", "L[1].C.B", "L[5].B", "L[boom()].B", "L[1 / 0].B", "L[rec(0)].M[rec(\"x\")][rec(1)]", + "M[K].b", "M[rec(K)][rec(\"b\")]", "M[\"l\"][0]", "M[\"l\"][rec(5)]", "M[1]", "S[K]", "S[\"k\"].B", + "A.M[\"x\"][1]", "P.M[\"x\"][0]", "N.M[\"x\"]", "I.M[\"x\"][0]", "PP.M[\"x\"][0]", + "D[rec(0)][boom()]", "D[boom()][rec(0)]", "D[0][rec(1)] + D[rec(1)][0]", + "L[0].B[0]", "A.B[0]", "D[0][0][0]", "M[K][K]", "D[nil]", + } { + assertEquivalent(t, src, equivStructEnvs()...) + } +} + +func TestEquivalenceFilterIndex(t *testing.T) { + envs := []any{ + map[string]any{"xs": []any{1, 2, 3, 4}, "I": 0}, + map[string]any{"xs": []int{5, 0, 7}, "I": 1}, + map[string]any{"xs": [3]int{1, 2, 3}, "I": 2}, + map[string]any{"xs": nil, "I": 0}, + map[string]any{"xs": "abc", "I": 0}, + map[string]any{"xs": []any{}, "I": 0}, + map[string]any{"xs": []any{1, 2}, "I": 0, "filter": func(xs any, f any) []any { return []any{"shadow"} }}, + } + // Each literal-index form must behave like the general path and like + // the same expression with the index in a variable. + for _, tc := range []struct{ lit, variable string }{ + {"filter(xs, rec(it) > 1)[0]", "filter(xs, rec(it) > 1)[I]"}, + {"filter(xs, rec(it) > 1)[1]", "filter(xs, rec(it) > 1)[I + 1]"}, + {"filter(xs, true)[2]", "filter(xs, true)[I + 2]"}, + {"filter(xs, false)[0]", "filter(xs, false)[I]"}, + {"filter(xs, x, rec(x) > 1)[0]", "filter(xs, x, rec(x) > 1)[I]"}, + {"filter(xs, 1 / it > 0)[0]", "filter(xs, 1 / it > 0)[I]"}, + {"filter(xs, it == 2 || boom())[0]", "filter(xs, it == 2 || boom())[I]"}, + {"filter(xs, rec(index) >= 0)[0x1]", "filter(xs, rec(index) >= 0)[I + 1]"}, + {"filter(xs, it)[99]", "filter(xs, it)[I + 99]"}, + {"filter(xs, 1, it)[0]", "filter(xs, 1, it)[I]"}, + {"filter(filter(xs, rec(it) > 1), rec(it) < 4)[0]", "filter(filter(xs, rec(it) > 1), rec(it) < 4)[I]"}, + } { + assertEquivalent(t, tc.lit, envs...) + assertEquivalent(t, tc.variable, envs...) + for i, env := range envs { + m := env.(map[string]any) + lit := runEquiv(tc.lit, env, false) + // Shift the variable so both forms read the same position. + shifted := map[string]any{} + for k, v := range m { + shifted[k] = v + } + shifted["I"] = 0 + variable := runEquiv(tc.variable, shifted, false) + if !sameOutcome(lit.mxOutcome, variable.mxOutcome) || fmt.Sprint(lit.calls) != fmt.Sprint(variable.calls) { + t.Errorf("env %d: literal and variable index disagree:\n %s: %v\n %s: %v", i, tc.lit, lit, tc.variable, variable) + } + } + } +} + +// TestApplyBinaryFastMatchesGeneral checks the typed fast path against +// the general operator implementation on every operand pair it handles. +func TestApplyBinaryFastMatchesGeneral(t *testing.T) { + values := []any{ + int64(0), int64(1), int64(-1), int64(7), int64(math.MaxInt64), int64(math.MinInt64), int64(1 << 53), int64(1<<53 + 1), + 0, 1, -3, math.MaxInt64, math.MinInt64, + 0.0, 1.0, -1.5, 2.5, 1e300, math.Inf(1), math.Inf(-1), math.NaN(), float64(1 << 53), + "", "a", "b", "ab", + true, false, + } + ops := []token.Token{token.ADD, token.SUB, token.MUL, token.QUO, token.REM, + token.EQL, token.NEQ, token.LSS, token.GTR, token.LEQ, token.GEQ, + token.AND, token.OR, token.XOR, token.SHL, token.SHR, token.AND_NOT} + checked := 0 + for _, op := range ops { + for _, l := range values { + for _, r := range values { + fv, ferr, ok := applyBinaryFast(op, l, r) + if !ok { + continue + } + checked++ + gv, gerr := applyBinaryGeneral(op, l, r) + same := fmt.Sprint(ferr) == fmt.Sprint(gerr) && sameNumber(fv, gv) + if !same { + t.Errorf("%#v %v %#v: fast = (%#v, %v), general = (%#v, %v)", l, op, r, fv, ferr, gv, gerr) + } + } + } + } + if checked < 500 { + t.Fatalf("only %d operand pairs reached the fast path", checked) + } +} + +// sameNumber is equality that treats NaN as equal to NaN, so the +// fast/general comparison is about which value is produced. +func sameNumber(a, b any) bool { + if fa, ok := a.(float64); ok { + if fb, ok := b.(float64); ok && math.IsNaN(fa) && math.IsNaN(fb) { + return true + } + } + return a == b +} + +// TestEquivalenceCorpus runs the shared value matrix expressions through +// the equivalence check too, with host-call logging, so every +// optimization sees the full set of host value shapes. +func TestEquivalenceCorpus(t *testing.T) { + for _, nv := range matrixValues() { + env := map[string]any{"V": nv.v, "K": "a", "I": 0} + for _, src := range mxExprs { + if strings.Contains(src, "sprintf") { + continue // covered by the matrix; formatting is not optimized + } + assertEquivalent(t, src, env, mxEnv{V: nv.v, K: "a", I: 0}) + } + } +} diff --git a/prepared.go b/prepared.go index 4077016..47ec8e0 100644 --- a/prepared.go +++ b/prepared.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "sync" ) // Func is a native dispatcher signature. Functions registered with @@ -17,37 +18,31 @@ import ( type Func func(ctx context.Context, args []any) (any, error) // preparedFunc holds everything callPrepared needs to invoke a -// registered function, built once at Engine construction. The native -// field is the fast path; when it is nil the reflect fields drive a -// fallback that reuses the cached type/value rather than re-deriving -// them on every call. +// registered function, built once when the option is applied. The +// native field is the fast path; when it is nil, fv and sig drive the +// reflect path shared with env functions and bound methods. type preparedFunc struct { name string // Native fast path. native Func - // Reflect fallback metadata (only populated when native is nil). - fv reflect.Value - ft reflect.Type - paramOff int // 1 if the function takes context.Context as its first param - declaredIn int // user-visible parameter count (excluding ctx) - fixed int // len(declaredIn) - 1 for variadic, else declaredIn - variadic bool - paramTypes []reflect.Type // cached ft.In(paramOff+i) for i in [0, declaredIn) - variadicElem reflect.Type // element type if variadic - numOut int - hasErrRet bool // true if second return is error + // Reflect path (only populated when native is nil). + fv reflect.Value + sig *callSig } // prepareFunc inspects fn and builds a preparedFunc. If fn already -// matches the native Func shape it is stored directly; otherwise the -// function's reflect metadata is cached for the fallback path. +// matches the native Func shape it is stored directly; otherwise its +// signature is validated and cached for the reflect path. func prepareFunc(name string, fn any) (*preparedFunc, error) { if fn == nil { return nil, fmt.Errorf("function %q is nil", name) } if nf, ok := fn.(Func); ok { + if nf == nil { + return nil, fmt.Errorf("function %q is a nil Func", name) + } return &preparedFunc{name: name, native: nf}, nil } fv := reflect.ValueOf(fn) @@ -57,100 +52,120 @@ func prepareFunc(name string, fn any) (*preparedFunc, error) { if fv.IsNil() { return nil, fmt.Errorf("function %q is a nil function value", name) } - ft := fv.Type() - p := &preparedFunc{ - name: name, - fv: fv, - ft: ft, - variadic: ft.IsVariadic(), - numOut: ft.NumOut(), - } - if ft.NumIn() > 0 && ft.In(0) == ctxType { - p.paramOff = 1 - } - p.declaredIn = ft.NumIn() - p.paramOff - if p.variadic { - p.fixed = p.declaredIn - 1 - p.variadicElem = ft.In(ft.NumIn() - 1).Elem() - } else { - p.fixed = p.declaredIn - } - p.paramTypes = make([]reflect.Type, p.declaredIn) - for i := 0; i < p.declaredIn; i++ { - p.paramTypes[i] = ft.In(p.paramOff + i) - } - if p.numOut == 2 { - if !ft.Out(1).Implements(errValType) { - return nil, fmt.Errorf("function %q: second return must be error, got %v", name, ft.Out(1)) - } - p.hasErrRet = true - } else if p.numOut > 2 { - return nil, fmt.Errorf("function %q returns %d values (expected 0, 1, or (T, error))", name, p.numOut) + sig, err := sigOf(fv.Type()) + if err != nil { + return nil, fmt.Errorf("function %q %v", name, err) } - return p, nil + return &preparedFunc{name: name, fv: fv, sig: sig}, nil } // callPrepared invokes pf with args. Uses the native dispatcher when -// available, otherwise takes the reflect fallback path using cached -// metadata. +// available, otherwise the shared reflect path. func callPrepared(ctx context.Context, pf *preparedFunc, args []any) (any, error) { if pf.native != nil { - return pf.native(ctx, args) + v, err := pf.native(ctx, args) + return v, wrapCallErr(pf.name, err) } - return callPreparedReflect(ctx, pf, args) + return invokeReflect(ctx, pf.name, pf.fv, pf.sig, args) } -func callPreparedReflect(ctx context.Context, pf *preparedFunc, args []any) (any, error) { - if pf.variadic { - if len(args) < pf.fixed { - return nil, fmt.Errorf("%w: %q expects at least %d args, got %d", ErrEvaluate, pf.name, pf.fixed, len(args)) +// callSig is what the reflect call path needs to know about a function +// type: how to build its arguments and read its results. It is computed +// once per type (sigOf) and shared by registered functions, env +// functions and bound methods, so the three apply the same argument +// conversions and result rules. +type callSig struct { + paramOff int // 1 if the function takes context.Context as its first param + declaredIn int // user-visible parameter count (excluding ctx) + fixed int // declaredIn - 1 for variadic, else declaredIn + variadic bool + paramTypes []reflect.Type // ft.In(paramOff+i) for i in [0, declaredIn) + variadicElem reflect.Type // element type if variadic + numOut int +} + +var callSigs sync.Map // reflect.Type -> *callSig or error + +// sigOf returns the callSig of a function type, or an error (without an +// ErrEvaluate/ErrCompile prefix or function name) for a result shape +// expr does not support. +func sigOf(ft reflect.Type) (*callSig, error) { + if cached, ok := callSigs.Load(ft); ok { + if err, isErr := cached.(error); isErr { + return nil, err } - } else if len(args) != pf.declaredIn { - return nil, fmt.Errorf("%w: %q expects %d args, got %d", ErrEvaluate, pf.name, pf.declaredIn, len(args)) + return cached.(*callSig), nil + } + sig, err := buildSig(ft) + if err != nil { + callSigs.Store(ft, err) + return nil, err + } + callSigs.Store(ft, sig) + return sig, nil +} + +func buildSig(ft reflect.Type) (*callSig, error) { + s := &callSig{variadic: ft.IsVariadic(), numOut: ft.NumOut()} + switch { + case s.numOut > 2: + return nil, fmt.Errorf("returns %d values (expected 0, 1, or (T, error))", s.numOut) + case s.numOut == 2 && !ft.Out(1).Implements(errValType): + return nil, fmt.Errorf("second return must be error, got %v", ft.Out(1)) + } + if ft.NumIn() > 0 && ft.In(0) == ctxType { + s.paramOff = 1 } + s.declaredIn = ft.NumIn() - s.paramOff + s.fixed = s.declaredIn + if s.variadic { + s.fixed-- + s.variadicElem = ft.In(ft.NumIn() - 1).Elem() + } + s.paramTypes = make([]reflect.Type, s.declaredIn) + for i := range s.paramTypes { + s.paramTypes[i] = ft.In(s.paramOff + i) + } + return s, nil +} - in := make([]reflect.Value, pf.paramOff+len(args)) - if pf.paramOff == 1 { +// invokeReflect checks and converts args for sig, calls fv, and reads +// its results. A runtime panic in the callee becomes an ErrEvaluate. +func invokeReflect(ctx context.Context, name string, fv reflect.Value, sig *callSig, args []any) (any, error) { + if sig.variadic { + if len(args) < sig.fixed { + return nil, fmt.Errorf("%w: %q expects at least %d args, got %d", ErrEvaluate, name, sig.fixed, len(args)) + } + } else if len(args) != sig.declaredIn { + return nil, fmt.Errorf("%w: %q expects %d args, got %d", ErrEvaluate, name, sig.declaredIn, len(args)) + } + in := make([]reflect.Value, sig.paramOff+len(args)) + if sig.paramOff == 1 { in[0] = reflect.ValueOf(ctx) } - for i := 0; i < pf.fixed; i++ { - v, err := convertArg(pf.name, i, args[i], pf.paramTypes[i]) + for i, a := range args { + want := sig.variadicElem + if i < sig.fixed || !sig.variadic { + want = sig.paramTypes[i] + } + v, err := convertArg(name, i, a, want) if err != nil { return nil, err } - in[pf.paramOff+i] = v - } - if pf.variadic { - for i := pf.fixed; i < len(args); i++ { - v, err := convertArg(pf.name, i, args[i], pf.variadicElem) - if err != nil { - return nil, err - } - in[pf.paramOff+i] = v - } - } else { - for i := pf.fixed; i < len(args); i++ { - v, err := convertArg(pf.name, i, args[i], pf.paramTypes[i]) - if err != nil { - return nil, err - } - in[pf.paramOff+i] = v - } + in[sig.paramOff+i] = v } - - out, err := callReflectRecoverRuntime(pf.name, pf.fv, in) + out, err := callReflectRecoverRuntime(name, fv, in) if err != nil { return nil, err } - switch pf.numOut { + switch sig.numOut { case 0: return nil, nil case 1: return out[0].Interface(), nil } - // (T, error) - if !out[1].IsNil() { - return nil, out[1].Interface().(error) + if err := callResultErr(out[1]); err != nil { + return nil, wrapCallErr(name, err) } return out[0].Interface(), nil } diff --git a/preprocess_growth_test.go b/preprocess_growth_test.go new file mode 100644 index 0000000..4f62fa5 --- /dev/null +++ b/preprocess_growth_test.go @@ -0,0 +1,149 @@ +package expr + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// growthShape builds a source of roughly n units of one syntax feature, +// to measure how compile cost grows with input size. +type growthShape struct { + name string + build func(n int) string + template bool +} + +func nest(n int, open, leaf, close string) string { + return strings.Repeat(open, n) + leaf + strings.Repeat(close, n) +} + +var growthShapes = []growthShape{ + {name: "optional_chain", build: func(n int) string { return "a" + strings.Repeat("?.a", n) }}, + {name: "optional_index_chain", build: func(n int) string { return "a" + strings.Repeat("?[0]", n) }}, + {name: "optional_paren_nest", build: func(n int) string { + // ((x)?.a)?.a: each `?` walks back over a growing group. + return nest(n, "(", "x", ")?.a") + }}, + {name: "optional_in_index", build: func(n int) string { return nest(n, "a?[", "0", "]") }}, + {name: "json_array_nest", build: func(n int) string { return nest(n, "[", "1", "]") }}, + {name: "json_object_nest", build: func(n int) string { return nest(n, `{"a": `, "1", "}") }}, + {name: "json_flat", build: func(n int) string { return "[" + strings.Repeat(`{"a": [1]}, `, n) + "]" }}, + {name: "typed_literal_optional", build: func(n int) string { return strings.Repeat("[]any{1}?[0] + ", n) + "0" }}, + {name: "keyword_fields", build: func(n int) string { return strings.Repeat("x.type + x?.map + ", n) + "0" }}, + {name: "map_if_keywords", build: func(n int) string { return nest(n, "if(c, map(xs, it), ", "0", ")") }}, + {name: "pipe_chain", build: func(n int) string { return "x" + strings.Repeat(" | f()", n) }}, + {name: "comments", build: func(n int) string { return strings.Repeat("x /* [ { ?. map */ ?.a + ", n) + "0" }}, + {name: "template_segments", template: true, build: func(n int) string { return strings.Repeat("a ${x?.a} b ", n) }}, +} + +func compileShape(s growthShape, src string) { + if s.template { + _, _ = NewTemplate(src) + return + } + _, _ = Compile(src) +} + +// sizeFor returns the largest unit count whose source fits in limit. +func sizeFor(s growthShape, limit int) int { + lo, hi := 1, limit + for lo < hi { + mid := (lo + hi + 1) / 2 + if len(s.build(mid)) <= limit { + lo = mid + } else { + hi = mid - 1 + } + } + return lo +} + +func allocatedBytes(f func()) uint64 { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + f() + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +// Compile (preprocessing, parse and tree passes) must grow about +// linearly with input size: doubling the input may at most ~2.5x the +// allocated bytes, measured up to MaxSourceLength. Allocation is +// deterministic, unlike wall time, so the bound does not flake. +func TestCompileGrowthIsLinear(t *testing.T) { + if testing.Short() { + t.Skip("allocates up to MaxSourceLength per shape") + } + for _, s := range growthShapes { + t.Run(s.name, func(t *testing.T) { + big := sizeFor(s, MaxSourceLength) + small := big / 2 + srcSmall, srcBig := s.build(small), s.build(big) + // Warm up so one-time initialization is not counted. + compileShape(s, srcSmall) + a := allocatedBytes(func() { compileShape(s, srcSmall) }) + b := allocatedBytes(func() { compileShape(s, srcBig) }) + ratio := float64(b) / float64(a) + lenRatio := float64(len(srcBig)) / float64(len(srcSmall)) + t.Logf("%d -> %d bytes of source: %d -> %d bytes allocated (x%.2f)", len(srcSmall), len(srcBig), a, b, ratio) + if ratio > 1.25*lenRatio { + t.Errorf("allocation grew x%.2f for x%.2f input: superlinear", ratio, lenRatio) + } + }) + } +} + +// Each preprocessing stage's output must stay within a constant factor +// of its input, so later stages and the parser see bounded input. +func TestRewriteOutputIsLinear(t *testing.T) { + for _, s := range growthShapes { + if s.template { + continue + } + src := s.build(sizeFor(s, MaxSourceLength)) + out, _ := rewriteSource(src) + // The largest expansion is `?[` → `__try_index__(` plus `, `: + // 16 bytes for 2. + if len(out) > 9*len(src) { + t.Errorf("%s: %d bytes rewritten to %d", s.name, len(src), len(out)) + } + } +} + +func BenchmarkCompileGrowth(b *testing.B) { + for _, s := range growthShapes { + for _, limit := range []int{MaxSourceLength / 16, MaxSourceLength / 4, MaxSourceLength} { + src := s.build(sizeFor(s, limit)) + b.Run(fmt.Sprintf("%s/%dKiB", s.name, limit/1024), func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + compileShape(s, src) + } + }) + } + } +} + +// BenchmarkRewriteGrowth measures the token rewrites alone, without the +// parser and tree passes. +func BenchmarkRewriteGrowth(b *testing.B) { + for _, s := range growthShapes { + if s.template { + continue + } + for _, limit := range []int{MaxSourceLength / 16, MaxSourceLength} { + src := s.build(sizeFor(s, limit)) + b.Run(fmt.Sprintf("%s/%dKiB", s.name, limit/1024), func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + rewriteSource(src) + } + }) + } + } +} diff --git a/program.go b/program.go index 815ab05..efdf396 100644 --- a/program.go +++ b/program.go @@ -42,6 +42,14 @@ type Program struct { // counter so concurrent Runs never share budget state. evalBudget int budgetLeft *int64 + + // noOptimize disables folding and fast paths (see + // withoutOptimizations). + noOptimize bool + + // deferred, when set, holds source and identifiers for a Program + // made by sub, computed on first use. + deferred *deferredInfo } // compile walks the root AST once to populate the lookup caches used @@ -118,7 +126,7 @@ func (p *Program) prewalk(node ast.Expr) ast.Expr { case *ast.ParenExpr: n.X = p.prewalk(n.X) // Unwrap parens around constants so constValue sees them. - if _, ok := p.constValue(n.X); ok { + if _, ok := p.constValue(n.X); ok && !p.noOptimize { return n.X } return n @@ -126,9 +134,10 @@ func (p *Program) prewalk(node ast.Expr) ast.Expr { n.X = p.prewalk(n.X) // `-9223372036854775808` (MinInt64) is special: the bare // literal overflows int64 on its own, so it never reaches - // litCache and the generic fold below can't see it. Parse the - // negation as a single signed literal instead, matching how - // users read it. + // litCache. Parse the negation as a single signed literal + // instead, matching how users read it. This is part of the + // language, not an optimization, so it runs even when folding + // is off. if n.Op == token.SUB { if lit, ok := n.X.(*ast.BasicLit); ok && lit.Kind == token.INT { if _, cached := p.litCache[lit]; !cached { @@ -140,6 +149,9 @@ func (p *Program) prewalk(node ast.Expr) ast.Expr { } } } + if p.noOptimize { + return n + } if xv, ok := p.constValue(n.X); ok { if v, err := applyUnary(n.Op, xv); err == nil { if wrapped, ok := p.wrapConst(v); ok { @@ -157,7 +169,7 @@ func (p *Program) prewalk(node ast.Expr) ast.Expr { // isn't worth a second code path. Skip == and != too: they // never fail, so folding `"a" == 1` to false would hide a // comparison that is never true from Check. - if n.Op != token.LAND && n.Op != token.LOR && n.Op != token.EQL && n.Op != token.NEQ { + if !p.noOptimize && n.Op != token.LAND && n.Op != token.LOR && n.Op != token.EQL && n.Op != token.NEQ { if xv, ok := p.constValue(n.X); ok { if yv, ok2 := p.constValue(n.Y); ok2 { if v, err := applyBinary(n.Op, xv, yv); err == nil { @@ -183,7 +195,7 @@ func (p *Program) prewalk(node ast.Expr) ast.Expr { // Keyword sentinels (`__expr_if__`, etc.) are translated back // to their user-visible names so the prepared lookup keys on // what the user actually wrote. - if ident, ok := n.Fun.(*ast.Ident); ok { + if ident, ok := n.Fun.(*ast.Ident); ok && !p.noOptimize { if _, isForm := higherOrderForms[ident.Name]; !isForm { name := displayIdent(ident.Name) if pf, ok := p.prepared[name]; ok && (pf.native != nil || pf.fv.IsValid()) { @@ -242,7 +254,10 @@ func applyUnary(op token.Token, v any) (any, error) { var _ runner = (*Program)(nil) // Source returns the original expression text. -func (p *Program) Source() string { return p.source } +func (p *Program) Source() string { + source, _ := p.resolveDeferred() + return source +} // Run evaluates the program against env. env may be a map[string]any, a // struct, or a pointer to a struct — identifier lookups resolve to map @@ -450,65 +465,6 @@ func evalIdent(n *ast.Ident, env any, funcs map[string]any, fieldTags *structTag ErrEvaluate, name, identHint(env, funcs, name, fieldTags)) } -// lookupEnv resolves a top-level identifier against env. env may be a -// map[string]any, any map with string keys, a struct, a pointer to a -// struct, or an *itEnv wrapping one of the above. For structs, configured -// field tags participate in field lookup, and fields are preferred over -// methods when both match. Methods are returned as bound function values -// so they can be invoked by a CallExpr node. An itEnv binds its element -// name (`it`, or the named binding of a three-arg form) and `index` -// ahead of anything in its parent. -func lookupEnv(env any, name string, fieldTags *structTagConfig) (any, bool, error) { - if env == nil { - return nil, false, nil - } - if it, ok := env.(*itEnv); ok { - switch name { - case it.elementName(): - return it.it, true, nil - case "index": - return it.index, true, nil - } - return lookupEnv(it.parent, name, fieldTags) - } - if m, ok := env.(map[string]any); ok { - v, ok := m[name] - return v, ok, nil - } - rv := reflect.ValueOf(env) - // Check methods on the original (possibly pointer) value first so - // pointer-receiver methods are visible — but prefer struct fields if - // both exist with the same name. - orig := rv - if rv.Kind() == reflect.Pointer { - if rv.IsNil() { - return nil, false, nil - } - rv = rv.Elem() - } - switch rv.Kind() { - case reflect.Struct: - fv, ok, err := structFieldByName(rv, name, fieldTags) - if err != nil { - return nil, false, err - } - if ok && fv.CanInterface() { - return fv.Interface(), true, nil - } - if mv := orig.MethodByName(name); mv.IsValid() { - return methodCallable(name, mv), true, nil - } - case reflect.Map: - if rv.Type().Key().Kind() == reflect.String { - mv := rv.MapIndex(mapStringKey(rv.Type().Key(), name)) - if mv.IsValid() { - return mv.Interface(), true, nil - } - } - } - return nil, false, nil -} - func (p *Program) evalUnary(ctx context.Context, n *ast.UnaryExpr, env any, depth int) (any, error) { v, err := p.eval(ctx, n.X, env, depth) if err != nil { @@ -569,75 +525,96 @@ func (p *Program) evalBinary(ctx context.Context, n *ast.BinaryExpr, env any, de if err != nil { return nil, err } + if p.noOptimize { + return applyBinaryGeneral(n.Op, lhs, rhs) + } return applyBinary(n.Op, lhs, rhs) } +// applyBinary applies a non-short-circuit binary operator. It tries +// applyBinaryFast, then applyBinaryGeneral; the two must agree wherever +// the fast path answers (TestApplyBinaryFastMatchesGeneral). func applyBinary(op token.Token, lhs, rhs any) (any, error) { - // Fast path: exact int64/int/float64/string/bool operand pairs cover - // the vast majority of expressions and skip the reflect-based - // coercion helpers below. Unsupported operators fall through so the - // general path produces its usual error (with the original types). + if v, err, ok := applyBinaryFast(op, lhs, rhs); ok { + return v, err + } + return applyBinaryGeneral(op, lhs, rhs) +} + +// applyBinaryFast handles exact int64/int/float64/string/bool operand +// pairs, which cover the vast majority of expressions, without the +// reflect-based coercion helpers. ok is false when it does not apply; +// unsupported operators fall through so the general path produces its +// usual error (with the original types). +func applyBinaryFast(op token.Token, lhs, rhs any) (any, error, bool) { switch l := lhs.(type) { case int64: switch r := rhs.(type) { case int64: if v, err, ok := applyInt64Binary(op, l, r); ok { - return v, err + return v, err, true } case int: if v, err, ok := applyInt64Binary(op, l, int64(r)); ok { - return v, err + return v, err, true } case float64: if v, err, ok := applyFloat64Binary(op, float64(l), r); ok { - return v, err + return v, err, true } } case int: switch r := rhs.(type) { case int64: if v, err, ok := applyInt64Binary(op, int64(l), r); ok { - return v, err + return v, err, true } case int: if v, err, ok := applyInt64Binary(op, int64(l), int64(r)); ok { - return v, err + return v, err, true } case float64: if v, err, ok := applyFloat64Binary(op, float64(l), r); ok { - return v, err + return v, err, true } } case float64: switch r := rhs.(type) { case int64: if v, err, ok := applyFloat64Binary(op, l, float64(r)); ok { - return v, err + return v, err, true } case int: if v, err, ok := applyFloat64Binary(op, l, float64(r)); ok { - return v, err + return v, err, true } case float64: if v, err, ok := applyFloat64Binary(op, l, r); ok { - return v, err + return v, err, true } } case string: if r, ok := rhs.(string); ok { - return applyStringBinary(op, l, r) + v, err := applyStringBinary(op, l, r) + return v, err, true } case bool: if r, ok := rhs.(bool); ok { switch op { case token.EQL: - return l == r, nil + return l == r, nil, true case token.NEQ: - return l != r, nil + return l != r, nil, true } } } + return nil, nil, false +} + +// applyBinaryGeneral applies op to any operand types, coercing numbers +// of every Go width and named string/bool types. +func applyBinaryGeneral(op token.Token, lhs, rhs any) (any, error) { // String concatenation and comparison. if ls, lok := asString(lhs); lok { if rs, rok := asString(rhs); rok { @@ -700,6 +677,20 @@ func applyBinary(op token.Token, lhs, rhs any) (any, error) { } } + // Integers outside int64 (large uint64s) still order exactly. + if c, ok := compareIntegers(lhs, rhs); ok { + switch op { + case token.LSS: + return c < 0, nil + case token.GTR: + return c > 0, nil + case token.LEQ: + return c <= 0, nil + case token.GEQ: + return c >= 0, nil + } + } + lf, lfOk := toFloat64(lhs) rf, rfOk := toFloat64(rhs) if lfOk && rfOk { @@ -742,9 +733,9 @@ func applyBinary(op token.Token, lhs, rhs any) (any, error) { func applyInt64Binary(op token.Token, l, r int64) (any, error, bool) { switch op { case token.EQL: - return float64(l) == float64(r), nil, true + return l == r, nil, true case token.NEQ: - return float64(l) != float64(r), nil, true + return l != r, nil, true case token.ADD: if v, ok := checkedAddInt64(l, r); ok { return v, nil, true @@ -877,8 +868,10 @@ func (p *Program) evalSelector(ctx context.Context, n *ast.SelectorExpr, env any // embedded array gets memcpy'd on each access. The fast path skips // plain map[string]any roots because the type-asserted map lookup // in selectField is already faster than reflect MapIndex. - if v, ok, err := evalSelectorChainRV(n, env, depth, p.fieldTags); ok { - return v, err + if !p.noOptimize { + if v, ok, err := evalSelectorChainRV(n, env, depth, p.fieldTags); ok { + return v, err + } } recv, err := p.eval(ctx, n.X, env, depth) if err != nil { @@ -943,164 +936,16 @@ func evalSelectorChainRV(n *ast.SelectorExpr, env any, depth int, fieldTags *str return rv.Interface(), true, nil } -// lookupEnvRV mirrors lookupEnv but returns a reflect.Value so callers -// can chain field accesses without boxing intermediate struct values. -// Map and itEnv lookups still go through any internally because their -// stored values already live behind interfaces. -func lookupEnvRV(env any, name string, fieldTags *structTagConfig) (reflect.Value, bool, error) { - if env == nil { - return reflect.Value{}, false, nil - } - if it, ok := env.(*itEnv); ok { - switch name { - case it.elementName(): - return reflect.ValueOf(it.it), true, nil - case "index": - return reflect.ValueOf(it.index), true, nil - } - return lookupEnvRV(it.parent, name, fieldTags) - } - if m, ok := env.(map[string]any); ok { - v, ok := m[name] - if !ok { - return reflect.Value{}, false, nil - } - return reflect.ValueOf(v), true, nil - } - rv := reflect.ValueOf(env) - orig := rv - if rv.Kind() == reflect.Pointer { - if rv.IsNil() { - return reflect.Value{}, false, nil - } - rv = rv.Elem() - } - switch rv.Kind() { - case reflect.Struct: - fv, ok, err := structFieldByName(rv, name, fieldTags) - if err != nil { - return reflect.Value{}, false, err - } - if ok && fv.CanInterface() { - return fv, true, nil - } - if mv := orig.MethodByName(name); mv.IsValid() { - return reflect.ValueOf(methodCallable(name, mv)), true, nil - } - case reflect.Map: - if rv.Type().Key().Kind() == reflect.String { - mv := rv.MapIndex(mapStringKey(rv.Type().Key(), name)) - if mv.IsValid() { - return mv, true, nil - } - } - } - return reflect.Value{}, false, nil -} - -// selectFieldRV reads a field/key from rv without boxing the parent -// through any. It mirrors selectField's behavior on structs and -// string-keyed maps, transparently dereffing pointers and unwrapping -// interface values. Errors are produced lazily — the cold path may -// call Interface() to build a "did you mean" hint. -func selectFieldRV(rv reflect.Value, name string, fieldTags *structTagConfig) (reflect.Value, error) { - for rv.Kind() == reflect.Interface && !rv.IsNil() { - rv = rv.Elem() - } - if !rv.IsValid() { - return reflect.Value{}, fmt.Errorf("%w: cannot access %q on nil", ErrEvaluate, name) - } - if rv.Kind() == reflect.Pointer { - if rv.IsNil() { - return reflect.Value{}, fmt.Errorf("%w: cannot access %q on nil pointer", ErrEvaluate, name) - } - rv = rv.Elem() - } - switch rv.Kind() { - case reflect.Struct: - fv, ok, err := structFieldByName(rv, name, fieldTags) - if err != nil { - return reflect.Value{}, err - } - if !ok || !fv.IsValid() || !fv.CanInterface() { - return reflect.Value{}, fmt.Errorf("%w: field %q not found on %v%s", - ErrEvaluate, name, rv.Type(), fieldHint(rv.Interface(), name, fieldTags)) - } - return fv, nil - case reflect.Map: - if rv.Type().Key().Kind() != reflect.String { - return reflect.Value{}, fmt.Errorf("%w: cannot select %q on map with non-string keys", ErrEvaluate, name) - } - mv := rv.MapIndex(mapStringKey(rv.Type().Key(), name)) - if !mv.IsValid() { - return reflect.Value{}, fmt.Errorf("%w: key %q not found%s", - ErrEvaluate, name, fieldHint(rv.Interface(), name, fieldTags)) - } - return mv, nil - } - return reflect.Value{}, fmt.Errorf("%w: cannot select %q on %v", ErrEvaluate, name, rv.Type()) -} - -func selectField(recv any, name string, fieldTags *structTagConfig) (any, error) { - // Translate rewritten identifiers (currently only `map`) back to - // their user-visible form so field/key/error lookups match what - // the user actually typed. - name = displayIdent(name) - if recv == nil { - return nil, fmt.Errorf("%w: cannot access %q on nil", ErrEvaluate, name) - } - if m, ok := recv.(map[string]any); ok { - v, ok := m[name] - if !ok { - return nil, fmt.Errorf("%w: key %q not found%s", - ErrEvaluate, name, fieldHint(recv, name, fieldTags)) - } - return v, nil - } - rv := reflect.ValueOf(recv) - switch rv.Kind() { - case reflect.Map: - if rv.Type().Key().Kind() != reflect.String { - return nil, fmt.Errorf("%w: cannot select %q on map with non-string keys", ErrEvaluate, name) - } - mv := rv.MapIndex(mapStringKey(rv.Type().Key(), name)) - if !mv.IsValid() { - return nil, fmt.Errorf("%w: key %q not found%s", - ErrEvaluate, name, fieldHint(recv, name, fieldTags)) - } - return mv.Interface(), nil - case reflect.Struct: - fv, ok, err := structFieldByName(rv, name, fieldTags) - if err != nil { - return nil, err - } - if !ok || !fv.IsValid() { - return nil, fmt.Errorf("%w: field %q not found on %T%s", - ErrEvaluate, name, recv, fieldHint(recv, name, fieldTags)) - } - if !fv.CanInterface() { - // Unexported field: deny access rather than panicking via Interface(). - return nil, fmt.Errorf("%w: field %q not found on %T%s", - ErrEvaluate, name, recv, fieldHint(recv, name, fieldTags)) +func (p *Program) evalIndex(ctx context.Context, n *ast.IndexExpr, env any, depth int) (any, error) { + // Fast path: filter(xs, p)[k] with a constant k streams the matches + // instead of materializing the full filtered slice. + if !p.noOptimize { + if v, ok, err := p.tryFilterIndex(ctx, n, env, depth); ok { + return v, err } - return fv.Interface(), nil - case reflect.Pointer: - if rv.IsNil() { - return nil, fmt.Errorf("%w: cannot access %q on nil pointer", ErrEvaluate, name) + if v, ok, err := p.evalIndexChainRV(ctx, n, env, depth); ok { + return v, err } - return selectField(rv.Elem().Interface(), name, fieldTags) - } - return nil, fmt.Errorf("%w: cannot select %q on %T", ErrEvaluate, name, recv) -} - -func (p *Program) evalIndex(ctx context.Context, n *ast.IndexExpr, env any, depth int) (any, error) { - // Fast path: filter(xs, p)[k] with a small constant k stops at the - // (k+1)-th match instead of materializing the full filtered slice. - if v, ok, err := p.tryFilterIndex(ctx, n, env, depth); ok { - return v, err - } - if v, ok, err := p.evalIndexChainRV(ctx, n, env, depth); ok { - return v, err } recv, err := p.eval(ctx, n.X, env, depth) if err != nil { @@ -1202,86 +1047,12 @@ walk: return rv.Interface(), true, nil } -// mapStringAnyType lets indexValueRV mirror indexValue's dedicated -// map[string]any branch, keeping error messages identical between the -// fast and general index paths. -var mapStringAnyType = reflect.TypeOf(map[string]any(nil)) - -// indexValueRV mirrors indexValue but keeps the receiver in reflect -// space so intermediate hops of a chain are never boxed through any. -func indexValueRV(rv reflect.Value, idx any) (reflect.Value, error) { - for rv.Kind() == reflect.Interface && !rv.IsNil() { - rv = rv.Elem() - } - if !rv.IsValid() || (rv.Kind() == reflect.Interface && rv.IsNil()) { - return reflect.Value{}, fmt.Errorf("%w: cannot index nil", ErrEvaluate) - } - if rv.Type() == mapStringAnyType { - key, ok := idx.(string) - if !ok { - return reflect.Value{}, fmt.Errorf("%w: map index must be string, got %T", ErrEvaluate, idx) - } - mv := rv.MapIndex(reflect.ValueOf(key)) - if !mv.IsValid() { - return reflect.Value{}, fmt.Errorf("%w: key %q not found%s", - ErrEvaluate, key, fieldHint(rv.Interface(), key, nil)) - } - return mv, nil - } - switch rv.Kind() { - case reflect.Slice, reflect.Array: - i, err := toIndexInt(idx) - if err != nil { - return reflect.Value{}, err - } - if i < 0 || i >= int64(rv.Len()) { - return reflect.Value{}, fmt.Errorf("%w: index %d out of range [0, %d)", ErrEvaluate, i, rv.Len()) - } - return rv.Index(int(i)), nil - case reflect.String: - i, err := toIndexInt(idx) - if err != nil { - return reflect.Value{}, err - } - runes := []rune(rv.String()) - if i < 0 || i >= int64(len(runes)) { - return reflect.Value{}, fmt.Errorf("%w: index %d out of range [0, %d)", ErrEvaluate, i, len(runes)) - } - return reflect.ValueOf(string(runes[i])), nil - case reflect.Map: - keyType := rv.Type().Key() - if idx == nil { - return reflect.Value{}, fmt.Errorf("%w: cannot use nil as map key %v", ErrEvaluate, keyType) - } - kv := reflect.ValueOf(idx) - if !kv.Type().AssignableTo(keyType) { - if isNumericKind(keyType.Kind()) && isNumericKind(kv.Kind()) { - converted, err := safeNumericConvert(kv, keyType) - if err != nil { - return reflect.Value{}, fmt.Errorf("%w: map key conversion: %v", ErrEvaluate, err) - } - kv = converted - } else if kv.Type().ConvertibleTo(keyType) { - kv = kv.Convert(keyType) - } else { - return reflect.Value{}, fmt.Errorf("%w: cannot use %T as map key %v", ErrEvaluate, idx, keyType) - } - } - mv := rv.MapIndex(kv) - if !mv.IsValid() { - return reflect.Value{}, fmt.Errorf("%w: key %v not found", ErrEvaluate, idx) - } - return mv, nil - } - return reflect.Value{}, fmt.Errorf("%w: cannot index %v", ErrEvaluate, rv.Type()) -} - // tryFilterIndex recognises `filter(xs, predicate)[N]` where N is a // non-negative integer literal and `filter` is the built-in special // form (not shadowed by env or funcs). When matched, it iterates xs -// and stops as soon as the N-th matching element is found, mirroring -// the result of the general path without allocating the intermediate -// slice. ok is true when the fast path was taken. +// and keeps the N-th matching element. The predicate still runs on every +// element, so errors and side effects match the general path; only the +// intermediate slice is skipped. ok is true when the fast path was taken. func (p *Program) tryFilterIndex(ctx context.Context, n *ast.IndexExpr, env any, depth int) (any, bool, error) { call, ok := n.X.(*ast.CallExpr) if !ok { @@ -1327,15 +1098,21 @@ func (p *Program) tryFilterIndex(ctx context.Context, n *ast.IndexExpr, env any, return nil, true, err } if coll == nil { - return nil, true, fmt.Errorf("%w: index %d out of range", ErrEvaluate, target) + // filter(nil, p) is an empty list. + return nil, true, fmt.Errorf("%w: index %d out of range [0, 0)", ErrEvaluate, target) } rv := reflect.ValueOf(coll) if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { return nil, true, fmt.Errorf("%w: filter expects a list as its first argument, got %T", ErrEvaluate, coll) } + // The predicate still runs on every element after the match, so the + // result and any predicate error (or side effect) are exactly those + // of the general path; only the intermediate slice is skipped. scope := &itEnv{parent: env, name: bindName} var seen int64 + var found bool + var result any for i := 0; i < rv.Len(); i++ { item := rv.Index(i).Interface() scope.it = item @@ -1348,79 +1125,15 @@ func (p *Program) tryFilterIndex(ctx context.Context, n *ast.IndexExpr, env any, continue } if seen == target { - return item, true, nil + found, result = true, item } seen++ } - return nil, true, fmt.Errorf("%w: index %d out of range", ErrEvaluate, target) -} - -func indexValue(recv, idx any) (any, error) { - if recv == nil { - return nil, fmt.Errorf("%w: cannot index nil", ErrEvaluate) - } - if m, ok := recv.(map[string]any); ok { - key, ok := idx.(string) - if !ok { - return nil, fmt.Errorf("%w: map index must be string, got %T", ErrEvaluate, idx) - } - v, ok := m[key] - if !ok { - return nil, fmt.Errorf("%w: key %q not found%s", - ErrEvaluate, key, fieldHint(recv, key, nil)) - } - return v, nil - } - rv := reflect.ValueOf(recv) - switch rv.Kind() { - case reflect.Slice, reflect.Array: - i, err := toIndexInt(idx) - if err != nil { - return nil, err - } - if i < 0 || i >= int64(rv.Len()) { - return nil, fmt.Errorf("%w: index %d out of range [0, %d)", ErrEvaluate, i, rv.Len()) - } - return rv.Index(int(i)).Interface(), nil - case reflect.String: - i, err := toIndexInt(idx) - if err != nil { - return nil, err - } - runes := []rune(rv.String()) - if i < 0 || i >= int64(len(runes)) { - return nil, fmt.Errorf("%w: index %d out of range [0, %d)", ErrEvaluate, i, len(runes)) - } - return string(runes[i]), nil - case reflect.Map: - keyType := rv.Type().Key() - if idx == nil { - // Untyped nil is only a valid key for nilable key kinds; rather - // than handling every combination via reflect we reject it - // — reflect.ValueOf(nil).Type() would otherwise panic. - return nil, fmt.Errorf("%w: cannot use nil as map key %v", ErrEvaluate, keyType) - } - kv := reflect.ValueOf(idx) - if !kv.Type().AssignableTo(keyType) { - if isNumericKind(keyType.Kind()) && isNumericKind(kv.Kind()) { - converted, err := safeNumericConvert(kv, keyType) - if err != nil { - return nil, fmt.Errorf("%w: map key conversion: %v", ErrEvaluate, err) - } - kv = converted - } else if kv.Type().ConvertibleTo(keyType) { - kv = kv.Convert(keyType) - } else { - return nil, fmt.Errorf("%w: cannot use %T as map key %v", ErrEvaluate, idx, keyType) - } - } - mv := rv.MapIndex(kv) - if !mv.IsValid() { - return nil, fmt.Errorf("%w: key %v not found", ErrEvaluate, idx) - } - return mv.Interface(), nil + if found { + return result, true, nil } - return nil, fmt.Errorf("%w: cannot index %T", ErrEvaluate, recv) + // seen is the length the filtered list would have had. + return nil, true, fmt.Errorf("%w: index %d out of range [0, %d)", ErrEvaluate, target, seen) } func (p *Program) evalCall(ctx context.Context, n *ast.CallExpr, env any, depth int) (any, error) { @@ -1594,6 +1307,6 @@ func resolveMethod(recv any, name string, fieldTags *structTagConfig) (any, erro func methodCallable(name string, mv reflect.Value) Func { return func(ctx context.Context, args []any) (any, error) { - return callMethodValue(ctx, name, mv, args) + return callFuncValue(ctx, name, mv, args) } } diff --git a/reflect.go b/reflect.go index 00c0249..5b88e9a 100644 --- a/reflect.go +++ b/reflect.go @@ -2,6 +2,7 @@ package expr import ( "context" + "errors" "fmt" "math" "reflect" @@ -23,27 +24,38 @@ var ( // and is not counted against the caller-supplied argument list. func callFunction(ctx context.Context, name string, fn any, args []any) (any, error) { if nf, ok := fn.(Func); ok { - return nf(ctx, args) + if nf == nil { + return nil, fmt.Errorf("%w: %q is a nil function value", ErrEvaluate, name) + } + v, err := nf(ctx, args) + return v, wrapCallErr(name, err) } - fv := reflect.ValueOf(fn) + return callFuncValue(ctx, name, reflect.ValueOf(fn), args) +} + +// callFuncValue invokes fv, an env function, struct field or bound +// method, through the reflect path registered functions use. +func callFuncValue(ctx context.Context, name string, fv reflect.Value, args []any) (any, error) { if fv.Kind() != reflect.Func { - return nil, fmt.Errorf("%w: %q is not a function (got %T)", ErrEvaluate, name, fn) + return nil, fmt.Errorf("%w: %q is not a function (got %v)", ErrEvaluate, name, describeValue(fv)) } if fv.IsNil() { return nil, fmt.Errorf("%w: %q is a nil function value", ErrEvaluate, name) } - ft := fv.Type() - - in, err := buildCallArgs(ctx, name, ft, args) + sig, err := sigOf(fv.Type()) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %q %v", ErrEvaluate, name, err) } + return invokeReflect(ctx, name, fv, sig, args) +} - out, err := callReflectRecoverRuntime(name, fv, in) - if err != nil { - return nil, err +// describeValue names rv's type for an error message: its Go type, or +// nil for an invalid Value. +func describeValue(rv reflect.Value) string { + if !rv.IsValid() { + return "nil" } - return finishCall(name, ft, out) + return rv.Type().String() } // callReflectRecoverRuntime invokes fv and converts runtime.Error @@ -69,95 +81,32 @@ func callReflectRecoverRuntime(name string, fv reflect.Value, in []reflect.Value return out, nil } -func callMethodValue(ctx context.Context, name string, fv reflect.Value, args []any) (any, error) { - if fv.Kind() != reflect.Func { - return nil, fmt.Errorf("%w: %q is not a function (got %v)", ErrEvaluate, name, fv.Kind()) - } - if fv.IsNil() { - return nil, fmt.Errorf("%w: %q is a nil function value", ErrEvaluate, name) - } - ft := fv.Type() - in, err := buildCallArgs(ctx, name, ft, args) - if err != nil { - return nil, err - } - out, err := callReflectRecoverRuntime(name, fv, in) - if err != nil { - return nil, err - } - return finishCall(name, ft, out) +// callResultErr extracts the error a function returned in v, whose type +// implements error. A nil interface or nil pointer-like value is no +// error; a value of a non-nilable kind (a struct implementing error) is +// always one, as it is in Go once boxed in an error interface. +func callResultErr(v reflect.Value) error { + switch v.Kind() { + case reflect.Interface, reflect.Pointer, reflect.Map, reflect.Slice, reflect.Chan, reflect.Func: + if v.IsNil() { + return nil + } + } + err, _ := v.Interface().(error) + return err } -func finishCall(name string, ft reflect.Type, out []reflect.Value) (any, error) { - switch ft.NumOut() { - case 0: - return nil, nil - case 1: - return out[0].Interface(), nil - case 2: - if !ft.Out(1).Implements(errValType) { - return nil, fmt.Errorf("%w: %q: second return must be error, got %v", ErrEvaluate, name, ft.Out(1)) - } - if !out[1].IsNil() { - return nil, out[1].Interface().(error) - } - return out[0].Interface(), nil - } - return nil, fmt.Errorf("%w: %q returns %d values (expected 0, 1, or (T, error))", ErrEvaluate, name, ft.NumOut()) -} - -func buildCallArgs(ctx context.Context, name string, ft reflect.Type, args []any) ([]reflect.Value, error) { - // Detect context.Context as the first declared parameter. If present, - // ctx is injected automatically and the remaining parameters are - // matched against the caller-supplied args. - injectCtx := ft.NumIn() > 0 && ft.In(0) == ctxType - paramOffset := 0 - if injectCtx { - paramOffset = 1 - } - declaredIn := ft.NumIn() - paramOffset - - if ft.IsVariadic() { - fixed := declaredIn - 1 - if len(args) < fixed { - return nil, fmt.Errorf("%w: %q expects at least %d args, got %d", ErrEvaluate, name, fixed, len(args)) - } - in := make([]reflect.Value, paramOffset+len(args)) - if injectCtx { - in[0] = reflect.ValueOf(ctx) - } - for i := 0; i < fixed; i++ { - v, err := convertArg(name, i, args[i], ft.In(paramOffset+i)) - if err != nil { - return nil, err - } - in[paramOffset+i] = v - } - variadicElem := ft.In(paramOffset + fixed).Elem() - for i := fixed; i < len(args); i++ { - v, err := convertArg(name, i, args[i], variadicElem) - if err != nil { - return nil, err - } - in[paramOffset+i] = v - } - return in, nil - } - if len(args) != declaredIn { - return nil, fmt.Errorf("%w: %q expects %d args, got %d", ErrEvaluate, name, declaredIn, len(args)) - } - in := make([]reflect.Value, paramOffset+len(args)) - if injectCtx { - in[0] = reflect.ValueOf(ctx) - } - for i, a := range args { - v, err := convertArg(name, i, a, ft.In(paramOffset+i)) - if err != nil { - return nil, err - } - in[paramOffset+i] = v - } - return in, nil +// wrapCallErr wraps an error returned by a host function in ErrEvaluate, +// keeping the original cause reachable through errors.Is / errors.As. +// Errors that already wrap ErrEvaluate (builtins, nested Runs) pass +// through unchanged, as do context cancellation errors, which try must +// not swallow. +func wrapCallErr(name string, err error) error { + if err == nil || errors.Is(err, ErrEvaluate) || + errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return fmt.Errorf("%w: %q: %w", ErrEvaluate, name, err) } func convertArg(name string, idx int, value any, want reflect.Type) (reflect.Value, error) { @@ -188,12 +137,81 @@ func convertArg(name string, idx int, value any, want reflect.Type) (reflect.Val if want.Kind() == reflect.Interface && rv.Type().Implements(want) { return rv, nil } - if rv.Type().ConvertibleTo(want) { - return rv.Convert(want), nil + if converted, ok := convertValue(rv, want); ok { + return converted, nil } return reflect.Value{}, fmt.Errorf("%w: %q arg %d: cannot use %T as %v", ErrEvaluate, name, idx, value, want) } +// convertValue converts rv to want when Go allows it for this value. +// reflect.Type.ConvertibleTo only checks types: converting a slice to an +// array (or array pointer) also needs the slice to be long enough, and +// reflect panics otherwise. +func convertValue(rv reflect.Value, want reflect.Type) (reflect.Value, bool) { + if !rv.Type().ConvertibleTo(want) { + return reflect.Value{}, false + } + if rv.Kind() == reflect.Slice { + switch { + case want.Kind() == reflect.Array: + if rv.Len() < want.Len() { + return reflect.Value{}, false + } + case want.Kind() == reflect.Pointer && want.Elem().Kind() == reflect.Array: + if rv.Len() < want.Elem().Len() { + return reflect.Value{}, false + } + } + } + return rv.Convert(want), true +} + +// mapKey converts idx to a key usable with a map of keyType. Beyond the +// type conversion, the key must be hashable at run time: an interface +// key type accepts any dynamic value, and hashing a slice (or a struct +// or array holding one) panics. +func mapKey(keyType reflect.Type, idx any) (reflect.Value, error) { + if idx == nil { + // Untyped nil is only a valid key for nilable key kinds; rather + // than handling every combination via reflect we reject it. + return reflect.Value{}, fmt.Errorf("%w: cannot use nil as map key %v", ErrEvaluate, keyType) + } + kv := reflect.ValueOf(idx) + if !kv.Type().AssignableTo(keyType) { + if isNumericKind(keyType.Kind()) && isNumericKind(kv.Kind()) { + // An integer key follows the index rule (xs[1.5] fails): + // a float must be integral, not truncated. + if isIntegerKind(keyType.Kind()) && (kv.Kind() == reflect.Float32 || kv.Kind() == reflect.Float64) { + if f := kv.Float(); f != math.Trunc(f) { + return reflect.Value{}, fmt.Errorf("%w: map key must be integer, got %v", ErrEvaluate, f) + } + } + converted, err := safeNumericConvert(kv, keyType) + if err != nil { + return reflect.Value{}, fmt.Errorf("%w: map key conversion: %v", ErrEvaluate, err) + } + kv = converted + } else if converted, ok := convertValue(kv, keyType); ok { + kv = converted + } else { + return reflect.Value{}, fmt.Errorf("%w: cannot use %T as map key %v", ErrEvaluate, idx, keyType) + } + } + if !canCompare(kv) { + return reflect.Value{}, fmt.Errorf("%w: cannot use unhashable %T as map key", ErrEvaluate, idx) + } + return kv, nil +} + +// canCompare reports whether Go's == (and map hashing) accepts v without +// panicking. Both checks are needed: Type.Comparable misses an +// interface field holding a slice, and Value.Comparable reports a +// zero-length array of uncomparable elements ([0]map[K]V) comparable +// although == on it panics. +func canCompare(v reflect.Value) bool { + return v.IsValid() && v.Type().Comparable() && v.Comparable() +} + func isNumericKind(k reflect.Kind) bool { switch k { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, @@ -491,6 +509,11 @@ func looseEqual(lhs, rhs any) (bool, bool) { if rhs == nil { return isNilValue(lhs), true } + // Two integers compare exactly, whatever their widths and + // signedness; float64 would round above 2^53. + if c, ok := compareIntegers(lhs, rhs); ok { + return c == 0, true + } if lf, lok := toFloat64(lhs); lok { if rf, rok := toFloat64(rhs); rok { return lf == rf, true @@ -506,20 +529,76 @@ func looseEqual(lhs, rhs any) (bool, bool) { return lb == rb, true } } - lt := reflect.TypeOf(lhs) - if !lt.Comparable() { + // Value.Comparable, unlike Type.Comparable, also looks inside + // interface fields and elements: a struct or array holding a slice + // has a comparable type but panics when compared. + lv, rv := reflect.ValueOf(lhs), reflect.ValueOf(rhs) + if !canCompare(lv) || !canCompare(rv) { return false, false } - rt := reflect.TypeOf(rhs) - if !rt.Comparable() { - return false, false - } - if lt != rt { + if lv.Type() != rv.Type() { return false, true } return lhs == rhs, true } +// asInteger reads v as an integer of any Go integer kind (including +// named types). A value above math.MaxInt64 comes back in u with big +// set; otherwise it is in i. +func asInteger(v any) (i int64, u uint64, big, ok bool) { + switch n := v.(type) { + case int64: + return n, 0, false, true + case int: + return int64(n), 0, false, true + case nil: + return 0, 0, false, false + } + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int(), 0, false, true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + if u := rv.Uint(); u > math.MaxInt64 { + return 0, u, true, true + } + return int64(rv.Uint()), 0, false, true + } + return 0, 0, false, false +} + +// compareIntegers compares two integers exactly, returning -1, 0 or +1. +// ok is false unless both are of an integer kind. +func compareIntegers(lhs, rhs any) (int, bool) { + li, lu, lbig, lok := asInteger(lhs) + if !lok { + return 0, false + } + ri, ru, rbig, rok := asInteger(rhs) + if !rok { + return 0, false + } + switch { + case lbig && rbig: + return cmpOrdered(lu, ru), true + case lbig: + return 1, true // above every int64 + case rbig: + return -1, true + } + return cmpOrdered(li, ri), true +} + +func cmpOrdered[T int64 | uint64](a, b T) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + } + return 0 +} + func asBool(v any) (bool, bool) { if b, ok := v.(bool); ok { return b, true diff --git a/runtime_regression_test.go b/runtime_regression_test.go new file mode 100644 index 0000000..b46ba30 --- /dev/null +++ b/runtime_regression_test.go @@ -0,0 +1,290 @@ +package expr + +import ( + "context" + "errors" + "testing" +) + +// Keep a runtime panic local to its subtest so every independent regression +// remains visible in one test run. +func runtimeRegressionNoPanic(t *testing.T) { + t.Helper() + if recovered := recover(); recovered != nil { + t.Errorf("evaluation panicked instead of returning a result or error: %v", recovered) + } +} + +func TestRuntimeRegressionMapKeyComparability(t *testing.T) { + for _, source := range []string{"m[[1]]", "root.m[[1]]", "m?[[1]]"} { + t.Run(source, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + m := map[any]any{} + p, err := Compile(source) + if err != nil { + t.Fatal(err) + } + _, err = p.Run(t.Context(), map[string]any{ + "m": m, "root": map[string]any{"m": m}, + }) + if !errors.Is(err, ErrEvaluate) { + t.Fatalf("unhashable map key: got %v, want ErrEvaluate", err) + } + }) + } +} + +func TestRuntimeRegressionEqualityComparability(t *testing.T) { + for _, value := range []struct { + name string + x any + }{ + {"struct", struct{ V any }{[]int{1}}}, + {"array", [1]any{[]int{1}}}, + } { + for _, source := range []string{"x == x", "x != x", "contains([x], x)"} { + t.Run(value.name+"/"+source, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + p, err := Compile(source, WithBuiltins()) + if err != nil { + t.Fatal(err) + } + got, err := p.Run(t.Context(), map[string]any{"x": value.x}) + if source == "contains([x], x)" { + // contains ignores elements that cannot be compared. + if err != nil || got != false { + t.Fatalf("contains incomparable value: got (%v, %v), want (false, nil)", got, err) + } + } else if !errors.Is(err, ErrEvaluate) { + t.Fatalf("incomparable equality: got %v, want ErrEvaluate", err) + } + }) + } + } +} + +func TestRuntimeRegressionValueDependentConversion(t *testing.T) { + for _, registered := range []bool{true, false} { + name := "env" + if registered { + name = "registered" + } + t.Run(name, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + called := false + fn := func([2]int) int { called = true; return 0 } + env := map[string]any{"xs": []int{1}} + var opts []Option + if registered { + opts = append(opts, WithFunctions(map[string]any{"f": fn})) + } else { + env["f"] = fn + } + p, err := Compile("f(xs)", opts...) + if err != nil { + t.Fatal(err) + } + _, err = p.Run(t.Context(), env) + if !errors.Is(err, ErrEvaluate) { + t.Fatalf("short slice conversion: got %v, want ErrEvaluate", err) + } + if called { + t.Fatal("function called with an invalid argument") + } + }) + } + for _, source := range []string{"m[xs]", "root.m[xs]", "m?[xs]"} { + t.Run(source, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + m := map[[2]int]string{} + p, err := Compile(source) + if err != nil { + t.Fatal(err) + } + _, err = p.Run(t.Context(), map[string]any{ + "m": m, "root": map[string]any{"m": m}, "xs": []int{1}, + }) + if !errors.Is(err, ErrEvaluate) { + t.Fatalf("short slice map key conversion: got %v, want ErrEvaluate", err) + } + }) + } +} + +type runtimeRegressionConcreteError struct{} + +func (runtimeRegressionConcreteError) Error() string { return "concrete error" } + +func TestRuntimeRegressionConcreteErrorReturn(t *testing.T) { + for _, registered := range []bool{true, false} { + name := "env" + if registered { + name = "registered" + } + t.Run(name, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + called := false + fn := func() (int, runtimeRegressionConcreteError) { + called = true + return 0, runtimeRegressionConcreteError{} + } + env := map[string]any{} + var opts []Option + if registered { + opts = append(opts, WithFunctions(map[string]any{"f": fn})) + } else { + env["f"] = fn + } + p, err := Compile("f()", opts...) + if err != nil { + if !registered || !errors.Is(err, ErrCompile) { + t.Fatalf("unexpected compilation failure: %v", err) + } + return // Rejecting this signature at registration is also safe. + } + _, err = p.Run(t.Context(), env) + if !errors.Is(err, ErrEvaluate) { + t.Fatalf("got %v, want ErrEvaluate for signature rejection or function error", err) + } + if called && !errors.Is(err, runtimeRegressionConcreteError{}) { + t.Fatalf("function error lost its original cause: %v", err) + } + }) + } +} + +func TestRuntimeRegressionNilNativeFunction(t *testing.T) { + for _, registered := range []bool{true, false} { + name := "env" + if registered { + name = "registered" + } + t.Run(name, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + env := map[string]any{} + var opts []Option + if registered { + opts = append(opts, WithFunctions(map[string]any{"f": Func(nil)})) + } else { + env["f"] = Func(nil) + } + p, err := Compile("f()", opts...) + if registered { + if !errors.Is(err, ErrCompile) { + t.Fatalf("nil native function registration: got %v, want ErrCompile", err) + } + return + } + if err != nil { + t.Fatal(err) + } + _, err = p.Run(t.Context(), env) + if !errors.Is(err, ErrEvaluate) { + t.Fatalf("nil native function: got %v, want ErrEvaluate", err) + } + }) + } +} + +func TestRuntimeRegressionFilterIndexEvaluation(t *testing.T) { + for _, source := range []string{ + "filter([1, 0], 1 / it)[0]", + "filter([1, 0], x, 1 / x)[0]", + "filter([1, 0], 1 / it)[i]", + } { + t.Run(source, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + p, err := Compile(source) + if err != nil { + t.Fatal(err) + } + _, err = p.Run(t.Context(), map[string]any{"i": 0}) + if !errors.Is(err, ErrEvaluate) { + t.Fatalf("filter must evaluate its later failing predicate before indexing: got %v", err) + } + }) + } + t.Run("predicate calls", func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + calls := 0 + p, err := Compile("filter([1, 2, 3], predicate(it))[0]", WithFunctions(map[string]any{ + "predicate": Func(func(context.Context, []any) (any, error) { + calls++ + return true, nil + }), + })) + if err != nil { + t.Fatal(err) + } + got, err := p.Run(t.Context(), nil) + if err != nil || got != int64(1) { + t.Fatalf("got (%v, %v), want (1, nil)", got, err) + } + if calls != 3 { + t.Fatalf("predicate called %d times, want 3", calls) + } + }) +} + +func TestRuntimeRegressionSumMixedNumericPromotion(t *testing.T) { + for _, source := range []string{ + "sum([9223372036854775807, 1, 0.0])", + "sum([0.0, 9223372036854775807, 1])", + } { + t.Run(source, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + p, err := Compile(source, WithFunctions(CollectionFuncs())) + if err != nil { + t.Fatal(err) + } + got, err := p.Run(t.Context(), nil) + if err != nil || got != float64(uint64(1)<<63) { + t.Fatalf("mixed sum: got (%v, %v), want (float64(2^63), nil)", got, err) + } + }) + } +} + +// Found by TestHostValuesRandom (seed 90): reflect.Value.Comparable +// reports a zero-length array whose elements are arrays (or structs, or +// interfaces) of uncomparable values comparable, since it checks each of +// no elements, but Go's == on it panics, so equality and map keys check +// the type too. +func TestRuntimeRegressionZeroLengthUncomparableArray(t *testing.T) { + v := [0][1]map[string]int{} + for _, source := range []string{"V == V", "V != V", "contains([V], V)", "m[V]", "m?[V]"} { + t.Run(source, func(t *testing.T) { + defer runtimeRegressionNoPanic(t) + p, err := Compile(source, WithBuiltins()) + if err != nil { + t.Fatal(err) + } + got, err := p.Run(t.Context(), map[string]any{"V": v, "m": map[any]any{}}) + if source == "contains([V], V)" { + if err != nil || got != false { + t.Fatalf("got (%v, %v), want (false, nil)", got, err) + } + return + } + if !errors.Is(err, ErrEvaluate) { + t.Fatalf("got (%v, %v), want ErrEvaluate", got, err) + } + }) + } +} + +// Found by TestHostValuesRandom (seed 2623): map keys that render alike +// (int 1 and uint16 1 in a map[any]V) sorted by rendered text alone, so +// their order followed map iteration and string(m) varied between runs. +func TestRuntimeRegressionFormatMapKeysRenderingAlike(t *testing.T) { + m := map[any]any{1: "int", uint16(1): "uint16", int64(1): "int64"} + want := safeFormatValue(m) + for i := 0; i < 50; i++ { + if got := safeFormatValue(m); got != want { + t.Fatalf("formatting varies: %q then %q", want, got) + } + } + if want != "map[1:int 1:int64 1:uint16]" { + t.Fatalf("got %q, want keys ordered by type name on a tie", want) + } +} diff --git a/suggest.go b/suggest.go index 152b99f..e847d6d 100644 --- a/suggest.go +++ b/suggest.go @@ -51,7 +51,11 @@ func formatHint(name string, candidates []string) string { if len(candidates) == 0 { return "" } + // Sort first: candidates often come from map iteration, and + // closestName keeps the first of equally close names, so the hint + // must not depend on that order. candidates = dedup(candidates) + sort.Strings(candidates) if closest, ok := closestName(name, candidates); ok { return fmt.Sprintf(" (did you mean %q?)", closest) } @@ -62,7 +66,6 @@ func formatHint(name string, candidates []string) string { if len(candidates) > maxList { return "" } - sort.Strings(candidates) return fmt.Sprintf(" (available: %s)", strings.Join(candidates, ", ")) } diff --git a/syntax_matrix_test.go b/syntax_matrix_test.go new file mode 100644 index 0000000..9b52137 --- /dev/null +++ b/syntax_matrix_test.go @@ -0,0 +1,348 @@ +package expr + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// The syntax matrix checks that expr's source rewrites (keyword fields, +// optional access, JSON literals, the map/if keywords, pipes and template +// delimiters) compose. Every receiver is crossed with every access +// operator, every surrounding context and every comment placement, and +// each expression must behave exactly like its explicitly parenthesized +// equivalent: the same value, or the same error category, or a compile +// error in both forms. + +// matrixReceiver is a primary expression. obj receivers hold the object +// {"a": {"b": 2}, "type": 3, "map": 4, "n": nil}; list receivers hold +// [1, 2, 3] (or the string "abc"). +type matrixReceiver struct { + src string + list bool +} + +var matrixReceivers = []matrixReceiver{ + {src: `x`}, + {src: `x.self`}, + {src: `x["self"]`}, + {src: `{"a": {"b": 2}, "type": 3, "map": 4, "n": nil}`}, + {src: `map[string]any{"a": {"b": 2}, "type": 3, "map": 4}`}, + {src: `id(x)`}, + {src: `if(true, x, nil)`}, + {src: `map([x], it)[0]`}, + {src: `[x][0]`}, + {src: `x?.self`}, + {src: `xs`, list: true}, + {src: `[1, 2, 3]`, list: true}, + {src: `[]any{1, 2, 3}`, list: true}, + {src: `[1, 2, 3][0:0]`, list: true}, // rejected slice syntax + {src: `map(xs, it)`, list: true}, + {src: `if(true, xs, nil)`, list: true}, + {src: `id([1, 2, 3])`, list: true}, + {src: `"abc"`, list: true}, + {src: `{"self": x}.self`}, + {src: `{"self": x}?.self`}, + {src: `[{"a": {"b": 2}, "type": 3}][0]`}, + {src: `[]any{x}[0]`}, + {src: `[]any{x}?[0]`}, + {src: `[x]?[0]`}, + {src: `[1]`, list: true}, + {src: `[6]`, list: true}, + {src: `[xs][0]`, list: true}, + {src: `[[1, 2, 3]][0]`, list: true}, + {src: `[[1, 2, 3]]?[0]`, list: true}, + {src: `[]any{[]any{1}}[0]`, list: true}, + {src: `[2]any{1, 2}`, list: true}, + {src: `{"k": [1, 2]}.k`, list: true}, + {src: `{"k": [1, 2]}?.k`, list: true}, + {src: `{"k": [1, 2]}["k"]`, list: true}, + {src: `n`}, + {src: `n?.self`}, +} + +var matrixObjectOps = []string{ + `.a.b`, `?.a.b`, `?.a?.b`, `.a?.b`, `["a"]["b"]`, `?["a"]?["b"]`, + `["a"]?.b`, `?["a"].b`, `.type`, `?.type`, `.map`, `?.map`, `.if`, + `?.missing`, `?.n?.b`, `.self.type`, `?.self?.map`, +} + +var matrixListOps = []string{ + `[0]`, `?[0]`, `[1 + 1]`, `?[5]`, `[len("ab")]`, `?[0]?.b`, `[-1]`, +} + +// matrixContexts surround a primary; %s is the primary. +var matrixContexts = []string{ + `%s`, + `%s * 2`, + `2 * %s`, + `%s - 1`, + `1 - %s`, + `-%s`, + `!%s`, + `%s == %s`, + `%s * %s`, + `%s * -%s`, + `[%s, %s]`, + `{"k": %s, "j": [%s]}`, + `id(%s) * 2`, + `%s | id() == 2`, + `%s | id()`, + `id(%s)`, + `id(1, %s)`, + `[%s]`, + `[1, %s]`, + `{"k": %s}`, + `map([1], %s)`, + `filter([1], %s != nil)`, + `if(%s == nil, 0, 1)`, + `[%s][0]`, + `{"k": %s}.k`, + `{"k": %s}?.k`, +} + +// matrixCommentContexts are the contexts crossed with every comment +// placement. +var matrixCommentContexts = []string{`%s`, `%s * 2`, `-%s`, `%s | id()`, `[1, %s]`, `{"k": %s}?.k`} + +// matrixComments are placements for comments inside a primary: before +// the access operator, after it (where no semicolon is inserted), and +// around the whole primary. +var matrixComments = []func(recv, op string) string{ + func(recv, op string) string { return recv + op }, + func(recv, op string) string { return recv + " /* c */ " + op }, + func(recv, op string) string { return "/* c */ " + recv + op + " /* c */" }, + func(recv, op string) string { return recv + insertAfterOperator(op, " /* c */ ") }, + func(recv, op string) string { return recv + insertAfterOperator(op, " // c\n ") }, +} + +// insertAfterOperator places c after the operator that starts op (`.`, +// `?.`, `[`, `?[`), where Go inserts no semicolon at a newline. +func insertAfterOperator(op, c string) string { + for _, p := range []string{"?.", "?[", ".", "["} { + if strings.HasPrefix(op, p) { + return p + c + op[len(p):] + } + } + return op +} + +func matrixEnv() map[string]any { + x := map[string]any{"a": map[string]any{"b": int64(2)}, "type": int64(3), "map": int64(4), "if": int64(5), "n": nil} + x["self"] = map[string]any{"a": map[string]any{"b": int64(2)}, "type": int64(3), "map": int64(4)} + return map[string]any{ + "x": x, + "xs": []any{int64(1), int64(2), int64(3)}, + "n": nil, + } +} + +var matrixOpts = []Option{WithFunctions(map[string]any{ + "id": func(args ...any) any { return args[len(args)-1] }, + "len": func(s string) int { + return len(s) + }, +})} + +type matrixOutcome struct { + compileErr bool + val any + errKind string +} + +func (o matrixOutcome) String() string { + switch { + case o.compileErr: + return "compile error" + case o.errKind != "": + return o.errKind + } + return fmt.Sprintf("%#v", o.val) +} + +func runMatrix(src string, env any) (matrixOutcome, error) { + p, err := Compile(src, matrixOpts...) + if err != nil { + if !errors.Is(err, ErrCompile) { + return matrixOutcome{errKind: "unwrapped compile error"}, err + } + return matrixOutcome{compileErr: true}, err + } + v, err := p.Run(context.Background(), env) + if err != nil { + if errors.Is(err, ErrEvaluate) { + return matrixOutcome{errKind: "evaluate error"}, err + } + return matrixOutcome{errKind: "other error"}, err + } + return matrixOutcome{val: v}, nil +} + +func sameSyntaxOutcome(a, b matrixOutcome) bool { + if a.compileErr || b.compileErr { + return a.compileErr == b.compileErr + } + if a.errKind != b.errKind { + return false + } + return reflect.DeepEqual(a.val, b.val) +} + +// matrixCases yields each expression with its parenthesized forms. +func matrixCases(yield func(name, src string, alts []string) bool) { + for _, r := range matrixReceivers { + ops := matrixObjectOps + if r.list { + ops = matrixListOps + } + for _, op := range ops { + for ci, comment := range matrixComments { + primary := comment(r.src, op) + parenRecv := comment("("+r.src+")", op) + contexts := matrixContexts + if ci > 0 { + // Comments are a lexical concern: a few contexts cover them. + contexts = matrixCommentContexts + } + for _, c := range contexts { + fill := func(p string) string { return strings.ReplaceAll(c, "%s", p) } + name := fmt.Sprintf("%s|%s|c%d|%s", r.src, op, ci, c) + if !yield(name, fill(primary), []string{fill(parenRecv), fill("(" + primary + ")")}) { + return + } + } + } + } + } +} + +func TestSyntaxMatrix(t *testing.T) { + env := matrixEnv() + var failures, total int + compiled := map[bool]int{} + matrixCases(func(name, src string, alts []string) bool { + total++ + want, werr := runMatrix(src, env) + compiled[!want.compileErr]++ + for _, alt := range alts { + got, gerr := runMatrix(alt, env) + if !sameSyntaxOutcome(want, got) { + failures++ + if failures <= 40 { + t.Errorf("%s\n %q -> %v (%v)\n %q -> %v (%v)", name, src, want, werr, alt, got, gerr) + } + } + } + return true + }) + if failures > 0 { + t.Errorf("%d of %d matrix cells disagree with their parenthesized form", failures, total) + } + // Guard against the matrix degenerating into compile errors. + if compiled[true] < total/2 { + t.Errorf("only %d of %d matrix expressions compile", compiled[true], total) + } + t.Logf("%d cells, %d compile", total, compiled[true]) +} + +// Templates must render each expression the same way whatever the +// delimiters, and the same as the parenthesized expression. +func TestSyntaxMatrixTemplates(t *testing.T) { + env := matrixEnv() + delims := [][2]string{{"${", "}"}, {"{{", "}}"}, {"${{", "}}"}, {"@{{{", "}}}"}} + var failures int + contexts := matrixContexts[:1] + for _, r := range matrixReceivers { + ops := matrixObjectOps + if r.list { + ops = matrixListOps + } + for _, op := range ops { + for ci, comment := range matrixComments { + for _, c := range contexts { + e := strings.ReplaceAll(c, "%s", comment(r.src, op)) + type out struct { + s string + err bool + } + var outs []out + var srcs []string + for _, d := range delims { + for _, body := range []string{e, "(" + e + ")", " " + e + " "} { + raw := "<" + d[0] + body + d[1] + ">" + var opts []Option + opts = append(opts, matrixOpts...) + if d[0] != "${" { + opts = append(opts, WithTemplateDelimiters(d[0], d[1])) + } + tmpl, err := NewTemplate(raw, opts...) + if err != nil { + outs = append(outs, out{err: true}) + srcs = append(srcs, raw+" (compile: "+err.Error()+")") + continue + } + s, err := tmpl.Render(context.Background(), env) + outs = append(outs, out{s: s, err: err != nil}) + srcs = append(srcs, raw) + } + } + for i := 1; i < len(outs); i++ { + if outs[i] != outs[0] { + failures++ + if failures <= 20 { + t.Errorf("c%d %q: %q -> %+v, but %q -> %+v", ci, e, srcs[0], outs[0], srcs[i], outs[i]) + } + break + } + } + } + } + } + } + if failures > 0 { + t.Errorf("%d template cells disagree", failures) + } +} + +// Compile errors must point at the source the user wrote, however many +// bytes the rewrites inserted before the error. Each matrix primary is +// followed by an operator expr rejects at validation (`^`) and a byte the +// scanner rejects (`$`); the error must name that token's line:column. +func TestSyntaxMatrixErrorPositions(t *testing.T) { + var failures int + for _, r := range matrixReceivers { + if _, err := Compile(r.src, matrixOpts...); err != nil { + continue // the receiver reports its own error first + } + ops := matrixObjectOps + if r.list { + ops = matrixListOps + } + for _, op := range ops { + for ci, comment := range matrixComments { + primary := comment(r.src, op) + for _, bad := range []string{" ^ 1", " $"} { + for _, src := range []string{primary + bad, "[" + primary + bad + "]", "id(" + primary + "," + bad + ")"} { + at := strings.LastIndexAny(src, "^$") + line := 1 + strings.Count(src[:at], "\n") + col := at - strings.LastIndex(src[:at], "\n") + want := fmt.Sprintf("%d:%d: ", line, col) + _, err := Compile(src, matrixOpts...) + if err == nil || !strings.Contains(err.Error(), want) { + failures++ + if failures <= 20 { + t.Errorf("c%d %q: error %v, want position %s", ci, src, err, want) + } + } + } + } + } + } + } + if failures > 0 { + t.Errorf("%d misplaced error positions", failures) + } +} diff --git a/syntax_regression_test.go b/syntax_regression_test.go new file mode 100644 index 0000000..1682a2b --- /dev/null +++ b/syntax_regression_test.go @@ -0,0 +1,144 @@ +package expr + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/deepnoodle-ai/expr/internal/require" +) + +// A multiplication operator after literal indexing must not make the +// JSON-literal rewrite interpret the receiver as a Go array type. +func TestSyntaxRegression_LiteralIndexMultiplication(t *testing.T) { + for _, source := range []string{ + `[6][0] * 2`, + `[6][0] * factor`, + `[[6]][0][0] * 2`, + } { + t.Run(source, func(t *testing.T) { + got, err := evalExpr(t.Context(), source, map[string]any{"factor": int64(2)}) + require.NoError(t, err) + require.Equal(t, int64(12), got) + }) + } +} + +// Braces belonging to an empty object count as expression content; +// only the template's closing delimiter is outside the expression. +func TestSyntaxRegression_TemplateEmptyObject(t *testing.T) { + for _, tc := range []struct { + name string + source string + opts []Option + }{ + {name: "default", source: `${{}}`}, + {name: "comment", source: `${ { /* empty object */ } }`}, + {name: "custom_delimiters", source: `${{ {} }}`, opts: []Option{WithTemplateDelimiters("${{", "}}")}}, + } { + t.Run(tc.name, func(t *testing.T) { + tmpl, err := NewTemplate(tc.source, tc.opts...) + require.NoError(t, err) + got, err := tmpl.Render(t.Context(), nil) + require.NoError(t, err) + require.Equal(t, "{}", got) + }) + } +} + +// Supported typed literals are primary expressions too, so optional +// access must work without adding parentheses around the receiver. +func TestSyntaxRegression_OptionalAccessTypedComposite(t *testing.T) { + for _, tc := range []struct { + source string + want any + }{ + {`[]any{6}?[0]`, int64(6)}, + {`[]any{}?[0]`, nil}, + {`map[string]any{"a": 1}?.a`, int64(1)}, + {`map[string]any{}?.missing`, nil}, + } { + t.Run(tc.source, func(t *testing.T) { + got, err := evalExpr(t.Context(), tc.source, nil) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +// Measure preprocessing separately from evaluation. These modest inputs +// expose repeated scanning/copying without a timing-dependent assertion. +func BenchmarkCompileOptionalAccessChain(b *testing.B) { + for _, count := range []int{100, 500, 1000} { + b.Run(fmt.Sprintf("operators_%d", count), func(b *testing.B) { + source := "a" + strings.Repeat("?.a", count) + b.ReportAllocs() + b.SetBytes(int64(len(source))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := Compile(source); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// Compile errors used to report positions in the rewritten source, so +// every byte a rewrite inserted (`map[string]any`, `__try_select__(`, +// `__expr_kw_`) shifted the column: `x.type ^ 1` reported 1:18. +func TestSyntaxRegression_ErrorPositionsAfterRewrites(t *testing.T) { + for _, tc := range []struct{ source, want string }{ + {`{"a" 1}`, "1:6: missing ','"}, + {`[1, 2 3]`, "1:7: missing ','"}, + {`x.type ^ 1`, "1:8: bitwise operator ^"}, + {`x?.type & 1`, "1:9: bitwise operator &"}, + {`if(c, 1) ^ 2`, "1:10: bitwise operator ^"}, + {`[1, 2] | 3`, "1:8: pipe operator |"}, + {"x?.a?.b +\n [1][0:1]", "2:6: slice expressions"}, + {`{"a": x?.b}.a $`, "1:15: illegal character"}, + } { + t.Run(tc.source, func(t *testing.T) { + _, err := Compile(tc.source) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), tc.want) + }) + } + // Template segments report positions within the segment. + _, err := NewTemplate(`a ${ {"k" 1} } b`) + require.ErrorIs(t, err, ErrCompile) + require.Contains(t, err.Error(), "1:6: missing ','") +} + +// jsonlit rescanned the contents of every nested `[` for a top-level +// comma, so a deeply nested array literal took quadratic time: ~1.8s to +// rewrite at MaxSourceLength. One pass now marks every bracket. +func TestSyntaxRegression_NestedArrayLiteralLinear(t *testing.T) { + n := (MaxSourceLength - 1) / 2 + src := strings.Repeat("[", n) + "1" + strings.Repeat("]", n) + start := time.Now() + rewriteSource(src) + if d := time.Since(start); d > time.Second { + t.Fatalf("rewriting %d nested literals took %v", n, d) + } +} + +// Parsing a template copied the rest of the source for every `${`, and +// counted lines from the start for every segment, so a template with many +// segments allocated quadratically (209 MB for 64 KiB of `${x?.a}`). +func TestSyntaxRegression_TemplateParseLinear(t *testing.T) { + build := func(n int) string { return strings.Repeat("a ${x} b\n", n) } + small := allocatedBytes(func() { _, _ = NewTemplate(build(2000)) }) + big := allocatedBytes(func() { _, _ = NewTemplate(build(8000)) }) + if ratio := float64(big) / float64(small); ratio > 5 { + t.Fatalf("4x the segments allocated x%.1f", ratio) + } + // Positions are still exact. + tmpl, err := NewTemplate("a\n ${x} ${y}") + require.NoError(t, err) + segs := tmpl.Segments() + require.Equal(t, 2, segs[1].Line) + require.Equal(t, 3, segs[1].Column) + require.Equal(t, 8, segs[3].Column) +} diff --git a/template.go b/template.go index e1794ed..bf19750 100644 --- a/template.go +++ b/template.go @@ -284,7 +284,10 @@ func formatTemplateValue(v any) string { case string: return x } - if isCompositeKind(v) { + // encoding/json has no depth or work bound; a value outside the + // formatting bounds renders through safeFormatValue, which marks + // where it stopped. + if isCompositeKind(v) && jsonWithinBounds(v) { if s, ok := marshalTemplateJSON(v); ok { return s } @@ -295,13 +298,9 @@ func formatTemplateValue(v any) string { // isCompositeKind reports whether v is a map, slice, array, or struct // (following pointers), the shapes that render as JSON in templates. func isCompositeKind(v any) bool { - rv := reflect.ValueOf(v) - for rv.Kind() == reflect.Pointer { - if rv.IsNil() { - return false - } - rv = rv.Elem() - } + // unwrapReceiver bounds the pointer walk, so a self-referential + // pointer type cannot loop here. + rv, _ := unwrapReceiver(reflect.ValueOf(v)) switch rv.Kind() { case reflect.Map, reflect.Slice, reflect.Array, reflect.Struct: return true @@ -370,6 +369,13 @@ func parseTemplateSegments(raw, open, close string, compile func(string) (runner var segs []templateSegment var lit strings.Builder litStart := 0 + // Offsets only grow, so one forward-moving line counter and one + // byte copy of raw keep parsing linear in the template's length. + lines := &lineCounter{raw: raw, line: 1} + rawBytes := []byte(raw) + // nextEsc is the absolute offset of the next escape at or after i, + // -1 when there is none left, or -2 when not yet searched. + nextEsc := -2 appendLit := func(s string, at int) { if lit.Len() == 0 { litStart = at @@ -378,7 +384,7 @@ func parseTemplateSegments(raw, open, close string, compile func(string) (runner } flushLit := func() { if lit.Len() > 0 { - line, col := templateLineCol(raw, litStart) + line, col := lines.at(litStart) segs = append(segs, templateSegment{literal: lit.String(), offset: litStart, line: line, column: col}) lit.Reset() } @@ -395,7 +401,15 @@ func parseTemplateSegments(raw, open, close string, compile func(string) (runner openIdx := strings.Index(raw[i:], open) escIdx := -1 if escape != "" { - escIdx = strings.Index(raw[i:], escape) + if nextEsc == -2 || (nextEsc >= 0 && nextEsc < i) { + nextEsc = -1 + if k := strings.Index(raw[i:], escape); k >= 0 { + nextEsc = i + k + } + } + if nextEsc >= 0 { + escIdx = nextEsc - i + } } if openIdx < 0 && escIdx < 0 { appendLit(raw[i:], i) @@ -419,11 +433,11 @@ func parseTemplateSegments(raw, open, close string, compile func(string) (runner flushLit() openOffset := i exprStart := i + len(open) - bodyEnd, hadContent, err := scanTemplateExprEnd(raw, exprStart, openOffset, open, close, braces) + line, col := lines.at(openOffset) + bodyEnd, hadContent, err := scanTemplateExprEnd(raw, rawBytes, exprStart, openOffset, open, close, braces, line, col) if err != nil { return nil, err } - line, col := templateLineCol(raw, openOffset) body := strings.TrimSpace(raw[exprStart:bodyEnd]) if body == "" || !hadContent { return nil, fmt.Errorf("template: empty expression `%s%s` at %d:%d (offset %d)", @@ -459,6 +473,32 @@ func templateLineCol(raw string, offset int) (line, col int) { return line, col } +// lineCounter converts increasing byte offsets in raw to 1-based lines +// and byte columns, counting each newline once. +type lineCounter struct { + raw string + off, line int + lastNewline int // offset of the last newline before off, or -1 + initialized bool +} + +func (c *lineCounter) at(offset int) (line, col int) { + if !c.initialized { + c.lastNewline, c.initialized = -1, true + } + offset = min(offset, len(c.raw)) + if offset < c.off { + return templateLineCol(c.raw, offset) + } + for ; c.off < offset; c.off++ { + if c.raw[c.off] == '\n' { + c.line++ + c.lastNewline = c.off + } + } + return c.line, offset - c.lastNewline +} + // scanTemplateExprEnd returns the byte offset within src of the first // `}` of the run that closes the opener whose body begins at start. // It drives go/scanner over src[start:] and counts brace depth, so @@ -473,13 +513,17 @@ func templateLineCol(raw string, offset int) (line, col int) { // reported as an error rather than silently splitting the closer. // // hadContent reports whether the scanner saw at least one token inside -// the body that wasn't a brace or auto-inserted semicolon, so that +// the body other than the closing braces or an auto-inserted semicolon, so that // comment-only bodies like `${/* hi */}` are rejected up front with // the same "empty expression" error as `${}`. // // openOffset is the byte offset of the opening delimiter in the // original raw template source; it's only used for error messages. -func scanTemplateExprEnd(src string, start, openOffset int, open, close string, braces int) (end int, hadContent bool, err error) { +// +// srcBytes is src as bytes, converted once by the caller so scanning +// each expression does not copy the rest of the template; line and col +// locate the opener for error messages. +func scanTemplateExprEnd(src string, srcBytes []byte, start, openOffset int, open, close string, braces, line, col int) (end int, hadContent bool, err error) { fset := token.NewFileSet() file := fset.AddFile("", fset.Base(), len(src)-start) @@ -487,9 +531,8 @@ func scanTemplateExprEnd(src string, start, openOffset int, open, close string, // A no-op error handler keeps the scanner advancing through any // lex errors. Compile will surface the real diagnostic when the // body is later fed to go/parser. - s.Init(file, []byte(src[start:]), func(token.Position, string) {}, 0) + s.Init(file, srcBytes[start:], func(token.Position, string) {}, 0) - line, col := templateLineCol(src, openOffset) depth := braces // we are already inside the opener's brace run for { pos, tok, _ := s.Scan() @@ -498,7 +541,10 @@ func scanTemplateExprEnd(src string, start, openOffset int, open, close string, return 0, false, fmt.Errorf("template: unclosed `%s` at %d:%d (offset %d, missing `%s`)", open, line, col, openOffset, close) case token.LBRACE: + // The opener's own braces are not scanned, so every `{` + // here belongs to the body (an object literal like `${{}}`). depth++ + hadContent = true case token.RBRACE: depth-- if depth == 0 { diff --git a/types.go b/types.go index 42454a7..ebe32c3 100644 --- a/types.go +++ b/types.go @@ -252,10 +252,17 @@ func union(a, b *Type) *Type { // empty is the element type of an empty list literal. A list with no // elements fits any element type, so joining it with another list keeps // that list's element type: the fallback in `xs || []` or `if(c, xs, [])` -// keeps xs's. Anywhere else, such as the result of first([]), it is -// unchecked, like Any. +// keeps xs's. first and last of such a list are nil. Anywhere else, such +// as an element bound by map([], ...), it is unchecked, like Any. var empty = &Type{Kind: KindAny} +// never is the type of an operation that always fails at run time, such +// as indexing with nil. It produces no value, so joining it with another +// type gives that type: try(x[nil], y) has y's type. Anywhere else it is +// unchecked, like Any, since the failure is reported where it happens +// (or caught by try). +var never = &Type{Kind: KindAny} + func orAny(t *Type) *Type { if t == nil { return Any @@ -270,6 +277,10 @@ func Join(a, b *Type) *Type { return b case b == nil: return a + case a == never: + return b + case b == never: + return a case a.isAny() || b.isAny(): return Any case a.Kind == KindNull: diff --git a/value_matrix_test.go b/value_matrix_test.go new file mode 100644 index 0000000..472fbe7 --- /dev/null +++ b/value_matrix_test.go @@ -0,0 +1,354 @@ +package expr + +import ( + "context" + "errors" + "fmt" + "math" + "reflect" + "strings" + "testing" +) + +// The value matrix runs one set of expressions over a common set of Go +// values through every evaluation path that applies, and checks that +// the paths agree and that the documented relations between operators +// hold. The paths are the env shape (a map, so lookups take the +// type-asserted branches, or a struct, so selector and index chains +// take the reflect-space fast paths) crossed with optimization (on, or +// off via withoutOptimizations). + +type mxInner struct{ B any } + +type mxEmb struct{ E int } + +type mxStruct struct { + A any + N int + L []mxInner + M map[string]any + P *mxInner + hidden int //nolint:unused // read reflectively: must stay invisible +} + +// mxWithEmbPtr promotes E through a nil embedded pointer. +type mxWithEmbPtr struct { + *mxEmb + A int +} + +type ( + mxID int64 + mxName string + mxFlag bool + mxHolder struct{ V any } + mxSelf *mxSelf +) + +type mxErr struct{ msg string } + +func (e mxErr) Error() string { return e.msg } + +type mxPtrErr struct{ msg string } + +func (e *mxPtrErr) Error() string { return e.msg } + +type namedValue struct { + name string + v any +} + +// matrixValues is the shared set of host values. Cyclic values are not +// here: a regression in cycle handling overflows the stack, which no +// in-process test survives, so they live in isolated_test.go. +func matrixValues() []namedValue { + s := mxStruct{A: map[string]any{"B": 1}, N: 2, L: []mxInner{{B: "b0"}, {B: nil}}, M: map[string]any{"a": 1}, P: &mxInner{B: 3}} + ps := &s + return []namedValue{ + {"untyped nil", nil}, + {"nil *struct", (*mxStruct)(nil)}, + {"struct", s}, + {"*struct", ps}, + {"**struct", &ps}, + {"zero struct", mxStruct{}}, + {"embedded nil ptr", mxWithEmbPtr{A: 1}}, + {"embedded ptr", mxWithEmbPtr{mxEmb: &mxEmb{E: 4}, A: 1}}, + {"holder of slice", mxHolder{V: []int{1}}}, + {"holder of int", mxHolder{V: 1}}, + {"array of slice", [1]any{[]int{1}}}, + {"[]int", []int{1, 2}}, + {"nil []int", []int(nil)}, + {"empty []any", []any{}}, + {"[]any of maps", []any{map[string]any{"a": 1}, nil}}, + {"[]any nested", []any{[]any{"x"}}}, + {"[3]int", [3]int{7, 8, 9}}, + {"*[3]int", &[3]int{7, 8, 9}}, + {"string", "héllo"}, + {"empty string", ""}, + {"named string", mxName("ab")}, + {"map[string]any", map[string]any{"a": 1, "A": map[string]any{"B": 2}, "B": nil}}, + {"map[string]int", map[string]int{"a": 1}}, + {"nil map[string]int", map[string]int(nil)}, + {"map[mxName]int", map[mxName]int{"a": 1}}, + {"map[any]any", map[any]any{1: "x", "a": 2, int64(0): "zero"}}, + {"map[[2]int]string", map[[2]int]string{{1, 2}: "x"}}, + {"map[int]string", map[int]string{0: "z", 1: "o"}}, + {"map[mxID]string", map[mxID]string{1: "one"}}, + {"map[int8]string", map[int8]string{1: "one"}}, + {"int", 1}, + {"int8", int8(5)}, + {"int64 min", int64(math.MinInt64)}, + {"uint64 max", uint64(math.MaxUint64)}, + {"uint64 max-1", uint64(math.MaxUint64 - 1)}, + {"mxID 2^53+1", mxID(1<<53 + 1)}, + {"float32", float32(1.5)}, + {"float64 int", float64(1)}, + {"bool", true}, + {"named bool", mxFlag(true)}, + {"mxErr", mxErr{"x"}}, + {"*mxPtrErr", &mxPtrErr{"y"}}, + {"nil *mxPtrErr", (*mxPtrErr)(nil)}, + {"nil func", (func() int)(nil)}, + {"func", func() int { return 1 }}, + {"nil Func", Func(nil)}, + {"nil chan", (chan int)(nil)}, + } +} + +// Expressions over V alone, and over V with a key K or an index I. +var ( + mxExprs = []string{ + "V", "V.A", "V?.A", "V.a", "V?.a", "V.A.B", "V?.A?.B", "V.E", "V?.E", + "V.hidden", "V?.hidden", "V.P.B", "V?.P?.B", "V.L[0].B", "V.L[1].B", "V?.L?[5]", + "V[0]", "V?[0]", "V[1]", "V?[1]", "V[5]", "V?[5]", `V["a"]`, `V?["a"]`, + "V[0][0]", "V?[0]?[0]", "V[0].a", "V?[0]?.a", + "V == V", "V != V", "V == nil", "V != nil", "!V", "bool(V)", "if(V, 1, 2)", + "len(V)", "string(V)", `sprintf("%v", V)`, "V | len()", + "V + 1", "V - 1", "V < 1", "V >= 0", "-V", "V == 9007199254740992", "V == 18446744073709551614.0", + "contains([V], V)", "[V][0]", `{"x": V}.x`, `try(V.A, "d")`, `try(V[0], "d")`, + "filter([V], true)[0]", "filter([V, V], true)[1]", "map([V], it)[0]", "count([V], it == V)", + "keys(V)", "has(V, \"a\")", + } + mxKeyExprs = []string{"V[K]", "V?[K]", "V.M[K]", "V?.M?[K]", "try(V[K], \"d\")"} + mxKeys = []any{"a", "A", "missing", 1, int64(0), mxName("a"), []int{1, 2}, []any{1}, nil, 1.5} + mxIndexExprs = []string{"V[I]", "V?[I]", "V.L[I]", "V?.L?[I]", "try(V[I], \"d\")"} + mxIndexes = []any{0, 1, 5, -1, 1.0, 1.5, "a", nil, uint64(math.MaxUint64), mxID(1)} +) + +// mxEnv is the struct form of the matrix env. +type mxEnv struct{ V, K, I any } + +type mxOutcome struct { + value any + category string // "", "evaluate", "compile", or "UNWRAPPED" + msg string + panicked any +} + +func (o mxOutcome) String() string { + if o.panicked != nil { + return fmt.Sprintf("panic(%v)", o.panicked) + } + if o.category != "" { + return o.category + ": " + o.msg + } + return fmt.Sprintf("%#v", o.value) +} + +func errCategory(err error) string { + switch { + case err == nil: + return "" + case errors.Is(err, ErrCompile): + return "compile" + case errors.Is(err, ErrEvaluate): + return "evaluate" + } + return "UNWRAPPED" +} + +func mxRun(src string, env any, opts ...Option) (out mxOutcome) { + defer func() { + if r := recover(); r != nil { + out = mxOutcome{panicked: r} + } + }() + p, err := Compile(src, append([]Option{WithBuiltins()}, opts...)...) + if err != nil { + return mxOutcome{category: errCategory(err), msg: err.Error()} + } + v, err := p.Run(context.Background(), env) + if err != nil { + return mxOutcome{category: errCategory(err), msg: err.Error()} + } + return mxOutcome{value: v} +} + +func sameOutcome(a, b mxOutcome) bool { + if a.panicked != nil || b.panicked != nil { + return false + } + return a.category == b.category && a.msg == b.msg && mxDeepEqual(a.value, b.value) +} + +// mxDeepEqual is reflect.DeepEqual, except that funcs are equal when +// they are the same function (DeepEqual only equates nil funcs). +func mxDeepEqual(a, b any) bool { + if a != nil && b != nil && reflect.TypeOf(a).Kind() == reflect.Func && reflect.TypeOf(a) == reflect.TypeOf(b) { + return reflect.ValueOf(a).Pointer() == reflect.ValueOf(b).Pointer() + } + return reflect.DeepEqual(a, b) +} + +type mxCase struct { + src string + k, i any +} + +func mxCases() []mxCase { + var cases []mxCase + for _, src := range mxExprs { + cases = append(cases, mxCase{src: src}) + } + for _, src := range mxKeyExprs { + for _, k := range mxKeys { + cases = append(cases, mxCase{src: src, k: k}) + } + } + for _, src := range mxIndexExprs { + for _, i := range mxIndexes { + cases = append(cases, mxCase{src: src, i: i}) + } + } + return cases +} + +// mxEval runs c over v through every path and returns the outcome +// they agree on, reporting any disagreement or panic. +func mxEval(t *testing.T, label string, c mxCase, v any) mxOutcome { + t.Helper() + mapEnv := map[string]any{"V": v, "K": c.k, "I": c.i} + structEnv := mxEnv{V: v, K: c.k, I: c.i} + paths := []struct { + name string + out mxOutcome + }{ + {"map", mxRun(c.src, mapEnv)}, + {"map/noopt", mxRun(c.src, mapEnv, withoutOptimizations())}, + {"struct", mxRun(c.src, structEnv)}, + {"struct/noopt", mxRun(c.src, &structEnv, withoutOptimizations())}, + } + for _, p := range paths { + if p.out.panicked != nil { + t.Errorf("%s: %s via %s panicked: %v", label, c.src, p.name, p.out.panicked) + } + if p.out.category == "UNWRAPPED" { + t.Errorf("%s: %s via %s: error does not wrap ErrEvaluate/ErrCompile: %s", label, c.src, p.name, p.out.msg) + } + } + for _, p := range paths[1:] { + if !sameOutcome(paths[0].out, p.out) { + t.Errorf("%s: %s (K=%#v I=%#v)\n map: %v\n %-8s %v", label, c.src, c.k, c.i, paths[0].out, p.name+":", p.out) + } + } + return paths[0].out +} + +func TestValueMatrixPathsAgree(t *testing.T) { + for _, nv := range matrixValues() { + t.Run(nv.name, func(t *testing.T) { + got := map[string]mxOutcome{} + for _, c := range mxCases() { + key := c.src + if c.k != nil || c.i != nil { + key = fmt.Sprintf("%s|%#v|%#v", c.src, c.k, c.i) + } + got[key] = mxEval(t, nv.name, c, nv.v) + } + checkMatrixRelations(t, nv, got) + }) + } +} + +// checkMatrixRelations asserts the documented relations between the +// outcomes of related expressions on one value. +func checkMatrixRelations(t *testing.T, nv namedValue, got map[string]mxOutcome) { + t.Helper() + ok := func(o mxOutcome) bool { return o.category == "" && o.panicked == nil } + + // Strict and optional access: ?. and ?[ give the strict result when + // there is one, nil when the receiver is nil or the value is + // missing, and the strict error otherwise. + pairs := [][2]string{ + {"V.A", "V?.A"}, {"V.a", "V?.a"}, {"V.E", "V?.E"}, {"V.hidden", "V?.hidden"}, + {"V[0]", "V?[0]"}, {"V[1]", "V?[1]"}, {`V["a"]`, `V?["a"]`}, + } + for _, k := range mxKeys { + pairs = append(pairs, [2]string{fmt.Sprintf("V[K]|%#v|", k), fmt.Sprintf("V?[K]|%#v|", k)}) + } + for _, i := range mxIndexes { + pairs = append(pairs, [2]string{fmt.Sprintf("V[I]||%#v", i), fmt.Sprintf("V?[I]||%#v", i)}) + } + for _, p := range pairs { + strict, opt := got[p[0]], got[p[1]] + switch { + case ok(strict) && !sameOutcome(strict, opt): + t.Errorf("%s: %s = %v but %s = %v", nv.name, p[0], strict, p[1], opt) + case !ok(strict) && ok(opt) && opt.value != nil: + t.Errorf("%s: %s fails (%v) but %s = %v; optional access may only add nil", nv.name, p[0], strict, p[1], opt) + case !ok(opt) && ok(strict): + t.Errorf("%s: %s fails (%v) but %s succeeds", nv.name, p[1], opt, p[0]) + case !ok(opt) && opt.msg != strict.msg: + t.Errorf("%s: %s and %s fail differently:\n %s\n %s", nv.name, p[0], p[1], strict.msg, opt.msg) + } + if !ok(strict) && ok(opt) && isNilValue(nv.v) == false && strings.Contains(strict.msg, "cannot select") { + t.Errorf("%s: wrong-kind %s became nil under %s", nv.name, p[0], p[1]) + } + } + + // try(x, d) is x when x succeeds and d when it fails. + for strict, tried := range map[string]string{"V.A": `try(V.A, "d")`, "V[0]": `try(V[0], "d")`} { + s, tr := got[strict], got[tried] + want := mxOutcome{value: "d"} + if ok(s) { + want = s + } + if !sameOutcome(want, tr) { + t.Errorf("%s: %s = %v, want %v (from %s = %v)", nv.name, tried, tr, want, strict, s) + } + } + + // != is exactly !(==), and the two fail together. + eq, ne := got["V == V"], got["V != V"] + if ok(eq) != ok(ne) || (ok(eq) && eq.value == ne.value) { + t.Errorf("%s: V == V is %v but V != V is %v", nv.name, eq, ne) + } + // Equality is reflexive on comparable values (no NaN here). + if ok(eq) && eq.value != true { + t.Errorf("%s: V == V = %v", nv.name, eq) + } + // == nil agrees with the documented typed-nil rule. + if n := got["V == nil"]; !ok(n) || n.value != isNilValue(nv.v) { + t.Errorf("%s: V == nil = %v, want %v", nv.name, n, isNilValue(nv.v)) + } + + // Truthiness is one rule, wherever it is consulted. + truthy := IsTruthy(nv.v) + for src, want := range map[string]any{"!V": !truthy, "bool(V)": truthy, "if(V, 1, 2)": map[bool]int64{true: 1, false: 2}[truthy]} { + if o := got[src]; !sameOutcome(o, mxOutcome{value: want}) { + t.Errorf("%s: %s = %v, want %v", nv.name, src, o, want) + } + } + + // Moving a value through a literal or a form does not change it. + for _, src := range []string{"[V][0]", `{"x": V}.x`, "filter([V], true)[0]", "filter([V, V], true)[1]", "map([V], it)[0]"} { + if o := got[src]; !sameOutcome(o, got["V"]) { + t.Errorf("%s: %s = %v, want %v", nv.name, src, o, got["V"]) + } + } + + // Formatting never fails, and both formatters agree. + if s, f := got["string(V)"], got[`sprintf("%v", V)`]; !ok(s) || !ok(f) { + t.Errorf("%s: formatting failed: string(V) = %v, sprintf = %v", nv.name, s, f) + } +}