diff --git a/docs/howto/analyze.md b/docs/howto/analyze.md index e7c2c66b41..87541d093e 100644 --- a/docs/howto/analyze.md +++ b/docs/howto/analyze.md @@ -106,4 +106,6 @@ reports the result columns and parameters: ] ``` -Pass `--ast` to also include each statement's parsed AST under an `ast` key. +Pass `--ast` to also include each statement's parsed AST under an `ast` key. It +has the same shape as the output of [`parse`](parse.md), with every node tagged +by type. diff --git a/docs/howto/parse.md b/docs/howto/parse.md index 2227c1f2b6..623e3b8b90 100644 --- a/docs/howto/parse.md +++ b/docs/howto/parse.md @@ -48,6 +48,7 @@ The output is a JSON array with one object per statement: "name": "GetAuthor", "cmd": ":one", "ast": { + "tag": "RawStmt", "Stmt": { "...": "..." }, @@ -60,3 +61,49 @@ The output is a JSON array with one object per statement: Statements without a `-- name:` annotation (for example schema DDL) omit the `name` and `cmd` fields. + +## Node types + +Every node in the AST carries a `tag` naming its type. Some nodes have no +fields of their own, so without it a star, a null literal and an untranslated +clause would all print as `{}`. + +```json +"Val": { + "tag": "ColumnRef", + "Name": "", + "Fields": { + "tag": "List", + "Items": [ + { + "tag": "A_Star" + } + ] + }, + "Location": 93 +} +``` + +A `tag` of `TODO` marks a clause the dialect's converter does not translate +yet. It means the clause was parsed but is not represented in the AST, not that +the clause was absent from the query. + +## Absent fields + +A field the statement does not use is left out rather than printed as `null`. +An `A_Const` carrying an integer reports only what it has: + +```json +"Val": { + "tag": "A_Const", + "Val": { + "tag": "Integer", + "Ival": 1 + }, + "Location": 30 +} +``` + +Only absent fields are omitted. A zero keeps its place, because zero is a value +the parser can find: `StmtLocation` is 0 for the first statement in a file, and +`LIMIT 0` parses to an `Ival` of 0. diff --git a/internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt index b74264f687..ab257bea23 100644 --- a/internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt @@ -24,23 +24,28 @@ } ], "ast": { + "tag": "RawStmt", "Stmt": { + "tag": "SelectStmt", "DistinctClause": { - "Items": null + "tag": "List" }, - "IntoClause": null, "TargetList": { + "tag": "List", "Items": [ { - "Name": null, + "tag": "ResTarget", "Indirection": { - "Items": null + "tag": "List" }, "Val": { + "tag": "ColumnRef", "Name": "", "Fields": { + "tag": "List", "Items": [ { + "tag": "String", "Str": "name" } ] @@ -52,32 +57,37 @@ ] }, "FromClause": { + "tag": "List", "Items": [ { - "Catalogname": null, - "Schemaname": null, + "tag": "RangeVar", "Relname": "authors", "Inh": true, "Relpersistence": 112, - "Alias": null, "Location": 45 } ] }, "WhereClause": { + "tag": "A_Expr", "Kind": 1, "Name": { + "tag": "List", "Items": [ { + "tag": "String", "Str": "=" } ] }, "Lexpr": { + "tag": "ColumnRef", "Name": "", "Fields": { + "tag": "List", "Items": [ { + "tag": "String", "Str": "id" } ] @@ -85,6 +95,7 @@ "Location": 59 }, "Rexpr": { + "tag": "ParamRef", "Number": 1, "Location": 64, "Dollar": true @@ -92,28 +103,31 @@ "Location": 62 }, "GroupClause": { - "Items": null + "tag": "List" + }, + "HavingClause": { + "tag": "TODO" }, - "HavingClause": {}, "WindowClause": { - "Items": null + "tag": "List" }, "ValuesLists": { - "Items": null + "tag": "List" }, "SortClause": { - "Items": null + "tag": "List" + }, + "LimitOffset": { + "tag": "TODO" + }, + "LimitCount": { + "tag": "TODO" }, - "LimitOffset": {}, - "LimitCount": {}, "LockingClause": { - "Items": null + "tag": "List" }, - "WithClause": null, "Op": 0, - "All": false, - "Larg": null, - "Rarg": null + "All": false }, "StmtLocation": 0, "StmtLen": 66 diff --git a/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt b/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt index 28a5ce7e1f..8b2e6a9f56 100644 --- a/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt @@ -3,16 +3,18 @@ "name": "GetValue", "cmd": ":one", "ast": { + "tag": "RawStmt", "Stmt": { - "DistinctClause": null, - "IntoClause": null, + "tag": "SelectStmt", "TargetList": { + "tag": "List", "Items": [ { - "Name": null, - "Indirection": null, + "tag": "ResTarget", "Val": { + "tag": "A_Const", "Val": { + "tag": "Integer", "Ival": 1 }, "Location": 31 @@ -21,21 +23,8 @@ } ] }, - "FromClause": null, - "WhereClause": null, - "GroupClause": null, - "HavingClause": null, - "WindowClause": null, - "ValuesLists": null, - "SortClause": null, - "LimitOffset": null, - "LimitCount": null, - "LockingClause": null, - "WithClause": null, "Op": 0, - "All": false, - "Larg": null, - "Rarg": null + "All": false }, "StmtLocation": 0, "StmtLen": 32 diff --git a/internal/endtoend/testdata/parse_basic/duckdb/stdout.txt b/internal/endtoend/testdata/parse_basic/duckdb/stdout.txt index ca847e616a..cae0296ac7 100644 --- a/internal/endtoend/testdata/parse_basic/duckdb/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/duckdb/stdout.txt @@ -3,16 +3,18 @@ "name": "GetValue", "cmd": ":one", "ast": { + "tag": "RawStmt", "Stmt": { - "DistinctClause": null, - "IntoClause": null, + "tag": "SelectStmt", "TargetList": { + "tag": "List", "Items": [ { - "Name": null, - "Indirection": null, + "tag": "ResTarget", "Val": { + "tag": "A_Const", "Val": { + "tag": "Integer", "Ival": 1 }, "Location": 30 @@ -21,21 +23,8 @@ } ] }, - "FromClause": null, - "WhereClause": null, - "GroupClause": null, - "HavingClause": null, - "WindowClause": null, - "ValuesLists": null, - "SortClause": null, - "LimitOffset": null, - "LimitCount": null, - "LockingClause": null, - "WithClause": null, "Op": 0, - "All": false, - "Larg": null, - "Rarg": null + "All": false }, "StmtLocation": 0, "StmtLen": 33 diff --git a/internal/endtoend/testdata/parse_basic/googlesql/stdout.txt b/internal/endtoend/testdata/parse_basic/googlesql/stdout.txt index 086f885402..715536dec9 100644 --- a/internal/endtoend/testdata/parse_basic/googlesql/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/googlesql/stdout.txt @@ -3,16 +3,18 @@ "name": "GetValue", "cmd": ":one", "ast": { + "tag": "RawStmt", "Stmt": { - "DistinctClause": null, - "IntoClause": null, + "tag": "SelectStmt", "TargetList": { + "tag": "List", "Items": [ { - "Name": null, - "Indirection": null, + "tag": "ResTarget", "Val": { + "tag": "A_Const", "Val": { + "tag": "Integer", "Ival": 1 }, "Location": 30 @@ -21,21 +23,8 @@ } ] }, - "FromClause": null, - "WhereClause": null, - "GroupClause": null, - "HavingClause": null, - "WindowClause": null, - "ValuesLists": null, - "SortClause": null, - "LimitOffset": null, - "LimitCount": null, - "LockingClause": null, - "WithClause": null, "Op": 0, - "All": false, - "Larg": null, - "Rarg": null + "All": false }, "StmtLocation": 0, "StmtLen": 31 diff --git a/internal/endtoend/testdata/parse_basic/mssql/stdout.txt b/internal/endtoend/testdata/parse_basic/mssql/stdout.txt index b20fbdcee5..39e5aa0df6 100644 --- a/internal/endtoend/testdata/parse_basic/mssql/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/mssql/stdout.txt @@ -3,16 +3,18 @@ "name": "GetValue", "cmd": ":one", "ast": { + "tag": "RawStmt", "Stmt": { - "DistinctClause": null, - "IntoClause": null, + "tag": "SelectStmt", "TargetList": { + "tag": "List", "Items": [ { - "Name": null, - "Indirection": null, + "tag": "ResTarget", "Val": { + "tag": "A_Const", "Val": { + "tag": "Integer", "Ival": 1 }, "Location": 30 @@ -21,21 +23,8 @@ } ] }, - "FromClause": null, - "WhereClause": null, - "GroupClause": null, - "HavingClause": null, - "WindowClause": null, - "ValuesLists": null, - "SortClause": null, - "LimitOffset": null, - "LimitCount": null, - "LockingClause": null, - "WithClause": null, "Op": 0, - "All": false, - "Larg": null, - "Rarg": null + "All": false }, "StmtLocation": 0, "StmtLen": 32 diff --git a/internal/endtoend/testdata/parse_basic/mysql/stdout.txt b/internal/endtoend/testdata/parse_basic/mysql/stdout.txt index e9ed28784f..5ee2116255 100644 --- a/internal/endtoend/testdata/parse_basic/mysql/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/mysql/stdout.txt @@ -3,16 +3,18 @@ "name": "GetValue", "cmd": ":one", "ast": { + "tag": "RawStmt", "Stmt": { - "DistinctClause": null, - "IntoClause": null, + "tag": "SelectStmt", "TargetList": { + "tag": "List", "Items": [ { - "Name": null, - "Indirection": null, + "tag": "ResTarget", "Val": { + "tag": "A_Const", "Val": { + "tag": "Integer", "Ival": 1 }, "Location": 30 @@ -22,26 +24,17 @@ ] }, "FromClause": { - "Items": null + "tag": "List" }, - "WhereClause": null, "GroupClause": { - "Items": null + "tag": "List" }, - "HavingClause": null, "WindowClause": { + "tag": "List", "Items": [] }, - "ValuesLists": null, - "SortClause": null, - "LimitOffset": null, - "LimitCount": null, - "LockingClause": null, - "WithClause": null, "Op": 0, - "All": false, - "Larg": null, - "Rarg": null + "All": false }, "StmtLocation": 0, "StmtLen": 31 diff --git a/internal/endtoend/testdata/parse_basic/postgresql/stdout.txt b/internal/endtoend/testdata/parse_basic/postgresql/stdout.txt index fe35a664c7..deaaea7d6c 100644 --- a/internal/endtoend/testdata/parse_basic/postgresql/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/postgresql/stdout.txt @@ -3,20 +3,24 @@ "name": "GetValue", "cmd": ":one", "ast": { + "tag": "RawStmt", "Stmt": { + "tag": "SelectStmt", "DistinctClause": { - "Items": null + "tag": "List" }, - "IntoClause": null, "TargetList": { + "tag": "List", "Items": [ { - "Name": null, + "tag": "ResTarget", "Indirection": { - "Items": null + "tag": "List" }, "Val": { + "tag": "A_Const", "Val": { + "tag": "Integer", "Ival": 1 }, "Location": 30 @@ -26,32 +30,37 @@ ] }, "FromClause": { - "Items": null + "tag": "List" + }, + "WhereClause": { + "tag": "TODO" }, - "WhereClause": {}, "GroupClause": { - "Items": null + "tag": "List" + }, + "HavingClause": { + "tag": "TODO" }, - "HavingClause": {}, "WindowClause": { - "Items": null + "tag": "List" }, "ValuesLists": { - "Items": null + "tag": "List" }, "SortClause": { - "Items": null + "tag": "List" + }, + "LimitOffset": { + "tag": "TODO" + }, + "LimitCount": { + "tag": "TODO" }, - "LimitOffset": {}, - "LimitCount": {}, "LockingClause": { - "Items": null + "tag": "List" }, - "WithClause": null, "Op": 0, - "All": false, - "Larg": null, - "Rarg": null + "All": false }, "StmtLocation": 0, "StmtLen": 31 diff --git a/internal/endtoend/testdata/parse_basic/sqlite/stdout.txt b/internal/endtoend/testdata/parse_basic/sqlite/stdout.txt index c1303a9a1e..50d4c72330 100644 --- a/internal/endtoend/testdata/parse_basic/sqlite/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/sqlite/stdout.txt @@ -3,16 +3,18 @@ "name": "GetValue", "cmd": ":one", "ast": { + "tag": "RawStmt", "Stmt": { - "DistinctClause": null, - "IntoClause": null, + "tag": "SelectStmt", "TargetList": { + "tag": "List", "Items": [ { - "Name": null, - "Indirection": null, + "tag": "ResTarget", "Val": { + "tag": "A_Const", "Val": { + "tag": "Integer", "Ival": 1 }, "Location": 30 @@ -22,28 +24,19 @@ ] }, "FromClause": { - "Items": null + "tag": "List" }, - "WhereClause": null, "GroupClause": { - "Items": null + "tag": "List" }, - "HavingClause": null, "WindowClause": { - "Items": null + "tag": "List" }, "ValuesLists": { - "Items": null + "tag": "List" }, - "SortClause": null, - "LimitOffset": null, - "LimitCount": null, - "LockingClause": null, - "WithClause": null, "Op": 0, - "All": false, - "Larg": null, - "Rarg": null + "All": false }, "StmtLocation": 0, "StmtLen": 31 diff --git a/internal/sql/ast/json.go b/internal/sql/ast/json.go new file mode 100644 index 0000000000..cf33cd69d7 --- /dev/null +++ b/internal/sql/ast/json.go @@ -0,0 +1,131 @@ +package ast + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" +) + +// TagKey is the JSON key under which a node's concrete type is reported. +// +// Node is an interface, so the JSON encoding of an AST would otherwise carry no +// record of which node a given object is. Nodes with no fields (A_Star, Null, +// TODO) all encode as "{}", and an empty List is indistinguishable from them. +// Every node object is emitted with this key first, holding the name of its Go +// type. +const TagKey = "tag" + +var nodeType = reflect.TypeOf((*Node)(nil)).Elem() + +// MarshalJSON encodes the statement and every node beneath it, tagging each +// node object with its type. RawStmt is the root of the AST that the parse and +// analyze commands print, so implementing it here is enough to tag a whole +// tree. +func (n *RawStmt) MarshalJSON() ([]byte, error) { + if n == nil { + return []byte("null"), nil + } + return marshalValue(reflect.ValueOf(n)) +} + +func marshalValue(v reflect.Value) ([]byte, error) { + switch v.Kind() { + case reflect.Invalid: + return []byte("null"), nil + + case reflect.Interface, reflect.Pointer: + if v.IsNil() { + return []byte("null"), nil + } + return marshalValue(v.Elem()) + + case reflect.Struct: + return marshalStruct(v) + + case reflect.Slice: + if v.IsNil() { + return []byte("null"), nil + } + fallthrough + case reflect.Array: + return marshalArray(v) + + default: + // Scalars, strings and anything else encoding/json already handles. + return json.Marshal(v.Interface()) + } +} + +func marshalStruct(v reflect.Value) ([]byte, error) { + var buf bytes.Buffer + buf.WriteByte('{') + + // A node's Pos method is declared on the pointer type. + if reflect.PointerTo(v.Type()).Implements(nodeType) { + fmt.Fprintf(&buf, "%q:%q", TagKey, v.Type().Name()) + } + + t := v.Type() + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if !field.IsExported() { + continue + } + name := field.Name + if tag, ok := field.Tag.Lookup("json"); ok { + tagName, _, _ := strings.Cut(tag, ",") + if tagName == "-" { + continue + } + if tagName != "" { + name = tagName + } + } + if isNil(v.Field(i)) { + continue + } + value, err := marshalValue(v.Field(i)) + if err != nil { + return nil, err + } + if buf.Len() > 1 { + buf.WriteByte(',') + } + fmt.Fprintf(&buf, "%q:", name) + buf.Write(value) + } + + buf.WriteByte('}') + return buf.Bytes(), nil +} + +// isNil reports whether a field is absent from the tree, which is the only +// thing left out of the encoding. Zero-valued scalars are kept: a Location of 0 +// is the start of the file and an Ival of 0 is the literal in "LIMIT 0", so +// dropping them would lose what the parser found. +func isNil(v reflect.Value) bool { + switch v.Kind() { + case reflect.Pointer, reflect.Interface, reflect.Slice, reflect.Map: + return v.IsNil() + } + return false +} + +func marshalArray(v reflect.Value) ([]byte, error) { + var buf bytes.Buffer + buf.WriteByte('[') + for i := 0; i < v.Len(); i++ { + if i > 0 { + buf.WriteByte(',') + } + item, err := marshalValue(v.Index(i)) + if err != nil { + return nil, err + } + buf.Write(item) + } + buf.WriteByte(']') + return buf.Bytes(), nil +} diff --git a/internal/sql/ast/tag_test.go b/internal/sql/ast/tag_test.go new file mode 100644 index 0000000000..3165ca06f2 --- /dev/null +++ b/internal/sql/ast/tag_test.go @@ -0,0 +1,49 @@ +package ast + +import ( + "go/ast" + "go/parser" + "go/token" + "strings" + "testing" +) + +// TestNoFieldShadowsTagKey guards the JSON encoding in json.go: every node +// object is emitted with a TagKey key naming its type, so no node may declare a +// field of its own that collides with it. encoding/json matches field names +// case-insensitively, so a field named "Tag" would silently capture the tag +// when the output is decoded back into a node. +// +// This is a unit test because the invariant is about the declarations in this +// package, not about anything sqlc produces. No SQL input can exercise a field +// that does not exist yet, so the end-to-end tests cannot catch the day someone +// adds one. +func TestNoFieldShadowsTagKey(t *testing.T) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", nil, 0) + if err != nil { + t.Fatalf("parsing package: %s", err) + } + + for _, pkg := range pkgs { + ast.Inspect(pkg, func(n ast.Node) bool { + spec, ok := n.(*ast.TypeSpec) + if !ok { + return true + } + structType, ok := spec.Type.(*ast.StructType) + if !ok { + return true + } + for _, field := range structType.Fields.List { + for _, name := range field.Names { + if strings.EqualFold(name.Name, TagKey) { + t.Errorf("%s declares a field %q, which collides with the %q key used to tag node types in JSON. Rename the field or pick a different TagKey.", + spec.Name.Name, name.Name, TagKey) + } + } + } + return true + }) + } +}