Support gts-spec v0.17 reference keywords

.github/workflows/ci.yml

@@ -162,9 +162,15 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          # Install httprunner first to get pydantic<1.9 constraint
-          pip install httprunner pytest requests
-          # Then install gts (will use already-installed pydantic v1)
+          # The gts-spec test runner requires pytest >= 9 (see
+          # .gts-spec/tests/pytest.ini). httprunner pins incompatible deps
+          # (pytest < 8, pydantic < 1.9), so install it with --no-deps and take
+          # the rest from the spec's requirements, mirroring the Makefile's
+          # py-env target. Installing gts then pins pydantic to v2.
+          grep -viE '^[[:space:]]*httprunner([^A-Za-z0-9_.-]|$)' \
+            .gts-spec/tests/requirements.txt > /tmp/spec-requirements.txt
+          pip install -r /tmp/spec-requirements.txt
+          pip install --no-deps 'httprunner>=4,<5'
           pip install -e ./gts
 
       - name: Run e2e tests

.gts-spec-version

@@ -1 +1 @@
-v0.15.0
+v0.17.0

Makefile

@@ -54,6 +54,21 @@ endif
 
 force-py-env:
 
+# Recreate the venv when the bootstrap interpreter's Python version no longer
+# matches the existing venv (e.g. the system python3 was upgraded, or the venv
+# was created by a different python3). Silently reusing a venv built with an
+# incompatible interpreter causes confusing failures: google-re2 ships wheels
+# only for 3.10-3.14, so a stale 3.9 venv tries to build it from source and
+# fails on missing RE2/Abseil headers.
+PY_ENV_VERSION := $(shell "$(PY_ENV_PYTHON)" -c 'import sys;print("%d.%d"%sys.version_info[:2])' 2>/dev/null)
+PY_BOOTSTRAP_VERSION := $(shell $(PYTHON_BOOTSTRAP) -c 'import sys;print("%d.%d"%sys.version_info[:2])' 2>/dev/null)
+ifneq ($(PY_ENV_VERSION),)
+ifneq ($(PY_ENV_VERSION),$(PY_BOOTSTRAP_VERSION))
+$(PY_ENV_PYTHON): force-py-env
+$(PY_ENV_STAMP): force-py-env
+endif
+endif
+
 $(PY_ENV_STAMP): $(PY_ENV_PYTHON) gts/pyproject.toml $(GTS_SPEC_DIR)/tests/requirements.txt requirements.txt Makefile
 	@echo "Creating/updating Python virtual environment in $(PY_ENV_DIR)..."
 	@rm -f "$@"

README.md

@@ -2,9 +2,9 @@
 
 # GTS Python Library
 
-A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts.
+A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/GlobalTypeSystem/gts-spec)) identifiers and JSON/JSON Schema artifacts.
 
-Current supported GTS spec version: `0.15.0`. Conformance CI pins the released gts-spec `v0.15.0` in [`.gts-spec-version`](.gts-spec-version) and the `.gts-spec` submodule; run `make update-spec` to check the pinned release out. The 0.15 breaking change is the [safe regular-expression profile](gts/README.md#regular-expressions).
+Targets gts-spec [v0.17.0](https://github.com/GlobalTypeSystem/gts-spec/releases/tag/v0.17.0) — recorded in [`.gts-spec-version`](.gts-spec-version) and pinned by the `.gts-spec` submodule. Run `make update-spec` to check the pinned release out.
 
 ## Roadmap
 
@@ -29,14 +29,16 @@ Featureset:
 
 See details in [gts/README.md](gts/README.md)
 
-Other GTS spec [Reference Implementation](https://github.com/globaltypesystem/gts-spec/blob/main/README.md#9-reference-implementation-recommendations) recommended features support:
+Other GTS spec [Reference Implementation](https://github.com/GlobalTypeSystem/gts-spec/blob/main/README.md#9-reference-implementation-recommendations) recommended features support:
 
 - [x] **In-memory entities registry** - simple GTS entities registry with optional GTS references validation on entity registration
 - [x] **CLI** - command-line interface for all GTS operations
 - [x] **Web server** - a non-production web-server with REST API for the operations processing and testing
-- [x] **x-gts-ref support** - to support special GTS entity reference annotation in schemas
+- [x] **x-gts-type-ref / x-gts-instance-ref** - kind-specific GTS entity references in schemas (spec §9.6.1)
+- [x] **x-gts-pattern-ref** - pattern-valued string fields: GTS identifier pattern syntax and coverage by an operand, independent of the registry (spec §9.6.2, §10)
+- [x] **x-gts-final / x-gts-abstract** - GTS Type Schema modifiers controlling inheritance and instantiation
 - [x] **YAML support** - to support YAML files (*.yml, *.yaml) as input files
-- [x] **UUID for instances** - to support UUID as ID in JSON instances
+- [x] **UUID for instances** - to support UUID as ID in JSON instances (anonymous instances)
 - [ ] **TypeSpec support** - direct support for [typespec.io](https://typespec.io/) files (*.tsp) input files
 
 Technical Backlog:
@@ -49,7 +51,7 @@ Technical Backlog:
 
 ## Installation
 
-GTS requires Python 3.9 or later.
+GTS requires Python 3.9 or later. `google-re2` ships wheels for CPython 3.10–3.14; on Python 3.9 pip builds it from source, which needs a C++17 compiler and the RE2 and Abseil development headers.
 
 ### Local development
 
@@ -82,39 +84,133 @@ pip install gts
 ### CLI
 
 ```bash
-gts <command> <args>
-
-# see available commands
+# Basic operations
+gts validate-id --gts-id 'gts.example.demo._.event.v1~'
+gts parse-id --gts-id 'gts.example.demo._.event.v1~'
+gts match-id-pattern --candidate 'gts.example.demo._.event.v1~' --pattern 'gts.example.*'
+gts uuid --gts-id 'gts.example.demo._.event.v1~'
+
+# Operations with loaded entities.
+# Placeholders: point --path at a directory holding your GTS schemas and
+# instances, and substitute their IDs (no ./examples directory ships with
+# this repository; the gts-spec submodule's .gts-spec/examples has samples).
+gts --path ./examples validate-instance --gts-id 'gts.example.demo._.event.v1~example.demo._.created.v1'
+gts --path ./examples validate-type-schema --gts-id 'gts.example.demo._.event.v1.1~'
+gts --path ./examples validate-entity --gts-id 'gts.example.demo._.event.v1~'
+gts --path ./examples resolve-relationships --gts-id 'gts.example.demo._.event.v1~'
+gts --path ./examples compatibility --old-schema-id 'gts.example.demo._.event.v1.0~' --new-schema-id 'gts.example.demo._.event.v1.1~'
+gts --path ./examples cast --from-id 'gts.example.demo._.event.v1~example.demo._.created.v1' --to-schema-id 'gts.example.demo._.event.v1.1~'
+gts --path ./examples query --expr 'gts.example.*' --limit 10
+gts --path ./examples attr --gts-with-path 'gts.example.demo._.event.v1~@properties.name'
+gts --path ./examples list --limit 100
+
+# See all commands and options
 gts --help
 ```
 
 ### Library
 
-See [gts/README.md](gts/README.md)
+`GtsOps` provides the high-level in-memory API:
 
-### Web server
+```python
+from gts.ops import GtsOps
+
+ops = GtsOps()
+
+# OP#1 - ID Validation
+result = ops.validate_id("gts.example.demo._.event.v1~")
+assert result.valid
+
+# OP#2 - ID Extraction
+extracted = ops.extract_id({"id": "gts.example.demo._.event.v1.0", "name": "Event"})
+print(extracted.id)
+
+# OP#3 - ID Parsing
+parsed = ops.parse_id("gts.example.demo._.event.v1~")
+for seg in parsed.segments:
+    print(seg.vendor, seg.package, seg.type, seg.ver_major)
+
+# OP#4 - Pattern Matching
+match = ops.match_id_pattern("gts.example.demo._.event.v1.0", "gts.example.*")
+assert match.match
+
+# OP#5 - ID to UUID Mapping
+uuid_res = ops.uuid("gts.example.demo._.event.v1~")
+print(uuid_res.uuid)
+```
+
+See [gts/README.md](gts/README.md) for full library documentation including schema validation, store operations, query execution, traits, and casting.
 
-The web server is a non-production web-server with REST API for the operations processing and testing. It implements reference API for gts-spec [tests](https://github.com/GlobalTypeSystem/gts-spec/tree/main/tests)
+### Web server
 
+The web server is a non-production web server with REST API for operations processing and testing. It implements the reference API for [gts-spec tests](https://github.com/GlobalTypeSystem/gts-spec/tree/main/tests).
 
 ```bash
-# start the web server, default location is http://127.0.0.1:8000
-gts serve
+# Start the web server (default: http://127.0.0.1:8000)
+gts server
 
-# start the web server on different port
+# Start the web server on a different port
 gts server --port 8081
 
-# pre-populate server with the JSON instancens and schemas from the gts-spec tests
-gts server --path {PATH_TO_JSON_FILES}
+# Pre-populate server with entities from files or directories
+gts --path {PATH_TO_JSON_FILES} server
+
+# Allow re-registering an entity with different content (default: reject with 409 Conflict)
+gts server --allow-entity-updates
 
-# Generate the OpenAPI schema
-curl -s http://127.0.0.1:8000/openapi.json | jq > ./gts/openapi.json
+# Generate the OpenAPI specification
+gts openapi-spec --out openapi.json
 
-# See the schema
+# Or fetch from running server
+curl -s http://127.0.0.1:8000/openapi.json | jq > ./openapi.json
 curl -s http://127.0.0.1:8000/openapi.json | jq | less -S
 ```
 
-### Testing
+#### API Endpoints
+
+##### Entity Management
+- `GET /entities` - List registered entities (`limit` query parameter, 1–1000)
+- `GET /entities/{gts_id}` - Retrieve a specific entity
+- `POST /entities` - Register a single entity (optional `?validate=true`)
+- `POST /entities/bulk` - Register multiple entities
+- `POST /type-schemas` - Register a batch of GTS Type Schemas (each `type_id` derived from its embedded `$id`)
+
+##### GTS Operations
+- `GET /validate-id?gts_id=<id>` - Validate GTS ID (OP#1)
+- `POST /extract-id` - Extract GTS ID from JSON (OP#2)
+- `GET /parse-id?gts_id=<id>` - Parse GTS ID (OP#3)
+- `GET /match-id-pattern?candidate=<id>&pattern=<pat>` - Match pattern (OP#4)
+- `GET /uuid?gts_id=<id>` - Generate UUID (OP#5)
+- `POST /validate-instance` - Validate registered instance (OP#6)
+- `POST /validate-json` - Validate transient instance or Type Schema JSON without registering (OP#6)
+- `POST /validate-json/{gts_type}` - Validate transient instance JSON against explicit type (OP#6)
+- `GET /resolve-relationships?gts_id=<id>` - Resolve relationships and detect broken references (OP#7)
+- `GET /compatibility?old_type_id=<id>&new_type_id=<id>` - Check Type Schema evolution compatibility (OP#8)
+- `POST /cast` - Cast instance to target schema (OP#9)
+- `GET /query?expr=<expr>&limit=<limit>` - Query entities (OP#10)
+- `GET /attr?gts_with_path=<id@path>` - Retrieve attribute value (OP#11)
+- `POST /validate-type-schema` - Validate derived Type Schema against base chain (OP#12)
+- `POST /validate-entity` - Validate entity by GTS ID (OP#12/OP#13)
+
+## Development
+
+Common development tasks are wrapped in the `Makefile`:
+
+```bash
+make py-env      # create/update virtualenv and install dependencies
+make install     # install gts in editable mode
+make build       # build source and wheel distributions into dist/
+make check       # run all quality checks (fmt, lint, mypy, test, e2e)
+make test        # run unit tests
+make coverage    # measure code coverage
+make fmt         # check code formatting (ruff)
+make dev-fmt     # auto-format code (ruff)
+make lint        # run linter (ruff)
+make mypy        # run type checker
+make security    # check dependencies for vulnerabilities (pip-audit)
+```
+
+## Testing
 
 You can test the gts-python library by utilizing the shared test suite from the [gts-spec](https://github.com/GlobalTypeSystem/gts-spec) specification and executing the tests against the web server.
 

gts/README.md

@@ -2,7 +2,7 @@
 
 Python helpers and a reference HTTP service for the [Global Type System (GTS)](https://github.com/globaltypesystem/gts-spec). The package supports GTS identifier parsing, JSON Schema-backed validation, schema compatibility and derivation checks, traits, casting, queries, file loading, a CLI, and a FastAPI application.
 
-The package targets GTS specification 0.15.0 and requires Python 3.9 or later. `google-re2` ships wheels for CPython 3.10-3.14; on Python 3.9 pip builds it from source, which needs a C++17 compiler and the RE2 and Abseil development headers.
+The package targets GTS specification 0.17.0 and requires Python 3.9 or later. `google-re2` ships wheels for CPython 3.10-3.14; on Python 3.9 pip builds it from source, which needs a C++17 compiler and the RE2 and Abseil development headers.
 
 ## Installation
 
@@ -166,8 +166,8 @@ TypeSpec (`.tsp`) inputs must be compiled to JSON Schema before loading.
 | `get(entity_id)` | Return `GtsEntity` or `None`. |
 | `items()` | Return an iterator over in-memory `(entity_id, entity)` pairs. |
 | `get_schema_content(type_id)` | Return a schema dictionary or raise `KeyError`. |
-| `validate_schema_basic(type_id)` | Check `$ref` format, `x-gts-ref` declarations, and GTS keyword placement. |
-| `validate_schema(type_id)` | Run full JSON Schema, derivation, final/abstract, x-gts-ref, and trait validation. |
+| `validate_schema_basic(type_id)` | Check `$ref` format, GTS reference declarations, and GTS keyword placement. |
+| `validate_schema(type_id)` | Run full JSON Schema, derivation, final/abstract, reference, and trait validation. |
 | `validate_instance(gts_id)` | Validate a well-known or UUID-addressed instance against its type schema. |
 | `is_minor_compatible(old_schema_id, new_schema_id)` | Return compatibility verdicts for two registered schemas. |
 | `cast(from_id, target_schema_id)` | Cast a registered **instance** to a target schema. |
@@ -215,11 +215,15 @@ The library validates schemas with the dialect named by `$schema`; if absent, it
 
 During instance validation, GTS references are resolved with `referencing.Registry`. Draft 2019-09 and Draft 2020-12 `$ref` sibling constraints are preserved, so a sibling such as `minLength` is enforced.
 
-### `x-gts-ref`
+### GTS references
 
-`x-gts-ref` restricts a string value to a GTS ID or pattern. It accepts an absolute `gts.` pattern or a JSON Pointer beginning with `/` that resolves to one. If a store is present, the referenced entity must be registered.
+GTS 0.17 replaces the former `x-gts-ref` with kind-specific entity references `x-gts-type-ref` and `x-gts-instance-ref`, and 0.16 adds `x-gts-pattern-ref`:
 
-`x-gts-ref` can appear in `oneOf`, `anyOf`, and `allOf`. For x-gts-ref-only combinator branches, GTS evaluates the x-gts-ref constraints as the combinator condition. For ordinary or mixed JSON Schema branches, normal JSON Schema structural matching determines which branch’s x-gts-ref constraints are applied.
+- `x-gts-type-ref`: restricts a string value to a GTS Type Identifier.
+- `x-gts-instance-ref`: restricts a string value to a GTS Instance Identifier.
+- `x-gts-pattern-ref`: restricts a string value to a GTS identifier pattern covered by the operand (`M(value) ⊆ M(operand)`), evaluated without consulting the registry.
+
+The former kind-blind `x-gts-ref` keyword was removed in GTS 0.16 and is rejected on schema validation like any unknown `x-gts-*` keyword.
 
 ### `x-gts-final` and `x-gts-abstract`
 

gts/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "gts"
-version = "0.16.0"
+version = "0.17.0"
 description = "Global Type System (GTS) helpers: identifiers, parsing, validation, and operations"
 readme = "README.md"
 authors = [{ name = "GTS Community" }]

gts/src/gts/_json_validation.py

@@ -13,7 +13,9 @@
 from .gts import GtsID
 from .store import GtsStore
 
-_X_GTS_REF_KEYWORD = "x-gts-ref"
+# Any GTS extension keyword (x-gts-type-ref, x-gts-pattern-ref, x-gts-final, ...)
+# marks a file as GTS-related even when it carries no gts. identifier.
+_X_GTS_KEYWORD_PREFIX = "x-gts-"
 
 
 @dataclass
@@ -131,7 +133,9 @@ def _on_walk_error(err: OSError) -> None:
     @staticmethod
     def _is_gts_marker(text: str) -> bool:
         return (
-            GTS_PREFIX in text or GTS_URI_PREFIX in text or _X_GTS_REF_KEYWORD in text
+            GTS_PREFIX in text
+            or GTS_URI_PREFIX in text
+            or _X_GTS_KEYWORD_PREFIX in text
         )
 
     def _read_file(self, file_path: Path) -> None:

gts/src/gts/compatibility.py

@@ -199,6 +199,17 @@ def _is_subschema(subset: Any, superset: Any) -> bool | None:
     lowered_superset = _lower_root_unevaluated_properties(superset)
     if lowered_subset is None or lowered_superset is None:
         return None
+    # Equal schemas accept equal instance sets, so inclusion holds. This also
+    # covers assertions the delegated engine cannot reason about (e.g. nested
+    # object-valued ``const``/``enum``), where it would otherwise report unknown.
+    # The GTS reference dimension is compared separately (README §9.6.1), so it
+    # is sound for this structural primitive to ignore ``x-gts-*`` here. JSON
+    # spelling distinguishes ``true`` from ``1`` (and ``false`` from ``0``), so
+    # compare canonical JSON rather than Python equality, which conflates them.
+    if _json_key(sanitize(_coerce_bool_schema(lowered_subset))) == _json_key(
+        sanitize(_coerce_bool_schema(lowered_superset))
+    ):
+        return True
     finite_result = _finite_subset(lowered_subset, lowered_superset)
     if finite_result is not None:
         return finite_result
@@ -430,18 +441,39 @@ def dialects_differ(old_schema: Any, new_schema: Any) -> bool:
     )
 
 
+def _inclusion_verdict(subset: Any, superset: Any) -> str:
+    """Structural inclusion folded with the GTS reference-scope dimension.
+
+    The reference scopes are compared separately (README §9.6.1) because the
+    delegated structural engine does not understand the reference keywords. A
+    proved reference exclusion is incompatible; an unknown reference relation is
+    unknown unless the structural dimension already disproves inclusion.
+    """
+    from . import ref_scope_compat
+
+    ref_rel = ref_scope_compat.schema_ref_relation(superset, subset)
+    if ref_rel == ref_scope_compat.EXCLUDED:
+        return INCOMPATIBLE
+    structural = _verdict(_is_subschema(subset, superset))
+    if structural == INCOMPATIBLE:
+        return INCOMPATIBLE
+    if ref_rel == ref_scope_compat.UNKNOWN:
+        return UNKNOWN
+    return structural
+
+
 def check_backward_compatibility(old_schema: Any, new_schema: Any) -> str:
     """new consumers read old data: ``Valid(old) subset-of Valid(new)``."""
     if dialects_differ(old_schema, new_schema):
         return UNKNOWN
-    return _verdict(_is_subschema(old_schema, new_schema))
+    return _inclusion_verdict(old_schema, new_schema)
 
 
 def check_forward_compatibility(old_schema: Any, new_schema: Any) -> str:
     """old consumers read new data: ``Valid(new) subset-of Valid(old)``."""
     if dialects_differ(old_schema, new_schema):
         return UNKNOWN
-    return _verdict(_is_subschema(new_schema, old_schema))
+    return _inclusion_verdict(new_schema, old_schema)
 
 
 def full_verdict(backward: str, forward: str) -> str:

gts/src/gts/derivation.py

@@ -18,6 +18,7 @@
 import copy
 from typing import Any
 
+from . import ref_scope_compat
 from .compatibility import boolean_schema_value, check_accepted_set_inclusion
 
 MAX_RECURSION_DEPTH = 64
@@ -38,6 +39,19 @@ def validate_derivation_compatibility(
             base_schema, derived_schema, base_id, derived_id
         )
     )
+    # GTS reference scopes (README §9.6.1): the derived scope must be provably
+    # included in the base's. Admission fails closed, so an unknown relation is
+    # rejected too.
+    if (
+        ref_scope_compat.schema_ref_relation(
+            base_schema, derived_schema, base_id, derived_id
+        )
+        != ref_scope_compat.INCLUDED
+    ):
+        errors.append(
+            f"derived schema '{derived_id}' is not included in base '{base_id}': "
+            "a GTS reference constraint is not included in the base constraint"
+        )
     return errors