From e6da57aa234f535510fcd72143f022771c0f9938 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 1 Nov 2026 15:12:02 +0100 Subject: [PATCH 1/3] test: simplify the extension examples and pin their null rows CustomExtensionIntegrationTest: validateVector states the one accepted storage shape as a record pattern instead of nested negations; the dtype helpers drop parameters always given the same value; readers opened only to assert a failure are closed; Chunk.column needs no cast. nullRows_readBackAsNull lists the null rows by id: the existing comparison against the vector() fixture still passes when the fixture stops writing nulls, so nothing caught it. CurrencyExtensionIntegrationTest asserts the expected code directly. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011fRbYrNfgxEVYukCoXYfre --- .../CurrencyExtensionIntegrationTest.java | 3 +- .../CustomExtensionIntegrationTest.java | 101 +++++++++++------- 2 files changed, 63 insertions(+), 41 deletions(-) diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java index 3f52b8f34..ed86f0a70 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java @@ -135,7 +135,8 @@ void allowUnknown_readsColumnAsItsCodes(@TempDir Path tmp) throws IOException { } // Then - assertThat(firstCode).isEqualTo(currency(1).getCurrencyCode()); + // row 1 is the second of the four cycling currencies + assertThat(firstCode).isEqualTo("USD"); } @Test diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java index 27f66e8a2..0ca293e19 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java @@ -40,8 +40,8 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -96,7 +96,7 @@ class RoundTrip { void fullScan_decodesEveryVectorAndNull(@TempDir Path tmp) throws IOException { // Given — three batches of nullable vectors written from float[] values Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, true)); + writeVectors(file, vectorDtype(true)); // When SequencedMap result = readVectors(file, ScanOptions.all()); @@ -106,11 +106,30 @@ void fullScan_decodesEveryVectorAndNull(@TempDir Path tmp) throws IOException { assertEachVectorMatchesItsId(result); } + @Test + void nullRows_readBackAsNull(@TempDir Path tmp) throws IOException { + // Given — a nullable column written with a null vector at every seventh row (3, 10, 17, …) + Path file = tmp.resolve("vectors.vortex"); + writeVectors(file, vectorDtype(true)); + + // When + SequencedMap result = readVectors(file, ScanOptions.all()); + + // Then — the null rows, listed by id rather than compared with vector(): a fixture that + // stops writing nulls still agrees with vector() and passes assertEachVectorMatchesItsId, + // so only this catches it; an encoder or decoder losing the validity fails here too + List nullRows = result.entrySet().stream() + .filter(row -> row.getValue() == null) + .map(Map.Entry::getKey) + .toList(); + assertThat(nullRows).isEqualTo(LongStream.range(0, ROWS).filter(row -> row % 7 == 3).boxed().toList()); + } + @Test void schema_keepsIdStorageAndMetadata(@TempDir Path tmp) throws IOException { // Given Path file = tmp.resolve("vectors.vortex"); - DType.Extension written = vectorDtype(DIM, true); + DType.Extension written = vectorDtype(true); writeVectors(file, written); DType result; @@ -131,7 +150,7 @@ void filteredScan_keepsEachVectorWithItsRow(@TempDir Path tmp) throws IOExceptio // cannot match, so the result is a superset of the matching rows; every vector must // still sit next to its own id. Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, true)); + writeVectors(file, vectorDtype(true)); long threshold = CHUNK_ROWS + CHUNK_ROWS / 2; // When @@ -148,7 +167,7 @@ void filteredScan_keepsEachVectorWithItsRow(@TempDir Path tmp) throws IOExceptio void limitedScan_returnsTheFirstRows(@TempDir Path tmp) throws IOException { // Given Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, true)); + writeVectors(file, vectorDtype(true)); int limit = CHUNK_ROWS + 7; // When @@ -164,7 +183,7 @@ void storageArray_withoutEncoder_isWrittenAsIs(@TempDir Path tmp) throws IOExcep // Given — no encoder registered: the caller passes the storage shape itself and the // metadata as the bytes a file stores, as for any extension id the writer does not know Path file = tmp.resolve("raw.vortex"); - DType.Struct schema = schema(opaqueVectorDtype(2, 2)); + DType.Struct schema = schema(opaqueVectorDtype(2)); float[] flat = {1f, 2f, 3f, 4f}; try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); var sut = VortexWriter.create(ch, schema, WriteOptions.defaults())) { @@ -186,11 +205,11 @@ void open_withoutDecoder_throws(@TempDir Path tmp) throws IOException { // Given — a reader with only the built-ins. Rust refuses an extension id its session // has no vtable for ("No such DType extension ID") unless unknown ids are allowed. Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); ReadRegistry builtInsOnly = ReadRegistry.loadAll(); // When / Then - assertThatThrownBy(() -> VortexReader.open(file, builtInsOnly)) + assertThatThrownBy(() -> VortexReader.open(file, builtInsOnly).close()) .isInstanceOf(VortexException.class) .hasMessageContaining("no extension decoder registered for example.vector"); } @@ -200,7 +219,7 @@ void allowUnknown_readsColumnAsStorageArray(@TempDir Path tmp) throws IOExceptio // Given — with unknown ids allowed the column reads as its fixed-size-list storage, // Rust's ForeignExtDType Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); ReadRegistry allowUnknown = ReadRegistry.builder().registerDefaults().allowUnknown().build(); // When @@ -208,19 +227,20 @@ void allowUnknown_readsColumnAsStorageArray(@TempDir Path tmp) throws IOExceptio try (VortexReader reader = VortexReader.open(file, allowUnknown); var iter = reader.scan(ScanOptions.all().withLimit(ScanLimit.of(2))); Chunk chunk = iter.next()) { - FixedSizeListArray storage = (FixedSizeListArray) chunk.column(EMBEDDING); + FixedSizeListArray storage = chunk.column(EMBEDDING); firstElementOfRow1 = ((FloatArray) storage.elements()).getFloat(DIM); } // Then - assertThat(firstElementOfRow1).isEqualTo(vector(1)[0]); + // row 1's vector is (1, -1.25, 1.5): its first element is the row number + assertThat(firstElementOfRow1).isEqualTo(1f); } @Test void as_withoutDecoder_throws(@TempDir Path tmp) throws IOException { // Given Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); ReadRegistry allowUnknown = ReadRegistry.builder().registerDefaults().allowUnknown().build(); try (VortexReader reader = VortexReader.open(file, allowUnknown); @@ -237,7 +257,7 @@ void as_withoutDecoder_throws(@TempDir Path tmp) throws IOException { void as_wrongDomainType_throws(@TempDir Path tmp) throws IOException { // Given Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); try (VortexReader reader = VortexReader.open(file, READ_REGISTRY); var iter = reader.scan(ScanOptions.all().withLimit(ScanLimit.of(2))); @@ -254,7 +274,7 @@ void vortexJni_withoutVtable_refusesTheFile(@TempDir Path tmp) throws IOExceptio // Given — the Rust reference, which has no example.vector vtable and whose JNI session // exposes no allow_unknown: the behavior the strict default above mirrors Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); String uri = file.toAbsolutePath().toUri().toString(); // When / Then @@ -274,7 +294,7 @@ void create_withoutEncoder_rejectsTypedMetadata(@TempDir Path tmp) throws IOExce try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { // When / Then - assertThatThrownBy(() -> VortexWriter.create(ch, schema(vectorDtype(DIM, false)), WriteOptions.defaults())) + assertThatThrownBy(() -> VortexWriter.create(ch, schema(vectorDtype(false)), WriteOptions.defaults())) .isInstanceOf(VortexException.class) .hasMessageContaining("no extension encoder registered for example.vector"); } @@ -298,7 +318,7 @@ void create_rejectsDtypeTheEncoderRefuses(@TempDir Path tmp) throws IOException @Test void open_rejectsDtypeTheDecoderRefuses(@TempDir Path tmp) throws IOException { // Given — a file from a writer that had no encoder, so nothing validated the dtype - DType.Extension invalid = opaqueVectorDtype(2, 5); + DType.Extension invalid = opaqueVectorDtype(5); Path file = tmp.resolve("invalid.vortex"); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); var writer = VortexWriter.create(ch, schema(invalid), WriteOptions.defaults())) { @@ -306,7 +326,7 @@ void open_rejectsDtypeTheDecoderRefuses(@TempDir Path tmp) throws IOException { } // When / Then — Rust validates an extension dtype when it deserializes it - assertThatThrownBy(() -> VortexReader.open(file, READ_REGISTRY)) + assertThatThrownBy(() -> VortexReader.open(file, READ_REGISTRY).close()) .isInstanceOf(VortexException.class) .hasMessageContaining("example.vector metadata dimension 5 != storage size 2"); } @@ -316,10 +336,10 @@ void writeChunk_nullInNonNullableColumn_throws(@TempDir Path tmp) throws IOExcep // Given Path file = tmp.resolve("nonnull.vortex"); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); - var sut = VortexWriter.create(ch, schema(vectorDtype(DIM, false)), WriteOptions.defaults(), WRITE_REGISTRY)) { + var sut = VortexWriter.create(ch, schema(vectorDtype(false)), WriteOptions.defaults(), WRITE_REGISTRY)) { // When / Then assertThatThrownBy(() -> sut.writeChunk(Map.of(ID, new long[]{0L}, - EMBEDDING, Arrays.asList((float[]) null)))) + EMBEDDING, Collections.singletonList((float[]) null)))) .isInstanceOf(VortexException.class) .hasMessageContaining("null element in non-nullable example.vector column"); } @@ -335,15 +355,16 @@ void writeChunk_nullInNonNullableColumn_throws(@TempDir Path tmp) throws IOExcep record VectorMetadata(int dimension) implements ExtensionMetadata { } - /// The `example.vector` dtype: `FixedSizeList` storage, dimension also in metadata. - static DType.Extension vectorDtype(int dim, boolean nullable) { - return new DType.Extension(VECTOR.id(), new DType.FixedSizeList(DTypes.F32, dim, nullable), - new VectorMetadata(dim), nullable); + /// The `example.vector` dtype: `FixedSizeList` storage, dimension also in metadata. + static DType.Extension vectorDtype(boolean nullable) { + return new DType.Extension(VECTOR.id(), new DType.FixedSizeList(DTypes.F32, DIM, nullable), + new VectorMetadata(DIM), nullable); } - /// The dtype as a writer without the encoder writes it: metadata as the bytes a file stores. - static DType.Extension opaqueVectorDtype(int storageDim, int metadataDim) { - return new DType.Extension(VECTOR.id(), new DType.FixedSizeList(DTypes.F32, storageDim, false), + /// The dtype as a writer without the encoder writes it, over 2-element vectors: metadata as the + /// bytes a file stores, here declaring `metadataDim` dimensions. + static DType.Extension opaqueVectorDtype(int metadataDim) { + return new DType.Extension(VECTOR.id(), new DType.FixedSizeList(DTypes.F32, 2, false), new OpaqueMetadata(dimensionBytes(metadataDim)), false); } @@ -355,18 +376,17 @@ static MemorySegment dimensionBytes(int dim) { /// Shared by encoder and decoder, as one Rust `ExtVTable::validate_dtype` serves both directions. static void validateVector(DType.Extension dtype) { - if (!(dtype.storageDType() instanceof DType.FixedSizeList list) - || !(list.elementType() instanceof DType.Primitive p && p.ptype() == PType.F32 && !p.nullable())) { - throw new VortexException("example.vector storage must be FixedSizeList, got " - + dtype.storageDType()); - } - if (!(dtype.metadata() instanceof VectorMetadata(int dim))) { - throw new VortexException("example.vector metadata must be a VectorMetadata, got " + dtype.metadata()); - } - if (dim != list.fixedSize()) { - throw new VortexException("example.vector metadata dimension " + dim - + " != storage size " + list.fixedSize()); + // the one accepted shape; anything else falls through to the error + if (dtype.storageDType() instanceof DType.FixedSizeList(DType.Primitive(PType ptype, boolean nullable), int size, var _) + && ptype == PType.F32 && !nullable + && dtype.metadata() instanceof VectorMetadata(int dim)) { + if (dim != size) { + throw new VortexException("example.vector metadata dimension " + dim + " != storage size " + size); + } + return; } + throw new VortexException("example.vector must be FixedSizeList storage with a VectorMetadata, got " + + dtype.storageDType() + " and " + dtype.metadata()); } static final class VectorEncoder implements ExtensionEncoder { @@ -378,7 +398,7 @@ public ExtensionId extensionId() { @Override public DType.Extension dtype(boolean nullable) { - return vectorDtype(DIM, nullable); + return vectorDtype(nullable); } @Override @@ -432,7 +452,7 @@ public ExtensionId extensionId() { @Override public DType.Extension dtype(boolean nullable) { - return vectorDtype(DIM, nullable); + return vectorDtype(nullable); } @Override @@ -505,6 +525,7 @@ private void writeVectors(Path file, DType.Extension vectorDtype) throws IOExcep long row = (long) chunk * CHUNK_ROWS + i; ids[i] = row; float[] vector = vector(row); + // a non-nullable column has no null rows: give them a zero vector vectors.add(vector == null && !vectorDtype.nullable() ? new float[DIM] : vector); } writer.writeChunk(Map.of(ID, ids, EMBEDDING, vectors)); @@ -521,7 +542,7 @@ private SequencedMap readVectors(Path file, ScanOptions options) while (iter.hasNext()) { try (Chunk chunk = iter.next()) { List vectors = chunk.as(EMBEDDING, float[].class); - LongArray ids = (LongArray) chunk.column(ID); + LongArray ids = chunk.column(ID); for (int i = 0; i < vectors.size(); i++) { out.put(ids.getLong(i), vectors.get(i)); } From 196977d26a26a1db801a3e839bbb91732f7d4f05 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 1 Nov 2026 15:21:53 +0100 Subject: [PATCH 2/3] refactor(core): DType.Extension carries an ExtensionId, not a String The id was parsed at the file boundary and then stored as its raw string, so most consumers re-parsed it (ExtensionId.parse(ext.extensionId())) or compared strings (ExtensionId.VORTEX_TIMESTAMP.id().equals(...)). DType.Extension#extensionId() now returns the typed id, matching ExtensionDecoder/ExtensionEncoder#extensionId(): consumers switch or compare by identity on it, the reader passes the id it already parsed, and only the FlatBuffer serializer turns it back into its wire string. Strings at the boundary, types inside. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011fRbYrNfgxEVYukCoXYfre --- AGENTS.md | 2 +- CHANGELOG.md | 1 + .../github/dfa1/vortex/calcite/Temporal.java | 2 +- .../dfa1/vortex/calcite/VortexEnumerator.java | 2 +- .../dfa1/vortex/calcite/VortexTable.java | 2 +- .../github/dfa1/vortex/cli/tui/GridRender.java | 2 +- .../dfa1/vortex/cli/tui/InspectorRender.java | 2 +- .../dfa1/vortex/cli/SchemaCommandTest.java | 5 +++-- .../github/dfa1/vortex/cli/VortexCliTest.java | 3 ++- .../dfa1/vortex/cli/tui/GridRenderTest.java | 5 +++-- .../vortex/cli/tui/InspectorRenderTest.java | 3 ++- .../github/dfa1/vortex/core/model/DType.java | 8 +++++--- .../dfa1/vortex/core/model/TimeDtype.java | 2 +- .../dfa1/vortex/core/model/TimestampDtype.java | 2 +- .../vortex/core/model/DTypeExtensionTest.java | 8 ++++---- .../core/model/DTypeWithNullableTest.java | 6 +++--- .../vortex/core/model/ExtensionIdTest.java | 10 +++++----- docs/how-to.md | 2 +- docs/reference.md | 2 +- .../github/dfa1/vortex/inspect/HtmlReport.java | 2 +- .../CurrencyExtensionIntegrationTest.java | 2 +- .../CustomExtensionIntegrationTest.java | 6 +++--- .../JavaWritesRustReadsIntegrationTest.java | 7 ++++--- ...lKnownExtensionsInteropIntegrationTest.java | 7 ++++--- .../github/dfa1/vortex/jdbc/JdbcImporter.java | 2 +- .../dfa1/vortex/parquet/ParquetExporter.java | 2 +- .../dfa1/vortex/parquet/ColumnBuilderTest.java | 3 ++- .../vortex/parquet/ParquetExporterTest.java | 3 ++- .../vortex/parquet/ParquetImporterTest.java | 2 +- .../io/github/dfa1/vortex/reader/Chunk.java | 3 +-- .../dfa1/vortex/reader/PostscriptParser.java | 2 +- .../dfa1/vortex/reader/ReadRegistry.java | 3 +-- .../reader/decode/VariantEncodingDecoder.java | 2 +- .../reader/extension/DateExtensionDecoder.java | 2 +- .../reader/extension/UuidExtensionDecoder.java | 2 +- .../PostscriptParserDTypeGuardsTest.java | 2 +- .../dfa1/vortex/reader/ReadRegistryTest.java | 6 +++--- .../dfa1/vortex/reader/compute/ValuesTest.java | 3 ++- .../decode/ConstantEncodingDecoderTest.java | 11 ++++++----- .../DateTimePartsEncodingDecoderTest.java | 5 +++-- .../decode/VariantEncodingDecoderTest.java | 2 +- .../extension/DateExtensionDecoderTest.java | 4 ++-- .../reader/extension/ExtensionStorageTest.java | 3 ++- .../reader/extension/ExtensionTestSupport.java | 3 ++- .../reader/layout/DictLayoutDecoderTest.java | 5 +++-- .../reader/layout/ZonedStatsSchemaTest.java | 3 ++- .../dfa1/vortex/writer/DTypeFbsSerializer.java | 4 ++-- .../dfa1/vortex/writer/VortexWriter.java | 6 ++---- .../writer/encode/DateExtensionEncoder.java | 2 +- .../encode/DateTimePartsEncodingEncoder.java | 2 +- .../writer/encode/UuidExtensionEncoder.java | 2 +- .../vortex/writer/DTypeFbsSerializerTest.java | 18 +++++++++--------- .../dfa1/vortex/writer/WriterZoneMapTest.java | 3 ++- .../vortex/writer/ZoneMapStatCodecTest.java | 3 ++- .../DateTimePartsEncodingEncoderTest.java | 5 +++-- .../writer/encode/ExtEncodingEncoderTest.java | 13 +++++++------ .../encode/TimeExtensionEncoderTest.java | 2 +- .../writer/encode/ZoneMapStatsFitnessTest.java | 3 ++- 58 files changed, 124 insertions(+), 105 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4f8adca17..bae6089ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -302,7 +302,7 @@ When stuck on encode/decode behavior, consult **in this order**: ## Design decisions - **DType is pluggable only via `Extension`.** `DType` is a sealed interface; downstream code must - not add variants. Use `new DType.Extension("ip.address", new DType.Primitive(PType.I32, false), + not add variants. Use `new DType.Extension(ExtensionId.parse("ip.address"), new DType.Primitive(PType.I32, false), OpaqueMetadata.EMPTY, false)` and register an `ExtensionDecoder`/`ExtensionEncoder` on `ReadRegistry`/`WriteRegistry`. Mirrors Rust's `ExtVTable` (`vortex.date`, `vortex.uuid`, …): an unregistered id fails the read unless `allowUnknown()`, then reads as storage. No SPI for DType diff --git a/CHANGELOG.md b/CHANGELOG.md index 89b3e563e..2bbe556ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** CSV import stores an empty cell in a number or boolean column as null instead of `0` or `false`, so inferred number and boolean columns are now nullable (read through `MaskedArray`), and an empty cell or a missing column fails the import where the schema declares the column non-nullable. ([#592](https://github.com/dfa1/vortex-java/issues/592)) - **Breaking:** the shared non-nullable dtypes move from `DType` to the new `DTypes` class: `DTypes.I64`, `DTypes.UTF8`, … ([#624](https://github.com/dfa1/vortex-java/pull/624)) - **Breaking:** an extension dtype's metadata is a typed `ExtensionMetadata` record instead of a `MemorySegment`, parsed when a file opens and serialized when it is written, as Rust does: `TimeUnitMetadata` for date and time, `TimestampMetadata` (its zone kept as stored) for timestamp, `OpaqueMetadata` for bytes nothing parses, and a third-party record through `ExtensionDecoder#parseMetadata` and `ExtensionEncoder#serializeMetadata`. ([#625](https://github.com/dfa1/vortex-java/pull/625)) +- **Breaking:** `DType.Extension#extensionId()` is an `ExtensionId` instead of a `String`; build one with `ExtensionId.VORTEX_DATE` or `ExtensionId.parse("acme.money")`. ([#626](https://github.com/dfa1/vortex-java/pull/626)) - **Breaking:** `MemorySize` moves from `io.github.dfa1.vortex.core.model` to `io.github.dfa1.vortex.core.io`. ([#625](https://github.com/dfa1/vortex-java/pull/625)) ### Fixed diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/Temporal.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/Temporal.java index 1bf168b48..ed86653a9 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/Temporal.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/Temporal.java @@ -31,7 +31,7 @@ record Temporal(ExtensionId.WellKnown kind, TimeUnit unit) { /// @throws VortexException for a unit the extension does not allow, as the decoders do static Temporal of(DType type) { if (!(type instanceof DType.Extension ext) - || !(ExtensionId.parse(ext.extensionId()) instanceof ExtensionId.WellKnown kind) + || !(ext.extensionId() instanceof ExtensionId.WellKnown kind) || kind != ExtensionId.WellKnown.VORTEX_DATE && kind != ExtensionId.WellKnown.VORTEX_TIME && kind != ExtensionId.WellKnown.VORTEX_TIMESTAMP) { return null; diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexEnumerator.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexEnumerator.java index da5e38219..9087cf210 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexEnumerator.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexEnumerator.java @@ -268,7 +268,7 @@ private static Object value(Object array, DType type, long r) { case DType.Decimal _ -> ((DecimalArray) array).getDecimal(r); // Calcite's internal forms: DATE is days since the epoch (int), TIME milliseconds of the // day (int), TIMESTAMP milliseconds since the epoch (long) - case DType.Extension ext -> switch (ExtensionId.parse(ext.extensionId())) { + case DType.Extension ext -> switch (ext.extensionId()) { case ExtensionId.WellKnown.VORTEX_DATE -> Math.toIntExact(DateExtensionDecoder.INSTANCE.decode(ext, (Array) array, r).toEpochDay()); case ExtensionId.WellKnown.VORTEX_TIME -> diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java index 4f122ef27..945328267 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java @@ -1557,7 +1557,7 @@ private static RelDataType toSqlType(RelDataTypeFactory factory, DType type) { case DType.Bool _ -> factory.createSqlType(SqlTypeName.BOOLEAN); case DType.Null _ -> factory.createSqlType(SqlTypeName.NULL); case DType.Decimal d -> factory.createSqlType(SqlTypeName.DECIMAL, d.precision(), d.scale()); - case DType.Extension ext -> switch (ExtensionId.parse(ext.extensionId())) { + case DType.Extension ext -> switch (ext.extensionId()) { case ExtensionId.WellKnown.VORTEX_DATE -> factory.createSqlType(SqlTypeName.DATE); // millisecond precision, the precision the scan delivers: at Calcite's default of 0 a // literal built for the column (a pushed-down MIN/MAX, a top-N row) lost its millis diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/GridRender.java b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/GridRender.java index 39b3c76a5..0f46379d1 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/GridRender.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/GridRender.java @@ -105,7 +105,7 @@ private static boolean isUnsigned(Array arr) { } private static Optional formatExtension(DType.Extension ext, Array storage, long i) { - if (!(ExtensionId.parse(ext.extensionId()) instanceof ExtensionId.WellKnown id)) { + if (!(ext.extensionId() instanceof ExtensionId.WellKnown id)) { return Optional.empty(); } return switch (id) { diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/InspectorRender.java b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/InspectorRender.java index 4d297f20a..b166f8748 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/InspectorRender.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/InspectorRender.java @@ -31,7 +31,7 @@ private InspectorRender() { /// @return formatted value, or an angle-bracketed fallback for unknown shapes static String formatValue(Array array, int i, DType declared) { if (declared instanceof DType.Extension ext - && ExtensionId.parse(ext.extensionId()) == ExtensionId.VORTEX_DATE) { + && ext.extensionId() == ExtensionId.VORTEX_DATE) { try { return DateExtensionDecoder.INSTANCE.decode(ext, array, i).toString(); } catch (RuntimeException _) { diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/SchemaCommandTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/SchemaCommandTest.java index 373f68bf8..dd305e21a 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/SchemaCommandTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/SchemaCommandTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.OpaqueMetadata; import io.github.dfa1.vortex.core.model.PType; @@ -91,8 +92,8 @@ static Stream dtypeCases() { arguments(new DType.Map(DTypes.UTF8, i64, true, false), "map"), arguments(new DType.Map(DTypes.UTF8, new DType.Primitive(PType.I64, true), false, false), "map"), - arguments(new DType.Extension("vortex.uuid", i64, OpaqueMetadata.EMPTY, false), "ext"), - arguments(new DType.Extension("vortex.uuid", i64, OpaqueMetadata.EMPTY, true), "ext?"), + arguments(new DType.Extension(ExtensionId.VORTEX_UUID, i64, OpaqueMetadata.EMPTY, false), "ext"), + arguments(new DType.Extension(ExtensionId.VORTEX_UUID, i64, OpaqueMetadata.EMPTY, true), "ext?"), arguments(DTypes.VARIANT, "variant"), arguments(new DType.Variant(true), "variant?"), arguments(new DType.Struct(List.of(ColumnName.of("a"), ColumnName.of("b")), List.of(i64, DTypes.UTF8), false), diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/VortexCliTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/VortexCliTest.java index cbbce164c..fa10e6600 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/VortexCliTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/VortexCliTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.writer.VortexWriter; import io.github.dfa1.vortex.writer.WriteOptions; @@ -160,7 +161,7 @@ void allowUnknown_doesNotLeakIntoTheNextInvocation(@TempDir Path tmp) throws IOE private static Path writeCustomExtensionVortex(Path dir) throws IOException { Path file = dir.resolve("money.vortex"); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("price")), - List.of(new DType.Extension("acme.money", DTypes.I64, + List.of(new DType.Extension(ExtensionId.parse("acme.money"), DTypes.I64, new io.github.dfa1.vortex.core.model.OpaqueMetadata( java.lang.foreign.MemorySegment.ofArray("EUR".getBytes(StandardCharsets.US_ASCII))), false)), false); diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/GridRenderTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/GridRenderTest.java index 93cd160a3..beb186436 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/GridRenderTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/GridRenderTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.TimeUnit; import io.github.dfa1.vortex.reader.array.Array; @@ -205,7 +206,7 @@ void truncatesBinaryHexBeyond16Bytes() { void extensionDecodeFailureRendersDiagnostic() { try (Arena arena = Arena.ofConfined()) { // Given — vortex.date declared but storage is F64, which epochInteger rejects - DType dateExt = new DType.Extension("vortex.date", + DType dateExt = new DType.Extension(ExtensionId.VORTEX_DATE, DTypes.I32, new io.github.dfa1.vortex.core.model.TimeUnitMetadata( io.github.dfa1.vortex.core.model.TimeUnit.Days), false); Array badStorage = ArrayFixtures.doubles(arena, 1.5); @@ -222,7 +223,7 @@ void extensionDecodeFailureRendersDiagnostic() { void unknownExtensionIdFallsThroughToStorageRendering() { try (Arena arena = Arena.ofConfined()) { // Given — an extension id this reader does not know: render the raw storage value - DType unknownExt = new DType.Extension("custom.thing", + DType unknownExt = new DType.Extension(ExtensionId.parse("custom.thing"), DTypes.I32, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); IntArray storage = ArrayFixtures.ints(arena, 42); diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/InspectorRenderTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/InspectorRenderTest.java index 8b8b483a9..cbeb91f4e 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/InspectorRenderTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/InspectorRenderTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.reader.array.Array; @@ -146,7 +147,7 @@ void rendersBinaryVarBinAsShortHex() { void dateExtensionDecodeFailureFallsThroughToGeneric() { try (Arena arena = Arena.ofConfined()) { // Given — vortex.date declared but storage is F64; the decode throws and is swallowed - DType dateExt = new DType.Extension("vortex.date", + DType dateExt = new DType.Extension(ExtensionId.VORTEX_DATE, DTypes.I32, new io.github.dfa1.vortex.core.model.TimeUnitMetadata( io.github.dfa1.vortex.core.model.TimeUnit.Days), false); Array badStorage = ArrayFixtures.doubles(arena, 1.5); diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/DType.java b/core/src/main/java/io/github/dfa1/vortex/core/model/DType.java index 062b83128..245b0c42a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/DType.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/DType.java @@ -335,12 +335,13 @@ public Struct entriesDtype() { /// Extension logical type with user-defined semantics layered over a storage type. /// - /// @param extensionId unique string identifier for the extension type (e.g. `"vortex.timestamp"`) + /// @param extensionId the extension type's id ([ExtensionId#VORTEX_TIMESTAMP], or a custom id + /// from [ExtensionId#parse(String)]) /// @param storageDType underlying storage dtype used for physical encoding /// @param metadata the extension's metadata, [OpaqueMetadata#EMPTY] for none /// @param nullable whether null values are permitted record Extension( - String extensionId, + ExtensionId extensionId, DType storageDType, ExtensionMetadata metadata, boolean nullable @@ -348,9 +349,10 @@ record Extension( /// Requires metadata that is a record, so the dtype compares and hashes by value. /// - /// @throws NullPointerException if `metadata` is `null` + /// @throws NullPointerException if `extensionId` or `metadata` is `null` /// @throws IllegalArgumentException if `metadata` is not a record public Extension { + Objects.requireNonNull(extensionId, "extensionId"); Objects.requireNonNull(metadata, "metadata"); if (!metadata.getClass().isRecord()) { throw new IllegalArgumentException( diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/TimeDtype.java b/core/src/main/java/io/github/dfa1/vortex/core/model/TimeDtype.java index cb8307fe1..69818d0dc 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/TimeDtype.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/TimeDtype.java @@ -32,7 +32,7 @@ public static DType.Extension of(TimeUnit unit, boolean nullable) { case Days -> throw new IllegalArgumentException("Days unit not valid for vortex.time"); }; return new DType.Extension( - ExtensionId.VORTEX_TIME.id(), + ExtensionId.VORTEX_TIME, new DType.Primitive(storage, nullable), new TimeUnitMetadata(unit), nullable); diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java b/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java index 7b5a89409..d769a24ed 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java @@ -28,7 +28,7 @@ public static DType.Extension of(boolean nullable) { /// @return matching extension dtype public static DType.Extension of(TimeUnit unit, ZoneId zone, boolean nullable) { return new DType.Extension( - ExtensionId.VORTEX_TIMESTAMP.id(), + ExtensionId.VORTEX_TIMESTAMP, new DType.Primitive(PType.I64, nullable), TimestampMetadata.of(unit, zone), nullable); diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java index 49efd3162..5bc0feefa 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java @@ -22,7 +22,7 @@ class Metadata { @Test void nullMetadata_rejected() { // Given / When / Then — no metadata is OpaqueMetadata.EMPTY, so a dtype always has one - assertThatThrownBy(() -> new DType.Extension("vortex.date", I32, null, false)) + assertThatThrownBy(() -> new DType.Extension(ExtensionId.VORTEX_DATE, I32, null, false)) .isInstanceOf(NullPointerException.class); } @@ -34,7 +34,7 @@ void nonRecordMetadata_rejected() { }; // When / Then - assertThatThrownBy(() -> new DType.Extension("acme.custom", I32, identity, false)) + assertThatThrownBy(() -> new DType.Extension(ExtensionId.parse("acme.custom"), I32, identity, false)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("must be a record"); } @@ -46,10 +46,10 @@ void sameOpaqueBytes_equalDtypes() { MemorySegment offHeap = Arena.ofAuto().allocate(2); offHeap.set(ValueLayout.JAVA_BYTE, 0, (byte) 1); offHeap.set(ValueLayout.JAVA_BYTE, 1, (byte) 2); - var built = new DType.Extension("acme.custom", I32, new OpaqueMetadata(MemorySegment.ofArray(new byte[]{1, 2})), false); + var built = new DType.Extension(ExtensionId.parse("acme.custom"), I32, new OpaqueMetadata(MemorySegment.ofArray(new byte[]{1, 2})), false); // When - var result = new DType.Extension("acme.custom", I32, new OpaqueMetadata(offHeap), false); + var result = new DType.Extension(ExtensionId.parse("acme.custom"), I32, new OpaqueMetadata(offHeap), false); // Then assertThat(result).isEqualTo(built).hasSameHashCodeAs(built); diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeWithNullableTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeWithNullableTest.java index 3bcf2fbbe..a4a80c596 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeWithNullableTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeWithNullableTest.java @@ -27,7 +27,7 @@ static Stream nonNullableSamples() { Arguments.of("Struct", new DType.Struct(List.of(ColumnName.of("a"), ColumnName.of("b")), List.of(i32, DTypes.UTF8), false)), Arguments.of("List", new DType.List(i32, false)), Arguments.of("FixedSizeList", new DType.FixedSizeList(i32, 4, false)), - Arguments.of("Extension", new DType.Extension("ip.address", i32, OpaqueMetadata.EMPTY, false)), + Arguments.of("Extension", new DType.Extension(ExtensionId.parse("ip.address"), i32, OpaqueMetadata.EMPTY, false)), Arguments.of("Variant", DTypes.VARIANT) ); } @@ -97,7 +97,7 @@ void withNullable_preservesCompoundPayload_struct() { void withNullable_preservesCompoundPayload_extensionAndFixedSizeList() { // Given DType storage = DTypes.I32; - DType.Extension ext = new DType.Extension("ip.address", storage, OpaqueMetadata.EMPTY, false); + DType.Extension ext = new DType.Extension(ExtensionId.parse("ip.address"), storage, OpaqueMetadata.EMPTY, false); DType.FixedSizeList fsl = new DType.FixedSizeList(storage, 16, false); // When @@ -105,7 +105,7 @@ void withNullable_preservesCompoundPayload_extensionAndFixedSizeList() { DType.FixedSizeList fslResult = (DType.FixedSizeList) fsl.withNullable(true); // Then - assertThat(extResult.extensionId()).isEqualTo("ip.address"); + assertThat(extResult.extensionId().id()).isEqualTo("ip.address"); assertThat(extResult.storageDType()).isEqualTo(storage); assertThat(extResult.metadata()).isEqualTo(OpaqueMetadata.EMPTY); assertThat(fslResult.fixedSize()).isEqualTo(16); diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/ExtensionIdTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/ExtensionIdTest.java index 0070875d6..780342568 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/ExtensionIdTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/ExtensionIdTest.java @@ -175,7 +175,7 @@ static Stream invalid() { @MethodSource("valid") void validate_validDtype_passes(String name, DType.Extension dtype) { // Given a dtype Rust accepts - ExtensionId.WellKnown sut = (ExtensionId.WellKnown) ExtensionId.parse(dtype.extensionId()); + ExtensionId.WellKnown sut = (ExtensionId.WellKnown) dtype.extensionId(); // When / Then sut.validate(dtype); @@ -185,7 +185,7 @@ void validate_validDtype_passes(String name, DType.Extension dtype) { @MethodSource("invalid") void validate_invalidDtype_throws(DType.Extension dtype, String message) { // Given a dtype Rust refuses - ExtensionId.WellKnown sut = (ExtensionId.WellKnown) ExtensionId.parse(dtype.extensionId()); + ExtensionId.WellKnown sut = (ExtensionId.WellKnown) dtype.extensionId(); // When / Then assertThatThrownBy(() -> sut.validate(dtype)) @@ -214,16 +214,16 @@ private static DType.Extension ext(ExtensionId id, DType storage, int... meta) { for (int i = 0; i < meta.length; i++) { bytes[i] = (byte) meta[i]; } - return new DType.Extension(id.id(), storage, new OpaqueMetadata(MemorySegment.ofArray(bytes)), + return new DType.Extension(id, storage, new OpaqueMetadata(MemorySegment.ofArray(bytes)), storage.nullable()); } private static DType.Extension unit(ExtensionId id, DType storage, TimeUnit unit) { - return new DType.Extension(id.id(), storage, new TimeUnitMetadata(unit), storage.nullable()); + return new DType.Extension(id, storage, new TimeUnitMetadata(unit), storage.nullable()); } private static DType.Extension timestamp(DType storage, TimeUnit unit, ZoneId zone) { - return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP.id(), storage, TimestampMetadata.of(unit, zone), + return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, storage, TimestampMetadata.of(unit, zone), storage.nullable()); } } diff --git a/docs/how-to.md b/docs/how-to.md index fe183cae4..bcacfe95b 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -1044,7 +1044,7 @@ final class MoneyEncoder implements ExtensionEncoder { static final ExtensionId ID = ExtensionId.parse("acme.money"); static DType.Extension dtype(Currency currency) { - return new DType.Extension(ID.id(), DTypes.I64, new MoneyMetadata(currency), false); + return new DType.Extension(ID, DTypes.I64, new MoneyMetadata(currency), false); } @Override diff --git a/docs/reference.md b/docs/reference.md index 54e1ce9ff..5037a8fb6 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -67,7 +67,7 @@ shortcut returning a nullable copy), `withNullable(boolean)`, `DType.Struct.fiel | Type | Shape | Notes | |------|-------|-------| | `EncodingId` | `sealed interface` — `WellKnown` enum + `Custom` record | Array-encoding identity; total `parse(String)` over non-blank ids; constants re-exported (`EncodingId.VORTEX_PRIMITIVE`, …) | -| `ExtensionId` | `sealed interface` — `WellKnown` enum + `Custom` record | Extension-type identity (`vortex.date`/`time`/`timestamp`/`uuid` well-known); same `parse(String)` and `Custom` policy as `EncodingId`. `WellKnown#validate(DType.Extension)` applies Rust's metadata and storage checks for the four spec types. `DType.Extension` still carries the raw string; the file parser rejects ids `parse` would refuse | +| `ExtensionId` | `sealed interface` — `WellKnown` enum + `Custom` record | Extension-type identity (`vortex.date`/`time`/`timestamp`/`uuid` well-known); same `parse(String)` and `Custom` policy as `EncodingId`. `WellKnown#validate(DType.Extension)` applies Rust's metadata and storage checks for the four spec types. `DType.Extension` carries the typed id; the file parser rejects ids `parse` would refuse | | `LayoutId` | `sealed interface` — `WellKnown` enum + `Custom` record | Layout identity (separate namespace from encodings; `vortex.flat` is layout-only); both zoned aliases `vortex.zoned`/`vortex.stats` | | `ColumnName` | `record ColumnName(String value)` | Validated column name: non-blank, no control characters. `ColumnName.violation(String)` is the policy chokepoint shared by builder, writer, and file parser | | `MemorySize` | `record MemorySize(long bytes)` | In `io.github.dfa1.vortex.core.io`. Validated non-negative byte count; `ofKiB`/`ofMiB`/`ofGiB` factories, `toGiB()` for display. Rejects negative values (`IllegalArgumentException`) — a programmer-error guard, not a `VortexException`-worthy untrusted-input check | diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java index ba64e3688..dcb1e9967 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java @@ -587,7 +587,7 @@ static Object readable(Object raw, DType type) { return raw; } try { - return switch (ExtensionId.parse(ext.extensionId())) { + return switch (ext.extensionId()) { case ExtensionId.WellKnown.VORTEX_TIMESTAMP -> ExtensionStorage.instantFromRaw(epoch, ExtensionStorage.readUnit(ext)); case ExtensionId.WellKnown.VORTEX_DATE -> DateExtensionDecoder.INSTANCE.date(ext, epoch); diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java index ed86f0a70..f383e972c 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java @@ -156,7 +156,7 @@ void vortexJni_withoutVtable_refusesTheFile(@TempDir Path tmp) throws IOExceptio /// The `acme.currency` dtype: `Utf8` storage of ISO 4217 codes, no metadata. static DType.Extension currencyDtype(boolean nullable) { - return new DType.Extension(CURRENCY.id(), nullable ? DTypes.UTF8.asNullable() : DTypes.UTF8, + return new DType.Extension(CURRENCY, nullable ? DTypes.UTF8.asNullable() : DTypes.UTF8, OpaqueMetadata.EMPTY, nullable); } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java index 0ca293e19..ccbd2b339 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java @@ -303,7 +303,7 @@ void create_withoutEncoder_rejectsTypedMetadata(@TempDir Path tmp) throws IOExce @Test void create_rejectsDtypeTheEncoderRefuses(@TempDir Path tmp) throws IOException { // Given — metadata says 4 dimensions, storage holds 3 - DType.Extension invalid = new DType.Extension(VECTOR.id(), + DType.Extension invalid = new DType.Extension(VECTOR, new DType.FixedSizeList(DTypes.F32, DIM, false), new VectorMetadata(4), false); Path file = tmp.resolve("invalid.vortex"); @@ -357,14 +357,14 @@ record VectorMetadata(int dimension) implements ExtensionMetadata { /// The `example.vector` dtype: `FixedSizeList` storage, dimension also in metadata. static DType.Extension vectorDtype(boolean nullable) { - return new DType.Extension(VECTOR.id(), new DType.FixedSizeList(DTypes.F32, DIM, nullable), + return new DType.Extension(VECTOR, new DType.FixedSizeList(DTypes.F32, DIM, nullable), new VectorMetadata(DIM), nullable); } /// The dtype as a writer without the encoder writes it, over 2-element vectors: metadata as the /// bytes a file stores, here declaring `metadataDim` dimensions. static DType.Extension opaqueVectorDtype(int metadataDim) { - return new DType.Extension(VECTOR.id(), new DType.FixedSizeList(DTypes.F32, 2, false), + return new DType.Extension(VECTOR, new DType.FixedSizeList(DTypes.F32, 2, false), new OpaqueMetadata(dimensionBytes(metadataDim)), false); } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java index 2e9bfb78d..1f44068b9 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java @@ -10,6 +10,7 @@ import dev.vortex.arrow.ArrowAllocation; import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.testing.OhlcData; @@ -2333,7 +2334,7 @@ void javaWriter_rustReader_extensionWithNullMetadata(@TempDir Path tmp) throws I // none, and Rust refuses an absent field ("failed to parse extension metadata") for the // whole file, even when only a sibling column is projected. Path file = tmp.resolve("java_json_null_metadata.vtx"); - DType.Extension json = new DType.Extension("vortex.json", DTypes.UTF8, + DType.Extension json = new DType.Extension(ExtensionId.VORTEX_JSON, DTypes.UTF8, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("id"), ColumnName.of("j")), List.of(DTypes.I64, json), false); @@ -2367,7 +2368,7 @@ void javaWriter_rustReader_extensionWithNullMetadata(@TempDir Path tmp) throws I void invalidBuiltInExtensionDtype_refusedByJavaAndRust(@TempDir Path tmp) throws IOException { // Given — vortex.timestamp over I32 storage, which Rust's validate_dtype refuses ("Timestamp // storage dtype must be i64"). Java used to write and read it. - DType.Extension invalid = new DType.Extension("vortex.timestamp", DTypes.I32, + DType.Extension invalid = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I32, new io.github.dfa1.vortex.core.model.TimestampMetadata(io.github.dfa1.vortex.core.model.TimeUnit.Milliseconds, null), false); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("ts")), List.of(invalid), false); @@ -2384,7 +2385,7 @@ void invalidBuiltInExtensionDtype_refusedByJavaAndRust(@TempDir Path tmp) throws // No Java write path emits it any more, so write the column under a custom id of the same // length and rename it in place: a FlatBuffer string is stored verbatim, once. DType.Struct disguised = new DType.Struct(List.of(ColumnName.of("ts")), List.of(new DType.Extension( - "vortex.timestamq", DTypes.I32, + ExtensionId.parse("vortex.timestamq"), DTypes.I32, new io.github.dfa1.vortex.core.model.OpaqueMetadata(MemorySegment.ofArray(new byte[]{2, 0, 0})), false)), false); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE, diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/WellKnownExtensionsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/WellKnownExtensionsInteropIntegrationTest.java index 98c00aac1..531ae7677 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/WellKnownExtensionsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/WellKnownExtensionsInteropIntegrationTest.java @@ -9,6 +9,7 @@ import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.OpaqueMetadata; import io.github.dfa1.vortex.reader.Chunk; @@ -56,9 +57,9 @@ class WellKnownExtensionsInteropIntegrationTest { private static final DType.Struct SCHEMA = new DType.Struct( List.of(ColumnName.of("id"), ColumnName.of("j"), ColumnName.of("g"), ColumnName.of("p")), List.of(DTypes.I64, - new DType.Extension("vortex.json", DTypes.UTF8, NO_METADATA, false), - new DType.Extension("vortex.st.wkb", DTypes.BINARY, NO_METADATA, false), - new DType.Extension("vortex.st.point", POINT_STORAGE, NO_METADATA, false)), + new DType.Extension(ExtensionId.VORTEX_JSON, DTypes.UTF8, NO_METADATA, false), + new DType.Extension(ExtensionId.VORTEX_ST_WKB, DTypes.BINARY, NO_METADATA, false), + new DType.Extension(ExtensionId.VORTEX_ST_POINT, POINT_STORAGE, NO_METADATA, false)), false); private static final int BATCHES = 2; private static final int BATCH_ROWS = 500; diff --git a/jdbc/src/main/java/io/github/dfa1/vortex/jdbc/JdbcImporter.java b/jdbc/src/main/java/io/github/dfa1/vortex/jdbc/JdbcImporter.java index e3fe6f86d..e92902bc9 100644 --- a/jdbc/src/main/java/io/github/dfa1/vortex/jdbc/JdbcImporter.java +++ b/jdbc/src/main/java/io/github/dfa1/vortex/jdbc/JdbcImporter.java @@ -218,7 +218,7 @@ private static boolean fillCell(Object buffer, int rowIdx, ResultSet rs, int col @SuppressWarnings("unchecked") private static void fillExtensionCell(List buffer, ResultSet rs, int colIdx, DType.Extension ext) throws SQLException { - if (!(ExtensionId.parse(ext.extensionId()) instanceof ExtensionId.WellKnown id)) { + if (!(ext.extensionId() instanceof ExtensionId.WellKnown id)) { throw new UnsupportedOperationException("unsupported extension: " + ext.extensionId()); } // SQL NULL → null in the buffer. Nullable extension columns round-trip through the diff --git a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java index ba0645a97..285ba5a50 100644 --- a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java +++ b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java @@ -195,7 +195,7 @@ private static void addPrimitiveColumn(FileSchema.Builder builder, String name, /// supported for export. private static void addTimestampColumn(FileSchema.Builder builder, String name, DType.Extension ext, RepetitionType rep) { - if (!ExtensionId.VORTEX_TIMESTAMP.id().equals(ext.extensionId())) { + if (ext.extensionId() != ExtensionId.VORTEX_TIMESTAMP) { throw new UnsupportedOperationException( "unsupported extension type for Parquet export: " + ext.extensionId() + " (column: " + name + ")"); } diff --git a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java index 6a9a3dcc4..99518f6f6 100644 --- a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java +++ b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java @@ -4,6 +4,7 @@ import dev.hardwood.row.PqStruct; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.writer.encode.ListData; import io.github.dfa1.vortex.writer.encode.NullableData; @@ -201,7 +202,7 @@ void f16_rejectedAtBuild() { void extensionDtype_rejectedAtConstruction() { // Given — nested Extension (timestamp) columns are unsupported; see class javadoc DType.Extension extensionDtype = new DType.Extension( - "vortex.date", new DType.Primitive(PType.I32, false), + ExtensionId.VORTEX_DATE, new DType.Primitive(PType.I32, false), new io.github.dfa1.vortex.core.model.TimeUnitMetadata(io.github.dfa1.vortex.core.model.TimeUnit.Days), false); // When / Then diff --git a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java index 4878f5a3c..19b5018a3 100644 --- a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java +++ b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java @@ -10,6 +10,7 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.TimeUnit; @@ -208,7 +209,7 @@ void timestamp_secondsOrDaysResolution_throws(TimeUnit unit) { @Test void nonTimestampExtension_throws() { // Given — a made-up extension id this exporter has no mapping for - DType.Extension ext = new DType.Extension("vortex.uuid", new DType.Primitive(PType.I64, false), + DType.Extension ext = new DType.Extension(ExtensionId.VORTEX_UUID, new DType.Primitive(PType.I64, false), io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); // When / Then diff --git a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetImporterTest.java b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetImporterTest.java index fd0ab3bde..2701df860 100644 --- a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetImporterTest.java +++ b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetImporterTest.java @@ -113,7 +113,7 @@ void int64_timestamp_mapsToTimestampExtensionOverI64(LogicalType.TimeUnit unit) // Then — vortex.timestamp extension over nullable I64 storage assertThat(result).isInstanceOf(DType.Extension.class); DType.Extension ext = (DType.Extension) result; - assertThat(ext.extensionId()).isEqualTo("vortex.timestamp"); + assertThat(ext.extensionId().id()).isEqualTo("vortex.timestamp"); assertThat(ext.storageDType()).isEqualTo(new DType.Primitive(PType.I64, true)); assertThat(ext.nullable()).isTrue(); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java b/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java index 4177ad2db..ca4fdeed6 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java @@ -6,7 +6,6 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.Mask; import io.github.dfa1.vortex.reader.array.SelectedArrays; -import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.reader.extension.ExtensionDecoder; import java.lang.foreign.Arena; @@ -220,7 +219,7 @@ public List as(ColumnName name, Class domainType) { if (!(col.dtype() instanceof DType.Extension ext)) { throw new VortexException("not an extension column: " + name); } - ExtensionDecoder decoder = registry.lookup(ExtensionId.parse(ext.extensionId())); + ExtensionDecoder decoder = registry.lookup(ext.extensionId()); if (decoder == null) { throw new VortexException("no extension decoder registered for " + ext.extensionId() + " (column '" + name + "')"); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java index 119bccde2..11c4a9019 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java @@ -325,7 +325,7 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs) { } ExtensionId id = ExtensionMetadataParser.parseId(e.id()); yield new DType.Extension( - id.id(), + id, storage, ExtensionMetadataParser.parse(id, metadata), storage.nullable()); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java index b722e0d6d..fe41f82af 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java @@ -53,7 +53,6 @@ import io.github.dfa1.vortex.reader.decode.ZstdEncodingDecoder; import io.github.dfa1.vortex.reader.extension.DateExtensionDecoder; import io.github.dfa1.vortex.reader.extension.ExtensionDecoder; -import io.github.dfa1.vortex.reader.extension.ExtensionMetadataParser; import io.github.dfa1.vortex.reader.extension.TimeExtensionDecoder; import io.github.dfa1.vortex.reader.extension.TimestampExtensionDecoder; import io.github.dfa1.vortex.reader.extension.UuidExtensionDecoder; @@ -164,7 +163,7 @@ public DType resolveExtensions(DType dtype) { return switch (dtype) { case null -> null; case DType.Extension ext -> { - ExtensionId id = ExtensionMetadataParser.parseId(ext.extensionId()); + ExtensionId id = ext.extensionId(); ExtensionDecoder decoder = extensions.get(id); DType storage = resolveExtensions(ext.storageDType()); DType.Extension resolved = ext; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java index 82c64d4fa..ad19a193f 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java @@ -129,7 +129,7 @@ static DType dtypeFromProto(io.github.dfa1.vortex.core.proto.ProtoDType proto) { ExtensionId id = ExtensionMetadataParser.parseId(proto.extension().id()); byte[] metadata = proto.extension().metadata() != null ? proto.extension().metadata() : new byte[0]; return new DType.Extension( - id.id(), + id, storage, ExtensionMetadataParser.parse(id, MemorySegment.ofArray(metadata)), storage.nullable()); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java index 6816b963c..130b6af70 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java @@ -44,7 +44,7 @@ public Class domainType() { @Override public DType.Extension dtype(boolean nullable) { return new DType.Extension( - ExtensionId.VORTEX_DATE.id(), + ExtensionId.VORTEX_DATE, new DType.Primitive(PType.I32, nullable), new io.github.dfa1.vortex.core.model.TimeUnitMetadata(TimeUnit.Days), nullable); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java index 8c7bc0ad3..414f021e0 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java @@ -49,7 +49,7 @@ public DType.Extension dtype(boolean nullable) { DType.Primitive u8 = DTypes.U8; // Rust vortex.uuid metadata: 0 bytes means "no specific version" return new DType.Extension( - ExtensionId.VORTEX_UUID.id(), + ExtensionId.VORTEX_UUID, new DType.FixedSizeList(u8, 16, nullable), io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, nullable); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java index de5d6af6b..d90fa98ce 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java @@ -196,7 +196,7 @@ void convertDType_customExtensionId_isAccepted() { // Then assertThat(result.dtype()).isInstanceOfSatisfying(DType.Extension.class, ext -> - assertThat(ext.extensionId()).isEqualTo("acme.geopoint")); + assertThat(ext.extensionId().id()).isEqualTo("acme.geopoint")); } @org.junit.jupiter.params.ParameterizedTest diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java index 9cb2bf02a..0c6fab827 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java @@ -344,7 +344,7 @@ void resolveExtensions_validateThrowsRuntimeException_wrapsInVortexException() { void resolveExtensions_wellKnownIdWithoutDecoder_passesStrictMode() { // Given — vortex.json is in the vortex-jni default session, so a strict reader must open it // (reading the column as its Utf8 storage) though Java registers no decoder for it - DType schema = new DType.Extension("vortex.json", DTypes.UTF8, OpaqueMetadata.EMPTY, false); + DType schema = new DType.Extension(ExtensionId.VORTEX_JSON, DTypes.UTF8, OpaqueMetadata.EMPTY, false); ReadRegistry sut = ReadRegistry.loadAll(); // When @@ -357,7 +357,7 @@ void resolveExtensions_wellKnownIdWithoutDecoder_passesStrictMode() { @Test void resolveExtensions_wellKnownIdWithoutDecoder_stillValidated() { // Given — no decoder, but Rust's rules for the id still apply - DType schema = new DType.Extension("vortex.json", DTypes.I32, OpaqueMetadata.EMPTY, false); + DType schema = new DType.Extension(ExtensionId.VORTEX_JSON, DTypes.I32, OpaqueMetadata.EMPTY, false); ReadRegistry sut = ReadRegistry.loadAll(); // When / Then @@ -371,7 +371,7 @@ private record Srid(int srid) implements ExtensionMetadata { } private static DType.Extension geopoint(ExtensionMetadata metadata) { - return new DType.Extension("acme.geopoint", DTypes.I32, metadata, false); + return new DType.Extension(ExtensionId.parse("acme.geopoint"), DTypes.I32, metadata, false); } private static EncodingDecoder decoder(EncodingId id) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java index b18fe9805..1383fcdb1 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.GenericArray; @@ -67,7 +68,7 @@ void signedNarrowColumnsSignExtend() { void extensionDtypePassesIntegerThrough() { // Given an integer column whose dtype is an Extension (not a Primitive) — the unsigned // widening must pass the raw value through rather than assume a primitive ptype - DType extension = new DType.Extension("test.ext", DTypes.I32, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); + DType extension = new DType.Extension(ExtensionId.parse("test.ext"), DTypes.I32, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); MemorySegment seg = ARENA.allocate(4, 4); seg.setAtIndex(VortexFormat.LE_INT, 0, -7); Array column = new MaterializedIntArray(extension, 1, seg); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java index 2d0a07d05..091f16022 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.model.OpaqueMetadata; @@ -107,13 +108,13 @@ static Stream maskableTypes() { cases.add(Arguments.of(DTypes.UTF8.asNullable(), VarBinArray.class)); cases.add(Arguments.of(DTypes.BINARY.asNullable(), VarBinArray.class)); cases.add(Arguments.of(new DType.Decimal((byte) 10, (byte) 2, true), DecimalArray.class)); - cases.add(Arguments.of(new DType.Extension("vortex.date", DTypes.I32, + cases.add(Arguments.of(new DType.Extension(ExtensionId.VORTEX_DATE, DTypes.I32, new TimeUnitMetadata(TimeUnit.Nanoseconds), true), IntArray.class)); - cases.add(Arguments.of(new DType.Extension("vortex.time", DTypes.I32, + cases.add(Arguments.of(new DType.Extension(ExtensionId.VORTEX_TIME, DTypes.I32, new TimeUnitMetadata(TimeUnit.Microseconds), true), IntArray.class)); - cases.add(Arguments.of(new DType.Extension("vortex.timestamp", DTypes.I64, + cases.add(Arguments.of(new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, new TimestampMetadata(TimeUnit.Microseconds, null), true), LongArray.class)); - cases.add(Arguments.of(new DType.Extension("acme.custom", DTypes.U16, OpaqueMetadata.EMPTY, true), + cases.add(Arguments.of(new DType.Extension(ExtensionId.parse("acme.custom"), DTypes.U16, OpaqueMetadata.EMPTY, true), ShortArray.class)); return cases.stream(); } @@ -155,7 +156,7 @@ static Stream unmaskableTypes() { new DType.Struct(List.of(ColumnName.of("a")), List.of(DTypes.I64), true), new DType.Map(DTypes.I64, DTypes.I64, false, true), new DType.Variant(true), - new DType.Extension("vortex.uuid", new DType.FixedSizeList(DTypes.U8, 16, false), OpaqueMetadata.EMPTY, true)); + new DType.Extension(ExtensionId.VORTEX_UUID, new DType.FixedSizeList(DTypes.U8, 16, false), OpaqueMetadata.EMPTY, true)); } @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java index c16cb8128..e2f4abee0 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader.decode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; @@ -35,7 +36,7 @@ private static MemorySegment i64Meta() { } private static DType timestampDType(TimeUnit unit, boolean nullable) { - return new DType.Extension("vortex.timestamp", + return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, new DType.Primitive(PType.I64, nullable), new io.github.dfa1.vortex.core.model.TimestampMetadata(unit, null), nullable); } @@ -111,7 +112,7 @@ void decode_nullableExtension_decodesNullableDaysChild() { @Test void decode_extensionMissingTimeUnitMetadata_throws() { // Given an extension whose metadata records no time unit - DType noUnit = new DType.Extension("vortex.timestamp", + DType noUnit = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); DecodeContext c = ctx(i64Meta(), noUnit, 1, TestSegments.leLongs(0L), TestSegments.leLongs(0L), TestSegments.leLongs(0L)); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java index abf7de3ff..464e75019 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java @@ -271,7 +271,7 @@ void extension_withMetadata() { // Then id, storage dtype, and metadata bytes are preserved assertThat(result).isInstanceOf(DType.Extension.class); DType.Extension ext = (DType.Extension) result; - assertThat(ext.extensionId()).isEqualTo("ip.address"); + assertThat(ext.extensionId().id()).isEqualTo("ip.address"); assertThat(ext.storageDType()).isEqualTo(DTypes.I32); assertThat(ext.metadata()).isInstanceOf(io.github.dfa1.vortex.core.model.OpaqueMetadata.class); assertThat(((io.github.dfa1.vortex.core.model.OpaqueMetadata) ext.metadata()).bytes().byteSize()).isEqualTo(3); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java index 2cee964e0..7f8808355 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java @@ -48,7 +48,7 @@ void dtype_isI32StorageWithDaysMetadata() { DType.Extension dtype = sut.dtype(false); // Then - assertThat(dtype.extensionId()).isEqualTo("vortex.date"); + assertThat(dtype.extensionId().id()).isEqualTo("vortex.date"); assertThat(dtype.storageDType()).isEqualTo(DTypes.I32); assertThat(dtype.metadata()).isEqualTo(new io.github.dfa1.vortex.core.model.TimeUnitMetadata(TimeUnit.Days)); } @@ -161,7 +161,7 @@ void date_unsupportedUnit_throws() { } private static DType.Extension dateDtype(TimeUnit unit, DType storage) { - return new DType.Extension("vortex.date", storage, + return new DType.Extension(ExtensionId.VORTEX_DATE, storage, new io.github.dfa1.vortex.core.model.TimeUnitMetadata(unit), false); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java index 51e1780ca..92a2a227b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader.extension; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.TimeUnit; @@ -102,7 +103,7 @@ void opaqueMetadata_throws() { } private DType.Extension ext(io.github.dfa1.vortex.core.model.ExtensionMetadata metadata) { - return new DType.Extension("vortex.timestamp", DTypes.I64, metadata, false); + return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, metadata, false); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java index b700b390e..bf91b22cf 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java @@ -27,7 +27,8 @@ private ExtensionTestSupport() { /// An extension dtype whose metadata is parsed from `meta` as a file's would be. static DType.Extension ext(String id, DType storage, MemorySegment meta) { - return new DType.Extension(id, storage, ExtensionMetadataParser.parse(ExtensionId.parse(id), meta), false); + ExtensionId extensionId = ExtensionId.parse(id); + return new DType.Extension(extensionId, storage, ExtensionMetadataParser.parse(extensionId, meta), false); } static MemorySegment unitByte(byte tag) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java index 1ec0da0ce..6cad8a1da 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.core.model.OpaqueMetadata; @@ -325,7 +326,7 @@ void fixedSizeListPool_throwsVortexException() { // neither VarBin- nor primitive-shaped. The fallback used to blind-cast it to VarBinArray, // which breaks the security contract (a raw ClassCastException on untrusted input). DType.FixedSizeList storage = new DType.FixedSizeList(DTypes.U8, 16, false); - DType uuid = new DType.Extension("vortex.uuid", storage, OpaqueMetadata.EMPTY, false); + DType uuid = new DType.Extension(ExtensionId.VORTEX_UUID, storage, OpaqueMetadata.EMPTY, false); Array values = new FixedSizeListArray(storage, 2, bytePool(new int[32])); Array codes = byteCodes(0, 1, 0); @@ -338,7 +339,7 @@ void fixedSizeListPool_throwsVortexException() { // ── harness ───────────────────────────────────────────────────────────────── private static Stream varBinPools() { - DType utf8Extension = new DType.Extension("ip.address", DTypes.UTF8, OpaqueMetadata.EMPTY, false); + DType utf8Extension = new DType.Extension(ExtensionId.parse("ip.address"), DTypes.UTF8, OpaqueMetadata.EMPTY, false); String[] byCode = {"ccc", "a", "bb", "a"}; String[] broadcast = {"zz", "zz", "zz", "zz"}; return Stream.of( diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java index 048457d15..6cb578a70 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java @@ -5,6 +5,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import org.junit.jupiter.api.Nested; @@ -206,7 +207,7 @@ void keepsNanCountForFloatColumn() { @Test void resolvesExtensionViaStorageDType() { // Given — ext over i32 storage; sum should widen using storage dtype (i32 → i64) - DType columnDtype = new DType.Extension("ip.address", + DType columnDtype = new DType.Extension(ExtensionId.parse("ip.address"), DTypes.I32, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); List present = List.of( ZonedStatsSchema.Stat.MIN, ZonedStatsSchema.Stat.SUM); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/DTypeFbsSerializer.java b/writer/src/main/java/io/github/dfa1/vortex/writer/DTypeFbsSerializer.java index 4eef4f951..31469fc03 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/DTypeFbsSerializer.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/DTypeFbsSerializer.java @@ -73,7 +73,7 @@ byte[] metadataBytes(DType.Extension ext) { } private byte[] serialize(DType.Extension ext) { - ExtensionId id = ExtensionId.parse(ext.extensionId()); + ExtensionId id = ext.extensionId(); byte[] bytes = switch (ext.metadata()) { case OpaqueMetadata(MemorySegment opaque) -> opaque.toArray(ValueLayout.JAVA_BYTE); case TimeUnitMetadata(TimeUnit unit) when id == ExtensionId.VORTEX_DATE || id == ExtensionId.VORTEX_TIME -> @@ -182,7 +182,7 @@ int serializeDType(FbsBuilder fbb, DType dtype) { yield FbsDType.createFbsDType(fbb, FbsType.FbsMap, inner); } case DType.Extension e -> { - int idOff = fbb.createString(e.extensionId()); + int idOff = fbb.createString(e.extensionId().id()); int storageDtypeOff = serializeDType(fbb, e.storageDType()); // Always present, empty when there is none: Rust refuses an extension dtype whose // metadata field is absent ("failed to parse extension metadata"), failing the file. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index a2db007d7..d228a6c79 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -315,8 +315,7 @@ private static Set editionExcluded( private void validateExtensions(DType dtype) { switch (dtype) { case DType.Extension ext -> { - io.github.dfa1.vortex.core.model.ExtensionId id = - io.github.dfa1.vortex.core.model.ExtensionId.parse(ext.extensionId()); + io.github.dfa1.vortex.core.model.ExtensionId id = ext.extensionId(); ExtensionEncoder encoder = extensionRegistry.lookup(id); if (encoder != null) { encoder.validate(ext); @@ -611,8 +610,7 @@ public void writeChunk(Map columns) throws IOException { // ExtEncoding wraps the storage child below — matches Rust's nested layout // (ExtEncoding → PrimitiveEncoding) and lets Registry skip its unwrap path. if (colDtype instanceof DType.Extension extDtype && data instanceof java.util.Collection coll) { - ExtensionEncoder impl = extensionRegistry.lookup( - io.github.dfa1.vortex.core.model.ExtensionId.parse(extDtype.extensionId())); + ExtensionEncoder impl = extensionRegistry.lookup(extDtype.extensionId()); if (impl == null) { // Without an encoder the Collection would reach the encoders unconverted and // fail far from the cause; raw storage arrays still pass through. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java index 974f87d7d..08faffb3e 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java @@ -35,7 +35,7 @@ public void validate(DType.Extension dtype) { @Override public DType.Extension dtype(boolean nullable) { return new DType.Extension( - ExtensionId.VORTEX_DATE.id(), + ExtensionId.VORTEX_DATE, new DType.Primitive(PType.I32, nullable), new TimeUnitMetadata(TimeUnit.Days), nullable); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java index facb69bfc..6c6af71a4 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java @@ -140,7 +140,7 @@ private static DateTimePartsData asParts(DType dtype, Object data) { return parts; } if (!(data instanceof long[] storage) || !(dtype instanceof DType.Extension ext) - || !ExtensionId.VORTEX_TIMESTAMP.id().equals(ext.extensionId()) + || ext.extensionId() != ExtensionId.VORTEX_TIMESTAMP || !(ext.metadata() instanceof TimestampMetadata(TimeUnit unit, var _)) || unit == TimeUnit.Days) { return null; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java index 1eb42b18a..33f4f50ef 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java @@ -36,7 +36,7 @@ public DType.Extension dtype(boolean nullable) { DType.Primitive u8 = DTypes.U8; // Rust vortex.uuid metadata: 0 bytes means "no specific version" return new DType.Extension( - ExtensionId.VORTEX_UUID.id(), + ExtensionId.VORTEX_UUID, new DType.FixedSizeList(u8, 16, nullable), OpaqueMetadata.EMPTY, nullable); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/DTypeFbsSerializerTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/DTypeFbsSerializerTest.java index 39c0383e0..7b0cb3fdb 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/DTypeFbsSerializerTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/DTypeFbsSerializerTest.java @@ -38,8 +38,8 @@ void builtInRecords_underTheirOwnIds_areWrittenAsTheReaderParsesThem() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().build()); // When - byte[] date = sut.metadataBytes(extension("vortex.date", new TimeUnitMetadata(TimeUnit.Days))); - byte[] timestamp = sut.metadataBytes(extension("vortex.timestamp", + byte[] date = sut.metadataBytes(extension(ExtensionId.VORTEX_DATE, new TimeUnitMetadata(TimeUnit.Days))); + byte[] timestamp = sut.metadataBytes(extension(ExtensionId.VORTEX_TIMESTAMP, new TimestampMetadata(TimeUnit.Milliseconds, "UTC"))); // Then — the unit tag; then for a timestamp the u16 LE zone length and the zone as stored @@ -56,7 +56,7 @@ void builtInRecord_underACustomId_goesThroughThatIdsEncoder() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().register(encoder).build()); // When - byte[] result = sut.metadataBytes(extension(GEOPOINT.id(), new TimeUnitMetadata(TimeUnit.Seconds))); + byte[] result = sut.metadataBytes(extension(GEOPOINT, new TimeUnitMetadata(TimeUnit.Seconds))); // Then assertThat(result).containsExactly(7); @@ -68,7 +68,7 @@ void encoderBytes_areSerializedOncePerExtension() { ExtensionEncoder encoder = encoder(GEOPOINT); given(encoder.serializeMetadata(any())).willReturn(new byte[]{1}); DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().register(encoder).build()); - DType.Extension ext = extension(GEOPOINT.id(), new Srid(4326)); + DType.Extension ext = extension(GEOPOINT, new Srid(4326)); // When sut.metadataBytes(ext); @@ -87,7 +87,7 @@ void oversizedMetadata_isRefused() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().register(encoder).build()); // When / Then - assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT.id(), new Srid(1)))) + assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT, new Srid(1)))) .isInstanceOf(VortexException.class) .hasMessageContaining("acme.geopoint metadata serializes to 65537 bytes"); } @@ -100,7 +100,7 @@ void encoderReturningNull_isRefusedNamingTheId() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().register(encoder).build()); // When / Then — a VortexException naming the extension, not an NPE inside the FlatBuffer builder - assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT.id(), new Srid(1)))) + assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT, new Srid(1)))) .isInstanceOf(VortexException.class) .hasMessageContaining("acme.geopoint encoder serialized its metadata to null"); } @@ -111,7 +111,7 @@ void customRecordWithoutEncoder_isRefused() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().build()); // When / Then - assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT.id(), new Srid(1)))) + assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT, new Srid(1)))) .isInstanceOf(VortexException.class) .hasMessageContaining("no extension encoder registered for acme.geopoint"); } @@ -123,13 +123,13 @@ void opaqueMetadata_isWrittenAsItsBytes() { byte[] bytes = "crs".getBytes(StandardCharsets.US_ASCII); // When - byte[] result = sut.metadataBytes(extension("vortex.st.wkb", new OpaqueMetadata(MemorySegment.ofArray(bytes)))); + byte[] result = sut.metadataBytes(extension(ExtensionId.VORTEX_ST_WKB, new OpaqueMetadata(MemorySegment.ofArray(bytes)))); // Then assertThat(result).isEqualTo(bytes); } - private static DType.Extension extension(String id, ExtensionMetadata metadata) { + private static DType.Extension extension(ExtensionId id, ExtensionMetadata metadata) { return new DType.Extension(id, DTypes.I64, metadata, false); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java index de2a94bae..96cb2d06e 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java @@ -8,6 +8,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.Editions; import io.github.dfa1.vortex.core.model.PType; @@ -300,7 +301,7 @@ void extensionColumn_emitsStorageMinMaxZoneMap(@TempDir Path tmp) throws IOExcep // two zones of two rows: zone 0 = [10, 11], zone 1 = [20, 21]. ExtEncoding propagates the // storage primitive's min/max, so the zone-map carries MAX+MIN+NULL_COUNT — same as I64. DType ext = new DType.Extension( - "test.ext", DTypes.I64, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); + ExtensionId.parse("test.ext"), DTypes.I64, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("t")), List.of(ext), false); WriteOptions opts = WriteOptions.cascading(0).withZoneMaps(true).withGlobalDict(false).withEdition(Editions.CORE_2025_10_0); Path file = tmp.resolve("ext.vtx"); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java index c958019db..86e26ae5a 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.writer.encode.MinMax; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.writer.encode.ComparableValues; @@ -119,7 +120,7 @@ void empty_returnsNull() { class ExtensionType { private static final DType TIMESTAMP_MS = - new DType.Extension("vortex.timestamp", DTypes.I64, + new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, new io.github.dfa1.vortex.core.model.TimestampMetadata( io.github.dfa1.vortex.core.model.TimeUnit.Nanoseconds, null), false); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java index bc63087d3..57e6192ef 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.core.model.DTypes; @@ -34,7 +35,7 @@ class DateTimePartsEncodingEncoderTest { private static final DType EXT_TIMESTAMP_NS = timestampDType(TimeUnit.Nanoseconds); private static DType timestampDType(TimeUnit unit) { - return new DType.Extension("vortex.timestamp", + return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, new io.github.dfa1.vortex.core.model.TimestampMetadata(unit, null), false); } @@ -87,7 +88,7 @@ void encode_producesThreeChildren_noBuffersAtRoot() { @Test void encode_missingMetadata_throws() { // Given - DType noMeta = new DType.Extension("vortex.timestamp", + DType noMeta = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); DateTimePartsData data = new DateTimePartsData(new long[]{0L}, false); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java index f0bb5ad9d..50c33e1ac 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.decode.ArrayNode; @@ -49,7 +50,7 @@ class ZoneMapStats { // pruning and MIN/MAX push-down purely for being a timestamp rather than a bare i64. private static final DType.Extension TIMESTAMP = - new DType.Extension("vortex.timestamp", DTypes.I64, TIMESTAMP_MS, false); + new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, TIMESTAMP_MS, false); @Test void encode_extensionColumn_reportsStorageSpaceBounds() throws Exception { @@ -98,7 +99,7 @@ class Encode { @Test void accepts_extensionDtype_returnsTrue() { // Given - DType extDType = new DType.Extension("vortex.timestamp", + DType extDType = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, TIMESTAMP_MS, false); // When / Then @@ -116,7 +117,7 @@ void encode_extensionWrappingI64_roundTrips() { // Given long[] data = {100L, 200L, 300L, 400L}; DType storageDType = DTypes.I64; - DType extDType = new DType.Extension("vortex.timestamp", storageDType, TIMESTAMP_MS, false); + DType extDType = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, storageDType, TIMESTAMP_MS, false); // When EncodeResult resultEncoded = ENCODER.encode(extDType, data, EncodeTestHelper.testCtx()); @@ -157,7 +158,7 @@ void encodeCascade_exposesStorageAsOpenChild() { // Given long[] data = {100L, 200L, 300L, 400L}; DType storageDType = DTypes.I64; - DType extDType = new DType.Extension("vortex.timestamp", storageDType, TIMESTAMP_MS, false); + DType extDType = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, storageDType, TIMESTAMP_MS, false); // When CascadeStep result = ENCODER.encodeCascade(extDType, data, EncodeTestHelper.testCtx()); @@ -195,7 +196,7 @@ void decode_extensionWrappingI64_returnsStorageArray() { MemorySegment buf = TestSegments.leLongs(values); DType storageDType = DTypes.I64; - DType extDType = new DType.Extension("vortex.timestamp", storageDType, TIMESTAMP_MS, false); + DType extDType = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, storageDType, TIMESTAMP_MS, false); ArrayNode primitiveNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); ArrayNode extNode = new ArrayNode(EncodingId.VORTEX_EXT, null, new ArrayNode[]{primitiveNode}, new int[0]); @@ -261,7 +262,7 @@ static Stream everyWellKnownExtension() { nulls)), Arguments.of(UuidExtensionEncoder.INSTANCE.dtype(true), UuidExtensionEncoder.INSTANCE.encodeAll(UuidExtensionEncoder.INSTANCE.dtype(true), nulls)), - Arguments.of(new DType.Extension("acme.custom", DTypes.U16, OpaqueMetadata.EMPTY, true), + Arguments.of(new DType.Extension(ExtensionId.parse("acme.custom"), DTypes.U16, OpaqueMetadata.EMPTY, true), new NullableData(new short[ROWS], new boolean[ROWS]))); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java index 4261770e9..04a040772 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java @@ -156,7 +156,7 @@ void encodeAll_nullInNonNullableColumn_throws() { void encodeAll_daysUnit_throws() { // Given a hand-built Days-tagged dtype (TimeDtype.of rejects Days, so build directly) DType.Extension dtype = new DType.Extension( - ExtensionId.VORTEX_TIME.id(), DTypes.I32, + ExtensionId.VORTEX_TIME, DTypes.I32, new io.github.dfa1.vortex.core.model.TimeUnitMetadata(TimeUnit.Days), false); List values = List.of(T); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsFitnessTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsFitnessTest.java index 1c476d6ed..d23deae5d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsFitnessTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsFitnessTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.writer.WriteRegistry; @@ -174,7 +175,7 @@ private static Stream boundsOwingSamples() { Arguments.of(EncodingId.VORTEX_PCO, DTypes.I64, ascending), Arguments.of(EncodingId.VORTEX_ZIGZAG, DTypes.I64, ascending), Arguments.of(EncodingId.VORTEX_EXT, - new DType.Extension("vortex.timestamp", DTypes.I64, new io.github.dfa1.vortex.core.model.TimestampMetadata( + new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, new io.github.dfa1.vortex.core.model.TimestampMetadata( io.github.dfa1.vortex.core.model.TimeUnit.Milliseconds, null), false), ascending), Arguments.of(EncodingId.VORTEX_SPARSE, DTypes.I64, sparse), Arguments.of(EncodingId.VORTEX_PATCHED, DTypes.I64, ascending), From aeaef042d932e4cd2bf8490bdfd4917b33ba5ea6 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 1 Nov 2026 15:42:16 +0100 Subject: [PATCH 3/3] fix: review findings on the ExtensionId change - RaincloudConformanceIntegrationTest compared the wire string with the typed id, which compiled and was always false, so timestamp and time columns lost their unit rescaling; it compares the ids. CI does not run the raincloud tag, so nothing caught it. - A null extension id is refused where the dtype is built, now under test. - The by-id null-row assertion moves into fullScan_decodesEveryVectorAndNull: one file write instead of two, and it still fails when the fixture stops writing nulls. - VortexWriter imports ExtensionId instead of spelling the package out; the reader's test helper takes an ExtensionId, so a mistyped id no longer becomes a custom one silently. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011fRbYrNfgxEVYukCoXYfre --- .../vortex/core/model/DTypeExtensionTest.java | 9 +++++++++ .../CustomExtensionIntegrationTest.java | 20 +++++-------------- .../RaincloudConformanceIntegrationTest.java | 4 ++-- .../extension/ExtensionTestSupport.java | 5 ++--- .../extension/TimeExtensionDecoderTest.java | 10 +++++----- .../TimestampExtensionDecoderTest.java | 4 ++-- .../dfa1/vortex/writer/VortexWriter.java | 5 +++-- 7 files changed, 28 insertions(+), 29 deletions(-) diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java index 5bc0feefa..08e6f375b 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java @@ -26,6 +26,15 @@ void nullMetadata_rejected() { .isInstanceOf(NullPointerException.class); } + @Test + void nullId_rejected() { + // Given / When / Then — refused where it is built, not later as an NPE in the serializer + // (`extensionId().id()`) or a registry lookup + assertThatThrownBy(() -> new DType.Extension(null, I32, OpaqueMetadata.EMPTY, false)) + .isInstanceOf(NullPointerException.class) + .hasMessage("extensionId"); + } + @Test void nonRecordMetadata_rejected() { // Given metadata whose equality is identity: a dtype holding it would not compare by diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java index ccbd2b339..c820f35da 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java @@ -94,7 +94,8 @@ class RoundTrip { @Test void fullScan_decodesEveryVectorAndNull(@TempDir Path tmp) throws IOException { - // Given — three batches of nullable vectors written from float[] values + // Given — three batches of nullable vectors written from float[] values, a null vector + // at every seventh row (3, 10, 17, …) Path file = tmp.resolve("vectors.vortex"); writeVectors(file, vectorDtype(true)); @@ -104,20 +105,9 @@ void fullScan_decodesEveryVectorAndNull(@TempDir Path tmp) throws IOException { // Then assertThat(result.keySet()).containsExactlyElementsOf(ids(0, ROWS)); assertEachVectorMatchesItsId(result); - } - - @Test - void nullRows_readBackAsNull(@TempDir Path tmp) throws IOException { - // Given — a nullable column written with a null vector at every seventh row (3, 10, 17, …) - Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(true)); - - // When - SequencedMap result = readVectors(file, ScanOptions.all()); - - // Then — the null rows, listed by id rather than compared with vector(): a fixture that - // stops writing nulls still agrees with vector() and passes assertEachVectorMatchesItsId, - // so only this catches it; an encoder or decoder losing the validity fails here too + // the null rows, listed by id rather than compared with vector(): a fixture that stops + // writing nulls still agrees with vector() and passes the check above, so only this + // catches it; an encoder or decoder losing the validity fails here too List nullRows = result.entrySet().stream() .filter(row -> row.getValue() == null) .map(Map.Entry::getKey) diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java index f1c9642a3..fc9e2e14b 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java @@ -236,9 +236,9 @@ private static TimeUnit[] vortexTimeUnits(Path vortex) throws IOException { TimeUnit[] units = new TimeUnit[struct.fieldTypes().size()]; for (int i = 0; i < units.length; i++) { if (struct.fieldTypes().get(i) instanceof DType.Extension ext) { - if (ExtensionId.VORTEX_TIMESTAMP.id().equals(ext.extensionId())) { + if (ext.extensionId() == ExtensionId.VORTEX_TIMESTAMP) { units[i] = TimestampDtype.readUnit(ext); - } else if (ExtensionId.VORTEX_TIME.id().equals(ext.extensionId())) { + } else if (ext.extensionId() == ExtensionId.VORTEX_TIME) { units[i] = TimeDtype.readUnit(ext); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java index bf91b22cf..c325b92e4 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java @@ -26,9 +26,8 @@ private ExtensionTestSupport() { } /// An extension dtype whose metadata is parsed from `meta` as a file's would be. - static DType.Extension ext(String id, DType storage, MemorySegment meta) { - ExtensionId extensionId = ExtensionId.parse(id); - return new DType.Extension(extensionId, storage, ExtensionMetadataParser.parse(extensionId, meta), false); + static DType.Extension ext(ExtensionId id, DType storage, MemorySegment meta) { + return new DType.Extension(id, storage, ExtensionMetadataParser.parse(id, meta), false); } static MemorySegment unitByte(byte tag) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java index a9a4a8949..0c74c0fd5 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java @@ -64,19 +64,19 @@ void decode_eachUnit() { // Given — round-trip a known time-of-day through every TimeUnit try (Arena arena = Arena.ofConfined()) { // Seconds: 3661 s = 01:01:01 - assertThat(sut.decode(ext("vortex.time", I32, unitByte((byte) 3)), + assertThat(sut.decode(ext(ExtensionId.VORTEX_TIME, I32, unitByte((byte) 3)), i32(arena, 3661), 0)) .isEqualTo(LocalTime.of(1, 1, 1)); // Milliseconds: 3_661_500 = 01:01:01.500 - assertThat(sut.decode(ext("vortex.time", I32, unitByte((byte) 2)), + assertThat(sut.decode(ext(ExtensionId.VORTEX_TIME, I32, unitByte((byte) 2)), i32(arena, 3_661_500), 0)) .isEqualTo(LocalTime.of(1, 1, 1, 500_000_000)); // Microseconds: 1_000_001 = 00:00:01.000001 - assertThat(sut.decode(ext("vortex.time", I64, unitByte((byte) 1)), + assertThat(sut.decode(ext(ExtensionId.VORTEX_TIME, I64, unitByte((byte) 1)), i64(arena, 1_000_001L), 0)) .isEqualTo(LocalTime.of(0, 0, 1, 1_000)); // Nanoseconds: 42 ns past midnight - assertThat(sut.decode(ext("vortex.time", I64, unitByte((byte) 0)), + assertThat(sut.decode(ext(ExtensionId.VORTEX_TIME, I64, unitByte((byte) 0)), i64(arena, 42L), 0)) .isEqualTo(LocalTime.ofNanoOfDay(42)); } @@ -86,7 +86,7 @@ void decode_eachUnit() { void decode_daysUnitThrows() { // Given — Days isn't a sub-second unit try (Arena arena = Arena.ofConfined()) { - DType.Extension dtype = ext("vortex.time", I32, unitByte((byte) 4)); + DType.Extension dtype = ext(ExtensionId.VORTEX_TIME, I32, unitByte((byte) 4)); IntArray storage = i32(arena, 0); // When / Then diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java index 59141af3b..9c1c1ba11 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java @@ -81,7 +81,7 @@ void instant_microsecondsPath_handlesNegativeRaw() { // Given — pre-epoch micros exercise the floorDiv / floorMod path long micros = -1_500_001L; // -1.500001s try (Arena arena = Arena.ofConfined()) { - DType.Extension dtype = ext("vortex.timestamp", I64, tzMeta((byte) 1, null)); + DType.Extension dtype = ext(ExtensionId.VORTEX_TIMESTAMP, I64, tzMeta((byte) 1, null)); // When Instant got = sut.instant(dtype, i64(arena, micros), 0); @@ -119,7 +119,7 @@ void truncatedMetadata_refusedWhenParsed() { MemorySegment meta = MemorySegment.ofArray(new byte[]{2, 5, 0, (byte) 0x55, (byte) 0x54, (byte) 0x43}); // When / Then — refused when the file's dtype is parsed, before any decoder reads the zone - assertThatThrownBy(() -> ext("vortex.timestamp", I64, meta)) + assertThatThrownBy(() -> ext(ExtensionId.VORTEX_TIMESTAMP, I64, meta)) .isInstanceOf(VortexException.class) .hasMessageContaining("truncated"); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index d228a6c79..83a6fb0a2 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -16,6 +16,7 @@ import io.github.dfa1.vortex.core.model.EditionFamily; import io.github.dfa1.vortex.core.model.Editions; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.writer.encode.EncodeContext; import io.github.dfa1.vortex.writer.encode.EncodeNode; @@ -315,11 +316,11 @@ private static Set editionExcluded( private void validateExtensions(DType dtype) { switch (dtype) { case DType.Extension ext -> { - io.github.dfa1.vortex.core.model.ExtensionId id = ext.extensionId(); + ExtensionId id = ext.extensionId(); ExtensionEncoder encoder = extensionRegistry.lookup(id); if (encoder != null) { encoder.validate(ext); - } else if (id instanceof io.github.dfa1.vortex.core.model.ExtensionId.WellKnown wellKnown) { + } else if (id instanceof ExtensionId.WellKnown wellKnown) { // written from its storage array; still held to Rust's rules for the id wellKnown.validate(ext); }