diff --git a/AGENTS.md b/AGENTS.md index 4f8adca17..bae6089ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -302,7 +302,7 @@ When stuck on encode/decode behavior, consult **in this order**: ## Design decisions - **DType is pluggable only via `Extension`.** `DType` is a sealed interface; downstream code must - not add variants. Use `new DType.Extension("ip.address", new DType.Primitive(PType.I32, false), + not add variants. Use `new DType.Extension(ExtensionId.parse("ip.address"), new DType.Primitive(PType.I32, false), OpaqueMetadata.EMPTY, false)` and register an `ExtensionDecoder`/`ExtensionEncoder` on `ReadRegistry`/`WriteRegistry`. Mirrors Rust's `ExtVTable` (`vortex.date`, `vortex.uuid`, …): an unregistered id fails the read unless `allowUnknown()`, then reads as storage. No SPI for DType diff --git a/CHANGELOG.md b/CHANGELOG.md index 89b3e563e..2bbe556ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** CSV import stores an empty cell in a number or boolean column as null instead of `0` or `false`, so inferred number and boolean columns are now nullable (read through `MaskedArray`), and an empty cell or a missing column fails the import where the schema declares the column non-nullable. ([#592](https://github.com/dfa1/vortex-java/issues/592)) - **Breaking:** the shared non-nullable dtypes move from `DType` to the new `DTypes` class: `DTypes.I64`, `DTypes.UTF8`, … ([#624](https://github.com/dfa1/vortex-java/pull/624)) - **Breaking:** an extension dtype's metadata is a typed `ExtensionMetadata` record instead of a `MemorySegment`, parsed when a file opens and serialized when it is written, as Rust does: `TimeUnitMetadata` for date and time, `TimestampMetadata` (its zone kept as stored) for timestamp, `OpaqueMetadata` for bytes nothing parses, and a third-party record through `ExtensionDecoder#parseMetadata` and `ExtensionEncoder#serializeMetadata`. ([#625](https://github.com/dfa1/vortex-java/pull/625)) +- **Breaking:** `DType.Extension#extensionId()` is an `ExtensionId` instead of a `String`; build one with `ExtensionId.VORTEX_DATE` or `ExtensionId.parse("acme.money")`. ([#626](https://github.com/dfa1/vortex-java/pull/626)) - **Breaking:** `MemorySize` moves from `io.github.dfa1.vortex.core.model` to `io.github.dfa1.vortex.core.io`. ([#625](https://github.com/dfa1/vortex-java/pull/625)) ### Fixed diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/Temporal.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/Temporal.java index 1bf168b48..ed86653a9 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/Temporal.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/Temporal.java @@ -31,7 +31,7 @@ record Temporal(ExtensionId.WellKnown kind, TimeUnit unit) { /// @throws VortexException for a unit the extension does not allow, as the decoders do static Temporal of(DType type) { if (!(type instanceof DType.Extension ext) - || !(ExtensionId.parse(ext.extensionId()) instanceof ExtensionId.WellKnown kind) + || !(ext.extensionId() instanceof ExtensionId.WellKnown kind) || kind != ExtensionId.WellKnown.VORTEX_DATE && kind != ExtensionId.WellKnown.VORTEX_TIME && kind != ExtensionId.WellKnown.VORTEX_TIMESTAMP) { return null; diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexEnumerator.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexEnumerator.java index da5e38219..9087cf210 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexEnumerator.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexEnumerator.java @@ -268,7 +268,7 @@ private static Object value(Object array, DType type, long r) { case DType.Decimal _ -> ((DecimalArray) array).getDecimal(r); // Calcite's internal forms: DATE is days since the epoch (int), TIME milliseconds of the // day (int), TIMESTAMP milliseconds since the epoch (long) - case DType.Extension ext -> switch (ExtensionId.parse(ext.extensionId())) { + case DType.Extension ext -> switch (ext.extensionId()) { case ExtensionId.WellKnown.VORTEX_DATE -> Math.toIntExact(DateExtensionDecoder.INSTANCE.decode(ext, (Array) array, r).toEpochDay()); case ExtensionId.WellKnown.VORTEX_TIME -> diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java index 4f122ef27..945328267 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java @@ -1557,7 +1557,7 @@ private static RelDataType toSqlType(RelDataTypeFactory factory, DType type) { case DType.Bool _ -> factory.createSqlType(SqlTypeName.BOOLEAN); case DType.Null _ -> factory.createSqlType(SqlTypeName.NULL); case DType.Decimal d -> factory.createSqlType(SqlTypeName.DECIMAL, d.precision(), d.scale()); - case DType.Extension ext -> switch (ExtensionId.parse(ext.extensionId())) { + case DType.Extension ext -> switch (ext.extensionId()) { case ExtensionId.WellKnown.VORTEX_DATE -> factory.createSqlType(SqlTypeName.DATE); // millisecond precision, the precision the scan delivers: at Calcite's default of 0 a // literal built for the column (a pushed-down MIN/MAX, a top-N row) lost its millis diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/GridRender.java b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/GridRender.java index 39b3c76a5..0f46379d1 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/GridRender.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/GridRender.java @@ -105,7 +105,7 @@ private static boolean isUnsigned(Array arr) { } private static Optional formatExtension(DType.Extension ext, Array storage, long i) { - if (!(ExtensionId.parse(ext.extensionId()) instanceof ExtensionId.WellKnown id)) { + if (!(ext.extensionId() instanceof ExtensionId.WellKnown id)) { return Optional.empty(); } return switch (id) { diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/InspectorRender.java b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/InspectorRender.java index 4d297f20a..b166f8748 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/InspectorRender.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/InspectorRender.java @@ -31,7 +31,7 @@ private InspectorRender() { /// @return formatted value, or an angle-bracketed fallback for unknown shapes static String formatValue(Array array, int i, DType declared) { if (declared instanceof DType.Extension ext - && ExtensionId.parse(ext.extensionId()) == ExtensionId.VORTEX_DATE) { + && ext.extensionId() == ExtensionId.VORTEX_DATE) { try { return DateExtensionDecoder.INSTANCE.decode(ext, array, i).toString(); } catch (RuntimeException _) { diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/SchemaCommandTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/SchemaCommandTest.java index 373f68bf8..dd305e21a 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/SchemaCommandTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/SchemaCommandTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.OpaqueMetadata; import io.github.dfa1.vortex.core.model.PType; @@ -91,8 +92,8 @@ static Stream dtypeCases() { arguments(new DType.Map(DTypes.UTF8, i64, true, false), "map"), arguments(new DType.Map(DTypes.UTF8, new DType.Primitive(PType.I64, true), false, false), "map"), - arguments(new DType.Extension("vortex.uuid", i64, OpaqueMetadata.EMPTY, false), "ext"), - arguments(new DType.Extension("vortex.uuid", i64, OpaqueMetadata.EMPTY, true), "ext?"), + arguments(new DType.Extension(ExtensionId.VORTEX_UUID, i64, OpaqueMetadata.EMPTY, false), "ext"), + arguments(new DType.Extension(ExtensionId.VORTEX_UUID, i64, OpaqueMetadata.EMPTY, true), "ext?"), arguments(DTypes.VARIANT, "variant"), arguments(new DType.Variant(true), "variant?"), arguments(new DType.Struct(List.of(ColumnName.of("a"), ColumnName.of("b")), List.of(i64, DTypes.UTF8), false), diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/VortexCliTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/VortexCliTest.java index cbbce164c..fa10e6600 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/VortexCliTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/VortexCliTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.writer.VortexWriter; import io.github.dfa1.vortex.writer.WriteOptions; @@ -160,7 +161,7 @@ void allowUnknown_doesNotLeakIntoTheNextInvocation(@TempDir Path tmp) throws IOE private static Path writeCustomExtensionVortex(Path dir) throws IOException { Path file = dir.resolve("money.vortex"); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("price")), - List.of(new DType.Extension("acme.money", DTypes.I64, + List.of(new DType.Extension(ExtensionId.parse("acme.money"), DTypes.I64, new io.github.dfa1.vortex.core.model.OpaqueMetadata( java.lang.foreign.MemorySegment.ofArray("EUR".getBytes(StandardCharsets.US_ASCII))), false)), false); diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/GridRenderTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/GridRenderTest.java index 93cd160a3..beb186436 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/GridRenderTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/GridRenderTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.TimeUnit; import io.github.dfa1.vortex.reader.array.Array; @@ -205,7 +206,7 @@ void truncatesBinaryHexBeyond16Bytes() { void extensionDecodeFailureRendersDiagnostic() { try (Arena arena = Arena.ofConfined()) { // Given — vortex.date declared but storage is F64, which epochInteger rejects - DType dateExt = new DType.Extension("vortex.date", + DType dateExt = new DType.Extension(ExtensionId.VORTEX_DATE, DTypes.I32, new io.github.dfa1.vortex.core.model.TimeUnitMetadata( io.github.dfa1.vortex.core.model.TimeUnit.Days), false); Array badStorage = ArrayFixtures.doubles(arena, 1.5); @@ -222,7 +223,7 @@ void extensionDecodeFailureRendersDiagnostic() { void unknownExtensionIdFallsThroughToStorageRendering() { try (Arena arena = Arena.ofConfined()) { // Given — an extension id this reader does not know: render the raw storage value - DType unknownExt = new DType.Extension("custom.thing", + DType unknownExt = new DType.Extension(ExtensionId.parse("custom.thing"), DTypes.I32, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); IntArray storage = ArrayFixtures.ints(arena, 42); diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/InspectorRenderTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/InspectorRenderTest.java index 8b8b483a9..cbeb91f4e 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/InspectorRenderTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/InspectorRenderTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.reader.array.Array; @@ -146,7 +147,7 @@ void rendersBinaryVarBinAsShortHex() { void dateExtensionDecodeFailureFallsThroughToGeneric() { try (Arena arena = Arena.ofConfined()) { // Given — vortex.date declared but storage is F64; the decode throws and is swallowed - DType dateExt = new DType.Extension("vortex.date", + DType dateExt = new DType.Extension(ExtensionId.VORTEX_DATE, DTypes.I32, new io.github.dfa1.vortex.core.model.TimeUnitMetadata( io.github.dfa1.vortex.core.model.TimeUnit.Days), false); Array badStorage = ArrayFixtures.doubles(arena, 1.5); diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/DType.java b/core/src/main/java/io/github/dfa1/vortex/core/model/DType.java index 062b83128..245b0c42a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/DType.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/DType.java @@ -335,12 +335,13 @@ public Struct entriesDtype() { /// Extension logical type with user-defined semantics layered over a storage type. /// - /// @param extensionId unique string identifier for the extension type (e.g. `"vortex.timestamp"`) + /// @param extensionId the extension type's id ([ExtensionId#VORTEX_TIMESTAMP], or a custom id + /// from [ExtensionId#parse(String)]) /// @param storageDType underlying storage dtype used for physical encoding /// @param metadata the extension's metadata, [OpaqueMetadata#EMPTY] for none /// @param nullable whether null values are permitted record Extension( - String extensionId, + ExtensionId extensionId, DType storageDType, ExtensionMetadata metadata, boolean nullable @@ -348,9 +349,10 @@ record Extension( /// Requires metadata that is a record, so the dtype compares and hashes by value. /// - /// @throws NullPointerException if `metadata` is `null` + /// @throws NullPointerException if `extensionId` or `metadata` is `null` /// @throws IllegalArgumentException if `metadata` is not a record public Extension { + Objects.requireNonNull(extensionId, "extensionId"); Objects.requireNonNull(metadata, "metadata"); if (!metadata.getClass().isRecord()) { throw new IllegalArgumentException( diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/TimeDtype.java b/core/src/main/java/io/github/dfa1/vortex/core/model/TimeDtype.java index cb8307fe1..69818d0dc 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/TimeDtype.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/TimeDtype.java @@ -32,7 +32,7 @@ public static DType.Extension of(TimeUnit unit, boolean nullable) { case Days -> throw new IllegalArgumentException("Days unit not valid for vortex.time"); }; return new DType.Extension( - ExtensionId.VORTEX_TIME.id(), + ExtensionId.VORTEX_TIME, new DType.Primitive(storage, nullable), new TimeUnitMetadata(unit), nullable); diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java b/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java index 7b5a89409..d769a24ed 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java @@ -28,7 +28,7 @@ public static DType.Extension of(boolean nullable) { /// @return matching extension dtype public static DType.Extension of(TimeUnit unit, ZoneId zone, boolean nullable) { return new DType.Extension( - ExtensionId.VORTEX_TIMESTAMP.id(), + ExtensionId.VORTEX_TIMESTAMP, new DType.Primitive(PType.I64, nullable), TimestampMetadata.of(unit, zone), nullable); diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java index 49efd3162..08e6f375b 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeExtensionTest.java @@ -22,10 +22,19 @@ class Metadata { @Test void nullMetadata_rejected() { // Given / When / Then — no metadata is OpaqueMetadata.EMPTY, so a dtype always has one - assertThatThrownBy(() -> new DType.Extension("vortex.date", I32, null, false)) + assertThatThrownBy(() -> new DType.Extension(ExtensionId.VORTEX_DATE, I32, null, false)) .isInstanceOf(NullPointerException.class); } + @Test + void nullId_rejected() { + // Given / When / Then — refused where it is built, not later as an NPE in the serializer + // (`extensionId().id()`) or a registry lookup + assertThatThrownBy(() -> new DType.Extension(null, I32, OpaqueMetadata.EMPTY, false)) + .isInstanceOf(NullPointerException.class) + .hasMessage("extensionId"); + } + @Test void nonRecordMetadata_rejected() { // Given metadata whose equality is identity: a dtype holding it would not compare by @@ -34,7 +43,7 @@ void nonRecordMetadata_rejected() { }; // When / Then - assertThatThrownBy(() -> new DType.Extension("acme.custom", I32, identity, false)) + assertThatThrownBy(() -> new DType.Extension(ExtensionId.parse("acme.custom"), I32, identity, false)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("must be a record"); } @@ -46,10 +55,10 @@ void sameOpaqueBytes_equalDtypes() { MemorySegment offHeap = Arena.ofAuto().allocate(2); offHeap.set(ValueLayout.JAVA_BYTE, 0, (byte) 1); offHeap.set(ValueLayout.JAVA_BYTE, 1, (byte) 2); - var built = new DType.Extension("acme.custom", I32, new OpaqueMetadata(MemorySegment.ofArray(new byte[]{1, 2})), false); + var built = new DType.Extension(ExtensionId.parse("acme.custom"), I32, new OpaqueMetadata(MemorySegment.ofArray(new byte[]{1, 2})), false); // When - var result = new DType.Extension("acme.custom", I32, new OpaqueMetadata(offHeap), false); + var result = new DType.Extension(ExtensionId.parse("acme.custom"), I32, new OpaqueMetadata(offHeap), false); // Then assertThat(result).isEqualTo(built).hasSameHashCodeAs(built); diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeWithNullableTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeWithNullableTest.java index 3bcf2fbbe..a4a80c596 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeWithNullableTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/DTypeWithNullableTest.java @@ -27,7 +27,7 @@ static Stream nonNullableSamples() { Arguments.of("Struct", new DType.Struct(List.of(ColumnName.of("a"), ColumnName.of("b")), List.of(i32, DTypes.UTF8), false)), Arguments.of("List", new DType.List(i32, false)), Arguments.of("FixedSizeList", new DType.FixedSizeList(i32, 4, false)), - Arguments.of("Extension", new DType.Extension("ip.address", i32, OpaqueMetadata.EMPTY, false)), + Arguments.of("Extension", new DType.Extension(ExtensionId.parse("ip.address"), i32, OpaqueMetadata.EMPTY, false)), Arguments.of("Variant", DTypes.VARIANT) ); } @@ -97,7 +97,7 @@ void withNullable_preservesCompoundPayload_struct() { void withNullable_preservesCompoundPayload_extensionAndFixedSizeList() { // Given DType storage = DTypes.I32; - DType.Extension ext = new DType.Extension("ip.address", storage, OpaqueMetadata.EMPTY, false); + DType.Extension ext = new DType.Extension(ExtensionId.parse("ip.address"), storage, OpaqueMetadata.EMPTY, false); DType.FixedSizeList fsl = new DType.FixedSizeList(storage, 16, false); // When @@ -105,7 +105,7 @@ void withNullable_preservesCompoundPayload_extensionAndFixedSizeList() { DType.FixedSizeList fslResult = (DType.FixedSizeList) fsl.withNullable(true); // Then - assertThat(extResult.extensionId()).isEqualTo("ip.address"); + assertThat(extResult.extensionId().id()).isEqualTo("ip.address"); assertThat(extResult.storageDType()).isEqualTo(storage); assertThat(extResult.metadata()).isEqualTo(OpaqueMetadata.EMPTY); assertThat(fslResult.fixedSize()).isEqualTo(16); diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/ExtensionIdTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/ExtensionIdTest.java index 0070875d6..780342568 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/ExtensionIdTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/ExtensionIdTest.java @@ -175,7 +175,7 @@ static Stream invalid() { @MethodSource("valid") void validate_validDtype_passes(String name, DType.Extension dtype) { // Given a dtype Rust accepts - ExtensionId.WellKnown sut = (ExtensionId.WellKnown) ExtensionId.parse(dtype.extensionId()); + ExtensionId.WellKnown sut = (ExtensionId.WellKnown) dtype.extensionId(); // When / Then sut.validate(dtype); @@ -185,7 +185,7 @@ void validate_validDtype_passes(String name, DType.Extension dtype) { @MethodSource("invalid") void validate_invalidDtype_throws(DType.Extension dtype, String message) { // Given a dtype Rust refuses - ExtensionId.WellKnown sut = (ExtensionId.WellKnown) ExtensionId.parse(dtype.extensionId()); + ExtensionId.WellKnown sut = (ExtensionId.WellKnown) dtype.extensionId(); // When / Then assertThatThrownBy(() -> sut.validate(dtype)) @@ -214,16 +214,16 @@ private static DType.Extension ext(ExtensionId id, DType storage, int... meta) { for (int i = 0; i < meta.length; i++) { bytes[i] = (byte) meta[i]; } - return new DType.Extension(id.id(), storage, new OpaqueMetadata(MemorySegment.ofArray(bytes)), + return new DType.Extension(id, storage, new OpaqueMetadata(MemorySegment.ofArray(bytes)), storage.nullable()); } private static DType.Extension unit(ExtensionId id, DType storage, TimeUnit unit) { - return new DType.Extension(id.id(), storage, new TimeUnitMetadata(unit), storage.nullable()); + return new DType.Extension(id, storage, new TimeUnitMetadata(unit), storage.nullable()); } private static DType.Extension timestamp(DType storage, TimeUnit unit, ZoneId zone) { - return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP.id(), storage, TimestampMetadata.of(unit, zone), + return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, storage, TimestampMetadata.of(unit, zone), storage.nullable()); } } diff --git a/docs/how-to.md b/docs/how-to.md index fe183cae4..bcacfe95b 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -1044,7 +1044,7 @@ final class MoneyEncoder implements ExtensionEncoder { static final ExtensionId ID = ExtensionId.parse("acme.money"); static DType.Extension dtype(Currency currency) { - return new DType.Extension(ID.id(), DTypes.I64, new MoneyMetadata(currency), false); + return new DType.Extension(ID, DTypes.I64, new MoneyMetadata(currency), false); } @Override diff --git a/docs/reference.md b/docs/reference.md index 54e1ce9ff..5037a8fb6 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -67,7 +67,7 @@ shortcut returning a nullable copy), `withNullable(boolean)`, `DType.Struct.fiel | Type | Shape | Notes | |------|-------|-------| | `EncodingId` | `sealed interface` — `WellKnown` enum + `Custom` record | Array-encoding identity; total `parse(String)` over non-blank ids; constants re-exported (`EncodingId.VORTEX_PRIMITIVE`, …) | -| `ExtensionId` | `sealed interface` — `WellKnown` enum + `Custom` record | Extension-type identity (`vortex.date`/`time`/`timestamp`/`uuid` well-known); same `parse(String)` and `Custom` policy as `EncodingId`. `WellKnown#validate(DType.Extension)` applies Rust's metadata and storage checks for the four spec types. `DType.Extension` still carries the raw string; the file parser rejects ids `parse` would refuse | +| `ExtensionId` | `sealed interface` — `WellKnown` enum + `Custom` record | Extension-type identity (`vortex.date`/`time`/`timestamp`/`uuid` well-known); same `parse(String)` and `Custom` policy as `EncodingId`. `WellKnown#validate(DType.Extension)` applies Rust's metadata and storage checks for the four spec types. `DType.Extension` carries the typed id; the file parser rejects ids `parse` would refuse | | `LayoutId` | `sealed interface` — `WellKnown` enum + `Custom` record | Layout identity (separate namespace from encodings; `vortex.flat` is layout-only); both zoned aliases `vortex.zoned`/`vortex.stats` | | `ColumnName` | `record ColumnName(String value)` | Validated column name: non-blank, no control characters. `ColumnName.violation(String)` is the policy chokepoint shared by builder, writer, and file parser | | `MemorySize` | `record MemorySize(long bytes)` | In `io.github.dfa1.vortex.core.io`. Validated non-negative byte count; `ofKiB`/`ofMiB`/`ofGiB` factories, `toGiB()` for display. Rejects negative values (`IllegalArgumentException`) — a programmer-error guard, not a `VortexException`-worthy untrusted-input check | diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java index ba64e3688..dcb1e9967 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java @@ -587,7 +587,7 @@ static Object readable(Object raw, DType type) { return raw; } try { - return switch (ExtensionId.parse(ext.extensionId())) { + return switch (ext.extensionId()) { case ExtensionId.WellKnown.VORTEX_TIMESTAMP -> ExtensionStorage.instantFromRaw(epoch, ExtensionStorage.readUnit(ext)); case ExtensionId.WellKnown.VORTEX_DATE -> DateExtensionDecoder.INSTANCE.date(ext, epoch); diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java index 3f52b8f34..f383e972c 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CurrencyExtensionIntegrationTest.java @@ -135,7 +135,8 @@ void allowUnknown_readsColumnAsItsCodes(@TempDir Path tmp) throws IOException { } // Then - assertThat(firstCode).isEqualTo(currency(1).getCurrencyCode()); + // row 1 is the second of the four cycling currencies + assertThat(firstCode).isEqualTo("USD"); } @Test @@ -155,7 +156,7 @@ void vortexJni_withoutVtable_refusesTheFile(@TempDir Path tmp) throws IOExceptio /// The `acme.currency` dtype: `Utf8` storage of ISO 4217 codes, no metadata. static DType.Extension currencyDtype(boolean nullable) { - return new DType.Extension(CURRENCY.id(), nullable ? DTypes.UTF8.asNullable() : DTypes.UTF8, + return new DType.Extension(CURRENCY, nullable ? DTypes.UTF8.asNullable() : DTypes.UTF8, OpaqueMetadata.EMPTY, nullable); } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java index 27f66e8a2..c820f35da 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CustomExtensionIntegrationTest.java @@ -40,8 +40,8 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -94,9 +94,10 @@ class RoundTrip { @Test void fullScan_decodesEveryVectorAndNull(@TempDir Path tmp) throws IOException { - // Given — three batches of nullable vectors written from float[] values + // Given — three batches of nullable vectors written from float[] values, a null vector + // at every seventh row (3, 10, 17, …) Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, true)); + writeVectors(file, vectorDtype(true)); // When SequencedMap result = readVectors(file, ScanOptions.all()); @@ -104,13 +105,21 @@ void fullScan_decodesEveryVectorAndNull(@TempDir Path tmp) throws IOException { // Then assertThat(result.keySet()).containsExactlyElementsOf(ids(0, ROWS)); assertEachVectorMatchesItsId(result); + // the null rows, listed by id rather than compared with vector(): a fixture that stops + // writing nulls still agrees with vector() and passes the check above, so only this + // catches it; an encoder or decoder losing the validity fails here too + List nullRows = result.entrySet().stream() + .filter(row -> row.getValue() == null) + .map(Map.Entry::getKey) + .toList(); + assertThat(nullRows).isEqualTo(LongStream.range(0, ROWS).filter(row -> row % 7 == 3).boxed().toList()); } @Test void schema_keepsIdStorageAndMetadata(@TempDir Path tmp) throws IOException { // Given Path file = tmp.resolve("vectors.vortex"); - DType.Extension written = vectorDtype(DIM, true); + DType.Extension written = vectorDtype(true); writeVectors(file, written); DType result; @@ -131,7 +140,7 @@ void filteredScan_keepsEachVectorWithItsRow(@TempDir Path tmp) throws IOExceptio // cannot match, so the result is a superset of the matching rows; every vector must // still sit next to its own id. Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, true)); + writeVectors(file, vectorDtype(true)); long threshold = CHUNK_ROWS + CHUNK_ROWS / 2; // When @@ -148,7 +157,7 @@ void filteredScan_keepsEachVectorWithItsRow(@TempDir Path tmp) throws IOExceptio void limitedScan_returnsTheFirstRows(@TempDir Path tmp) throws IOException { // Given Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, true)); + writeVectors(file, vectorDtype(true)); int limit = CHUNK_ROWS + 7; // When @@ -164,7 +173,7 @@ void storageArray_withoutEncoder_isWrittenAsIs(@TempDir Path tmp) throws IOExcep // Given — no encoder registered: the caller passes the storage shape itself and the // metadata as the bytes a file stores, as for any extension id the writer does not know Path file = tmp.resolve("raw.vortex"); - DType.Struct schema = schema(opaqueVectorDtype(2, 2)); + DType.Struct schema = schema(opaqueVectorDtype(2)); float[] flat = {1f, 2f, 3f, 4f}; try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); var sut = VortexWriter.create(ch, schema, WriteOptions.defaults())) { @@ -186,11 +195,11 @@ void open_withoutDecoder_throws(@TempDir Path tmp) throws IOException { // Given — a reader with only the built-ins. Rust refuses an extension id its session // has no vtable for ("No such DType extension ID") unless unknown ids are allowed. Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); ReadRegistry builtInsOnly = ReadRegistry.loadAll(); // When / Then - assertThatThrownBy(() -> VortexReader.open(file, builtInsOnly)) + assertThatThrownBy(() -> VortexReader.open(file, builtInsOnly).close()) .isInstanceOf(VortexException.class) .hasMessageContaining("no extension decoder registered for example.vector"); } @@ -200,7 +209,7 @@ void allowUnknown_readsColumnAsStorageArray(@TempDir Path tmp) throws IOExceptio // Given — with unknown ids allowed the column reads as its fixed-size-list storage, // Rust's ForeignExtDType Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); ReadRegistry allowUnknown = ReadRegistry.builder().registerDefaults().allowUnknown().build(); // When @@ -208,19 +217,20 @@ void allowUnknown_readsColumnAsStorageArray(@TempDir Path tmp) throws IOExceptio try (VortexReader reader = VortexReader.open(file, allowUnknown); var iter = reader.scan(ScanOptions.all().withLimit(ScanLimit.of(2))); Chunk chunk = iter.next()) { - FixedSizeListArray storage = (FixedSizeListArray) chunk.column(EMBEDDING); + FixedSizeListArray storage = chunk.column(EMBEDDING); firstElementOfRow1 = ((FloatArray) storage.elements()).getFloat(DIM); } // Then - assertThat(firstElementOfRow1).isEqualTo(vector(1)[0]); + // row 1's vector is (1, -1.25, 1.5): its first element is the row number + assertThat(firstElementOfRow1).isEqualTo(1f); } @Test void as_withoutDecoder_throws(@TempDir Path tmp) throws IOException { // Given Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); ReadRegistry allowUnknown = ReadRegistry.builder().registerDefaults().allowUnknown().build(); try (VortexReader reader = VortexReader.open(file, allowUnknown); @@ -237,7 +247,7 @@ void as_withoutDecoder_throws(@TempDir Path tmp) throws IOException { void as_wrongDomainType_throws(@TempDir Path tmp) throws IOException { // Given Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); try (VortexReader reader = VortexReader.open(file, READ_REGISTRY); var iter = reader.scan(ScanOptions.all().withLimit(ScanLimit.of(2))); @@ -254,7 +264,7 @@ void vortexJni_withoutVtable_refusesTheFile(@TempDir Path tmp) throws IOExceptio // Given — the Rust reference, which has no example.vector vtable and whose JNI session // exposes no allow_unknown: the behavior the strict default above mirrors Path file = tmp.resolve("vectors.vortex"); - writeVectors(file, vectorDtype(DIM, false)); + writeVectors(file, vectorDtype(false)); String uri = file.toAbsolutePath().toUri().toString(); // When / Then @@ -274,7 +284,7 @@ void create_withoutEncoder_rejectsTypedMetadata(@TempDir Path tmp) throws IOExce try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { // When / Then - assertThatThrownBy(() -> VortexWriter.create(ch, schema(vectorDtype(DIM, false)), WriteOptions.defaults())) + assertThatThrownBy(() -> VortexWriter.create(ch, schema(vectorDtype(false)), WriteOptions.defaults())) .isInstanceOf(VortexException.class) .hasMessageContaining("no extension encoder registered for example.vector"); } @@ -283,7 +293,7 @@ void create_withoutEncoder_rejectsTypedMetadata(@TempDir Path tmp) throws IOExce @Test void create_rejectsDtypeTheEncoderRefuses(@TempDir Path tmp) throws IOException { // Given — metadata says 4 dimensions, storage holds 3 - DType.Extension invalid = new DType.Extension(VECTOR.id(), + DType.Extension invalid = new DType.Extension(VECTOR, new DType.FixedSizeList(DTypes.F32, DIM, false), new VectorMetadata(4), false); Path file = tmp.resolve("invalid.vortex"); @@ -298,7 +308,7 @@ void create_rejectsDtypeTheEncoderRefuses(@TempDir Path tmp) throws IOException @Test void open_rejectsDtypeTheDecoderRefuses(@TempDir Path tmp) throws IOException { // Given — a file from a writer that had no encoder, so nothing validated the dtype - DType.Extension invalid = opaqueVectorDtype(2, 5); + DType.Extension invalid = opaqueVectorDtype(5); Path file = tmp.resolve("invalid.vortex"); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); var writer = VortexWriter.create(ch, schema(invalid), WriteOptions.defaults())) { @@ -306,7 +316,7 @@ void open_rejectsDtypeTheDecoderRefuses(@TempDir Path tmp) throws IOException { } // When / Then — Rust validates an extension dtype when it deserializes it - assertThatThrownBy(() -> VortexReader.open(file, READ_REGISTRY)) + assertThatThrownBy(() -> VortexReader.open(file, READ_REGISTRY).close()) .isInstanceOf(VortexException.class) .hasMessageContaining("example.vector metadata dimension 5 != storage size 2"); } @@ -316,10 +326,10 @@ void writeChunk_nullInNonNullableColumn_throws(@TempDir Path tmp) throws IOExcep // Given Path file = tmp.resolve("nonnull.vortex"); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); - var sut = VortexWriter.create(ch, schema(vectorDtype(DIM, false)), WriteOptions.defaults(), WRITE_REGISTRY)) { + var sut = VortexWriter.create(ch, schema(vectorDtype(false)), WriteOptions.defaults(), WRITE_REGISTRY)) { // When / Then assertThatThrownBy(() -> sut.writeChunk(Map.of(ID, new long[]{0L}, - EMBEDDING, Arrays.asList((float[]) null)))) + EMBEDDING, Collections.singletonList((float[]) null)))) .isInstanceOf(VortexException.class) .hasMessageContaining("null element in non-nullable example.vector column"); } @@ -335,15 +345,16 @@ void writeChunk_nullInNonNullableColumn_throws(@TempDir Path tmp) throws IOExcep record VectorMetadata(int dimension) implements ExtensionMetadata { } - /// The `example.vector` dtype: `FixedSizeList` storage, dimension also in metadata. - static DType.Extension vectorDtype(int dim, boolean nullable) { - return new DType.Extension(VECTOR.id(), new DType.FixedSizeList(DTypes.F32, dim, nullable), - new VectorMetadata(dim), nullable); + /// The `example.vector` dtype: `FixedSizeList` storage, dimension also in metadata. + static DType.Extension vectorDtype(boolean nullable) { + return new DType.Extension(VECTOR, new DType.FixedSizeList(DTypes.F32, DIM, nullable), + new VectorMetadata(DIM), nullable); } - /// The dtype as a writer without the encoder writes it: metadata as the bytes a file stores. - static DType.Extension opaqueVectorDtype(int storageDim, int metadataDim) { - return new DType.Extension(VECTOR.id(), new DType.FixedSizeList(DTypes.F32, storageDim, false), + /// The dtype as a writer without the encoder writes it, over 2-element vectors: metadata as the + /// bytes a file stores, here declaring `metadataDim` dimensions. + static DType.Extension opaqueVectorDtype(int metadataDim) { + return new DType.Extension(VECTOR, new DType.FixedSizeList(DTypes.F32, 2, false), new OpaqueMetadata(dimensionBytes(metadataDim)), false); } @@ -355,18 +366,17 @@ static MemorySegment dimensionBytes(int dim) { /// Shared by encoder and decoder, as one Rust `ExtVTable::validate_dtype` serves both directions. static void validateVector(DType.Extension dtype) { - if (!(dtype.storageDType() instanceof DType.FixedSizeList list) - || !(list.elementType() instanceof DType.Primitive p && p.ptype() == PType.F32 && !p.nullable())) { - throw new VortexException("example.vector storage must be FixedSizeList, got " - + dtype.storageDType()); - } - if (!(dtype.metadata() instanceof VectorMetadata(int dim))) { - throw new VortexException("example.vector metadata must be a VectorMetadata, got " + dtype.metadata()); - } - if (dim != list.fixedSize()) { - throw new VortexException("example.vector metadata dimension " + dim - + " != storage size " + list.fixedSize()); + // the one accepted shape; anything else falls through to the error + if (dtype.storageDType() instanceof DType.FixedSizeList(DType.Primitive(PType ptype, boolean nullable), int size, var _) + && ptype == PType.F32 && !nullable + && dtype.metadata() instanceof VectorMetadata(int dim)) { + if (dim != size) { + throw new VortexException("example.vector metadata dimension " + dim + " != storage size " + size); + } + return; } + throw new VortexException("example.vector must be FixedSizeList storage with a VectorMetadata, got " + + dtype.storageDType() + " and " + dtype.metadata()); } static final class VectorEncoder implements ExtensionEncoder { @@ -378,7 +388,7 @@ public ExtensionId extensionId() { @Override public DType.Extension dtype(boolean nullable) { - return vectorDtype(DIM, nullable); + return vectorDtype(nullable); } @Override @@ -432,7 +442,7 @@ public ExtensionId extensionId() { @Override public DType.Extension dtype(boolean nullable) { - return vectorDtype(DIM, nullable); + return vectorDtype(nullable); } @Override @@ -505,6 +515,7 @@ private void writeVectors(Path file, DType.Extension vectorDtype) throws IOExcep long row = (long) chunk * CHUNK_ROWS + i; ids[i] = row; float[] vector = vector(row); + // a non-nullable column has no null rows: give them a zero vector vectors.add(vector == null && !vectorDtype.nullable() ? new float[DIM] : vector); } writer.writeChunk(Map.of(ID, ids, EMBEDDING, vectors)); @@ -521,7 +532,7 @@ private SequencedMap readVectors(Path file, ScanOptions options) while (iter.hasNext()) { try (Chunk chunk = iter.next()) { List vectors = chunk.as(EMBEDDING, float[].class); - LongArray ids = (LongArray) chunk.column(ID); + LongArray ids = chunk.column(ID); for (int i = 0; i < vectors.size(); i++) { out.put(ids.getLong(i), vectors.get(i)); } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java index 2e9bfb78d..1f44068b9 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java @@ -10,6 +10,7 @@ import dev.vortex.arrow.ArrowAllocation; import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.testing.OhlcData; @@ -2333,7 +2334,7 @@ void javaWriter_rustReader_extensionWithNullMetadata(@TempDir Path tmp) throws I // none, and Rust refuses an absent field ("failed to parse extension metadata") for the // whole file, even when only a sibling column is projected. Path file = tmp.resolve("java_json_null_metadata.vtx"); - DType.Extension json = new DType.Extension("vortex.json", DTypes.UTF8, + DType.Extension json = new DType.Extension(ExtensionId.VORTEX_JSON, DTypes.UTF8, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("id"), ColumnName.of("j")), List.of(DTypes.I64, json), false); @@ -2367,7 +2368,7 @@ void javaWriter_rustReader_extensionWithNullMetadata(@TempDir Path tmp) throws I void invalidBuiltInExtensionDtype_refusedByJavaAndRust(@TempDir Path tmp) throws IOException { // Given — vortex.timestamp over I32 storage, which Rust's validate_dtype refuses ("Timestamp // storage dtype must be i64"). Java used to write and read it. - DType.Extension invalid = new DType.Extension("vortex.timestamp", DTypes.I32, + DType.Extension invalid = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I32, new io.github.dfa1.vortex.core.model.TimestampMetadata(io.github.dfa1.vortex.core.model.TimeUnit.Milliseconds, null), false); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("ts")), List.of(invalid), false); @@ -2384,7 +2385,7 @@ void invalidBuiltInExtensionDtype_refusedByJavaAndRust(@TempDir Path tmp) throws // No Java write path emits it any more, so write the column under a custom id of the same // length and rename it in place: a FlatBuffer string is stored verbatim, once. DType.Struct disguised = new DType.Struct(List.of(ColumnName.of("ts")), List.of(new DType.Extension( - "vortex.timestamq", DTypes.I32, + ExtensionId.parse("vortex.timestamq"), DTypes.I32, new io.github.dfa1.vortex.core.model.OpaqueMetadata(MemorySegment.ofArray(new byte[]{2, 0, 0})), false)), false); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE, diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java index f1c9642a3..fc9e2e14b 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java @@ -236,9 +236,9 @@ private static TimeUnit[] vortexTimeUnits(Path vortex) throws IOException { TimeUnit[] units = new TimeUnit[struct.fieldTypes().size()]; for (int i = 0; i < units.length; i++) { if (struct.fieldTypes().get(i) instanceof DType.Extension ext) { - if (ExtensionId.VORTEX_TIMESTAMP.id().equals(ext.extensionId())) { + if (ext.extensionId() == ExtensionId.VORTEX_TIMESTAMP) { units[i] = TimestampDtype.readUnit(ext); - } else if (ExtensionId.VORTEX_TIME.id().equals(ext.extensionId())) { + } else if (ext.extensionId() == ExtensionId.VORTEX_TIME) { units[i] = TimeDtype.readUnit(ext); } } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/WellKnownExtensionsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/WellKnownExtensionsInteropIntegrationTest.java index 98c00aac1..531ae7677 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/WellKnownExtensionsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/WellKnownExtensionsInteropIntegrationTest.java @@ -9,6 +9,7 @@ import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.OpaqueMetadata; import io.github.dfa1.vortex.reader.Chunk; @@ -56,9 +57,9 @@ class WellKnownExtensionsInteropIntegrationTest { private static final DType.Struct SCHEMA = new DType.Struct( List.of(ColumnName.of("id"), ColumnName.of("j"), ColumnName.of("g"), ColumnName.of("p")), List.of(DTypes.I64, - new DType.Extension("vortex.json", DTypes.UTF8, NO_METADATA, false), - new DType.Extension("vortex.st.wkb", DTypes.BINARY, NO_METADATA, false), - new DType.Extension("vortex.st.point", POINT_STORAGE, NO_METADATA, false)), + new DType.Extension(ExtensionId.VORTEX_JSON, DTypes.UTF8, NO_METADATA, false), + new DType.Extension(ExtensionId.VORTEX_ST_WKB, DTypes.BINARY, NO_METADATA, false), + new DType.Extension(ExtensionId.VORTEX_ST_POINT, POINT_STORAGE, NO_METADATA, false)), false); private static final int BATCHES = 2; private static final int BATCH_ROWS = 500; diff --git a/jdbc/src/main/java/io/github/dfa1/vortex/jdbc/JdbcImporter.java b/jdbc/src/main/java/io/github/dfa1/vortex/jdbc/JdbcImporter.java index e3fe6f86d..e92902bc9 100644 --- a/jdbc/src/main/java/io/github/dfa1/vortex/jdbc/JdbcImporter.java +++ b/jdbc/src/main/java/io/github/dfa1/vortex/jdbc/JdbcImporter.java @@ -218,7 +218,7 @@ private static boolean fillCell(Object buffer, int rowIdx, ResultSet rs, int col @SuppressWarnings("unchecked") private static void fillExtensionCell(List buffer, ResultSet rs, int colIdx, DType.Extension ext) throws SQLException { - if (!(ExtensionId.parse(ext.extensionId()) instanceof ExtensionId.WellKnown id)) { + if (!(ext.extensionId() instanceof ExtensionId.WellKnown id)) { throw new UnsupportedOperationException("unsupported extension: " + ext.extensionId()); } // SQL NULL → null in the buffer. Nullable extension columns round-trip through the diff --git a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java index ba0645a97..285ba5a50 100644 --- a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java +++ b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java @@ -195,7 +195,7 @@ private static void addPrimitiveColumn(FileSchema.Builder builder, String name, /// supported for export. private static void addTimestampColumn(FileSchema.Builder builder, String name, DType.Extension ext, RepetitionType rep) { - if (!ExtensionId.VORTEX_TIMESTAMP.id().equals(ext.extensionId())) { + if (ext.extensionId() != ExtensionId.VORTEX_TIMESTAMP) { throw new UnsupportedOperationException( "unsupported extension type for Parquet export: " + ext.extensionId() + " (column: " + name + ")"); } diff --git a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java index 6a9a3dcc4..99518f6f6 100644 --- a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java +++ b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java @@ -4,6 +4,7 @@ import dev.hardwood.row.PqStruct; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.writer.encode.ListData; import io.github.dfa1.vortex.writer.encode.NullableData; @@ -201,7 +202,7 @@ void f16_rejectedAtBuild() { void extensionDtype_rejectedAtConstruction() { // Given — nested Extension (timestamp) columns are unsupported; see class javadoc DType.Extension extensionDtype = new DType.Extension( - "vortex.date", new DType.Primitive(PType.I32, false), + ExtensionId.VORTEX_DATE, new DType.Primitive(PType.I32, false), new io.github.dfa1.vortex.core.model.TimeUnitMetadata(io.github.dfa1.vortex.core.model.TimeUnit.Days), false); // When / Then diff --git a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java index 4878f5a3c..19b5018a3 100644 --- a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java +++ b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java @@ -10,6 +10,7 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.TimeUnit; @@ -208,7 +209,7 @@ void timestamp_secondsOrDaysResolution_throws(TimeUnit unit) { @Test void nonTimestampExtension_throws() { // Given — a made-up extension id this exporter has no mapping for - DType.Extension ext = new DType.Extension("vortex.uuid", new DType.Primitive(PType.I64, false), + DType.Extension ext = new DType.Extension(ExtensionId.VORTEX_UUID, new DType.Primitive(PType.I64, false), io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); // When / Then diff --git a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetImporterTest.java b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetImporterTest.java index fd0ab3bde..2701df860 100644 --- a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetImporterTest.java +++ b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetImporterTest.java @@ -113,7 +113,7 @@ void int64_timestamp_mapsToTimestampExtensionOverI64(LogicalType.TimeUnit unit) // Then — vortex.timestamp extension over nullable I64 storage assertThat(result).isInstanceOf(DType.Extension.class); DType.Extension ext = (DType.Extension) result; - assertThat(ext.extensionId()).isEqualTo("vortex.timestamp"); + assertThat(ext.extensionId().id()).isEqualTo("vortex.timestamp"); assertThat(ext.storageDType()).isEqualTo(new DType.Primitive(PType.I64, true)); assertThat(ext.nullable()).isTrue(); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java b/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java index 4177ad2db..ca4fdeed6 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java @@ -6,7 +6,6 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.Mask; import io.github.dfa1.vortex.reader.array.SelectedArrays; -import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.reader.extension.ExtensionDecoder; import java.lang.foreign.Arena; @@ -220,7 +219,7 @@ public List as(ColumnName name, Class domainType) { if (!(col.dtype() instanceof DType.Extension ext)) { throw new VortexException("not an extension column: " + name); } - ExtensionDecoder decoder = registry.lookup(ExtensionId.parse(ext.extensionId())); + ExtensionDecoder decoder = registry.lookup(ext.extensionId()); if (decoder == null) { throw new VortexException("no extension decoder registered for " + ext.extensionId() + " (column '" + name + "')"); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java index 119bccde2..11c4a9019 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java @@ -325,7 +325,7 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs) { } ExtensionId id = ExtensionMetadataParser.parseId(e.id()); yield new DType.Extension( - id.id(), + id, storage, ExtensionMetadataParser.parse(id, metadata), storage.nullable()); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java index b722e0d6d..fe41f82af 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java @@ -53,7 +53,6 @@ import io.github.dfa1.vortex.reader.decode.ZstdEncodingDecoder; import io.github.dfa1.vortex.reader.extension.DateExtensionDecoder; import io.github.dfa1.vortex.reader.extension.ExtensionDecoder; -import io.github.dfa1.vortex.reader.extension.ExtensionMetadataParser; import io.github.dfa1.vortex.reader.extension.TimeExtensionDecoder; import io.github.dfa1.vortex.reader.extension.TimestampExtensionDecoder; import io.github.dfa1.vortex.reader.extension.UuidExtensionDecoder; @@ -164,7 +163,7 @@ public DType resolveExtensions(DType dtype) { return switch (dtype) { case null -> null; case DType.Extension ext -> { - ExtensionId id = ExtensionMetadataParser.parseId(ext.extensionId()); + ExtensionId id = ext.extensionId(); ExtensionDecoder decoder = extensions.get(id); DType storage = resolveExtensions(ext.storageDType()); DType.Extension resolved = ext; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java index 82c64d4fa..ad19a193f 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoder.java @@ -129,7 +129,7 @@ static DType dtypeFromProto(io.github.dfa1.vortex.core.proto.ProtoDType proto) { ExtensionId id = ExtensionMetadataParser.parseId(proto.extension().id()); byte[] metadata = proto.extension().metadata() != null ? proto.extension().metadata() : new byte[0]; return new DType.Extension( - id.id(), + id, storage, ExtensionMetadataParser.parse(id, MemorySegment.ofArray(metadata)), storage.nullable()); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java index 6816b963c..130b6af70 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java @@ -44,7 +44,7 @@ public Class domainType() { @Override public DType.Extension dtype(boolean nullable) { return new DType.Extension( - ExtensionId.VORTEX_DATE.id(), + ExtensionId.VORTEX_DATE, new DType.Primitive(PType.I32, nullable), new io.github.dfa1.vortex.core.model.TimeUnitMetadata(TimeUnit.Days), nullable); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java index 8c7bc0ad3..414f021e0 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java @@ -49,7 +49,7 @@ public DType.Extension dtype(boolean nullable) { DType.Primitive u8 = DTypes.U8; // Rust vortex.uuid metadata: 0 bytes means "no specific version" return new DType.Extension( - ExtensionId.VORTEX_UUID.id(), + ExtensionId.VORTEX_UUID, new DType.FixedSizeList(u8, 16, nullable), io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, nullable); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java index de5d6af6b..d90fa98ce 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java @@ -196,7 +196,7 @@ void convertDType_customExtensionId_isAccepted() { // Then assertThat(result.dtype()).isInstanceOfSatisfying(DType.Extension.class, ext -> - assertThat(ext.extensionId()).isEqualTo("acme.geopoint")); + assertThat(ext.extensionId().id()).isEqualTo("acme.geopoint")); } @org.junit.jupiter.params.ParameterizedTest diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java index 9cb2bf02a..0c6fab827 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java @@ -344,7 +344,7 @@ void resolveExtensions_validateThrowsRuntimeException_wrapsInVortexException() { void resolveExtensions_wellKnownIdWithoutDecoder_passesStrictMode() { // Given — vortex.json is in the vortex-jni default session, so a strict reader must open it // (reading the column as its Utf8 storage) though Java registers no decoder for it - DType schema = new DType.Extension("vortex.json", DTypes.UTF8, OpaqueMetadata.EMPTY, false); + DType schema = new DType.Extension(ExtensionId.VORTEX_JSON, DTypes.UTF8, OpaqueMetadata.EMPTY, false); ReadRegistry sut = ReadRegistry.loadAll(); // When @@ -357,7 +357,7 @@ void resolveExtensions_wellKnownIdWithoutDecoder_passesStrictMode() { @Test void resolveExtensions_wellKnownIdWithoutDecoder_stillValidated() { // Given — no decoder, but Rust's rules for the id still apply - DType schema = new DType.Extension("vortex.json", DTypes.I32, OpaqueMetadata.EMPTY, false); + DType schema = new DType.Extension(ExtensionId.VORTEX_JSON, DTypes.I32, OpaqueMetadata.EMPTY, false); ReadRegistry sut = ReadRegistry.loadAll(); // When / Then @@ -371,7 +371,7 @@ private record Srid(int srid) implements ExtensionMetadata { } private static DType.Extension geopoint(ExtensionMetadata metadata) { - return new DType.Extension("acme.geopoint", DTypes.I32, metadata, false); + return new DType.Extension(ExtensionId.parse("acme.geopoint"), DTypes.I32, metadata, false); } private static EncodingDecoder decoder(EncodingId id) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java index b18fe9805..1383fcdb1 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.GenericArray; @@ -67,7 +68,7 @@ void signedNarrowColumnsSignExtend() { void extensionDtypePassesIntegerThrough() { // Given an integer column whose dtype is an Extension (not a Primitive) — the unsigned // widening must pass the raw value through rather than assume a primitive ptype - DType extension = new DType.Extension("test.ext", DTypes.I32, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); + DType extension = new DType.Extension(ExtensionId.parse("test.ext"), DTypes.I32, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); MemorySegment seg = ARENA.allocate(4, 4); seg.setAtIndex(VortexFormat.LE_INT, 0, -7); Array column = new MaterializedIntArray(extension, 1, seg); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java index 2d0a07d05..091f16022 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.model.OpaqueMetadata; @@ -107,13 +108,13 @@ static Stream maskableTypes() { cases.add(Arguments.of(DTypes.UTF8.asNullable(), VarBinArray.class)); cases.add(Arguments.of(DTypes.BINARY.asNullable(), VarBinArray.class)); cases.add(Arguments.of(new DType.Decimal((byte) 10, (byte) 2, true), DecimalArray.class)); - cases.add(Arguments.of(new DType.Extension("vortex.date", DTypes.I32, + cases.add(Arguments.of(new DType.Extension(ExtensionId.VORTEX_DATE, DTypes.I32, new TimeUnitMetadata(TimeUnit.Nanoseconds), true), IntArray.class)); - cases.add(Arguments.of(new DType.Extension("vortex.time", DTypes.I32, + cases.add(Arguments.of(new DType.Extension(ExtensionId.VORTEX_TIME, DTypes.I32, new TimeUnitMetadata(TimeUnit.Microseconds), true), IntArray.class)); - cases.add(Arguments.of(new DType.Extension("vortex.timestamp", DTypes.I64, + cases.add(Arguments.of(new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, new TimestampMetadata(TimeUnit.Microseconds, null), true), LongArray.class)); - cases.add(Arguments.of(new DType.Extension("acme.custom", DTypes.U16, OpaqueMetadata.EMPTY, true), + cases.add(Arguments.of(new DType.Extension(ExtensionId.parse("acme.custom"), DTypes.U16, OpaqueMetadata.EMPTY, true), ShortArray.class)); return cases.stream(); } @@ -155,7 +156,7 @@ static Stream unmaskableTypes() { new DType.Struct(List.of(ColumnName.of("a")), List.of(DTypes.I64), true), new DType.Map(DTypes.I64, DTypes.I64, false, true), new DType.Variant(true), - new DType.Extension("vortex.uuid", new DType.FixedSizeList(DTypes.U8, 16, false), OpaqueMetadata.EMPTY, true)); + new DType.Extension(ExtensionId.VORTEX_UUID, new DType.FixedSizeList(DTypes.U8, 16, false), OpaqueMetadata.EMPTY, true)); } @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java index c16cb8128..e2f4abee0 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader.decode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; @@ -35,7 +36,7 @@ private static MemorySegment i64Meta() { } private static DType timestampDType(TimeUnit unit, boolean nullable) { - return new DType.Extension("vortex.timestamp", + return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, new DType.Primitive(PType.I64, nullable), new io.github.dfa1.vortex.core.model.TimestampMetadata(unit, null), nullable); } @@ -111,7 +112,7 @@ void decode_nullableExtension_decodesNullableDaysChild() { @Test void decode_extensionMissingTimeUnitMetadata_throws() { // Given an extension whose metadata records no time unit - DType noUnit = new DType.Extension("vortex.timestamp", + DType noUnit = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); DecodeContext c = ctx(i64Meta(), noUnit, 1, TestSegments.leLongs(0L), TestSegments.leLongs(0L), TestSegments.leLongs(0L)); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java index abf7de3ff..464e75019 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java @@ -271,7 +271,7 @@ void extension_withMetadata() { // Then id, storage dtype, and metadata bytes are preserved assertThat(result).isInstanceOf(DType.Extension.class); DType.Extension ext = (DType.Extension) result; - assertThat(ext.extensionId()).isEqualTo("ip.address"); + assertThat(ext.extensionId().id()).isEqualTo("ip.address"); assertThat(ext.storageDType()).isEqualTo(DTypes.I32); assertThat(ext.metadata()).isInstanceOf(io.github.dfa1.vortex.core.model.OpaqueMetadata.class); assertThat(((io.github.dfa1.vortex.core.model.OpaqueMetadata) ext.metadata()).bytes().byteSize()).isEqualTo(3); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java index 2cee964e0..7f8808355 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoderTest.java @@ -48,7 +48,7 @@ void dtype_isI32StorageWithDaysMetadata() { DType.Extension dtype = sut.dtype(false); // Then - assertThat(dtype.extensionId()).isEqualTo("vortex.date"); + assertThat(dtype.extensionId().id()).isEqualTo("vortex.date"); assertThat(dtype.storageDType()).isEqualTo(DTypes.I32); assertThat(dtype.metadata()).isEqualTo(new io.github.dfa1.vortex.core.model.TimeUnitMetadata(TimeUnit.Days)); } @@ -161,7 +161,7 @@ void date_unsupportedUnit_throws() { } private static DType.Extension dateDtype(TimeUnit unit, DType storage) { - return new DType.Extension("vortex.date", storage, + return new DType.Extension(ExtensionId.VORTEX_DATE, storage, new io.github.dfa1.vortex.core.model.TimeUnitMetadata(unit), false); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java index 51e1780ca..92a2a227b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionStorageTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader.extension; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.TimeUnit; @@ -102,7 +103,7 @@ void opaqueMetadata_throws() { } private DType.Extension ext(io.github.dfa1.vortex.core.model.ExtensionMetadata metadata) { - return new DType.Extension("vortex.timestamp", DTypes.I64, metadata, false); + return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, metadata, false); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java index b700b390e..c325b92e4 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java @@ -26,8 +26,8 @@ private ExtensionTestSupport() { } /// An extension dtype whose metadata is parsed from `meta` as a file's would be. - static DType.Extension ext(String id, DType storage, MemorySegment meta) { - return new DType.Extension(id, storage, ExtensionMetadataParser.parse(ExtensionId.parse(id), meta), false); + static DType.Extension ext(ExtensionId id, DType storage, MemorySegment meta) { + return new DType.Extension(id, storage, ExtensionMetadataParser.parse(id, meta), false); } static MemorySegment unitByte(byte tag) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java index a9a4a8949..0c74c0fd5 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoderTest.java @@ -64,19 +64,19 @@ void decode_eachUnit() { // Given — round-trip a known time-of-day through every TimeUnit try (Arena arena = Arena.ofConfined()) { // Seconds: 3661 s = 01:01:01 - assertThat(sut.decode(ext("vortex.time", I32, unitByte((byte) 3)), + assertThat(sut.decode(ext(ExtensionId.VORTEX_TIME, I32, unitByte((byte) 3)), i32(arena, 3661), 0)) .isEqualTo(LocalTime.of(1, 1, 1)); // Milliseconds: 3_661_500 = 01:01:01.500 - assertThat(sut.decode(ext("vortex.time", I32, unitByte((byte) 2)), + assertThat(sut.decode(ext(ExtensionId.VORTEX_TIME, I32, unitByte((byte) 2)), i32(arena, 3_661_500), 0)) .isEqualTo(LocalTime.of(1, 1, 1, 500_000_000)); // Microseconds: 1_000_001 = 00:00:01.000001 - assertThat(sut.decode(ext("vortex.time", I64, unitByte((byte) 1)), + assertThat(sut.decode(ext(ExtensionId.VORTEX_TIME, I64, unitByte((byte) 1)), i64(arena, 1_000_001L), 0)) .isEqualTo(LocalTime.of(0, 0, 1, 1_000)); // Nanoseconds: 42 ns past midnight - assertThat(sut.decode(ext("vortex.time", I64, unitByte((byte) 0)), + assertThat(sut.decode(ext(ExtensionId.VORTEX_TIME, I64, unitByte((byte) 0)), i64(arena, 42L), 0)) .isEqualTo(LocalTime.ofNanoOfDay(42)); } @@ -86,7 +86,7 @@ void decode_eachUnit() { void decode_daysUnitThrows() { // Given — Days isn't a sub-second unit try (Arena arena = Arena.ofConfined()) { - DType.Extension dtype = ext("vortex.time", I32, unitByte((byte) 4)); + DType.Extension dtype = ext(ExtensionId.VORTEX_TIME, I32, unitByte((byte) 4)); IntArray storage = i32(arena, 0); // When / Then diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java index 59141af3b..9c1c1ba11 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java @@ -81,7 +81,7 @@ void instant_microsecondsPath_handlesNegativeRaw() { // Given — pre-epoch micros exercise the floorDiv / floorMod path long micros = -1_500_001L; // -1.500001s try (Arena arena = Arena.ofConfined()) { - DType.Extension dtype = ext("vortex.timestamp", I64, tzMeta((byte) 1, null)); + DType.Extension dtype = ext(ExtensionId.VORTEX_TIMESTAMP, I64, tzMeta((byte) 1, null)); // When Instant got = sut.instant(dtype, i64(arena, micros), 0); @@ -119,7 +119,7 @@ void truncatedMetadata_refusedWhenParsed() { MemorySegment meta = MemorySegment.ofArray(new byte[]{2, 5, 0, (byte) 0x55, (byte) 0x54, (byte) 0x43}); // When / Then — refused when the file's dtype is parsed, before any decoder reads the zone - assertThatThrownBy(() -> ext("vortex.timestamp", I64, meta)) + assertThatThrownBy(() -> ext(ExtensionId.VORTEX_TIMESTAMP, I64, meta)) .isInstanceOf(VortexException.class) .hasMessageContaining("truncated"); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java index 1ec0da0ce..6cad8a1da 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.core.model.OpaqueMetadata; @@ -325,7 +326,7 @@ void fixedSizeListPool_throwsVortexException() { // neither VarBin- nor primitive-shaped. The fallback used to blind-cast it to VarBinArray, // which breaks the security contract (a raw ClassCastException on untrusted input). DType.FixedSizeList storage = new DType.FixedSizeList(DTypes.U8, 16, false); - DType uuid = new DType.Extension("vortex.uuid", storage, OpaqueMetadata.EMPTY, false); + DType uuid = new DType.Extension(ExtensionId.VORTEX_UUID, storage, OpaqueMetadata.EMPTY, false); Array values = new FixedSizeListArray(storage, 2, bytePool(new int[32])); Array codes = byteCodes(0, 1, 0); @@ -338,7 +339,7 @@ void fixedSizeListPool_throwsVortexException() { // ── harness ───────────────────────────────────────────────────────────────── private static Stream varBinPools() { - DType utf8Extension = new DType.Extension("ip.address", DTypes.UTF8, OpaqueMetadata.EMPTY, false); + DType utf8Extension = new DType.Extension(ExtensionId.parse("ip.address"), DTypes.UTF8, OpaqueMetadata.EMPTY, false); String[] byCode = {"ccc", "a", "bb", "a"}; String[] broadcast = {"zz", "zz", "zz", "zz"}; return Stream.of( diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java index 048457d15..6cb578a70 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java @@ -5,6 +5,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.PType; import org.junit.jupiter.api.Nested; @@ -206,7 +207,7 @@ void keepsNanCountForFloatColumn() { @Test void resolvesExtensionViaStorageDType() { // Given — ext over i32 storage; sum should widen using storage dtype (i32 → i64) - DType columnDtype = new DType.Extension("ip.address", + DType columnDtype = new DType.Extension(ExtensionId.parse("ip.address"), DTypes.I32, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); List present = List.of( ZonedStatsSchema.Stat.MIN, ZonedStatsSchema.Stat.SUM); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/DTypeFbsSerializer.java b/writer/src/main/java/io/github/dfa1/vortex/writer/DTypeFbsSerializer.java index 4eef4f951..31469fc03 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/DTypeFbsSerializer.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/DTypeFbsSerializer.java @@ -73,7 +73,7 @@ byte[] metadataBytes(DType.Extension ext) { } private byte[] serialize(DType.Extension ext) { - ExtensionId id = ExtensionId.parse(ext.extensionId()); + ExtensionId id = ext.extensionId(); byte[] bytes = switch (ext.metadata()) { case OpaqueMetadata(MemorySegment opaque) -> opaque.toArray(ValueLayout.JAVA_BYTE); case TimeUnitMetadata(TimeUnit unit) when id == ExtensionId.VORTEX_DATE || id == ExtensionId.VORTEX_TIME -> @@ -182,7 +182,7 @@ int serializeDType(FbsBuilder fbb, DType dtype) { yield FbsDType.createFbsDType(fbb, FbsType.FbsMap, inner); } case DType.Extension e -> { - int idOff = fbb.createString(e.extensionId()); + int idOff = fbb.createString(e.extensionId().id()); int storageDtypeOff = serializeDType(fbb, e.storageDType()); // Always present, empty when there is none: Rust refuses an extension dtype whose // metadata field is absent ("failed to parse extension metadata"), failing the file. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index a2db007d7..83a6fb0a2 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -16,6 +16,7 @@ import io.github.dfa1.vortex.core.model.EditionFamily; import io.github.dfa1.vortex.core.model.Editions; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.writer.encode.EncodeContext; import io.github.dfa1.vortex.writer.encode.EncodeNode; @@ -315,12 +316,11 @@ private static Set editionExcluded( private void validateExtensions(DType dtype) { switch (dtype) { case DType.Extension ext -> { - io.github.dfa1.vortex.core.model.ExtensionId id = - io.github.dfa1.vortex.core.model.ExtensionId.parse(ext.extensionId()); + ExtensionId id = ext.extensionId(); ExtensionEncoder encoder = extensionRegistry.lookup(id); if (encoder != null) { encoder.validate(ext); - } else if (id instanceof io.github.dfa1.vortex.core.model.ExtensionId.WellKnown wellKnown) { + } else if (id instanceof ExtensionId.WellKnown wellKnown) { // written from its storage array; still held to Rust's rules for the id wellKnown.validate(ext); } @@ -611,8 +611,7 @@ public void writeChunk(Map columns) throws IOException { // ExtEncoding wraps the storage child below — matches Rust's nested layout // (ExtEncoding → PrimitiveEncoding) and lets Registry skip its unwrap path. if (colDtype instanceof DType.Extension extDtype && data instanceof java.util.Collection coll) { - ExtensionEncoder impl = extensionRegistry.lookup( - io.github.dfa1.vortex.core.model.ExtensionId.parse(extDtype.extensionId())); + ExtensionEncoder impl = extensionRegistry.lookup(extDtype.extensionId()); if (impl == null) { // Without an encoder the Collection would reach the encoders unconverted and // fail far from the cause; raw storage arrays still pass through. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java index 974f87d7d..08faffb3e 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateExtensionEncoder.java @@ -35,7 +35,7 @@ public void validate(DType.Extension dtype) { @Override public DType.Extension dtype(boolean nullable) { return new DType.Extension( - ExtensionId.VORTEX_DATE.id(), + ExtensionId.VORTEX_DATE, new DType.Primitive(PType.I32, nullable), new TimeUnitMetadata(TimeUnit.Days), nullable); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java index facb69bfc..6c6af71a4 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java @@ -140,7 +140,7 @@ private static DateTimePartsData asParts(DType dtype, Object data) { return parts; } if (!(data instanceof long[] storage) || !(dtype instanceof DType.Extension ext) - || !ExtensionId.VORTEX_TIMESTAMP.id().equals(ext.extensionId()) + || ext.extensionId() != ExtensionId.VORTEX_TIMESTAMP || !(ext.metadata() instanceof TimestampMetadata(TimeUnit unit, var _)) || unit == TimeUnit.Days) { return null; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java index 1eb42b18a..33f4f50ef 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/UuidExtensionEncoder.java @@ -36,7 +36,7 @@ public DType.Extension dtype(boolean nullable) { DType.Primitive u8 = DTypes.U8; // Rust vortex.uuid metadata: 0 bytes means "no specific version" return new DType.Extension( - ExtensionId.VORTEX_UUID.id(), + ExtensionId.VORTEX_UUID, new DType.FixedSizeList(u8, 16, nullable), OpaqueMetadata.EMPTY, nullable); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/DTypeFbsSerializerTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/DTypeFbsSerializerTest.java index 39c0383e0..7b0cb3fdb 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/DTypeFbsSerializerTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/DTypeFbsSerializerTest.java @@ -38,8 +38,8 @@ void builtInRecords_underTheirOwnIds_areWrittenAsTheReaderParsesThem() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().build()); // When - byte[] date = sut.metadataBytes(extension("vortex.date", new TimeUnitMetadata(TimeUnit.Days))); - byte[] timestamp = sut.metadataBytes(extension("vortex.timestamp", + byte[] date = sut.metadataBytes(extension(ExtensionId.VORTEX_DATE, new TimeUnitMetadata(TimeUnit.Days))); + byte[] timestamp = sut.metadataBytes(extension(ExtensionId.VORTEX_TIMESTAMP, new TimestampMetadata(TimeUnit.Milliseconds, "UTC"))); // Then — the unit tag; then for a timestamp the u16 LE zone length and the zone as stored @@ -56,7 +56,7 @@ void builtInRecord_underACustomId_goesThroughThatIdsEncoder() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().register(encoder).build()); // When - byte[] result = sut.metadataBytes(extension(GEOPOINT.id(), new TimeUnitMetadata(TimeUnit.Seconds))); + byte[] result = sut.metadataBytes(extension(GEOPOINT, new TimeUnitMetadata(TimeUnit.Seconds))); // Then assertThat(result).containsExactly(7); @@ -68,7 +68,7 @@ void encoderBytes_areSerializedOncePerExtension() { ExtensionEncoder encoder = encoder(GEOPOINT); given(encoder.serializeMetadata(any())).willReturn(new byte[]{1}); DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().register(encoder).build()); - DType.Extension ext = extension(GEOPOINT.id(), new Srid(4326)); + DType.Extension ext = extension(GEOPOINT, new Srid(4326)); // When sut.metadataBytes(ext); @@ -87,7 +87,7 @@ void oversizedMetadata_isRefused() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().register(encoder).build()); // When / Then - assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT.id(), new Srid(1)))) + assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT, new Srid(1)))) .isInstanceOf(VortexException.class) .hasMessageContaining("acme.geopoint metadata serializes to 65537 bytes"); } @@ -100,7 +100,7 @@ void encoderReturningNull_isRefusedNamingTheId() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().register(encoder).build()); // When / Then — a VortexException naming the extension, not an NPE inside the FlatBuffer builder - assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT.id(), new Srid(1)))) + assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT, new Srid(1)))) .isInstanceOf(VortexException.class) .hasMessageContaining("acme.geopoint encoder serialized its metadata to null"); } @@ -111,7 +111,7 @@ void customRecordWithoutEncoder_isRefused() { DTypeFbsSerializer sut = new DTypeFbsSerializer(WriteRegistry.builder().build()); // When / Then - assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT.id(), new Srid(1)))) + assertThatThrownBy(() -> sut.metadataBytes(extension(GEOPOINT, new Srid(1)))) .isInstanceOf(VortexException.class) .hasMessageContaining("no extension encoder registered for acme.geopoint"); } @@ -123,13 +123,13 @@ void opaqueMetadata_isWrittenAsItsBytes() { byte[] bytes = "crs".getBytes(StandardCharsets.US_ASCII); // When - byte[] result = sut.metadataBytes(extension("vortex.st.wkb", new OpaqueMetadata(MemorySegment.ofArray(bytes)))); + byte[] result = sut.metadataBytes(extension(ExtensionId.VORTEX_ST_WKB, new OpaqueMetadata(MemorySegment.ofArray(bytes)))); // Then assertThat(result).isEqualTo(bytes); } - private static DType.Extension extension(String id, ExtensionMetadata metadata) { + private static DType.Extension extension(ExtensionId id, ExtensionMetadata metadata) { return new DType.Extension(id, DTypes.I64, metadata, false); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java index de2a94bae..96cb2d06e 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java @@ -8,6 +8,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.Editions; import io.github.dfa1.vortex.core.model.PType; @@ -300,7 +301,7 @@ void extensionColumn_emitsStorageMinMaxZoneMap(@TempDir Path tmp) throws IOExcep // two zones of two rows: zone 0 = [10, 11], zone 1 = [20, 21]. ExtEncoding propagates the // storage primitive's min/max, so the zone-map carries MAX+MIN+NULL_COUNT — same as I64. DType ext = new DType.Extension( - "test.ext", DTypes.I64, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); + ExtensionId.parse("test.ext"), DTypes.I64, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("t")), List.of(ext), false); WriteOptions opts = WriteOptions.cascading(0).withZoneMaps(true).withGlobalDict(false).withEdition(Editions.CORE_2025_10_0); Path file = tmp.resolve("ext.vtx"); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java index c958019db..86e26ae5a 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapStatCodecTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.writer.encode.MinMax; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.writer.encode.ComparableValues; @@ -119,7 +120,7 @@ void empty_returnsNull() { class ExtensionType { private static final DType TIMESTAMP_MS = - new DType.Extension("vortex.timestamp", DTypes.I64, + new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, new io.github.dfa1.vortex.core.model.TimestampMetadata( io.github.dfa1.vortex.core.model.TimeUnit.Nanoseconds, null), false); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java index bc63087d3..57e6192ef 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.core.model.DTypes; @@ -34,7 +35,7 @@ class DateTimePartsEncodingEncoderTest { private static final DType EXT_TIMESTAMP_NS = timestampDType(TimeUnit.Nanoseconds); private static DType timestampDType(TimeUnit unit) { - return new DType.Extension("vortex.timestamp", + return new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, new io.github.dfa1.vortex.core.model.TimestampMetadata(unit, null), false); } @@ -87,7 +88,7 @@ void encode_producesThreeChildren_noBuffersAtRoot() { @Test void encode_missingMetadata_throws() { // Given - DType noMeta = new DType.Extension("vortex.timestamp", + DType noMeta = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); DateTimePartsData data = new DateTimePartsData(new long[]{0L}, false); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java index f0bb5ad9d..50c33e1ac 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.decode.ArrayNode; @@ -49,7 +50,7 @@ class ZoneMapStats { // pruning and MIN/MAX push-down purely for being a timestamp rather than a bare i64. private static final DType.Extension TIMESTAMP = - new DType.Extension("vortex.timestamp", DTypes.I64, TIMESTAMP_MS, false); + new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, TIMESTAMP_MS, false); @Test void encode_extensionColumn_reportsStorageSpaceBounds() throws Exception { @@ -98,7 +99,7 @@ class Encode { @Test void accepts_extensionDtype_returnsTrue() { // Given - DType extDType = new DType.Extension("vortex.timestamp", + DType extDType = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, TIMESTAMP_MS, false); // When / Then @@ -116,7 +117,7 @@ void encode_extensionWrappingI64_roundTrips() { // Given long[] data = {100L, 200L, 300L, 400L}; DType storageDType = DTypes.I64; - DType extDType = new DType.Extension("vortex.timestamp", storageDType, TIMESTAMP_MS, false); + DType extDType = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, storageDType, TIMESTAMP_MS, false); // When EncodeResult resultEncoded = ENCODER.encode(extDType, data, EncodeTestHelper.testCtx()); @@ -157,7 +158,7 @@ void encodeCascade_exposesStorageAsOpenChild() { // Given long[] data = {100L, 200L, 300L, 400L}; DType storageDType = DTypes.I64; - DType extDType = new DType.Extension("vortex.timestamp", storageDType, TIMESTAMP_MS, false); + DType extDType = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, storageDType, TIMESTAMP_MS, false); // When CascadeStep result = ENCODER.encodeCascade(extDType, data, EncodeTestHelper.testCtx()); @@ -195,7 +196,7 @@ void decode_extensionWrappingI64_returnsStorageArray() { MemorySegment buf = TestSegments.leLongs(values); DType storageDType = DTypes.I64; - DType extDType = new DType.Extension("vortex.timestamp", storageDType, TIMESTAMP_MS, false); + DType extDType = new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, storageDType, TIMESTAMP_MS, false); ArrayNode primitiveNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); ArrayNode extNode = new ArrayNode(EncodingId.VORTEX_EXT, null, new ArrayNode[]{primitiveNode}, new int[0]); @@ -261,7 +262,7 @@ static Stream everyWellKnownExtension() { nulls)), Arguments.of(UuidExtensionEncoder.INSTANCE.dtype(true), UuidExtensionEncoder.INSTANCE.encodeAll(UuidExtensionEncoder.INSTANCE.dtype(true), nulls)), - Arguments.of(new DType.Extension("acme.custom", DTypes.U16, OpaqueMetadata.EMPTY, true), + Arguments.of(new DType.Extension(ExtensionId.parse("acme.custom"), DTypes.U16, OpaqueMetadata.EMPTY, true), new NullableData(new short[ROWS], new boolean[ROWS]))); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java index 4261770e9..04a040772 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/TimeExtensionEncoderTest.java @@ -156,7 +156,7 @@ void encodeAll_nullInNonNullableColumn_throws() { void encodeAll_daysUnit_throws() { // Given a hand-built Days-tagged dtype (TimeDtype.of rejects Days, so build directly) DType.Extension dtype = new DType.Extension( - ExtensionId.VORTEX_TIME.id(), DTypes.I32, + ExtensionId.VORTEX_TIME, DTypes.I32, new io.github.dfa1.vortex.core.model.TimeUnitMetadata(TimeUnit.Days), false); List values = List.of(T); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsFitnessTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsFitnessTest.java index 1c476d6ed..d23deae5d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsFitnessTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsFitnessTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.writer.WriteRegistry; @@ -174,7 +175,7 @@ private static Stream boundsOwingSamples() { Arguments.of(EncodingId.VORTEX_PCO, DTypes.I64, ascending), Arguments.of(EncodingId.VORTEX_ZIGZAG, DTypes.I64, ascending), Arguments.of(EncodingId.VORTEX_EXT, - new DType.Extension("vortex.timestamp", DTypes.I64, new io.github.dfa1.vortex.core.model.TimestampMetadata( + new DType.Extension(ExtensionId.VORTEX_TIMESTAMP, DTypes.I64, new io.github.dfa1.vortex.core.model.TimestampMetadata( io.github.dfa1.vortex.core.model.TimeUnit.Milliseconds, null), false), ascending), Arguments.of(EncodingId.VORTEX_SPARSE, DTypes.I64, sparse), Arguments.of(EncodingId.VORTEX_PATCHED, DTypes.I64, ascending),