From de6a5f2ce3209001e74d4cdf6cfda024d5e469f4 Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 13:42:21 -0700 Subject: [PATCH 01/10] fix: Paths, grammar --- documentation/getting_started_guide/GETTING_STARTED.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/getting_started_guide/GETTING_STARTED.ipynb b/documentation/getting_started_guide/GETTING_STARTED.ipynb index c6a2b1d..03d857f 100644 --- a/documentation/getting_started_guide/GETTING_STARTED.ipynb +++ b/documentation/getting_started_guide/GETTING_STARTED.ipynb @@ -577,7 +577,7 @@ "\n", "### Advanced conversion\n", "\n", - "[`convert_tfe_data`](https://github.com/AllenCell/tfe-data/blob/main/tfe_data/converter.py#L390) includes detailed documentation for additional configuration options and parameters.\n", + "[`convert_colorizer_data`](https://github.com/AllenCell/tfe-data/blob/main/colorizer_data/converter.py#L390) includes detailed documentation for additional configuration options and parameters.\n", "\n", "This includes support for:\n", "- **Outliers**, objects that should have unique visualization (e.g. dead cells)\n", From b34e8e9945ae0feb1f467e3071daf3b08632ac78 Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 13:52:36 -0700 Subject: [PATCH 02/10] fix: TFE link in Python notebook --- documentation/getting_started_guide/GETTING_STARTED.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/getting_started_guide/GETTING_STARTED.ipynb b/documentation/getting_started_guide/GETTING_STARTED.ipynb index 03d857f..c6a2b1d 100644 --- a/documentation/getting_started_guide/GETTING_STARTED.ipynb +++ b/documentation/getting_started_guide/GETTING_STARTED.ipynb @@ -577,7 +577,7 @@ "\n", "### Advanced conversion\n", "\n", - "[`convert_colorizer_data`](https://github.com/AllenCell/tfe-data/blob/main/colorizer_data/converter.py#L390) includes detailed documentation for additional configuration options and parameters.\n", + "[`convert_tfe_data`](https://github.com/AllenCell/tfe-data/blob/main/tfe_data/converter.py#L390) includes detailed documentation for additional configuration options and parameters.\n", "\n", "This includes support for:\n", "- **Outliers**, objects that should have unique visualization (e.g. dead cells)\n", From 9d67cd43518ec9fe05bf45702aa02903e96e4412 Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 11:41:01 -0700 Subject: [PATCH 03/10] refactor: Renamed ColorizerMetadata to DatasetMetadata, deprecated ColorizerMetadata --- documentation/DATA_FORMAT.md | 2 +- .../example_scripts/getting_started_guide.py | 4 ++-- tests/test_types.py | 6 ++--- tests/test_writer.py | 22 ++++++++--------- tfe_data/converter.py | 8 +++---- tfe_data/types.py | 24 ++++++++++++------- tfe_data/utils.py | 6 ++--- tfe_data/writer.py | 12 +++++----- 8 files changed, 45 insertions(+), 39 deletions(-) diff --git a/documentation/DATA_FORMAT.md b/documentation/DATA_FORMAT.md index 32ccf03..2055989 100644 --- a/documentation/DATA_FORMAT.md +++ b/documentation/DATA_FORMAT.md @@ -841,6 +841,6 @@ frame_paths = generate_frame_paths(num_frames, start_frame=starting_timepoint) writer.set_frame_paths(frame_paths) # Including starting frame number in the metadata -metadata = ColorizerMetadata(start_frame_num=starting_timepoint) +metadata = DatasetMetadata(start_frame_num=starting_timepoint) writer.write_manifest(metadata) ``` diff --git a/documentation/bin/example_scripts/getting_started_guide.py b/documentation/bin/example_scripts/getting_started_guide.py index 85a0b6e..a15cc56 100644 --- a/documentation/bin/example_scripts/getting_started_guide.py +++ b/documentation/bin/example_scripts/getting_started_guide.py @@ -31,7 +31,7 @@ convert_colorizer_data, FeatureInfo, FeatureType, - ColorizerMetadata, + DatasetMetadata, CollectionMetadata, update_collection, ) @@ -90,7 +90,7 @@ def main(): dataset_out_dir.mkdir(parents=True, exist_ok=True) # Dataset metadata - metadata = ColorizerMetadata( + metadata = DatasetMetadata( name="Example dataset {}".format(i + 1), description="An example dataset for the Timelapse Feature Explorer.", author="Author name et al.", diff --git a/tests/test_types.py b/tests/test_types.py index 234a48d..f740754 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -1,10 +1,10 @@ -from tfe_data.types import ColorizerMetadata +from tfe_data.types import DatasetMetadata def test_manifest_from_dict_handles_empty_dict(): manifest_dict = {} # Test that this does not crash - metadata = ColorizerMetadata.from_dict(manifest_dict) + metadata = DatasetMetadata.from_dict(manifest_dict) def test_manifest_from_dict_handles_partial_frame_dims(): @@ -19,7 +19,7 @@ def test_manifest_from_dict_handles_partial_frame_dims(): for frame_dims in frame_dimensions: manifest_dict = {"frameDims": frame_dims} - metadata = ColorizerMetadata.from_dict(manifest_dict) + metadata = DatasetMetadata.from_dict(manifest_dict) # Check for default values for nested fields if "width" in frame_dims.keys(): diff --git a/tests/test_writer.py b/tests/test_writer.py index 22a3c75..17a0bdb 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -9,7 +9,7 @@ from tfe_data.types import ( CURRENT_VERSION, - ColorizerMetadata, + DatasetMetadata, DatasetManifest, FeatureInfo, ) @@ -83,7 +83,7 @@ def test_metadata_uses_frame_dims_subfield(tmp_path): writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) writer.write_manifest( - metadata=ColorizerMetadata( + metadata=DatasetMetadata( frame_width=width, frame_height=height, frame_units=units, @@ -101,14 +101,14 @@ def test_metadata_uses_frame_dims_subfield(tmp_path): assert manifest["metadata"]["frameDims"]["height"] == height assert manifest["metadata"]["frameDims"]["units"] == units - # Expect ColorizerMetadata fields to not be written directly + # Expect DatasetMetadata fields to not be written directly assert "frame_width" not in manifest["metadata"].keys() assert "frame_height" not in manifest["metadata"].keys() assert "frame_units" not in manifest["metadata"].keys() - # Expect manifest structure can be parsed to ColorizerMetadata + # Expect manifest structure can be parsed to DatasetMetadata # with correct fields - metadata = ColorizerMetadata.from_dict(manifest["metadata"]) + metadata = DatasetMetadata.from_dict(manifest["metadata"]) assert metadata.frame_width == width assert metadata.frame_height == height @@ -143,7 +143,7 @@ def test_writer_handles_renamed_fields(existing_manifest): writer, tmp_path, manifest_path = existing_manifest writer.write_manifest( - metadata=ColorizerMetadata( + metadata=DatasetMetadata( start_time_sec=1.0, start_frame_num=2.0, frame_duration_sec=3.0 ) ) @@ -151,7 +151,7 @@ def test_writer_handles_renamed_fields(existing_manifest): with open(manifest_path, "r") as f: manifest: DatasetManifest = json.load(f) metadata_dict = manifest["metadata"] - metadata = ColorizerMetadata.from_dict(metadata_dict) + metadata = DatasetMetadata.from_dict(metadata_dict) # Expect fields to be written out in the dictionary assert metadata_dict["startingTimeSeconds"] == 1.0 @@ -167,9 +167,7 @@ def test_writer_handles_renamed_fields(existing_manifest): def test_writer_keeps_manifest_metadata(existing_manifest): # Should keep name, author, time of creation, description writer, tmp_path, manifest_path = existing_manifest - writer.write_manifest( - metadata=ColorizerMetadata(start_time_sec=5, start_frame_num=6) - ) + writer.write_manifest(metadata=DatasetMetadata(start_time_sec=5, start_frame_num=6)) with open(manifest_path, "r") as f: manifest: DatasetManifest = json.load(f) @@ -193,7 +191,7 @@ def test_writer_overrides_metadata_fields(existing_manifest): writer, tmp_path, manifest_path = existing_manifest writer.write_manifest( - metadata=ColorizerMetadata( + metadata=DatasetMetadata( name="new name", description="new description", author="geoff", @@ -228,7 +226,7 @@ def test_writer_updates_fields_when_metadata_is_missing(blank_manifest): with open(manifest_path, "r") as f: manifest: DatasetManifest = json.load(f) - metadata: ColorizerMetadata = ColorizerMetadata.from_dict(manifest["metadata"]) + metadata: DatasetMetadata = DatasetMetadata.from_dict(manifest["metadata"]) assert metadata.name == DEFAULT_DATASET_NAME assert metadata.date_created is not None diff --git a/tfe_data/converter.py b/tfe_data/converter.py index 05a040e..57c467e 100644 --- a/tfe_data/converter.py +++ b/tfe_data/converter.py @@ -17,7 +17,7 @@ from tfe_data.types import ( BackdropMetadata, - ColorizerMetadata, + DatasetMetadata, DataFileType, FeatureInfo, Frames3dMetadata, @@ -392,7 +392,7 @@ def convert_colorizer_data( output_dir: Union[str, pathlib.Path], *, source_dir: Optional[Union[str, pathlib.Path]] = None, - metadata: Optional[ColorizerMetadata] = None, + metadata: Optional[DatasetMetadata] = None, object_id_column: str = "ID", # DEPRECATED segmentation_id_column: str = None, times_column: str = "Frame", @@ -427,9 +427,9 @@ def convert_colorizer_data( relative paths in the `data` DataFrame will be resolved relative to this directory. Absolute paths will not be affected by this value. If `None`, the current working directory (`.`) will be used. - metadata (ColorizerMetadata | None): Metadata to include in the dataset's manifest, such + metadata (DatasetMetadata | None): Metadata to include in the dataset's manifest, such as the dataset name, author, dataset description, frame resolution, and time units. - See `ColorizerMetadata` for more information. Note that some information will be + See `DatasetMetadata` for more information. Note that some information will be written automatically, such as a timestamp and revision number. object_id_column (str): DEPRECATED. The name of the column containing the segmentation ID of a given object in the frame or image data. Overridden by `segmentation_id_column` diff --git a/tfe_data/types.py b/tfe_data/types.py index 7502ba2..e352dae 100644 --- a/tfe_data/types.py +++ b/tfe_data/types.py @@ -4,6 +4,7 @@ from dataclasses_json.core import _decode_dataclass from enum import Enum from typing import Dict, List, Optional, Type, TypeVar, TypedDict, Union +from warnings import deprecated Json = Union[dict, str, int, float, bool, None] @@ -182,10 +183,11 @@ class Frames3dMetadata(DataClassJsonMixin): @dataclass -class ColorizerMetadata(DataClassJsonMixin): +class DatasetMetadata(DataClassJsonMixin): """ - Data class representation of metadata for a Colorizer dataset. - Can be converted to and from camelCase JSON format; see https://pypi.org/project/dataclasses-json/. + Data class representation of metadata for a Colorizer dataset. Can be + converted to and from camelCase JSON format; see + https://pypi.org/project/dataclasses-json/. """ dataclass_json_config = config(letter_case=LetterCase.CAMEL, undefined=None)[ @@ -255,11 +257,11 @@ def from_dict( infer_missing=True, ) -> A: # Hacky. This is what DataClassJsonMixin.from_dict() calls internally, passing in the - # inferred class. In this case, we want to explicitly pass in this class (ColorizerMetadata) + # inferred class. In this case, we want to explicitly pass in this class (DatasetMetadata) # and use the parent behavior, but we can't call DataClassJsonMixin.from_dict() directly - # because it is unaware of ColorizerMetadata's dataclass fields. - metadata: ColorizerMetadata = _decode_dataclass( - ColorizerMetadata, kvs, infer_missing + # because it is unaware of DatasetMetadata's dataclass fields. + metadata: DatasetMetadata = _decode_dataclass( + DatasetMetadata, kvs, infer_missing ) if "frameDims" in kvs.keys() and isinstance(kvs["frameDims"], dict): @@ -281,6 +283,12 @@ def from_dict( return metadata +@deprecated(reason="Use DatasetMetadata instead.") +@dataclass +class ColorizerMetadata(DatasetMetadata): + pass + + class DatasetManifest(TypedDict): features: List[FeatureMetadata] outliers: str @@ -292,7 +300,7 @@ class DatasetManifest(TypedDict): # snake_case, but JSON convention is camelCase. segIds: str bounds: str - metadata: ColorizerMetadata + metadata: DatasetMetadata frames: List[str] backdrops: List[BackdropMetadata] frames3d: Optional[Frames3dMetadata] diff --git a/tfe_data/utils.py b/tfe_data/utils.py index ca61a6b..f1f0600 100644 --- a/tfe_data/utils.py +++ b/tfe_data/utils.py @@ -22,7 +22,7 @@ DATETIME_FORMAT, CollectionManifest, CollectionMetadata, - ColorizerMetadata, + DatasetMetadata, FeatureInfo, FeatureType, ) @@ -212,7 +212,7 @@ def remap_segmented_image( def update_metadata( - metadata: Optional[Union[CollectionMetadata, ColorizerMetadata]], + metadata: Optional[Union[CollectionMetadata, DatasetMetadata]], *, default_name: Optional[str] = None, ): @@ -225,7 +225,7 @@ def update_metadata( - name: Sets name if none exists and a `default_name` argument was provided. Args: - metadata (CollectionMetadata | ColorizerMetadata): The metadata object to update. + metadata (CollectionMetadata | DatasetMetadata): The metadata object to update. default_name (str): The name of the collection or dataset to use if the metadata has none. """ current_time = datetime.now(timezone.utc).strftime(DATETIME_FORMAT) diff --git a/tfe_data/writer.py b/tfe_data/writer.py index 1319689..e2062d9 100644 --- a/tfe_data/writer.py +++ b/tfe_data/writer.py @@ -11,7 +11,7 @@ from tfe_data.types import ( BackdropMetadata, - ColorizerMetadata, + DatasetMetadata, DatasetManifest, FeatureInfo, FeatureMetadata, @@ -63,7 +63,7 @@ class ColorizerDatasetWriter: outpath: pathlib.Path default_dataset_name: str manifest: DatasetManifest - metadata: ColorizerMetadata + metadata: DatasetMetadata backdrops: Dict[str, BackdropMetadata] features: Dict[str, FeatureMetadata] scale: float @@ -111,9 +111,9 @@ def __init__( self.default_dataset_name = dataset if "metadata" not in self.manifest: # New default metadata - self.metadata = ColorizerMetadata() + self.metadata = DatasetMetadata() else: - self.metadata = ColorizerMetadata.from_dict(self.manifest["metadata"]) + self.metadata = DatasetMetadata.from_dict(self.manifest["metadata"]) def write_categorical_feature( self, @@ -517,7 +517,7 @@ def set_3d_frame_data(self, data: Frames3dMetadata) -> None: def write_manifest( self, num_frames: int = None, - metadata: ColorizerMetadata = None, + metadata: DatasetMetadata = None, ): """ Writes the final manifest file for the dataset in the configured output directory. @@ -526,7 +526,7 @@ def write_manifest( Args: num_frames (int): DEPRECATED. Define to generate the expected paths for frame images. - metadata (ColorizerMetadata): Metadata to be written with the dataset. Leave fields blank to use existing default values. + metadata (DatasetMetadata): Metadata to be written with the dataset. Leave fields blank to use existing default values. Note that some metadata fields (like `last_modified`, `_writer_version`, `_revision`, and `date_created`) will be automatically updated. Add definitions for these fields in the `metadata` argument to override this behavior. From 4ceb40c660c546627dd30eb443a225c4b40b9ee3 Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 11:46:48 -0700 Subject: [PATCH 04/10] refactor: Renamed `convert_tfe_data`, aliased as `convert_colorizer_data` with deprecation warning --- tfe_data/converter.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tfe_data/converter.py b/tfe_data/converter.py index 57c467e..12e3426 100644 --- a/tfe_data/converter.py +++ b/tfe_data/converter.py @@ -1,3 +1,4 @@ +from functools import wraps import logging import math import multiprocessing @@ -6,6 +7,7 @@ import shutil import time from typing import Dict, List, Optional, Union +from warnings import deprecated from bioio import BioImage from dataclasses import dataclass @@ -14,7 +16,6 @@ import pandas as pd import numpy as np - from tfe_data.types import ( BackdropMetadata, DatasetMetadata, @@ -387,7 +388,7 @@ def _handle_3d_frames( writer.set_3d_frame_data(config.frames_3d) -def convert_colorizer_data( +def convert_tfe_data( data: DataFrame, output_dir: Union[str, pathlib.Path], *, @@ -625,3 +626,11 @@ def convert_colorizer_data( finally: # Restore working directory os.chdir(original_cwd) + + +@deprecated( + "Will be removed in the next major release. Please use convert_tfe_data instead." +) +@wraps(convert_tfe_data) +def convert_colorizer_data(*args, **kwargs): + return convert_tfe_data(*args, **kwargs) From c582e410b4a66904484d73b327632fb5522cdb3f Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 11:56:45 -0700 Subject: [PATCH 05/10] refactor: Renamed `convert_colorizer_data` to `convert_tfe_data`, deprecated `convert_colorizer_data`. --- README.md | 4 +- .../example_scripts/check_conversion_api.py | 4 +- .../example_scripts/getting_started_guide.py | 4 +- .../GETTING_STARTED.ipynb | 24 +++---- tests/test_converter.py | 62 +++++++++---------- tfe_data/converter.py | 8 +-- 6 files changed, 52 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 55e2b83..7146c1d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ into the correct format for the Timelapse Feature Explorer. ```python import pandas as pd from io import StringIO -from tfe_data import convert_colorizer_data +from tfe_data import convert_tfe_data from pathlib import Path @@ -49,7 +49,7 @@ source_dir = Path("some/source/directory") data: pd.DataFrame = pd.read_csv(StringIO(csv)) output_dir = Path("some/directory/my-dataset") -convert_colorizer_data( +convert_tfe_data( data, output_dir, source_dir=source_dir, diff --git a/documentation/bin/example_scripts/check_conversion_api.py b/documentation/bin/example_scripts/check_conversion_api.py index d088085..69403f0 100644 --- a/documentation/bin/example_scripts/check_conversion_api.py +++ b/documentation/bin/example_scripts/check_conversion_api.py @@ -1,6 +1,6 @@ import pathlib import pandas as pd -from tfe_data import convert_colorizer_data +from tfe_data import convert_tfe_data """ To test, run the following commands from the root directory. @@ -14,7 +14,7 @@ source_dir = pathlib.Path("./documentation/getting_started_guide/raw_dataset") data = pd.read_csv(source_dir / "data.csv") output_dir = pathlib.Path("./documentation/getting_started_guide/processed_data") - convert_colorizer_data( + convert_tfe_data( data, output_dir, source_dir=source_dir, diff --git a/documentation/bin/example_scripts/getting_started_guide.py b/documentation/bin/example_scripts/getting_started_guide.py index a15cc56..606d3d5 100644 --- a/documentation/bin/example_scripts/getting_started_guide.py +++ b/documentation/bin/example_scripts/getting_started_guide.py @@ -28,7 +28,7 @@ from pathlib import Path from tfe_data import ( - convert_colorizer_data, + convert_tfe_data, FeatureInfo, FeatureType, DatasetMetadata, @@ -107,7 +107,7 @@ def main(): # Convert the dataset data: pd.DataFrame = pd.read_csv(dataset_src_dir / "data.csv") - convert_colorizer_data( + convert_tfe_data( data, dataset_out_dir, source_dir=dataset_src_dir, diff --git a/documentation/getting_started_guide/GETTING_STARTED.ipynb b/documentation/getting_started_guide/GETTING_STARTED.ipynb index c6a2b1d..cae5565 100644 --- a/documentation/getting_started_guide/GETTING_STARTED.ipynb +++ b/documentation/getting_started_guide/GETTING_STARTED.ipynb @@ -162,7 +162,7 @@ "source": [ "from pathlib import Path\n", "\n", - "from tfe_data import convert_colorizer_data\n", + "from tfe_data import convert_tfe_data\n", "import pandas as pd\n", "\n", "source_directory = Path(\"raw_datasets/dataset_1\")\n", @@ -171,12 +171,12 @@ "# Load the dataset\n", "data: pd.DataFrame = pd.read_csv(source_directory / \"data.csv\")\n", "\n", - "# `convert_colorizer_data` is a helper function for dataset conversion! You can pass in a pandas DataFrame\n", + "# `convert_tfe_data` is a helper function for dataset conversion! You can pass in a pandas DataFrame\n", "# and the output directory, as well as the names of the data columns for the id, time, track, and image data.\n", "# Any other columns are treated as features. We'll show off more advanced behavior in later steps, but this will\n", "# output a complete readable dataset.\n", "\n", - "convert_colorizer_data(\n", + "convert_tfe_data(\n", " data,\n", " output_directory,\n", " source_dir=source_directory,\n", @@ -206,7 +206,7 @@ "\n", "By default, any columns that aren't mapped to data columns (e.g. object ID, times, tracks, etc.) are automatically handled as feature data. You can specify which columns are features by passing a list of column names to the `feature_columns` parameter.\n", "\n", - "While `convert_colorizer_data` will infer feature types for you, you can also provide metadata about each feature, such as its label, key, type, units, and description using the `feature_info` argument.\n", + "While `convert_tfe_data` will infer feature types for you, you can also provide metadata about each feature, such as its label, key, type, units, and description using the `feature_info` argument.\n", "\n", "Features can be one of three types:\n", "\n", @@ -262,7 +262,7 @@ "# Providing it is optional, and `feature_info` will still update metadata without it.\n", "feature_column_names = [\"area\", \"radius\", \"location\"]\n", "\n", - "convert_colorizer_data(\n", + "convert_tfe_data(\n", " data,\n", " output_directory,\n", " source_dir=source_directory,\n", @@ -293,11 +293,11 @@ "outputs": [], "source": [ "from tfe_data import (\n", - " ColorizerMetadata,\n", + " DatasetMetadata,\n", ")\n", "\n", "# Define the metadata for this dataset.\n", - "metadata = ColorizerMetadata(\n", + "metadata = DatasetMetadata(\n", " name=\"Example dataset 1\",\n", " description=\"An example dataset for the Timelapse Feature Explorer!\",\n", " author=\"Jane Doe et al.\",\n", @@ -311,7 +311,7 @@ " frame_duration_sec=1,\n", ")\n", "\n", - "convert_colorizer_data(\n", + "convert_tfe_data(\n", " data,\n", " output_directory,\n", " source_dir=source_directory,\n", @@ -359,7 +359,7 @@ "output_directory = Path(\"processed_dataset/dataset_2\")\n", "data: pd.DataFrame = pd.read_csv(source_directory / \"data.csv\")\n", "\n", - "convert_colorizer_data(\n", + "convert_tfe_data(\n", " data,\n", " output_directory,\n", " source_dir=source_directory,\n", @@ -501,7 +501,7 @@ "# If the download doesn't start (can be browser-dependent), you can manually\n", "# navigate to and download the zip file from the file browser on the left side\n", "# of the screen. It will be in\n", - "# `tfe-data/documentation/getting_started_guide/`." + "# `tfe_data/documentation/getting_started_guide/`." ] }, { @@ -587,7 +587,7 @@ "For example, for 3D data, you can specify a 3D frame source like this:\n", "\n", "```python\n", - "from tfe_data import convert_colorizer_data\n", + "from tfe_data import convert_tfe_data\n", "from tfe_data.types import Frames3dMetadata, Backdrop3dMetadata\n", "\n", "frames_3d = Frames3dMetadata(\n", @@ -620,7 +620,7 @@ " ]\n", ")\n", "\n", - "convert_colorizer_data(\n", + "convert_tfe_data(\n", " ...\n", " frames_3d=frames_3d,\n", " image_column=None, # Disable 2D frame generation\n", diff --git a/tests/test_converter.py b/tests/test_converter.py index 7c85987..80b42b5 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -8,7 +8,7 @@ import pytest from typing import Dict, List, Union -from tfe_data import convert_colorizer_data +from tfe_data import convert_tfe_data from tfe_data.types import DataFileType, FeatureMetadata, Frames3dMetadata from tfe_data.utils import read_data_array_file @@ -44,7 +44,7 @@ def existing_dataset(tmp_path_factory) -> pathlib.Path: tmp_path = tmp_path_factory.mktemp("dataset") csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, output_format=DataFileType.JSON, image_column="File Path" ) return tmp_path @@ -144,16 +144,14 @@ def validate_default_dataset( def test_handles_simple_csv(tmp_path): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data( - csv_data, tmp_path / "dataset", output_format=DataFileType.JSON - ) + convert_tfe_data(csv_data, tmp_path / "dataset", output_format=DataFileType.JSON) validate_default_dataset(tmp_path / "dataset") def test_handles_renamed_columns(tmp_path): csv_content = f"{sample_csv_headers_alternate}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path / "dataset", object_id_column="object_id", @@ -173,7 +171,7 @@ def test_handles_renamed_columns(tmp_path): def test_handles_default_csv_parquet(tmp_path): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path / "dataset", output_format=DataFileType.PARQUET, @@ -190,7 +188,7 @@ def test_fails_if_no_features_given(tmp_path): ["Continuous Feature", "Discrete Feature", "Categorical Feature"], axis=1 ) with pytest.raises(Exception): - convert_colorizer_data(csv_data, tmp_path) + convert_tfe_data(csv_data, tmp_path) def test_fails_if_no_objects_exist(tmp_path): @@ -198,14 +196,14 @@ def test_fails_if_no_objects_exist(tmp_path): csv_data = pd.read_csv(StringIO(csv_content)) with pytest.raises(Exception): - convert_colorizer_data(csv_data, tmp_path) + convert_tfe_data(csv_data, tmp_path) def test_handles_missing_centroid_and_outlier_columns(tmp_path): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) csv_data = csv_data.drop(["Centroid X", "Centroid Y", "Outlier"], axis=1) - convert_colorizer_data(csv_data, tmp_path, output_format=DataFileType.JSON) + convert_tfe_data(csv_data, tmp_path, output_format=DataFileType.JSON) # Outliers and centroids should not be written assert not os.path.exists(tmp_path / "outliers.json") @@ -223,7 +221,7 @@ def test_throws_error_if_all_values_are_outliers(tmp_path): csv_data = pd.read_csv(StringIO(csv_content)) csv_data["Outlier"] = 1 with pytest.raises(Exception): - convert_colorizer_data(csv_data, tmp_path) + convert_tfe_data(csv_data, tmp_path) def test_throws_error_if_times_column_is_missing(tmp_path): @@ -231,14 +229,14 @@ def test_throws_error_if_times_column_is_missing(tmp_path): csv_data = pd.read_csv(StringIO(csv_content)) csv_data = csv_data.drop(["Frame"], axis=1) with pytest.raises(Exception): - convert_colorizer_data(csv_data, tmp_path) + convert_tfe_data(csv_data, tmp_path) def test_uses_id_as_track_if_track_is_missing(tmp_path): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) csv_data = csv_data.drop(["Track"], axis=1) - convert_colorizer_data(csv_data, tmp_path, output_format=DataFileType.JSON) + convert_tfe_data(csv_data, tmp_path, output_format=DataFileType.JSON) manifest = {} with open(tmp_path / "manifest.json", "r") as f: @@ -259,14 +257,14 @@ def test_uses_source_dir_to_evaluate_relative_paths(tmp_path): # Check that conversion fails if no source directory is provided with pytest.raises(Exception): - convert_colorizer_data( + convert_tfe_data( data, tmp_path, ) # Current working directory is tmp_path, but all the assets are in # the asset_path directory. - convert_colorizer_data( + convert_tfe_data( data, tmp_path, source_dir=asset_path, @@ -278,7 +276,7 @@ def test_uses_source_dir_to_evaluate_relative_paths(tmp_path): def test_uses_default_segmentation_ids(tmp_path): data = pd.read_csv(StringIO(f"{sample_csv_headers}\n{sample_csv_data}")) - convert_colorizer_data( + convert_tfe_data( data, tmp_path, output_format=DataFileType.JSON, @@ -308,7 +306,7 @@ def test_writes_3d_centroids(tmp_path): csv_data = pd.read_csv(StringIO(csv_content)) dataset_dir = tmp_path / "dataset" - convert_colorizer_data( + convert_tfe_data( csv_data, dataset_dir, output_format=DataFileType.JSON, image_column="File Path" ) # Validate that centroids are written correctly @@ -334,7 +332,7 @@ def test_does_not_rewrite_existing_frames(existing_dataset): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data(csv_data, existing_dataset, force_frame_generation=False) + convert_tfe_data(csv_data, existing_dataset, force_frame_generation=False) # Frames + bbox data should not be modified assert os.path.getmtime(existing_dataset / "frame_0.png") == frame_0_time @@ -357,7 +355,7 @@ def test_detects_missing_frames(existing_dataset): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data(csv_data, existing_dataset, force_frame_generation=False) + convert_tfe_data(csv_data, existing_dataset, force_frame_generation=False) # Frames should be regenerated assert os.path.exists(existing_dataset / "frame_0.png") @@ -375,7 +373,7 @@ def test_force_image_generation_flag_works(existing_dataset): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data(csv_data, existing_dataset, force_frame_generation=True) + convert_tfe_data(csv_data, existing_dataset, force_frame_generation=True) # Frame 0 and 1 should both have newer write times assert os.path.getmtime(existing_dataset / "frame_0.png") > frame_0_time @@ -389,7 +387,7 @@ def test_rewrites_images_when_object_count_changes(existing_dataset): csv_content = f"{sample_csv_headers}\n{sample_csv_data}\n4,3,1,70,80,0.9,4,D,1,f{asset_path / 'test_csv/frame_1.tiff'}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data(csv_data, existing_dataset, force_frame_generation=False) + convert_tfe_data(csv_data, existing_dataset, force_frame_generation=False) # Frame 0 and 1 should both have newer write times assert os.path.getmtime(existing_dataset / "frame_0.png") > frame_0_time @@ -406,7 +404,7 @@ def test_regenerates_frames_if_missing_times_file(existing_dataset): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data(csv_data, existing_dataset, force_frame_generation=False) + convert_tfe_data(csv_data, existing_dataset, force_frame_generation=False) # Frame 0 and 1 should both have newer write times assert os.path.getmtime(existing_dataset / "frame_0.png") > frame_0_time @@ -420,7 +418,7 @@ def test_skips_frame_generation_if_no_image_column(existing_dataset): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data( + convert_tfe_data( csv_data, existing_dataset, force_frame_generation=False, image_column=None ) @@ -436,7 +434,7 @@ def test_writes_3d_data(tmp_path): csv_content = f"{sample_csv_headers}\n{sample_csv_data}" csv_data = pd.read_csv(StringIO(csv_content)) - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, frames_3d=Frames3dMetadata( @@ -477,7 +475,7 @@ def test_writes_backdrop_images_using_column_names(self, tmp_path): StringIO(self.backdrop_headers + "\n" + self.backdrop_csv_data) ) - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, backdrop_column_names=["Backdrop Image 1", "Backdrop Image 2"], @@ -537,7 +535,7 @@ def test_does_not_rewrite_existing_backdrop_images(self, tmp_path): csv_data = pd.read_csv( StringIO(self.backdrop_headers + "\n" + local_backdrop_csv_data) ) - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, backdrop_column_names=["Backdrop Image 1", "Backdrop Image 2"], @@ -593,7 +591,7 @@ def test_override_some_backdrop_metadata(self, tmp_path): "key": "new_backdrop_key", } } - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, backdrop_column_names=["Backdrop Image 1", "Backdrop Image 2"], @@ -641,7 +639,7 @@ def test_override_backdrop_paths(self, tmp_path): ] } } - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, backdrop_column_names=["Backdrop Image 1", "Backdrop Image 2"], @@ -690,7 +688,7 @@ def test_writes_backdrops_that_are_not_in_column_names(self, tmp_path): "A backdrop": backdrop_info_1, "Some other backdrop": backdrop_info_2, } - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, backdrop_info=backdrop_info, @@ -739,7 +737,7 @@ def test_throws_error_if_backdrop_does_not_exist(self, tmp_path): } } with pytest.raises(Exception): - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, backdrop_info=backdrop_info, @@ -757,7 +755,7 @@ def test_sanitizes_backdrop_key_names(self, tmp_path): ], } } - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, backdrop_info=backdrop_info, @@ -778,7 +776,7 @@ def test_sanitizes_backdrop_key_names(self, tmp_path): def test_handles_none_backdrop(self, tmp_path): csv_data = pd.read_csv(StringIO(sample_csv_headers + "\n" + sample_csv_data)) - convert_colorizer_data( + convert_tfe_data( csv_data, tmp_path, backdrop_column_names=["Nonexistent Backdrop"], diff --git a/tfe_data/converter.py b/tfe_data/converter.py index 12e3426..ff4bca0 100644 --- a/tfe_data/converter.py +++ b/tfe_data/converter.py @@ -488,15 +488,15 @@ def convert_tfe_data( Example: ```python import pandas as pd - from tfe_data import convert_colorizer_data + from tfe_data import convert_tfe_data # 1. Assuming CSV data has default columns "ID", "Track", # "Frame", and "File Path": data = pd.read_csv("some/path/data.csv") - convert_colorizer_data(data, "dataset_dir/dataset_name") + convert_tfe_data(data, "dataset_dir/dataset_name") # 2. If not, you can specify the column names: - convert_colorizer_data( + convert_tfe_data( data, "dataset_dir/dataset_name", object_id_column="my_id_column", @@ -523,7 +523,7 @@ def convert_tfe_data( categories=["Colony", "Edge", "Migratory"], ), } - convert_colorizer_data( + convert_tfe_data( data, "dataset_dir/dataset_name", feature_info=feature_info, From 9e89363a4ffb073922ff00620fe76fa1896ea3de Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 11:57:08 -0700 Subject: [PATCH 06/10] feat: Added a migration guide --- documentation/MIGRATION.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 documentation/MIGRATION.md diff --git a/documentation/MIGRATION.md b/documentation/MIGRATION.md new file mode 100644 index 0000000..684a413 --- /dev/null +++ b/documentation/MIGRATION.md @@ -0,0 +1,21 @@ +# Migration + +## From v1 to v2 + +1. Update imports to reflect the new package name and repository location: + +``` +# pip +pip install git+https://github.com/AllenCell/tfe-data.git@v2.0.0 + +# requirements.txt +tfe_data @ git+https://github.com/AllenCell/tfe-data.git@v2.0.0 +``` + +2. Rename the package in any imports from `colorizer_data` to `tfe_data`. + +3. Replace renamed methods and classes. The old methods/classes will continue to + work, but are marked as deprecated and will be removed in the next major + version. + - Replace `convert_colorizer_data()` with `convert_tfe_data()`. + - Replace `ColorizerMetadata` with `DatasetMetadata`. From 2f4c74b878cde6d134b83b197a451b3c97424ad0 Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 12:08:42 -0700 Subject: [PATCH 07/10] refactor: Renamed `ColorizerDatasetWriter` to `TfeDatasetWriter`. --- documentation/DATA_FORMAT.md | 2 +- documentation/MIGRATION.md | 1 + tests/test_writer.py | 38 ++++++++++++++++++------------------ tfe_data/converter.py | 22 ++++++++++----------- tfe_data/types.py | 4 ++-- tfe_data/writer.py | 18 +++++++++++------ 6 files changed, 46 insertions(+), 39 deletions(-) diff --git a/documentation/DATA_FORMAT.md b/documentation/DATA_FORMAT.md index 2055989..e490403 100644 --- a/documentation/DATA_FORMAT.md +++ b/documentation/DATA_FORMAT.md @@ -832,7 +832,7 @@ A collection file can also include optional metadata fields, saved under the `me Once you get the first frame in your dataset, you'll need to save the `starting_timepoint` to your dataset's metadata and include the information when generating the frame paths. ```python -writer = ColorizerDatasetWriter(...) +writer = TfeDatasetWriter(...) starting_timepoint = 5 num_frames = 100 diff --git a/documentation/MIGRATION.md b/documentation/MIGRATION.md index 684a413..c6dd499 100644 --- a/documentation/MIGRATION.md +++ b/documentation/MIGRATION.md @@ -19,3 +19,4 @@ tfe_data @ git+https://github.com/AllenCell/tfe-data.git@v2.0.0 version. - Replace `convert_colorizer_data()` with `convert_tfe_data()`. - Replace `ColorizerMetadata` with `DatasetMetadata`. + - Replace `ColorizerDatasetWriter` with `TfeDatasetWriter`. diff --git a/tests/test_writer.py b/tests/test_writer.py index 17a0bdb..d9236e0 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -13,7 +13,7 @@ DatasetManifest, FeatureInfo, ) -from tfe_data.writer import ColorizerDatasetWriter +from tfe_data.writer import TfeDatasetWriter DEFAULT_DATASET_NAME = "dataset" @@ -42,7 +42,7 @@ } -def setup_dummy_writer_data(writer: ColorizerDatasetWriter): +def setup_dummy_writer_data(writer: TfeDatasetWriter): writer.write_data(times=np.ndarray([0]), write_json=True) writer.set_frame_paths([""]) @@ -53,18 +53,18 @@ def setup_manifest_and_writer(path, content): manifest_path = path / DEFAULT_DATASET_NAME / "manifest.json" with open(manifest_path, "w") as f: json.dump(content, f, indent=2) - writer = ColorizerDatasetWriter(path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) return writer, path, manifest_path @pytest.fixture -def existing_manifest(tmp_path) -> Tuple[ColorizerDatasetWriter, Path, Path]: +def existing_manifest(tmp_path) -> Tuple[TfeDatasetWriter, Path, Path]: return setup_manifest_and_writer(tmp_path, EXISTING_MANIFEST_CONTENT) @pytest.fixture -def blank_manifest(tmp_path) -> Tuple[ColorizerDatasetWriter, Path, Path]: +def blank_manifest(tmp_path) -> Tuple[TfeDatasetWriter, Path, Path]: return setup_manifest_and_writer(tmp_path, BLANK_MANIFEST_CONTENT) @@ -80,7 +80,7 @@ def test_metadata_uses_frame_dims_subfield(tmp_path): for i in range(len(frame_dimensions)): width, height, units = frame_dimensions[i] - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) writer.write_manifest( metadata=DatasetMetadata( @@ -236,7 +236,7 @@ def test_writer_updates_fields_when_metadata_is_missing(blank_manifest): def test_writer_overwrites_duplicate_feature_keys(tmp_path): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_1_info = FeatureInfo(key="shared_feature_key", label="Feature 1") @@ -254,7 +254,7 @@ def test_writer_overwrites_duplicate_feature_keys(tmp_path): def test_writer_overwrites_duplicate_backdrop_keys(tmp_path): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) writer.add_backdrops("Backdrop 1", [], "shared_backdrop_key") @@ -273,7 +273,7 @@ class TestWriteFeature: def test_write_feature_ignores_outliers_when_calculating_feature_min_max( self, tmp_path ): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_info = FeatureInfo(key="feature", label="Feature") @@ -296,7 +296,7 @@ def test_write_feature_ignores_outliers_when_calculating_feature_min_max( def test_writer_ignores_infinity_values_for_feature_min_max(self, tmp_path): # Test that infinity values are ignored when calculating min/max - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_info = FeatureInfo(key="feature", label="Feature") @@ -319,7 +319,7 @@ def test_writer_ignores_infinity_values_for_feature_min_max(self, tmp_path): def test_write_feature_uses_overrides_when_calculating_feature_min_max( self, tmp_path ): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_info = FeatureInfo(key="feature", label="Feature", min=-5, max=3) @@ -338,7 +338,7 @@ def test_write_feature_uses_overrides_when_calculating_feature_min_max( assert feature_data["data"] == [0, 1, 2, 3, 4] def test_writer_throws_error_if_feature_has_no_finite_values(self, tmp_path): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_info = FeatureInfo(key="feature", label="Feature") @@ -346,7 +346,7 @@ def test_writer_throws_error_if_feature_has_no_finite_values(self, tmp_path): writer.write_feature(np.array([-np.inf, np.nan, np.inf]), feature_info) def test_writer_throws_error_if_feature_has_only_outlier_values(self, tmp_path): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_info = FeatureInfo(key="feature", label="Feature") @@ -361,7 +361,7 @@ def test_writer_serializes_infinity_values_as_strings(self, tmp_path): # Generally we want to ignore infinity values when calculating min/max, # but if the user explicitly sets min/max to infinity, we should # serialize them as strings in the JSON file to avoid parsing errors. - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_info = FeatureInfo( @@ -381,7 +381,7 @@ def test_writer_serializes_infinity_values_as_strings(self, tmp_path): assert feature_data["data"] == [0, 1, 2, np.inf, -np.inf, 4] def test_write_feature_writes_parquet_data(self, tmp_path): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_info = FeatureInfo(key="feature", label="Feature") @@ -406,7 +406,7 @@ def test_write_feature_writes_parquet_data(self, tmp_path): assert table.to_pandas()["data"].tolist() == data.tolist() def write_features_can_write_nan_to_parquet(self, tmp_path): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) setup_dummy_writer_data(writer) feature_info = FeatureInfo(key="feature", label="Feature") @@ -446,7 +446,7 @@ class TestWriteData: ), } - def write_default_data(self, writer: ColorizerDatasetWriter, write_json: bool): + def write_default_data(self, writer: TfeDatasetWriter, write_json: bool): writer.write_data( tracks=self.default_data["tracks"], times=self.default_data["times"], @@ -482,13 +482,13 @@ def validate_default_data(self, dataset_root: str, write_json: bool): self.validate_parquet(data_path, self.default_data[key]) def test_write_data_writes_json_data(self, tmp_path): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) self.write_default_data(writer, write_json=True) writer.write_manifest() self.validate_default_data(tmp_path / DEFAULT_DATASET_NAME, write_json=True) def test_write_data_writes_parquet_data(self, tmp_path): - writer = ColorizerDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) + writer = TfeDatasetWriter(tmp_path, DEFAULT_DATASET_NAME) self.write_default_data(writer, write_json=False) writer.write_manifest() self.validate_default_data(tmp_path / DEFAULT_DATASET_NAME, write_json=False) diff --git a/tfe_data/converter.py b/tfe_data/converter.py index ff4bca0..1d1fdad 100644 --- a/tfe_data/converter.py +++ b/tfe_data/converter.py @@ -35,7 +35,7 @@ sanitize_path_by_platform, scale_image, ) -from tfe_data.writer import ColorizerDatasetWriter +from tfe_data.writer import TfeDatasetWriter @dataclass @@ -67,7 +67,7 @@ def _get_image_from_row(row: pd.DataFrame, config: ConverterConfig) -> BioImage: def _make_frame( frame: pd.DataFrame, scale: float, - writer: ColorizerDatasetWriter, + writer: TfeDatasetWriter, config: ConverterConfig, ): start_time = time.time() @@ -96,7 +96,7 @@ def _make_frame( def _make_frames_parallel( grouped_frames: DataFrameGroupBy, scale: float, - writer: ColorizerDatasetWriter, + writer: TfeDatasetWriter, config: ConverterConfig, ): """ @@ -130,7 +130,7 @@ def _get_data_or_none( def _write_data( dataset: pd.DataFrame, - writer: ColorizerDatasetWriter, + writer: TfeDatasetWriter, config: ConverterConfig, ): outliers_data = _get_data_or_none(dataset, config.outlier_column) @@ -192,7 +192,7 @@ def _get_raw_backdrop_paths( def _write_backdrop_from_column( backdrop_column: str, grouped_frames: DataFrameGroupBy, - writer: ColorizerDatasetWriter, + writer: TfeDatasetWriter, config: ConverterConfig, ): backdrop_metadata = BackdropMetadata( @@ -243,7 +243,7 @@ def _write_backdrop_from_column( def _write_backdrops( dataset: pd.DataFrame, - writer: ColorizerDatasetWriter, + writer: TfeDatasetWriter, config: ConverterConfig, ): grouped_frames = dataset.groupby(config.times_column) @@ -284,7 +284,7 @@ def _get_reserved_column_names(config: ConverterConfig) -> List[str]: def _write_features( dataset: pd.DataFrame, - writer: ColorizerDatasetWriter, + writer: TfeDatasetWriter, config: ConverterConfig, ): # Detect all features @@ -324,7 +324,7 @@ def _write_features( def _should_regenerate_frames( - writer: ColorizerDatasetWriter, data: DataFrame, config: ConverterConfig + writer: TfeDatasetWriter, data: DataFrame, config: ConverterConfig ) -> bool: if "frames" not in writer.manifest: logging.info("No frames found in dataset manifest. Regenerating all frames.") @@ -362,7 +362,7 @@ def _should_regenerate_frames( return False -def _validate_manifest(writer: ColorizerDatasetWriter): +def _validate_manifest(writer: TfeDatasetWriter): if len(writer.features) == 0: raise ValueError( "No features found in dataset. At least one feature is required." @@ -370,7 +370,7 @@ def _validate_manifest(writer: ColorizerDatasetWriter): def _handle_3d_frames( - data: DataFrame, writer: ColorizerDatasetWriter, config: ConverterConfig + data: DataFrame, writer: TfeDatasetWriter, config: ConverterConfig ) -> None: # Check for 3D frame src (TODO: safe to assume Zarr?) # If 3D frame src is provided, go to 3D source (using bioio) and check the number of frames. @@ -568,7 +568,7 @@ def convert_tfe_data( source_dir = pathlib.Path.cwd() original_cwd = pathlib.Path.cwd() - writer = ColorizerDatasetWriter(parent_directory, dataset_name) + writer = TfeDatasetWriter(parent_directory, dataset_name) try: # Change source directory for evaluating relative paths diff --git a/tfe_data/types.py b/tfe_data/types.py index e352dae..7e9b4bd 100644 --- a/tfe_data/types.py +++ b/tfe_data/types.py @@ -185,7 +185,7 @@ class Frames3dMetadata(DataClassJsonMixin): @dataclass class DatasetMetadata(DataClassJsonMixin): """ - Data class representation of metadata for a Colorizer dataset. Can be + Data class representation of metadata for a TFE dataset. Can be converted to and from camelCase JSON format; see https://pypi.org/project/dataclasses-json/. """ @@ -309,7 +309,7 @@ class DatasetManifest(TypedDict): @dataclass class CollectionMetadata(DataClassJsonMixin): """ - Data class representation of metadata for a Colorizer collection file. + Data class representation of metadata for a TFE collection file. Can be converted to and from camelCase JSON format; see https://pypi.org/project/dataclasses-json/. """ diff --git a/tfe_data/writer.py b/tfe_data/writer.py index e2062d9..4afa2fe 100644 --- a/tfe_data/writer.py +++ b/tfe_data/writer.py @@ -5,6 +5,7 @@ import pathlib import shutil from typing import Dict, List, Optional, Union +from warnings import deprecated import numpy as np from PIL import Image @@ -41,9 +42,9 @@ MAX_SEG_ID_GAP = 10 -class ColorizerDatasetWriter: +class TfeDatasetWriter: """ - Writes provided data as Colorizer-compatible dataset files to the configured output directory. + Writes provided data as TFE-compatible dataset files to the configured output directory. Args: output_dir (`str | pathlib.Path`): The output directory to write the dataset to. @@ -234,7 +235,7 @@ def write_feature( fmin = np.nanmin(filtered_data) except ValueError: raise ValueError( - "ColorizerDatasetWriter.write_feature: Feature '{}' had no finite, non-outlier values when calculating min/max bounds.".format( + "TfeDatasetWriter.write_feature: Feature '{}' had no finite, non-outlier values when calculating min/max bounds.".format( info.get_name() ) + " Provide a min and max property in FeatureInfo to override automatic bounds calculation." @@ -245,7 +246,7 @@ def write_feature( fmax = np.nanmax(filtered_data) except ValueError: raise ValueError( - "ColorizerDatasetWriter.write_feature: Feature '{}' has no finite, non-outlier values when calculating min/max bounds.".format( + "TfeDatasetWriter.write_feature: Feature '{}' has no finite, non-outlier values when calculating min/max bounds.".format( info.get_name() ) + " Provide a min and max property in FeatureInfo to override automatic bounds calculation." @@ -507,7 +508,7 @@ def set_frame_paths(self, paths: List[str]) -> None: def set_3d_frame_data(self, data: Frames3dMetadata) -> None: if data.total_frames is None: logging.info( - "ColorizerDatasetWriter: The `total_frames` property of the Frames3dMetadata object is `None`. Will attempt to infer the number of frames from the provided data." + "TfeDatasetWriter: The `total_frames` property of the Frames3dMetadata object is `None`. Will attempt to infer the number of frames from the provided data." ) data.total_frames = _get_frame_count_from_3d_source(data.source) self.manifest["frames3d"] = data.to_dict() @@ -536,7 +537,7 @@ def write_manifest( if num_frames is not None and "frames" not in self.manifest: logging.warning( - "ColorizerDatasetWriter: The argument `num_frames` on `write_manifest` is deprecated and will be removed in the future! Please call `set_frame_paths(generate_frame_paths(num_frames))` instead." + "TfeDatasetWriter: The argument `num_frames` on `write_manifest` is deprecated and will be removed in the future! Please call `set_frame_paths(generate_frame_paths(num_frames))` instead." ) self.set_frame_paths(generate_frame_paths(num_frames)) @@ -722,3 +723,8 @@ def validate_dataset( check_file_source( f"3D frames backdrop {i} source", backdrop_source, self.outpath ) + + +@deprecated("ColorizerDatasetWriter is deprecated, use TfeDatasetWriter instead.") +class ColorizerDatasetWriter(TfeDatasetWriter): + pass From fedb875e23502f8cc03184a4be69dd21a05356c8 Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 13:29:48 -0700 Subject: [PATCH 08/10] refactor: Use `typing_extensions` for backwards-compatibility of @deprecated decorator --- tfe_data/converter.py | 2 +- tfe_data/types.py | 2 +- tfe_data/writer.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tfe_data/converter.py b/tfe_data/converter.py index 1d1fdad..2154db3 100644 --- a/tfe_data/converter.py +++ b/tfe_data/converter.py @@ -7,7 +7,7 @@ import shutil import time from typing import Dict, List, Optional, Union -from warnings import deprecated +from typing_extensions import deprecated from bioio import BioImage from dataclasses import dataclass diff --git a/tfe_data/types.py b/tfe_data/types.py index 7e9b4bd..c893b21 100644 --- a/tfe_data/types.py +++ b/tfe_data/types.py @@ -4,7 +4,7 @@ from dataclasses_json.core import _decode_dataclass from enum import Enum from typing import Dict, List, Optional, Type, TypeVar, TypedDict, Union -from warnings import deprecated +from typing_extensions import deprecated Json = Union[dict, str, int, float, bool, None] diff --git a/tfe_data/writer.py b/tfe_data/writer.py index 4afa2fe..cd49f54 100644 --- a/tfe_data/writer.py +++ b/tfe_data/writer.py @@ -5,7 +5,7 @@ import pathlib import shutil from typing import Dict, List, Optional, Union -from warnings import deprecated +from typing_extensions import deprecated import numpy as np from PIL import Image From db39afba6351d8563beb4086636f4ad56f9fb196 Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 13:33:34 -0700 Subject: [PATCH 09/10] fix: Removed reason argument in deprecation warning --- tfe_data/converter.py | 2 +- tfe_data/types.py | 4 +++- tfe_data/writer.py | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tfe_data/converter.py b/tfe_data/converter.py index 2154db3..e108705 100644 --- a/tfe_data/converter.py +++ b/tfe_data/converter.py @@ -629,7 +629,7 @@ def convert_tfe_data( @deprecated( - "Will be removed in the next major release. Please use convert_tfe_data instead." + "convert_colorizer_data is deprecated and will be removed in the next major release. Please use convert_tfe_data instead." ) @wraps(convert_tfe_data) def convert_colorizer_data(*args, **kwargs): diff --git a/tfe_data/types.py b/tfe_data/types.py index c893b21..3c14055 100644 --- a/tfe_data/types.py +++ b/tfe_data/types.py @@ -283,7 +283,9 @@ def from_dict( return metadata -@deprecated(reason="Use DatasetMetadata instead.") +@deprecated( + "ColorizerMetadata is deprecated and will be removed in the next major release. Please use DatasetMetadata instead." +) @dataclass class ColorizerMetadata(DatasetMetadata): pass diff --git a/tfe_data/writer.py b/tfe_data/writer.py index cd49f54..90a6283 100644 --- a/tfe_data/writer.py +++ b/tfe_data/writer.py @@ -725,6 +725,8 @@ def validate_dataset( ) -@deprecated("ColorizerDatasetWriter is deprecated, use TfeDatasetWriter instead.") +@deprecated( + "ColorizerDatasetWriter is deprecated and will be removed in the next major release. Please use TfeDatasetWriter instead." +) class ColorizerDatasetWriter(TfeDatasetWriter): pass From 1f57bb8e4c60f78be973e7c50a648496276d5846 Mon Sep 17 00:00:00 2001 From: Peyton Lee Date: Mon, 14 Sep 2026 13:51:43 -0700 Subject: [PATCH 10/10] refactor: Removed types when converting dataclass methods, added typing-extensions dependency --- pyproject.toml | 3 ++- tfe_data/types.py | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 745b3c0..46f25b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,8 +34,9 @@ dependencies = [ "pandas", "pillow", "pyarrow", - "scikit-image", "requests", + "scikit-image", + "typing-extensions", ] [project.optional-dependencies] diff --git a/tfe_data/types.py b/tfe_data/types.py index 3c14055..7711820 100644 --- a/tfe_data/types.py +++ b/tfe_data/types.py @@ -260,9 +260,7 @@ def from_dict( # inferred class. In this case, we want to explicitly pass in this class (DatasetMetadata) # and use the parent behavior, but we can't call DataClassJsonMixin.from_dict() directly # because it is unaware of DatasetMetadata's dataclass fields. - metadata: DatasetMetadata = _decode_dataclass( - DatasetMetadata, kvs, infer_missing - ) + metadata = _decode_dataclass(cls, kvs, infer_missing) if "frameDims" in kvs.keys() and isinstance(kvs["frameDims"], dict): if "width" in kvs["frameDims"].keys():