From 617684e29cd2c24344fabf2b0c085035c6e09de8 Mon Sep 17 00:00:00 2001 From: Theodoros Katzalis Date: Mon, 14 Oct 2024 14:54:18 +0200 Subject: [PATCH 1/4] bump bioimageio spec/0.5.3.3 core/0.6.9 --- setup.py | 4 ++-- tiktorch/server/session/process.py | 5 +---- vendor/core-bioimage-io-python | 2 +- vendor/spec-bioimage-io | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 29dab19c..88ddb1db 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,8 @@ ], packages=find_packages(exclude=["tests"]), # Required install_requires=[ - "bioimageio.spec==0.5.3.2", - "bioimageio.core==0.6.8", + "bioimageio.spec==0.5.3.3", + "bioimageio.core==0.6.9", "grpcio>=1.31", "numpy<2", # pytorch 2.2.2-py3.9_0 for macos is compiled with numpy 1.* "protobuf", diff --git a/tiktorch/server/session/process.py b/tiktorch/server/session/process.py index 42ebd802..6224e23a 100644 --- a/tiktorch/server/session/process.py +++ b/tiktorch/server/session/process.py @@ -9,7 +9,6 @@ from bioimageio.core import PredictionPipeline, Tensor, create_prediction_pipeline from bioimageio.spec import InvalidDescr, load_description from bioimageio.spec.model import v0_5 -from bioimageio.spec.model.v0_5 import BatchAxis from tiktorch import log from tiktorch.rpc import Shutdown @@ -37,9 +36,7 @@ def _check_shape(self, tensor_id: str, tensor: Tensor): for axis in spec.axes: source_axis_size = axis.size target_axis_size = tensor.sizes[axis.id] - if axis.id not in tensor.sizes: - ValueError(f"{axis.id} not found in {tensor.sizes}") - if isinstance(axis, BatchAxis) and axis.size is None: + if source_axis_size is None: continue try: self._validate_size(source_axis_size, target_axis_size) diff --git a/vendor/core-bioimage-io-python b/vendor/core-bioimage-io-python index 3179f83c..c59a1f8c 160000 --- a/vendor/core-bioimage-io-python +++ b/vendor/core-bioimage-io-python @@ -1 +1 @@ -Subproject commit 3179f83cda85351f1ff4c00639580ffe9d0e32b6 +Subproject commit c59a1f8c994f3720994821d0bc80633c895d9b80 diff --git a/vendor/spec-bioimage-io b/vendor/spec-bioimage-io index daa1794a..af7ab2b1 160000 --- a/vendor/spec-bioimage-io +++ b/vendor/spec-bioimage-io @@ -1 +1 @@ -Subproject commit daa1794a68dba7dee0e4661859b66221b78e8cc8 +Subproject commit af7ab2b102776352aca8f1488606d6def51cf5f8 From 20bcd835dcf69333532a72531d2eb631fccbba72 Mon Sep 17 00:00:00 2001 From: Theodoros Katzalis Date: Tue, 15 Oct 2024 21:46:52 +0200 Subject: [PATCH 2/4] include tensor id when converting numpy to pb tensor --- tests/test_converters.py | 4 ++-- tiktorch/converters.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_converters.py b/tests/test_converters.py index a197a617..cf5a51d2 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -16,11 +16,11 @@ from tiktorch.proto import inference_pb2 -def _numpy_to_pb_tensor(arr): +def _numpy_to_pb_tensor(arr, tensor_id: str = "dummy_tensor_name"): """ Makes sure that tensor was serialized/deserialized """ - tensor = numpy_to_pb_tensor(arr) + tensor = numpy_to_pb_tensor(tensor_id, arr) parsed = inference_pb2.Tensor() parsed.ParseFromString(tensor.SerializeToString()) return parsed diff --git a/tiktorch/converters.py b/tiktorch/converters.py index 5a6669b8..3159c05c 100644 --- a/tiktorch/converters.py +++ b/tiktorch/converters.py @@ -34,12 +34,12 @@ def sample_to_pb_tensors(sample: Sample) -> List[inference_pb2.Tensor]: return [xarray_to_pb_tensor(tensor_id, res_tensor.data) for tensor_id, res_tensor in sample.members.items()] -def numpy_to_pb_tensor(array: np.ndarray, axistags=None) -> inference_pb2.Tensor: +def numpy_to_pb_tensor(tensor_id: str, array: np.ndarray, axistags=None) -> inference_pb2.Tensor: if axistags: shape = [inference_pb2.NamedInt(size=dim, name=name) for dim, name in zip(array.shape, axistags)] else: shape = [inference_pb2.NamedInt(size=dim) for dim in array.shape] - return inference_pb2.Tensor(dtype=str(array.dtype), shape=shape, buffer=bytes(array)) + return inference_pb2.Tensor(tensorId=tensor_id, dtype=str(array.dtype), shape=shape, buffer=bytes(array)) def xarray_to_pb_tensor(tensor_id: str, array: xr.DataArray) -> inference_pb2.Tensor: From c4e17fdb57a9f810b72d1b32a821819591bbd423 Mon Sep 17 00:00:00 2001 From: Theodoros Katzalis Date: Fri, 18 Oct 2024 14:48:58 +0200 Subject: [PATCH 3/4] bump bioimageio core/0.6.10 --- .gitignore | 1 + setup.py | 2 +- vendor/core-bioimage-io-python | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index e7f4af11..567c0490 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ tiktorch/__pycache/ /#wrapper.py# /.#wrapper.py# .py~ +.vscode diff --git a/setup.py b/setup.py index 88ddb1db..d7210962 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ packages=find_packages(exclude=["tests"]), # Required install_requires=[ "bioimageio.spec==0.5.3.3", - "bioimageio.core==0.6.9", + "bioimageio.core==0.6.10", "grpcio>=1.31", "numpy<2", # pytorch 2.2.2-py3.9_0 for macos is compiled with numpy 1.* "protobuf", diff --git a/vendor/core-bioimage-io-python b/vendor/core-bioimage-io-python index c59a1f8c..878c6469 160000 --- a/vendor/core-bioimage-io-python +++ b/vendor/core-bioimage-io-python @@ -1 +1 @@ -Subproject commit c59a1f8c994f3720994821d0bc80633c895d9b80 +Subproject commit 878c6469d6996298d95db815e2c5ae6a0db5ec3a From 4d473bc3ffbd9b1722707a4bf2e5a06e49bcdf42 Mon Sep 17 00:00:00 2001 From: k-dominik Date: Mon, 9 Dec 2024 13:05:49 +0100 Subject: [PATCH 4/4] Move deps to environment.yml that were pip installed * for more reproducible development environments, used conda to solve all dependencies * Change submodule installation order - core depends on spec, so install spec first fixup! Move deps to environment.yml that were pip installed Try strict priority --- .github/workflows/ci.yml | 8 ++++---- Makefile | 2 +- environment.yml | 27 +++++++++++++++++++++------ setup.py | 1 + 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1481fd8e..95037b0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: auto-activate-base: false activate-environment: tiktorch-server-env environment-file: environment.yml - channel-priority: flexible + channel-priority: strict miniforge-variant: Miniforge3 - name: Get the latest commit hash and target ref run: | @@ -51,7 +51,7 @@ jobs: auto-activate-base: false activate-environment: tiktorch-server-env environment-file: environment.yml - channel-priority: flexible + channel-priority: strict miniforge-variant: Miniforge3 - name: conda diagnostics run: | @@ -89,7 +89,7 @@ jobs: with: auto-update-conda: true auto-activate-base: true - channel-priority: flexible + channel-priority: strict miniforge-variant: Miniforge3 - name: install common conda dependencies run: conda install -n base -c conda-forge conda-build setuptools_scm -y @@ -133,7 +133,7 @@ jobs: with: auto-update-conda: true auto-activate-base: true - channel-priority: flexible + channel-priority: strict miniforge-variant: Miniforge3 - name: install common conda dependencies run: conda install -n base -c conda-forge conda-build setuptools_scm -y diff --git a/Makefile b/Makefile index 9dc9c0bf..a0c713d8 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL=/bin/bash ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) TIKTORCH_ENV_NAME ?= tiktorch-server-env -SUBMODULES = ./vendor/core-bioimage-io-python ./vendor/spec-bioimage-io +SUBMODULES = ./vendor/spec-bioimage-io ./vendor/core-bioimage-io-python protos: python -m grpc_tools.protoc -I./proto --python_out=tiktorch/proto/ --grpc_python_out=tiktorch/proto/ ./proto/*.proto diff --git a/environment.yml b/environment.yml index aba40a3f..c968d26b 100644 --- a/environment.yml +++ b/environment.yml @@ -1,13 +1,14 @@ name: tiktorch-server-env channels: - - ilastik-forge - pytorch + - ilastik-forge - conda-forge + - nodefaults dependencies: # - bioimage.spec via submodule # - bioimage.core via submodule - python 3.9.* - - numpy + - numpy >=1.21,<2 - grpcio=1.44 # protobuf 5 requires protoc version > 3.19.0 that requires grpcio >= 1.44 - marshmallow-union - marshmallow=3.12.* @@ -15,7 +16,6 @@ dependencies: - protobuf - pyyaml=5.3.* - requests - - ruamel.yaml - scikit-learn - scipy - typing-extensions @@ -32,17 +32,32 @@ dependencies: - cpuonly # - cudatoolkit >=10.2 # - cudnn - # - tochvision + - torchvision # tensorflow (1.14 is the latest 1.x version on cf) # so far we don't have any 2.x models in the model zoo # tensorflow skipped for now, as it conflicts with grpcio version 1.41 # - tensorflow >=2.9,<3.0 - # convenient to use bioiamgeio.core tools - - imageio + # bioimageio.spec / bioimageio.core dependencies: + - annotated-types >=0.5.0,<1 + - email_validator + - h5py + - imageio >=2.10 + - loguru + - packaging >=17.0 + - pooch >=1.5,<2 + - pydantic >=2.7.0,<2.10 + - pydantic-core + - pydantic-settings >=2.5 + - python-dateutil + - rich + - ruyaml + - tifffile - tqdm - typer + - zipp + # dev stuff - pytest diff --git a/setup.py b/setup.py index d7210962..7aebd69c 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ "grpcio>=1.31", "numpy<2", # pytorch 2.2.2-py3.9_0 for macos is compiled with numpy 1.* "protobuf", + "pydantic>=2.7.0,<2.10", "pyyaml", "xarray", ],