From 87664216f54f3581b186c4a68fbba8f76086f112 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Tue, 31 Dec 2024 17:02:26 +0100 Subject: [PATCH 1/4] Build documentation WIP --- .github/workflows/build_docs.yaml | 66 +++++++++++++++++++++++++++++++ build_doc.py | 16 ++++++++ 2 files changed, 82 insertions(+) create mode 100644 .github/workflows/build_docs.yaml create mode 100644 build_doc.py diff --git a/.github/workflows/build_docs.yaml b/.github/workflows/build_docs.yaml new file mode 100644 index 0000000..71ee2d8 --- /dev/null +++ b/.github/workflows/build_docs.yaml @@ -0,0 +1,66 @@ +name: Build and Deploy Docs + +on: + push: + paths: + - "doc/*.md" # Trigger on changes to any markdown file + - "elf/**/*.py" # Optionally include changes in Python files + branches: + - master # Run the workflow only on pushes to the main branch + workflow_dispatch: + +# security: restrict permissions for CI jobs. +permissions: + contents: read + +jobs: + build: + name: Build Documentation + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Micromamba + uses: mamba-org/setup-micromamba@v2 + with: + micromamba-version: "latest" + environment-file: environment.yaml + init-shell: bash + cache-environment: true + post-cleanup: 'all' + + - name: Install package + shell: bash -l {0} + run: pip install -e . + + - name: Install pdoc + shell: bash -l {0} + run: pip install pdoc + + - name: Generate Documentation + shell: bash -l {0} + run: pdoc elf/ -d google -o doc/ + + - name: Verify Documentation Output + run: ls -la doc/ + + - name: Upload Documentation Artifact + uses: actions/upload-pages-artifact@v3 + with: + path: doc/ + + deploy: + name: Deploy Documentation + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GiHub Pages + uses: actions/deploy-pages@v4 diff --git a/build_doc.py b/build_doc.py new file mode 100644 index 0000000..1bc311b --- /dev/null +++ b/build_doc.py @@ -0,0 +1,16 @@ +import argparse +from subprocess import run + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--out", "-o", action="store_true") + args = parser.parse_args() + + cmd = ["pdoc", "--docformat", "google"] + + if args.out: + cmd.extend(["--out", "tmp/"]) + cmd.append("elf/") + + run(cmd) From dbffc67cd4123886bcecb50a4882e3859621a4c2 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Wed, 1 Jan 2025 11:20:43 +0100 Subject: [PATCH 2/4] Try to fix ellipsis type --- elf/util.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/elf/util.py b/elf/util.py index def6625..880142c 100644 --- a/elf/util.py +++ b/elf/util.py @@ -1,7 +1,16 @@ import numbers +import sys + from itertools import product from math import ceil -from typing import List, Sequence, Tuple, Union, TYPE_CHECKING +from typing import List, Literal, Sequence, Tuple, Union, TYPE_CHECKING + +# Ellipsis as type annontation is only supported for python >= 3.11. +# To support earlier version we use Literal[...] as type annotation, according to ChatGPT that should work. +if sys.version_info.minor < 11: + EllipsisType = Ellipsis +else: + EllipsisType = Literal[...] if TYPE_CHECKING: import numpy as np @@ -68,7 +77,7 @@ def int_to_start_stop(i: int, size: int) -> slice: # But it's worth taking a look at @clbarnes more general implementation too # https://github.com/clbarnes/h5py_like def normalize_index( - index: Union[int, slice, Ellipsis, Tuple[Union[int, slice, Ellipsis], ...]], + index: Union[int, slice, EllipsisType, Tuple[Union[int, slice, EllipsisType], ...]], shape: Tuple[int, ...] ) -> Tuple[Tuple[slice, ...], Tuple[int, ...]]: """Normalize an index for a given shape, so that is expressed as tuple of slices with correct coordinates. From cf0661f9e28d85302237f762df38b9c434987f00 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Wed, 1 Jan 2025 12:00:43 +0100 Subject: [PATCH 3/4] Update root doc strings --- .github/workflows/build.yml | 2 +- .github/workflows/build_docs.yaml | 3 +- elf/__init__.py | 49 +++++++++++++++++++++++++++++++ elf/color/__init__.py | 3 ++ elf/evaluation/__init__.py | 3 ++ elf/htm/__init__.py | 5 ++++ elf/ilastik/__init__.py | 2 ++ elf/io/__init__.py | 3 ++ elf/label_multiset/__init__.py | 3 ++ elf/mesh/__init__.py | 3 ++ elf/parallel/__init__.py | 3 ++ elf/segmentation/__init__.py | 5 ++++ elf/skeleton/__init__.py | 3 ++ elf/tracking/__init__.py | 5 ++++ elf/transformation/__init__.py | 3 ++ elf/visualisation/__init__.py | 3 ++ elf/wrapper/__init__.py | 3 ++ 17 files changed, 98 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d19e7fe..cd6e6a5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout diff --git a/.github/workflows/build_docs.yaml b/.github/workflows/build_docs.yaml index 71ee2d8..0765057 100644 --- a/.github/workflows/build_docs.yaml +++ b/.github/workflows/build_docs.yaml @@ -3,8 +3,7 @@ name: Build and Deploy Docs on: push: paths: - - "doc/*.md" # Trigger on changes to any markdown file - - "elf/**/*.py" # Optionally include changes in Python files + - "elf/**/*.py" # Trigger on any changes in python files. branches: - master # Run the workflow only on pushes to the main branch workflow_dispatch: diff --git a/elf/__init__.py b/elf/__init__.py index 9226fe7..3261940 100644 --- a/elf/__init__.py +++ b/elf/__init__.py @@ -1 +1,50 @@ +"""[elf](https://github.com/constantinpape/elf) implements image analysis functionality for large microscopy data. + +# Overview + +`elf` provides functionality for different image analysis tasks. The main functionality is: +- `elf.evaluation`: Common metrics for evaluating segmentation results. +- `elf.io`: Common interface for reading file formats for large microscopy data. +- `elf.parallel`: Parallel implementations of image analysis functions. +- `elf.segmentation`: Segmentation functions based on clustering, (lifted) multicut, mutex watershed and more. +- `elf.tracking`: Graph-based tracking algorithms. +- `elf.wrapper`: Wrapper for large microscopy data to enable on-the-fly processing. + +# Installation + +`elf` is available on conda-forge. You can install it into an existing conda environment via: +``` +conda install -c conda-forge python-elf +``` + +We also provide a environment for a development environment. To set it up: +1. Clone the elf repository: +``` +git clone https://github.com/constantinpape/elf +``` +2. Enter the root elf directory: +``` +cd elf +``` +3. Create the development environment: +``` +conda create -f environment.yaml +``` +4. Activate the environment: +``` +conda activate elf-dev +``` +5. Install elf in development mode: +``` +pip install -e . +``` + +# Usage & Examples + +Example scripts for many of `elf`'s features can be found in [example](https://github.com/constantinpape/elf/tree/master/example). + +`elf` also provides command line functionality. Currently provided are the command line interfaces: +- `view_container`: Visualize the content of any file supported by `elf.io` with napari. +""" + from .__version__ import __version__ diff --git a/elf/color/__init__.py b/elf/color/__init__.py index bf8b37c..c426245 100644 --- a/elf/color/__init__.py +++ b/elf/color/__init__.py @@ -1 +1,4 @@ +"""Glasbey and random color palettes for visualizing instance segmentations. +""" + from .palette import glasbey, random_colors diff --git a/elf/evaluation/__init__.py b/elf/evaluation/__init__.py index 9682acd..fe4dbbb 100644 --- a/elf/evaluation/__init__.py +++ b/elf/evaluation/__init__.py @@ -1,3 +1,6 @@ +"""Common metrics for evaluating instance segmentation results. +""" + from .cremi_score import cremi_score from .dice import dice_score, symmetric_best_dice_score from .rand_index import rand_index diff --git a/elf/htm/__init__.py b/elf/htm/__init__.py index 57dbcdc..33847c4 100644 --- a/elf/htm/__init__.py +++ b/elf/htm/__init__.py @@ -1,2 +1,7 @@ +"""Visualization of high-throughput / high-content microscopy data via napari. + +Examples for using this functionality are in [example/htm](https://github.com/constantinpape/elf/tree/master/example/htm). +""" + from .parser import parse_simple_htm from .visualization import view_plate, view_positional_images diff --git a/elf/ilastik/__init__.py b/elf/ilastik/__init__.py index e69de29..a914161 100644 --- a/elf/ilastik/__init__.py +++ b/elf/ilastik/__init__.py @@ -0,0 +1,2 @@ +"""Convenience functionality for parsing outputs from the ilastik carving workflow. +""" diff --git a/elf/io/__init__.py b/elf/io/__init__.py index de02313..740c9a4 100644 --- a/elf/io/__init__.py +++ b/elf/io/__init__.py @@ -1,2 +1,5 @@ +"""Common interface for reading and writing microscopy data formats for large imaging data. +""" + from .files import open_file, supported_extensions from .files import is_group, is_dataset, is_h5py, is_z5py, is_knossos diff --git a/elf/label_multiset/__init__.py b/elf/label_multiset/__init__.py index b7f4979..c0a27b1 100644 --- a/elf/label_multiset/__init__.py +++ b/elf/label_multiset/__init__.py @@ -1,3 +1,6 @@ +"""Implementation of the label multiset format used by paintera. +""" + from .create import create_multiset_from_labels, downsample_multiset, merge_multisets from .label_multiset import LabelMultiset from .serialize import serialize_multiset, deserialize_multiset, deserialize_labels diff --git a/elf/mesh/__init__.py b/elf/mesh/__init__.py index 0053355..f00c930 100644 --- a/elf/mesh/__init__.py +++ b/elf/mesh/__init__.py @@ -1 +1,4 @@ +"""Functionality for converting 3D segmentation masks to and from meshes. +""" + from .mesh import marching_cubes diff --git a/elf/parallel/__init__.py b/elf/parallel/__init__.py index 13466d5..30bb0c7 100644 --- a/elf/parallel/__init__.py +++ b/elf/parallel/__init__.py @@ -1,3 +1,6 @@ +"""Parallel implementations of image analysis functionality. +""" + from .copy_dataset import copy_dataset from .distance_transform import distance_transform from .operations import (apply_operation, add, divide, multiply, subtract, diff --git a/elf/segmentation/__init__.py b/elf/segmentation/__init__.py index 9cdef3e..167c073 100644 --- a/elf/segmentation/__init__.py +++ b/elf/segmentation/__init__.py @@ -1,3 +1,8 @@ +"""Graph-based instance segmentation based on clustering, (lifted) multicut, mutex watershed and more. + +Examples for graph-based segmentation with `elf.segmentation` can be found in [example/segmentation](https://github.com/constantinpape/elf/tree/master/example/segmentation) and for embedding-based segmentation in [example/embeddings](https://github.com/constantinpape/elf/tree/master/example/embeddings). +""" + from .clustering import (agglomerative_clustering, cluster_segmentation, cluster_segmentation_mala, mala_clustering) diff --git a/elf/skeleton/__init__.py b/elf/skeleton/__init__.py index 8b946ba..07fe270 100644 --- a/elf/skeleton/__init__.py +++ b/elf/skeleton/__init__.py @@ -1 +1,4 @@ +"""Functionality for skeletonizing 3D segmentations and reading / writing skeletons to different file formats. +""" + from .skeletonize import skeletonize, get_method_names diff --git a/elf/tracking/__init__.py b/elf/tracking/__init__.py index e69de29..710edb1 100644 --- a/elf/tracking/__init__.py +++ b/elf/tracking/__init__.py @@ -0,0 +1,5 @@ +"""Graph-based tracking functionality. The currently available implementation is based on [motile](https://github.com/funkelab/motile). + +For using this functionality, you have to install motile as an additional dependency. +An example for how to use `elf.tracking` can be found in [example/tracking](https://github.com/constantinpape/elf/tree/master/example/tracking). +""" diff --git a/elf/transformation/__init__.py b/elf/transformation/__init__.py index 5e4658b..86e1c67 100644 --- a/elf/transformation/__init__.py +++ b/elf/transformation/__init__.py @@ -1,3 +1,6 @@ +"""Functionality for applying affine and resize transformations to large image data. +""" + from .affine import (affine_matrix_2d, affine_matrix_3d, compute_affine_matrix, scale_from_matrix, translation_from_matrix, transform_subvolume_affine, transform_roi_with_affine) diff --git a/elf/visualisation/__init__.py b/elf/visualisation/__init__.py index 84e492f..9b118eb 100644 --- a/elf/visualisation/__init__.py +++ b/elf/visualisation/__init__.py @@ -1,3 +1,6 @@ +"""Napari-based visualization of segmentation results and other visualization functionality. +""" + from .edge_visualisation import visualise_edges, visualise_attractive_and_repulsive_edges from .grid_views import simple_grid_view from .object_visualisation import visualise_iou_scores, visualise_dice_scores, visualise_voi_scores diff --git a/elf/wrapper/__init__.py b/elf/wrapper/__init__.py index 1c9c099..e95db66 100644 --- a/elf/wrapper/__init__.py +++ b/elf/wrapper/__init__.py @@ -1,2 +1,5 @@ +"""Wrapper for large image data to implement lazy data processing. +""" + from .base import SimpleTransformationWrapper, TransformationWrapper from .generic import NormalizeWrapper, ThresholdWrapper, RoiWrapper From b8e443f6dc0b14c968b6777060c43dd79d8e2ef3 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Wed, 1 Jan 2025 12:04:55 +0100 Subject: [PATCH 4/4] Fix typo --- elf/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elf/util.py b/elf/util.py index 880142c..ddf8c72 100644 --- a/elf/util.py +++ b/elf/util.py @@ -7,7 +7,7 @@ # Ellipsis as type annontation is only supported for python >= 3.11. # To support earlier version we use Literal[...] as type annotation, according to ChatGPT that should work. -if sys.version_info.minor < 11: +if sys.version_info.minor >= 11: EllipsisType = Ellipsis else: EllipsisType = Literal[...]