Merge pull request #73 from AdaptiveParticles/particle_dtypes

- Add IntParticles (Note: these are not yet supported in most pyapr processing functionality)
- Add astype and __array__ methods to APRSlicer
- Remove unsafe math optimizations from default CMake build
- Update CI and deployment workflows

.github/workflows/build-deploy.yml

@@ -45,21 +45,20 @@ jobs:
       # Indicates the location of the vcpkg as a Git submodule of the project repository.
       VCPKG_ROOT: ${{ github.workspace }}/external/LibAPR/vcpkg
       CIBW_ENVIRONMENT_WINDOWS: EXTRA_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=D:\\a\\pyapr\\pyapr\\external\\LibAPR\\vcpkg\\scripts\\buildsystems\\vcpkg.cmake -DVCPKG_MANIFEST_DIR=D:\\a\\pyapr\\pyapr\\external\\LibAPR\\"
-      CIBW_BUILD: "cp36-* cp37-* cp38-* cp39-* cp310-*"
+      CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-*"
       CIBW_SKIP: "*musllinux*"
       CIBW_ARCHS: "auto64"
-      CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_24"
       CIBW_BUILD_VERBOSITY: 1
-      CIBW_REPAIR_WHEEL_COMMAND_MACOS: "pip uninstall -y delocate && pip install git+https://github.com/Chia-Network/delocate.git && delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} -v {wheel}"
+      CIBW_REPAIR_WHEEL_COMMAND_MACOS: "pip install -U delocate && delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} -v {wheel}"
       CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "pip install -U wheel delvewheel && python fix_windows_wheel.py {wheel} {dest_dir}"
       CIBW_TEST_REQUIRES: "pytest pytest-qt pytest-xvfb"
       CIBW_TEST_COMMAND: "python3 -m pytest -vv {project}/pyapr/tests"
       CIBW_TEST_SKIP: "*-win_amd64"  # windows tests are run separately
-      CIBW_BEFORE_BUILD_LINUX: "apt update && apt install -y libtiff5-dev libhdf5-dev"
+      CIBW_BEFORE_BUILD_LINUX: "yum makecache && yum install -y libtiff-devel hdf5-devel"
       CIBW_ENVIRONMENT_MACOS: CPPFLAGS="-I/usr/local/opt/llvm/include" LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib" CXX="/usr/local/opt/llvm/bin/clang++" CC="/usr/local/opt/llvm/bin/clang"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: true
@@ -109,7 +108,7 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python3 -m pip install cibuildwheel==2.5.0
+          python3 -m pip install cibuildwheel
 
       - name: Install OpenMP dependencies with brew for OSX
         if: contains(matrix.os,'macos')
@@ -139,11 +138,11 @@ jobs:
       fail-fast: false
       matrix:
         os: [ windows-latest ]
-        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: false
 

.github/workflows/deploy-docs.yml

@@ -32,7 +32,7 @@ jobs:
       VCPKG_ROOT: ${{ github.workspace }}/external/LibAPR/vcpkg
       EXTRA_CMAKE_ARGS: "-DCMAKE_TOOLCHAIN_FILE='${{ github.workspace }}/external/LibAPR/vcpkg/scripts/buildsystems/vcpkg.cmake'"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: true
@@ -75,7 +75,7 @@ jobs:
 
       - name: Check file existence
         id: check_files
-        uses: andstor/file-existence-action@v1
+        uses: andstor/file-existence-action@v2
         with:
           files: "${{ env.VCPKG_ROOT }}/vcpkg"
 

.github/workflows/quick-test.yml

@@ -37,7 +37,7 @@ jobs:
       VCPKG_ROOT: ${{ github.workspace }}/external/LibAPR/vcpkg
       EXTRA_CMAKE_ARGS: "-DCMAKE_TOOLCHAIN_FILE='${{ github.workspace }}/external/LibAPR/vcpkg/scripts/buildsystems/vcpkg.cmake'"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: true
@@ -80,7 +80,7 @@ jobs:
 
       - name: Check file existence
         id: check_files
-        uses: andstor/file-existence-action@v1
+        uses: andstor/file-existence-action@v2
         with:
           files: "${{ env.VCPKG_ROOT }}/vcpkg"
 

CMakeLists.txt

@@ -101,25 +101,25 @@ if(WIN32)
         message(STATUS "Compiling on windows with CLANG!")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fcxx-exceptions")
         set(CMAKE_CXX_FLAGS_DEBUG "/MD /Z7")
-        set(CMAKE_CXX_FLAGS_RELEASE "/MD /EHsc /std:c++17 /arch:AVX2 -Xclang -O3 /nologo /fp:fast") #-flto=thin -march=native /O2 /Ob2
+        set(CMAKE_CXX_FLAGS_RELEASE "/MD /EHsc /std:c++17 /arch:AVX2 -Xclang -O3 /nologo") #-flto=thin -march=native /O2 /Ob2
     endif()
 
     if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
         message(STATUS "Compiling on windows with MSVC!")
-        set(CMAKE_CXX_FLAGS_RELEASE "/MD /EHsc /std:c++17 /arch:AVX2 /O2 /Ob2 /nologo /fp:fast")
+        set(CMAKE_CXX_FLAGS_RELEASE "/MD /EHsc /std:c++17 /arch:AVX2 /O2 /Ob2 /nologo")
         set(CMAKE_CXX_FLAGS_DEBUG "/MD /Z7")
     endif()
 
 else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -pedantic ")
     if(CMAKE_COMPILER_IS_GNUCC)
-        set(CMAKE_CXX_FLAGS_RELEASE "-O4 -ffast-math")
+        set(CMAKE_CXX_FLAGS_RELEASE "-O3")
         set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")
         set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Bdynamic")
         set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -ldl -lz")
 
     elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-        set(CMAKE_CXX_FLAGS_RELEASE "-O3 -ffast-math")
+        set(CMAKE_CXX_FLAGS_RELEASE "-O3")
         set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")
         set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lz")
     endif()

README.md

@@ -5,49 +5,89 @@
 [![License](https://img.shields.io/pypi/l/pyapr.svg?color=green)](https://raw.githubusercontent.com/AdaptiveParticles/pyapr/master/LICENSE)
 [![Python Version](https://img.shields.io/pypi/pyversions/pyapr.svg?color=blue)]((https://python.org))
 [![PyPI](https://img.shields.io/pypi/v/pyapr.svg?color=green)](https://pypi.org/project/pyapr/)
-![PowerShell Gallery](https://img.shields.io/powershellgallery/p/DNS.1.1.1.1)
-[![DOI](https://zenodo.org/badge/184399854.svg)](https://zenodo.org/badge/latestdoi/184399854)
+[![Downloads](https://static.pepy.tech/badge/pyapr)](https://pepy.tech/project/pyapr)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7304045.svg)](https://doi.org/10.5281/zenodo.7304045)
 
 Documentation can be found [here](https://adaptiveparticles.github.io/pyapr/index.html).
 
-Content-adaptive storage and processing of large volumetric microscopy data using 
-the Adaptive Particle Representation (APR).
+Content-adaptive storage and processing of large volumetric microscopy data using the Adaptive Particle Representation (APR).
+
+The APR is an adaptive image representation designed primarily for large 3D fluorescence microscopy datasets. By replacing pixels with particles positioned according to the image content, it enables orders-of-magnitude compression of sparse image data while maintaining image quality. However, unlike most compression formats, the APR can be used directly in a wide range of processing tasks - even on the GPU!
+
+| Pixels | APR |
+| :--: | :--: |
+| ![img.png](./docs/images/pix_joined.png) | ![img.png](./docs/images/apr_joined.png) |
+| Uniform sampling | Adaptive sampling |
+
+*[image source](https://bbbc.broadinstitute.org/bbbc/BBBC032), 
+[illustration source](https://ieeexplore.ieee.org/abstract/document/9796006)*
 
-The APR is an adaptive image representation designed primarily for large 3D fluorescence
-microscopy datasets. By replacing pixels with particles positioned according to the
-image content, it enables orders-of-magnitude compression of sparse image data
-while maintaining image quality. However, unlike most compression formats, the APR
-can be used directly in a wide range of processing tasks - even on the GPU!
 
 For more detailed information about the APR and its use, see:
 - [Adaptive particle representation of fluorescence microscopy images](https://www.nature.com/articles/s41467-018-07390-9) (nature communications)
 - [Parallel Discrete Convolutions on Adaptive Particle Representations of Images](https://ieeexplore.ieee.org/abstract/document/9796006) (IEEE Transactions on Image Processing)
 
 **pyapr** is built on top of the C++ library [LibAPR] using [pybind11].
 
-## Installation
-For Windows 10, OSX, and Linux and Python versions 3.7-3.9 direct installation with OpenMP support should work via [pip]:
+## Quick start guide
+
+Convert images to APR using minimal amounts of code (*see [get_apr_demo](demo/get_apr_demo.py) and [get_apr_interactive_demo](demo/get_apr_interactive_demo.py) for additional options*).
+
+```python
+import pyapr
+from skimage import io
+
+# read image into numpy array
+img = io.imread('my_image.tif')
+
+# convert to APR using default settings
+apr, parts = pyapr.converter.get_apr(img)
+
+# write APR to file
+pyapr.io.write('my_image.apr', apr, parts)
 ```
-pip install pyapr
+
+![img.png](./docs/images/apr_file.png)
+
+To return to the pixel representation:
+```python
+# reconstruct pixel image
+img = pyapr.reconstruction.reconstruct_constant(apr, parts)
 ```
-Note: Due to the use of OpenMP, it is encouraged to install as part of a virtualenv.
 
-See [INSTALL] for manual build instructions.
 
-## Exclusive features
+Inspect APRs using our makeshift image viewers (*see [napari-apr-viewer] for less experimental visualization options*).
+
+```python
+# read APR from file
+apr, parts = pyapr.io.read('my_image.apr')
 
-In addition to providing wrappers for most of the functionality of LibAPR, we provide a number of
-new features that simplify the generation and handling of the APR. For example:
+# launch viewer
+pyapr.viewer.parts_viewer(apr, parts)
+```
+![img.png](./docs/images/view_apr.png)
 
-* Interactive APR conversion (see [get_apr_interactive_demo](demo/get_apr_interactive_demo.py) and 
-  [get_apr_by_block_interactive_demo](demo/get_apr_by_block_interactive_demo.py))
-* Interactive APR z-slice viewer (see [viewer_demo](demo/viewer_demo.py))
-* Interactive APR raycast (maximum intensity projection) viewer (see [raycast_demo](demo/raycast_demo.py))
-* Interactive lossy compression of particle intensities (see [compress_particles_demo](demo/compress_particles_demo.py))
+The `View Level` toggle allows you to see the adaptation (brighter = higher resolution).
 
-For further examples see the [demo scripts].
+![img.png](./docs/images/view_level.png)
 
-Also be sure to check out our (experimental) [napari] plugin: [napari-apr-viewer].
+Or view the result in 3D using APR-native maximum intensity projection raycast (cpu).
+```python
+# launch raycast viewer
+pyapr.viewer.raycast_viewer(apr, parts)
+```
+![img.png](./docs/images/raycast.png)
+
+See the [demo scripts] for more examples.
+
+## Installation
+For Windows 10, OSX, and Linux direct installation with OpenMP support should work via [pip]:
+```
+pip install pyapr
+```
+Note: Due to the use of OpenMP, it is encouraged to install as part of a virtualenv.
+
+See [INSTALL] for manual build instructions.
 
 
 ## License

pyapr/data_containers/__init__.py

@@ -1,12 +1,13 @@
 from _pyaprwrapper.data_containers import APR, APRParameters, FloatParticles, ShortParticles, LongParticles, \
-    ByteParticles, ReconPatch, PixelDataByte, PixelDataShort, PixelDataFloat, APRPtrVector, LazyAccess, \
+    ByteParticles, IntParticles, ReconPatch, PixelDataByte, PixelDataShort, PixelDataFloat, APRPtrVector, LazyAccess, \
     LazyDataByte, LazyDataShort, LazyDataLong, LazyDataFloat, LazyIterator, LinearIterator
 
 __all__ = [
     'APR',
     'APRParameters',
     'ByteParticles',
     'ShortParticles',
+    'IntParticles',
     'FloatParticles',
     'LongParticles',
     'ReconPatch',

pyapr/data_containers/src/BindParticleData.hpp

@@ -19,6 +19,7 @@ struct TypeParseTraits{
 
 REGISTER_PARSE_TYPE(uint8);
 REGISTER_PARSE_TYPE(uint16);
+REGISTER_PARSE_TYPE(int32);
 REGISTER_PARSE_TYPE(uint64);
 REGISTER_PARSE_TYPE(float);
 

pyapr/reconstruction/APRSlicer.py

@@ -1,6 +1,6 @@
-from _pyaprwrapper.data_containers import APR, ReconPatch, ByteParticles, ShortParticles, FloatParticles, LongParticles
+from _pyaprwrapper.data_containers import APR, ReconPatch, ByteParticles, ShortParticles, FloatParticles, LongParticles, IntParticles
 from _pyaprwrapper.tree import fill_tree_mean, fill_tree_max
-from ..utils import particles_to_type
+from ..utils import particles_to_type, type_to_particles
 from .._common import _check_input
 from . import reconstruct_constant, reconstruct_level, reconstruct_smooth
 import numpy as np
@@ -23,7 +23,7 @@ class APRSlicer:
     """
     def __init__(self,
                  apr: APR,
-                 parts: Union[ByteParticles, ShortParticles, LongParticles, FloatParticles],
+                 parts: Union[ByteParticles, ShortParticles, LongParticles, FloatParticles, IntParticles],
                  mode: str = 'constant',
                  level_delta: int = 0,
                  tree_mode: str = 'mean'):
@@ -66,6 +66,14 @@ def shape(self):
     @property
     def ndim(self):
         return 3
+
+    def __array__(self):
+        # allows things like np.max(APRSlicer)
+        return np.array(self.parts)
+
+    def astype(self, typespec):
+        parts = type(type_to_particles(typespec))(np.array(self.parts).astype(typespec))
+        return APRSlicer(self.apr, parts, mode=self.mode, level_delta=self.patch.level_delta, tree_mode='max' if typespec in [int, np.int32, 'int', 'int32'] else 'mean')
 
     def new_empty_slice(self):
         return np.zeros((self.patch.z_end-self.patch.z_begin,

pyapr/reconstruction/src/BindReconstruction.hpp

@@ -275,11 +275,13 @@ void AddReconstruction(py::module &m) {
     bindReconstruct<uint16_t>(m);
     bindReconstruct<uint64_t>(m);
     bindReconstruct<float>(m);
+    bindReconstruct<int32_t>(m);
 
     bindReconstructPatch<uint8_t>(m);
     bindReconstructPatch<uint16_t>(m);
     bindReconstructPatch<uint64_t>(m);
     bindReconstructPatch<float>(m);
+    bindReconstructPatch<int32_t>(m);
 
     bindReconstructLazy<uint8_t>(m);
     bindReconstructLazy<uint16_t>(m);

pyapr/tests/test_reconstruction.py

@@ -41,3 +41,14 @@ def test_reconstruction(mode, ndim):
 
         assert np.allclose(slicer[0], lazy_slicer[0])
         assert np.allclose(slicer[0, float(0), :], lazy_slicer[0, float(0), :])
+
+    assert np.max(slicer) == parts.max() == np.max(np.array(parts))
+
+    slicer = slicer.astype(np.float32)
+    assert isinstance(slicer.parts, pyapr.FloatParticles)
+
+    slicer = slicer.astype(int)
+    assert isinstance(slicer.parts, pyapr.IntParticles)
+
+    with pytest.raises(ValueError):
+        slicer = slicer.astype(np.int8)

pyapr/tests/test_utils.py

@@ -7,6 +7,7 @@
 def test_utils():
     assert isinstance(pyapr.utils.type_to_particles(np.uint8), pyapr.ByteParticles)
     assert isinstance(pyapr.utils.type_to_particles(np.uint16), pyapr.ShortParticles)
+    assert isinstance(pyapr.utils.type_to_particles(np.int32), pyapr.IntParticles)
     assert isinstance(pyapr.utils.type_to_particles(np.uint64), pyapr.LongParticles)
     assert isinstance(pyapr.utils.type_to_particles(np.float32), pyapr.FloatParticles)
 
@@ -17,6 +18,7 @@ def test_utils():
 
     assert pyapr.utils.particles_to_type(pyapr.ByteParticles()) is np.uint8
     assert pyapr.utils.particles_to_type(pyapr.ShortParticles()) is np.uint16
+    assert pyapr.utils.particles_to_type(pyapr.IntParticles()) is np.int32
     assert pyapr.utils.particles_to_type(pyapr.LongParticles()) is np.uint64
     assert pyapr.utils.particles_to_type(pyapr.FloatParticles()) is np.float32
 
@@ -29,7 +31,7 @@ def test_utils():
         pyapr.utils.particles_to_type(np.zeros(5, dtype=np.uint16))
 
     with pytest.raises(ValueError):
-        pyapr.utils.type_to_particles(np.int32)
+        pyapr.utils.type_to_particles(np.int64)
 
     with pytest.raises(ValueError):
         pyapr.utils.type_to_lazy_particles(np.float64)

pyapr/tree/src/BindFillTree.hpp

@@ -103,16 +103,19 @@ void AddFillTree(py::module &m) {
     bindFillTreeMean<uint16_t>(m);
     bindFillTreeMean<uint64_t>(m);
     bindFillTreeMean<float>(m);
+    bindFillTreeMean<int32_t>(m);
 
     bindFillTreeMin<uint8_t>(m);
     bindFillTreeMin<uint16_t>(m);
     bindFillTreeMin<uint64_t>(m);
     bindFillTreeMin<float>(m);
+    bindFillTreeMin<int32_t>(m);
 
     bindFillTreeMax<uint8_t>(m);
     bindFillTreeMax<uint16_t>(m);
     bindFillTreeMax<uint64_t>(m);
     bindFillTreeMax<float>(m);
+    bindFillTreeMax<int32_t>(m);
 
     bindSampleFromTree<uint8_t>(m);
     bindSampleFromTree<uint16_t>(m);

pyapr/utils/types.py

@@ -1,4 +1,4 @@
-from _pyaprwrapper.data_containers import ByteParticles, ShortParticles, FloatParticles, LongParticles, \
+from _pyaprwrapper.data_containers import ByteParticles, ShortParticles, FloatParticles, LongParticles, IntParticles, \
                                           LazyDataByte, LazyDataShort, LazyDataFloat, LazyDataLong
 import numpy as np
 from typing import Union
@@ -32,8 +32,10 @@ def type_to_particles(typespec: Union[str, type]) -> ParticleData:
         return ByteParticles()
     if typespec in ('uint64', np.uint64):
         return LongParticles()
+    if typespec in (int, 'int', 'int32', np.int32):
+        return IntParticles()
     raise ValueError(f'Type {typespec} is currently not supported. Valid types are \'uint8\', \'uint16\', '
-                     f'\'uint64\' and \'float\'')
+                     f'\'uint64\', \'int\' and \'float\'')
 
 
 def type_to_lazy_particles(typespec: Union[str, type]) -> LazyData:
@@ -87,4 +89,6 @@ def particles_to_type(parts: Union[ParticleData, LazyData]) -> type:
         return np.uint8
     if isinstance(parts, (LongParticles, LazyDataLong)):
         return np.uint64
+    if isinstance(parts, IntParticles):
+        return np.int32
     raise TypeError(f'Input must be of type {ParticleData} or {LazyData} ({type(parts)} was provided)')