diff --git a/.github/workflows/build_and_test_mac.yml b/.github/workflows/build_and_test_mac.yml index 2720bba5..a5a6bb60 100644 --- a/.github/workflows/build_and_test_mac.yml +++ b/.github/workflows/build_and_test_mac.yml @@ -30,7 +30,7 @@ jobs: with: activate-environment: anaconda-client-env miniconda-version: "latest" - python-version: "3.10" + python-version: "3.11" channels: conda-forge channel-priority: strict auto-activate-base: false @@ -73,7 +73,7 @@ jobs: with: activate-environment: anaconda-client-env miniconda-version: "latest" - python-version: "3.10" + python-version: "3.11" channels: conda-forge channel-priority: strict auto-activate-base: false diff --git a/.github/workflows/build_and_test_ubuntu.yml b/.github/workflows/build_and_test_ubuntu.yml index cc9b0ff9..e54db4cb 100644 --- a/.github/workflows/build_and_test_ubuntu.yml +++ b/.github/workflows/build_and_test_ubuntu.yml @@ -31,7 +31,7 @@ jobs: with: activate-environment: anaconda-client-env miniconda-version: "latest" - python-version: "3.10" + python-version: "3.11" channels: conda-forge auto-activate-base: false @@ -67,7 +67,7 @@ jobs: with: activate-environment: anaconda-client-env miniconda-version: "latest" - python-version: "3.10" + python-version: "3.11" channels: conda-forge auto-activate-base: false diff --git a/.github/workflows/build_and_test_windows.yml b/.github/workflows/build_and_test_windows.yml index 48f51872..39c67228 100644 --- a/.github/workflows/build_and_test_windows.yml +++ b/.github/workflows/build_and_test_windows.yml @@ -31,7 +31,7 @@ jobs: with: activate-environment: anaconda-client-env miniconda-version: "latest" - python-version: "3.10" + python-version: "3.11" channels: conda-forge auto-activate-base: false @@ -71,7 +71,7 @@ jobs: with: activate-environment: anaconda-client-env miniconda-version: "latest" - python-version: "3.10" + python-version: "3.11" channels: conda-forge auto-activate-base: false diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 44a6ad2a..9010fc04 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -39,7 +39,7 @@ jobs: - name: Copy 3rd-party libraries to staging directory working-directory: ${{github.workspace}} run: | - bash ci-utils/docker_copy_3rd_party_libs.sh ${{github.workspace}}/miniconda-for-nyxus/envs/nyxus-3.9 + bash ci-utils/docker_copy_3rd_party_libs.sh ${{github.workspace}}/miniconda-for-nyxus/envs/nyxus-3.11 - name: Set version from .version file run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 9679d11c..71318bc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,28 +83,16 @@ if(USEGPU) endif() if(USE_Z5) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - #since xtensor does not built with GCC 7.5 and lower - if(NOT(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.5)) - set(USE_Z5 OFF) - message(WARNING "OMEZarr support is not available with GCC 7.5 and older compiler.") - endif() - endif() find_package(ZLIB) if (NOT ZLIB_FOUND) set(USE_Z5 OFF) - message(WARNING "zlib not found. Blosc and hereby OMEZarr support will not be available.") + message(WARNING "zlib not found. OMEZarr support will not be available.") endif() find_package(BLOSC) if(NOT BLOSC_FOUND) set(USE_Z5 OFF) message(WARNING "Blosc not found. OMEZarr support will not be available.") endif() - find_package(Boost) - if(NOT Boost_FOUND) - set(USE_Z5 OFF) - message(WARNING "Boost not found. OMEZarr support will not be available.") - endif() find_package(nlohmann_json) if(NOT nlohmann_json_FOUND) @@ -282,10 +270,6 @@ if(USE_Z5) list(APPEND Nyxus_LIBRARIES ${BLOSC_LIBRARIES}) endif() - if(Boost_FOUND) - include_directories(${Boost_INCLUDE_DIR}) - endif() - if (ZLIB_FOUND) list(APPEND Nyxus_LIBRARIES ${ZLIB_LIBRARIES} ) include_directories (${ZLIB_INCLUDE_DIR}) @@ -398,15 +382,30 @@ if(USE_ARROW) endif() find_package(Parquet) - if (NOT Parquet_FOUND) + if (NOT Parquet_FOUND) message(WARNING "The Parquet library was not found. The build will continue without Arrow support.") set(USE_ARROW OFF) endif() + # Building Arrow support pulls in Boost headers. Try to locate Boost so we can + # add its include dirs, but do NOT gate Arrow on find_package(Boost) succeeding: + # some prereq layouts ship Boost as headers-only under the common include prefix + # (no CMake config / no Boost_INCLUDE_DIR), which is enough to build against. + find_package(Boost QUIET) + if(NOT Boost_FOUND) + message(STATUS "Boost was not found via find_package; relying on Boost headers on the include path for Arrow support.") + endif() + endif() if(USE_ARROW) add_definitions(-DUSE_ARROW) + if(Boost_FOUND) + include_directories(${Boost_INCLUDE_DIRS}) + elseif(DEFINED ENV{NYXUS_DEP_DIR}) + # headers-only Boost shipped under the dependency prefix (no CMake config) + include_directories("$ENV{NYXUS_DEP_DIR}/include") + endif() list(APPEND Nyxus_LIBRARIES arrow_shared) list(APPEND Nyxus_LIBRARIES parquet_shared) endif() diff --git a/Dockerfile b/Dockerfile index 16a995d2..612c26ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:11.5.2-runtime-ubuntu20.04 +FROM nvidia/cuda:12.6.0-runtime-ubuntu22.04 COPY VERSION / diff --git a/ci-utils/build_conda.sh b/ci-utils/build_conda.sh index eb3e6c80..bd3466e5 100755 --- a/ci-utils/build_conda.sh +++ b/ci-utils/build_conda.sh @@ -17,7 +17,7 @@ MINICONDA=$PWD/miniconda-for-nyxus # Modify this to your preferred location for CPP_BUILD_DIR=$PWD SRC_ROOT=$1 #source dir location NYXUS_ROOT=$SRC_ROOT -PYTHON=3.9 +PYTHON=3.11 git config --global --add safe.directory $NYXUS_ROOT @@ -66,6 +66,7 @@ mkdir -p $CPP_BUILD_DIR pushd $CPP_BUILD_DIR cmake -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \ -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ + -DCMAKE_CUDA_COMPILER=$CONDA_PREFIX/bin/nvcc \ -DBUILD_CLI=ON \ -DUSEGPU=ON \ -DALLEXTRAS=ON \ diff --git a/ci-utils/envs/conda_cpp.txt b/ci-utils/envs/conda_cpp.txt index 8a963274..4772fbaa 100644 --- a/ci-utils/envs/conda_cpp.txt +++ b/ci-utils/envs/conda_cpp.txt @@ -1,11 +1,9 @@ -z5py +z5py >=3.0.1 libtiff <=4.7.0 boost >=1.63 nlohmann_json blosc pybind11 >= 2.12.0 -xtensor >=0.26,<0.27 -xsimd >=13,<14 cmake dcmtk >=3.6.9 fmjpeg2koj >=1.0.3 diff --git a/ci-utils/envs/conda_cpp_mac.txt b/ci-utils/envs/conda_cpp_mac.txt index 8a074ebe..b041fd16 100644 --- a/ci-utils/envs/conda_cpp_mac.txt +++ b/ci-utils/envs/conda_cpp_mac.txt @@ -1,13 +1,11 @@ # Mac CI does not exercise DICOM support, and conda-forge does not provide a # satisfiable combination of dcmtk/fmjpeg2koj with the pinned libtiff stack. -z5py +z5py >=3.0.1 libtiff <=4.7.0 boost >=1.63 nlohmann_json blosc pybind11 >= 2.12.0 -xtensor >=0.26,<0.27 -xsimd >=13,<14 cmake libarrow libparquet diff --git a/ci-utils/envs/conda_linux_gpu.txt b/ci-utils/envs/conda_linux_gpu.txt index 35545b83..da5841d9 100644 --- a/ci-utils/envs/conda_linux_gpu.txt +++ b/ci-utils/envs/conda_linux_gpu.txt @@ -1,2 +1,4 @@ -cudatoolkit=11.5.0 -cudatoolkit-dev=11.5.0 \ No newline at end of file +cuda-version=12.6 +cuda-nvcc=12.6 +cuda-cudart-dev=12.6 +libcufft-dev diff --git a/ci-utils/install_prereq_linux.sh b/ci-utils/install_prereq_linux.sh index 74a41817..9b1782a4 100755 --- a/ci-utils/install_prereq_linux.sh +++ b/ci-utils/install_prereq_linux.sh @@ -61,9 +61,21 @@ unzip zlib131.zip cd zlib-1.3.1 mkdir build_man cd build_man -cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ .. -cmake --build . -cmake --build . --target install +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ .. +cmake --build . +cmake --build . --target install +cd ../../ + +# libdeflate is a base dependency (used by libtiff) and is also the gzip/zlib +# backend for z5 3.0.1 (find_package(libdeflate CONFIG REQUIRED)), so install it +# before the z5 build below. +curl -L https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.zip -o v1.19.zip +unzip v1.19.zip +cd libdeflate-1.19 +mkdir build_man +cd build_man +cmake -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ -DCMAKE_PREFIX_PATH=../../"$LOCAL_INSTALL_DIR"/ .. +make install -j4 cd ../../ if [[ $BUILD_BOOST_DEP -eq 1 ]]; then @@ -93,48 +105,18 @@ if [[ $BUILD_Z5_DEP -eq 1 ]]; then cmake --build . --target install cd ../../ - curl -L https://github.com/xtensor-stack/xtl/archive/refs/tags/0.8.0.zip -o 0.8.0.zip - unzip 0.8.0.zip - cd xtl-0.8.0 - mkdir build_man - cd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ .. - cmake --build . - cmake --build . --target install - cd ../../ - - curl -L https://github.com/xtensor-stack/xtensor/archive/refs/tags/0.26.0.zip -o 0.26.0.zip - unzip 0.26.0.zip - cd xtensor-0.26.0 - mkdir build_man - cd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ .. - cmake --build . - cmake --build . --target install - cd ../../ - - curl -L https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.2.0.zip -o 13.2.0.zip - unzip 13.2.0.zip - cd xsimd-13.2.0 - mkdir build_man - cd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ .. - cmake --build . - cmake --build . --target install - cd ../../ - curl -L https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.zip -o v3.11.2.zip - unzip v3.11.2.zip + unzip v3.11.2.zip cd json-3.11.2 mkdir build_man cd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ .. + cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ .. make install/fast cd ../../ - curl -L https://github.com/constantinpape/z5/archive/refs/tags/2.0.20.zip -o 2.0.20.zip - unzip 2.0.20.zip - cd z5-2.0.20 + curl -L https://github.com/constantinpape/z5/archive/refs/tags/3.0.1.zip -o 3.0.1.zip + unzip 3.0.1.zip + cd z5-3.0.1 mkdir build_man cd build_man cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ -DCMAKE_PREFIX_PATH=../../"$LOCAL_INSTALL_DIR"/ -DWITH_BLOSC=ON -DBUILD_Z5PY=OFF .. @@ -172,16 +154,6 @@ if [[ $BULD_DCMTK_DEP -eq 1 ]]; then cd ../../ fi -curl -L https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.zip -o v1.19.zip -unzip v1.19.zip -cd libdeflate-1.19 -mkdir build_man -cd build_man -cmake -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ -DCMAKE_PREFIX_PATH=../../"$LOCAL_INSTALL_DIR"/ .. -make install -j4 -cd ../../ - - for i in {1..5} do curl -L https://download.osgeo.org/libtiff/tiff-4.7.0.zip -o tiff-4.7.0.zip diff --git a/ci-utils/install_prereq_win.bat b/ci-utils/install_prereq_win.bat index aefed80a..d17b8b43 100644 --- a/ci-utils/install_prereq_win.bat +++ b/ci-utils/install_prereq_win.bat @@ -40,8 +40,21 @@ tar -xvf zlib131.zip pushd zlib-1.3.1 mkdir build_man pushd build_man -cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ .. -cmake --build . --config Release --target install --parallel 4 +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ .. +cmake --build . --config Release --target install --parallel 4 +popd +popd + +rem libdeflate is a base dependency (used by libtiff) and is also the gzip/zlib +rem backend for z5 3.0.1 (find_package(libdeflate CONFIG REQUIRED)), so install it +rem before the z5 build below. +curl -L https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.zip -o v1.19.zip +tar -xf v1.19.zip +pushd libdeflate-1.19 +mkdir build_man +pushd build_man +cmake -DCMAKE_INSTALL_PREFIX=../../local_install/ -DCMAKE_PREFIX_PATH=%ABS_INSTALL% .. +cmake --build . --config Release --target install --parallel 4 popd popd @@ -71,49 +84,19 @@ if "%BUILD_Z5_DEP%" == "1" ( popd popd - curl -L https://github.com/xtensor-stack/xtl/archive/refs/tags/0.8.0.zip -o 0.8.0.zip - tar -xf 0.8.0.zip - pushd xtl-0.8.0 - mkdir build_man - pushd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ .. - cmake --build . --config Release --target install - popd - popd - - curl -L https://github.com/xtensor-stack/xtensor/archive/refs/tags/0.26.0.zip -o 0.26.0.zip - tar -xf 0.26.0.zip - pushd xtensor-0.26.0 - mkdir build_man - pushd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ .. - cmake --build . --config Release --target install - popd - popd - - curl -L https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.2.0.zip -o 13.2.0.zip - tar -xf 13.2.0.zip - pushd xsimd-13.2.0 - mkdir build_man - pushd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ .. - cmake --build . --config Release --target install - popd - popd - curl -L https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.zip -o v3.11.2.zip - tar -xf v3.11.2.zip + tar -xf v3.11.2.zip pushd json-3.11.2 mkdir build_man pushd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ -DJSON_BuildTests=OFF .. + cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ -DJSON_BuildTests=OFF .. cmake --build . --config Release --target install --parallel 4 popd popd - curl -L https://github.com/constantinpape/z5/archive/refs/tags/2.0.20.zip -o 2.0.20.zip - tar -xf 2.0.20.zip - pushd z5-2.0.20 + curl -L https://github.com/constantinpape/z5/archive/refs/tags/3.0.1.zip -o 3.0.1.zip + tar -xf 3.0.1.zip + pushd z5-3.0.1 mkdir build_man pushd build_man cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ -DCMAKE_PREFIX_PATH=%ABS_INSTALL% -DWITH_BLOSC=ON -DBUILD_Z5PY=OFF .. @@ -163,16 +146,6 @@ if "%BUILD_DCMTK_DEP%" == "1" ( popd ) -curl -L https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.zip -o v1.19.zip -tar -xf v1.19.zip -pushd libdeflate-1.19 -mkdir build_man -pushd build_man -cmake -DCMAKE_INSTALL_PREFIX=../../local_install/ -DCMAKE_PREFIX_PATH=%ABS_INSTALL% .. -cmake --build . --config Release --target install --parallel 4 -popd -popd - curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip -o 3.1.0.zip tar -xf 3.1.0.zip pushd libjpeg-turbo-3.1.0 diff --git a/ci-utils/validate_prereqs.sh b/ci-utils/validate_prereqs.sh index dc7a7fbb..3bd9b5f9 100755 --- a/ci-utils/validate_prereqs.sh +++ b/ci-utils/validate_prereqs.sh @@ -25,11 +25,14 @@ check() { echo "=== Validating prereq installation in $LOCAL_INSTALL ===" echo "" -echo "-- Z5 support (CMakeLists.txt: find_package ZLIB, BLOSC, Boost, nlohmann_json; find_file z5/z5.hxx) --" +echo "-- Z5 support (CMakeLists.txt: find_package ZLIB, BLOSC, nlohmann_json; find_file z5/z5.hxx) --" check "Z5 header" "$LOCAL_INSTALL/include/z5/z5.hxx" +check "Z5 array_access header" "$LOCAL_INSTALL/include/z5/multiarray/array_access.hxx" check "nlohmann_json headers" "$LOCAL_INSTALL/include/nlohmann/json.hpp" -check "xtensor headers" "$LOCAL_INSTALL/include/xtensor/containers/xtensor.hpp" check "blosc header" "$LOCAL_INSTALL/include/blosc.h" +# z5 3.0.1 uses libdeflate as its gzip/zlib backend (find_package(libdeflate CONFIG REQUIRED)) +check "libdeflate CMake config" "$LOCAL_INSTALL/lib/cmake/libdeflate/libdeflate-config.cmake" \ + "$LOCAL_INSTALL/lib64/cmake/libdeflate/libdeflate-config.cmake" echo "" echo "-- Arrow support (CMakeLists.txt: find_package Arrow, Parquet) --" diff --git a/environment.yml b/environment.yml index 717e7bf4..85937eba 100644 --- a/environment.yml +++ b/environment.yml @@ -6,11 +6,9 @@ dependencies: - python>=3.8 - compilers - boost[version='>=1.63'] - - z5py + - z5py[version='>=3.0.1'] - nlohmann_json - - xsimd[version='>=8,<9'] - cmake - - xtensor[version='>=0.24,<0.25'] - libtiff - pybind11[version='>=2.10.0'] - dcmtk[version='>=3.6.9'] diff --git a/src/nyx/omezarr.h b/src/nyx/omezarr.h index 5c02f737..d3a72553 100644 --- a/src/nyx/omezarr.h +++ b/src/nyx/omezarr.h @@ -5,16 +5,19 @@ #include #include "abs_tile_loader.h" #include "nlohmann/json.hpp" -#include "xtensor/containers/xarray.hpp" // factory functions to create files, groups and datasets #include "z5/factory.hxx" // handles for z5 filesystem objects #include "z5/filesystem/handle.hxx" -// io for xtensor multi-arrays -#include "z5/multiarray/xtensor_access.hxx" +// z5 multiarray API (ArrayView-based, no xtensor) +#include "z5/multiarray/array_view.hxx" +#include "z5/multiarray/array_access.hxx" +// z5 types +#include "z5/types/types.hxx" // attribute functionality #include "z5/attributes.hxx" + /// @brief Tile Loader for OMEZarr /// @tparam DataType AbstractView's internal type template @@ -125,25 +128,35 @@ class NyxusOmeZarrLoader : public AbstractTileLoader } template - void loadTile(std::shared_ptr> &dest, size_t pixel_row_index, size_t pixel_col_index, size_t pixel_layer_index){ - std::vector datasets; + void loadTile(std::shared_ptr> &dest, size_t pixel_row_index, + size_t pixel_col_index, size_t pixel_layer_index) { auto ds = z5::openDataset(*zarr_ptr_, ds_name_); + size_t data_height = tile_height_, data_width = tile_width_; - if (pixel_row_index + data_height > full_height_) {data_height = full_height_ - pixel_row_index;} - if (pixel_col_index + data_width > full_width_) {data_width = full_width_ - pixel_col_index;} - - typename xt::xarray::shape_type shape = {1,1,1,data_height,data_width }; - z5::types::ShapeType offset = { 0,0,pixel_layer_index, pixel_row_index, pixel_col_index }; - xt::xarray array(shape); - z5::multiarray::readSubarray(ds, array, offset.begin()); - std::vector tmp = std::vector (array.begin(), array.end()); + if (pixel_row_index + data_height > full_height_) { + data_height = full_height_ - pixel_row_index; + } + if (pixel_col_index + data_width > full_width_) { + data_width = full_width_ - pixel_col_index; + } + // Create a buffer to hold the read data + std::vector buffer(data_height * data_width); - for (size_t k=0;kbegin()+k*tile_width_); + // Create an ArrayView into the buffer (z5 3.0.1 uses ArrayView instead of xtensor) + z5::types::ShapeType shape = {1, 1, 1, data_height, data_width}; + auto view = z5::multiarray::makeView(buffer.data(), shape); + z5::types::ShapeType offset = {0, 0, pixel_layer_index, pixel_row_index, pixel_col_index}; + + // Read subarray from z5 dataset + z5::multiarray::readSubarray(*ds, view, offset.begin()); + + // Copy from buffer to destination tile, handling partial tiles + for (size_t k = 0; k < data_height; ++k) { + std::copy(buffer.begin() + k * data_width, + buffer.begin() + (k + 1) * data_width, + dest->begin() + k * tile_width_); } - //*dest = std::vector (array.begin(), array.end()); } /// @brief Tiff file height @@ -193,4 +206,4 @@ class NyxusOmeZarrLoader : public AbstractTileLoader std::unique_ptr zarr_ptr_; std::string ds_name_; }; -#endif //OMEZARR_SUPPORT \ No newline at end of file +#endif //OMEZARR_SUPPORT diff --git a/src/nyx/raw_omezarr.h b/src/nyx/raw_omezarr.h index 9430a9de..4cb36b22 100644 --- a/src/nyx/raw_omezarr.h +++ b/src/nyx/raw_omezarr.h @@ -4,14 +4,16 @@ #include #include "nlohmann/json.hpp" -#include "xtensor/containers/xarray.hpp" // factory functions to create files, groups and datasets #include "z5/factory.hxx" // handles for z5 filesystem objects #include "z5/filesystem/handle.hxx" -// io for xtensor multi-arrays -#include "z5/multiarray/xtensor_access.hxx" +// z5 multiarray API (ArrayView-based, no xtensor) +#include "z5/multiarray/array_view.hxx" +#include "z5/multiarray/array_access.hxx" +// z5 types +#include "z5/types/types.hxx" // attribute functionality #include "z5/attributes.hxx" @@ -129,34 +131,37 @@ class RawOmezarrLoader: public RawFormatLoader } template - void loadTile (size_t pixel_row_index, size_t pixel_col_index, size_t pixel_layer_index) + void loadTile(size_t pixel_row_index, size_t pixel_col_index, size_t pixel_layer_index) { - std::vector datasets; auto ds = z5::openDataset(*zarr_ptr_, ds_name_); + size_t data_height = tile_height_, data_width = tile_width_; - - if (pixel_row_index + data_height > full_height_) - { - data_height = full_height_ - pixel_row_index; + if (pixel_row_index + data_height > full_height_) { + data_height = full_height_ - pixel_row_index; } - if (pixel_col_index + data_width > full_width_) - { - data_width = full_width_ - pixel_col_index; + if (pixel_col_index + data_width > full_width_) { + data_width = full_width_ - pixel_col_index; } - typename xt::xarray::shape_type shape = { 1,1,1,data_height,data_width }; - z5::types::ShapeType offset = { 0,0,pixel_layer_index, pixel_row_index, pixel_col_index }; - xt::xarray array(shape); - z5::multiarray::readSubarray(ds, array, offset.begin()); - std::vector tmp = std::vector (array.begin(), array.end()); - + // Create a buffer to hold the read data + std::vector buffer(data_height * data_width); + + // Create an ArrayView into the buffer (z5 3.0.1 uses ArrayView instead of xtensor) + z5::types::ShapeType shape = {1, 1, 1, data_height, data_width}; + auto view = z5::multiarray::makeView(buffer.data(), shape); + z5::types::ShapeType offset = {0, 0, pixel_layer_index, pixel_row_index, pixel_col_index}; + + // Read subarray from z5 dataset + z5::multiarray::readSubarray(*ds, view, offset.begin()); + // zero-fill the buffer foreseeing its partial filling at incomplete (tail) tiles - std::fill (dest.begin(), dest.end(), 0); + std::fill(dest.begin(), dest.end(), 0); - // save this chunk of z5 tile data in the buffer - for (size_t k = 0; k < data_height; ++k) - { - std::copy(tmp.begin() + k * data_width, tmp.begin() + (k + 1) * data_width, dest.begin() + k * tile_width_); + // Copy from buffer to destination tile, handling partial tiles and type conversion + for (size_t k = 0; k < data_height; ++k) { + for (size_t j = 0; j < data_width; ++j) { + dest[k * tile_width_ + j] = static_cast(buffer[k * data_width + j]); + } } } @@ -209,4 +214,4 @@ class RawOmezarrLoader: public RawFormatLoader std::vector dest; }; -#endif //OMEZARR_SUPPORT \ No newline at end of file +#endif //OMEZARR_SUPPORT diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2dbb3e79..65e3eaa5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,6 +21,7 @@ set(TEST_SRC test_gabor.h test_initialization.h test_3d_nifti.h + test_omezarr.h ${TEST_SOURCE_FILES} ) add_executable(runAllTests ${TEST_SRC}) diff --git a/tests/data/omezarr/README.md b/tests/data/omezarr/README.md new file mode 100644 index 00000000..c3bcac8d --- /dev/null +++ b/tests/data/omezarr/README.md @@ -0,0 +1,54 @@ +# OME-Zarr test datasets + +These datasets are consumed by `tests/test_omezarr.h`, which exercises the two +z5-based OME-Zarr readers: + +- `NyxusOmeZarrLoader` (`src/nyx/omezarr.h`) — the Hedgehog tile-loader +- `RawOmezarrLoader` (`src/nyx/raw_omezarr.h`) — the raw-format loader + +Both readers expect a **zarr v2** store laid out as OME-NGFF: a root group whose +`.zattrs` carries `multiscales[0].datasets[0].path`, pointing at a 5D +`(T, C, Z, Y, X)` array. The loaders read `shape[2..4]` as depth/height/width +and `chunks[2..4]` as the tile sizes. + +## Datasets + +| Store | Size (HxW) | dtype | Chunk | Tile grid | Pixel value | +|------------------|-----------:|--------|-----------|-----------|------------------------| +| `test.ome.zarr` | 512x512 | uint16 | 1024x1024 | 1x1 | `(row + col) % 65536` | +| `multi.ome.zarr` | 1500x1200 | uint16 | 1024x1024 | 2x2* | `(row*7 + col*3) % 65536` | + +\* `multi.ome.zarr` has partial edge tiles (1500 = 1024 + 476, 1200 = 1024 + 176), +so it also exercises the loaders' partial-tile clipping path. + +Deterministic checksums asserted by the tests: + +- `test.ome.zarr` sum of all pixels = `133955584` +- `multi.ome.zarr` sum of all pixels = `12681000000` + +## Regeneration + +The datasets were generated with [bfio](https://pypi.org/project/bfio/) (writes +zarr v2 OME-Zarr by default): + +```python +import numpy as np +from bfio import BioWriter + +# test.ome.zarr +H, W = 512, 512 +Y, X = np.meshgrid(np.arange(H), np.arange(W), indexing='ij') +img = ((Y.astype(np.uint32) + X.astype(np.uint32)) % 65536).astype(np.uint16) +with BioWriter("test.ome.zarr", X=W, Y=H, Z=1, C=1, T=1, dtype=np.uint16) as bw: + bw[:] = img[..., np.newaxis, np.newaxis, np.newaxis] + +# multi.ome.zarr +H, W = 1500, 1200 +Y, X = np.meshgrid(np.arange(H), np.arange(W), indexing='ij') +img = ((Y.astype(np.uint32) * 7 + X.astype(np.uint32) * 3) % 65536).astype(np.uint16) +with BioWriter("multi.ome.zarr", X=W, Y=H, Z=1, C=1, T=1, dtype=np.uint16) as bw: + bw[:] = img[..., np.newaxis, np.newaxis, np.newaxis] +``` + +bfio's default tile size (1024) determines the chunk size; the image dimensions +above were chosen so `multi.ome.zarr` produces a 2x2 grid with partial edges. diff --git a/tests/data/omezarr/multi.ome.zarr/.zattrs b/tests/data/omezarr/multi.ome.zarr/.zattrs new file mode 100644 index 00000000..f00e24a2 --- /dev/null +++ b/tests/data/omezarr/multi.ome.zarr/.zattrs @@ -0,0 +1,16 @@ +{ + "multiscales": [ + { + "version": "0.1", + "name": "multi.ome.zarr", + "datasets": [ + { + "path": "0" + } + ], + "metadata": { + "method": "mean" + } + } + ] +} \ No newline at end of file diff --git a/tests/data/omezarr/multi.ome.zarr/.zgroup b/tests/data/omezarr/multi.ome.zarr/.zgroup new file mode 100644 index 00000000..cab13da6 --- /dev/null +++ b/tests/data/omezarr/multi.ome.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/tests/data/omezarr/multi.ome.zarr/.zmetadata b/tests/data/omezarr/multi.ome.zarr/.zmetadata new file mode 100644 index 00000000..3b3434d2 --- /dev/null +++ b/tests/data/omezarr/multi.ome.zarr/.zmetadata @@ -0,0 +1 @@ +{"metadata": {".zgroup": {"zarr_format": 2}, ".zattrs": {"multiscales": [{"version": "0.1", "name": "multi.ome.zarr", "datasets": [{"path": "0"}], "metadata": {"method": "mean"}}]}, "0/.zattrs": {}, "0/.zarray": {"shape": [1, 1, 1, 1500, 1200], "chunks": [1, 1, 1, 1024, 1024], "dtype": " + + + + + + + + diff --git a/tests/data/omezarr/test.ome.zarr/.zattrs b/tests/data/omezarr/test.ome.zarr/.zattrs new file mode 100644 index 00000000..6b0836a2 --- /dev/null +++ b/tests/data/omezarr/test.ome.zarr/.zattrs @@ -0,0 +1,16 @@ +{ + "multiscales": [ + { + "version": "0.1", + "name": "test.ome.zarr", + "datasets": [ + { + "path": "0" + } + ], + "metadata": { + "method": "mean" + } + } + ] +} \ No newline at end of file diff --git a/tests/data/omezarr/test.ome.zarr/.zgroup b/tests/data/omezarr/test.ome.zarr/.zgroup new file mode 100644 index 00000000..cab13da6 --- /dev/null +++ b/tests/data/omezarr/test.ome.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/tests/data/omezarr/test.ome.zarr/.zmetadata b/tests/data/omezarr/test.ome.zarr/.zmetadata new file mode 100644 index 00000000..b6b71b39 --- /dev/null +++ b/tests/data/omezarr/test.ome.zarr/.zmetadata @@ -0,0 +1 @@ +{"metadata": {".zgroup": {"zarr_format": 2}, ".zattrs": {"multiscales": [{"version": "0.1", "name": "test.ome.zarr", "datasets": [{"path": "0"}], "metadata": {"method": "mean"}}]}, "0/.zattrs": {}, "0/.zarray": {"shape": [1, 1, 1, 512, 512], "chunks": [1, 1, 1, 1024, 1024], "dtype": " + + + + + + + + diff --git a/tests/test_all.cc b/tests/test_all.cc index f099b67e..41c52864 100644 --- a/tests/test_all.cc +++ b/tests/test_all.cc @@ -27,6 +27,7 @@ #include "test_roi_blacklist.h" #include "test_image_quality.h" #include "test_3d_nifti.h" +#include "test_omezarr.h" #include "test_3d_shape.h" #include "test_3d_gldzm.h" #include "test_3d_ngldm.h" @@ -2315,6 +2316,37 @@ TEST(TEST_NYXUS, TEST_3D_NIFTY_DACC_CONSISTENCY) { } +//***** OME-Zarr i/o ***** + +#ifdef OMEZARR_SUPPORT + +TEST(TEST_NYXUS, TEST_OMEZARR_TILELOADER_GEOMETRY) { + ASSERT_NO_THROW (test_omezarr_tileloader_geometry()); +} + +TEST(TEST_NYXUS, TEST_OMEZARR_TILELOADER_CONTENT) { + ASSERT_NO_THROW (test_omezarr_tileloader_content()); +} + +TEST(TEST_NYXUS, TEST_OMEZARR_TILELOADER_MULTITILE) { + ASSERT_NO_THROW (test_omezarr_tileloader_multitile()); +} + +TEST(TEST_NYXUS, TEST_RAW_OMEZARR_GEOMETRY) { + ASSERT_NO_THROW (test_raw_omezarr_geometry()); +} + +TEST(TEST_NYXUS, TEST_RAW_OMEZARR_CONTENT) { + ASSERT_NO_THROW (test_raw_omezarr_content()); +} + +TEST(TEST_NYXUS, TEST_RAW_OMEZARR_MULTITILE) { + ASSERT_NO_THROW (test_raw_omezarr_multitile()); +} + +#endif // OMEZARR_SUPPORT + + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/tests/test_omezarr.h b/tests/test_omezarr.h new file mode 100644 index 00000000..cdd2eb12 --- /dev/null +++ b/tests/test_omezarr.h @@ -0,0 +1,219 @@ +#pragma once + +#include + +#ifdef OMEZARR_SUPPORT + +#include +#include +#include "../src/nyx/omezarr.h" +#include "../src/nyx/raw_omezarr.h" +#include "../src/nyx/helpers/fsystem.h" + +// The OME-Zarr test datasets under tests/data/omezarr are generated with bfio +// (see tests/data/omezarr/README.md). They are zarr-v2 stores with the 5D +// (T, C, Z, Y, X) layout that the nyxus z5-based loaders expect. +// +// test.ome.zarr : 512 x 512 uint16, single 1024x1024 chunk (one tile) +// pixel value = (row + col) % 65536 +// multi.ome.zarr : 1500 x 1200 uint16, 1024x1024 chunks (2x2 tile grid, +// partial edge tiles) ; value = (row*7 + col*3) % 65536 + +static inline fs::path omezarr_data_path(const char* name) +{ + fs::path p(__FILE__); + fs::path pp = p.parent_path(); + fs::path rel(std::string("/data/omezarr/") + name); + return fs::path(pp.string() + rel.make_preferred().string()); +} + +// --------------------------------------------------------------------------- +// NyxusOmeZarrLoader (AbstractTileLoader) tests +// --------------------------------------------------------------------------- + +// Geometry: dimensions and chunk/tile sizes are read correctly from metadata. +void test_omezarr_tileloader_geometry() +{ + fs::path ds = omezarr_data_path("test.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)); + + auto ldr = NyxusOmeZarrLoader(1, ds.string()); + + ASSERT_EQ(ldr.fullHeight(0), 512u); + ASSERT_EQ(ldr.fullWidth(0), 512u); + ASSERT_EQ(ldr.fullDepth(0), 1u); + ASSERT_EQ(ldr.tileHeight(0), 1024u); + ASSERT_EQ(ldr.tileWidth(0), 1024u); + ASSERT_EQ(ldr.tileDepth(0), 1u); + ASSERT_EQ(ldr.numberPyramidLevels(), 1u); +} + +// Content: read the single tile and verify exact pixel values and the checksum. +void test_omezarr_tileloader_content() +{ + fs::path ds = omezarr_data_path("test.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)); + + auto ldr = NyxusOmeZarrLoader(1, ds.string()); + + size_t th = ldr.tileHeight(0); + size_t tw = ldr.tileWidth(0); + auto tile = std::make_shared>(th * tw, 0u); + + ASSERT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0)); + + const std::vector& buf = *tile; + + // pixel value = (row + col) % 65536, laid out row-major with stride tileWidth + ASSERT_EQ(buf[0 * tw + 0], 0u); // (0,0) + ASSERT_EQ(buf[0 * tw + 511], 511u); // (0,511) + ASSERT_EQ(buf[100 * tw + 50], 150u); // (100,50) + ASSERT_EQ(buf[511 * tw + 511], 1022u); // (511,511) + + // full-image checksum over the valid 512x512 region + unsigned long long total = 0; + for (size_t r = 0; r < ldr.fullHeight(0); ++r) + for (size_t c = 0; c < ldr.fullWidth(0); ++c) + total += buf[r * tw + c]; + ASSERT_EQ(total, 133955584ull); +} + +// Multi-tile: 2x2 tile grid with partial edge tiles. +void test_omezarr_tileloader_multitile() +{ + fs::path ds = omezarr_data_path("multi.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)); + + auto ldr = NyxusOmeZarrLoader(1, ds.string()); + + const size_t H = ldr.fullHeight(0); + const size_t W = ldr.fullWidth(0); + const size_t th = ldr.tileHeight(0); + const size_t tw = ldr.tileWidth(0); + ASSERT_EQ(H, 1500u); + ASSERT_EQ(W, 1200u); + + const size_t nRows = (H + th - 1) / th; // 2 + const size_t nCols = (W + tw - 1) / tw; // 2 + ASSERT_EQ(nRows, 2u); + ASSERT_EQ(nCols, 2u); + + auto tile = std::make_shared>(th * tw, 0u); + + unsigned long long total = 0; + for (size_t tr = 0; tr < nRows; ++tr) + { + for (size_t tc = 0; tc < nCols; ++tc) + { + std::fill(tile->begin(), tile->end(), 0u); + ASSERT_NO_THROW(ldr.loadTileFromFile(tile, tr, tc, 0, 0)); + + const std::vector& buf = *tile; + const size_t row0 = tr * th; + const size_t col0 = tc * tw; + const size_t validH = std::min(th, H - row0); + const size_t validW = std::min(tw, W - col0); + + for (size_t r = 0; r < validH; ++r) + { + for (size_t c = 0; c < validW; ++c) + { + size_t gr = row0 + r; + size_t gc = col0 + c; + uint32_t expected = static_cast((gr * 7 + gc * 3) % 65536); + ASSERT_EQ(buf[r * tw + c], expected) + << "mismatch at global (" << gr << "," << gc << ")"; + total += buf[r * tw + c]; + } + } + } + } + ASSERT_EQ(total, 12681000000ull); +} + +// --------------------------------------------------------------------------- +// RawOmezarrLoader (RawFormatLoader) tests +// --------------------------------------------------------------------------- + +void test_raw_omezarr_geometry() +{ + fs::path ds = omezarr_data_path("test.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)); + + auto ldr = RawOmezarrLoader(ds.string()); + + ASSERT_EQ(ldr.fullHeight(0), 512u); + ASSERT_EQ(ldr.fullWidth(0), 512u); + ASSERT_EQ(ldr.fullDepth(0), 1u); + ASSERT_EQ(ldr.tileHeight(0), 1024u); + ASSERT_EQ(ldr.tileWidth(0), 1024u); +} + +void test_raw_omezarr_content() +{ + fs::path ds = omezarr_data_path("test.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)); + + auto ldr = RawOmezarrLoader(ds.string()); + const size_t tw = ldr.tileWidth(0); + + ASSERT_NO_THROW(ldr.loadTileFromFile(0, 0, 0, 0)); + + // get_uint32_pixel indexes into the internal tile buffer (stride tileWidth) + ASSERT_EQ(ldr.get_uint32_pixel(0 * tw + 0), 0u); + ASSERT_EQ(ldr.get_uint32_pixel(0 * tw + 511), 511u); + ASSERT_EQ(ldr.get_uint32_pixel(100 * tw + 50), 150u); + ASSERT_EQ(ldr.get_uint32_pixel(511 * tw + 511), 1022u); + + // get_dpequiv_pixel returns the same values as double + ASSERT_DOUBLE_EQ(ldr.get_dpequiv_pixel(100 * tw + 50), 150.0); + + unsigned long long total = 0; + for (size_t r = 0; r < ldr.fullHeight(0); ++r) + for (size_t c = 0; c < ldr.fullWidth(0); ++c) + total += ldr.get_uint32_pixel(r * tw + c); + ASSERT_EQ(total, 133955584ull); +} + +// Multi-tile read through the RawFormatLoader path, including partial tiles. +void test_raw_omezarr_multitile() +{ + fs::path ds = omezarr_data_path("multi.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)); + + auto ldr = RawOmezarrLoader(ds.string()); + const size_t H = ldr.fullHeight(0); + const size_t W = ldr.fullWidth(0); + const size_t th = ldr.tileHeight(0); + const size_t tw = ldr.tileWidth(0); + + const size_t nRows = (H + th - 1) / th; + const size_t nCols = (W + tw - 1) / tw; + + unsigned long long total = 0; + for (size_t tr = 0; tr < nRows; ++tr) + { + for (size_t tc = 0; tc < nCols; ++tc) + { + ASSERT_NO_THROW(ldr.loadTileFromFile(tr, tc, 0, 0)); + const size_t row0 = tr * th; + const size_t col0 = tc * tw; + const size_t validH = std::min(th, H - row0); + const size_t validW = std::min(tw, W - col0); + for (size_t r = 0; r < validH; ++r) + { + for (size_t c = 0; c < validW; ++c) + { + size_t gr = row0 + r, gc = col0 + c; + uint32_t expected = static_cast((gr * 7 + gc * 3) % 65536); + ASSERT_EQ(ldr.get_uint32_pixel(r * tw + c), expected) + << "mismatch at global (" << gr << "," << gc << ")"; + total += ldr.get_uint32_pixel(r * tw + c); + } + } + } + } + ASSERT_EQ(total, 12681000000ull); +} + +#endif // OMEZARR_SUPPORT