chore: upgrade z5 library to 3.0.1 (remove xtensor dependency)

.github/workflows/build_and_test_mac.yml

@@ -30,7 +30,7 @@ jobs:
         with:
           activate-environment: anaconda-client-env
           miniconda-version: "latest"
-          python-version: "3.10"
+          python-version: "3.11"
           channels: conda-forge
           channel-priority: strict
           auto-activate-base: false
@@ -73,7 +73,7 @@ jobs:
         with:
           activate-environment: anaconda-client-env
           miniconda-version: "latest"
-          python-version: "3.10"
+          python-version: "3.11"
           channels: conda-forge
           channel-priority: strict
           auto-activate-base: false

.github/workflows/build_and_test_ubuntu.yml

@@ -31,7 +31,7 @@ jobs:
       with:
         activate-environment: anaconda-client-env
         miniconda-version: "latest"
-        python-version: "3.10"
+        python-version: "3.11"
         channels: conda-forge
         auto-activate-base: false
 
@@ -67,7 +67,7 @@ jobs:
       with:
         activate-environment: anaconda-client-env
         miniconda-version: "latest"
-        python-version: "3.10"
+        python-version: "3.11"
         channels: conda-forge
         auto-activate-base: false
 

.github/workflows/build_and_test_windows.yml

@@ -31,7 +31,7 @@ jobs:
       with:
         activate-environment: anaconda-client-env
         miniconda-version: "latest"
-        python-version: "3.10"
+        python-version: "3.11"
         channels: conda-forge
         auto-activate-base: false
 
@@ -71,7 +71,7 @@ jobs:
       with:
         activate-environment: anaconda-client-env
         miniconda-version: "latest"
-        python-version: "3.10"
+        python-version: "3.11"
         channels: conda-forge
         auto-activate-base: false
 

CMakeLists.txt

@@ -83,28 +83,16 @@ if(USEGPU)
 endif()
 
 if(USE_Z5)
-	if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-		#since xtensor does not built with GCC 7.5 and lower
-		if(NOT(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.5)) 
-			set(USE_Z5 OFF)
-			message(WARNING "OMEZarr support is not available with GCC 7.5 and older compiler.")
-		endif()
-	endif()
 	find_package(ZLIB)
 	if (NOT ZLIB_FOUND)
 		set(USE_Z5 OFF)
-		message(WARNING "zlib not found. Blosc and hereby OMEZarr support will not be available.")
+		message(WARNING "zlib not found. OMEZarr support will not be available.")
 	endif()
 	find_package(BLOSC)
 	if(NOT BLOSC_FOUND)
 		set(USE_Z5 OFF)
 		message(WARNING "Blosc not found. OMEZarr support will not be available.")
 	endif()
-	find_package(Boost)
-	if(NOT Boost_FOUND)
-		set(USE_Z5 OFF)
-		message(WARNING "Boost not found. OMEZarr support will not be available.")
-	endif()
 
 	find_package(nlohmann_json)
 	if(NOT nlohmann_json_FOUND)
@@ -282,10 +270,6 @@ if(USE_Z5)
 		list(APPEND Nyxus_LIBRARIES ${BLOSC_LIBRARIES})
 	endif()
 
-	if(Boost_FOUND)
-		include_directories(${Boost_INCLUDE_DIR})
-	endif()
-
 	if (ZLIB_FOUND)
 		list(APPEND Nyxus_LIBRARIES ${ZLIB_LIBRARIES} )
 		include_directories (${ZLIB_INCLUDE_DIR})
@@ -392,21 +376,36 @@ endif()
 if(USE_ARROW)
 	# Look for installed packages the system
 	find_package(Arrow)
-	if (NOT Arrow_FOUND) 
+	if (NOT Arrow_FOUND)
 		message(WARNING "The Arrow library was not found. The build will continue without Arrow support.")
 		set(USE_ARROW OFF)
 	endif()
-	
+
 	find_package(Parquet)
-	if (NOT Parquet_FOUND) 
+	if (NOT Parquet_FOUND)
 		message(WARNING "The Parquet library was not found. The build will continue without Arrow support.")
 		set(USE_ARROW OFF)
 	endif()
 
+	# Building Arrow support pulls in Boost headers. Try to locate Boost so we can
+	# add its include dirs, but do NOT gate Arrow on find_package(Boost) succeeding:
+	# some prereq layouts ship Boost as headers-only under the common include prefix
+	# (no CMake config / no Boost_INCLUDE_DIR), which is enough to build against.
+	find_package(Boost QUIET)
+	if(NOT Boost_FOUND)
+		message(STATUS "Boost was not found via find_package; relying on Boost headers on the include path for Arrow support.")
+	endif()
+
 endif()
 
 if(USE_ARROW)
 	add_definitions(-DUSE_ARROW)
+	if(Boost_FOUND)
+		include_directories(${Boost_INCLUDE_DIRS})
+	elseif(DEFINED ENV{NYXUS_DEP_DIR})
+		# headers-only Boost shipped under the dependency prefix (no CMake config)
+		include_directories("$ENV{NYXUS_DEP_DIR}/include")
+	endif()
 	list(APPEND Nyxus_LIBRARIES arrow_shared)
 	list(APPEND Nyxus_LIBRARIES parquet_shared)
 endif()

ci-utils/envs/conda_cpp.txt

@@ -1,11 +1,9 @@
-z5py 
+z5py >=3.0.1
 libtiff <=4.7.0
 boost >=1.63
 nlohmann_json 
 blosc 
 pybind11 >= 2.12.0 
-xtensor >=0.26,<0.27
-xsimd >=13,<14
 cmake
 dcmtk >=3.6.9
 fmjpeg2koj >=1.0.3

ci-utils/envs/conda_cpp_mac.txt

@@ -1,13 +1,11 @@
 # Mac CI does not exercise DICOM support, and conda-forge does not provide a
 # satisfiable combination of dcmtk/fmjpeg2koj with the pinned libtiff stack.
-z5py
+z5py >=3.0.1
 libtiff <=4.7.0
 boost >=1.63
 nlohmann_json
 blosc
 pybind11 >= 2.12.0
-xtensor >=0.26,<0.27
-xsimd >=13,<14
 cmake
 libarrow
 libparquet

ci-utils/install_prereq_linux.sh

@@ -61,9 +61,21 @@ unzip zlib131.zip
 cd zlib-1.3.1
 mkdir build_man
 cd build_man
-cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ ..  
-cmake --build . 
-cmake --build . --target install 
+cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ ..
+cmake --build .
+cmake --build . --target install
+cd ../../
+
+# libdeflate is a base dependency (used by libtiff) and is also the gzip/zlib
+# backend for z5 3.0.1 (find_package(libdeflate CONFIG REQUIRED)), so install it
+# before the z5 build below.
+curl -L https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.zip -o v1.19.zip
+unzip v1.19.zip
+cd libdeflate-1.19
+mkdir build_man
+cd build_man
+cmake -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/   -DCMAKE_PREFIX_PATH=../../"$LOCAL_INSTALL_DIR"/ ..
+make install -j4
 cd ../../
 
 if [[ $BUILD_BOOST_DEP -eq 1 ]]; then
@@ -93,48 +105,18 @@ if [[ $BUILD_Z5_DEP -eq 1 ]]; then
     cmake --build . --target install 
     cd ../../
 
-    curl -L https://github.com/xtensor-stack/xtl/archive/refs/tags/0.8.0.zip -o 0.8.0.zip
-    unzip 0.8.0.zip
-    cd xtl-0.8.0 
-    mkdir build_man
-    cd build_man
-    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ ..  
-    cmake --build . 
-    cmake --build . --target install 
-    cd ../../
-
-    curl -L https://github.com/xtensor-stack/xtensor/archive/refs/tags/0.26.0.zip -o 0.26.0.zip
-    unzip 0.26.0.zip
-    cd xtensor-0.26.0
-    mkdir build_man
-    cd build_man
-    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ ..  
-    cmake --build . 
-    cmake --build . --target install 
-    cd ../../
-
-    curl -L https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.2.0.zip -o 13.2.0.zip
-    unzip 13.2.0.zip 
-    cd xsimd-13.2.0 
-    mkdir build_man
-    cd build_man
-    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ ..  
-    cmake --build . 
-    cmake --build . --target install 
-    cd ../../
-
     curl -L https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.zip -o v3.11.2.zip
-    unzip v3.11.2.zip 
+    unzip v3.11.2.zip
     cd json-3.11.2
     mkdir build_man
     cd build_man
-    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ ..  
+    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/ ..
     make install/fast
     cd ../../
 
-    curl -L https://github.com/constantinpape/z5/archive/refs/tags/2.0.20.zip -o 2.0.20.zip
-    unzip 2.0.20.zip
-    cd z5-2.0.20
+    curl -L https://github.com/constantinpape/z5/archive/refs/tags/3.0.1.zip -o 3.0.1.zip
+    unzip 3.0.1.zip
+    cd z5-3.0.1
     mkdir build_man
     cd build_man
     cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/   -DCMAKE_PREFIX_PATH=../../"$LOCAL_INSTALL_DIR"/ -DWITH_BLOSC=ON -DBUILD_Z5PY=OFF  ..
@@ -172,16 +154,6 @@ if [[ $BULD_DCMTK_DEP -eq 1 ]]; then
     cd ../../
 fi
 
-curl -L https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.zip -o v1.19.zip
-unzip v1.19.zip
-cd libdeflate-1.19
-mkdir build_man
-cd build_man
-cmake -DCMAKE_INSTALL_PREFIX=../../"$LOCAL_INSTALL_DIR"/   -DCMAKE_PREFIX_PATH=../../"$LOCAL_INSTALL_DIR"/ ..
-make install -j4
-cd ../../
-
-
 for i in {1..5}
 do
     curl -L https://download.osgeo.org/libtiff/tiff-4.7.0.zip -o tiff-4.7.0.zip 

ci-utils/install_prereq_win.bat

@@ -40,8 +40,21 @@ tar -xvf zlib131.zip
 pushd zlib-1.3.1
 mkdir build_man
 pushd build_man
-cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ ..  
-cmake --build . --config Release --target install --parallel 4  
+cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ ..
+cmake --build . --config Release --target install --parallel 4
+popd
+popd
+
+rem libdeflate is a base dependency (used by libtiff) and is also the gzip/zlib
+rem backend for z5 3.0.1 (find_package(libdeflate CONFIG REQUIRED)), so install it
+rem before the z5 build below.
+curl -L https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.zip -o v1.19.zip
+tar -xf v1.19.zip
+pushd libdeflate-1.19
+mkdir build_man
+pushd build_man
+cmake -DCMAKE_INSTALL_PREFIX=../../local_install/   -DCMAKE_PREFIX_PATH=%ABS_INSTALL%  ..
+cmake --build . --config Release --target install --parallel 4
 popd
 popd
 
@@ -71,49 +84,19 @@ if "%BUILD_Z5_DEP%" == "1" (
     popd
     popd
 
-    curl -L https://github.com/xtensor-stack/xtl/archive/refs/tags/0.8.0.zip -o 0.8.0.zip
-    tar -xf 0.8.0.zip
-    pushd xtl-0.8.0 
-    mkdir build_man
-    pushd build_man
-    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ ..  
-    cmake --build . --config Release --target install 
-    popd
-    popd
-
-    curl -L https://github.com/xtensor-stack/xtensor/archive/refs/tags/0.26.0.zip -o 0.26.0.zip
-    tar -xf 0.26.0.zip
-    pushd xtensor-0.26.0
-    mkdir build_man
-    pushd build_man
-    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ ..  
-    cmake --build . --config Release --target install 
-    popd
-    popd
-
-    curl -L https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.2.0.zip -o 13.2.0.zip
-    tar -xf 13.2.0.zip 
-    pushd xsimd-13.2.0 
-    mkdir build_man
-    pushd build_man
-    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ ..  
-    cmake --build . --config Release --target install  
-    popd
-    popd
-
     curl -L https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.zip -o v3.11.2.zip
-    tar -xf v3.11.2.zip 
+    tar -xf v3.11.2.zip
     pushd json-3.11.2
     mkdir build_man
     pushd build_man
-    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ -DJSON_BuildTests=OFF ..  
+    cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/ -DJSON_BuildTests=OFF ..
     cmake --build . --config Release --target install  --parallel 4
     popd
     popd
 
-    curl -L https://github.com/constantinpape/z5/archive/refs/tags/2.0.20.zip -o 2.0.20.zip
-    tar -xf 2.0.20.zip 
-    pushd z5-2.0.20
+    curl -L https://github.com/constantinpape/z5/archive/refs/tags/3.0.1.zip -o 3.0.1.zip
+    tar -xf 3.0.1.zip
+    pushd z5-3.0.1
     mkdir build_man
     pushd build_man
     cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=../../local_install/   -DCMAKE_PREFIX_PATH=%ABS_INSTALL% -DWITH_BLOSC=ON -DBUILD_Z5PY=OFF ..
@@ -163,16 +146,6 @@ if "%BUILD_DCMTK_DEP%" == "1" (
     popd
 )
 
-curl -L https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.zip -o v1.19.zip
-tar -xf v1.19.zip
-pushd libdeflate-1.19
-mkdir build_man
-pushd build_man
-cmake -DCMAKE_INSTALL_PREFIX=../../local_install/   -DCMAKE_PREFIX_PATH=%ABS_INSTALL%  ..
-cmake --build . --config Release --target install --parallel 4
-popd
-popd
-
 curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip -o 3.1.0.zip
 tar -xf 3.1.0.zip
 pushd libjpeg-turbo-3.1.0

ci-utils/validate_prereqs.sh

@@ -25,11 +25,14 @@ check() {
 echo "=== Validating prereq installation in $LOCAL_INSTALL ==="
 
 echo ""
-echo "-- Z5 support (CMakeLists.txt: find_package ZLIB, BLOSC, Boost, nlohmann_json; find_file z5/z5.hxx) --"
+echo "-- Z5 support (CMakeLists.txt: find_package ZLIB, BLOSC, nlohmann_json; find_file z5/z5.hxx) --"
 check "Z5 header"             "$LOCAL_INSTALL/include/z5/z5.hxx"
+check "Z5 array_access header" "$LOCAL_INSTALL/include/z5/multiarray/array_access.hxx"
 check "nlohmann_json headers" "$LOCAL_INSTALL/include/nlohmann/json.hpp"
-check "xtensor headers"       "$LOCAL_INSTALL/include/xtensor/containers/xtensor.hpp"
 check "blosc header"          "$LOCAL_INSTALL/include/blosc.h"
+# z5 3.0.1 uses libdeflate as its gzip/zlib backend (find_package(libdeflate CONFIG REQUIRED))
+check "libdeflate CMake config" "$LOCAL_INSTALL/lib/cmake/libdeflate/libdeflate-config.cmake" \
+                                "$LOCAL_INSTALL/lib64/cmake/libdeflate/libdeflate-config.cmake"
 
 echo ""
 echo "-- Arrow support (CMakeLists.txt: find_package Arrow, Parquet) --"

environment.yml

@@ -6,11 +6,9 @@ dependencies:
   - python>=3.8
   - compilers
   - boost[version='>=1.63']
-  - z5py
+  - z5py[version='>=3.0.1']
   - nlohmann_json
-  - xsimd[version='>=8,<9']
   - cmake
-  - xtensor[version='>=0.24,<0.25']
   - libtiff
   - pybind11[version='>=2.10.0']
   - dcmtk[version='>=3.6.9']