From b88ff39b03694c4d2b7d449ddee8ce933f5dfbe0 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 09:31:47 +0200 Subject: [PATCH 01/46] Added .github/workflows/gpu_ci_trigger.yml --- .github/workflows/gpu_ci_trigger.yml | 23 ++++++ .gitlab-ci.yml | 104 +++++++++++++-------------- 2 files changed, 75 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/gpu_ci_trigger.yml diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml new file mode 100644 index 0000000..c8da567 --- /dev/null +++ b/.github/workflows/gpu_ci_trigger.yml @@ -0,0 +1,23 @@ +name: Trigger GitLab GPU CI + +on: + push: + branches: [main, devel] + pull_request: + branches: [main, devel] + workflow_dispatch: # Allows manual triggering + +jobs: + trigger-gitlab: + runs-on: ubuntu-latest + steps: + - name: Trigger GitLab Pipeline + run: | + # Use curl to call the GitLab Trigger API + # We pass the GitHub SHA and Repository to GitLab so it knows exactly what to clone. + curl --request POST \ + --form token=${{ secrets.GITLAB_TRIGGER_TOKEN }} \ + --form ref=main \ + --form "variables[GH_SHA]=${{ github.event.pull_request.head.sha || github.sha }}" \ + --form "variables[GH_REPO]=${{ github.repository }}" \ + "https://gitlab.mpcdf.mpg.de/api/v4/projects/${{ secrets.GITLAB_PROJECT_ID }}/trigger/pipeline" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7de373c..d607cc8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,57 +1,57 @@ -# This file is a template, and might need editing before it works on your project. -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml - -# Official language image. Look for the different tagged releases at: -# https://hub.docker.com/r/library/python/tags/ -image: python:latest - -# Change pip's cache directory to be inside the project directory since we can -# only cache local items. variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - -# https://pip.pypa.io/en/stable/topics/caching/ -cache: - paths: - - .cache/pip - -before_script: - - python --version ; pip --version # For debugging - - pip install virtualenv - - virtualenv venv - - source venv/bin/activate - -test: - script: - - pip install ruff tox # you can also use tox - - pip install --editable ".[test]" - - tox -e py,ruff - -run: + # Base image with CUDA and C++ tools + CUDA_IMAGE: "nvidia/cuda:12.4.1-devel-ubuntu22.04" + +stages: + - checkout + - test + +# 1. Checkout Step: Clone the canonical GitHub repo at the exact SHA +checkout_github: + stage: checkout + image: alpine:latest + tags: + - mpcdf-shared script: - - pip install . - # run the command here + - apk add --no-cache git + - echo "Cloning GitHub repo ${GH_REPO} at SHA ${GH_SHA}" + - git clone https://x-access-token:${GH_PAT}@github.com/${GH_REPO}.git workspace + - cd workspace + - git checkout ${GH_SHA} artifacts: paths: - - build/* - -pages: + - workspace/ + expire_in: 1 hour + +# 2. GPU Test Step: Run CUDA sanity checks, build, and test +gpu_tests: + stage: test + image: ${CUDA_IMAGE} + tags: + - gpu-nvidia + - gpu-nvidia-cc80 + dependencies: + - checkout_github + before_script: + - apt-get update && apt-get install -y cmake python3-pip git + - cd workspace script: - - pip install sphinx sphinx-rtd-theme - - cd doc - - make html - - mv build/html/ ../public/ - artifacts: - paths: - - public - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - -deploy: - stage: deploy - script: echo "Define your deployment script!" - environment: production - + - echo "--- CUDA Sanity Check ---" + - nvidia-smi + + - echo "--- Detect Compute Capability ---" + # Simple check for CC 8.0 (A100) + - nvidia-smi --query-gpu=compute_cap --format=csv,noheader + + - echo "--- CMake Build ---" + # Example for C++ components if they exist + - if [ -f "CMakeLists.txt" ]; then + mkdir build && cd build; + cmake ..; + make -j$(nproc); + cd ..; + fi + + - echo "--- Pytest Execution ---" + - pip3 install .[test] + - pytest tests/unit/ From 9f0c910e0fae50c7ad41e48fb817b7b409123f5a Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 09:47:33 +0200 Subject: [PATCH 02/46] run the gitlab workflow on the same branch --- .github/workflows/gpu_ci_trigger.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index c8da567..94c4bbc 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -13,11 +13,13 @@ jobs: steps: - name: Trigger GitLab Pipeline run: | - # Use curl to call the GitLab Trigger API - # We pass the GitHub SHA and Repository to GitLab so it knows exactly what to clone. + # Use the branch name that triggered this workflow + # github.head_ref is used for PRs, github.ref_name for pushes + BRANCH_NAME="${{ github.head_ref || github.ref_name }}" + curl --request POST \ --form token=${{ secrets.GITLAB_TRIGGER_TOKEN }} \ - --form ref=main \ + --form "ref=$BRANCH_NAME" \ --form "variables[GH_SHA]=${{ github.event.pull_request.head.sha || github.sha }}" \ --form "variables[GH_REPO]=${{ github.repository }}" \ "https://gitlab.mpcdf.mpg.de/api/v4/projects/${{ secrets.GITLAB_PROJECT_ID }}/trigger/pipeline" From ec18a9455a1808ca522f779e3aacdfb0ba0559a6 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 09:50:23 +0200 Subject: [PATCH 03/46] push from github->gitlab --- .github/workflows/gpu_ci_trigger.yml | 41 +++++++++++++++++++--------- .gitlab-ci.yml | 26 ++---------------- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index 94c4bbc..e8e841d 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -1,25 +1,40 @@ -name: Trigger GitLab GPU CI +name: Sync to GitLab and Run GPU CI on: push: branches: [main, devel] pull_request: branches: [main, devel] - workflow_dispatch: # Allows manual triggering + workflow_dispatch: jobs: - trigger-gitlab: + sync-and-test: runs-on: ubuntu-latest steps: - - name: Trigger GitLab Pipeline + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Push to GitLab + env: + # You will need to add this secret to GitHub + GITLAB_PUSH_TOKEN: ${{ secrets.GITLAB_PUSH_TOKEN }} run: | - # Use the branch name that triggered this workflow - # github.head_ref is used for PRs, github.ref_name for pushes - BRANCH_NAME="${{ github.head_ref || github.ref_name }}" + # Determine the branch name + # For PRs, we push to a branch named 'pr-' on GitLab + # For regular pushes, we use the actual branch name + if [ "${{ github.event_name }}" == "pull_request" ]; then + TARGET_BRANCH="pr-${{ github.event.number }}" + else + TARGET_BRANCH="${{ github.ref_name }}" + fi + + echo "Pushing to GitLab branch: $TARGET_BRANCH" + + # Add GitLab as a remote using the Project Access Token + # Format: https://oauth2:TOKEN@gitlab.mpcdf.mpg.de/path/to/repo.git + git remote add gitlab "https://oauth2:${GITLAB_PUSH_TOKEN}@gitlab.mpcdf.mpg.de/maxlin/cunumpy.git" - curl --request POST \ - --form token=${{ secrets.GITLAB_TRIGGER_TOKEN }} \ - --form "ref=$BRANCH_NAME" \ - --form "variables[GH_SHA]=${{ github.event.pull_request.head.sha || github.sha }}" \ - --form "variables[GH_REPO]=${{ github.repository }}" \ - "https://gitlab.mpcdf.mpg.de/api/v4/projects/${{ secrets.GITLAB_PROJECT_ID }}/trigger/pipeline" + # Force push the current HEAD to the target branch on GitLab + git push -f gitlab HEAD:refs/heads/$TARGET_BRANCH diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d607cc8..9706eaf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,48 +3,26 @@ variables: CUDA_IMAGE: "nvidia/cuda:12.4.1-devel-ubuntu22.04" stages: - - checkout - test -# 1. Checkout Step: Clone the canonical GitHub repo at the exact SHA -checkout_github: - stage: checkout - image: alpine:latest - tags: - - mpcdf-shared - script: - - apk add --no-cache git - - echo "Cloning GitHub repo ${GH_REPO} at SHA ${GH_SHA}" - - git clone https://x-access-token:${GH_PAT}@github.com/${GH_REPO}.git workspace - - cd workspace - - git checkout ${GH_SHA} - artifacts: - paths: - - workspace/ - expire_in: 1 hour - -# 2. GPU Test Step: Run CUDA sanity checks, build, and test +# GPU Test Step: Run CUDA sanity checks, build, and test +# Since GitHub now pushes the code directly to GitLab, we can just use the local repo. gpu_tests: stage: test image: ${CUDA_IMAGE} tags: - gpu-nvidia - gpu-nvidia-cc80 - dependencies: - - checkout_github before_script: - apt-get update && apt-get install -y cmake python3-pip git - - cd workspace script: - echo "--- CUDA Sanity Check ---" - nvidia-smi - echo "--- Detect Compute Capability ---" - # Simple check for CC 8.0 (A100) - nvidia-smi --query-gpu=compute_cap --format=csv,noheader - echo "--- CMake Build ---" - # Example for C++ components if they exist - if [ -f "CMakeLists.txt" ]; then mkdir build && cd build; cmake ..; From e23cef93ba2d0a6b71a8d2cde33834a2388a18b0 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 09:57:54 +0200 Subject: [PATCH 04/46] trigger CI From 1c9322fb1db177c680c40578d69858fa50833b2d Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:05:05 +0200 Subject: [PATCH 05/46] push with ssh --- .github/workflows/gpu_ci_trigger.yml | 33 +++++++++++++++------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index e8e841d..b421ebb 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -14,27 +14,30 @@ jobs: - name: Checkout Code uses: actions/checkout@v4 with: - fetch-depth: 0 # Fetch all history for all branches and tags + fetch-depth: 0 - - name: Push to GitLab - env: - # You will need to add this secret to GitHub - GITLAB_PUSH_TOKEN: ${{ secrets.GITLAB_PUSH_TOKEN }} + - name: Install SSH Key + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} + + - name: Push to GitLab via SSH run: | - # Determine the branch name - # For PRs, we push to a branch named 'pr-' on GitLab - # For regular pushes, we use the actual branch name + # 1. Setup SSH known hosts to prevent interactive prompts + mkdir -p ~/.ssh + ssh-keyscan gitlab.mpcdf.mpg.de >> ~/.ssh/known_hosts + + # 2. Determine target branch if [ "${{ github.event_name }}" == "pull_request" ]; then TARGET_BRANCH="pr-${{ github.event.number }}" else TARGET_BRANCH="${{ github.ref_name }}" fi - + echo "Pushing to GitLab branch: $TARGET_BRANCH" - - # Add GitLab as a remote using the Project Access Token - # Format: https://oauth2:TOKEN@gitlab.mpcdf.mpg.de/path/to/repo.git - git remote add gitlab "https://oauth2:${GITLAB_PUSH_TOKEN}@gitlab.mpcdf.mpg.de/maxlin/cunumpy.git" - - # Force push the current HEAD to the target branch on GitLab + + # 3. Add GitLab SSH remote + git remote add gitlab git@gitlab.mpcdf.mpg.de:maxlin/cunumpy.git + + # 4. Force push git push -f gitlab HEAD:refs/heads/$TARGET_BRANCH From 7d2bd4ff098f9fc90a62c43349b5a5734f05d3df Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:11:07 +0200 Subject: [PATCH 06/46] Updated .gitlab-ci.yml --- .gitlab-ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9706eaf..d569820 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,12 +1,12 @@ variables: # Base image with CUDA and C++ tools CUDA_IMAGE: "nvidia/cuda:12.4.1-devel-ubuntu22.04" + # Instruct pip to install to a directory in the path, or avoid the 'UNKNOWN' warning + PIP_DISABLE_PIP_VERSION_CHECK: "1" stages: - test -# GPU Test Step: Run CUDA sanity checks, build, and test -# Since GitHub now pushes the code directly to GitLab, we can just use the local repo. gpu_tests: stage: test image: ${CUDA_IMAGE} @@ -14,6 +14,8 @@ gpu_tests: - gpu-nvidia - gpu-nvidia-cc80 before_script: + # Set non-interactive timezone to prevent tzdata prompting + - export DEBIAN_FRONTEND=noninteractive - apt-get update && apt-get install -y cmake python3-pip git script: - echo "--- CUDA Sanity Check ---" @@ -31,5 +33,11 @@ gpu_tests: fi - echo "--- Pytest Execution ---" - - pip3 install .[test] - - pytest tests/unit/ + # Ensure pip is up to date and install test requirements directly + - python3 -m pip install --upgrade pip + - pip3 install pytest cupy-cuda12x + # Install the package itself + - pip3 install -e . + # Set the backend to cupy for tests and run them + - export ARRAY_BACKEND=cupy + - python3 -m pytest tests/unit/ From 856ef94951e88a8ea9704b982827cf78ab434d98 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:17:03 +0200 Subject: [PATCH 07/46] use a venv on gitalb ci --- .gitlab-ci.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d569820..9a622ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,21 +23,18 @@ gpu_tests: - echo "--- Detect Compute Capability ---" - nvidia-smi --query-gpu=compute_cap --format=csv,noheader - - - echo "--- CMake Build ---" - - if [ -f "CMakeLists.txt" ]; then - mkdir build && cd build; - cmake ..; - make -j$(nproc); - cd ..; - fi - + + - ls + - pwd + - echo "--- Pytest Execution ---" # Ensure pip is up to date and install test requirements directly - - python3 -m pip install --upgrade pip - - pip3 install pytest cupy-cuda12x + - python3 -m venv .venv + - source .venv/bin/activate + - pip install --upgrade pip + - pip install pytest cupy-cuda12x # Install the package itself - - pip3 install -e . + - pip install -e . # Set the backend to cupy for tests and run them - export ARRAY_BACKEND=cupy - python3 -m pytest tests/unit/ From 677d04cf5fdbccaf5b47ab956f703d5e36806ca9 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:21:46 +0200 Subject: [PATCH 08/46] Use gitlab-registry.mpcdf.mpg.de/mpcdf/ci-module-image/nvhpcsdk_26:2026 --- .gitlab-ci.yml | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a622ed..3e11419 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,6 @@ variables: - # Base image with CUDA and C++ tools - CUDA_IMAGE: "nvidia/cuda:12.4.1-devel-ubuntu22.04" - # Instruct pip to install to a directory in the path, or avoid the 'UNKNOWN' warning + # Use the specialized MPCDF HPC image + CUDA_IMAGE: "gitlab-registry.mpcdf.mpg.de/mpcdf/ci-module-image/nvhpcsdk_26:2026" PIP_DISABLE_PIP_VERSION_CHECK: "1" stages: @@ -13,28 +12,34 @@ gpu_tests: tags: - gpu-nvidia - gpu-nvidia-cc80 - before_script: - # Set non-interactive timezone to prevent tzdata prompting - - export DEBIAN_FRONTEND=noninteractive - - apt-get update && apt-get install -y cmake python3-pip git script: - echo "--- CUDA Sanity Check ---" - nvidia-smi - echo "--- Detect Compute Capability ---" - nvidia-smi --query-gpu=compute_cap --format=csv,noheader - - - ls - - pwd - + + - echo "--- Tool Versions ---" + - cmake --version + - python3 --version + - git --version + + - echo "--- CMake Build ---" + - if [ -f "CMakeLists.txt" ]; then + mkdir -p build && cd build; + cmake ..; + make -j$(nproc); + cd ..; + fi + - echo "--- Pytest Execution ---" - # Ensure pip is up to date and install test requirements directly - - python3 -m venv .venv - - source .venv/bin/activate - - pip install --upgrade pip - - pip install pytest cupy-cuda12x - # Install the package itself - - pip install -e . - # Set the backend to cupy for tests and run them + # The MPCDF image likely has a specific python environment. + # We install our dependencies into the user directory or a virtualenv. + - python3 -m pip install --user pytest cupy-cuda12x + - python3 -m pip install --user -e . + + # Add the user bin to PATH for pytest + - export PATH="$HOME/.local/bin:$PATH" - export ARRAY_BACKEND=cupy + - python3 -m pytest tests/unit/ From 15ecddb321091b50d146397d5e60f93c28d37e47 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:24:30 +0200 Subject: [PATCH 09/46] load modules --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e11419..80ce53f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,11 @@ gpu_tests: tags: - gpu-nvidia - gpu-nvidia-cc80 + before_script: + - echo "Running GPU tests on image: ${CUDA_IMAGE}" + # Load modules + - module load python-waterboa/2025.06 + - module load nvhpcsdk/26 script: - echo "--- CUDA Sanity Check ---" - nvidia-smi From 8a55a188edc9ef24bae1ef3c2d4966ee491cf9d8 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:28:16 +0200 Subject: [PATCH 10/46] cleanup --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 80ce53f..48cd289 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,6 @@ gpu_tests: - gpu-nvidia - gpu-nvidia-cc80 before_script: - - echo "Running GPU tests on image: ${CUDA_IMAGE}" - # Load modules - module load python-waterboa/2025.06 - module load nvhpcsdk/26 script: From ba45341d36a2d0267e2b5c912ed50bbf7d5bc28e Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:29:58 +0200 Subject: [PATCH 11/46] cleanup --- .gitlab-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 48cd289..15971fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,14 +27,6 @@ gpu_tests: - python3 --version - git --version - - echo "--- CMake Build ---" - - if [ -f "CMakeLists.txt" ]; then - mkdir -p build && cd build; - cmake ..; - make -j$(nproc); - cd ..; - fi - - echo "--- Pytest Execution ---" # The MPCDF image likely has a specific python environment. # We install our dependencies into the user directory or a virtualenv. From 32a2362ef4126721fdc60010d33de61e9d271f75 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:31:27 +0200 Subject: [PATCH 12/46] Skip parity test --- tests/unit/test_app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index 53e095e..1eb6e61 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -18,6 +18,9 @@ def test_numpy_symbols_accessible(): This validates the runtime behaviour that the stub file (__init__.pyi) declares to Pylance so that `xp.` shows numpy completions in VS Code. """ + if xp.cupy_backend: + pytest.skip("CuPy does not have 100% symbol parity with NumPy.") + # Exclude our custom methods from the numpy check custom_methods = [ "to_numpy", From a08068da98ca616ef7ce1046abf4e20dda1752cc Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:32:48 +0200 Subject: [PATCH 13/46] Wait for gitlab job completion --- .github/workflows/gpu_ci_trigger.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index b421ebb..9cbe903 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -41,3 +41,26 @@ jobs: # 4. Force push git push -f gitlab HEAD:refs/heads/$TARGET_BRANCH + + # Store branch for next step + echo "TARGET_BRANCH=$TARGET_BRANCH" >> $GITHUB_ENV + + - name: Trigger GitLab Pipeline & Provide Link + run: | + # Trigger the pipeline and capture the JSON response + RESPONSE=$(curl --silent --request POST \ + --form token=${{ secrets.GITLAB_TRIGGER_TOKEN }} \ + --form "ref=$TARGET_BRANCH" \ + "https://gitlab.mpcdf.mpg.de/api/v4/projects/${{ secrets.GITLAB_PROJECT_ID }}/trigger/pipeline") + + # Extract the pipeline web_url using python (built-in to runner) + PIPELINE_URL=$(echo $RESPONSE | python3 -c "import sys, json; print(json.load(sys.stdin).get('web_url', ''))") + + if [ -z "$PIPELINE_URL" ]; then + echo "::error::Failed to trigger GitLab pipeline. Response: $RESPONSE" + exit 1 + fi + + echo "::notice::GitLab GPU CI Pipeline triggered successfully!" + echo "::notice::View Pipeline: $PIPELINE_URL" + echo "PIPELINE_URL=$PIPELINE_URL" >> $GITHUB_ENV From e2557dbcd9b5a33c56a632f4555cbde14d966ae3 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:35:05 +0200 Subject: [PATCH 14/46] Split tests --- .gitlab-ci.yml | 2 +- CHANGELOG.md | 1 + tests/unit/test_app.py | 134 ------------------------------------- tests/unit/test_cunumpy.py | 51 ++++++++++++++ tests/unit/test_cupy.py | 34 ++++++++++ tests/unit/test_numpy.py | 40 +++++++++++ 6 files changed, 127 insertions(+), 135 deletions(-) delete mode 100644 tests/unit/test_app.py create mode 100644 tests/unit/test_cunumpy.py create mode 100644 tests/unit/test_cupy.py create mode 100644 tests/unit/test_numpy.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15971fc..b76c09b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,7 @@ gpu_tests: - echo "--- Pytest Execution ---" # The MPCDF image likely has a specific python environment. # We install our dependencies into the user directory or a virtualenv. - - python3 -m pip install --user pytest cupy-cuda12x + - python3 -m pip install --user cupy-cuda12x - python3 -m pip install --user -e . # Add the user bin to PATH for pytest diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fae537..ec7876c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `xp.synchronize()`: Blocks until GPU operations are complete (no-op on CPU). Essential for accurate benchmarking. - **Developer Experience**: - Added `isort` configuration to `pyproject.toml` with `black` profile compatibility. + - Reorganized test suite into specialized files: `test_numpy.py`, `test_cupy.py`, and `test_cunumpy.py`. ### Changed - **Dynamic Dispatch Architecture**: Refactored `src/cunumpy/xp.py` to use module-level `__getattr__`. This ensures that `cunumpy.` calls always resolve to the currently active backend module, enabling seamless runtime switching via `set_backend`. diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py deleted file mode 100644 index 1eb6e61..0000000 --- a/tests/unit/test_app.py +++ /dev/null @@ -1,134 +0,0 @@ -import numpy as np -import pytest - -import cunumpy as xp - - -def test_xp_array(): - - arr = xp.array([1, 2]) - arr *= 2 - - print(f"{arr = } {type(arr) = }") - - -def test_numpy_symbols_accessible(): - """All public numpy symbols must be reachable via cunumpy. - - This validates the runtime behaviour that the stub file (__init__.pyi) - declares to Pylance so that `xp.` shows numpy completions in VS Code. - """ - if xp.cupy_backend: - pytest.skip("CuPy does not have 100% symbol parity with NumPy.") - - # Exclude our custom methods from the numpy check - custom_methods = [ - "to_numpy", - "to_cupy", - "to_cunumpy", - "get_backend", - "is_gpu", - "is_cpu", - "use_backend", - "set_backend", - "synchronize", - "numpy_backend", - "cupy_backend", - "xp", - ] - missing = [ - name - for name in np.__all__ - if not hasattr(xp, name) and name not in custom_methods - ] - assert missing == [], f"Symbols not accessible via cunumpy: {missing}" - - -def test_to_numpy(): - arr = xp.array([1, 2, 3]) - # Even if it's already numpy, to_numpy should work - arr_np = xp.to_numpy(arr) - assert isinstance(arr_np, np.ndarray) - assert np.array_equal(arr_np, [1, 2, 3]) - - -def test_to_cupy_not_available(): - try: - import cupy - - pytest.skip("CuPy is installed, cannot test missing cupy error") - except ImportError: - pass - - arr = np.array([1, 2, 3]) - - with pytest.raises(ImportError): - xp.to_cupy(arr) - - -def test_to_cunumpy(): - arr = np.array([1, 2, 3]) - arr_xp = xp.to_cunumpy(arr) - # Backend is numpy in tests usually - assert isinstance(arr_xp, (np.ndarray, xp.ndarray)) - - -def test_get_backend_and_is_gpu_cpu(): - arr = np.array([1, 2, 3]) - assert xp.get_backend(arr) == "numpy" - assert xp.is_gpu(arr) is False - assert xp.is_cpu(arr) is True - - -def test_use_backend(): - # Initial backend should be numpy (default) in this test environment - # Accessing xp.xp triggers the dynamic __getattr__ in xp.py - assert "numpy" in xp.xp.__name__ - - with xp.use_backend("numpy"): - assert "numpy" in xp.xp.__name__ - arr = xp.zeros(10) - assert isinstance(arr, np.ndarray) - - assert "numpy" in xp.xp.__name__ - - -def test_set_backend(): - # Set to numpy - xp.set_backend("numpy") - assert "numpy" in xp.xp.__name__ - arr = xp.array([1]) - assert isinstance(arr, np.ndarray) - - # Set to cupy (falls back to numpy if not available) - xp.set_backend("cupy") - # If cupy is not installed, xp.xp will be numpy module - # We just verify it doesn't crash and we can still call things - arr2 = xp.array([2]) - assert arr2 is not None - - -def test_synchronize(): - # Should not crash on any backend - xp.synchronize() - - with xp.use_backend("numpy"): - xp.synchronize() - - with xp.use_backend("cupy"): - xp.synchronize() - - -def test_backend_bools(): - with xp.use_backend("numpy"): - assert xp.numpy_backend is True - assert xp.cupy_backend is False - - # Note: in test env without cupy, cupy_backend might be false - # even inside use_backend('cupy') if fallback occurs. - # Our implementation of use_backend calls _load_backend which returns np if cp missing. - - -if __name__ == "__main__": - test_xp_array() - test_numpy_symbols_accessible() diff --git a/tests/unit/test_cunumpy.py b/tests/unit/test_cunumpy.py new file mode 100644 index 0000000..e2b0302 --- /dev/null +++ b/tests/unit/test_cunumpy.py @@ -0,0 +1,51 @@ +import numpy as np +import pytest +import cunumpy as xp + +def test_to_numpy(): + arr = xp.array([1, 2, 3]) + # Even if it's already numpy, to_numpy should work + arr_np = xp.to_numpy(arr) + assert isinstance(arr_np, np.ndarray) + assert np.array_equal(arr_np, [1, 2, 3]) + +def test_to_cunumpy(): + arr = np.array([1, 2, 3]) + arr_xp = xp.to_cunumpy(arr) + # Backend is numpy in tests usually + assert isinstance(arr_xp, (np.ndarray, xp.ndarray)) + +def test_get_backend_and_is_gpu_cpu(): + arr = np.array([1, 2, 3]) + assert xp.get_backend(arr) == "numpy" + assert xp.is_gpu(arr) is False + assert xp.is_cpu(arr) is True + +def test_use_backend(): + # Initial backend should be numpy (default) in this test environment + assert "numpy" in xp.xp.__name__ + + with xp.use_backend("numpy"): + assert "numpy" in xp.xp.__name__ + arr = xp.zeros(10) + assert isinstance(arr, np.ndarray) + + assert "numpy" in xp.xp.__name__ + +def test_set_backend(): + # Set to numpy + xp.set_backend("numpy") + assert "numpy" in xp.xp.__name__ + arr = xp.array([1]) + assert isinstance(arr, np.ndarray) + + # Set to cupy (falls back to numpy if not available) + xp.set_backend("cupy") + # If cupy is not installed, xp.xp will be numpy module + arr2 = xp.array([2]) + assert arr2 is not None + +def test_backend_bools(): + with xp.use_backend("numpy"): + assert xp.numpy_backend is True + assert xp.cupy_backend is False diff --git a/tests/unit/test_cupy.py b/tests/unit/test_cupy.py new file mode 100644 index 0000000..f1d2afb --- /dev/null +++ b/tests/unit/test_cupy.py @@ -0,0 +1,34 @@ +import pytest +import cunumpy as xp +import numpy as np + +def test_to_cupy_available(): + try: + import cupy as cp + except ImportError: + pytest.skip("CuPy not installed") + + arr = np.array([1, 2, 3]) + arr_cp = xp.to_cupy(arr) + assert isinstance(arr_cp, cp.ndarray) + +def test_to_cupy_not_available(): + try: + import cupy + pytest.skip("CuPy is installed, cannot test missing cupy error") + except ImportError: + pass + + arr = np.array([1, 2, 3]) + with pytest.raises(ImportError): + xp.to_cupy(arr) + +def test_synchronize(): + # Should not crash on any backend + xp.synchronize() + + with xp.use_backend("numpy"): + xp.synchronize() + + with xp.use_backend("cupy"): + xp.synchronize() diff --git a/tests/unit/test_numpy.py b/tests/unit/test_numpy.py new file mode 100644 index 0000000..bd481ee --- /dev/null +++ b/tests/unit/test_numpy.py @@ -0,0 +1,40 @@ +import numpy as np +import pytest +import cunumpy as xp + +def test_xp_array(): + arr = xp.array([1, 2]) + arr *= 2 + assert isinstance(arr, np.ndarray) + assert np.array_equal(arr, [2, 4]) + +def test_numpy_symbols_accessible(): + """All public numpy symbols must be reachable via cunumpy. + + This validates the runtime behaviour that the stub file (__init__.pyi) + declares to Pylance so that `xp.` shows numpy completions in VS Code. + """ + if xp.cupy_backend: + pytest.skip("CuPy does not have 100% symbol parity with NumPy.") + + # Exclude our custom methods from the numpy check + custom_methods = [ + "to_numpy", + "to_cupy", + "to_cunumpy", + "get_backend", + "is_gpu", + "is_cpu", + "use_backend", + "set_backend", + "synchronize", + "numpy_backend", + "cupy_backend", + "xp", + ] + missing = [ + name + for name in np.__all__ + if not hasattr(xp, name) and name not in custom_methods + ] + assert missing == [], f"Symbols not accessible via cunumpy: {missing}" From f190154f486c6897f0451181df5e5ee77168d9fa Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:35:30 +0200 Subject: [PATCH 15/46] Fix gitlab ci trigger --- .github/workflows/gpu_ci_trigger.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index 9cbe903..05457ec 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -47,20 +47,28 @@ jobs: - name: Trigger GitLab Pipeline & Provide Link run: | + # Give GitLab a moment to index the newly pushed branch + sleep 2 + # Trigger the pipeline and capture the JSON response - RESPONSE=$(curl --silent --request POST \ + # We use -w to get the HTTP status code as well + RESPONSE_DATA=$(curl --silent --show-error --write-out "\n%{http_code}" --request POST \ --form token=${{ secrets.GITLAB_TRIGGER_TOKEN }} \ --form "ref=$TARGET_BRANCH" \ "https://gitlab.mpcdf.mpg.de/api/v4/projects/${{ secrets.GITLAB_PROJECT_ID }}/trigger/pipeline") - # Extract the pipeline web_url using python (built-in to runner) - PIPELINE_URL=$(echo $RESPONSE | python3 -c "import sys, json; print(json.load(sys.stdin).get('web_url', ''))") + HTTP_STATUS=$(echo "$RESPONSE_DATA" | tail -n1) + RESPONSE_BODY=$(echo "$RESPONSE_DATA" | sed '$d') - if [ -z "$PIPELINE_URL" ]; then - echo "::error::Failed to trigger GitLab pipeline. Response: $RESPONSE" + if [ "$HTTP_STATUS" != "201" ]; then + echo "::error::GitLab API returned $HTTP_STATUS" + echo "::error::Response: $RESPONSE_BODY" + echo "::error::Check if GITLAB_PROJECT_ID (${{ secrets.GITLAB_PROJECT_ID }}) and GITLAB_TRIGGER_TOKEN are correct." exit 1 fi + # Extract the pipeline web_url + PIPELINE_URL=$(echo $RESPONSE_BODY | python3 -c "import sys, json; print(json.load(sys.stdin).get('web_url', ''))") + echo "::notice::GitLab GPU CI Pipeline triggered successfully!" echo "::notice::View Pipeline: $PIPELINE_URL" - echo "PIPELINE_URL=$PIPELINE_URL" >> $GITHUB_ENV From 6be17e80a948ee95831ab764137108a6b8fb1818 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:37:03 +0200 Subject: [PATCH 16/46] Sleep for 10s --- .github/workflows/gpu_ci_trigger.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index 05457ec..6cb16fc 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -48,7 +48,7 @@ jobs: - name: Trigger GitLab Pipeline & Provide Link run: | # Give GitLab a moment to index the newly pushed branch - sleep 2 + sleep 10 # Trigger the pipeline and capture the JSON response # We use -w to get the HTTP status code as well From 0210104c1e4bea8aaa16244a76dcd982b1a98678 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:39:11 +0200 Subject: [PATCH 17/46] Simplify workflow --- .github/workflows/gpu_ci_trigger.yml | 43 ++++++---------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index 6cb16fc..9ed33d7 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -21,9 +21,9 @@ jobs: with: ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} - - name: Push to GitLab via SSH + - name: Push to GitLab via SSH & Provide Link run: | - # 1. Setup SSH known hosts to prevent interactive prompts + # 1. Setup SSH known hosts mkdir -p ~/.ssh ssh-keyscan gitlab.mpcdf.mpg.de >> ~/.ssh/known_hosts @@ -34,41 +34,16 @@ jobs: TARGET_BRANCH="${{ github.ref_name }}" fi - echo "Pushing to GitLab branch: $TARGET_BRANCH" - # 3. Add GitLab SSH remote git remote add gitlab git@gitlab.mpcdf.mpg.de:maxlin/cunumpy.git - # 4. Force push + # 4. Force push (This automatically starts the GitLab Pipeline) git push -f gitlab HEAD:refs/heads/$TARGET_BRANCH - - # Store branch for next step - echo "TARGET_BRANCH=$TARGET_BRANCH" >> $GITHUB_ENV - - name: Trigger GitLab Pipeline & Provide Link - run: | - # Give GitLab a moment to index the newly pushed branch - sleep 10 - - # Trigger the pipeline and capture the JSON response - # We use -w to get the HTTP status code as well - RESPONSE_DATA=$(curl --silent --show-error --write-out "\n%{http_code}" --request POST \ - --form token=${{ secrets.GITLAB_TRIGGER_TOKEN }} \ - --form "ref=$TARGET_BRANCH" \ - "https://gitlab.mpcdf.mpg.de/api/v4/projects/${{ secrets.GITLAB_PROJECT_ID }}/trigger/pipeline") - - HTTP_STATUS=$(echo "$RESPONSE_DATA" | tail -n1) - RESPONSE_BODY=$(echo "$RESPONSE_DATA" | sed '$d') - - if [ "$HTTP_STATUS" != "201" ]; then - echo "::error::GitLab API returned $HTTP_STATUS" - echo "::error::Response: $RESPONSE_BODY" - echo "::error::Check if GITLAB_PROJECT_ID (${{ secrets.GITLAB_PROJECT_ID }}) and GITLAB_TRIGGER_TOKEN are correct." - exit 1 - fi - - # Extract the pipeline web_url - PIPELINE_URL=$(echo $RESPONSE_BODY | python3 -c "import sys, json; print(json.load(sys.stdin).get('web_url', ''))") - - echo "::notice::GitLab GPU CI Pipeline triggered successfully!" + # 5. Provide the direct link + # We construct the URL manually since the push triggers the pipeline automatically + PIPELINE_URL="https://gitlab.mpcdf.mpg.de/maxlin/cunumpy/-/pipelines?ref=$TARGET_BRANCH" + + echo "::notice::GitLab GPU CI Pipeline started automatically via Push!" echo "::notice::View Pipeline: $PIPELINE_URL" + From dbfa33a5a24700a500cf0096e99f3ae5d210740d Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:43:11 +0200 Subject: [PATCH 18/46] specify backend in tests --- tests/unit/test_cupy.py | 26 ++++++++++++++----- tests/unit/test_numpy.py | 55 ++++++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/tests/unit/test_cupy.py b/tests/unit/test_cupy.py index f1d2afb..b0cc42f 100644 --- a/tests/unit/test_cupy.py +++ b/tests/unit/test_cupy.py @@ -8,9 +8,10 @@ def test_to_cupy_available(): except ImportError: pytest.skip("CuPy not installed") - arr = np.array([1, 2, 3]) - arr_cp = xp.to_cupy(arr) - assert isinstance(arr_cp, cp.ndarray) + with xp.use_backend("cupy"): + arr = np.array([1, 2, 3]) + arr_cp = xp.to_cupy(arr) + assert isinstance(arr_cp, cp.ndarray) def test_to_cupy_not_available(): try: @@ -19,9 +20,10 @@ def test_to_cupy_not_available(): except ImportError: pass - arr = np.array([1, 2, 3]) - with pytest.raises(ImportError): - xp.to_cupy(arr) + with xp.use_backend("cupy"): + arr = np.array([1, 2, 3]) + with pytest.raises(ImportError): + xp.to_cupy(arr) def test_synchronize(): # Should not crash on any backend @@ -32,3 +34,15 @@ def test_synchronize(): with xp.use_backend("cupy"): xp.synchronize() + +def test_xp_array_cupy(): + try: + import cupy as cp + except ImportError: + pytest.skip("CuPy not installed") + + with xp.use_backend("cupy"): + arr = xp.array([1, 2]) + arr *= 2 + assert isinstance(arr, cp.ndarray) + assert cp.asnumpy(arr).tolist() == [2, 4] diff --git a/tests/unit/test_numpy.py b/tests/unit/test_numpy.py index bd481ee..5572eda 100644 --- a/tests/unit/test_numpy.py +++ b/tests/unit/test_numpy.py @@ -3,10 +3,11 @@ import cunumpy as xp def test_xp_array(): - arr = xp.array([1, 2]) - arr *= 2 - assert isinstance(arr, np.ndarray) - assert np.array_equal(arr, [2, 4]) + with xp.use_backend("numpy"): + arr = xp.array([1, 2]) + arr *= 2 + assert isinstance(arr, np.ndarray) + assert np.array_equal(arr, [2, 4]) def test_numpy_symbols_accessible(): """All public numpy symbols must be reachable via cunumpy. @@ -14,27 +15,25 @@ def test_numpy_symbols_accessible(): This validates the runtime behaviour that the stub file (__init__.pyi) declares to Pylance so that `xp.` shows numpy completions in VS Code. """ - if xp.cupy_backend: - pytest.skip("CuPy does not have 100% symbol parity with NumPy.") - - # Exclude our custom methods from the numpy check - custom_methods = [ - "to_numpy", - "to_cupy", - "to_cunumpy", - "get_backend", - "is_gpu", - "is_cpu", - "use_backend", - "set_backend", - "synchronize", - "numpy_backend", - "cupy_backend", - "xp", - ] - missing = [ - name - for name in np.__all__ - if not hasattr(xp, name) and name not in custom_methods - ] - assert missing == [], f"Symbols not accessible via cunumpy: {missing}" + with xp.use_backend("numpy"): + # Exclude our custom methods from the numpy check + custom_methods = [ + "to_numpy", + "to_cupy", + "to_cunumpy", + "get_backend", + "is_gpu", + "is_cpu", + "use_backend", + "set_backend", + "synchronize", + "numpy_backend", + "cupy_backend", + "xp", + ] + missing = [ + name + for name in np.__all__ + if not hasattr(xp, name) and name not in custom_methods + ] + assert missing == [], f"Symbols not accessible via cunumpy: {missing}" From 89769ee8812dca594a48f7fc990387d563c72c60 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:43:23 +0200 Subject: [PATCH 19/46] formatting --- tests/unit/test_cunumpy.py | 7 +++++++ tests/unit/test_cupy.py | 16 +++++++++++----- tests/unit/test_numpy.py | 3 +++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_cunumpy.py b/tests/unit/test_cunumpy.py index e2b0302..8f07ed5 100644 --- a/tests/unit/test_cunumpy.py +++ b/tests/unit/test_cunumpy.py @@ -1,7 +1,9 @@ import numpy as np import pytest + import cunumpy as xp + def test_to_numpy(): arr = xp.array([1, 2, 3]) # Even if it's already numpy, to_numpy should work @@ -9,18 +11,21 @@ def test_to_numpy(): assert isinstance(arr_np, np.ndarray) assert np.array_equal(arr_np, [1, 2, 3]) + def test_to_cunumpy(): arr = np.array([1, 2, 3]) arr_xp = xp.to_cunumpy(arr) # Backend is numpy in tests usually assert isinstance(arr_xp, (np.ndarray, xp.ndarray)) + def test_get_backend_and_is_gpu_cpu(): arr = np.array([1, 2, 3]) assert xp.get_backend(arr) == "numpy" assert xp.is_gpu(arr) is False assert xp.is_cpu(arr) is True + def test_use_backend(): # Initial backend should be numpy (default) in this test environment assert "numpy" in xp.xp.__name__ @@ -32,6 +37,7 @@ def test_use_backend(): assert "numpy" in xp.xp.__name__ + def test_set_backend(): # Set to numpy xp.set_backend("numpy") @@ -45,6 +51,7 @@ def test_set_backend(): arr2 = xp.array([2]) assert arr2 is not None + def test_backend_bools(): with xp.use_backend("numpy"): assert xp.numpy_backend is True diff --git a/tests/unit/test_cupy.py b/tests/unit/test_cupy.py index b0cc42f..935a27a 100644 --- a/tests/unit/test_cupy.py +++ b/tests/unit/test_cupy.py @@ -1,21 +1,25 @@ +import numpy as np import pytest + import cunumpy as xp -import numpy as np + def test_to_cupy_available(): try: import cupy as cp except ImportError: pytest.skip("CuPy not installed") - + with xp.use_backend("cupy"): arr = np.array([1, 2, 3]) arr_cp = xp.to_cupy(arr) assert isinstance(arr_cp, cp.ndarray) + def test_to_cupy_not_available(): try: import cupy + pytest.skip("CuPy is installed, cannot test missing cupy error") except ImportError: pass @@ -25,22 +29,24 @@ def test_to_cupy_not_available(): with pytest.raises(ImportError): xp.to_cupy(arr) + def test_synchronize(): # Should not crash on any backend xp.synchronize() - + with xp.use_backend("numpy"): xp.synchronize() - + with xp.use_backend("cupy"): xp.synchronize() + def test_xp_array_cupy(): try: import cupy as cp except ImportError: pytest.skip("CuPy not installed") - + with xp.use_backend("cupy"): arr = xp.array([1, 2]) arr *= 2 diff --git a/tests/unit/test_numpy.py b/tests/unit/test_numpy.py index 5572eda..30f00d7 100644 --- a/tests/unit/test_numpy.py +++ b/tests/unit/test_numpy.py @@ -1,7 +1,9 @@ import numpy as np import pytest + import cunumpy as xp + def test_xp_array(): with xp.use_backend("numpy"): arr = xp.array([1, 2]) @@ -9,6 +11,7 @@ def test_xp_array(): assert isinstance(arr, np.ndarray) assert np.array_equal(arr, [2, 4]) + def test_numpy_symbols_accessible(): """All public numpy symbols must be reachable via cunumpy. From 8d65bac2b688e80b79797d521d391f9d4455175d Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:55:06 +0200 Subject: [PATCH 20/46] Added some instructions to the ci trigger --- .github/workflows/gpu_ci_trigger.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index 9ed33d7..74ac41f 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -1,3 +1,22 @@ +# SETUP INSTRUCTIONS: +# ------------------ +# This workflow synchronizes the code to GitLab via SSH to trigger GPU-enabled CI. +# +# 1. GENERATE SSH KEY PAIR (on your local machine): +# ssh-keygen -t ed25519 -f ~/.ssh/gitlab_sync_key -N "" -C "github-to-gitlab-sync" +# +# 2. CONFIGURE GITLAB (The Target): +# - Go to GitLab project > Settings > Repository > Deploy keys. +# - Add the content of '~/.ssh/gitlab_sync_key.pub'. +# - IMPORTANT: Check "Allow write access to this repository". +# +# 3. CONFIGURE GITHUB (The Source): +# - Go to GitHub repo > Settings > Secrets and variables > Actions. +# - Add a new Repository Secret: +# - Name: GITLAB_SSH_PRIVATE_KEY +# - Value: Paste the entire content of '~/.ssh/gitlab_sync_key'. +# + name: Sync to GitLab and Run GPU CI on: From 241a6481e78c4f7f24821661a4847325aa712ccb Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 May 2026 10:55:28 +0200 Subject: [PATCH 21/46] Added more tests --- tests/unit/test_benchmarks.py | 79 +++++++++++++++++ tests/unit/test_features.py | 153 +++++++++++++++++++++++++++++++++ tests/unit/test_integration.py | 79 +++++++++++++++++ 3 files changed, 311 insertions(+) create mode 100644 tests/unit/test_benchmarks.py create mode 100644 tests/unit/test_features.py create mode 100644 tests/unit/test_integration.py diff --git a/tests/unit/test_benchmarks.py b/tests/unit/test_benchmarks.py new file mode 100644 index 0000000..3e8de19 --- /dev/null +++ b/tests/unit/test_benchmarks.py @@ -0,0 +1,79 @@ +import numpy as np +import pytest +import cunumpy as xp +import time + +def has_cupy(): + try: + import cupy + import cupy.cuda + return cupy.cuda.is_available() + except ImportError: + return False + +@pytest.mark.skipif(not has_cupy(), reason="CuPy/GPU not available") +def test_benchmark_matmul(): + """Benchmark matrix multiplication to show CuPy performance gain.""" + size = 2000 + + # --- Benchmark NumPy --- + with xp.use_backend("numpy"): + a_np = xp.random.rand(size, size).astype(xp.float32) + b_np = xp.random.rand(size, size).astype(xp.float32) + + start_np = time.perf_counter() + c_np = a_np @ b_np + # No sync needed for NumPy as it is synchronous + end_np = time.perf_counter() + t_np = end_np - start_np + + # --- Benchmark CuPy --- + with xp.use_backend("cupy"): + a_cp = xp.random.rand(size, size).astype(xp.float32) + b_cp = xp.random.rand(size, size).astype(xp.float32) + + # Warm up + _ = a_cp @ b_cp + xp.synchronize() + + start_cp = time.perf_counter() + c_cp = a_cp @ b_cp + xp.synchronize() # CRITICAL for benchmarking GPU + end_cp = time.perf_counter() + t_cp = end_cp - start_cp + + print(f"\n[Benchmark] Size: {size}x{size}") + print(f"NumPy time: {t_np:.4f}s") + print(f"CuPy time: {t_cp:.4f}s") + print(f"Speedup: {t_np/t_cp:.2f}x") + + # On a real GPU (A100/A30), CuPy should be significantly faster + # We use a conservative threshold of 1.5x for the test to pass on various hardware + assert t_cp < t_np, f"CuPy ({t_cp:.4f}s) was not faster than NumPy ({t_np:.4f}s)" + +@pytest.mark.skipif(not has_cupy(), reason="CuPy/GPU not available") +def test_benchmark_fft(): + """Benchmark FFT performance.""" + size = 2**22 # ~4 million elements + + with xp.use_backend("numpy"): + data_np = xp.random.rand(size).astype(xp.complex64) + start = time.perf_counter() + _ = xp.fft.fft(data_np) + t_np = time.perf_counter() - start + + with xp.use_backend("cupy"): + data_cp = xp.random.rand(size).astype(xp.complex64) + # Warm up + _ = xp.fft.fft(data_cp) + xp.synchronize() + + start = time.perf_counter() + _ = xp.fft.fft(data_cp) + xp.synchronize() + t_cp = time.perf_counter() - start + + print(f"\n[Benchmark] FFT Size: {size}") + print(f"NumPy time: {t_np:.4f}s") + print(f"CuPy time: {t_cp:.4f}s") + assert t_cp < t_np diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py new file mode 100644 index 0000000..bd85591 --- /dev/null +++ b/tests/unit/test_features.py @@ -0,0 +1,153 @@ +import numpy as np +import pytest +import cunumpy as xp + +def has_cupy(): + try: + import cupy + return True + except ImportError: + return False + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_matrix_multiplication(backend): + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + # Test basic @ operator and matmul + a = xp.array([[1, 2], [3, 4]], dtype=float) + b = xp.array([[5, 6], [7, 8]], dtype=float) + c = a @ b + + expected = np.array([[19, 22], [43, 50]]) + assert xp.array_equal(xp.to_numpy(c), expected) + + # Test linalg.norm + norm = xp.linalg.norm(a) + assert np.isclose(float(norm), np.linalg.norm([[1, 2], [3, 4]])) + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_reductions_and_axes(backend): + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + a = xp.array([[1, 10, 100], [2, 20, 200]], dtype=float) + + assert xp.sum(a) == 333 + assert xp.array_equal(xp.to_numpy(xp.max(a, axis=0)), [2, 20, 200]) + assert xp.array_equal(xp.to_numpy(xp.min(a, axis=1)), [1, 2]) + assert xp.mean(a) == 333 / 6 + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_complex_elementwise(backend): + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + a = xp.array([-1, 0, 1], dtype=float) + + # Exp and Log + exp_a = xp.exp(a) + assert np.allclose(xp.to_numpy(exp_a), np.exp([-1, 0, 1])) + + # Trig + b = xp.array([0, xp.pi/2], dtype=float) + assert np.allclose(xp.to_numpy(xp.cos(b)), [1, 0], atol=1e-7) + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_broadcasting_logic(backend): + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + # 3D + 1D broadcasting + a = xp.ones((2, 3, 4)) + b = xp.arange(4) + c = a * b + + assert c.shape == (2, 3, 4) + assert xp.array_equal(xp.to_numpy(c[0, 0]), [0, 1, 2, 3]) + assert xp.array_equal(xp.to_numpy(c[1, 2]), [0, 1, 2, 3]) + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_fft_parity(backend): + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + # Create a signal with two frequencies + t = xp.linspace(0, 1, 128) + sig = xp.sin(2 * xp.pi * 5 * t) + 0.5 * xp.sin(2 * xp.pi * 20 * t) + + freqs = xp.fft.fft(sig) + inv = xp.fft.ifft(freqs) + + # ifft(fft(x)) == x + assert np.allclose(xp.to_numpy(inv.real), xp.to_numpy(sig)) + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_realistic_normalization_workflow(backend): + """Workflow: Load data -> Compute Stats -> Normalize -> Mask Outliers.""" + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + # 1. Create dummy data with clear outliers + data = xp.array([1.0, 2.0, 3.0, 4.0, 100.0, -100.0]) + + # 2. Normalize + mean = xp.mean(data) + std = xp.std(data) + norm_data = (data - mean) / std + + # 3. Mask outliers (abs > 1.0 in this specific small set) + mask = xp.abs(norm_data) < 1.0 + clean_data = data[mask] + + # Verify: -100 and 100 should be gone + res = xp.to_numpy(xp.sort(clean_data)) + assert np.array_equal(res, [1.0, 2.0, 3.0, 4.0]) + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_stacking_and_concatenation(backend): + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + a = xp.array([1, 2, 3]) + b = xp.array([4, 5, 6]) + + res_cat = xp.concatenate([a, b]) + assert xp.array_equal(xp.to_numpy(res_cat), [1, 2, 3, 4, 5, 6]) + + res_stack = xp.stack([a, b]) + assert res_stack.shape == (2, 3) + assert xp.array_equal(xp.to_numpy(res_stack[1]), [4, 5, 6]) + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_advanced_indexing(backend): + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + a = xp.arange(10).reshape(2, 5) + + # Pick specific elements: (0,1) and (1,3) + rows = xp.array([0, 1]) + cols = xp.array([1, 3]) + + indexed = a[rows, cols] + assert xp.array_equal(xp.to_numpy(indexed), [1, 8]) + +@pytest.mark.parametrize("backend", ["numpy", "cupy"]) +def test_random_generation(backend): + if backend == "cupy" and not has_cupy(): + pytest.skip("CuPy not installed") + + with xp.use_backend(backend): + # Test reproducibility if we were to add seed (checking existing proxy) + a = xp.random.normal(0, 1, size=(100, 100)) + assert a.shape == (100, 100) + assert xp.abs(xp.mean(a)) < 0.5 # Basic statistical sanity diff --git a/tests/unit/test_integration.py b/tests/unit/test_integration.py new file mode 100644 index 0000000..1baa595 --- /dev/null +++ b/tests/unit/test_integration.py @@ -0,0 +1,79 @@ +import numpy as np +import pytest +import cunumpy as xp + +def has_cupy(): + try: + import cupy + return True + except ImportError: + return False + +def test_data_movement_chain(): + """Test CPU -> GPU -> CPU multi-hop movement.""" + if not has_cupy(): + pytest.skip("CuPy not installed") + + # 1. Start on CPU + data_orig = np.random.rand(100, 100).astype(np.float32) + + # 2. Move to GPU + data_gpu = xp.to_cupy(data_orig) + assert xp.is_gpu(data_gpu) + + # 3. Do operation on GPU + with xp.use_backend("cupy"): + res_gpu = xp.sin(data_gpu) ** 2 + xp.cos(data_gpu) ** 2 + + # 4. Move back to CPU + res_cpu = xp.to_numpy(res_gpu) + assert isinstance(res_cpu, np.ndarray) + assert np.allclose(res_cpu, 1.0) + +def test_synchronize_logic(): + """Verify synchronize can be called and handles errors gracefully.""" + # This is more of a smoke test to ensure the path doesn't crash + xp.synchronize() + + if has_cupy(): + import cupy as cp + with xp.use_backend("cupy"): + a = xp.random.rand(100) + xp.synchronize() + assert xp.is_gpu(a) + +def test_fft_interop(): + """Test FFT between backends.""" + if not has_cupy(): + pytest.skip("CuPy not installed") + + # Create signal on CPU + sig_cpu = np.random.rand(1024).astype(np.complex128) + + # Move to GPU and transform + sig_gpu = xp.to_cupy(sig_cpu) + freq_gpu = xp.fft.fft(sig_gpu) + + # Move frequencies to CPU and transform back + freq_cpu = xp.to_numpy(freq_gpu) + sig_reconstructed = np.fft.ifft(freq_cpu) + + assert np.allclose(sig_cpu, sig_reconstructed) + +def test_mixed_backend_errors(): + """Verify that mixing backends in operations raises errors (standard NumPy/CuPy behavior).""" + if not has_cupy(): + pytest.skip("CuPy not installed") + + a_cpu = np.array([1, 2, 3]) + a_gpu = xp.to_cupy(a_cpu) + + # This should fail because you can't add CPU and GPU arrays directly + with pytest.raises(Exception): + _ = a_cpu + a_gpu + + # But to_cunumpy should fix it + a_gpu_fixed = xp.to_cunumpy(a_cpu) + with xp.use_backend("cupy"): + res = a_gpu + a_gpu_fixed + assert xp.is_gpu(res) From 6e48cb7b592887c190e761a3f6e156999743a96c Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 11:18:02 +0200 Subject: [PATCH 22/46] Fixed tests --- tests/unit/test_features.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py index bd85591..0754fa8 100644 --- a/tests/unit/test_features.py +++ b/tests/unit/test_features.py @@ -36,8 +36,8 @@ def test_reductions_and_axes(backend): a = xp.array([[1, 10, 100], [2, 20, 200]], dtype=float) assert xp.sum(a) == 333 - assert xp.array_equal(xp.to_numpy(xp.max(a, axis=0)), [2, 20, 200]) - assert xp.array_equal(xp.to_numpy(xp.min(a, axis=1)), [1, 2]) + assert np.array_equal(xp.to_numpy(xp.max(a, axis=0)), [2, 20, 200]) + assert np.array_equal(xp.to_numpy(xp.min(a, axis=1)), [1, 2]) assert xp.mean(a) == 333 / 6 @pytest.mark.parametrize("backend", ["numpy", "cupy"]) @@ -68,8 +68,8 @@ def test_broadcasting_logic(backend): c = a * b assert c.shape == (2, 3, 4) - assert xp.array_equal(xp.to_numpy(c[0, 0]), [0, 1, 2, 3]) - assert xp.array_equal(xp.to_numpy(c[1, 2]), [0, 1, 2, 3]) + assert np.array_equal(xp.to_numpy(c[0, 0]), [0, 1, 2, 3]) + assert np.array_equal(xp.to_numpy(c[1, 2]), [0, 1, 2, 3]) @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_fft_parity(backend): @@ -120,11 +120,11 @@ def test_stacking_and_concatenation(backend): b = xp.array([4, 5, 6]) res_cat = xp.concatenate([a, b]) - assert xp.array_equal(xp.to_numpy(res_cat), [1, 2, 3, 4, 5, 6]) + assert np.array_equal(xp.to_numpy(res_cat), [1, 2, 3, 4, 5, 6]) res_stack = xp.stack([a, b]) assert res_stack.shape == (2, 3) - assert xp.array_equal(xp.to_numpy(res_stack[1]), [4, 5, 6]) + assert np.array_equal(xp.to_numpy(res_stack[1]), [4, 5, 6]) @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_advanced_indexing(backend): @@ -139,7 +139,7 @@ def test_advanced_indexing(backend): cols = xp.array([1, 3]) indexed = a[rows, cols] - assert xp.array_equal(xp.to_numpy(indexed), [1, 8]) + assert np.array_equal(xp.to_numpy(indexed), [1, 8]) @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_random_generation(backend): From ca6746f7e0df96864eeb9ef479c05bf0d58affc0 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 11:18:17 +0200 Subject: [PATCH 23/46] formatting --- tests/unit/test_benchmarks.py | 26 ++++++++------ tests/unit/test_features.py | 64 ++++++++++++++++++++-------------- tests/unit/test_integration.py | 32 ++++++++++------- 3 files changed, 74 insertions(+), 48 deletions(-) diff --git a/tests/unit/test_benchmarks.py b/tests/unit/test_benchmarks.py index 3e8de19..6d4aa6a 100644 --- a/tests/unit/test_benchmarks.py +++ b/tests/unit/test_benchmarks.py @@ -1,26 +1,31 @@ +import time + import numpy as np import pytest + import cunumpy as xp -import time + def has_cupy(): try: import cupy import cupy.cuda + return cupy.cuda.is_available() except ImportError: return False + @pytest.mark.skipif(not has_cupy(), reason="CuPy/GPU not available") def test_benchmark_matmul(): """Benchmark matrix multiplication to show CuPy performance gain.""" size = 2000 - + # --- Benchmark NumPy --- with xp.use_backend("numpy"): a_np = xp.random.rand(size, size).astype(xp.float32) b_np = xp.random.rand(size, size).astype(xp.float32) - + start_np = time.perf_counter() c_np = a_np @ b_np # No sync needed for NumPy as it is synchronous @@ -31,14 +36,14 @@ def test_benchmark_matmul(): with xp.use_backend("cupy"): a_cp = xp.random.rand(size, size).astype(xp.float32) b_cp = xp.random.rand(size, size).astype(xp.float32) - + # Warm up _ = a_cp @ b_cp xp.synchronize() - + start_cp = time.perf_counter() c_cp = a_cp @ b_cp - xp.synchronize() # CRITICAL for benchmarking GPU + xp.synchronize() # CRITICAL for benchmarking GPU end_cp = time.perf_counter() t_cp = end_cp - start_cp @@ -46,16 +51,17 @@ def test_benchmark_matmul(): print(f"NumPy time: {t_np:.4f}s") print(f"CuPy time: {t_cp:.4f}s") print(f"Speedup: {t_np/t_cp:.2f}x") - + # On a real GPU (A100/A30), CuPy should be significantly faster # We use a conservative threshold of 1.5x for the test to pass on various hardware assert t_cp < t_np, f"CuPy ({t_cp:.4f}s) was not faster than NumPy ({t_np:.4f}s)" + @pytest.mark.skipif(not has_cupy(), reason="CuPy/GPU not available") def test_benchmark_fft(): """Benchmark FFT performance.""" - size = 2**22 # ~4 million elements - + size = 2**22 # ~4 million elements + with xp.use_backend("numpy"): data_np = xp.random.rand(size).astype(xp.complex64) start = time.perf_counter() @@ -67,7 +73,7 @@ def test_benchmark_fft(): # Warm up _ = xp.fft.fft(data_cp) xp.synchronize() - + start = time.perf_counter() _ = xp.fft.fft(data_cp) xp.synchronize() diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py index 0754fa8..73b32f1 100644 --- a/tests/unit/test_features.py +++ b/tests/unit/test_features.py @@ -1,153 +1,165 @@ import numpy as np import pytest + import cunumpy as xp + def has_cupy(): try: import cupy + return True except ImportError: return False + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_matrix_multiplication(backend): if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): # Test basic @ operator and matmul a = xp.array([[1, 2], [3, 4]], dtype=float) b = xp.array([[5, 6], [7, 8]], dtype=float) c = a @ b - + expected = np.array([[19, 22], [43, 50]]) assert xp.array_equal(xp.to_numpy(c), expected) - + # Test linalg.norm norm = xp.linalg.norm(a) assert np.isclose(float(norm), np.linalg.norm([[1, 2], [3, 4]])) + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_reductions_and_axes(backend): if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): a = xp.array([[1, 10, 100], [2, 20, 200]], dtype=float) - + assert xp.sum(a) == 333 assert np.array_equal(xp.to_numpy(xp.max(a, axis=0)), [2, 20, 200]) assert np.array_equal(xp.to_numpy(xp.min(a, axis=1)), [1, 2]) assert xp.mean(a) == 333 / 6 + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_complex_elementwise(backend): if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): a = xp.array([-1, 0, 1], dtype=float) - + # Exp and Log exp_a = xp.exp(a) assert np.allclose(xp.to_numpy(exp_a), np.exp([-1, 0, 1])) - + # Trig - b = xp.array([0, xp.pi/2], dtype=float) + b = xp.array([0, xp.pi / 2], dtype=float) assert np.allclose(xp.to_numpy(xp.cos(b)), [1, 0], atol=1e-7) + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_broadcasting_logic(backend): if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): # 3D + 1D broadcasting a = xp.ones((2, 3, 4)) b = xp.arange(4) c = a * b - + assert c.shape == (2, 3, 4) assert np.array_equal(xp.to_numpy(c[0, 0]), [0, 1, 2, 3]) assert np.array_equal(xp.to_numpy(c[1, 2]), [0, 1, 2, 3]) + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_fft_parity(backend): if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): # Create a signal with two frequencies t = xp.linspace(0, 1, 128) sig = xp.sin(2 * xp.pi * 5 * t) + 0.5 * xp.sin(2 * xp.pi * 20 * t) - + freqs = xp.fft.fft(sig) inv = xp.fft.ifft(freqs) - + # ifft(fft(x)) == x assert np.allclose(xp.to_numpy(inv.real), xp.to_numpy(sig)) + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_realistic_normalization_workflow(backend): """Workflow: Load data -> Compute Stats -> Normalize -> Mask Outliers.""" if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): # 1. Create dummy data with clear outliers data = xp.array([1.0, 2.0, 3.0, 4.0, 100.0, -100.0]) - + # 2. Normalize mean = xp.mean(data) std = xp.std(data) norm_data = (data - mean) / std - + # 3. Mask outliers (abs > 1.0 in this specific small set) mask = xp.abs(norm_data) < 1.0 clean_data = data[mask] - + # Verify: -100 and 100 should be gone res = xp.to_numpy(xp.sort(clean_data)) assert np.array_equal(res, [1.0, 2.0, 3.0, 4.0]) + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_stacking_and_concatenation(backend): if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): a = xp.array([1, 2, 3]) b = xp.array([4, 5, 6]) - + res_cat = xp.concatenate([a, b]) assert np.array_equal(xp.to_numpy(res_cat), [1, 2, 3, 4, 5, 6]) - + res_stack = xp.stack([a, b]) assert res_stack.shape == (2, 3) assert np.array_equal(xp.to_numpy(res_stack[1]), [4, 5, 6]) + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_advanced_indexing(backend): if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): a = xp.arange(10).reshape(2, 5) - + # Pick specific elements: (0,1) and (1,3) rows = xp.array([0, 1]) cols = xp.array([1, 3]) - + indexed = a[rows, cols] assert np.array_equal(xp.to_numpy(indexed), [1, 8]) + @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_random_generation(backend): if backend == "cupy" and not has_cupy(): pytest.skip("CuPy not installed") - + with xp.use_backend(backend): # Test reproducibility if we were to add seed (checking existing proxy) a = xp.random.normal(0, 1, size=(100, 100)) assert a.shape == (100, 100) - assert xp.abs(xp.mean(a)) < 0.5 # Basic statistical sanity + assert xp.abs(xp.mean(a)) < 0.5 # Basic statistical sanity diff --git a/tests/unit/test_integration.py b/tests/unit/test_integration.py index 1baa595..14a3f57 100644 --- a/tests/unit/test_integration.py +++ b/tests/unit/test_integration.py @@ -1,77 +1,85 @@ import numpy as np import pytest + import cunumpy as xp + def has_cupy(): try: import cupy + return True except ImportError: return False + def test_data_movement_chain(): """Test CPU -> GPU -> CPU multi-hop movement.""" if not has_cupy(): pytest.skip("CuPy not installed") - + # 1. Start on CPU data_orig = np.random.rand(100, 100).astype(np.float32) - + # 2. Move to GPU data_gpu = xp.to_cupy(data_orig) assert xp.is_gpu(data_gpu) - + # 3. Do operation on GPU with xp.use_backend("cupy"): res_gpu = xp.sin(data_gpu) ** 2 + xp.cos(data_gpu) ** 2 - + # 4. Move back to CPU res_cpu = xp.to_numpy(res_gpu) assert isinstance(res_cpu, np.ndarray) assert np.allclose(res_cpu, 1.0) + def test_synchronize_logic(): """Verify synchronize can be called and handles errors gracefully.""" # This is more of a smoke test to ensure the path doesn't crash xp.synchronize() - + if has_cupy(): import cupy as cp + with xp.use_backend("cupy"): a = xp.random.rand(100) xp.synchronize() assert xp.is_gpu(a) + def test_fft_interop(): """Test FFT between backends.""" if not has_cupy(): pytest.skip("CuPy not installed") - + # Create signal on CPU sig_cpu = np.random.rand(1024).astype(np.complex128) - + # Move to GPU and transform sig_gpu = xp.to_cupy(sig_cpu) freq_gpu = xp.fft.fft(sig_gpu) - + # Move frequencies to CPU and transform back freq_cpu = xp.to_numpy(freq_gpu) sig_reconstructed = np.fft.ifft(freq_cpu) - + assert np.allclose(sig_cpu, sig_reconstructed) + def test_mixed_backend_errors(): """Verify that mixing backends in operations raises errors (standard NumPy/CuPy behavior).""" if not has_cupy(): pytest.skip("CuPy not installed") - + a_cpu = np.array([1, 2, 3]) a_gpu = xp.to_cupy(a_cpu) - + # This should fail because you can't add CPU and GPU arrays directly with pytest.raises(Exception): _ = a_cpu + a_gpu - + # But to_cunumpy should fix it a_gpu_fixed = xp.to_cunumpy(a_cpu) with xp.use_backend("cupy"): From c7307093e2aba77d94732e5ea8f17af385e4648e Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 11:52:16 +0200 Subject: [PATCH 24/46] fix tests --- src/cunumpy/__init__.py | 2 ++ src/cunumpy/xp.py | 20 +++++++++++++++ tests/unit/test_benchmarks.py | 18 +++++--------- tests/unit/test_cupy.py | 24 ++++++++---------- tests/unit/test_features.py | 45 ++++++++++++++-------------------- tests/unit/test_integration.py | 23 ++++++----------- 6 files changed, 63 insertions(+), 69 deletions(-) diff --git a/src/cunumpy/__init__.py b/src/cunumpy/__init__.py index c6592bd..8fc3d8f 100644 --- a/src/cunumpy/__init__.py +++ b/src/cunumpy/__init__.py @@ -2,6 +2,7 @@ from . import xp from .xp import ( get_backend, + has_cupy, is_cpu, is_gpu, set_backend, @@ -14,6 +15,7 @@ __all__ = [ "xp", + "has_cupy", "to_numpy", "to_cupy", "to_cunumpy", diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index 36d3b9c..b3a6519 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -92,6 +92,26 @@ def set_backend(backend: BackendType) -> None: array_backend._xp = array_backend._load_backend(backend) +def has_cupy() -> bool: + """Check if CuPy is available and functional.""" + try: + import cupy as cp + + # Check if a GPU is available + if not cp.is_available(): + return False + + # Verify that essential libraries are loadable by performing a small operation. + # This prevents failures in environments where CuPy is installed but CUDA + # libraries (like libcublas or libcufft) are missing. + a = cp.array([1.0], dtype=cp.float32) + _ = a @ a + + return True + except (ImportError, Exception): + return False + + def _cupy_backend() -> bool: """Check if the active global backend is CuPy.""" return array_backend.backend == "cupy" diff --git a/tests/unit/test_benchmarks.py b/tests/unit/test_benchmarks.py index 6d4aa6a..5667954 100644 --- a/tests/unit/test_benchmarks.py +++ b/tests/unit/test_benchmarks.py @@ -6,17 +6,9 @@ import cunumpy as xp -def has_cupy(): - try: - import cupy - import cupy.cuda - - return cupy.cuda.is_available() - except ImportError: - return False - - -@pytest.mark.skipif(not has_cupy(), reason="CuPy/GPU not available") +@pytest.mark.skipif( + not xp.has_cupy(), reason="CuPy/GPU not available or not functional" +) def test_benchmark_matmul(): """Benchmark matrix multiplication to show CuPy performance gain.""" size = 2000 @@ -57,7 +49,9 @@ def test_benchmark_matmul(): assert t_cp < t_np, f"CuPy ({t_cp:.4f}s) was not faster than NumPy ({t_np:.4f}s)" -@pytest.mark.skipif(not has_cupy(), reason="CuPy/GPU not available") +@pytest.mark.skipif( + not xp.has_cupy(), reason="CuPy/GPU not available or not functional" +) def test_benchmark_fft(): """Benchmark FFT performance.""" size = 2**22 # ~4 million elements diff --git a/tests/unit/test_cupy.py b/tests/unit/test_cupy.py index 935a27a..04a100d 100644 --- a/tests/unit/test_cupy.py +++ b/tests/unit/test_cupy.py @@ -5,10 +5,10 @@ def test_to_cupy_available(): - try: - import cupy as cp - except ImportError: - pytest.skip("CuPy not installed") + if not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") + + import cupy as cp with xp.use_backend("cupy"): arr = np.array([1, 2, 3]) @@ -17,12 +17,8 @@ def test_to_cupy_available(): def test_to_cupy_not_available(): - try: - import cupy - - pytest.skip("CuPy is installed, cannot test missing cupy error") - except ImportError: - pass + if xp.has_cupy(): + pytest.skip("CuPy is installed and functional, cannot test missing cupy error") with xp.use_backend("cupy"): arr = np.array([1, 2, 3]) @@ -42,10 +38,10 @@ def test_synchronize(): def test_xp_array_cupy(): - try: - import cupy as cp - except ImportError: - pytest.skip("CuPy not installed") + if not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") + + import cupy as cp with xp.use_backend("cupy"): arr = xp.array([1, 2]) diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py index 73b32f1..c920bc6 100644 --- a/tests/unit/test_features.py +++ b/tests/unit/test_features.py @@ -4,19 +4,10 @@ import cunumpy as xp -def has_cupy(): - try: - import cupy - - return True - except ImportError: - return False - - @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_matrix_multiplication(backend): - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): # Test basic @ operator and matmul @@ -34,8 +25,8 @@ def test_matrix_multiplication(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_reductions_and_axes(backend): - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): a = xp.array([[1, 10, 100], [2, 20, 200]], dtype=float) @@ -48,8 +39,8 @@ def test_reductions_and_axes(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_complex_elementwise(backend): - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): a = xp.array([-1, 0, 1], dtype=float) @@ -65,8 +56,8 @@ def test_complex_elementwise(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_broadcasting_logic(backend): - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): # 3D + 1D broadcasting @@ -81,8 +72,8 @@ def test_broadcasting_logic(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_fft_parity(backend): - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): # Create a signal with two frequencies @@ -99,8 +90,8 @@ def test_fft_parity(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_realistic_normalization_workflow(backend): """Workflow: Load data -> Compute Stats -> Normalize -> Mask Outliers.""" - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): # 1. Create dummy data with clear outliers @@ -122,8 +113,8 @@ def test_realistic_normalization_workflow(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_stacking_and_concatenation(backend): - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): a = xp.array([1, 2, 3]) @@ -139,8 +130,8 @@ def test_stacking_and_concatenation(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_advanced_indexing(backend): - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): a = xp.arange(10).reshape(2, 5) @@ -155,8 +146,8 @@ def test_advanced_indexing(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_random_generation(backend): - if backend == "cupy" and not has_cupy(): - pytest.skip("CuPy not installed") + if backend == "cupy" and not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): # Test reproducibility if we were to add seed (checking existing proxy) diff --git a/tests/unit/test_integration.py b/tests/unit/test_integration.py index 14a3f57..f3138d3 100644 --- a/tests/unit/test_integration.py +++ b/tests/unit/test_integration.py @@ -4,19 +4,10 @@ import cunumpy as xp -def has_cupy(): - try: - import cupy - - return True - except ImportError: - return False - - def test_data_movement_chain(): """Test CPU -> GPU -> CPU multi-hop movement.""" - if not has_cupy(): - pytest.skip("CuPy not installed") + if not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") # 1. Start on CPU data_orig = np.random.rand(100, 100).astype(np.float32) @@ -40,7 +31,7 @@ def test_synchronize_logic(): # This is more of a smoke test to ensure the path doesn't crash xp.synchronize() - if has_cupy(): + if xp.has_cupy(): import cupy as cp with xp.use_backend("cupy"): @@ -51,8 +42,8 @@ def test_synchronize_logic(): def test_fft_interop(): """Test FFT between backends.""" - if not has_cupy(): - pytest.skip("CuPy not installed") + if not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") # Create signal on CPU sig_cpu = np.random.rand(1024).astype(np.complex128) @@ -70,8 +61,8 @@ def test_fft_interop(): def test_mixed_backend_errors(): """Verify that mixing backends in operations raises errors (standard NumPy/CuPy behavior).""" - if not has_cupy(): - pytest.skip("CuPy not installed") + if not xp.has_cupy(): + pytest.skip("CuPy not installed or not functional") a_cpu = np.array([1, 2, 3]) a_gpu = xp.to_cupy(a_cpu) From f5ecc7a8cb69fbb41595f6307a2d6910727bba11 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 11:57:50 +0200 Subject: [PATCH 25/46] load fft --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b76c09b..b3100e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ gpu_tests: before_script: - module load python-waterboa/2025.06 - module load nvhpcsdk/26 + - module load fftw-serial/3.3.10 script: - echo "--- CUDA Sanity Check ---" - nvidia-smi From a59f36c1324f21c96e45cd94a64b9add1d4efb39 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 12:03:43 +0200 Subject: [PATCH 26/46] temp: added has cupy cache --- src/cunumpy/xp.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index b3a6519..6201f27 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -92,13 +92,21 @@ def set_backend(backend: BackendType) -> None: array_backend._xp = array_backend._load_backend(backend) +_HAS_CUPY_CACHE = None + + def has_cupy() -> bool: """Check if CuPy is available and functional.""" + global _HAS_CUPY_CACHE + if _HAS_CUPY_CACHE is not None: + return _HAS_CUPY_CACHE + try: import cupy as cp # Check if a GPU is available if not cp.is_available(): + _HAS_CUPY_CACHE = False return False # Verify that essential libraries are loadable by performing a small operation. @@ -107,8 +115,10 @@ def has_cupy() -> bool: a = cp.array([1.0], dtype=cp.float32) _ = a @ a + _HAS_CUPY_CACHE = True return True except (ImportError, Exception): + _HAS_CUPY_CACHE = False return False @@ -143,12 +153,12 @@ def to_numpy(array: Any) -> np.ndarray: def to_cupy(array: Any) -> Any: """Convert an array to a CuPy array.""" - try: - import cupy as cp + if not has_cupy(): + raise ImportError("CuPy is not available or not functional.") + + import cupy as cp - return cp.asarray(array) - except ImportError: - raise ImportError("CuPy is not available.") + return cp.asarray(array) def to_cunumpy(array: Any) -> Any: From 834bbc74e77de549770b0da23de36bbd32be9de4 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 12:34:59 +0200 Subject: [PATCH 27/46] More robust cupy import --- src/cunumpy/xp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index 6201f27..e5adeb2 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -27,13 +27,13 @@ def __init__( def _load_backend(self, backend: BackendType, verbose: bool = False) -> ModuleType: if backend == "cupy": - try: + if has_cupy(): import cupy as cp return cp - except ImportError: + else: if verbose: - print("CuPy not available.") + print("CuPy not available or not functional.") return np import numpy as np_mod @@ -163,7 +163,7 @@ def to_cupy(array: Any) -> Any: def to_cunumpy(array: Any) -> Any: """Convert an array to the currently active backend.""" - if array_backend.backend == "cupy": + if array_backend.backend == "cupy" and has_cupy(): return to_cupy(array) return to_numpy(array) From 903467c060edc1370a699cb0cad854ea53783d1a Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 13:30:32 +0200 Subject: [PATCH 28/46] Added has_cupy() method --- src/cunumpy/xp.py | 60 +++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index e5adeb2..7da64e5 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -8,6 +8,36 @@ BackendType = Literal["numpy", "cupy"] +_HAS_CUPY_CACHE = None + + +def has_cupy() -> bool: + """Check if CuPy is available and functional.""" + global _HAS_CUPY_CACHE + if _HAS_CUPY_CACHE is not None: + return _HAS_CUPY_CACHE + + try: + import cupy as cp + + # Check if a GPU is available + if not cp.is_available(): + _HAS_CUPY_CACHE = False + return False + + # Verify that essential libraries are loadable by performing a small operation. + # This prevents failures in environments where CuPy is installed but CUDA + # libraries (like libcublas or libcufft) are missing. + a = cp.array([1.0], dtype=cp.float32) + _ = a @ a + + _HAS_CUPY_CACHE = True + return True + except (ImportError, Exception): + _HAS_CUPY_CACHE = False + return False + + class ArrayBackend: def __init__( self, @@ -92,36 +122,6 @@ def set_backend(backend: BackendType) -> None: array_backend._xp = array_backend._load_backend(backend) -_HAS_CUPY_CACHE = None - - -def has_cupy() -> bool: - """Check if CuPy is available and functional.""" - global _HAS_CUPY_CACHE - if _HAS_CUPY_CACHE is not None: - return _HAS_CUPY_CACHE - - try: - import cupy as cp - - # Check if a GPU is available - if not cp.is_available(): - _HAS_CUPY_CACHE = False - return False - - # Verify that essential libraries are loadable by performing a small operation. - # This prevents failures in environments where CuPy is installed but CUDA - # libraries (like libcublas or libcufft) are missing. - a = cp.array([1.0], dtype=cp.float32) - _ = a @ a - - _HAS_CUPY_CACHE = True - return True - except (ImportError, Exception): - _HAS_CUPY_CACHE = False - return False - - def _cupy_backend() -> bool: """Check if the active global backend is CuPy.""" return array_backend.backend == "cupy" From 851a0b2b5f12864be32cc9b8a319bf2073055cd5 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 13:40:01 +0200 Subject: [PATCH 29/46] Run pytest -xvs --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3100e3..62e9599 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,4 +38,4 @@ gpu_tests: - export PATH="$HOME/.local/bin:$PATH" - export ARRAY_BACKEND=cupy - - python3 -m pytest tests/unit/ + - pytest -xvs . From ec04a25856fbd2b82d67d32d49edc1d6e94820f5 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 13:43:42 +0200 Subject: [PATCH 30/46] Renamed has_cupy to cupy_available --- src/cunumpy/__init__.py | 4 ++-- src/cunumpy/__init__.pyi | 1 + src/cunumpy/xp.py | 22 +++++++++++----------- tests/unit/test_benchmarks.py | 5 +++-- tests/unit/test_cupy.py | 6 +++--- tests/unit/test_features.py | 18 +++++++++--------- tests/unit/test_integration.py | 8 ++++---- 7 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/cunumpy/__init__.py b/src/cunumpy/__init__.py index 8fc3d8f..383f32a 100644 --- a/src/cunumpy/__init__.py +++ b/src/cunumpy/__init__.py @@ -1,8 +1,8 @@ # cunumpy/__init__.py from . import xp from .xp import ( + cupy_available, get_backend, - has_cupy, is_cpu, is_gpu, set_backend, @@ -15,7 +15,7 @@ __all__ = [ "xp", - "has_cupy", + "cupy_available", "to_numpy", "to_cupy", "to_cunumpy", diff --git a/src/cunumpy/__init__.pyi b/src/cunumpy/__init__.pyi index 2a4fb33..8f51ee6 100644 --- a/src/cunumpy/__init__.pyi +++ b/src/cunumpy/__init__.pyi @@ -12,6 +12,7 @@ from . import xp def to_numpy(array: Any) -> np.ndarray: ... def to_cupy(array: Any) -> Any: ... def to_cunumpy(array: Any) -> Any: ... +def cupy_available() -> bool: ... def get_backend(array: Any) -> str: ... def is_gpu(array: Any) -> bool: ... def is_cpu(array: Any) -> bool: ... diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index 7da64e5..92137ff 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -8,21 +8,21 @@ BackendType = Literal["numpy", "cupy"] -_HAS_CUPY_CACHE = None +_CUPY_AVAILABLE_CACHE = None -def has_cupy() -> bool: +def cupy_available() -> bool: """Check if CuPy is available and functional.""" - global _HAS_CUPY_CACHE - if _HAS_CUPY_CACHE is not None: - return _HAS_CUPY_CACHE + global _CUPY_AVAILABLE_CACHE + if _CUPY_AVAILABLE_CACHE is not None: + return _CUPY_AVAILABLE_CACHE try: import cupy as cp # Check if a GPU is available if not cp.is_available(): - _HAS_CUPY_CACHE = False + _CUPY_AVAILABLE_CACHE = False return False # Verify that essential libraries are loadable by performing a small operation. @@ -31,10 +31,10 @@ def has_cupy() -> bool: a = cp.array([1.0], dtype=cp.float32) _ = a @ a - _HAS_CUPY_CACHE = True + _CUPY_AVAILABLE_CACHE = True return True except (ImportError, Exception): - _HAS_CUPY_CACHE = False + _CUPY_AVAILABLE_CACHE = False return False @@ -57,7 +57,7 @@ def __init__( def _load_backend(self, backend: BackendType, verbose: bool = False) -> ModuleType: if backend == "cupy": - if has_cupy(): + if cupy_available(): import cupy as cp return cp @@ -153,7 +153,7 @@ def to_numpy(array: Any) -> np.ndarray: def to_cupy(array: Any) -> Any: """Convert an array to a CuPy array.""" - if not has_cupy(): + if not cupy_available(): raise ImportError("CuPy is not available or not functional.") import cupy as cp @@ -163,7 +163,7 @@ def to_cupy(array: Any) -> Any: def to_cunumpy(array: Any) -> Any: """Convert an array to the currently active backend.""" - if array_backend.backend == "cupy" and has_cupy(): + if array_backend.backend == "cupy" and cupy_available(): return to_cupy(array) return to_numpy(array) diff --git a/tests/unit/test_benchmarks.py b/tests/unit/test_benchmarks.py index 5667954..f9a9e35 100644 --- a/tests/unit/test_benchmarks.py +++ b/tests/unit/test_benchmarks.py @@ -7,7 +7,7 @@ @pytest.mark.skipif( - not xp.has_cupy(), reason="CuPy/GPU not available or not functional" + not xp.cupy_available(), reason="CuPy/GPU not available or not functional" ) def test_benchmark_matmul(): """Benchmark matrix multiplication to show CuPy performance gain.""" @@ -50,7 +50,7 @@ def test_benchmark_matmul(): @pytest.mark.skipif( - not xp.has_cupy(), reason="CuPy/GPU not available or not functional" + not xp.cupy_available(), reason="CuPy/GPU not available or not functional" ) def test_benchmark_fft(): """Benchmark FFT performance.""" @@ -76,4 +76,5 @@ def test_benchmark_fft(): print(f"\n[Benchmark] FFT Size: {size}") print(f"NumPy time: {t_np:.4f}s") print(f"CuPy time: {t_cp:.4f}s") + print(f"Speedup: {t_np/t_cp:.2f}x") assert t_cp < t_np diff --git a/tests/unit/test_cupy.py b/tests/unit/test_cupy.py index 04a100d..a8e804d 100644 --- a/tests/unit/test_cupy.py +++ b/tests/unit/test_cupy.py @@ -5,7 +5,7 @@ def test_to_cupy_available(): - if not xp.has_cupy(): + if not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") import cupy as cp @@ -17,7 +17,7 @@ def test_to_cupy_available(): def test_to_cupy_not_available(): - if xp.has_cupy(): + if xp.cupy_available(): pytest.skip("CuPy is installed and functional, cannot test missing cupy error") with xp.use_backend("cupy"): @@ -38,7 +38,7 @@ def test_synchronize(): def test_xp_array_cupy(): - if not xp.has_cupy(): + if not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") import cupy as cp diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py index c920bc6..1e0aced 100644 --- a/tests/unit/test_features.py +++ b/tests/unit/test_features.py @@ -6,7 +6,7 @@ @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_matrix_multiplication(backend): - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): @@ -25,7 +25,7 @@ def test_matrix_multiplication(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_reductions_and_axes(backend): - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): @@ -39,7 +39,7 @@ def test_reductions_and_axes(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_complex_elementwise(backend): - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): @@ -56,7 +56,7 @@ def test_complex_elementwise(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_broadcasting_logic(backend): - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): @@ -72,7 +72,7 @@ def test_broadcasting_logic(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_fft_parity(backend): - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): @@ -90,7 +90,7 @@ def test_fft_parity(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_realistic_normalization_workflow(backend): """Workflow: Load data -> Compute Stats -> Normalize -> Mask Outliers.""" - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): @@ -113,7 +113,7 @@ def test_realistic_normalization_workflow(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_stacking_and_concatenation(backend): - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): @@ -130,7 +130,7 @@ def test_stacking_and_concatenation(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_advanced_indexing(backend): - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): @@ -146,7 +146,7 @@ def test_advanced_indexing(backend): @pytest.mark.parametrize("backend", ["numpy", "cupy"]) def test_random_generation(backend): - if backend == "cupy" and not xp.has_cupy(): + if backend == "cupy" and not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") with xp.use_backend(backend): diff --git a/tests/unit/test_integration.py b/tests/unit/test_integration.py index f3138d3..d13764f 100644 --- a/tests/unit/test_integration.py +++ b/tests/unit/test_integration.py @@ -6,7 +6,7 @@ def test_data_movement_chain(): """Test CPU -> GPU -> CPU multi-hop movement.""" - if not xp.has_cupy(): + if not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") # 1. Start on CPU @@ -31,7 +31,7 @@ def test_synchronize_logic(): # This is more of a smoke test to ensure the path doesn't crash xp.synchronize() - if xp.has_cupy(): + if xp.cupy_available(): import cupy as cp with xp.use_backend("cupy"): @@ -42,7 +42,7 @@ def test_synchronize_logic(): def test_fft_interop(): """Test FFT between backends.""" - if not xp.has_cupy(): + if not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") # Create signal on CPU @@ -61,7 +61,7 @@ def test_fft_interop(): def test_mixed_backend_errors(): """Verify that mixing backends in operations raises errors (standard NumPy/CuPy behavior).""" - if not xp.has_cupy(): + if not xp.cupy_available(): pytest.skip("CuPy not installed or not functional") a_cpu = np.array([1, 2, 3]) From 64ffb33c6c520addcf85832f769c572d5c697a8f Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 13:45:19 +0200 Subject: [PATCH 31/46] skip matmul test to check if cupy is available --- src/cunumpy/xp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index 92137ff..adbfc51 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -28,8 +28,8 @@ def cupy_available() -> bool: # Verify that essential libraries are loadable by performing a small operation. # This prevents failures in environments where CuPy is installed but CUDA # libraries (like libcublas or libcufft) are missing. - a = cp.array([1.0], dtype=cp.float32) - _ = a @ a + # a = cp.array([1.0], dtype=cp.float32) + # _ = a @ a _CUPY_AVAILABLE_CACHE = True return True From 97e3b1efdcc3dea20cbaeb20b896aa21a8110541 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 14:28:58 +0200 Subject: [PATCH 32/46] Install nvidia-cublas-cu12 --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62e9599..79d4881 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,10 +32,15 @@ gpu_tests: # The MPCDF image likely has a specific python environment. # We install our dependencies into the user directory or a virtualenv. - python3 -m pip install --user cupy-cuda12x + - python3 -m pip install --user nvidia-cublas-cu12 nvidia-cufft-cu12 nvidia-curand-cu12 nvidia-cusolver-cu12 nvidia-cusparse-cu12 - python3 -m pip install --user -e . # Add the user bin to PATH for pytest - export PATH="$HOME/.local/bin:$PATH" + + # Try to find libcublas and other libraries in the HPC environment + - export LD_LIBRARY_PATH=$(find /mpcdf/soft /opt/nvidia -name libcublas.so.12 -exec dirname {} \; 2>/dev/null | head -n 1):$LD_LIBRARY_PATH + - export ARRAY_BACKEND=cupy - pytest -xvs . From 54f5baaba5e3248ab0c702aa80c671ac2442469b Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 14:31:40 +0200 Subject: [PATCH 33/46] Remove the matmul test to check for _CUPY_AVAILABLE_CACHE --- src/cunumpy/xp.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index adbfc51..8e92e9a 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -21,18 +21,8 @@ def cupy_available() -> bool: import cupy as cp # Check if a GPU is available - if not cp.is_available(): - _CUPY_AVAILABLE_CACHE = False - return False - - # Verify that essential libraries are loadable by performing a small operation. - # This prevents failures in environments where CuPy is installed but CUDA - # libraries (like libcublas or libcufft) are missing. - # a = cp.array([1.0], dtype=cp.float32) - # _ = a @ a - - _CUPY_AVAILABLE_CACHE = True - return True + _CUPY_AVAILABLE_CACHE = cp.is_available() + return _CUPY_AVAILABLE_CACHE except (ImportError, Exception): _CUPY_AVAILABLE_CACHE = False return False From 30350d0d95dbf2565fd23b04e3df0c2603a4d2b8 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 15:23:14 +0200 Subject: [PATCH 34/46] Run gitlab ci using the CLI --- .github/workflows/gpu_ci_trigger.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index 74ac41f..cc249d6 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -12,9 +12,11 @@ # # 3. CONFIGURE GITHUB (The Source): # - Go to GitHub repo > Settings > Secrets and variables > Actions. -# - Add a new Repository Secret: +# - Add new Repository Secrets: # - Name: GITLAB_SSH_PRIVATE_KEY -# - Value: Paste the entire content of '~/.ssh/gitlab_sync_key'. +# Value: Paste the entire content of '~/.ssh/gitlab_sync_key'. +# - Name: GITLAB_TOKEN +# Value: Your GitLab Personal Access Token (with 'api' and 'read_repository' scopes). # name: Sync to GitLab and Run GPU CI @@ -41,6 +43,7 @@ jobs: ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} - name: Push to GitLab via SSH & Provide Link + id: push run: | # 1. Setup SSH known hosts mkdir -p ~/.ssh @@ -52,6 +55,7 @@ jobs: else TARGET_BRANCH="${{ github.ref_name }}" fi + echo "TARGET_BRANCH=$TARGET_BRANCH" >> $GITHUB_ENV # 3. Add GitLab SSH remote git remote add gitlab git@gitlab.mpcdf.mpg.de:maxlin/cunumpy.git @@ -60,9 +64,17 @@ jobs: git push -f gitlab HEAD:refs/heads/$TARGET_BRANCH # 5. Provide the direct link - # We construct the URL manually since the push triggers the pipeline automatically PIPELINE_URL="https://gitlab.mpcdf.mpg.de/maxlin/cunumpy/-/pipelines?ref=$TARGET_BRANCH" echo "::notice::GitLab GPU CI Pipeline started automatically via Push!" echo "::notice::View Pipeline: $PIPELINE_URL" + - name: Wait for GitLab Pipeline + uses: docker://gitlab/glab:latest + env: + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GITLAB_HOST: gitlab.mpcdf.mpg.de + with: + entrypoint: glab + args: ci status --wait --branch ${{ env.TARGET_BRANCH }} --repo maxlin/cunumpy + From 0572cc4d13df357cf184cc101a67b4b8ad3c123e Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 15:43:29 +0200 Subject: [PATCH 35/46] Replaced glab --wait with --live --- .github/workflows/gpu_ci_trigger.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index cc249d6..e40d19d 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -76,5 +76,4 @@ jobs: GITLAB_HOST: gitlab.mpcdf.mpg.de with: entrypoint: glab - args: ci status --wait --branch ${{ env.TARGET_BRANCH }} --repo maxlin/cunumpy - + args: ci status --live --branch ${{ env.TARGET_BRANCH }} --repo maxlin/cunumpy From e78cef7ac24bb83d8acc336d9b8adc3d22757074 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 May 2026 15:54:59 +0200 Subject: [PATCH 36/46] Added gpu-test prefix --- .github/workflows/gpu_ci_trigger.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gpu_ci_trigger.yml b/.github/workflows/gpu_ci_trigger.yml index e40d19d..4418074 100644 --- a/.github/workflows/gpu_ci_trigger.yml +++ b/.github/workflows/gpu_ci_trigger.yml @@ -51,9 +51,11 @@ jobs: # 2. Determine target branch if [ "${{ github.event_name }}" == "pull_request" ]; then - TARGET_BRANCH="pr-${{ github.event.number }}" + TARGET_BRANCH="gpu-test-pr-${{ github.event.number }}" else - TARGET_BRANCH="${{ github.ref_name }}" + SOURCE_REF="${{ github.ref_name }}" + SAFE_REF="${SOURCE_REF//\//-}" + TARGET_BRANCH="gpu-test-${SAFE_REF}" fi echo "TARGET_BRANCH=$TARGET_BRANCH" >> $GITHUB_ENV From c4536d5261498e159f9ba32ddd7e4464fe6cc9ac Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:04:11 +0200 Subject: [PATCH 37/46] Set self._backend to cupy correctly and improved check_backend --- src/cunumpy/xp.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index 36d3b9c..2241a39 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -30,13 +30,16 @@ def _load_backend(self, backend: BackendType, verbose: bool = False) -> ModuleTy try: import cupy as cp + self._backend = "cupy" return cp except ImportError: if verbose: - print("CuPy not available.") + print("CuPy not available. Falling back to NumPy.") + self._backend = "numpy" return np import numpy as np_mod + self._backend = "numpy" return np_mod def __init_post__(self, verbose: bool = False) -> None: @@ -115,7 +118,7 @@ def synchronize() -> None: def to_numpy(array: Any) -> np.ndarray: """Convert an array to a NumPy array.""" - if hasattr(array, "get"): + if get_backend(array) == "cupy": return array.get() return np.asarray(array) From 383ef876bcb4a329519c6ac4ff68aac40729be2c Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:05:02 +0200 Subject: [PATCH 38/46] Added __version__ --- src/cunumpy/__init__.py | 8 ++++++++ src/cunumpy/__init__.pyi | 1 + 2 files changed, 9 insertions(+) diff --git a/src/cunumpy/__init__.py b/src/cunumpy/__init__.py index c6592bd..15c4636 100644 --- a/src/cunumpy/__init__.py +++ b/src/cunumpy/__init__.py @@ -1,4 +1,6 @@ # cunumpy/__init__.py +from importlib.metadata import PackageNotFoundError, version + from . import xp from .xp import ( get_backend, @@ -12,8 +14,14 @@ use_backend, ) +try: + __version__ = version("cunumpy") +except PackageNotFoundError: + __version__ = "0.0.0+unknown" + __all__ = [ "xp", + "__version__", "to_numpy", "to_cupy", "to_cunumpy", diff --git a/src/cunumpy/__init__.pyi b/src/cunumpy/__init__.pyi index 2a4fb33..b30a97b 100644 --- a/src/cunumpy/__init__.pyi +++ b/src/cunumpy/__init__.pyi @@ -22,3 +22,4 @@ def synchronize() -> None: ... numpy_backend: bool cupy_backend: bool +__version__: str From 3a6e908ae916fc30d9de8df13b362ab928cb0a32 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:05:12 +0200 Subject: [PATCH 39/46] Added tests/unit/test_cunumpy.py --- tests/unit/test_cunumpy.py | 68 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/unit/test_cunumpy.py b/tests/unit/test_cunumpy.py index 8f07ed5..6413af7 100644 --- a/tests/unit/test_cunumpy.py +++ b/tests/unit/test_cunumpy.py @@ -2,6 +2,7 @@ import pytest import cunumpy as xp +import cunumpy.xp as cxp # the internal submodule, to inspect array_backend directly def test_to_numpy(): @@ -56,3 +57,70 @@ def test_backend_bools(): with xp.use_backend("numpy"): assert xp.numpy_backend is True assert xp.cupy_backend is False + + +def test_version_is_own_package_version(): + # __version__ must resolve to cunumpy's own version, not be silently + # proxied to the active backend module's __version__ via __getattr__. + assert isinstance(xp.__version__, str) + assert xp.__version__ != "" + assert xp.__version__ != np.__version__ + + +def test_set_backend_cupy_fallback_reports_effective_backend(): + """array_backend.backend must reflect what actually loaded, not what was requested.""" + try: + import cupy # noqa: F401 + + cupy_installed = True + except ImportError: + cupy_installed = False + + xp.set_backend("cupy") + + if cupy_installed: + # Only true in CI on MPCDF, where CuPy is actually available. + assert cxp.array_backend.backend == "cupy" + assert xp.cupy_backend is True + else: + # No silent lie: requesting cupy without it installed must fall + # back to numpy *and* report "numpy", not "cupy". + assert cxp.array_backend.backend == "numpy" + assert xp.numpy_backend is True + assert xp.cupy_backend is False + + xp.set_backend("numpy") + + +def test_use_backend_cupy_fallback_restores_correctly(): + try: + import cupy # noqa: F401 + + pytest.skip("CuPy is installed; fallback behaviour is not exercised here") + except ImportError: + pass + + assert cxp.array_backend.backend == "numpy" + + with xp.use_backend("cupy"): + # Falls back to numpy since CuPy isn't available here. + assert cxp.array_backend.backend == "numpy" + + # And the previous state is restored afterwards. + assert cxp.array_backend.backend == "numpy" + + +def test_to_numpy_does_not_misdetect_get_method_as_gpu_array(): + """Objects exposing a `.get` method (e.g. dict-like objects) must not be + mistaken for CuPy arrays just because they happen to have a `.get`.""" + + class MappingLike: + def get(self, key, default=None): + raise AssertionError(".get() should not be called for non-cupy objects") + + def __array__(self): + return np.array([1, 2, 3]) + + arr = xp.to_numpy(MappingLike()) + assert isinstance(arr, np.ndarray) + assert np.array_equal(arr, [1, 2, 3]) From daef0a5231104481f75ac7f03c465ecd62f61e11 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:14:40 +0200 Subject: [PATCH 40/46] Removed unneccesary post_init --- src/cunumpy/xp.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index 2241a39..c399f37 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -42,13 +42,6 @@ def _load_backend(self, backend: BackendType, verbose: bool = False) -> ModuleTy self._backend = "numpy" return np_mod - def __init_post__(self, verbose: bool = False) -> None: - # This is now redundant but kept for compatibility if called - self._xp = self._load_backend(self._backend, verbose) - assert isinstance(self._xp, ModuleType) - if verbose: - print(f"Using {self._xp.__name__} backend.") - @property def backend(self) -> BackendType: return self._backend @@ -73,15 +66,12 @@ def use_backend(self, backend: BackendType) -> Generator[None, None, None]: self._xp = old_xp -# TODO: Make this configurable via environment variable or config file. array_backend = ArrayBackend( backend=( "cupy" if os.getenv("ARRAY_BACKEND", "numpy").lower() == "cupy" else "numpy" ), verbose=False, ) -# Re-run initialization logic properly after backend selection -array_backend.__init_post__(verbose=False) def use_backend(backend: BackendType) -> Generator[None, None, None]: From 1c052ecd6dcc716ee0e760f7f011d31dee4cd7f5 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:43:31 +0200 Subject: [PATCH 41/46] assert --> Raise, added docstring --- src/cunumpy/__init__.py | 18 ++++++++++-------- src/cunumpy/__init__.pyi | 1 + src/cunumpy/xp.py | 22 ++++++++++++++++++---- tests/unit/test_cunumpy.py | 22 ++++++++++++++++++++++ 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/src/cunumpy/__init__.py b/src/cunumpy/__init__.py index 15c4636..81875ad 100644 --- a/src/cunumpy/__init__.py +++ b/src/cunumpy/__init__.py @@ -7,6 +7,7 @@ is_cpu, is_gpu, set_backend, + set_device, synchronize, to_cunumpy, to_cupy, @@ -20,19 +21,20 @@ __version__ = "0.0.0+unknown" __all__ = [ - "xp", "__version__", - "to_numpy", - "to_cupy", - "to_cunumpy", + "cupy_backend", "get_backend", - "is_gpu", "is_cpu", - "use_backend", + "is_gpu", + "numpy_backend", "set_backend", + "set_device", "synchronize", - "numpy_backend", - "cupy_backend", + "to_cunumpy", + "to_cupy", + "to_numpy", + "use_backend", + "xp", ] diff --git a/src/cunumpy/__init__.pyi b/src/cunumpy/__init__.pyi index b30a97b..b945d93 100644 --- a/src/cunumpy/__init__.pyi +++ b/src/cunumpy/__init__.pyi @@ -18,6 +18,7 @@ def is_cpu(array: Any) -> bool: ... @contextmanager def use_backend(backend: str) -> Generator[None, None, None]: ... def set_backend(backend: str) -> None: ... +def set_device(device_id: int) -> None: ... def synchronize() -> None: ... numpy_backend: bool diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index c399f37..26dc845 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -9,15 +9,21 @@ class ArrayBackend: + """Holds the process-wide active backend (NumPy or CuPy). + + Not thread-safe: `set_backend`/`use_backend` mutate this single shared + instance in place, so concurrent code (threads, async tasks) switching + backends independently will race. Safe for the typical single-threaded + script/notebook usage this library targets. + """ + def __init__( self, backend: BackendType = "numpy", verbose: bool = False, ) -> None: - assert backend.lower() in [ - "numpy", - "cupy", - ], "Array backend must be either 'numpy' or 'cupy'." + if backend.lower() not in ("numpy", "cupy"): + raise ValueError("Array backend must be either 'numpy' or 'cupy'.") self._backend: BackendType = "cupy" if backend.lower() == "cupy" else "numpy" self._xp: ModuleType = np # Placeholder @@ -95,6 +101,14 @@ def _numpy_backend() -> bool: return array_backend.backend == "numpy" +def set_device(device_id: int) -> None: + """Select the active CUDA device for the current process (no-op on NumPy).""" + if array_backend.backend == "cupy": + import cupy as cp + + cp.cuda.Device(device_id).use() + + def synchronize() -> None: """Wait for all kernels in all streams on current device to complete.""" if array_backend.backend == "cupy": diff --git a/tests/unit/test_cunumpy.py b/tests/unit/test_cunumpy.py index 6413af7..7b89298 100644 --- a/tests/unit/test_cunumpy.py +++ b/tests/unit/test_cunumpy.py @@ -124,3 +124,25 @@ def __array__(self): arr = xp.to_numpy(MappingLike()) assert isinstance(arr, np.ndarray) assert np.array_equal(arr, [1, 2, 3]) + + +def test_invalid_backend_raises_value_error(): + with pytest.raises(ValueError): + cxp.ArrayBackend(backend="tensorflow") + + +def test_set_device_is_noop_on_numpy(): + with xp.use_backend("numpy"): + # Must not raise even though there's no GPU to select on the CPU backend. + xp.set_device(0) + + +def test_set_device_selects_cuda_device(): + try: + import cupy as cp + except ImportError: + pytest.skip("CuPy not installed") + + with xp.use_backend("cupy"): + xp.set_device(0) + assert cp.cuda.Device().id == 0 From 365cc657a8dd83e600efff70c248e1d2b893aaf3 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:44:31 +0200 Subject: [PATCH 42/46] Updated version number --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e5c7bfa..fea3ee8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "setuptools", "wheel" ] [project] name = "cunumpy" -version = "0.1.2" +version = "0.1.3" description = "Simple wrapper for numpy and cupy. Replace `import numpy as np` with `import cunumpy as xp`." readme = "README.md" keywords = [ "python" ] From c137c41616bce69f40d5a7fbabe6dcfd1d33bac7 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:54:30 +0200 Subject: [PATCH 43/46] Updated CI and changelog + small fixes --- .github/workflows/static_analysis.yml | 3 +-- .github/workflows/testing.yml | 11 +++++++--- CHANGELOG.md | 16 ++++++++++++++ src/cunumpy/__init__.pyi | 4 ++-- src/cunumpy/xp.py | 21 +++++++++++++++---- tests/unit/test_cunumpy.py | 30 +++++++++++++++++++++++++++ tests/unit/test_numpy.py | 1 - 7 files changed, 74 insertions(+), 12 deletions(-) diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 61b3cb6..151b4bb 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -78,7 +78,6 @@ jobs: ruff: runs-on: ubuntu-latest - continue-on-error: true steps: - name: Checkout the code uses: actions/checkout@v4 @@ -86,7 +85,7 @@ jobs: - name: Linting with ruff run: | pip install ruff - ruff check src/ || true + ruff check src/ pylint: runs-on: ubuntu-latest diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 0c86b54..2b15a25 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -14,6 +14,11 @@ jobs: build: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.10", "3.13"] + steps: # Checkout the repository - name: Checkout code @@ -23,16 +28,16 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.10' # Adjust as needed + python-version: ${{ matrix.python-version }} # Cache pip dependencies - name: Cache pip uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }} + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} restore-keys: | - ${{ runner.os }}-pip- + ${{ runner.os }}-pip-${{ matrix.python-version }}- - name: Install dependencies run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index ec7876c..4634055 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to the `cunumpy` library are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.3] - 2026-07-31 + +### Added +- `xp.set_device(device_id)`: Select the active CUDA device for the current process (no-op on the NumPy backend). +- `xp.__version__`: Reports the installed `cunumpy` package version. + +### Fixed +- `ArrayBackend` no longer silently reports `"cupy"` as the active backend when CuPy was requested but is unavailable; it now correctly falls back to reporting `"numpy"` so `xp.cupy_backend`/`xp.numpy_backend` reflect what actually loaded. +- `to_numpy()` no longer misdetects CPU objects that merely expose a `.get` method (e.g. dict-like objects) as CuPy arrays; it now checks `get_backend()` instead of `hasattr(array, "get")`. +- Invalid backend names now raise `ValueError` instead of relying on a bare `assert`, which was previously stripped under `python -O`. + +### Changed +- Removed the redundant `ArrayBackend.__init_post__` double-initialization path. +- `ArrayBackend` documents that it is not thread-safe (global mutable backend state). +- CI now runs the test suite across a Python 3.8/3.10/3.13 matrix instead of only 3.10, and `ruff` is now an enforced check rather than advisory. + ## [0.1.2] - 2026-05-27 ### Added diff --git a/src/cunumpy/__init__.pyi b/src/cunumpy/__init__.pyi index ac50708..8379680 100644 --- a/src/cunumpy/__init__.pyi +++ b/src/cunumpy/__init__.pyi @@ -7,7 +7,7 @@ from typing import Any, Generator import numpy as np from numpy import * -from . import xp +from . import xp as xp def to_numpy(array: Any) -> np.ndarray: ... def to_cupy(array: Any) -> Any: ... @@ -17,7 +17,7 @@ def get_backend(array: Any) -> str: ... def is_gpu(array: Any) -> bool: ... def is_cpu(array: Any) -> bool: ... @contextmanager -def use_backend(backend: str) -> Generator[None, None, None]: ... +def use_backend(backend: str) -> Generator[None]: ... def set_backend(backend: str) -> None: ... def set_device(device_id: int) -> None: ... def synchronize() -> None: ... diff --git a/src/cunumpy/xp.py b/src/cunumpy/xp.py index fcc4eef..718cea1 100644 --- a/src/cunumpy/xp.py +++ b/src/cunumpy/xp.py @@ -1,4 +1,5 @@ import os +import warnings from contextlib import contextmanager from types import ModuleType from typing import TYPE_CHECKING, Any, Generator, Literal @@ -23,7 +24,7 @@ def cupy_available() -> bool: # Check if a GPU is available _CUPY_AVAILABLE_CACHE = cp.is_available() return _CUPY_AVAILABLE_CACHE - except (ImportError, Exception): + except Exception: # noqa: BLE001 - tolerate any driver/runtime failure _CUPY_AVAILABLE_CACHE = False return False @@ -60,7 +61,9 @@ def _load_backend(self, backend: BackendType, verbose: bool = False) -> ModuleTy return cp else: if verbose: - print("CuPy not available or not functional. Falling back to NumPy.") + print( + "CuPy not available or not functional. Falling back to NumPy." + ) self._backend = "numpy" return np import numpy as np_mod @@ -68,6 +71,9 @@ def _load_backend(self, backend: BackendType, verbose: bool = False) -> ModuleTy self._backend = "numpy" return np_mod + def __repr__(self) -> str: + return f"ArrayBackend(backend={self._backend!r}, module={self._xp.__name__!r})" + @property def backend(self) -> BackendType: return self._backend @@ -136,8 +142,15 @@ def synchronize() -> None: import cupy as cp cp.cuda.Device().synchronize() - except (ImportError, AttributeError): + except ImportError: pass + except AttributeError as e: + warnings.warn( + f"CuPy synchronize() failed unexpectedly, this may indicate a " + f"CuPy API mismatch: {e}", + RuntimeWarning, + stacklevel=2, + ) def to_numpy(array: Any) -> np.ndarray: @@ -184,7 +197,7 @@ def is_cpu(array: Any) -> bool: # TYPE_CHECKING is True when type checking (e.g., mypy), but False at runtime. # This allows us to use autocompletion for xp (i.e., numpy/cupy) as if numpy was imported. if TYPE_CHECKING: - import numpy as xp + import numpy as xp # noqa: F401 - type-checker-only alias for autocompletion else: # Use module-level __getattr__ for dynamic xp (Python 3.7+) def __getattr__(name): diff --git a/tests/unit/test_cunumpy.py b/tests/unit/test_cunumpy.py index 7b89298..4c3a8b1 100644 --- a/tests/unit/test_cunumpy.py +++ b/tests/unit/test_cunumpy.py @@ -1,3 +1,5 @@ +import sys + import numpy as np import pytest @@ -146,3 +148,31 @@ def test_set_device_selects_cuda_device(): with xp.use_backend("cupy"): xp.set_device(0) assert cp.cuda.Device().id == 0 + + +def test_array_backend_repr_reports_active_backend(): + with xp.use_backend("numpy"): + assert ( + repr(cxp.array_backend) == "ArrayBackend(backend='numpy', module='numpy')" + ) + + +def test_synchronize_warns_on_attribute_error(monkeypatch): + """An AttributeError from CuPy's synchronize call (e.g. API mismatch) + must surface as a warning, not be swallowed silently.""" + + class BrokenDevice: + def synchronize(self): + raise AttributeError("simulated CuPy API mismatch") + + class FakeCupy: + cuda = type("cuda", (), {"Device": staticmethod(lambda: BrokenDevice())}) + + monkeypatch.setitem(sys.modules, "cupy", FakeCupy) + monkeypatch.setattr(cxp.array_backend, "_backend", "cupy") + + try: + with pytest.warns(RuntimeWarning, match="CuPy API mismatch"): + xp.synchronize() + finally: + monkeypatch.setattr(cxp.array_backend, "_backend", "numpy") diff --git a/tests/unit/test_numpy.py b/tests/unit/test_numpy.py index 30f00d7..2247f10 100644 --- a/tests/unit/test_numpy.py +++ b/tests/unit/test_numpy.py @@ -1,5 +1,4 @@ import numpy as np -import pytest import cunumpy as xp From aa09be310034c760e99acf8f002a6b4d3420473e Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:54:49 +0200 Subject: [PATCH 44/46] Formatting --- README.md | 3 ++- docs/source/quickstart.md | 2 +- tests/unit/test_benchmarks.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0fe4acd..9fd85b7 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ export ARRAY_BACKEND=cupy ```python import cunumpy as xp -arr = xp.array([1,2]) + +arr = xp.array([1, 2]) print(type(arr)) print(xp.__version__) diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index 773224c..99970f8 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -41,7 +41,7 @@ xp.set_backend("cupy") # Scoped backend switching with xp.use_backend("numpy"): arr_cpu = xp.zeros(10) - print(xp.is_cpu(arr_cpu)) # True + print(xp.is_cpu(arr_cpu)) # True # Explicit conversion arr_np = xp.to_numpy(arr) diff --git a/tests/unit/test_benchmarks.py b/tests/unit/test_benchmarks.py index f9a9e35..bf856fb 100644 --- a/tests/unit/test_benchmarks.py +++ b/tests/unit/test_benchmarks.py @@ -42,7 +42,7 @@ def test_benchmark_matmul(): print(f"\n[Benchmark] Size: {size}x{size}") print(f"NumPy time: {t_np:.4f}s") print(f"CuPy time: {t_cp:.4f}s") - print(f"Speedup: {t_np/t_cp:.2f}x") + print(f"Speedup: {t_np / t_cp:.2f}x") # On a real GPU (A100/A30), CuPy should be significantly faster # We use a conservative threshold of 1.5x for the test to pass on various hardware @@ -76,5 +76,5 @@ def test_benchmark_fft(): print(f"\n[Benchmark] FFT Size: {size}") print(f"NumPy time: {t_np:.4f}s") print(f"CuPy time: {t_cp:.4f}s") - print(f"Speedup: {t_np/t_cp:.2f}x") + print(f"Speedup: {t_np / t_cp:.2f}x") assert t_cp < t_np From b61fc82df6ef537c10623b4c12db12ab3acb4e5a Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 08:57:10 +0200 Subject: [PATCH 45/46] ruff check fixes --- tests/unit/test_benchmarks.py | 5 ++--- tests/unit/test_integration.py | 6 ++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_benchmarks.py b/tests/unit/test_benchmarks.py index bf856fb..4f4e165 100644 --- a/tests/unit/test_benchmarks.py +++ b/tests/unit/test_benchmarks.py @@ -1,6 +1,5 @@ import time -import numpy as np import pytest import cunumpy as xp @@ -19,7 +18,7 @@ def test_benchmark_matmul(): b_np = xp.random.rand(size, size).astype(xp.float32) start_np = time.perf_counter() - c_np = a_np @ b_np + _ = a_np @ b_np # No sync needed for NumPy as it is synchronous end_np = time.perf_counter() t_np = end_np - start_np @@ -34,7 +33,7 @@ def test_benchmark_matmul(): xp.synchronize() start_cp = time.perf_counter() - c_cp = a_cp @ b_cp + _ = a_cp @ b_cp xp.synchronize() # CRITICAL for benchmarking GPU end_cp = time.perf_counter() t_cp = end_cp - start_cp diff --git a/tests/unit/test_integration.py b/tests/unit/test_integration.py index d13764f..2e1078c 100644 --- a/tests/unit/test_integration.py +++ b/tests/unit/test_integration.py @@ -38,6 +38,7 @@ def test_synchronize_logic(): a = xp.random.rand(100) xp.synchronize() assert xp.is_gpu(a) + assert isinstance(a, cp.ndarray) def test_fft_interop(): @@ -67,8 +68,9 @@ def test_mixed_backend_errors(): a_cpu = np.array([1, 2, 3]) a_gpu = xp.to_cupy(a_cpu) - # This should fail because you can't add CPU and GPU arrays directly - with pytest.raises(Exception): + # This should fail because you can't add CPU and GPU arrays directly. + # The exact exception type is NumPy/CuPy-version dependent, hence the broad catch. + with pytest.raises(Exception): # noqa: B017 _ = a_cpu + a_gpu # But to_cunumpy should fix it From 7bfe1ea1e5e00996a13fe19ce6322e8579ccdf37 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 31 Jul 2026 09:02:41 +0200 Subject: [PATCH 46/46] Fix cupy test, backend remained from old test --- tests/unit/test_integration.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_integration.py b/tests/unit/test_integration.py index 2e1078c..9cb4486 100644 --- a/tests/unit/test_integration.py +++ b/tests/unit/test_integration.py @@ -73,8 +73,10 @@ def test_mixed_backend_errors(): with pytest.raises(Exception): # noqa: B017 _ = a_cpu + a_gpu - # But to_cunumpy should fix it - a_gpu_fixed = xp.to_cunumpy(a_cpu) + # But to_cunumpy should fix it: it must run inside the cupy backend context + # so it actually converts a_cpu to a CuPy array, not whatever the global + # backend happened to be left as by an earlier test. with xp.use_backend("cupy"): + a_gpu_fixed = xp.to_cunumpy(a_cpu) res = a_gpu + a_gpu_fixed assert xp.is_gpu(res)