diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 264fb5398c..684bbc71b3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -64,6 +64,98 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/build_cpp.sh + # Exploratory (#1817): one self-contained classifier JAR per CUDA major and architecture, + # gathered into a single Maven-repository artifact for publishing. + java-static-build-matrix: + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + matrix_name: conda-cpp-build + matrix_filter: 'map(. + {CUDA_MAJOR: (.CUDA_VER | split(".") | .[0])}) | unique_by([.ARCH, .CUDA_MAJOR])' + + java-static-build: + needs: [build-details, java-static-build-matrix] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-static-build-matrix.outputs.matrix) }} + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "cpu16" + arch: ${{ matrix.ARCH }} + container_image: "rapidsai/ci-conda:26.10-latest" + script: "ci/build_java_static.sh" + artifact-name: "cuopt_java_${{ matrix.ARCH }}_cu${{ matrix.CUDA_MAJOR }}" + file_to_upload: "java/cuopt/classifier-jars/" + + # Combines every classifier into one Maven-repository-layout artifact, which is the form a + # publishing workflow consumes. See rapidsai/build-infra#379. + # Runs the full Java suite against each classifier JAR on a GPU, with no libcuopt installed, + # so a JAR that loads but computes wrong answers fails here rather than at a user. + java-static-test: + needs: [java-static-build, java-static-build-matrix] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-static-build-matrix.outputs.matrix) }} + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-l4-latest-1" + arch: ${{ matrix.ARCH }} + container_image: "rapidsai/ci-conda:26.10-latest" + script: "ci/test_java_static.sh" + + java-static-gather: + needs: [java-static-build] + runs-on: linux-amd64-cpu4 + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Download per-classifier JAR artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cuopt_java_* + path: ${{ runner.temp }}/jars + merge-multiple: true + - name: Assemble Maven repository layout + run: | + ./java/cuopt/ci/assemble_maven_repo.sh \ + --jars-dir "${RUNNER_TEMP}/jars" \ + --output-dir "${RUNNER_TEMP}/maven-repo" + - name: Upload combined Maven repository artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cuopt_java_maven_repo + path: ${{ runner.temp }}/maven-repo + if-no-files-found: error + java-build: needs: cpp-build permissions: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ba50abcbce..6634509de1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,6 +24,10 @@ jobs: - conda-cpp-build - conda-cpp-tests - java-build + - java-static-build-matrix + - java-static-build + - java-static-test + - java-static-gather - conda-python-build - conda-python-tests - docs-build @@ -113,6 +117,7 @@ jobs: - '!SECURITY.md' - '!ci/build_wheel*.sh' - '!ci/build_java.sh' + - '!ci/build_java_static.sh' - '!ci/check_style.sh' - '!ci/docker/**' - '!ci/release/**' @@ -178,6 +183,7 @@ jobs: - '!agents/**' - '!ci/build_docs.sh' - '!ci/build_java.sh' + - '!ci/build_java_static.sh' - '!ci/build_python.sh' - '!ci/build_wheel*.sh' - '!ci/check_style.sh' @@ -213,6 +219,7 @@ jobs: test_java: - 'java/**' - 'ci/build_java.sh' + - 'ci/build_java_static.sh' - 'ci/test_java.sh' - 'dependencies.yaml' - '.github/workflows/pr.yaml' @@ -262,6 +269,7 @@ jobs: - '!agents/**' - '!ci/build_docs.sh' - '!ci/build_java.sh' + - '!ci/build_java_static.sh' - '!ci/build_wheel*.sh' - '!ci/check_style.sh' - '!ci/docker/**' @@ -337,6 +345,7 @@ jobs: - '!ci/build_docs.sh' - '!ci/build_python.sh' - '!ci/build_java.sh' + - '!ci/build_java_static.sh' - '!ci/check_style.sh' - '!ci/docker/**' - '!ci/release/**' @@ -477,6 +486,97 @@ jobs: artifact-name: "cuopt_docs" container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/build_docs.sh" + # Exploratory (#1817): one self-contained classifier JAR per CUDA major and architecture, + # gathered into a single Maven-repository artifact. Compiles libcuopt from source, so it does + # not need conda-cpp-build. + java-static-build-matrix: + needs: changed-files + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + if: >- + fromJSON(needs.changed-files.outputs.changed_file_groups).test_java || + fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + matrix_name: conda-cpp-build + matrix_filter: 'map(. + {CUDA_MAJOR: (.CUDA_VER | split(".") | .[0])}) | unique_by([.ARCH, .CUDA_MAJOR])' + + java-static-build: + needs: [java-static-build-matrix, changed-files] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-static-build-matrix.outputs.matrix) }} + if: >- + fromJSON(needs.changed-files.outputs.changed_file_groups).test_java || + fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + node_type: "cpu16" + arch: ${{ matrix.ARCH }} + container_image: "rapidsai/ci-conda:26.10-latest" + script: "ci/build_java_static.sh" + artifact-name: "cuopt_java_${{ matrix.ARCH }}_cu${{ matrix.CUDA_MAJOR }}" + file_to_upload: "java/cuopt/classifier-jars/" + + # Runs the full Java suite against each classifier JAR on a GPU, with no libcuopt installed, + # so a JAR that loads but computes wrong answers fails here rather than at a user. + java-static-test: + needs: [java-static-build, java-static-build-matrix] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-static-build-matrix.outputs.matrix) }} + with: + build_type: pull-request + node_type: "gpu-l4-latest-1" + arch: ${{ matrix.ARCH }} + container_image: "rapidsai/ci-conda:26.10-latest" + script: "ci/test_java_static.sh" + + java-static-gather: + needs: [java-static-build] + runs-on: linux-amd64-cpu4 + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Download per-classifier JAR artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cuopt_java_* + path: ${{ runner.temp }}/jars + merge-multiple: true + - name: Assemble Maven repository layout + run: | + ./java/cuopt/ci/assemble_maven_repo.sh \ + --jars-dir "${RUNNER_TEMP}/jars" \ + --output-dir "${RUNNER_TEMP}/maven-repo" + - name: Upload combined Maven repository artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cuopt_java_maven_repo + path: ${{ runner.temp }}/maven-repo + if-no-files-found: error + java-build: needs: [conda-cpp-build, changed-files] permissions: diff --git a/ci/build_java_static.sh b/ci/build_java_static.sh new file mode 100755 index 0000000000..e23dbee5d0 --- /dev/null +++ b/ci/build_java_static.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Builds a self-contained Java classifier JAR and checks that it is actually self-contained. +# +# Unlike ci/build_java.sh, which installs a prebuilt libcuopt and links it as a shared library, +# this compiles libcuopt from source as a static archive and embeds it, so the JAR is the only +# thing a consumer installs. See #1817. + +set -euo pipefail + +if [[ -e /opt/conda/etc/profile.d/conda.sh ]]; then + . /opt/conda/etc/profile.d/conda.sh +fi + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating Java static build dependencies" +ENV_YAML_DIR=$(mktemp -d) +rapids-dependency-file-generator \ + --output conda \ + --file-key java_static \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java_static + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate java_static +set -u + +rapids-print-env + +export CUOPT_PREFIX="${CONDA_PREFIX}" +STATIC_BUILD_DIR="${PWD}/cpp/build-static" +JNI_BUILD_DIR="${PWD}/java/cuopt/build/native-static" +JAR_OUTPUT_DIR="${PWD}/java/cuopt/classifier-jars" + +rapids-logger "Building the scoped static libcuopt" +BUILD_DIR="${STATIC_BUILD_DIR}" bash java/cuopt/ci/build_static_libcuopt.sh + +rapids-logger "Linking libcuopt into cuopt_jni" +cmake -S java/cuopt -B "${JNI_BUILD_DIR}" -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCUOPT_PREFIX="${CUOPT_PREFIX}" \ + -DCUOPT_STATIC_BUILD_DIR="${STATIC_BUILD_DIR}" \ + -DCUOPT_EXTRA_INCLUDE_DIRS="${PWD}/cpp/include;${STATIC_BUILD_DIR}/include" +cmake --build "${JNI_BUILD_DIR}" --parallel "${PARALLEL_LEVEL:-$(nproc)}" + +rapids-logger "Packaging the classifier JAR" +bash java/cuopt/ci/build_cuopt_java_jar.sh \ + --native-lib "${JNI_BUILD_DIR}/libcuopt_jni.so" \ + --cuda-version "${RAPIDS_CUDA_VERSION}" \ + --output-dir "${JAR_OUTPUT_DIR}" + +# The JAR looking fine on this machine proves nothing: the build environment supplies every +# dependency by construction. This resolves them the way a consumer's machine would. +rapids-logger "Verifying the JAR is self-contained" +CLASSIFIER_JAR=$(find "${JAR_OUTPUT_DIR}" -name 'cuopt-*.jar' \ + ! -name '*-sources.jar' ! -name '*-javadoc.jar' -print -quit) +bash java/cuopt/ci/verify_jar_dependencies.sh --jar "${CLASSIFIER_JAR}" + +# The gather job combines the classifier directories from every matrix entry into one Maven +# repository layout; this job uploads its own directory as-is. +rapids-logger "Result" +du -h "${CLASSIFIER_JAR}" | sed 's/^/ /' +find "${JAR_OUTPUT_DIR}" -type f | sed "s|^${JAR_OUTPUT_DIR}/| |" | sort diff --git a/ci/test_java_static.sh b/ci/test_java_static.sh new file mode 100755 index 0000000000..47182bde59 --- /dev/null +++ b/ci/test_java_static.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Runs the Java test suite against an already-packaged classifier JAR, on a GPU, with no +# libcuopt installed. See #1817. +# +# ci/build_java_static.sh checks the JAR's dependencies statically; this is the other half, +# that the libraries it carries actually load and produce correct answers. +# +# Activates -Ppackaged-jar-tests so main compilation is skipped and the JAR supplies the classes +# and the native libraries. PackagedJarOriginCheck then asserts that is genuinely where they came +# from, so a stray target/classes cannot make this pass while testing the wrong thing. +# +# CUOPT_JAVA_JAR may be set to a classifier JAR to skip the download and test it directly. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=java/cuopt/ci/java_classifier.sh +. "${REPO_ROOT}/java/cuopt/ci/java_classifier.sh" +# shellcheck source=java/cuopt/scripts/maven.sh +. "${REPO_ROOT}/java/cuopt/scripts/maven.sh" +cuopt_maven_args + +if [[ -e /opt/conda/etc/profile.d/conda.sh ]]; then + . /opt/conda/etc/profile.d/conda.sh +fi + +if [[ -z "${CUOPT_JAVA_JAR:-}" ]]; then + case "$(arch)" in + x86_64) JOB_ARCH=amd64 ;; + aarch64) JOB_ARCH=arm64 ;; + *) echo "unsupported architecture $(arch)" >&2; exit 1 ;; + esac + ARTIFACT="cuopt_java_${JOB_ARCH}_cu${RAPIDS_CUDA_VERSION%%.*}" + rapids-logger "Downloading ${ARTIFACT}" + JAVA_PKG="$(rapids-download-from-github "${ARTIFACT}")" + CUOPT_JAVA_JAR="$(cuopt_java_resolve_artifact_jar "${JAVA_PKG}")" +fi +rapids-logger "Testing $(basename "${CUOPT_JAVA_JAR}")" + +# A JDK, Maven and the CUDA runtime only. Installing libcuopt would defeat the test, since the +# JAR is supposed to carry its own copy. +rapids-logger "Creating a consumer-like environment" +ENV_YAML_DIR=$(mktemp -d) +cat > "${ENV_YAML_DIR}/env.yaml" << EOF +name: java_static_test +channels: + - conda-forge +dependencies: + - openjdk=11.* + - maven + - cuda-version=${RAPIDS_CUDA_VERSION%.*} + - libcublas + - libcusparse +EOF +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java_static_test + +set +u +conda activate java_static_test +set -u + +if [[ -e "${CONDA_PREFIX}/lib/libcuopt.so" ]]; then + echo "ERROR: libcuopt.so is present in the test environment, so passing here would not show" >&2 + echo " that the JAR is self-contained." >&2 + exit 1 +fi + +rapids-print-env +nvidia-smi + +rapids-logger "Running the suite against the packaged JAR" +cuopt_mvn -B -f "${REPO_ROOT}/java/cuopt/pom.xml" test \ + -Ppackaged-jar-tests \ + "-Dcuopt.jar.path=${CUOPT_JAVA_JAR}" + +rapids-logger "Classifier JAR verified end to end" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b375cc4c56..eccf0cc00f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -52,6 +52,7 @@ option(BUILD_LP_ONLY "Build only linear programming components, exclude routing option(SKIP_C_PYTHON_ADAPTERS "Skip building C and Python adapter files (cython_solve.cu and cuopt_c.cpp)" OFF) option(SKIP_ROUTING_BUILD "Skip building routing components" OFF) option(SKIP_GRPC_BUILD "Skip building gRPC and protobuf components" OFF) +option(CUOPT_BUILD_STATIC_LIB "Build libcuopt_static.a for embedding into a self-contained consumer" OFF) option(WRITE_FATBIN "Enable fatbin writing" ON) option(HOST_LINEINFO "Build with debug line information for host code" OFF) @@ -292,11 +293,21 @@ FetchContent_MakeAvailable(papilo) # PSLP - Lightweight C presolver for linear programs # https://github.com/dance858/PSLP +# +# v0.0.11 ignores its own verbose=false setting for one message: run_presolver() prints +# "PSLP declares problem as infeasible[.| or unbounded.]" unconditionally, unlike every other +# console message in that function, which are all gated on verbose. cuOpt sets verbose = false +# (see third_party_presolve.cpp) precisely so PSLP stays silent, so this writes unexpectedly +# straight to the process's native stdout -- observed corrupting Maven Surefire's forked-JVM +# protocol in the Java bindings, which also uses stdout as its own channel. Patched upstream at +# https://github.com/dance858/PSLP/pull/55; drop this patch once a release containing it is +# available and this GIT_TAG is bumped past it. FetchContent_Declare( pslp GIT_REPOSITORY "https://github.com/dance858/PSLP.git" GIT_TAG "v0.0.11" GIT_PROGRESS TRUE + PATCH_COMMAND sh -c "git apply --check '${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/pslp/respect_verbose_for_infeasible_message.patch' 2>/dev/null && git apply '${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/pslp/respect_verbose_for_infeasible_message.patch'; true" EXCLUDE_FROM_ALL SYSTEM ) @@ -774,9 +785,10 @@ target_link_libraries(cuopt_objs ) # ################################################################################################## -# - generate tests -------------------------------------------------------------------------------- -if (BUILD_TESTS) - include(CTest) +# - static library -------------------------------------------------------------------------------- +# Built for the internal tests, and for consumers that embed cuOpt into a single self-contained +# shared object rather than linking the shared libcuopt (see the Java classifier JARs). +if (BUILD_TESTS OR CUOPT_BUILD_STATIC_LIB) add_library(cuopt_static STATIC $) target_link_libraries(cuopt_static PUBLIC @@ -817,6 +829,12 @@ if (BUILD_TESTS) if (TARGET KaMinPar) add_dependencies(cuopt_static KaMinPar) endif () +endif (BUILD_TESTS OR CUOPT_BUILD_STATIC_LIB) + +# ################################################################################################## +# - generate tests -------------------------------------------------------------------------------- +if (BUILD_TESTS) + include(CTest) add_subdirectory(tests) endif (BUILD_TESTS) diff --git a/cpp/cmake/patches/pslp/respect_verbose_for_infeasible_message.patch b/cpp/cmake/patches/pslp/respect_verbose_for_infeasible_message.patch new file mode 100644 index 0000000000..784ab23370 --- /dev/null +++ b/cpp/cmake/patches/pslp/respect_verbose_for_infeasible_message.patch @@ -0,0 +1,15 @@ +diff --git a/src/core/Presolver.c b/src/core/Presolver.c +index c0bdc9e..426008e 100644 +--- a/src/core/Presolver.c ++++ b/src/core/Presolver.c +@@ -720,6 +720,9 @@ PresolveStatus run_presolver(Presolver *presolver) + if (status != UNCHANGED) + { + // problem detected to be infeasible or unbounded +- print_infeas_or_unbnd_message(status); ++ if (stgs->verbose) ++ { ++ print_infeas_or_unbnd_message(status); ++ } + return status; + } diff --git a/cpp/src/utilities/logger.cpp b/cpp/src/utilities/logger.cpp index 217f9c64cb..7b2170db34 100644 --- a/cpp/src/utilities/logger.cpp +++ b/cpp/src/utilities/logger.cpp @@ -1,6 +1,6 @@ /* clang-format off */ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ /* clang-format on */ @@ -55,6 +55,25 @@ log_buffer& global_log_buffer() return buffer; } +// Overrides the sink used when log_to_console is true. Null (the default) keeps writing to +// std::cout; set by language bindings whose host runtime cannot safely receive writes to the +// native stdout stream -- for example Java, where a raw write there bypasses System.out and can +// corrupt tools that intercept it, such as Maven Surefire's forked-process protocol. +static std::mutex g_console_callback_mutex; +static log_console_callback_t g_console_callback = nullptr; + +void set_console_log_callback(log_console_callback_t callback) +{ + std::lock_guard lock(g_console_callback_mutex); + g_console_callback = callback; +} + +static log_console_callback_t console_log_callback() +{ + std::lock_guard lock(g_console_callback_mutex); + return g_console_callback; +} + // Callback function for the buffer sink static void buffer_log_callback(int lvl, const char* msg) { @@ -161,8 +180,13 @@ init_logger_t::init_logger_t(std::string log_file, bool log_to_console) // re-initialize sinks if (log_to_console) { - cuopt::default_logger().sinks().push_back( - std::make_shared(std::cout)); + if (auto callback = console_log_callback(); callback != nullptr) { + cuopt::default_logger().sinks().push_back( + std::make_shared(callback)); + } else { + cuopt::default_logger().sinks().push_back( + std::make_shared(std::cout)); + } } if (!log_file.empty()) { cuopt::default_logger().sinks().push_back( diff --git a/cpp/src/utilities/logger.hpp b/cpp/src/utilities/logger.hpp index 2f9053b05f..cc0f79175e 100644 --- a/cpp/src/utilities/logger.hpp +++ b/cpp/src/utilities/logger.hpp @@ -38,6 +38,19 @@ rapids_logger::logger& default_logger(); */ void reset_default_logger(); +using log_console_callback_t = void (*)(int level, const char* message); + +/** + * @brief Overrides the sink used for console logging (settings.log_to_console == true). + * + * Passing nullptr (the default) restores writing to std::cout. Intended for language bindings + * whose host runtime cannot safely receive a raw write to the native stdout stream -- see the + * definition site in logger.cpp for why that matters. + * + * @param callback The callback to invoke for each logged line, or nullptr to restore std::cout. + */ +void set_console_log_callback(log_console_callback_t callback); + // Ref-counted logger initializer class init_logger_t { // Using shared_ptr for ref-counting diff --git a/dependencies.yaml b/dependencies.yaml index c484aa2180..bf74995554 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -58,6 +58,19 @@ files: - depends_on_librmm - depends_on_rapids_logger - java + java_static: + output: none + includes: + # libcuopt is compiled from source here rather than installed, so this needs the C++ + # build dependencies (tbb-devel, nccl, zlib, bzip2) and not depends_on_libcuopt. + - build_common + - build_cpp + - cuda + - cuda_version + - depends_on_libraft_headers + - depends_on_librmm + - depends_on_rapids_logger + - java test_python: output: none includes: diff --git a/java/cuopt/CMakeLists.txt b/java/cuopt/CMakeLists.txt index 7b1a40c714..1109807c92 100644 --- a/java/cuopt/CMakeLists.txt +++ b/java/cuopt/CMakeLists.txt @@ -33,13 +33,27 @@ set(CUOPT_INCLUDE_DIR "${CUOPT_PREFIX}/include") # (rmm::_RMM_26_10), so mixing a different copy links cleanly and then fails at dlopen with an # undefined symbol. set(CUOPT_LIBRARY "" CACHE FILEPATH "Path to libcuopt.so (defaults to CUOPT_PREFIX/lib)") +# Embedding cuOpt into libcuopt_jni.so instead of linking the shared library, so a published +# classifier JAR carries everything it needs. See #1817. +set(CUOPT_STATIC_BUILD_DIR "" CACHE PATH + "Build tree holding libcuopt_static.a; when set, cuOpt is linked into cuopt_jni statically") set(CUOPT_EXTRA_INCLUDE_DIRS "" CACHE STRING "Extra include directories searched before CUOPT_PREFIX/include") -if(NOT CUOPT_LIBRARY) +if(CUOPT_STATIC_BUILD_DIR) + set(CUOPT_STATIC_ARCHIVE "${CUOPT_STATIC_BUILD_DIR}/libcuopt_static.a") + if(NOT EXISTS "${CUOPT_STATIC_ARCHIVE}") + message(FATAL_ERROR "libcuopt_static.a was not found in ${CUOPT_STATIC_BUILD_DIR}") + endif() + # Fetched and built alongside cuOpt rather than installed, so they are found by path. + file(GLOB CUOPT_STATIC_DEP_ARCHIVES + "${CUOPT_STATIC_BUILD_DIR}/_deps/pslp-build/libPSLP.a" + "${CUOPT_STATIC_BUILD_DIR}/_deps/kaminpar-build/kaminpar-shm/libKaMinPar.a" + "${CUOPT_STATIC_BUILD_DIR}/_deps/kaminpar-build/kaminpar-common/libKaMinParCommon.a") +elseif(NOT CUOPT_LIBRARY) set(CUOPT_LIBRARY "${CUOPT_PREFIX}/lib/libcuopt.so") endif() -if(NOT EXISTS "${CUOPT_LIBRARY}") +if(NOT CUOPT_STATIC_BUILD_DIR AND NOT EXISTS "${CUOPT_LIBRARY}") message(FATAL_ERROR "cuOpt shared library was not found at ${CUOPT_LIBRARY}") endif() @@ -60,7 +74,29 @@ target_include_directories(cuopt_jni PRIVATE ${CUDAToolkit_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/../../cpp/src) -target_link_libraries(cuopt_jni PRIVATE "${CUOPT_LIBRARY}" CUDA::cudart) +if(CUOPT_STATIC_BUILD_DIR) + find_package(OpenMP REQUIRED) + # The JNI layer references only a fraction of cuOpt directly; the rest is reached through + # registrations and virtual dispatch, so the archive has to be kept whole. + target_link_libraries(cuopt_jni PRIVATE + -Wl,--whole-archive "${CUOPT_STATIC_ARCHIVE}" -Wl,--no-whole-archive + ${CUOPT_STATIC_DEP_ARCHIVES} + CUDA::cublas CUDA::cusparse CUDA::cusolver CUDA::cudart_static + OpenMP::OpenMP_CXX + # rmm is mostly header-only, but its exception types are defined in librmm.so, and + # rapids_logger likewise. Conda ships no static variant of either, so they are linked + # shared and packaged next to cuopt_jni, which finds them through its $ORIGIN RPATH. + "${CUOPT_PREFIX}/lib/librmm.so" + "${CUOPT_PREFIX}/lib/librapids_logger.so" + # KaMinPar, which the static archive pulls in, throws through TBB. + "${CUOPT_PREFIX}/lib/libtbb.so.12" + # PDLP's distributed path references NCCL unconditionally; cpp/CMakeLists.txt has no + # switch to compile it out, so it has to be linked even for a single-GPU JAR. + "${CUOPT_PREFIX}/lib/libnccl.so.2" + "${CUOPT_PREFIX}/lib/libcudss.so.0") +else() + target_link_libraries(cuopt_jni PRIVATE "${CUOPT_LIBRARY}" CUDA::cudart) +endif() # The Java module is built outside the main cuOpt build. Keep its native # loader self-contained while allowing the script to add the cuOpt runtime diff --git a/java/cuopt/ci/argparse.sh b/java/cuopt/ci/argparse.sh new file mode 100755 index 0000000000..8fc52d6e00 --- /dev/null +++ b/java/cuopt/ci/argparse.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Argument-handling helpers shared by the Java CI scripts, so a missing or empty flag fails the +# same way everywhere rather than surfacing later as an unbound variable. + +# require_value — the flag was given but its value is missing. +require_value() { + local flag=$1 + local value=${2:-} + if [[ -z ${value} ]]; then + echo "Error: ${flag} requires a value" >&2 + exit 1 + fi +} + +# require_arg — the flag itself is mandatory. +require_arg() { + local flag=$1 + local value=${2:-} + if [[ -z ${value} ]]; then + echo "Error: ${flag} is required." >&2 + if declare -F print_help > /dev/null; then + print_help >&2 + fi + exit 1 + fi +} diff --git a/java/cuopt/ci/assemble_maven_repo.sh b/java/cuopt/ci/assemble_maven_repo.sh new file mode 100755 index 0000000000..292a213807 --- /dev/null +++ b/java/cuopt/ci/assemble_maven_repo.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Gathers per-classifier JARs into one Maven-repository-layout tree, which is the form a +# publishing workflow consumes. +# +# Input: one directory per classifier, as build_cuopt_java_jar.sh writes them, each holding +# cuopt--.jar +# cuopt-.pom +# Output: com/nvidia/cuopt/cuopt// holding every classifier JAR, the sources and +# javadoc JARs, and the POM named cuopt-.pom. +# +# The POM must be named after the artifact rather than left as pom.xml, and the sources and +# javadoc JARs are required by Maven Central, so a bundle missing either is rejected late. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=java/cuopt/ci/argparse.sh +source "${SCRIPT_DIR}/argparse.sh" + +GROUP_PATH="com/nvidia/cuopt" +ARTIFACT_ID="cuopt" +JARS_DIR="" +OUTPUT_DIR="" +EXTRA_JARS_DIR="" + +print_help() { + cat << 'EOF' +Usage: assemble_maven_repo.sh --jars-dir --output-dir [--extra-jars-dir ] + +REQUIRED: + -j, --jars-dir Parent directory holding one subdirectory per classifier. + -o, --output-dir Directory to receive the Maven-repository layout. Must not exist + or must be empty, so a stale artifact cannot be published. + +OPTIONS: + -e, --extra-jars-dir Directory holding the sources and javadoc JARs, normally + java/cuopt/target. + -h, --help Show this message. +EOF +} + +while [[ $# -gt 0 ]]; do + case $1 in + -h | --help) print_help; exit 0 ;; + -j | --jars-dir) require_value "$1" "${2:-}"; JARS_DIR=$2; shift 2 ;; + -o | --output-dir) require_value "$1" "${2:-}"; OUTPUT_DIR=$2; shift 2 ;; + -e | --extra-jars-dir) require_value "$1" "${2:-}"; EXTRA_JARS_DIR=$2; shift 2 ;; + *) echo "Unknown argument: $1" >&2; print_help >&2; exit 2 ;; + esac +done + +require_arg --jars-dir "${JARS_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" + +if [[ ! -d "${JARS_DIR}" ]]; then + echo "jars directory not found: ${JARS_DIR}" >&2 + exit 1 +fi +if [[ -d "${OUTPUT_DIR}" && -n "$(ls -A "${OUTPUT_DIR}" 2>/dev/null)" ]]; then + echo "output directory ${OUTPUT_DIR} is not empty; remove it before re-running" >&2 + exit 1 +fi + +# The version is read from a JAR name rather than the POM, so the layout can only ever describe +# artifacts that are actually present. +first_jar="$(find "${JARS_DIR}" -name "${ARTIFACT_ID}-*.jar" -print -quit)" +if [[ -z "${first_jar}" ]]; then + echo "no ${ARTIFACT_ID}-*.jar found under ${JARS_DIR}" >&2 + exit 1 +fi +VERSION="$(basename "${first_jar}" | sed -E "s/^${ARTIFACT_ID}-([0-9][^-]*)-.*\.jar$/\1/")" +if [[ -z "${VERSION}" || "${VERSION}" == "$(basename "${first_jar}")" ]]; then + echo "could not read a version from $(basename "${first_jar}")" >&2 + exit 1 +fi + +TARGET="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${VERSION}" +mkdir -p "${TARGET}" +echo "Assembling ${GROUP_PATH}/${ARTIFACT_ID}/${VERSION}" + +classifiers=0 +while IFS= read -r jar; do + cp "${jar}" "${TARGET}/" + echo " $(basename "${jar}")" + classifiers=$((classifiers + 1)) +done < <(find "${JARS_DIR}" -name "${ARTIFACT_ID}-${VERSION}-*.jar" ! -name '*-sources.jar' ! -name '*-javadoc.jar' | sort) + +if [[ "${classifiers}" -eq 0 ]]; then + echo "no classifier JARs found for version ${VERSION}" >&2 + exit 1 +fi + +pom="$(find "${JARS_DIR}" -name "${ARTIFACT_ID}-${VERSION}.pom" -print -quit)" +if [[ -z "${pom}" ]]; then + echo "no ${ARTIFACT_ID}-${VERSION}.pom found under ${JARS_DIR}" >&2 + exit 1 +fi +cp "${pom}" "${TARGET}/${ARTIFACT_ID}-${VERSION}.pom" +echo " ${ARTIFACT_ID}-${VERSION}.pom" + +for kind in sources javadoc; do + extra="" + if [[ -n "${EXTRA_JARS_DIR}" ]]; then + extra="$(find "${EXTRA_JARS_DIR}" -name "${ARTIFACT_ID}-${VERSION}-${kind}.jar" -print -quit)" + fi + if [[ -z "${extra}" ]]; then + extra="$(find "${JARS_DIR}" -name "${ARTIFACT_ID}-${VERSION}-${kind}.jar" -print -quit)" + fi + if [[ -z "${extra}" ]]; then + echo "WARNING: no ${kind} JAR found; Maven Central requires one before release" >&2 + continue + fi + cp "${extra}" "${TARGET}/" + echo " ${ARTIFACT_ID}-${VERSION}-${kind}.jar" +done + +echo +echo "Maven repository layout at ${OUTPUT_DIR}" +find "${OUTPUT_DIR}" -type f | sed "s|^${OUTPUT_DIR}/| |" | sort diff --git a/java/cuopt/ci/build_cuopt_java_jar.sh b/java/cuopt/ci/build_cuopt_java_jar.sh new file mode 100755 index 0000000000..f74ab93e8d --- /dev/null +++ b/java/cuopt/ci/build_cuopt_java_jar.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Packages one classifier JAR: the Java classes plus the native library for a single +# CUDA-major/architecture pair, laid out where NativeLibraryLoader looks for it. +# +# The library placed here must be self-contained, because the JAR is the only thing a consumer +# installs. Build it with build_static_libcuopt.sh; see #1817. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=java/cuopt/ci/argparse.sh +source "${SCRIPT_DIR}/argparse.sh" +# shellcheck source=java/cuopt/scripts/maven.sh +source "${SCRIPT_DIR}/../scripts/maven.sh" +cuopt_maven_args +MODULE_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=java/cuopt/ci/java_classifier.sh +source "${SCRIPT_DIR}/java_classifier.sh" + +NATIVE_LIB="" +CUDA_VERSION="" +OUTPUT_DIR="" +ARCH="$(uname -m)" + +print_help() { + cat << 'EOF' +Usage: build_cuopt_java_jar.sh --native-lib --cuda-version --output-dir + +Packages a single self-contained cuOpt Java classifier JAR. + +REQUIRED: + -n, --native-lib Path to the built libcuopt_jni.so to embed. + -c, --cuda-version CUDA version the library was built against, e.g. 13.0.3 or 13. + Its major version becomes part of the classifier. + -o, --output-dir Directory to receive / with the JAR and its POM. + +OPTIONS: + -a, --arch Target architecture (default: uname -m). + -h, --help Show this message. +EOF +} + +while [[ $# -gt 0 ]]; do + case $1 in + -h | --help) print_help; exit 0 ;; + -n | --native-lib) require_value "$1" "${2:-}"; NATIVE_LIB=$2; shift 2 ;; + -c | --cuda-version) require_value "$1" "${2:-}"; CUDA_VERSION=$2; shift 2 ;; + -o | --output-dir) require_value "$1" "${2:-}"; OUTPUT_DIR=$2; shift 2 ;; + -a | --arch) require_value "$1" "${2:-}"; ARCH=$2; shift 2 ;; + *) echo "Unknown argument: $1" >&2; print_help >&2; exit 2 ;; + esac +done + +require_arg --native-lib "${NATIVE_LIB}" +require_arg --cuda-version "${CUDA_VERSION}" +require_arg --output-dir "${OUTPUT_DIR}" + +if [[ ! -f "${NATIVE_LIB}" ]]; then + echo "native library not found: ${NATIVE_LIB}" >&2 + exit 1 +fi + +CLASSIFIER="$(cuopt_java_classifier "${CUDA_VERSION}" "${ARCH}")" +RESOURCE_DIR="$(cuopt_java_native_resource_dir "${ARCH}")" + +# A library that still needs libcuopt.so alongside it would load on the build machine and fail +# for a consumer who installed nothing else, so refuse to ship one. +if readelf -d "${NATIVE_LIB}" 2>/dev/null | grep -q 'NEEDED.*libcuopt\.so'; then + echo "ERROR: ${NATIVE_LIB} still has a DT_NEEDED on libcuopt.so." >&2 + echo " A classifier JAR must embed a self-contained library; link the static" >&2 + echo " archive from build_static_libcuopt.sh instead. See #1817." >&2 + exit 1 +fi + +STAGING="$(mktemp -d)" +trap 'rm -rf "${STAGING}"' EXIT +mkdir -p "${STAGING}/${RESOURCE_DIR}" +cp "${NATIVE_LIB}" "${STAGING}/${RESOURCE_DIR}/libcuopt_jni.so" + +echo "Packaging classifier ${CLASSIFIER}" +echo " native library -> ${RESOURCE_DIR}/libcuopt_jni.so" + +# rmm and rapids_logger define the exception types cuOpt throws and have no static build, so +# they ship beside the JNI library, which finds them through its $ORIGIN RPATH. +for companion in librmm.so librapids_logger.so libtbb.so.12 libnccl.so.2 libcudss.so.0; do + companion_path="${CUOPT_PREFIX:-}/lib/${companion}" + if [[ ! -f "${companion_path}" ]]; then + echo "ERROR: ${companion} not found at ${companion_path}; set CUOPT_PREFIX" >&2 + exit 1 + fi + # Dereference, since the conda entries are symlinks into a versioned file. + cp -L "${companion_path}" "${STAGING}/${RESOURCE_DIR}/${companion}" + echo " companion -> ${RESOURCE_DIR}/${companion}" +done + +mkdir -p "${OUTPUT_DIR}/${CLASSIFIER}" +cuopt_mvn -f "${MODULE_DIR}/pom.xml" -B \ + -DskipTests \ + -Dcuopt.jar.classifier="${CLASSIFIER}" \ + -Dcuopt.native.resources="${STAGING}" \ + package + +# Read straight from the POM rather than asking Maven: this needs no network, and +# ci/release/update-version.sh keeps the marker in step with the version. +VERSION="$(sed -n 's/.*VERSION_UPDATE_MARKER_START-->\([^<]*\)<\/version>.*/\1/p' \ + "${MODULE_DIR}/pom.xml")" +if [[ -z "${VERSION}" ]]; then + echo "could not read the version from ${MODULE_DIR}/pom.xml" >&2 + exit 1 +fi + +# Each classifier directory carries everything Maven Central needs for the artifact, so the +# gather step can work from the classifier directories alone. +cp "${MODULE_DIR}/target/cuopt-${VERSION}-${CLASSIFIER}.jar" "${OUTPUT_DIR}/${CLASSIFIER}/" +cp "${MODULE_DIR}/pom.xml" "${OUTPUT_DIR}/${CLASSIFIER}/cuopt-${VERSION}.pom" +for kind in sources javadoc; do + if [[ -f "${MODULE_DIR}/target/cuopt-${VERSION}-${kind}.jar" ]]; then + cp "${MODULE_DIR}/target/cuopt-${VERSION}-${kind}.jar" "${OUTPUT_DIR}/${CLASSIFIER}/" + else + echo "WARNING: no ${kind} JAR in ${MODULE_DIR}/target; Maven Central requires one" >&2 + fi +done + +jar_mb=$(( $(stat -c%s "${OUTPUT_DIR}/${CLASSIFIER}/cuopt-${VERSION}-${CLASSIFIER}.jar") / 1048576 )) +echo " wrote ${OUTPUT_DIR}/${CLASSIFIER}/cuopt-${VERSION}-${CLASSIFIER}.jar (${jar_mb} MB)" diff --git a/java/cuopt/ci/build_static_libcuopt.sh b/java/cuopt/ci/build_static_libcuopt.sh new file mode 100755 index 0000000000..0888ecd525 --- /dev/null +++ b/java/cuopt/ci/build_static_libcuopt.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Builds libcuopt_static.a scoped to what the Java bindings actually expose, and reports its +# size. See #1817. +# +# The Java API covers LP, MIP and QP only, so routing, the gRPC server and NCCL's distributed +# PDLP path are all excluded. That matters because the shared libcuopt is 554 MB against 29 +# DT_NEEDED entries, and Maven Central caps an upload bundle at 1 GB — a self-contained JAR is +# only viable if the embedded library is scoped first. +# +# This script does not produce a JAR. It exists to measure whether one is feasible. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +BUILD_DIR="${BUILD_DIR:-${REPO_ROOT}/cpp/build-static}" +PARALLEL_LEVEL="${PARALLEL_LEVEL:-$(nproc)}" +CUDA_ARCHS="${CUOPT_CMAKE_CUDA_ARCHITECTURES:-RAPIDS}" + +# Routing and gRPC are excluded here rather than in a Java-specific fork of the build, because +# cpp/CMakeLists.txt already offers the switches. +cmake_args=( + -S "${REPO_ROOT}/cpp" + -B "${BUILD_DIR}" + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCUOPT_BUILD_STATIC_LIB=ON + -DBUILD_TESTS=OFF + -DSKIP_ROUTING_BUILD=ON + -DSKIP_GRPC_BUILD=ON + -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" +) + +echo "Configuring scoped static build in ${BUILD_DIR}" +cmake "${cmake_args[@]}" + +echo "Building cuopt_static with ${PARALLEL_LEVEL} jobs" +cmake --build "${BUILD_DIR}" --target cuopt_static --parallel "${PARALLEL_LEVEL}" + +archive="$(find "${BUILD_DIR}" -name 'libcuopt_static.a' -print -quit)" +if [[ -z "${archive}" ]]; then + echo "cuopt_static built but libcuopt_static.a was not found under ${BUILD_DIR}" >&2 + exit 1 +fi + +# The archive is an upper bound, not the shipped size: linking it into a shared object keeps +# only the objects that are actually referenced. +size_mb=$(( $(stat -c%s "${archive}") / 1048576 )) +echo +echo " archive : ${archive}" +echo " size : ${size_mb} MB (unlinked upper bound)" +echo +echo "Link this into libcuopt_jni.so to get the figure that decides whether a" +echo "self-contained classifier JAR fits inside the 1 GB Maven Central bundle limit." diff --git a/java/cuopt/ci/java_classifier.sh b/java/cuopt/ci/java_classifier.sh new file mode 100755 index 0000000000..7537948ebd --- /dev/null +++ b/java/cuopt/ci/java_classifier.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Derives the Maven classifier for a self-contained cuOpt Java JAR. +# +# A classifier names the one combination of CUDA major version and CPU architecture that the +# JAR's embedded native library will run on. x86_64 carries no architecture suffix, matching +# the scheme cuDF publishes under (cuda12, cuda12-arm64, cuda13, cuda13-arm64). + +# cuopt_java_classifier [arch] +# cuda-version full or major-only, e.g. "13.0.3" or "13" +# arch defaults to the host's uname -m +cuopt_java_classifier() { + local cuda_version="${1:?missing cuda version}" + local arch="${2:-$(uname -m)}" + local cuda_major="${cuda_version%%.*}" + + case "${arch}" in + x86_64 | amd64) printf 'cuda%s\n' "${cuda_major}" ;; + aarch64 | arm64) printf 'cuda%s-arm64\n' "${cuda_major}" ;; + *) + echo "unsupported architecture '${arch}'; expected x86_64 or aarch64" >&2 + return 1 + ;; + esac +} + +# The directory a JAR for this classifier expects its native library under, which is also the +# resource path the loader searches at run time. +cuopt_java_native_resource_dir() { + local arch="${1:-$(uname -m)}" + case "${arch}" in + x86_64 | amd64) printf 'amd64/Linux\n' ;; + aarch64 | arm64) printf 'aarch64/Linux\n' ;; + *) + echo "unsupported architecture '${arch}'; expected x86_64 or aarch64" >&2 + return 1 + ;; + esac +} + +# Finds the classifier JAR inside a downloaded build artifact, ignoring the sources and javadoc +# JARs that sit beside it. +cuopt_java_resolve_artifact_jar() { + local artifact_dir="${1:?missing artifact directory}" + local jar + jar="$(find "${artifact_dir}" -name 'cuopt-*.jar' \ + ! -name '*-sources.jar' ! -name '*-javadoc.jar' -print -quit)" + if [[ -z "${jar}" ]]; then + echo "no classifier JAR found under ${artifact_dir}" >&2 + return 1 + fi + printf '%s\n' "${jar}" +} diff --git a/java/cuopt/ci/verify_jar_dependencies.sh b/java/cuopt/ci/verify_jar_dependencies.sh new file mode 100755 index 0000000000..ebffb1dda9 --- /dev/null +++ b/java/cuopt/ci/verify_jar_dependencies.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Checks that a classifier JAR can satisfy its own native dependencies. +# +# A JAR is only self-contained if every library it needs is either inside it, part of the CUDA +# toolkit the classifier names, or part of the base system. Anything else resolves on a build +# machine, because the build environment happens to have it, and fails for a consumer who +# installed only the JAR. Linking libcuopt statically surfaced four such libraries one at a time +# (rmm, TBB, NCCL, cuDSS), each as an UnsatisfiedLinkError at run time; this catches that class +# of gap at build time instead. +# +# The check reads DT_NEEDED rather than resolving against a directory, because a build +# environment's lib directory contains every dependency by construction and would make any JAR +# look self-contained. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=java/cuopt/ci/argparse.sh +source "${SCRIPT_DIR}/argparse.sh" + +JAR="" + +# Supplied by the CUDA toolkit a consumer installs for the classifier's CUDA major version. +ALLOWED_CUDA_LIBRARIES=( + libcublas.so libcublasLt.so libcusparse.so libcusolver.so libcudart.so libcurand.so + libnvJitLink.so libnvrtc.so libcuda.so +) + +# Present on any Linux that can run a JVM. +ALLOWED_SYSTEM_LIBRARIES=( + libc.so libm.so libdl.so librt.so libpthread.so libstdc++.so libgcc_s.so libgomp.so + ld-linux-x86-64.so ld-linux-aarch64.so libresolv.so +) + +print_help() { + cat << 'EOF' +Usage: verify_jar_dependencies.sh --jar + +Fails if the JAR's native libraries need anything that is neither packaged inside it, nor part +of the CUDA toolkit, nor part of the base system. + +REQUIRED: + -j, --jar Classifier JAR to check. +EOF +} + +while [[ $# -gt 0 ]]; do + case $1 in + -h | --help) print_help; exit 0 ;; + -j | --jar) require_value "$1" "${2:-}"; JAR=$2; shift 2 ;; + *) echo "Unknown argument: $1" >&2; print_help >&2; exit 2 ;; + esac +done + +require_arg --jar "${JAR}" +if [[ ! -f "${JAR}" ]]; then + echo "JAR not found: ${JAR}" >&2 + exit 1 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT +unzip -q "${JAR}" '*/Linux/*.so*' -d "${WORK}" 2>/dev/null || true + +JNI_LIB="$(find "${WORK}" -name 'libcuopt_jni.so' -print -quit)" +if [[ -z "${JNI_LIB}" ]]; then + echo "ERROR: ${JAR} contains no libcuopt_jni.so" >&2 + exit 1 +fi +NATIVE_DIR="$(dirname "${JNI_LIB}")" + +echo "Packaged libraries:" +while read -r lib; do + printf ' %6s MB %s\n' "$(( $(stat -c%s "${NATIVE_DIR}/${lib}") / 1048576 ))" "${lib}" +done < <(cd "${NATIVE_DIR}" && ls -S ./*.so* | sed 's|^\./||') + +# Strip the version suffix so libnccl.so.2 matches an allowlist entry of libnccl.so. +soname_stem() { sed -E 's/\.so\.[0-9.]+$/.so/' <<< "$1"; } + +allowed_external=("${ALLOWED_CUDA_LIBRARIES[@]}" "${ALLOWED_SYSTEM_LIBRARIES[@]}") +unsatisfied=() + +echo +echo "Checking DT_NEEDED of every packaged library" +for lib in "${NATIVE_DIR}"/*.so*; do + while read -r needed; do + [[ -z "${needed}" ]] && continue + # Packaged beside it, so the $ORIGIN RPATH resolves it. + if [[ -e "${NATIVE_DIR}/${needed}" ]]; then + continue + fi + stem="$(soname_stem "${needed}")" + permitted=false + for allowed in "${allowed_external[@]}"; do + if [[ "${stem}" == "${allowed}" ]]; then + permitted=true + break + fi + done + if [[ "${permitted}" == false ]]; then + unsatisfied+=("$(basename "${lib}") needs ${needed}") + fi + done < <(readelf -d "${lib}" 2>/dev/null | sed -n 's/.*NEEDED.*\[\(.*\)\]/\1/p') +done + +if [[ ${#unsatisfied[@]} -gt 0 ]]; then + echo >&2 + echo "ERROR: the JAR is not self-contained. Unsatisfied dependencies:" >&2 + printf ' %s\n' "${unsatisfied[@]}" | sort -u >&2 + echo >&2 + echo "Each must be linked into the JNI library or packaged beside it; see" >&2 + echo "build_cuopt_java_jar.sh. A consumer installs nothing but this JAR." >&2 + exit 1 +fi + +# libcuopt.so reappearing means the static link silently fell back to the shared library. +if readelf -d "${JNI_LIB}" | grep -q 'NEEDED.*libcuopt\.so'; then + echo "ERROR: libcuopt_jni.so depends on libcuopt.so; it was not linked statically." >&2 + exit 1 +fi + +echo +echo "Self-contained: every dependency is packaged, CUDA toolkit, or base system." diff --git a/java/cuopt/pom.xml b/java/cuopt/pom.xml index 019cc08433..f6e8c66f9c 100644 --- a/java/cuopt/pom.xml +++ b/java/cuopt/pom.xml @@ -43,6 +43,15 @@ SPDX-License-Identifier: Apache-2.0 11 UTF-8 5.11.4 + + + + ${project.basedir}/src/main/no-native + + **/PackagedJarOriginCheck.java @@ -55,11 +64,29 @@ SPDX-License-Identifier: Apache-2.0 + + + ${cuopt.native.resources} + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + ${cuopt.surefire.excludes} + + + org.apache.maven.plugins maven-jar-plugin 3.4.2 + + ${cuopt.jar.classifier} + org.apache.maven.plugins @@ -154,4 +181,47 @@ SPDX-License-Identifier: Apache-2.0 + + + + + packaged-jar-tests + + true + **/NothingIsExcludedHere.java + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + + **/*Test.java + **/PackagedJarOriginCheck.java + + + + + + + + com.nvidia.cuopt + cuopt-packaged + ${project.version} + system + ${cuopt.jar.path} + + + + diff --git a/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeCuOpt.java b/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeCuOpt.java index abfa374dbe..8c0c29d1b1 100644 --- a/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeCuOpt.java +++ b/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeCuOpt.java @@ -4,16 +4,9 @@ */ package com.nvidia.cuopt.mathematicaloptimization; -import java.nio.file.Path; - final class NativeCuOpt { static { - String nativeDir = System.getProperty("cuopt.native.dir"); - if (nativeDir == null || nativeDir.isBlank()) { - System.loadLibrary("cuopt_jni"); - } else { - System.load(Path.of(nativeDir, System.mapLibraryName("cuopt_jni")).toAbsolutePath().toString()); - } + NativeLibraryLoader.load(); } private NativeCuOpt() {} diff --git a/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeLibraryLoader.java b/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeLibraryLoader.java new file mode 100644 index 0000000000..1d92a2a497 --- /dev/null +++ b/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeLibraryLoader.java @@ -0,0 +1,136 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuopt.mathematicaloptimization; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +/** + * Locates and loads {@code libcuopt_jni}, in three steps. + * + *
    + *
  1. {@code -Dcuopt.native.dir}, for a library built from source; + *
  2. a copy embedded in this JAR, which is how the classifier artifacts ship; + *
  3. {@code System.loadLibrary}, for a library already on the library path. + *
+ */ +final class NativeLibraryLoader { + private static final String LIBRARY_NAME = "cuopt_jni"; + + /** rmm, rapids_logger and TBB have no static build, so they travel beside the JNI library. */ + private static final String[] COMPANION_LIBRARIES = {"librmm.so", "librapids_logger.so", "libtbb.so.12", "libnccl.so.2", "libcudss.so.0"}; + + private NativeLibraryLoader() {} + + static void load() { + String nativeDir = System.getProperty("cuopt.native.dir"); + if (nativeDir != null && !nativeDir.isBlank()) { + System.load( + Path.of(nativeDir, System.mapLibraryName(LIBRARY_NAME)).toAbsolutePath().toString()); + return; + } + + Path embedded = extractEmbeddedLibraries(); + if (embedded != null) { + System.load(embedded.toString()); + return; + } + + System.loadLibrary(LIBRARY_NAME); + } + + /** + * The path an embedded library occupies, which is also the layout the packaging step writes. + * {@code os.arch} reports {@code amd64} on x86_64 JVMs and {@code aarch64} on ARM ones. + */ + static String resourcePath(String osArch, String libraryFileName) { + String directory; + switch (osArch) { + case "amd64": + case "x86_64": + directory = "amd64"; + break; + case "aarch64": + case "arm64": + directory = "aarch64"; + break; + default: + throw new IllegalStateException( + "cuOpt has no native library for architecture '" + osArch + "'"); + } + return "/" + directory + "/Linux/" + libraryFileName; + } + + /** + * Copies the packaged libraries out of the JAR and returns the path of the JNI one, or null when + * this JAR does not carry them. + * + *

The companions are not loaded here. The JNI library's {@code $ORIGIN} RPATH resolves them + * once they sit in the same directory, so they only have to be on disk before it is loaded. + */ + private static Path extractEmbeddedLibraries() { + String osArch = System.getProperty("os.arch", ""); + String fileName = System.mapLibraryName(LIBRARY_NAME); + String resource; + try { + resource = resourcePath(osArch, fileName); + } catch (IllegalStateException e) { + return null; + } + if (NativeLibraryLoader.class.getResource(resource) == null) { + return null; + } + + try { + Path directory = + Path.of( + System.getProperty("java.io.tmpdir"), + "cuopt-native-" + System.getProperty("user.name", "shared")); + Files.createDirectories(directory); + + for (String companion : COMPANION_LIBRARIES) { + extractResource(resourcePath(osArch, companion), directory, companion); + } + return extractResource(resource, directory, fileName); + } catch (IOException e) { + throw new UncheckedIOException("failed to extract native libraries from the cuOpt JAR", e); + } + } + + /** + * Copies one packaged file into {@code directory} and returns it, or null when the JAR does not + * contain it. + * + *

A file already there with the expected size is reused rather than rewritten, because the JNI + * library is hundreds of megabytes and re-extracting it on every JVM start would dominate + * startup. It is written to a sibling and moved into place, so an interrupted run cannot leave a + * truncated library behind for the next one to load. + */ + private static Path extractResource(String resource, Path directory, String fileName) + throws IOException { + URL url = NativeLibraryLoader.class.getResource(resource); + if (url == null) { + return null; + } + Path target = directory.resolve(fileName); + long expectedSize = url.openConnection().getContentLengthLong(); + if (expectedSize >= 0 && Files.isRegularFile(target) && Files.size(target) == expectedSize) { + return target; + } + Path staging = Files.createTempFile(directory, fileName + ".", ".part"); + try (InputStream in = url.openStream()) { + Files.copy(in, staging, StandardCopyOption.REPLACE_EXISTING); + Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING); + } finally { + Files.deleteIfExists(staging); + } + return target; + } +} diff --git a/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeLogSink.java b/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeLogSink.java new file mode 100644 index 0000000000..1656aaea6f --- /dev/null +++ b/java/cuopt/src/main/java/com/nvidia/cuopt/mathematicaloptimization/NativeLogSink.java @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuopt.mathematicaloptimization; + +/** + * Receives cuOpt's console log lines from native code and writes them through {@link + * System#out}, rather than the native library writing to the process's stdout stream directly. + * + *

A direct native write bypasses {@code System.out}, so it is invisible to anything that + * intercepts or redirects it -- {@link System#setOut}, a logging framework bridge, or Maven + * Surefire, which uses the forked JVM's stdout as its own communication channel and can + * misinterpret an unexpected raw write on it as the forked process having crashed. + * + *

Called from {@code cuopt_jni.cpp}; not part of the public API. + */ +final class NativeLogSink { + private NativeLogSink() {} + + static void onLogLine(String message) { + System.out.print(message); + } +} diff --git a/java/cuopt/src/main/native/cuopt_jni.cpp b/java/cuopt/src/main/native/cuopt_jni.cpp index 2f7cc27ef1..439ac63cd1 100644 --- a/java/cuopt/src/main/native/cuopt_jni.cpp +++ b/java/cuopt/src/main/native/cuopt_jni.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include @@ -378,6 +379,57 @@ void mip_set_solution_callback(cuopt_float_t* solution, if (detach) { g_jvm->DetachCurrentThread(); } } +jclass g_log_sink_class = nullptr; +jmethodID g_log_sink_method = nullptr; +std::once_flag g_log_sink_once; + +// cuopt::log_console_callback_t: forwards a console log line to NativeLogSink.onLogLine, so it +// is written through System.out instead of directly to the native stdout stream. See +// register_console_log_sink for why that distinction matters. +void console_log_callback(int /* level */, const char* message) +{ + if (g_log_sink_class == nullptr || g_log_sink_method == nullptr) { return; } + + bool detach = false; + JNIEnv* env = get_callback_env(detach); + if (env == nullptr) { return; } + + jstring line = env->NewStringUTF(message); + if (line != nullptr) { + env->CallStaticVoidMethod(g_log_sink_class, g_log_sink_method, line); + // A logging call is not the place to raise a Java exception; drop it rather than leave it + // pending for whatever JNI call happens to run next on this thread. + if (env->ExceptionCheck() == JNI_TRUE) { env->ExceptionClear(); } + env->DeleteLocalRef(line); + } + + if (detach) { g_jvm->DetachCurrentThread(); } +} + +// Registers console_log_callback with the native logger, once. Done lazily on first use (rather +// than in JNI_OnLoad) because FindClass needs the caller's classloader, which JNI_OnLoad does not +// reliably have. +void register_console_log_sink(JNIEnv* env) +{ + std::call_once(g_log_sink_once, [env]() { + jclass local_cls = env->FindClass("com/nvidia/cuopt/mathematicaloptimization/NativeLogSink"); + if (local_cls == nullptr) { + env->ExceptionClear(); + return; + } + jmethodID method = env->GetStaticMethodID(local_cls, "onLogLine", "(Ljava/lang/String;)V"); + if (method == nullptr) { + env->ExceptionClear(); + env->DeleteLocalRef(local_cls); + return; + } + g_log_sink_class = static_cast(env->NewGlobalRef(local_cls)); + g_log_sink_method = method; + env->DeleteLocalRef(local_cls); + cuopt::set_console_log_callback(&console_log_callback); + }); +} + } // namespace extern "C" jint JNI_OnLoad(JavaVM* vm, void*) @@ -421,6 +473,7 @@ Java_com_nvidia_cuopt_mathematicaloptimization_NativeCuOpt_readProblemWithFormat extern "C" JNIEXPORT jlong JNICALL Java_com_nvidia_cuopt_mathematicaloptimization_NativeCuOpt_createSolverSettings(JNIEnv* env, jclass) { + register_console_log_sink(env); cuOptSolverSettings settings = nullptr; if (!check_status(env, cuOptCreateSolverSettings(&settings), "cuOptCreateSolverSettings")) { return 0; diff --git a/java/cuopt/src/main/no-native/.gitkeep b/java/cuopt/src/main/no-native/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/NativeLibraryLoaderTest.java b/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/NativeLibraryLoaderTest.java new file mode 100644 index 0000000000..780f8d0536 --- /dev/null +++ b/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/NativeLibraryLoaderTest.java @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuopt.mathematicaloptimization; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** + * The resource path has to agree with the layout the packaging step writes, and neither side can + * see the other, so pin it here. + */ +final class NativeLibraryLoaderTest { + @Test + void mapsJvmArchitecturesToThePackagedResourcePath() { + // os.arch reports amd64 on an x86_64 JVM, but the build host reports x86_64; both appear. + assertEquals( + "/amd64/Linux/libcuopt_jni.so", + NativeLibraryLoader.resourcePath("amd64", "libcuopt_jni.so")); + assertEquals( + "/amd64/Linux/libcuopt_jni.so", + NativeLibraryLoader.resourcePath("x86_64", "libcuopt_jni.so")); + assertEquals( + "/aarch64/Linux/libcuopt_jni.so", + NativeLibraryLoader.resourcePath("aarch64", "libcuopt_jni.so")); + assertEquals( + "/aarch64/Linux/libcuopt_jni.so", + NativeLibraryLoader.resourcePath("arm64", "libcuopt_jni.so")); + } + + @Test + void rejectsAnArchitectureCuOptDoesNotPublish() { + IllegalStateException error = + assertThrows( + IllegalStateException.class, + () -> NativeLibraryLoader.resourcePath("ppc64le", "libcuopt_jni.so")); + assertTrue(error.getMessage().contains("ppc64le")); + } + + @Test + void theLibraryThisSuiteRunsAgainstIsLoadable() { + // Reaching any native method proves whichever strategy applied here resolved the library: + // cuopt.native.dir for a source build, the embedded copy for a classifier JAR. + assertEquals(8, NativeCuOpt.getFloatSize()); + } +} diff --git a/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/NativeTestSupport.java b/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/NativeTestSupport.java index 541f413e7f..ab8ecff965 100644 --- a/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/NativeTestSupport.java +++ b/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/NativeTestSupport.java @@ -13,12 +13,33 @@ final class NativeTestSupport { private NativeTestSupport() {} + /** + * Skips when there is no native library to load, by either route the loader accepts: a + * directory named by {@code cuopt.native.dir} for a source build, or a copy embedded in a + * classifier JAR on the classpath. Requiring the property alone would silently skip the whole + * native suite when it runs against a JAR, which is where it matters most. + */ static void assumeNativeLibrary() { + String fileName = System.mapLibraryName("cuopt_jni"); String nativeDir = System.getProperty("cuopt.native.dir"); - Assumptions.assumeTrue(nativeDir != null && !nativeDir.isBlank(), "cuopt.native.dir is unset"); + if (nativeDir != null && !nativeDir.isBlank()) { + Assumptions.assumeTrue( + Files.exists(Path.of(nativeDir, fileName)), "libcuopt_jni is not built"); + return; + } Assumptions.assumeTrue( - Files.exists(Path.of(nativeDir, System.mapLibraryName("cuopt_jni"))), - "libcuopt_jni is not built"); + embeddedLibraryPresent(fileName), + "no libcuopt_jni: cuopt.native.dir is unset and no copy is embedded on the classpath"); + } + + private static boolean embeddedLibraryPresent(String fileName) { + try { + String resource = + NativeLibraryLoader.resourcePath(System.getProperty("os.arch", ""), fileName); + return NativeTestSupport.class.getResource(resource) != null; + } catch (IllegalStateException unsupportedArchitecture) { + return false; + } } private static final long NVIDIA_SMI_TIMEOUT_SECONDS = 30; diff --git a/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/PackagedJarOriginCheck.java b/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/PackagedJarOriginCheck.java new file mode 100644 index 0000000000..40da9005db --- /dev/null +++ b/java/cuopt/src/test/java/com/nvidia/cuopt/mathematicaloptimization/PackagedJarOriginCheck.java @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuopt.mathematicaloptimization; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URL; +import org.junit.jupiter.api.Test; + +/** + * Confirms the suite is exercising a packaged classifier JAR rather than classes built from + * source. + * + *

Excluded by default and run only under {@code -Ppackaged-jar-tests}. Without it a stray + * {@code target/classes} on the classpath would shadow the JAR and the run would pass while + * testing the wrong thing entirely — which is the failure this whole job exists to rule out. + */ +final class PackagedJarOriginCheck { + @Test + void classesComeFromAJarRatherThanADirectory() { + URL location = Problem.class.getProtectionDomain().getCodeSource().getLocation(); + assertNotNull(location, "no code source for Problem"); + String path = location.getPath(); + assertTrue( + path.endsWith(".jar"), + "expected Problem to be loaded from a packaged JAR, but it came from " + path); + } + + @Test + void theNativeLibraryIsEmbeddedInThatJar() { + String resource = + NativeLibraryLoader.resourcePath( + System.getProperty("os.arch", ""), System.mapLibraryName("cuopt_jni")); + URL embedded = PackagedJarOriginCheck.class.getResource(resource); + assertNotNull(embedded, "no " + resource + " on the classpath"); + assertTrue( + "jar".equals(embedded.getProtocol()), + "expected the native library to come from a JAR, but it came from " + embedded); + } + + @Test + void noNativeDirectoryOverrideIsInEffect() { + String nativeDir = System.getProperty("cuopt.native.dir"); + assertTrue( + nativeDir == null || nativeDir.isBlank(), + "cuopt.native.dir is set to '" + nativeDir + "', so the JAR's own library was bypassed"); + } +}