From 79e7e2b62c3bccefceecb15e86a938871399a124 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 23 Jun 2026 21:44:07 +0000 Subject: [PATCH 01/15] Add instructions for running TPC-H benchmarks --- docs/cudf/source/cudf_polars/benchmarks.md | 40 ++++++++++++++++++++++ docs/cudf/source/cudf_polars/index.md | 3 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 docs/cudf/source/cudf_polars/benchmarks.md diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md new file mode 100644 index 000000000000..2356c73c5133 --- /dev/null +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -0,0 +1,40 @@ +# Benchmarks + +## PDS-H (TPC-H variant) + +The steps below reproduce the PDS-H benchmark results using the Polars GPU engine. +No cudf source checkout is required — the benchmark script is included in the +installed `cudf-polars` package. + +### Setup + +Create and activate a fresh virtual environment (conda, mamba, or Python venv), then install the dependencies: + +```bash +CUDA_MAJOR=$(nvidia-smi | grep -oP 'CUDA Version: \K[0-9]+') +pip install tpchgen-cli +pip install --extra-index-url https://pypi.anaconda.org/rapidsai-wheels-nightly/simple "cudf-polars-cu${CUDA_MAJOR}>=0.0.0a0" +``` + +This installs: + +- **`tpchgen-cli`** — a Rust-based TPC-H data generator used to produce the benchmark dataset as Parquet files. +- **`cudf-polars`** — the Polars GPU engine, along with its dependencies including `polars`. Because the GPU engine pins to a tested range of Polars versions, the nightly wheel will install the highest Polars version that the GPU engine currently supports, which may not be the latest Polars release. + +### Generate data + +```bash +tpchgen-cli --output-dir="data/tables/scale-1000.0" --format=parquet -s 1000.0 +``` + +### Run + +```bash +# --frontend options: in-memory (single GPU), polars-cpu (CPU only), dask (multi-GPU) +python -m cudf_polars.streaming.benchmarks.pdsh all \ + --frontend in-memory \ + --path data/tables/scale-1000.0 +``` + +The `--path` value must match the `--output-dir` used during data generation. +Update both consistently when changing scale factors (e.g. `scale-100.0`). diff --git a/docs/cudf/source/cudf_polars/index.md b/docs/cudf/source/cudf_polars/index.md index 9e01902e31b5..969d20ca0de2 100644 --- a/docs/cudf/source/cudf_polars/index.md +++ b/docs/cudf/source/cudf_polars/index.md @@ -92,7 +92,7 @@ On a single GPU, you can run TB-scale workloads with significant speedups compar PDS-DS (SF3K) ``` -For more information on the benchmarks being run, see the PDS-DS queries in the [cuDF GitHub repository](https://github.com/rapidsai/cudf/tree/release/26.06/python/cudf_polars/cudf_polars/streaming/benchmarks). +To reproduce these results, see {doc}`benchmarks`. ## Learn More @@ -109,6 +109,7 @@ options profiling other_engines memory_errors +benchmarks api developer_docs ``` From 41df94c745349705fcb3421edd0ba3f21aa34c5e Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Mon, 29 Jun 2026 13:55:13 +0000 Subject: [PATCH 02/15] add env vars and tuning section --- docs/cudf/source/cudf_polars/benchmarks.md | 64 +++++++++++++++++++--- docs/cudf/source/cudf_polars/index.md | 3 +- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index 2356c73c5133..39ac89f2095f 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -1,14 +1,14 @@ # Benchmarks + + ## PDS-H (TPC-H variant) The steps below reproduce the PDS-H benchmark results using the Polars GPU engine. -No cudf source checkout is required — the benchmark script is included in the -installed `cudf-polars` package. ### Setup -Create and activate a fresh virtual environment (conda, mamba, or Python venv), then install the dependencies: +Create and activate a fresh environment ([conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html), [mamba](https://mamba.readthedocs.io/en/latest/user_guide/mamba.html), [Python venv](https://docs.python.org/3/library/venv.html), or [uv](https://docs.astral.sh/uv/)), then install the dependencies: ```bash CUDA_MAJOR=$(nvidia-smi | grep -oP 'CUDA Version: \K[0-9]+') @@ -18,8 +18,8 @@ pip install --extra-index-url https://pypi.anaconda.org/rapidsai-wheels-nightly/ This installs: -- **`tpchgen-cli`** — a Rust-based TPC-H data generator used to produce the benchmark dataset as Parquet files. -- **`cudf-polars`** — the Polars GPU engine, along with its dependencies including `polars`. Because the GPU engine pins to a tested range of Polars versions, the nightly wheel will install the highest Polars version that the GPU engine currently supports, which may not be the latest Polars release. +- **`tpchgen-cli`**: a Rust-based TPC-H data generator used to produce the benchmark dataset as Parquet files. +- **`cudf-polars`**: the Polars GPU engine, along with its dependencies including `polars`. Because the GPU engine pins to a tested range of Polars versions, the nightly wheel will install the highest Polars version that the GPU engine currently supports, which may not be the latest Polars release. ### Generate data @@ -29,12 +29,62 @@ tpchgen-cli --output-dir="data/tables/scale-1000.0" --format=parquet -s 1000.0 ### Run +Set these environment variables before running to match the configuration used for the published results: + +```bash +export POLARS_MAX_THREADS=1 +export OMP_NUM_THREADS=1 +export LIBCUDF_NUM_HOST_WORKERS=4 +export KVIKIO_NTHREADS=8 +export RAPIDSMPF_num_streaming_threads=8 +``` + +**Single GPU** (`--frontend spmd`, single-process streaming executor, equivalent to `collect(engine="gpu")`): + +```bash +python -m cudf_polars.streaming.benchmarks.pdsh all \ + --frontend spmd \ + --path data/tables/scale-1000.0 +``` + +**Multi GPU** (`--frontend ray`, Ray-managed distributed streaming executor): + +If running inside a Docker container, increase `/dev/shm` by passing `--shm-size=16g` to +`docker run`. All multi-GPU frontends use UCX for intra-node communication, which relies on +POSIX shared memory (`/dev/shm`) for GPU-to-GPU transfers. Docker's default `/dev/shm` is +64MB, which is far too small and will cause failures on any non-trivial workload. + +By default all visible GPUs are used. To select specific devices, set `CUDA_VISIBLE_DEVICES`. +To limit the number of GPUs, use `--num-gpus`: + ```bash -# --frontend options: in-memory (single GPU), polars-cpu (CPU only), dask (multi-GPU) +# All visible GPUs python -m cudf_polars.streaming.benchmarks.pdsh all \ - --frontend in-memory \ + --frontend ray \ + --path data/tables/scale-1000.0 + +# Specific devices +CUDA_VISIBLE_DEVICES=0,1,2,3 python -m cudf_polars.streaming.benchmarks.pdsh all \ + --frontend ray \ + --path data/tables/scale-1000.0 + +# Limit to N GPUs +python -m cudf_polars.streaming.benchmarks.pdsh all \ + --frontend ray \ + --num-gpus 4 \ --path data/tables/scale-1000.0 ``` The `--path` value must match the `--output-dir` used during data generation. Update both consistently when changing scale factors (e.g. `scale-100.0`). + +### Tuning + +The commands above use default settings, which gives a realistic baseline without manual tuning. The most impactful options to adjust are: + +| Option | Description | +|--------|-------------| +| `--target-partition-size` | Target IO chunk size in bytes fed to the GPU. The most impactful lever; tune this first if query performance is below expectations. Default: auto. | +| `--broadcast-limit` | Maximum table size in bytes for broadcast joins instead of shuffle. Increasing this can significantly speed up join-heavy queries. Default: auto. | +| `--spill-device-limit` | GPU memory usage percentage before spilling to host. Lower this if hitting out-of-memory errors. Default: 80%. | +| `--pinned-memory` / `--pinned-max-pool-size` | Enable and size a pinned host memory pool for faster CPU to GPU transfers. | diff --git a/docs/cudf/source/cudf_polars/index.md b/docs/cudf/source/cudf_polars/index.md index 969d20ca0de2..894efeba4a70 100644 --- a/docs/cudf/source/cudf_polars/index.md +++ b/docs/cudf/source/cudf_polars/index.md @@ -92,7 +92,8 @@ On a single GPU, you can run TB-scale workloads with significant speedups compar PDS-DS (SF3K) ``` -To reproduce these results, see {doc}`benchmarks`. + +For more information on the benchmarks being run, see the PDS queries in the [cuDF GitHub repository](https://github.com/rapidsai/cudf/tree/main/python/cudf_polars/cudf_polars/streaming/benchmarks). ## Learn More From 6a84f1a153313fea4b02ffef8b5f02262066655d Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Mon, 29 Jun 2026 14:02:58 +0000 Subject: [PATCH 03/15] clarify example scale factor --- docs/cudf/source/cudf_polars/benchmarks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index 39ac89f2095f..49cfac593fcf 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -23,6 +23,8 @@ This installs: ### Generate data +The following generates SF1000 (scale factor 1000, roughly 1TB of data): + ```bash tpchgen-cli --output-dir="data/tables/scale-1000.0" --format=parquet -s 1000.0 ``` From 1de60ed251a603c5516d5c89713127b4076eb0db Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Mon, 29 Jun 2026 14:32:37 +0000 Subject: [PATCH 04/15] add cpu section and results sections --- docs/cudf/source/cudf_polars/benchmarks.md | 32 ++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index 49cfac593fcf..6c9c04db60ef 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -41,6 +41,14 @@ export KVIKIO_NTHREADS=8 export RAPIDSMPF_num_streaming_threads=8 ``` +**CPU** (`--frontend polars-cpu`, Polars CPU streaming engine): + +```bash +python -m cudf_polars.streaming.benchmarks.pdsh all \ + --frontend polars-cpu \ + --path data/tables/scale-1000.0 +``` + **Single GPU** (`--frontend spmd`, single-process streaming executor, equivalent to `collect(engine="gpu")`): ```bash @@ -80,6 +88,30 @@ python -m cudf_polars.streaming.benchmarks.pdsh all \ The `--path` value must match the `--output-dir` used during data generation. Update both consistently when changing scale factors (e.g. `scale-100.0`). +### Results + +Results are written to `pdsh_results.jsonl` in the current directory by default (override with `-o`). +Each run appends one JSON line containing metadata and a `records` field with per-query, +per-iteration timings: + +```json +{ + "engine_name": "cudf-polars", + "frontend": "spmd", + "dataset_path": "data/tables/scale-1000.0", + "scale_factor": 1000, + "records": { + "1": [ + {"query": 1, "iteration": 0, "duration": 0.79, "status": "success"}, + {"query": 1, "iteration": 1, "duration": 0.55, "status": "success"} + ] + } +} +``` + +`duration` is in seconds. Running multiple frontends with the same `-o` file appends each as a +separate line, making it easy to compare CPU and GPU results in one file. + ### Tuning The commands above use default settings, which gives a realistic baseline without manual tuning. The most impactful options to adjust are: From 74defc6a634f67bb9d6033aea77c48c786ac1a7c Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Mon, 29 Jun 2026 23:37:07 +0000 Subject: [PATCH 05/15] address reviews --- docs/cudf/source/cudf_polars/benchmarks.md | 58 ++++++++++--------- .../cudf_polars/streaming/benchmarks/pdsh.py | 8 ++- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index 6c9c04db60ef..720b7af16055 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -8,45 +8,50 @@ The steps below reproduce the PDS-H benchmark results using the Polars GPU engin ### Setup -Create and activate a fresh environment ([conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html), [mamba](https://mamba.readthedocs.io/en/latest/user_guide/mamba.html), [Python venv](https://docs.python.org/3/library/venv.html), or [uv](https://docs.astral.sh/uv/)), then install the dependencies: +Install `cudf-polars` following the [RAPIDS installation guide](https://docs.rapids.ai/install). +For nightly wheels, install with the `ray` extra (required for multi-GPU benchmarking): ```bash CUDA_MAJOR=$(nvidia-smi | grep -oP 'CUDA Version: \K[0-9]+') -pip install tpchgen-cli -pip install --extra-index-url https://pypi.anaconda.org/rapidsai-wheels-nightly/simple "cudf-polars-cu${CUDA_MAJOR}>=0.0.0a0" +pip install --extra-index-url https://pypi.anaconda.org/rapidsai-wheels-nightly/simple \ + "cudf-polars-cu${CUDA_MAJOR}[ray]>=0.0.0a0" ``` -This installs: - -- **`tpchgen-cli`**: a Rust-based TPC-H data generator used to produce the benchmark dataset as Parquet files. -- **`cudf-polars`**: the Polars GPU engine, along with its dependencies including `polars`. Because the GPU engine pins to a tested range of Polars versions, the nightly wheel will install the highest Polars version that the GPU engine currently supports, which may not be the latest Polars release. - -### Generate data +Because `cudf-polars` pins to a tested range of Polars versions, the nightly wheel will install +the highest Polars version the GPU engine currently supports, which may not be the latest +Polars release. -The following generates SF1000 (scale factor 1000, roughly 1TB of data): + +Then install `tpchgen-cli`, a Rust-based TPC-H data generator used to produce the benchmark +dataset as Parquet files: ```bash -tpchgen-cli --output-dir="data/tables/scale-1000.0" --format=parquet -s 1000.0 +pip install tpchgen-cli ``` -### Run +### Generate data -Set these environment variables before running to match the configuration used for the published results: +Set the scale factor once and reuse it across all steps. The following generates SF1000 +(scale factor 1000, roughly 1TB of data): ```bash -export POLARS_MAX_THREADS=1 -export OMP_NUM_THREADS=1 -export LIBCUDF_NUM_HOST_WORKERS=4 -export KVIKIO_NTHREADS=8 -export RAPIDSMPF_num_streaming_threads=8 +export SCALE_FACTOR=1000.0 +export DATA_PATH="data/tables/scale-${SCALE_FACTOR}" + +tpchgen-cli --output-dir="${DATA_PATH}" --format=parquet -s ${SCALE_FACTOR} ``` +### Run + **CPU** (`--frontend polars-cpu`, Polars CPU streaming engine): ```bash python -m cudf_polars.streaming.benchmarks.pdsh all \ --frontend polars-cpu \ - --path data/tables/scale-1000.0 + --path "${DATA_PATH}" ``` **Single GPU** (`--frontend spmd`, single-process streaming executor, equivalent to `collect(engine="gpu")`): @@ -54,7 +59,7 @@ python -m cudf_polars.streaming.benchmarks.pdsh all \ ```bash python -m cudf_polars.streaming.benchmarks.pdsh all \ --frontend spmd \ - --path data/tables/scale-1000.0 + --path "${DATA_PATH}" ``` **Multi GPU** (`--frontend ray`, Ray-managed distributed streaming executor): @@ -71,23 +76,20 @@ To limit the number of GPUs, use `--num-gpus`: # All visible GPUs python -m cudf_polars.streaming.benchmarks.pdsh all \ --frontend ray \ - --path data/tables/scale-1000.0 + --path "${DATA_PATH}" # Specific devices CUDA_VISIBLE_DEVICES=0,1,2,3 python -m cudf_polars.streaming.benchmarks.pdsh all \ --frontend ray \ - --path data/tables/scale-1000.0 + --path "${DATA_PATH}" # Limit to N GPUs python -m cudf_polars.streaming.benchmarks.pdsh all \ --frontend ray \ --num-gpus 4 \ - --path data/tables/scale-1000.0 + --path "${DATA_PATH}" ``` -The `--path` value must match the `--output-dir` used during data generation. -Update both consistently when changing scale factors (e.g. `scale-100.0`). - ### Results Results are written to `pdsh_results.jsonl` in the current directory by default (override with `-o`). @@ -118,7 +120,7 @@ The commands above use default settings, which gives a realistic baseline withou | Option | Description | |--------|-------------| -| `--target-partition-size` | Target IO chunk size in bytes fed to the GPU. The most impactful lever; tune this first if query performance is below expectations. Default: auto. | -| `--broadcast-limit` | Maximum table size in bytes for broadcast joins instead of shuffle. Increasing this can significantly speed up join-heavy queries. Default: auto. | +| `--target-partition-size` | Target IO chunk size in bytes fed to the GPU. The most impactful lever; tune this first if query performance is below expectations. Default: `min(2.5% of smallest GPU memory, 1.5GB)`. | +| `--broadcast-limit` | Maximum table size in bytes for broadcast joins instead of shuffle. Increasing this can significantly speed up join-heavy queries. Default: `min(15% of smallest GPU memory, 16GB)`. | | `--spill-device-limit` | GPU memory usage percentage before spilling to host. Lower this if hitting out-of-memory errors. Default: 80%. | | `--pinned-memory` / `--pinned-max-pool-size` | Enable and size a pinned host memory pool for faster CPU to GPU transfers. | diff --git a/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py b/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py index 79260dc096a8..50cfa4f05eec 100644 --- a/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py +++ b/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """ @@ -42,6 +42,9 @@ # on each worker takes ~15 sec extra os.environ["KVIKIO_COMPAT_MODE"] = os.environ.get("KVIKIO_COMPAT_MODE", "on") os.environ["KVIKIO_NTHREADS"] = os.environ.get("KVIKIO_NTHREADS", "8") +os.environ["RAPIDSMPF_NUM_STREAMING_THREADS"] = os.environ.get( + "RAPIDSMPF_NUM_STREAMING_THREADS", "8" +) # The pre-computed expected results come from DuckDB, which has # different casting rules than Polars. For example, in polars @@ -1795,4 +1798,7 @@ def q22(run_config: RunConfig) -> str: if __name__ == "__main__": parser = build_parser(num_queries=22) args = parse_args(parser=parser) + if args.frontend != "polars-cpu": + os.environ["POLARS_MAX_THREADS"] = os.environ.get("POLARS_MAX_THREADS", "1") + os.environ["OMP_NUM_THREADS"] = os.environ.get("OMP_NUM_THREADS", "1") run_polars(PDSHQueries, args) From 05ea9ed314ba85427390b4ba41b9e32ac14711d0 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Mon, 29 Jun 2026 23:49:03 +0000 Subject: [PATCH 06/15] specify pinned memory defaults --- docs/cudf/source/cudf_polars/benchmarks.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index 720b7af16055..ad15478b6583 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -122,5 +122,5 @@ The commands above use default settings, which gives a realistic baseline withou |--------|-------------| | `--target-partition-size` | Target IO chunk size in bytes fed to the GPU. The most impactful lever; tune this first if query performance is below expectations. Default: `min(2.5% of smallest GPU memory, 1.5GB)`. | | `--broadcast-limit` | Maximum table size in bytes for broadcast joins instead of shuffle. Increasing this can significantly speed up join-heavy queries. Default: `min(15% of smallest GPU memory, 16GB)`. | -| `--spill-device-limit` | GPU memory usage percentage before spilling to host. Lower this if hitting out-of-memory errors. Default: 80%. | -| `--pinned-memory` / `--pinned-max-pool-size` | Enable and size a pinned host memory pool for faster CPU to GPU transfers. | +| `--spill-device-limit` | GPU memory usage percentage before spilling to host. Lower this if hitting out-of-memory errors. Default: `80%`. | +| `--pinned-memory` / `--pinned-initial-pool-size` | Enable a pinned host memory pool for faster CPU-to-GPU transfers. Off by default. When enabled, the pool starts empty and grows up to 80% of host memory per GPU; set `--pinned-initial-pool-size` (bytes) to pre-allocate capacity upfront. | From be44f90e17715639b8b1f08b2f3737a04f1642c1 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 30 Jun 2026 00:04:26 +0000 Subject: [PATCH 07/15] add a TODO for num_streaming_threads=8 default outside the benchmark runner --- python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py b/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py index 50cfa4f05eec..b3b226d76d2d 100644 --- a/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py +++ b/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py @@ -42,6 +42,7 @@ # on each worker takes ~15 sec extra os.environ["KVIKIO_COMPAT_MODE"] = os.environ.get("KVIKIO_COMPAT_MODE", "on") os.environ["KVIKIO_NTHREADS"] = os.environ.get("KVIKIO_NTHREADS", "8") +# TODO: consider raising the rapidsmpf built-in default from 1 to 8. os.environ["RAPIDSMPF_NUM_STREAMING_THREADS"] = os.environ.get( "RAPIDSMPF_NUM_STREAMING_THREADS", "8" ) From e28cc86728a52eb0dfe82cc64312cdab777234bb Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 30 Jun 2026 00:11:50 +0000 Subject: [PATCH 08/15] dont set cpu threds=1 for cpu engines, include env in tracked list --- .../cudf_polars/streaming/benchmarks/pdsds.py | 10 +++++++++- .../cudf_polars/streaming/benchmarks/pdsh.py | 3 ++- .../cudf_polars/streaming/benchmarks/utils.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsds.py b/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsds.py index bf43487bf8dc..19ac159b76e0 100644 --- a/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsds.py +++ b/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsds.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """ @@ -22,6 +22,7 @@ try: from cudf_polars.streaming.benchmarks.utils import ( COUNT_DTYPE, + _CPU_ENGINES, build_parser, parse_args, run_polars, @@ -37,6 +38,10 @@ # on each worker takes ~15 sec extra os.environ["KVIKIO_COMPAT_MODE"] = os.environ.get("KVIKIO_COMPAT_MODE", "on") os.environ["KVIKIO_NTHREADS"] = os.environ.get("KVIKIO_NTHREADS", "8") +# TODO: consider raising the rapidsmpf built-in default from 1 to 8. +os.environ["RAPIDSMPF_NUM_STREAMING_THREADS"] = os.environ.get( + "RAPIDSMPF_NUM_STREAMING_THREADS", "8" +) def valid_query(name: str) -> bool: @@ -328,4 +333,7 @@ class PDSDSDuckDBQueries(PDSDSQueries): if __name__ == "__main__": parser = build_parser(num_queries=99) args = parse_args(parser=parser) + if args.frontend not in _CPU_ENGINES: + os.environ["POLARS_MAX_THREADS"] = os.environ.get("POLARS_MAX_THREADS", "1") + os.environ["OMP_NUM_THREADS"] = os.environ.get("OMP_NUM_THREADS", "1") run_polars(PDSDSPolarsQueries, args) diff --git a/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py b/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py index b3b226d76d2d..f51d05d6c23f 100644 --- a/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py +++ b/python/cudf_polars/cudf_polars/streaming/benchmarks/pdsh.py @@ -22,6 +22,7 @@ try: from cudf_polars.streaming.benchmarks.utils import ( COUNT_DTYPE, + _CPU_ENGINES, QueryResult, RunConfig, build_parser, @@ -1799,7 +1800,7 @@ def q22(run_config: RunConfig) -> str: if __name__ == "__main__": parser = build_parser(num_queries=22) args = parse_args(parser=parser) - if args.frontend != "polars-cpu": + if args.frontend not in _CPU_ENGINES: os.environ["POLARS_MAX_THREADS"] = os.environ.get("POLARS_MAX_THREADS", "1") os.environ["OMP_NUM_THREADS"] = os.environ.get("OMP_NUM_THREADS", "1") run_polars(PDSHQueries, args) diff --git a/python/cudf_polars/cudf_polars/streaming/benchmarks/utils.py b/python/cudf_polars/cudf_polars/streaming/benchmarks/utils.py index 42e1722bbc89..d68a0fb08746 100644 --- a/python/cudf_polars/cudf_polars/streaming/benchmarks/utils.py +++ b/python/cudf_polars/cudf_polars/streaming/benchmarks/utils.py @@ -2045,7 +2045,7 @@ def build_parser(num_queries: int = 22) -> argparse.ArgumentParser: parser.add_argument( "--capture-env-vars", type=str, - default="CUDF_POLARS_LOG_TRACES_MEMORY,CUDF_POLARS_LOG_TRACES,DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT,DASK_DISTRIBUTED__COMM__UCX__CONNECT_TIMEOUT,KVIKIO_NTHREADS,LIBCUDF_NUM_HOST_WORKERS,OMP_NUM_THREADS,POLARS_MAX_THREADS,RAPIDSMPF_num_streaming_threads,UCX_MAX_RNDV_RAILS,UCX_PROTO_ENABLE,UCX_RNDV_FRAG_MEM_TYPES,UCX_RNDV_MTYPE_WORKER_FC_ENABLE,UCX_RNDV_MTYPE_WORKER_MAX_MEM,UCX_RNDV_PIPELINE_ERROR_HANDLING", + default="CUDF_POLARS_LOG_TRACES_MEMORY,CUDF_POLARS_LOG_TRACES,DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT,DASK_DISTRIBUTED__COMM__UCX__CONNECT_TIMEOUT,KVIKIO_NTHREADS,LIBCUDF_NUM_HOST_WORKERS,OMP_NUM_THREADS,POLARS_MAX_THREADS,RAPIDSMPF_NUM_STREAMING_THREADS,UCX_MAX_RNDV_RAILS,UCX_PROTO_ENABLE,UCX_RNDV_FRAG_MEM_TYPES,UCX_RNDV_MTYPE_WORKER_FC_ENABLE,UCX_RNDV_MTYPE_WORKER_MAX_MEM,UCX_RNDV_PIPELINE_ERROR_HANDLING", help="Comma-separated list of environment variables to capture. Written to ``extra_info.environment``.", ) From 9d31e22d2c27d339431301c54ae6009eb279fb87 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 1 Jul 2026 17:48:32 +0000 Subject: [PATCH 09/15] add cpu-only instructions --- docs/cudf/source/cudf_polars/benchmarks.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index ad15478b6583..49c4700d7b5e 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -8,7 +8,7 @@ The steps below reproduce the PDS-H benchmark results using the Polars GPU engin ### Setup -Install `cudf-polars` following the [RAPIDS installation guide](https://docs.rapids.ai/install). +**GPU machines:** Install `cudf-polars` following the [RAPIDS installation guide](https://docs.rapids.ai/install). For nightly wheels, install with the `ray` extra (required for multi-GPU benchmarking): ```bash @@ -25,6 +25,16 @@ Polars release. (and possibly structlog) so benchmark dependencies can be installed in one step: pip install "cudf-polars-cu${CUDA_MAJOR}[ray,benchmark]>=0.0.0a0" Requires changes to pyproject.toml and dependencies.yaml. --> + +**CPU-only machines (no CUDA):** The `--frontend polars-cpu` benchmark runs on any machine. +Since `cudf-polars` GPU wheels are not available for non-CUDA platforms, install from source: + +```bash +git clone --depth=1 https://github.com/rapidsai/cudf.git +pip install --no-deps ./cudf/python/cudf_polars +pip install polars nvtx +``` + Then install `tpchgen-cli`, a Rust-based TPC-H data generator used to produce the benchmark dataset as Parquet files: From 90c2f7988796d2336c95d42199f9f8919ef1a2d4 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 1 Jul 2026 17:54:36 +0000 Subject: [PATCH 10/15] update framing --- docs/cudf/source/cudf_polars/benchmarks.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index 49c4700d7b5e..dd10cad5d0de 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -8,8 +8,9 @@ The steps below reproduce the PDS-H benchmark results using the Polars GPU engin ### Setup -**GPU machines:** Install `cudf-polars` following the [RAPIDS installation guide](https://docs.rapids.ai/install). -For nightly wheels, install with the `ray` extra (required for multi-GPU benchmarking): +**GPU machines** can run both CPU and GPU benchmarks. Install `cudf-polars` following the +[RAPIDS installation guide](https://docs.rapids.ai/install). For nightly wheels, install with +the `ray` extra (required for multi-GPU benchmarking): ```bash CUDA_MAJOR=$(nvidia-smi | grep -oP 'CUDA Version: \K[0-9]+') @@ -26,8 +27,8 @@ Polars release. pip install "cudf-polars-cu${CUDA_MAJOR}[ray,benchmark]>=0.0.0a0" Requires changes to pyproject.toml and dependencies.yaml. --> -**CPU-only machines (no CUDA):** The `--frontend polars-cpu` benchmark runs on any machine. -Since `cudf-polars` GPU wheels are not available for non-CUDA platforms, install from source: +**CPU-only machines** (no CUDA) can only run the `--frontend polars-cpu` benchmark. Since the +`cudf-polars` GPU wheels require CUDA, install from source instead: ```bash git clone --depth=1 https://github.com/rapidsai/cudf.git From 79bcb628c8e107a1977d7e5a500761ad353eaedd Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 2 Jul 2026 14:04:25 +0000 Subject: [PATCH 11/15] Add instructions on running PDS-H for pandas --- docs/cudf/source/cudf_pandas/benchmarks.md | 82 ++++++++++++++++++++++ docs/cudf/source/cudf_polars/benchmarks.md | 11 +-- 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/docs/cudf/source/cudf_pandas/benchmarks.md b/docs/cudf/source/cudf_pandas/benchmarks.md index fb162d7ebee2..18d414ccf065 100644 --- a/docs/cudf/source/cudf_pandas/benchmarks.md +++ b/docs/cudf/source/cudf_pandas/benchmarks.md @@ -109,3 +109,85 @@ index f39beb0..a9ad651 100755 ./_launcher/solution.R --solution=pandas --task=join --nrow=1e7 ./_launcher/solution.R --solution=pandas --task=join --nrow=1e8 ``` + +## PDS-H (TPC-H variant) + +The steps below reproduce the PDS-H benchmark results using cudf.pandas. + +**Note on data types:** `tpchgen-cli` generates Decimal columns (e.g. `l_extendedprice`, +`l_discount`) and Python `datetime.date` columns. The benchmark reads these as float64 and +timestamp respectively, which is faster and avoids Python object overhead. This means results +may differ slightly from a strictly spec-compliant TPC-H run that preserves Decimal +precision. + +### Setup + +Install `cudf` following the +[RAPIDS installation guide](https://docs.rapids.ai/install). For nightly wheels: + +```bash +CUDA_MAJOR=$(nvidia-smi | grep -oP 'CUDA Version: \K[0-9]+') +pip install --extra-index-url https://pypi.anaconda.org/rapidsai-wheels-nightly/simple \ + "cudf-cu${CUDA_MAJOR}>=0.0.0a0" +``` + +Then install `tpchgen-cli`, a Rust-based TPC-H data generator used to produce the benchmark +dataset as Parquet files: + +```bash +pip install tpchgen-cli +``` + +### Generate data + +Set the scale factor once and reuse it across all steps. The following generates SF50 +(scale factor 50, roughly 50GB of data): + +```bash +export SCALE_FACTOR=50.0 +export DATA_PATH="data/tables/scale-${SCALE_FACTOR}" + +tpchgen-cli --output-dir="${DATA_PATH}" --format=parquet -s ${SCALE_FACTOR} +``` + +### Run + +**CPU** (`--executor cpu`, pandas): + +```bash +python -m cudf.pandas._benchmarks.pdsh all \ + --executor cpu \ + --path "${DATA_PATH}" +``` + +**GPU** (`--executor in-memory`, cudf.pandas): + +```bash +python -m cudf.pandas._benchmarks.pdsh all \ + --executor in-memory \ + --path "${DATA_PATH}" +``` + +### Results + +Results are written to `pdsh_results.jsonl` in the current directory by default (override with `-o`). +Each run appends one JSON line containing metadata and a `records` field with per-query, +per-iteration timings: + +```json +{ + "query_set": "pdsh", + "executor": "in-memory", + "dataset_path": "data/tables/scale-50.0", + "scale_factor": 50, + "records": { + "1": [ + {"query": 1, "iteration": 0, "duration": 0.79, "status": "success"}, + {"query": 1, "iteration": 1, "duration": 0.55, "status": "success"} + ] + } +} +``` + +`duration` is in seconds. Running multiple executors with the same `-o` file appends each as a +separate line, making it easy to compare CPU and GPU results in one file. diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index dd10cad5d0de..8621d4f32def 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -8,7 +8,7 @@ The steps below reproduce the PDS-H benchmark results using the Polars GPU engin ### Setup -**GPU machines** can run both CPU and GPU benchmarks. Install `cudf-polars` following the +Install `cudf-polars` following the [RAPIDS installation guide](https://docs.rapids.ai/install). For nightly wheels, install with the `ray` extra (required for multi-GPU benchmarking): @@ -27,15 +27,6 @@ Polars release. pip install "cudf-polars-cu${CUDA_MAJOR}[ray,benchmark]>=0.0.0a0" Requires changes to pyproject.toml and dependencies.yaml. --> -**CPU-only machines** (no CUDA) can only run the `--frontend polars-cpu` benchmark. Since the -`cudf-polars` GPU wheels require CUDA, install from source instead: - -```bash -git clone --depth=1 https://github.com/rapidsai/cudf.git -pip install --no-deps ./cudf/python/cudf_polars -pip install polars nvtx -``` - Then install `tpchgen-cli`, a Rust-based TPC-H data generator used to produce the benchmark dataset as Parquet files: From 6d749cc03c9c5741bbba711767826ab0fa1a3c76 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 2 Jul 2026 15:12:41 +0000 Subject: [PATCH 12/15] pandas tpch datagen --- docs/cudf/source/cudf_pandas/benchmarks.md | 30 +++++++++++++++++++- docs/cudf/source/cudf_polars/benchmarks.md | 2 +- python/cudf/cudf/pandas/_benchmarks/utils.py | 14 +++++++-- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/cudf/source/cudf_pandas/benchmarks.md b/docs/cudf/source/cudf_pandas/benchmarks.md index 18d414ccf065..68ef5767bcea 100644 --- a/docs/cudf/source/cudf_pandas/benchmarks.md +++ b/docs/cudf/source/cudf_pandas/benchmarks.md @@ -147,7 +147,35 @@ Set the scale factor once and reuse it across all steps. The following generates export SCALE_FACTOR=50.0 export DATA_PATH="data/tables/scale-${SCALE_FACTOR}" -tpchgen-cli --output-dir="${DATA_PATH}" --format=parquet -s ${SCALE_FACTOR} +tpchgen-cli parquet -o "${DATA_PATH}" -s ${SCALE_FACTOR} +``` + +`tpchgen-cli` generates Decimal and `datetime.date` columns. pandas cannot use these types +in arithmetic, so convert them to float64 and timestamp before running the benchmark: + +```python +from pathlib import Path +import pyarrow as pa +import pyarrow.parquet as pq +import os + +data_path = Path(os.environ["DATA_PATH"]) +tables = ["lineitem", "orders", "customer", "supplier", "part", "partsupp", "nation", "region"] + +def cast_schema(schema): + return pa.schema( + f.with_type(pa.float64()) if pa.types.is_decimal(f.type) + else f.with_type(pa.timestamp("ms")) if pa.types.is_date(f.type) + else f + for f in schema + ) + +for table in tables: + table_path = data_path / f"{table}.parquet" + parts = [table_path] if table_path.is_file() else sorted(table_path.glob("*.parquet")) + for part in parts: + tbl = pq.read_table(part, schema=cast_schema(pq.read_schema(part))) + pq.write_table(tbl, part) ``` ### Run diff --git a/docs/cudf/source/cudf_polars/benchmarks.md b/docs/cudf/source/cudf_polars/benchmarks.md index 8621d4f32def..2a267250fbd9 100644 --- a/docs/cudf/source/cudf_polars/benchmarks.md +++ b/docs/cudf/source/cudf_polars/benchmarks.md @@ -43,7 +43,7 @@ Set the scale factor once and reuse it across all steps. The following generates export SCALE_FACTOR=1000.0 export DATA_PATH="data/tables/scale-${SCALE_FACTOR}" -tpchgen-cli --output-dir="${DATA_PATH}" --format=parquet -s ${SCALE_FACTOR} +tpchgen-cli parquet -o "${DATA_PATH}" -s ${SCALE_FACTOR} ``` ### Run diff --git a/python/cudf/cudf/pandas/_benchmarks/utils.py b/python/cudf/cudf/pandas/_benchmarks/utils.py index 0058e6d95aec..ec1b1d133f4e 100644 --- a/python/cudf/cudf/pandas/_benchmarks/utils.py +++ b/python/cudf/cudf/pandas/_benchmarks/utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Utility functions/classes for running the PDS-H and PDS-DS benchmarks.""" @@ -406,8 +406,16 @@ def get_data( suffix: str = "", columns: list[str] | None = None, ) -> pd.DataFrame: - """Get table from dataset.""" - return pd.read_parquet(f"{path}/{table_name}{suffix}", columns=columns) + """Get table from dataset. + + The parquet files must have Decimal columns pre-converted to float64 and + date columns pre-converted to timestamp. tpchgen-cli generates Decimal and + date types that pandas cannot use in arithmetic. See schema-overhead.md for + a conversion script and benchmarks showing the performance impact. + """ + return pd.read_parquet( + Path(path) / f"{table_name}{suffix}", columns=columns + ) def execute_query( From 1984137a343103313009d0e0cac5e22d0172eb9e Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 2 Jul 2026 15:18:55 +0000 Subject: [PATCH 13/15] outdated instructions --- docs/cudf/source/cudf_pandas/benchmarks.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/cudf/source/cudf_pandas/benchmarks.md b/docs/cudf/source/cudf_pandas/benchmarks.md index 68ef5767bcea..f3d140b6788e 100644 --- a/docs/cudf/source/cudf_pandas/benchmarks.md +++ b/docs/cudf/source/cudf_pandas/benchmarks.md @@ -114,12 +114,6 @@ index f39beb0..a9ad651 100755 The steps below reproduce the PDS-H benchmark results using cudf.pandas. -**Note on data types:** `tpchgen-cli` generates Decimal columns (e.g. `l_extendedprice`, -`l_discount`) and Python `datetime.date` columns. The benchmark reads these as float64 and -timestamp respectively, which is faster and avoids Python object overhead. This means results -may differ slightly from a strictly spec-compliant TPC-H run that preserves Decimal -precision. - ### Setup Install `cudf` following the From 6fb28c7482e062dc6546077a3bd57818df7f5950 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 2 Jul 2026 15:21:43 +0000 Subject: [PATCH 14/15] revert --- python/cudf/cudf/pandas/_benchmarks/utils.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/python/cudf/cudf/pandas/_benchmarks/utils.py b/python/cudf/cudf/pandas/_benchmarks/utils.py index ec1b1d133f4e..0058e6d95aec 100644 --- a/python/cudf/cudf/pandas/_benchmarks/utils.py +++ b/python/cudf/cudf/pandas/_benchmarks/utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 """Utility functions/classes for running the PDS-H and PDS-DS benchmarks.""" @@ -406,16 +406,8 @@ def get_data( suffix: str = "", columns: list[str] | None = None, ) -> pd.DataFrame: - """Get table from dataset. - - The parquet files must have Decimal columns pre-converted to float64 and - date columns pre-converted to timestamp. tpchgen-cli generates Decimal and - date types that pandas cannot use in arithmetic. See schema-overhead.md for - a conversion script and benchmarks showing the performance impact. - """ - return pd.read_parquet( - Path(path) / f"{table_name}{suffix}", columns=columns - ) + """Get table from dataset.""" + return pd.read_parquet(f"{path}/{table_name}{suffix}", columns=columns) def execute_query( From 3a2f4fdfd131312a6d8053d3677b0c48b33936f4 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 2 Jul 2026 18:09:34 +0000 Subject: [PATCH 15/15] add a note about future of date support --- docs/cudf/source/cudf_pandas/benchmarks.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/cudf/source/cudf_pandas/benchmarks.md b/docs/cudf/source/cudf_pandas/benchmarks.md index f3d140b6788e..983c17ade8b9 100644 --- a/docs/cudf/source/cudf_pandas/benchmarks.md +++ b/docs/cudf/source/cudf_pandas/benchmarks.md @@ -145,7 +145,8 @@ tpchgen-cli parquet -o "${DATA_PATH}" -s ${SCALE_FACTOR} ``` `tpchgen-cli` generates Decimal and `datetime.date` columns. pandas cannot use these types -in arithmetic, so convert them to float64 and timestamp before running the benchmark: +in arithmetic, so convert them to float64 and timestamp before running the benchmark. This +conversion step may not be needed in the future (see [#21204](https://github.com/rapidsai/cudf/issues/21204)). ```python from pathlib import Path