diff --git a/CMakeLists.txt b/CMakeLists.txt index 58bf8eb4..14792714 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -404,6 +404,9 @@ if(NOT CUCASCADE_TOPOLOGY_ONLY AND CUCASCADE_BUILD_CUDF) cudf::cudf) target_include_directories(cucascade_cudf_objects PUBLIC ${CUCASCADE_PUBLIC_INCLUDE_DIRS}) + # Internal headers of the core library that the converters build on. + target_include_directories(cucascade_cudf_objects + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/memory) target_compile_definitions(cucascade_cudf_objects PUBLIC ${CUCASCADE_PUBLIC_COMPILE_DEFS}) target_compile_features(cucascade_cudf_objects PUBLIC cxx_std_20) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 075c4cbd..4c57eba6 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -59,6 +59,7 @@ add_executable(cucascade_benchmarks ${BENCHMARK_SOURCES}) target_include_directories( cucascade_benchmarks PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../test + ${PROJECT_SOURCE_DIR}/src/cudf ${PROJECT_SOURCE_DIR}/src/memory ${benchmark_SOURCE_DIR}/include) # Apply warning flags to benchmark executable. Suppress -Wnull-dereference: diff --git a/benchmark/benchmark_gpu_to_gpu_converter.cpp b/benchmark/benchmark_gpu_to_gpu_converter.cpp index acb8105c..3125205d 100644 --- a/benchmark/benchmark_gpu_to_gpu_converter.cpp +++ b/benchmark/benchmark_gpu_to_gpu_converter.cpp @@ -29,7 +29,13 @@ // The trailing wait on each result's writer event keeps the measurement valid if the converter // stops blocking the host before the copy completes. BM_RawPeerCopy copies matching total bytes // in one contiguous buffer per input, with a single host sync: a lower bound for the converter. +// +// BM_ConvertGpuToGpu needs peer access between the two devices. The Staged benchmarks force the +// route through pinned host memory instead, which is what device pairs without peer access take, +// so they run on any two GPUs. BM_ConvertGpuToGpuStagedPairs runs one such consumer per directed +// device pair at the same time and reports their combined throughput. +#include "builtin_converters_detail.hpp" #include "cucascade/cudf/builtin_converters.hpp" #include "cucascade/cudf/gpu_data_representation.hpp" #include "cucascade/data/representation_converter.hpp" @@ -41,6 +47,7 @@ #include #include +#include #include #include @@ -56,8 +63,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -93,12 +102,25 @@ constexpr std::size_t kInputBytes = constexpr std::size_t kPeakBytesPerGpu = (kNumInputs + 1) * kInputBytes; constexpr std::size_t kGpuCapacityPerDevice = kPeakBytesPerGpu + kPeakBytesPerGpu / 4; -// Host staging is unused (benchmarks skip without peer access); the host space still pins -// 4 x 128 x 1 MiB = 512 MiB at startup, so cap just above that. +// The host space pins 4 x 128 x 1 MiB = 512 MiB at startup, so cap just above that. A staged copy +// borrows host blocks for one column buffer at a time, a few MiB per consumer at kMaxRows. constexpr std::size_t kHostCapacityPerRegion = 1 * kGiB; static std::unique_ptr g_manager; static std::unique_ptr g_registry; +// Same converters, with every GPU -> GPU conversion forced through pinned host memory. +static std::unique_ptr g_staged_registry; + +void make_registries() +{ + g_registry = std::make_unique(); + register_builtin_converters(*g_registry); + + g_staged_registry = std::make_unique(); + cucascade::detail::register_builtin_converters( + *g_staged_registry, + cucascade::detail::gpu_to_gpu_converter_options{.force_staged_route = true}); +} /** * @brief Build a two-GPU manager with NUMA-local host spaces and peer-granted GPU pools. @@ -116,12 +138,36 @@ void DoSetup([[maybe_unused]] const benchmark::State& state) .set_per_numa_region_capacity(kHostCapacityPerRegion); g_manager = std::make_unique(builder.build()); - g_registry = std::make_unique(); - register_builtin_converters(*g_registry); + make_registries(); +} + +/** + * @brief Build a manager over every visible GPU, for the benchmark that runs several device pairs. + * + * Each GPU holds the inputs of one pair and the outputs of another, so it gets twice the capacity + * of the two-GPU setup. + */ +void DoSetupAllGpus([[maybe_unused]] const benchmark::State& state) +{ + int const devices = test::cuda_device_count(); + if (devices < 2) { return; } + + std::vector gpu_ids(static_cast(devices)); + std::iota(gpu_ids.begin(), gpu_ids.end(), 0); + + reservation_manager_configurator builder; + builder.set_gpu_ids(gpu_ids) + .set_gpu_usage_limit(2 * kGpuCapacityPerDevice) + .use_numa_id_as_host_id() + .set_per_numa_region_capacity(kHostCapacityPerRegion); + g_manager = std::make_unique(builder.build()); + + make_registries(); } void DoTeardown([[maybe_unused]] const benchmark::State& state) { + g_staged_registry.reset(); g_registry.reset(); g_manager.reset(); } @@ -170,14 +216,42 @@ bool skip_without_peers(benchmark::State& state) } /** - * @brief Build @p count source representations on the source GPU, each carrying a writer event + * @brief Skip @p state unless two devices exist. + * @return True if the benchmark was skipped. + */ +bool skip_without_two_devices(benchmark::State& state) +{ + if (test::cuda_device_count() < 2) { + state.SkipWithError("requires two CUDA devices"); + return true; + } + return false; +} + +/** + * @brief Peak MiB the HOST spaces have handed out since the manager was built. + * + * Staged copies borrow their host blocks there and nothing else in these benchmarks does. + */ +double peak_host_mib() +{ + std::size_t bytes = 0; + for (memory_space const* space : g_manager->get_memory_spaces_for_tier(Tier::HOST)) { + bytes += space->get_memory_resource_of()->get_peak_total_allocated_bytes(); + } + return static_cast(bytes) / static_cast(kMiB); +} + +/** + * @brief Build @p count source representations on @p src_device, each carrying a writer event * recorded on @p producer, as a Sirius partition task would leave them. */ std::vector> make_inputs(std::size_t count, cudf::size_type rows, - ::cuda::stream_ref producer) + ::cuda::stream_ref producer, + int src_device = kSrcDeviceId) { - memory_space* src_space = g_manager->get_memory_space(Tier::GPU, kSrcDeviceId); + memory_space* src_space = g_manager->get_memory_space(Tier::GPU, src_device); std::vector> inputs; inputs.reserve(count); @@ -204,13 +278,13 @@ void set_counters(benchmark::State& state, } /** - * @brief Sequentially convert N source-GPU partitions to the destination GPU on one thread. + * @brief Sequentially convert N source-GPU partitions to the destination GPU on one thread, with + * the converters of @p registry. * @param state.range(0) Rows per input (fixed schema and partition count) */ -void BM_ConvertGpuToGpu(benchmark::State& state) +void run_sequential_conversions(benchmark::State& state, + representation_converter_registry& registry) { - if (skip_without_peers(state)) { return; } - cudf::size_type const rows = static_cast(state.range(0)); rmm::cuda_stream producer = [] { @@ -231,7 +305,7 @@ void BM_ConvertGpuToGpu(benchmark::State& state) // Warm-up: first touch of the destination pool and the peer-route probe. { - auto warm = g_registry->convert(*inputs[0], dst_space, consumer); + auto warm = registry.convert(*inputs[0], dst_space, consumer); warm->wait_for_writer(consumer); consumer.synchronize(); } @@ -240,7 +314,7 @@ void BM_ConvertGpuToGpu(benchmark::State& state) outputs.reserve(kNumInputs); for (auto _ : state) { for (auto const& input : inputs) { - outputs.push_back(g_registry->convert(*input, dst_space, consumer)); + outputs.push_back(registry.convert(*input, dst_space, consumer)); } // Consumer may read every output only after its writer event. @@ -261,6 +335,161 @@ void BM_ConvertGpuToGpu(benchmark::State& state) state.counters["rows"] = rows; } +/** + * @brief The consumer fan-in over the peer route. + * @param state.range(0) Rows per input + */ +void BM_ConvertGpuToGpu(benchmark::State& state) +{ + if (skip_without_peers(state)) { return; } + + run_sequential_conversions(state, *g_registry); +} + +/** + * @brief The same consumer fan-in with every conversion staged through pinned host memory. + * @param state.range(0) Rows per input + */ +void BM_ConvertGpuToGpuStaged(benchmark::State& state) +{ + if (skip_without_two_devices(state)) { return; } + + run_sequential_conversions(state, *g_staged_registry); + state.counters["staging_peak_MiB"] = peak_host_mib(); +} + +/// One directed device pair of BM_ConvertGpuToGpuStagedPairs, with its own streams and tables. +struct staged_pair { + int src_device; + int dst_device; + rmm::cuda_stream producer; ///< On src_device; wrote the inputs + rmm::cuda_stream consumer; ///< On dst_device; reads the outputs + std::vector> inputs; + std::vector> outputs; +}; + +/** + * @brief Directed pairs for @p devices GPUs, each device paired with the one half the count away. + * + * Both directions of a pair are adjacent, e.g. for 4 devices: 0>2, 2>0, 1>3, 3>1. On a machine + * with two GPUs behind each of two PCIe switches these are the pairs that cross switches. + */ +std::vector> directed_pairs(int devices) +{ + std::vector> pairs; + int const half = devices / 2; + for (int low = 0; low < half; ++low) { + pairs.emplace_back(low, low + half); + pairs.emplace_back(low + half, low); + } + return pairs; +} + +/** + * @brief Run the consumer fan-in on several directed device pairs at once, one thread per pair, + * with every conversion staged through pinned host memory. + * @param state.range(0) Rows per input + * @param state.range(1) Number of directed pairs running concurrently + */ +void BM_ConvertGpuToGpuStagedPairs(benchmark::State& state) +{ + if (skip_without_two_devices(state)) { return; } + + cudf::size_type const rows = static_cast(state.range(0)); + auto const num_pairs = static_cast(state.range(1)); + + auto const candidates = directed_pairs(test::cuda_device_count()); + if (candidates.size() < num_pairs) { + state.SkipWithError("not enough CUDA devices for this many pairs"); + return; + } + + auto const make_stream = [](int device) { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{device}}; + return rmm::cuda_stream{}; + }; + + std::vector> pairs; + pairs.reserve(num_pairs); + std::size_t bytes_per_iteration = 0; + for (std::size_t i = 0; i < num_pairs; ++i) { + auto const [src_device, dst_device] = candidates[i]; + pairs.emplace_back(new staged_pair{ + src_device, dst_device, make_stream(src_device), make_stream(dst_device), {}, {}}); + staged_pair& pair = *pairs.back(); + + pair.inputs = make_inputs(kNumInputs, rows, pair.producer, src_device); + pair.outputs.reserve(kNumInputs); + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{src_device}}; + pair.producer.synchronize(); + } + for (auto const& input : pair.inputs) { + bytes_per_iteration += input->get_size_in_bytes(); + } + } + + auto const convert_pair = [](staged_pair& pair) { + const memory_space* dst_space = g_manager->get_memory_space(Tier::GPU, pair.dst_device); + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{pair.dst_device}}; + for (auto const& input : pair.inputs) { + pair.outputs.push_back( + g_staged_registry->convert(*input, dst_space, pair.consumer)); + } + // The consumer may read every output only after its writer event. + for (auto const& output : pair.outputs) { + output->wait_for_writer(pair.consumer); + } + pair.consumer.synchronize(); + }; + + auto const free_outputs = [](staged_pair& pair) { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{pair.dst_device}}; + pair.outputs.clear(); + CUCASCADE_CUDA_TRY(cudaDeviceSynchronize()); + }; + + // Warm-up: first touch of each destination pool. + for (auto const& pair : pairs) { + convert_pair(*pair); + free_outputs(*pair); + } + + std::vector> workers; + workers.reserve(num_pairs); + for (auto _ : state) { + for (auto const& pair : pairs) { + workers.push_back(std::async(std::launch::async, convert_pair, std::ref(*pair))); + } + for (auto& worker : workers) { + worker.get(); + } + + // Free outside the timed region. + state.PauseTiming(); + workers.clear(); + for (auto const& pair : pairs) { + free_outputs(*pair); + } + state.ResumeTiming(); + } + + set_counters(state, + bytes_per_iteration, + kNumInputs * num_pairs, + pairs.front()->inputs.front()->get_table_view().num_columns()); + state.counters["rows"] = rows; + state.counters["pairs"] = static_cast(num_pairs); + state.counters["staging_peak_MiB"] = peak_host_mib(); + + // Free each pair's inputs with their device current, before the streams go away. + for (auto const& pair : pairs) { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{pair->src_device}}; + pair->inputs.clear(); + CUCASCADE_CUDA_TRY(cudaDeviceSynchronize()); + } +} + /** * @brief Lower bound: peer-copy the same buffers, all queued on one stream, one host sync. * @param state.range(0) Rows per input, sized using the converter's actual table allocation @@ -324,6 +553,16 @@ void gpu_to_gpu_args(benchmark::internal::Benchmark* b) b->ArgName("rows"); } +void staged_pairs_args(benchmark::internal::Benchmark* b) +{ + for (int64_t pairs : {1, 2, 4}) { + for (int64_t rows : kRowCounts) { + b->Args({rows, pairs}); + } + } + b->ArgNames({"rows", "pairs"}); +} + BENCHMARK(BM_ConvertGpuToGpu) ->Setup(DoSetup) ->Teardown(DoTeardown) @@ -331,6 +570,20 @@ BENCHMARK(BM_ConvertGpuToGpu) ->Unit(benchmark::kMillisecond) ->UseRealTime(); +BENCHMARK(BM_ConvertGpuToGpuStaged) + ->Setup(DoSetup) + ->Teardown(DoTeardown) + ->Apply(gpu_to_gpu_args) + ->Unit(benchmark::kMillisecond) + ->UseRealTime(); + +BENCHMARK(BM_ConvertGpuToGpuStagedPairs) + ->Setup(DoSetupAllGpus) + ->Teardown(DoTeardown) + ->Apply(staged_pairs_args) + ->Unit(benchmark::kMillisecond) + ->UseRealTime(); + BENCHMARK(BM_RawPeerCopy) ->Setup(DoSetup) ->Teardown(DoTeardown) diff --git a/docs/data-management.md b/docs/data-management.md index 9f38e559..74c9e326 100644 --- a/docs/data-management.md +++ b/docs/data-management.md @@ -331,9 +331,42 @@ Registered via `register_builtin_converters()`: | `host_data_representation` | `gpu_table_representation` | Allocates `rmm::device_buffer` per column buffer, `cudaMemcpyBatchAsync` (H2D), reconstructs `cudf::column` tree | | `gpu_table_representation` | `host_data_packed_representation` | `cudf::pack()` on GPU -> `cudaMemcpyAsync` (D2H) to multi-block host allocation | | `host_data_packed_representation` | `gpu_table_representation` | `cudaMemcpyAsync` (H2D) from host blocks -> `cudf::unpack()` on device | -| `gpu_table_representation` | `gpu_table_representation` | `cudf::pack()` -> `cudaMemcpyPeerAsync` -> `cudf::unpack()` (cross-device copy) | +| `gpu_table_representation` | `gpu_table_representation` | Same device: `clone()`. Verified peer route: `cudaMemcpyPeerAsync` per column buffer. Otherwise a staged copy through pinned host memory (below) | | `host_data_packed_representation` | `host_data_packed_representation` | Block-by-block `std::memcpy` + metadata copy (cross-NUMA copy) | +### Staged GPU to GPU Copies + +When two devices cannot copy to each other directly (see the peer route verification in +[memory-management.md](memory-management.md)), the GPU to GPU converter sends the table through +pinned host memory: + +1. It validates the table and records every buffer, without touching the device. +2. It reads the character sizes of all STRING columns in one device round trip. +3. It allocates every destination buffer. +4. It moves all buffers as one stream of chunks through a small ring of host slots. The copy to + the host runs on a stream of the source device and the copy from the host on a stream of the + destination device, so one chunk is read while an earlier one is still being written. +5. It builds the `cudf::column` tree on the host from the filled buffers, with no kernels. + +The calling thread waits at most three times per conversion, whatever the schema: for the source +device if the source has no writer event, for the STRING sizes if any STRING column has rows, and +for the last chunk. A table without rows or columns is built on the host and waits for nothing. +When the conversion returns, no read of the source is pending. + +The two streams are ordered only by events that were recorded before they are waited on, so +conversions in any mix of directions run concurrently without locks and cannot deadlock. + +The ring holds at most `ring_depth` chunks of `chunk_bytes` (four chunks of 4 MiB by default; set +with `register_builtin_converters(registry, staged_copy_settings{...})`), and about the size of the +table when the table is smaller. It comes from the HOST pool of the destination device's NUMA +node. If the pool cannot supply it, smaller rings are tried, down to one block; only then does the +conversion fail with `rmm::out_of_memory`. Without a HOST memory space the ring is pinned for the +duration of the call. + +Columns must be whole on this route. A view that starts past row 0, or one that ends early over a +STRING, LIST, STRUCT or DICTIONARY32 column, is rejected with `std::invalid_argument` in every +build type. The peer route does not check this yet. + ### Conversion Flow GPU to HOST (direct copy — preferred): diff --git a/include/cucascade/cudf/builtin_converters.hpp b/include/cucascade/cudf/builtin_converters.hpp index 25daa341..2e6a3e33 100644 --- a/include/cucascade/cudf/builtin_converters.hpp +++ b/include/cucascade/cudf/builtin_converters.hpp @@ -19,8 +19,44 @@ #include +#include + namespace cucascade { +/** + * @brief Bounds on the pinned host memory one GPU -> GPU conversion borrows when the two devices + * cannot copy to each other directly. + * + * Such a conversion moves the table through a ring of host buffers: up to @ref ring_depth chunks + * of up to @ref chunk_bytes each are in flight at once, so it never holds more than their product + * (rounded up to the HOST pool's block size), and a table smaller than that holds about its own + * size. Larger chunks and a deeper ring trade host memory for fewer, longer copies. + */ +class staged_copy_settings { + public: + static constexpr std::size_t default_chunk_bytes = std::size_t{4} << 20; ///< 4 MiB + static constexpr std::size_t default_ring_depth = 4; + static constexpr std::size_t max_chunk_bytes = std::size_t{1} << 30; ///< 1 GiB + static constexpr std::size_t max_ring_depth = 64; + + /** @brief The defaults: four chunks of 4 MiB. */ + staged_copy_settings() = default; + + /** + * @param chunk_bytes Most bytes in one chunk, from 1 to @ref max_chunk_bytes + * @param ring_depth Most chunks in flight at once, from 1 to @ref max_ring_depth + * @throws std::invalid_argument if a value is zero or above its maximum + */ + staged_copy_settings(std::size_t chunk_bytes, std::size_t ring_depth); + + [[nodiscard]] std::size_t chunk_bytes() const noexcept { return _chunk_bytes; } + [[nodiscard]] std::size_t ring_depth() const noexcept { return _ring_depth; } + + private: + std::size_t _chunk_bytes{default_chunk_bytes}; + std::size_t _ring_depth{default_ring_depth}; +}; + /** * @brief Initialize the cudf-backed built-in representation converters. * @@ -41,4 +77,13 @@ namespace cucascade { */ void register_builtin_converters(representation_converter_registry& registry); +/** + * @brief register_builtin_converters() with explicit bounds for host-staged GPU -> GPU conversions. + * + * @param registry The converter registry to register converters with. + * @param staged_copy Bounds copied into the registered GPU -> GPU converter. + */ +void register_builtin_converters(representation_converter_registry& registry, + staged_copy_settings const& staged_copy); + } // namespace cucascade diff --git a/src/cudf/builtin_converters_detail.hpp b/src/cudf/builtin_converters_detail.hpp new file mode 100644 index 00000000..6aef862b --- /dev/null +++ b/src/cudf/builtin_converters_detail.hpp @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "staged_copy.hpp" + +#include + +namespace cucascade { +namespace detail { + +/** + * @brief Options of the GPU -> GPU converter beyond what the public registration exposes. + * + * They belong to the registry they were registered with, so two registries in one process can + * route differently. + */ +struct gpu_to_gpu_converter_options { + /** + * @brief Stage every conversion through pinned host memory. + * + * Applies to device pairs whose peer route is verified and to conversions within one device, so + * a machine with a single GPU, or with working peer access, still covers the staged route. + */ + bool force_staged_route{false}; + + /// Bounds on the host memory a staged conversion borrows. + memory::detail::staged_copy_ring_settings ring_settings{}; + + /// Hooks and counters handed to every staged conversion; must outlive the registry. Null in + /// production. + memory::detail::staged_copy_instrumentation* instrumentation{nullptr}; +}; + +/** + * @brief register_builtin_converters() with explicit GPU -> GPU converter options. + * + * @param registry The converter registry to register converters with. + * @param gpu_to_gpu_options Options copied into the registered GPU -> GPU converter. + */ +void register_builtin_converters(representation_converter_registry& registry, + gpu_to_gpu_converter_options const& gpu_to_gpu_options); + +} // namespace detail +} // namespace cucascade diff --git a/src/cudf/representation_converter_builtins.cpp b/src/cudf/representation_converter_builtins.cpp index 731dab8b..055bd56b 100644 --- a/src/cudf/representation_converter_builtins.cpp +++ b/src/cudf/representation_converter_builtins.cpp @@ -15,7 +15,9 @@ * limitations under the License. */ +#include "builtin_converters_detail.hpp" #include "cudf_compat.hpp" +#include "staged_copy.hpp" #include #include @@ -43,6 +45,7 @@ #include #include #include +#include #include #include @@ -64,8 +67,11 @@ #include #include #include +#include +#include #include #include +#include #include #include #include @@ -531,49 +537,10 @@ std::unique_ptr convert_gpu_to_host_fast( // ============================================================================= /** - * @brief Best-effort lookup of the NUMA node a GPU's PCI device is attached to. - * Returns -1 if the device id is invalid, the PCI bus id is unknown, or - * /sys/bus/pci/devices//numa_node is unreadable / -1. Pure topology - * query — no allocation, no side effects. - */ -int resolve_gpu_numa_node(int device_id) noexcept -{ - if (device_id < 0) { return -1; } - char pci_buf[32] = {0}; - if (cudaDeviceGetPCIBusId(pci_buf, sizeof(pci_buf), device_id) != cudaSuccess) { return -1; } - std::string pci(pci_buf); - for (auto& c : pci) { - c = static_cast(std::tolower(c)); - } - std::ifstream f("/sys/bus/pci/devices/" + pci + "/numa_node"); - if (!f.is_open()) { return -1; } - std::string content; - std::getline(f, content); - if (content.empty()) { return -1; } - try { - return std::stoi(content); - } catch (...) { - return -1; - } -} - -/** - * @brief Copy @p size bytes from @p src_ptr - * (on @p src_device) into a buffer. - * - * Routing is decided per direction by cucascade::memory::probe_peer_dma_works(), which verifies a - * peer copy between two mutually granted private memory pools: - * - verified: cudaMemcpyPeerAsync (direct only when the pools are granted peer access) - * - wrong bytes or a CUDA error during verification: explicit host-staged copy + * @brief Enqueue a direct copy of @p size_in_bytes bytes from @p src_ptr on @p src_device into + * @p buf on @p dst_device, on @p target_stream. * - * The host-staged copy goes through the pre-pinned HOST memory_space pool (registered by - * memory_space's HOST constructor in register_host_pool) and chunks across the pool's fixed block - * size, exactly like convert_gpu_to_host. We do the staging ourselves rather than relying on the - * driver's auto-fallback, so the correctness path is identical and observable regardless of - * driver/CUDA version quirks. - * - * NEVER calls cudaHostAlloc per transfer — the pool is allocated once at - * memory_reservation_manager construction time. + * Only for device pairs whose direct route cucascade::memory::probe_peer_dma_works() verified. */ template Buffer peer_copy_into_async(Buffer buf, @@ -585,129 +552,8 @@ Buffer peer_copy_into_async(Buffer buf, { if (size_in_bytes == 0 || src_ptr == nullptr) { return buf; } - if (memory::probe_peer_dma_works(src_device, dst_device)) { - // Real peer DMA works on this hardware — direct path. - CUCASCADE_CUDA_TRY(cudaMemcpyPeerAsync( - buf.data(), dst_device, src_ptr, src_device, size_in_bytes, target_stream.get())); - return buf; - } - - // Peer DMA broken — stage through the pre-pinned HOST pool. - // TODO: these paths currently remain untested. Add tests to cover these. - // - // Block layout: the host pool allocates non-contiguous fixed-size blocks - // (1 MB by default). We chunk the D2H and H2D legs across blocks just like - // convert_gpu_to_host does. Stream-ordering inside the loop preserves the - // same-stream invariant the original implementation relied on. - memory::fixed_size_host_memory_resource* host_pool = - memory::find_host_pool(resolve_gpu_numa_node(dst_device)); - // Without working peer DMA, each leg must run with its own device current (the D2H leg - // fails with cudaErrorInvalidValue otherwise). Both legs stay on target_stream, so they are - // still ordered. - - if (host_pool == nullptr) { - // No HOST memory_space has been registered. This should only happen in - // standalone test builds that exercise the converter without constructing - // a memory_reservation_manager. Fall back to a one-shot pinned allocation - // (slow path — preserves correctness). - memory::numa_region_pinned_host_memory_resource mr(resolve_gpu_numa_node(dst_device), - /*make_portable=*/true); - void* host_buf = mr.allocate_sync(size_in_bytes); - - try { - { - rmm::cuda_set_device_raii src_guard{rmm::cuda_device_id{src_device}}; - CUCASCADE_CUDA_TRY(cudaMemcpyAsync( - host_buf, src_ptr, size_in_bytes, cudaMemcpyDeviceToHost, target_stream.get())); - } - { - rmm::cuda_set_device_raii dst_guard{rmm::cuda_device_id{dst_device}}; - CUCASCADE_CUDA_TRY(cudaMemcpyAsync( - buf.data(), host_buf, size_in_bytes, cudaMemcpyHostToDevice, target_stream.get())); - } - target_stream.sync(); - } catch (...) { - // best-effort drain before deallocating host side resources; a second failure must not - // replace the original exception or stay pending. - if (cudaStreamSynchronize(target_stream.get()) != cudaSuccess) { - std::ignore = cudaGetLastError(); - } - mr.deallocate_sync(host_buf, size_in_bytes); - throw; - } - mr.deallocate_sync(host_buf, size_in_bytes); - return buf; - } - - { - auto allocation = host_pool->allocate_multiple_blocks(size_in_bytes); - const std::size_t block_sz = allocation->block_size(); - - try { - // Pass 1: device-to-host, chunked across blocks. Issued on target_stream - { - rmm::cuda_set_device_raii src_guard{rmm::cuda_device_id{src_device}}; - std::size_t block_index = 0; - std::size_t block_offset = 0; - std::size_t src_offset = 0; - while (src_offset < size_in_bytes) { - const std::size_t remaining = size_in_bytes - src_offset; - const std::size_t bytes_left = block_sz - block_offset; - const std::size_t to_copy = std::min(remaining, bytes_left); - std::span block = allocation->at(block_index); - CUCASCADE_CUDA_TRY(cudaMemcpyAsync(block.data() + block_offset, - static_cast(src_ptr) + src_offset, - to_copy, - cudaMemcpyDeviceToHost, - target_stream.get())); - src_offset += to_copy; - block_offset += to_copy; - if (block_offset == block_sz) { - ++block_index; - block_offset = 0; - } - } - } - - // Pass 2: host-to-device, chunked across the same blocks. - { - rmm::cuda_set_device_raii dst_guard{rmm::cuda_device_id{dst_device}}; - std::size_t block_index = 0; - std::size_t block_offset = 0; - std::size_t dst_offset = 0; - while (dst_offset < size_in_bytes) { - const std::size_t remaining = size_in_bytes - dst_offset; - const std::size_t bytes_left = block_sz - block_offset; - const std::size_t to_copy = std::min(remaining, bytes_left); - std::span block = allocation->at(block_index); - CUCASCADE_CUDA_TRY( - cudaMemcpyAsync(reinterpret_cast(buf.data()) + dst_offset, - block.data() + block_offset, - to_copy, - cudaMemcpyHostToDevice, - target_stream.get())); - dst_offset += to_copy; - block_offset += to_copy; - if (block_offset == block_sz) { - ++block_index; - block_offset = 0; - } - } - } - - target_stream.sync(); - } catch (...) { - // best-effort drain before the blocks return to the pool; a second failure must not - // replace the original exception or stay pending. - if (cudaStreamSynchronize(target_stream.get()) != cudaSuccess) { - std::ignore = cudaGetLastError(); - } - throw; - } - - // allocation's destructor returns blocks to the host pool — no cudaFreeHost. - } - + CUCASCADE_CUDA_TRY(cudaMemcpyPeerAsync( + buf.data(), dst_device, src_ptr, src_device, size_in_bytes, target_stream.get())); return buf; } @@ -865,6 +711,351 @@ std::unique_ptr reconstruct_column_p2p_async(const cudf::column_vi src.type(), src.size(), std::move(data_buf), std::move(null_mask), null_count); } +// ============================================================================= +// Host-staged GPU -> GPU converter (device pairs without a working direct route) +// ============================================================================= + +/** + * @brief One column of a table that is about to be copied through host memory. + * + * Built in two passes over the source. The first validates the column and records what it holds; + * the second, once the sizes of STRING characters are known, allocates the destination buffers. + */ +struct staged_column { + cudf::data_type type{cudf::type_id::EMPTY}; + cudf::size_type rows{0}; + cudf::size_type null_count{0}; + + void const* source_mask{nullptr}; + std::size_t mask_bytes{0}; ///< The bitmask words covering `rows`; zero if not nullable + void const* source_data{nullptr}; + std::size_t data_bytes{0}; ///< Unknown for STRING characters until the size probe is read + std::optional size_probe; ///< Which probe holds the size of the characters + + /// Set by the second pass; holds nothing if the column is not nullable. + std::optional mask; + rmm::device_buffer data; + std::vector children; + + /// A column without rows copies nothing: the second pass builds `empty` from `source`. + cudf::column_view source; + std::unique_ptr empty; +}; + +/** @brief What the first planning pass learns about the table as a whole. */ +struct staged_table_summary { + std::vector size_probes; + bool reads_source{false}; ///< Whether any byte of the source is read at all +}; + +[[noreturn]] void reject_staged_column(std::string const& reason) +{ + throw std::invalid_argument("host-staged GPU to GPU conversion: " + reason); +} + +/// A null mask that holds nothing, for a column that is not nullable. Allocates no device memory. +cudf_compat::null_mask_buffer make_absent_null_mask(::cuda::stream_ref stream, + rmm::device_async_resource_ref mr) +{ + return cudf::create_null_mask(0, cudf::mask_state::UNALLOCATED, stream, mr); +} + +/** + * @brief An empty column with the type of @p view, built without touching the device. + * + * Nested types keep their shape: a LIST has an empty offsets child and an empty child of its + * element type, a STRUCT has empty fields, and a DICTIONARY32 has empty indices and empty keys of + * its key type (the keys of a dictionary without rows are not copied). A STRING has no children. + */ +std::unique_ptr make_empty_like(cudf::column_view const& view, + ::cuda::stream_ref stream, + rmm::device_async_resource_ref mr) +{ + auto const empty_leaf = [&](cudf::data_type type) { + return std::make_unique( + type, 0, rmm::device_buffer{}, make_absent_null_mask(stream, mr), 0); + }; + + std::vector> children; + switch (view.type().id()) { + case cudf::type_id::LIST: + case cudf::type_id::DICTIONARY32: + // child 0 is the offsets (LIST) or the indices (DICTIONARY32), a column without children. + if (view.num_children() == 2) { + children.push_back(empty_leaf(view.child(0).type())); + children.push_back(make_empty_like(view.child(1), stream, mr)); + } + break; + case cudf::type_id::STRUCT: + for (cudf::size_type i = 0; i < view.num_children(); ++i) { + children.push_back(make_empty_like(view.child(i), stream, mr)); + } + break; + default: break; + } + return std::make_unique(view.type(), + 0, + rmm::device_buffer{}, + make_absent_null_mask(stream, mr), + 0, + std::move(children)); +} + +/// An offsets child must be whole: one entry more than its parent has rows, starting at entry 0. +void require_whole_offsets(cudf::column_view const& offsets, + cudf::size_type parent_rows, + char const* parent_type) +{ + auto const id = offsets.type().id(); + if ((id != cudf::type_id::INT32 && id != cudf::type_id::INT64) || offsets.offset() != 0 || + static_cast(offsets.size()) != static_cast(parent_rows) + 1) { + reject_staged_column(std::string{"the offsets of a "} + parent_type + + " column are sliced or malformed"); + } +} + +/** + * @brief First pass: validate @p view and record its buffers, without touching the device. + * + * A column must be whole. A view that starts past row 0 is rejected for every type, and so is a + * view that ends early when the column has children, because the children would not match it. + * + * @throws std::invalid_argument if the column is sliced, malformed or of an unsupported type + */ +staged_column plan_staged_column(cudf::column_view const& view, staged_table_summary& summary) +{ + staged_column plan; + plan.type = view.type(); + plan.rows = view.size(); + if (view.size() == 0) { + plan.source = view; + return plan; + } + + if (view.offset() != 0) { reject_staged_column("a column view that starts past row 0"); } + + auto const rows = static_cast(view.size()); + plan.null_count = view.null_count(); + if (view.nullable()) { + plan.source_mask = view.null_mask(); + plan.mask_bytes = + static_cast(cudf::num_bitmask_words(view.size())) * sizeof(cudf::bitmask_type); + summary.reads_source = true; + } + + switch (view.type().id()) { + case cudf::type_id::STRING: { + if (view.num_children() != 1) { + reject_staged_column("a STRING column with rows needs an offsets child"); + } + auto const offsets = view.child(0); + require_whole_offsets(offsets, view.size(), "STRING"); + // The last offset is the size of the characters, known only on the device. + plan.source_data = view.head(); + plan.size_probe = summary.size_probes.size(); + auto const width = cudf::size_of(offsets.type()); + summary.size_probes.push_back( + {static_cast(offsets.head()) + rows * width, width}); + plan.children.push_back(plan_staged_column(offsets, summary)); + break; + } + case cudf::type_id::LIST: { + if (view.num_children() != 2) { + reject_staged_column("a LIST column needs an offsets child and a values child"); + } + require_whole_offsets(view.child(0), view.size(), "LIST"); + plan.children.push_back(plan_staged_column(view.child(0), summary)); + plan.children.push_back(plan_staged_column(view.child(1), summary)); + break; + } + case cudf::type_id::STRUCT: { + for (cudf::size_type i = 0; i < view.num_children(); ++i) { + if (view.child(i).size() != view.size()) { + reject_staged_column("a STRUCT field differs in length from its column"); + } + plan.children.push_back(plan_staged_column(view.child(i), summary)); + } + break; + } + case cudf::type_id::DICTIONARY32: { + if (view.num_children() != 2 || view.child(0).size() != view.size()) { + reject_staged_column("the indices of a DICTIONARY32 column are missing or sliced"); + } + plan.children.push_back(plan_staged_column(view.child(0), summary)); + plan.children.push_back(plan_staged_column(view.child(1), summary)); + break; + } + default: { + if (cudf::is_fixed_width(view.type())) { + if (view.head() == nullptr) { reject_staged_column("a column with rows has no data"); } + plan.source_data = view.head(); + plan.data_bytes = rows * cudf::size_of(view.type()); + summary.reads_source = true; + } else if (view.type().id() != cudf::type_id::EMPTY) { + reject_staged_column("unsupported column type " + + std::to_string(static_cast(view.type().id()))); + } + break; + } + } + return plan; +} + +/** + * @brief Second pass: allocate the destination buffers of @p plan on @p stream and append one + * segment per buffer to @p segments, in the order mask, data, children. + * + * Only the words of a mask that cover the rows are copied, which is all a source mask is sure to + * hold. A column without rows gets its empty column here and no segment. + */ +void allocate_staged_column(staged_column& plan, + std::span string_sizes, + ::cuda::stream_ref stream, + rmm::device_async_resource_ref mr, + std::vector& segments) +{ + if (plan.rows == 0) { + plan.empty = make_empty_like(plan.source, stream, mr); + return; + } + + if (plan.mask_bytes > 0) { + plan.mask.emplace( + cudf::create_null_mask(plan.rows, cudf::mask_state::UNINITIALIZED, stream, mr)); + if (cudf_compat::get_null_mask_size_bytes(*plan.mask) < plan.mask_bytes) { + reject_staged_column("a null mask is smaller than its rows need"); + } + segments.push_back({plan.source_mask, plan.mask->data(), plan.mask_bytes}); + } else { + plan.mask.emplace(make_absent_null_mask(stream, mr)); + } + if (plan.size_probe) { + std::int64_t const characters = string_sizes[*plan.size_probe]; + if (characters < 0) { reject_staged_column("the last offset of a STRING column is negative"); } + plan.data_bytes = static_cast(characters); + if (plan.data_bytes > 0 && plan.source_data == nullptr) { + reject_staged_column("a STRING column has offsets past its characters"); + } + } + if (plan.data_bytes > 0) { + plan.data = rmm::device_buffer(plan.data_bytes, stream, mr); + segments.push_back({plan.source_data, plan.data.data(), plan.data_bytes}); + } + for (auto& child : plan.children) { + allocate_staged_column(child, string_sizes, stream, mr, segments); + } +} + +/** + * @brief Turn a planned column, whose buffers have been filled, into a cudf column. + * + * Uses the cudf::column constructor for every type, which only takes ownership. The cudf factory + * functions are avoided because some launch kernels that read the buffers, on a stream this + * conversion does not control. + */ +std::unique_ptr assemble_staged_column(staged_column&& plan) +{ + if (plan.rows == 0) { return std::move(plan.empty); } + + std::vector> children; + children.reserve(plan.children.size()); + for (auto& child : plan.children) { + children.push_back(assemble_staged_column(std::move(child))); + } + return std::make_unique(plan.type, + plan.rows, + std::move(plan.data), + std::move(*plan.mask), + plan.null_count, + std::move(children)); +} + +/** + * @brief Copy @p source to the device of @p target_memory_space through pinned host memory. + * + * The table is validated and measured first, every destination buffer is allocated next, and all + * buffers then travel in one memory::detail::staged_copy() call. The calling thread waits at most + * three times, whatever the schema: for the source's writer if that is unknown, for the sizes of + * the STRING columns if there are any with rows, and for the last byte to arrive. A table without + * rows or without columns is built on the host and waits for nothing. + * + * The source may be freed as soon as this returns: no read of it is pending. + * + * @throws std::invalid_argument if a column is sliced, malformed or of an unsupported type + */ +std::unique_ptr convert_gpu_to_gpu_staged( + gpu_table_representation const& source, + memory::memory_space& target_memory_space, + detail::gpu_to_gpu_converter_options const& options) +{ + nvtx_scope staged_range{"convert:gpu_to_gpu:host_staging"}; + + int const src_device = source.get_device_id(); + int const dst_device = target_memory_space.get_device_id(); + + // Declared before anything that owns destination memory, so that memory is freed with its + // device current when an exception unwinds this frame. + rmm::cuda_set_device_raii const target_guard{rmm::cuda_device_id{dst_device}}; + + // Each leg runs on a stream of the device it reads or writes. + ::cuda::stream_ref const src_stream = source.get_memory_space().acquire_stream(); + ::cuda::stream_ref const dst_stream = target_memory_space.acquire_stream(); + + cudf::table_view const src_view = source.get_table_view(); + staged_table_summary summary; + std::vector plans; + plans.reserve(static_cast(src_view.num_columns())); + for (cudf::column_view const& column : src_view) { + plans.push_back(plan_staged_column(column, summary)); + } + bool const reads_source = summary.reads_source || !summary.size_probes.empty(); + + std::vector string_sizes; + if (reads_source) { + { + // Order the reads after the source's writer. The stream that is made to wait belongs to the + // source device, so that device must be current. An unknown writer makes this wait for the + // whole source device. + rmm::cuda_set_device_raii const source_guard{rmm::cuda_device_id{src_device}}; + if (source.get_writer_event() == nullptr) { + nvtx_scope readiness_range{"staged:readiness"}; + source.wait_for_writer(src_stream); + if (options.instrumentation != nullptr) { ++options.instrumentation->readiness_syncs; } + } else { + source.wait_for_writer(src_stream); + } + } + string_sizes = + memory::detail::read_device_integers(summary.size_probes, + {src_device, src_stream}, + source.get_memory_space().get_default_allocator(), + options.instrumentation); + } + + std::vector segments; + auto const mr = target_memory_space.get_default_allocator(); + for (auto& plan : plans) { + allocate_staged_column(plan, string_sizes, dst_stream, mr, segments); + } + + memory::detail::staged_copy(segments, + {src_device, src_stream}, + {dst_device, dst_stream}, + options.ring_settings, + options.instrumentation); + + std::vector> columns; + columns.reserve(plans.size()); + for (auto& plan : plans) { + columns.push_back(assemble_staged_column(std::move(plan))); + } + + // The destination buffers were written on dst_stream; the representation records an event on it + // for readers on other streams. + return std::make_unique( + std::make_unique(std::move(columns)), target_memory_space, dst_stream); +} + /** * @brief Convert gpu_table_representation to gpu_table_representation (cross-GPU copy). * @@ -880,7 +1071,8 @@ std::unique_ptr convert_gpu_to_gpu( const idata_representation& source, const memory::memory_space* target_memory_space, ::cuda::stream_ref stream, - [[maybe_unused]] memory::reservation* reservation) + [[maybe_unused]] memory::reservation* reservation, + detail::gpu_to_gpu_converter_options const& options) { nvtx_scope convert_range{"convert:gpu_to_gpu"}; // no sync on the caller's stream needed here: wait_for_writer() below, which is the real @@ -890,7 +1082,8 @@ std::unique_ptr convert_gpu_to_gpu( auto& gpu_source = source.cast(); // Same-device case: clone via source's own clone() method. - if (source.get_device_id() == target_memory_space->get_device_id()) { + if (source.get_device_id() == target_memory_space->get_device_id() && + !options.force_staged_route) { // A null stream handle represents the default stream of the current device: ensure it's the // device attached to the representation, or clone() will throw. rmm::cuda_set_device_raii guard{rmm::cuda_device_id{target_memory_space->get_device_id()}}; @@ -899,10 +1092,14 @@ std::unique_ptr convert_gpu_to_gpu( auto const src_device_id = gpu_source.get_device_id(); auto const dst_device_id = target_memory_space->get_device_id(); - auto const copy_path = memory::probe_peer_dma_works(src_device_id, dst_device_id) - ? "convert:gpu_to_gpu:peer_copy" - : "convert:gpu_to_gpu:host_staging"; - nvtx_scope copy_path_range{copy_path}; + + // The route is chosen once for the whole table: a direct copy where the pair's peer route is + // verified, a copy through pinned host memory otherwise. + if (options.force_staged_route || !memory::probe_peer_dma_works(src_device_id, dst_device_id)) { + return convert_gpu_to_gpu_staged( + gpu_source, *const_cast(target_memory_space), options); + } + nvtx_scope copy_path_range{"convert:gpu_to_gpu:peer_copy"}; rmm::cuda_set_device_raii target_guard{rmm::cuda_device_id{dst_device_id}}; @@ -1881,15 +2078,49 @@ static std::unique_ptr convert_disk_to_gpu( } // namespace +staged_copy_settings::staged_copy_settings(std::size_t chunk_bytes, std::size_t ring_depth) + : _chunk_bytes(chunk_bytes), _ring_depth(ring_depth) +{ + if (chunk_bytes == 0 || chunk_bytes > max_chunk_bytes || ring_depth == 0 || + ring_depth > max_ring_depth) { + throw std::invalid_argument( + "staged_copy_settings: chunk_bytes must be 1 B to 1 GiB and ring_depth 1 to 64"); + } +} + void register_builtin_converters(representation_converter_registry& registry) +{ + register_builtin_converters(registry, staged_copy_settings{}); +} + +void register_builtin_converters(representation_converter_registry& registry, + staged_copy_settings const& staged_copy) +{ + detail::gpu_to_gpu_converter_options options; + options.ring_settings.chunk_bytes = staged_copy.chunk_bytes(); + options.ring_settings.depth = staged_copy.ring_depth(); + detail::register_builtin_converters(registry, options); +} + +namespace detail { + +void register_builtin_converters(representation_converter_registry& registry, + gpu_to_gpu_converter_options const& gpu_to_gpu_options) { // GPU -> GPU (cross-device copy). convert_gpu_to_gpu copies each column buffer with // cudaMemcpyPeerAsync when cucascade::memory::probe_peer_dma_works() verifies the direction, and - // stages through pinned host memory otherwise. The cached probe copies bytes between two mutually - // granted private pools, so a verification failure means the pool peer route delivered wrong - // bytes, as on platforms whose direct route reports success but leaves stale data. + // sends the whole table through a ring of pinned host memory otherwise. The cached probe copies + // bytes between two mutually granted private pools, so a verification failure means the pool + // peer route delivered wrong bytes, as on platforms whose direct route reports success but + // leaves stale data. registry.register_converter( - convert_gpu_to_gpu); + [gpu_to_gpu_options](const idata_representation& source, + const memory::memory_space* target_memory_space, + ::cuda::stream_ref stream, + memory::reservation* reservation) { + return convert_gpu_to_gpu( + source, target_memory_space, stream, reservation, gpu_to_gpu_options); + }); // GPU -> HOST registry.register_converter( @@ -1932,4 +2163,6 @@ void register_builtin_converters(representation_converter_registry& registry) convert_disk_to_gpu); } +} // namespace detail + } // namespace cucascade diff --git a/src/memory/CMakeLists.txt b/src/memory/CMakeLists.txt index 21958de7..5865f10e 100644 --- a/src/memory/CMakeLists.txt +++ b/src/memory/CMakeLists.txt @@ -31,6 +31,7 @@ if(TARGET cucascade_objects) ${CMAKE_CURRENT_SOURCE_DIR}/reservation_aware_resource_adaptor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/reservation_manager_configurator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/small_pinned_host_memory_resource.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/staged_copy.cpp ${CMAKE_CURRENT_SOURCE_DIR}/stream_pool.cpp ${CMAKE_CURRENT_SOURCE_DIR}/error.cpp) endif() diff --git a/src/memory/staged_copy.cpp b/src/memory/staged_copy.cpp new file mode 100644 index 00000000..244eac4b --- /dev/null +++ b/src/memory/staged_copy.cpp @@ -0,0 +1,466 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "staged_copy.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cucascade { +namespace memory { +namespace detail { + +namespace { + +using nvtx_range = nvtx3::scoped_range_in; + +[[nodiscard]] constexpr std::size_t ceil_div(std::size_t value, std::size_t divisor) noexcept +{ + return value / divisor + (value % divisor != 0 ? 1U : 0U); +} + +void require_stream_of_device(staged_copy_endpoint const& endpoint, char const* role) +{ + int stream_device = -1; + CUCASCADE_CUDA_TRY(cudaStreamGetDevice(endpoint.stream.get(), &stream_device)); + if (stream_device != endpoint.device_id) { + throw std::invalid_argument(std::string{"staged copy: the "} + role + + " stream belongs to device " + std::to_string(stream_device) + + ", not to device " + std::to_string(endpoint.device_id)); + } +} + +/** + * @brief Wait for all work enqueued so far on @p endpoint's stream, on a path that must not throw. + * + * @param endpoint The stream and its device + * @param marker An event of that device, recorded here and then waited on + * @return false if the work could not be confirmed complete + */ +[[nodiscard]] bool wait_for_enqueued_work(staged_copy_endpoint const& endpoint, + cuda::cuda_event const& marker) noexcept +{ + int previous_device = -1; + if (cudaGetDevice(&previous_device) != cudaSuccess || + cudaSetDevice(endpoint.device_id) != cudaSuccess) { + std::ignore = cudaGetLastError(); + return false; + } + bool complete = cudaEventRecord(marker.get(), endpoint.stream.get()) == cudaSuccess && + cudaEventSynchronize(marker.get()) == cudaSuccess; + if (cudaSetDevice(previous_device) != cudaSuccess) { complete = false; } + if (!complete) { std::ignore = cudaGetLastError(); } + return complete; +} + +/// Pinned host memory for one copy, used when no HOST pool is registered. +class temporary_pinned_memory { + public: + temporary_pinned_memory(int numa_node, std::size_t bytes) + : _mr(numa_node, /*make_portable=*/true), + _bytes(bytes), + _base(static_cast(_mr.allocate_sync(bytes))) + { + } + + ~temporary_pinned_memory() + { + if (_base != nullptr) { _mr.deallocate_sync(_base, _bytes); } + } + + temporary_pinned_memory(temporary_pinned_memory const&) = delete; + temporary_pinned_memory& operator=(temporary_pinned_memory const&) = delete; + + [[nodiscard]] std::byte* data() const noexcept { return _base; } + + /// Keep the memory pinned and mapped for the rest of the process. + void abandon() noexcept { _base = nullptr; } + + private: + numa_region_pinned_host_memory_resource _mr; + std::size_t _bytes; + std::byte* _base; +}; + +/// The host slots of one staged copy. Gives them back on destruction unless abandoned. +class staging_ring { + public: + staging_ring(std::size_t total_bytes, + int destination_device, + staged_copy_ring_settings const& settings) + { + int const numa_node = gpu_numa_node(destination_device); + if (fixed_size_host_memory_resource* pool = find_host_pool(numa_node); pool != nullptr) { + _block_bytes = pool->get_block_size(); + _geometry = plan_staged_copy_ring(total_bytes, _block_bytes, settings); + for (;;) { + try { + _pool_blocks = pool->allocate_multiple_blocks(bytes()); + break; + } catch (rmm::out_of_memory const&) { + if (!shrink_staged_copy_ring(_geometry)) { throw; } + } + } + } else { + _block_bytes = fixed_size_host_memory_resource::default_block_size; + _geometry = plan_staged_copy_ring(total_bytes, _block_bytes, settings); + _temporary = std::make_unique(numa_node, bytes()); + } + } + + staging_ring(staging_ring const&) = delete; + staging_ring& operator=(staging_ring const&) = delete; + + [[nodiscard]] std::size_t slots() const noexcept { return _geometry.slots; } + [[nodiscard]] std::size_t block_bytes() const noexcept { return _block_bytes; } + [[nodiscard]] std::size_t slot_bytes() const noexcept + { + return _geometry.blocks_per_slot * _block_bytes; + } + [[nodiscard]] std::size_t bytes() const noexcept { return _geometry.slots * slot_bytes(); } + + /// Block @p index of slot @p slot; a slot's blocks need not be adjacent in memory. + [[nodiscard]] std::byte* block(std::size_t slot, std::size_t index) const noexcept + { + std::size_t const position = slot * _geometry.blocks_per_slot + index; + if (_pool_blocks) { return _pool_blocks->get_blocks()[position]; } + return _temporary->data() + position * _block_bytes; + } + + /// Never give the memory back: device work that could not be waited for may still write to it. + void abandon() noexcept + { + if (_pool_blocks) { + // The blocks stay allocated in the pool's accounting, which is what keeps them unused. + std::ignore = _pool_blocks->release_blocks(); + } + if (_temporary) { _temporary->abandon(); } + } + + private: + staged_copy_ring_geometry _geometry{}; + std::size_t _block_bytes{0}; + fixed_size_host_memory_resource::fixed_multiple_blocks_allocation _pool_blocks; + std::unique_ptr _temporary; +}; + +/** + * @brief The events of one leg: one per slot, recorded when the leg's copy into or out of that + * slot is enqueued, and one to wait on after a failure. + * + * An event can only be recorded on a stream of the device that was current when it was created. + */ +class leg_events { + public: + leg_events(int device_id, std::size_t slots) + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{device_id}}; + _events.reserve(slots + 1); + for (std::size_t i = 0; i < slots + 1; ++i) { + _events.emplace_back(); + } + } + + [[nodiscard]] cuda::cuda_event& slot(std::size_t index) noexcept { return _events[index]; } + [[nodiscard]] cuda::cuda_event const& after_failure() const noexcept { return _events.back(); } + + private: + std::vector _events; +}; + +void call_before_leg(staged_copy_instrumentation* instrumentation, + staged_copy_leg leg, + std::size_t chunk_index, + ::cuda::stream_ref stream) +{ + if (instrumentation != nullptr && instrumentation->before_leg) { + instrumentation->before_leg(leg, chunk_index, stream); + } +} + +} // namespace + +int gpu_numa_node(int device_id) noexcept +{ + if (device_id < 0) { return -1; } + char pci_buf[32] = {0}; + if (cudaDeviceGetPCIBusId(pci_buf, static_cast(sizeof(pci_buf)), device_id) != cudaSuccess) { + std::ignore = cudaGetLastError(); + return -1; + } + try { + // CUDA reports the bus id in upper case; sysfs names the device in lower case. + std::string pci(pci_buf); + for (auto& c : pci) { + c = static_cast(std::tolower(static_cast(c))); + } + std::ifstream file("/sys/bus/pci/devices/" + pci + "/numa_node"); + std::string content; + if (!std::getline(file, content) || content.empty()) { return -1; } + return std::stoi(content); + } catch (...) { + return -1; + } +} + +staged_copy_ring_geometry plan_staged_copy_ring(std::size_t total_bytes, + std::size_t block_bytes, + staged_copy_ring_settings const& settings) +{ + if (total_bytes == 0 || block_bytes == 0) { + throw std::invalid_argument("staged copy: the payload and the block size must not be zero"); + } + if (settings.chunk_bytes == 0 || + settings.chunk_bytes > staged_copy_ring_settings::max_chunk_bytes || settings.depth == 0 || + settings.depth > staged_copy_ring_settings::max_depth) { + throw std::invalid_argument( + "staged copy: chunk capacity must be 1 B to 1 GiB and ring depth 1 to 64"); + } + + std::size_t const chunk_blocks = ceil_div(settings.chunk_bytes, block_bytes); + std::size_t const payload_blocks = ceil_div(total_bytes, block_bytes); + std::size_t const slots = std::min(settings.depth, payload_blocks); + std::size_t const blocks_per_slot = + std::min(chunk_blocks, ceil_div(ceil_div(total_bytes, slots), block_bytes)); + + if (blocks_per_slot > std::numeric_limits::max() / slots / block_bytes) { + throw std::invalid_argument("staged copy: the ring size overflows"); + } + return staged_copy_ring_geometry{slots, blocks_per_slot}; +} + +bool shrink_staged_copy_ring(staged_copy_ring_geometry& geometry) noexcept +{ + if (geometry.blocks_per_slot > 1) { + geometry.blocks_per_slot /= 2; + return true; + } + if (geometry.slots > 2) { + geometry.slots = 2; + return true; + } + if (geometry.slots > 1) { + geometry.slots = 1; + return true; + } + return false; +} + +void staged_copy(std::span segments, + staged_copy_endpoint source, + staged_copy_endpoint destination, + staged_copy_ring_settings const& settings, + staged_copy_instrumentation* instrumentation) +{ + CUCASCADE_FUNC_RANGE(); + + std::size_t total_bytes = 0; + for (auto const& segment : segments) { + if (segment.bytes == 0) { continue; } + if (segment.source == nullptr || segment.destination == nullptr) { + throw std::invalid_argument("staged copy: a segment with bytes has a null address"); + } + if (segment.bytes > std::numeric_limits::max() - total_bytes) { + throw std::invalid_argument("staged copy: the payload size overflows"); + } + total_bytes += segment.bytes; + } + if (total_bytes == 0) { return; } + + require_stream_of_device(source, "source"); + require_stream_of_device(destination, "destination"); + + staging_ring ring{total_bytes, destination.device_id, settings}; + if (instrumentation != nullptr) { + ++instrumentation->rings; + instrumentation->last_ring_bytes = ring.bytes(); + } + + // Everything the loop needs is created before the first copy is enqueued. + leg_events source_done{source.device_id, ring.slots()}; + leg_events destination_done{destination.device_id, ring.slots()}; + cuda::device_copy_batch to_host; + cuda::device_copy_batch to_device; + + try { + std::size_t segment_index = 0; + std::size_t segment_offset = 0; + std::size_t chunk_index = 0; + std::size_t remaining = total_bytes; + + while (remaining > 0) { + std::size_t const slot = chunk_index % ring.slots(); + std::size_t const chunk_bytes = std::min(ring.slot_bytes(), remaining); + + // Lay the next chunk_bytes of the segments out over the slot's blocks. A copy may cross + // neither a segment boundary nor a block boundary. + to_host.clear(); + to_device.clear(); + std::size_t filled = 0; + while (filled < chunk_bytes) { + auto const& segment = segments[segment_index]; + if (segment_offset == segment.bytes) { + ++segment_index; + segment_offset = 0; + continue; + } + std::size_t const block_offset = filled % ring.block_bytes(); + std::size_t const piece = std::min({segment.bytes - segment_offset, + ring.block_bytes() - block_offset, + chunk_bytes - filled}); + std::byte* const host = ring.block(slot, filled / ring.block_bytes()) + block_offset; + to_host.add(host, static_cast(segment.source) + segment_offset, piece); + to_device.add(static_cast(segment.destination) + segment_offset, host, piece); + filled += piece; + segment_offset += piece; + } + + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{source.device_id}}; + call_before_leg(instrumentation, staged_copy_leg::SOURCE, chunk_index, source.stream); + // The slot may only be overwritten once the chunk that used it last has left it. + if (chunk_index >= ring.slots()) { destination_done.slot(slot).wait(source.stream); } + CUCASCADE_CUDA_TRY(to_host.enqueue(source.stream)); + source_done.slot(slot).record(source.stream); + } + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{destination.device_id}}; + call_before_leg( + instrumentation, staged_copy_leg::DESTINATION, chunk_index, destination.stream); + source_done.slot(slot).wait(destination.stream); + CUCASCADE_CUDA_TRY(to_device.enqueue(destination.stream)); + destination_done.slot(slot).record(destination.stream); + } + + remaining -= chunk_bytes; + ++chunk_index; + } + + // The destination stream runs its legs in order and each waited for its source leg, so the + // last destination leg finishing means every byte has arrived. + { + nvtx_range const range{"staged:final"}; + destination_done.slot((chunk_index - 1) % ring.slots()).synchronize(); + } + if (instrumentation != nullptr) { ++instrumentation->final_syncs; } + } catch (...) { + // Copies already enqueued still read the source and write the ring and the destination. + bool const source_idle = wait_for_enqueued_work(source, source_done.after_failure()); + bool const destination_idle = + wait_for_enqueued_work(destination, destination_done.after_failure()); + bool const report_failure = instrumentation != nullptr && instrumentation->fail_cleanup.load(); + if (!source_idle || !destination_idle || report_failure) { + ring.abandon(); + if (instrumentation != nullptr) { ++instrumentation->abandoned_rings; } + CUCASCADE_LOG_ERROR( + "staged copy from device {} to device {} failed and its pending work could not be " + "waited for; {} bytes of pinned host memory are not returned", + source.device_id, + destination.device_id, + ring.bytes()); + } + throw; + } +} + +std::vector read_device_integers(std::span probes, + staged_copy_endpoint source, + rmm::device_async_resource_ref scratch_mr, + staged_copy_instrumentation* instrumentation) +{ + if (probes.empty()) { return {}; } + for (auto const& probe : probes) { + if (probe.address == nullptr || + (probe.width != sizeof(std::int32_t) && probe.width != sizeof(std::int64_t))) { + throw std::invalid_argument( + "device integer read: a probe needs an address and a width of 4 or 8 bytes"); + } + } + require_stream_of_device(source, "source"); + + constexpr std::size_t slot_bytes = sizeof(std::int64_t); + std::vector values(probes.size()); + std::vector host(probes.size() * slot_bytes); + + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{source.device_id}}; + cuda::cuda_event gathered; + rmm::device_buffer scratch(host.size(), source.stream, scratch_mr); + cuda::device_copy_batch gather; + gather.reserve(probes.size()); + for (std::size_t i = 0; i < probes.size(); ++i) { + gather.add( + static_cast(scratch.data()) + i * slot_bytes, probes[i].address, probes[i].width); + } + + try { + CUCASCADE_CUDA_TRY(gather.enqueue(source.stream)); + CUCASCADE_CUDA_TRY(cudaMemcpyAsync( + host.data(), scratch.data(), host.size(), cudaMemcpyDeviceToHost, source.stream.get())); + gathered.record(source.stream); + nvtx_range const range{"staged:metadata"}; + gathered.synchronize(); + } catch (...) { + // The scratch buffer and the host bytes are freed when this frame unwinds. + if (!wait_for_enqueued_work(source, gathered)) { + CUCASCADE_LOG_ERROR( + "reading {} integers from device {} failed and the pending copies could not be waited for", + probes.size(), + source.device_id); + } + throw; + } + if (instrumentation != nullptr) { ++instrumentation->metadata_syncs; } + + for (std::size_t i = 0; i < probes.size(); ++i) { + std::byte const* const bytes = host.data() + i * slot_bytes; + if (probes[i].width == sizeof(std::int32_t)) { + std::int32_t narrow = 0; + std::memcpy(&narrow, bytes, sizeof(narrow)); + values[i] = narrow; + } else { + std::memcpy(&values[i], bytes, sizeof(std::int64_t)); + } + } + return values; +} + +} // namespace detail +} // namespace memory +} // namespace cucascade diff --git a/src/memory/staged_copy.hpp b/src/memory/staged_copy.hpp new file mode 100644 index 00000000..cc8e91f0 --- /dev/null +++ b/src/memory/staged_copy.hpp @@ -0,0 +1,220 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file + * Staged copies: moving bytes from one GPU to another through pinned host memory, for device pairs + * that cannot copy to each other directly. + * + * A staged copy has two legs. The source leg copies device memory to host memory on a stream of + * the source device; the destination leg copies that host memory to device memory on a stream of + * the destination device. The bytes travel in chunks through a ring of host slots, so the source + * leg of one chunk runs while the destination leg of an earlier chunk is still in flight. + * + * The two streams are ordered against each other only by events: a destination leg waits for the + * event its source leg recorded, and a source leg that reuses a slot waits for the event of the + * destination leg that last read the slot. Every wait names an event recorded earlier, so staged + * copies in any mix of directions cannot deadlock, and none of them needs a stream to itself. + */ + +#pragma once + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace cucascade { +namespace memory { +namespace detail { + +/** @brief One run of bytes to move from the source device to the destination device. */ +struct staged_copy_segment { + void const* source{nullptr}; ///< First byte, in the source device's memory + void* destination{nullptr}; ///< First byte, in the destination device's memory + std::size_t bytes{0}; ///< Number of bytes; an empty segment is skipped +}; + +/** @brief One side of a staged copy: a device and the stream of that device its leg runs on. */ +struct staged_copy_endpoint { + int device_id{-1}; + ::cuda::stream_ref stream{cudaStream_t{nullptr}}; +}; + +/** @brief Upper bounds on the host memory one staged copy borrows. */ +struct staged_copy_ring_settings { + static constexpr std::size_t default_chunk_bytes = std::size_t{4} << 20; + static constexpr std::size_t default_depth = 4; + static constexpr std::size_t max_chunk_bytes = std::size_t{1} << 30; + static constexpr std::size_t max_depth = 64; + + std::size_t chunk_bytes{default_chunk_bytes}; ///< Most bytes in one chunk + std::size_t depth{default_depth}; ///< Most chunks in flight at once +}; + +/** @brief Shape of a ring: @p slots slots of @p blocks_per_slot host blocks each. */ +struct staged_copy_ring_geometry { + std::size_t slots{0}; + std::size_t blocks_per_slot{0}; + + [[nodiscard]] bool operator==(staged_copy_ring_geometry const&) const = default; +}; + +/** + * @brief The ring a copy of @p total_bytes asks for first. + * + * With block size `b`, chunk capacity `s` (rounded up to whole blocks) and depth `d`, the ring has + * `n = min(d, ceil(total / b))` slots of `min(s, round_up(ceil(total / n), b))` bytes. A ring is + * therefore never larger than `d * s`, nor than the payload rounded up to blocks plus `n - 1` + * blocks, so a small payload borrows about its own size. + * + * @param total_bytes Payload size; must not be zero + * @param block_bytes Size of one host block; must not be zero + * @param settings Chunk capacity and depth + * @throws std::invalid_argument if an argument is zero, a setting exceeds its maximum, or the ring + * size overflows + */ +[[nodiscard]] staged_copy_ring_geometry plan_staged_copy_ring( + std::size_t total_bytes, std::size_t block_bytes, staged_copy_ring_settings const& settings); + +/** + * @brief Step @p geometry to the next smaller ring, for a copy whose ring could not be allocated. + * + * Slots are halved (in whole blocks) down to one block each, then the ring drops to two slots, then + * to one: 4x4 -> 4x2 -> 4x1 -> 2x1 -> 1x1 for the default settings and 1 MiB blocks. + * + * @return false if @p geometry is already one slot of one block, and is left unchanged + */ +[[nodiscard]] bool shrink_staged_copy_ring(staged_copy_ring_geometry& geometry) noexcept; + +/** + * @brief The NUMA node the PCI device of GPU @p device_id is attached to, or -1 if unknown. + * + * A staged copy to that GPU takes its ring from the HOST pool registered for this node. + */ +[[nodiscard]] int gpu_numa_node(int device_id) noexcept; + +/** @brief Which half of a chunk's journey a hook is called for. */ +enum class staged_copy_leg { + SOURCE, ///< Device to host, on the source stream + DESTINATION ///< Host to device, on the destination stream +}; + +/** + * @brief Hooks and counters for tests; production callers pass none. + * + * One object may be shared by concurrent copies. @ref before_leg must be thread-safe, must not + * start a staged copy and must not block on another one. + */ +struct staged_copy_instrumentation { + /// Called before each leg of each chunk is enqueued, with that leg's device current. An + /// exception it throws fails the copy as an enqueue failure would. + std::function + before_leg; + + /// Report the wait after a failed copy as failed, after really waiting, so the ring is abandoned + /// as it would be if the wait had failed. + std::atomic fail_cleanup{false}; + + std::atomic readiness_syncs{0}; ///< Host waits for a source with an unknown writer + std::atomic metadata_syncs{0}; ///< Host waits for sizes read from the device + std::atomic final_syncs{0}; ///< Host waits for the last chunk of a copy + std::atomic rings{0}; ///< Rings acquired + std::atomic last_ring_bytes{0}; ///< Size of the most recently acquired ring + std::atomic abandoned_rings{0}; ///< Rings never given back +}; + +/** + * @brief Copy @p segments from @p source to @p destination through pinned host memory, and wait + * until every byte has arrived. + * + * The ring comes from the HOST pool registered for the destination device's NUMA node (any + * registered pool otherwise), and from a temporary pinned allocation when no pool is registered. + * If the pool cannot supply the ring, smaller rings are tried, down to one block; only then does + * the copy fail. Nothing is copied and no host memory is taken when the segments hold no bytes. + * + * The calling thread waits once, for the last chunk. On return, the ring has been given back and + * no work enqueued by this call is pending on either stream. The caller's current device is + * restored on every exit. + * + * If the copy fails after enqueueing work, it waits for that work before giving the ring back, and + * rethrows the original error. If that wait fails too, the ring is abandoned instead of given back + * (its blocks stay allocated) because device work may still write to it; the same work may still + * read the source and write the destination. + * + * @pre The source bytes are final in the order of @p source's stream: whatever wrote them is + * enqueued on that stream or the stream already waits for it. + * @pre The destination buffers were allocated on @p destination's stream. + * @pre The source and destination buffers, and the HOST pool, outlive the call. + * + * @param segments The byte runs to move, in the order they are sent + * @param source Source device and the stream its legs run on + * @param destination Destination device and the stream its legs run on + * @param settings Bounds for the ring + * @param instrumentation Test hooks and counters, or null + * @throws std::invalid_argument if a non-empty segment has a null address, a stream does not + * belong to its endpoint's device, or @p settings is invalid + * @throws rmm::out_of_memory if not even one host block is available + * @throws cucascade::cuda_error if a CUDA runtime call fails + */ +void staged_copy(std::span segments, + staged_copy_endpoint source, + staged_copy_endpoint destination, + staged_copy_ring_settings const& settings = {}, + staged_copy_instrumentation* instrumentation = nullptr); + +/** @brief Where to read one signed integer of @p width bytes (4 or 8) from device memory. */ +struct device_integer_probe { + void const* address{nullptr}; + std::size_t width{0}; +}; + +/** + * @brief Read one integer per probe from @p source's device with a single wait of the calling + * thread, however many probes there are. + * + * The values are gathered on the device into a scratch buffer, which is then copied to the host in + * one piece. An empty @p probes returns at once without touching the device. + * + * @pre The probed values are final in the order of @p source's stream, as for staged_copy(). + * + * @param probes Addresses in the source device's memory + * @param source Device to read from and the stream to read on + * @param scratch_mr Resource of that device for the scratch buffer + * @param instrumentation Test counters, or null + * @return The values, sign-extended, in the order of @p probes + * @throws std::invalid_argument if a probe has a null address or a width other than 4 or 8, or the + * stream does not belong to the device + * @throws cucascade::cuda_error if a CUDA runtime call fails + */ +[[nodiscard]] std::vector read_device_integers( + std::span probes, + staged_copy_endpoint source, + rmm::device_async_resource_ref scratch_mr, + staged_copy_instrumentation* instrumentation = nullptr); + +} // namespace detail +} // namespace memory +} // namespace cucascade diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6d600130..3eb6f8de 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -37,6 +37,7 @@ if(NOT CUCASCADE_TOPOLOGY_ONLY) memory/test_reservation_aware_resource_adaptor.cpp memory/test_reservation_manager_configurator.cpp memory/test_small_pinned_host_memory_resource.cpp + memory/test_staged_copy.cpp memory/test_gpu_kernels.cu # Data tests data/test_data_repository.cpp @@ -183,8 +184,9 @@ if(NOT CUCASCADE_TOPOLOGY_ONLY AND CUCASCADE_BUILD_CUDF) # Set include directories target_include_directories( - cucascade_cudf_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/utils) + cucascade_cudf_tests + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/utils + ${PROJECT_SOURCE_DIR}/src/cudf ${PROJECT_SOURCE_DIR}/src/memory) # Link dependencies target_link_libraries( diff --git a/test/cudf/test_gpu_to_gpu_converter.cpp b/test/cudf/test_gpu_to_gpu_converter.cpp index a8bf1f62..70ac6ea5 100644 --- a/test/cudf/test_gpu_to_gpu_converter.cpp +++ b/test/cudf/test_gpu_to_gpu_converter.cpp @@ -3,26 +3,42 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "builtin_converters_detail.hpp" #include "cucascade/cudf/builtin_converters.hpp" #include "cucascade/cudf/gpu_data_representation.hpp" #include "cucascade/data/representation_converter.hpp" +#include "cucascade/memory/fixed_size_host_memory_resource.hpp" #include "cucascade/memory/memory_reservation_manager.hpp" #include "cucascade/memory/memory_space.hpp" #include "cucascade/memory/reservation_manager_configurator.hpp" #include "cucascade/utils/cleanup_utils.hpp" +#include "utils/cuda_device_utils.hpp" #include "utils/cuda_test_utils.hpp" #include "utils/cudf_test_utils.hpp" #include "utils/mixed_table.hpp" #include "utils/peer_access_utils.hpp" +#include +#include +#include +#include +#include + +#include +#include #include +#include #include #include +#include #include #include +#include +#include #include +#include namespace { @@ -30,6 +46,7 @@ using namespace cucascade::memory; constexpr uint64_t kKiB = 1024ULL; constexpr uint64_t kMiB = 1024ULL * kKiB; +constexpr uint64_t kGiB = 1024ULL * kMiB; constexpr int kSourceDevice = 0; constexpr int kTargetDevice = 1; @@ -59,15 +76,109 @@ void require_peer_pools(memory_space const* source, memory_space const* destinat } } +// Bytes currently handed out by the HOST spaces of @p manager, which staged copies borrow from. +std::size_t host_bytes_in_use(memory_reservation_manager const& manager) +{ + std::size_t bytes = 0; + for (memory_space const* space : manager.get_memory_spaces_for_tier(Tier::HOST)) { + bytes += space->get_memory_resource_of()->get_total_allocated_bytes(); + } + return bytes; +} + +// Highest number of bytes any HOST space of @p manager has handed out at once. +std::size_t host_peak_bytes(memory_reservation_manager const& manager) +{ + std::size_t bytes = 0; + for (memory_space const* space : manager.get_memory_spaces_for_tier(Tier::HOST)) { + bytes += space->get_memory_resource_of()->get_peak_total_allocated_bytes(); + } + return bytes; +} + +// Convert a mixed table from @p source_device to @p target_device by the host-staged route and +// compare it with the same table built on the target. The two devices may be the same. +void check_staged_conversion(memory_reservation_manager& manager, + int source_device, + int target_device, + cudf::size_type rows) +{ + using cucascade::gpu_table_representation; + CAPTURE(source_device, target_device, rows); + + cucascade::representation_converter_registry registry; + cucascade::detail::register_builtin_converters( + registry, cucascade::detail::gpu_to_gpu_converter_options{.force_staged_route = true}); + + auto* source_space = manager.get_memory_space(Tier::GPU, source_device); + auto* target_space = manager.get_memory_space(Tier::GPU, target_device); + REQUIRE(source_space != nullptr); + REQUIRE(target_space != nullptr); + + // Streams outlive the tables written on them; each is created with its device current. + rmm::cuda_stream producer = [&] { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{source_device}}; + return rmm::cuda_stream{rmm::cuda_stream::flags::non_blocking}; + }(); + rmm::cuda_stream consumer = [&] { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{target_device}}; + return rmm::cuda_stream{rmm::cuda_stream::flags::non_blocking}; + }(); + + std::unique_ptr source; + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{source_device}}; + source = std::make_unique( + cucascade::test::make_mixed_table(rows, producer, source_space->get_default_allocator()), + *source_space, + producer); + } + + std::size_t const host_bytes_before = host_bytes_in_use(manager); + + std::unique_ptr converted; + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{target_device}}; + converted = registry.convert(*source, target_space, consumer); + // The converter switches devices for its two legs and must put the caller's device back. + CHECK(rmm::get_current_cuda_device().value() == target_device); + } + REQUIRE(converted != nullptr); + CHECK(converted->get_device_id() == target_device); + + // The copy went through this manager's HOST pool and gave every staging block back. + CHECK(host_peak_bytes(manager) > host_bytes_before); + CHECK(host_bytes_in_use(manager) == host_bytes_before); + + // The result must not depend on the source after the conversion returns. + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{source_device}}; + source.reset(); + } + + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{target_device}}; + auto expected = + cucascade::test::make_mixed_table(rows, consumer, target_space->get_default_allocator()); + cucascade::test::expect_cudf_tables_equal_on_stream( + expected->view(), converted->get_table_view(), consumer); + expected.reset(); + converted.reset(); + consumer.synchronize(); + } +} + } // namespace namespace cucascade { -TEST_CASE("gpu_to_gpu conversion copies content correctly across GPUs", - "[gpu-to-gpu-converter][multi-device][cross-gpu-correctness]") -{ - if (test::cuda_device_count() < 2) { SKIP("requires at least two GPUs"); } +namespace { +// Two converters read one source whose last write is still held back on the producer stream; +// both must wait for that write. With @p force_staged_route the copies go through host memory, +// otherwise the device pair must have peer access. +void check_conversion_waits_for_writer(bool force_staged_route) +{ reservation_manager_configurator builder; builder.set_gpu_ids({kSourceDevice, kTargetDevice}) .set_gpu_usage_limit(64 * kMiB) @@ -76,11 +187,14 @@ TEST_CASE("gpu_to_gpu conversion copies content correctly across GPUs", .set_host_portability(true); memory_reservation_manager manager(builder.build()); - require_peer_pools(manager.get_memory_space(Tier::GPU, kSourceDevice), - manager.get_memory_space(Tier::GPU, kTargetDevice)); + if (!force_staged_route) { + require_peer_pools(manager.get_memory_space(Tier::GPU, kSourceDevice), + manager.get_memory_space(Tier::GPU, kTargetDevice)); + } representation_converter_registry registry; - register_builtin_converters(registry); + detail::register_builtin_converters( + registry, detail::gpu_to_gpu_converter_options{.force_staged_route = force_staged_route}); rmm::cuda_set_device_raii source_guard{rmm::cuda_device_id{kSourceDevice}}; rmm::cuda_stream source_producer_stream{rmm::cuda_stream::flags::non_blocking}; @@ -166,4 +280,543 @@ TEST_CASE("gpu_to_gpu conversion copies content correctly across GPUs", } } +} // namespace + +TEST_CASE("gpu_to_gpu conversion copies content correctly across GPUs", + "[gpu-to-gpu-converter][multi-device][cross-gpu-correctness]") +{ + if (test::cuda_device_count() < 2) { SKIP("requires at least two GPUs"); } + + check_conversion_waits_for_writer(/*force_staged_route=*/false); +} + +TEST_CASE("gpu_to_gpu staged route waits for the source's writer across GPUs", + "[gpu-to-gpu-converter][staged-copy][multi-device]") +{ + if (test::cuda_device_count() < 2) { SKIP("requires at least two GPUs"); } + + check_conversion_waits_for_writer(/*force_staged_route=*/true); +} + +TEST_CASE("gpu_to_gpu staged route copies a mixed table within one GPU", + "[gpu-to-gpu-converter][staged-copy]") +{ + if (test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + // 200,000 rows give buffers of several MiB, so a buffer spans more than one 1 MiB host block. + auto const rows = GENERATE(cudf::size_type{1}, cudf::size_type{1024}, cudf::size_type{200'000}); + + reservation_manager_configurator builder; + builder.set_gpu_ids({kSourceDevice}) + .set_gpu_usage_limit(1 * kGiB) + .use_numa_id_as_host_id() + .set_per_numa_region_capacity(256 * kMiB) + .set_host_portability(true); + memory_reservation_manager manager(builder.build()); + + check_staged_conversion(manager, kSourceDevice, kSourceDevice, rows); +} + +TEST_CASE("gpu_to_gpu staged route copies a mixed table across GPUs in both directions", + "[gpu-to-gpu-converter][staged-copy][multi-device]") +{ + if (test::cuda_device_count() < 2) { SKIP("requires at least two GPUs"); } + + auto const rows = GENERATE(cudf::size_type{1024}, cudf::size_type{200'000}); + + reservation_manager_configurator builder; + builder.set_gpu_ids({kSourceDevice, kTargetDevice}) + .set_gpu_usage_limit(1 * kGiB) + .use_numa_id_as_host_id() + .set_per_numa_region_capacity(256 * kMiB) + .set_host_portability(true); + memory_reservation_manager manager(builder.build()); + + check_staged_conversion(manager, kSourceDevice, kTargetDevice, rows); + check_staged_conversion(manager, kTargetDevice, kSourceDevice, rows); +} + +namespace { + +// A registry whose GPU -> GPU converter always stages, with the counters of its staged copies. +struct staged_registry { + explicit staged_registry(memory::detail::staged_copy_ring_settings ring_settings = {}) + { + detail::register_builtin_converters( + registry, + detail::gpu_to_gpu_converter_options{ + .force_staged_route = true, .ring_settings = ring_settings, .instrumentation = &counters}); + } + + memory::detail::staged_copy_instrumentation counters; + representation_converter_registry registry; +}; + +auto one_gpu_spaces(std::size_t host_capacity_bytes = 256 * kMiB) +{ + reservation_manager_configurator builder; + builder.set_gpu_ids({kSourceDevice}) + .set_gpu_usage_limit(1 * kGiB) + .use_numa_id_as_host_id() + .set_per_numa_region_capacity(host_capacity_bytes) + .set_host_portability(true); + return builder.build(); +} + +// Convert one mixed table per entry of @p directions (source device, target device), each +// @p repeats times on a thread of its own, all starting together, and compare every result with +// the same table built on its target. Tables are built and compared on the calling thread; only +// the conversions run concurrently. +void check_concurrent_staged_conversions(memory_reservation_manager& manager, + std::vector> const& directions, + cudf::size_type rows, + int repeats) +{ + staged_registry staged; + + struct job { + int source_device; + int target_device; + rmm::cuda_stream producer; + rmm::cuda_stream consumer; + std::unique_ptr source; + std::vector> converted; + }; + auto const make_stream = [](int device) { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{device}}; + return rmm::cuda_stream{rmm::cuda_stream::flags::non_blocking}; + }; + + std::vector> jobs; + for (auto const& [source_device, target_device] : directions) { + auto* const source_space = manager.get_memory_space(Tier::GPU, source_device); + REQUIRE(source_space != nullptr); + REQUIRE(manager.get_memory_space(Tier::GPU, target_device) != nullptr); + jobs.push_back(std::unique_ptr(new job{source_device, + target_device, + make_stream(source_device), + make_stream(target_device), + nullptr, + {}})); + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{source_device}}; + jobs.back()->source = std::make_unique( + test::make_mixed_table(rows, jobs.back()->producer, source_space->get_default_allocator()), + *source_space, + jobs.back()->producer); + } + + std::size_t const host_before = host_bytes_in_use(manager); + + std::latch ready{static_cast(jobs.size())}; + std::latch start{1}; + std::vector> workers; + for (auto& current : jobs) { + workers.push_back(std::async(std::launch::async, [&, work = current.get()] { + ready.count_down(); + start.wait(); + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{work->target_device}}; + auto* const target_space = manager.get_memory_space(Tier::GPU, work->target_device); + for (int i = 0; i < repeats; ++i) { + work->converted.push_back(staged.registry.convert( + *work->source, target_space, work->consumer)); + } + })); + } + ready.wait(); + start.count_down(); + for (auto& worker : workers) { + REQUIRE(worker.wait_for(std::chrono::seconds{120}) == std::future_status::ready); + } + for (auto& worker : workers) { + REQUIRE_NOTHROW(worker.get()); + } + + CHECK(staged.counters.rings.load() == jobs.size() * static_cast(repeats)); + CHECK(staged.counters.abandoned_rings.load() == 0U); + CHECK(host_bytes_in_use(manager) == host_before); + + for (auto& current : jobs) { + CAPTURE(current->source_device, current->target_device); + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{current->source_device}}; + current->source.reset(); + current->producer.synchronize(); + } + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{current->target_device}}; + auto* const target_space = manager.get_memory_space(Tier::GPU, current->target_device); + auto expected = + test::make_mixed_table(rows, current->consumer, target_space->get_default_allocator()); + REQUIRE(current->converted.size() == static_cast(repeats)); + for (auto const& converted : current->converted) { + REQUIRE(converted != nullptr); + test::expect_cudf_tables_equal_on_stream( + expected->view(), converted->get_table_view(), current->consumer); + } + expected.reset(); + current->converted.clear(); + current->consumer.synchronize(); + } +} + +// The six fixed-width columns of test::make_mixed_table(); the other ten have children. +const std::vector kFixedWidthColumns{0, 1, 2, 3, 4, 5}; + +} // namespace + +TEST_CASE("gpu_to_gpu staged route waits a fixed number of times and borrows a bounded ring", + "[gpu-to-gpu-converter][staged-copy]") +{ + if (test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + memory_reservation_manager manager(one_gpu_spaces()); + auto* const space = manager.get_memory_space(Tier::GPU, kSourceDevice); + REQUIRE(space != nullptr); + + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{kSourceDevice}}; + rmm::cuda_stream producer{rmm::cuda_stream::flags::non_blocking}; + rmm::cuda_stream consumer{rmm::cuda_stream::flags::non_blocking}; + ::cuda::stream_ref const producer_ref = producer; + auto const mr = space->get_default_allocator(); + + // Tens of MiB in 16 columns, several of them nested: far more than one ring. + constexpr cudf::size_type kRows = 200'000; + constexpr std::size_t kLargestRing = + staged_copy_settings::default_chunk_bytes * staged_copy_settings::default_ring_depth; + auto table = test::make_mixed_table(kRows, producer, mr); + + staged_registry staged; + std::size_t const host_before = host_bytes_in_use(manager); + std::unique_ptr converted; + + SECTION("a known writer: one wait for the string sizes and one for the last byte") + { + gpu_table_representation source{std::move(table), *space, producer}; + converted = staged.registry.convert(source, space, consumer); + + CHECK(staged.counters.readiness_syncs.load() == 0U); + CHECK(staged.counters.metadata_syncs.load() == 1U); + } + + SECTION("an unknown writer adds one wait for the source device") + { + producer.synchronize(); + gpu_table_representation source{ + std::move(table), *space, ::cuda::stream_ref{cudaStream_t{nullptr}}}; + converted = staged.registry.convert(source, space, consumer); + + CHECK(staged.counters.readiness_syncs.load() == 1U); + CHECK(staged.counters.metadata_syncs.load() == 1U); + } + + SECTION("fixed-width columns need no sizes from the device") + { + gpu_table_representation source{ + std::make_unique(table->select(kFixedWidthColumns), producer_ref, mr), + *space, + producer}; + converted = staged.registry.convert(source, space, consumer); + + CHECK(staged.counters.readiness_syncs.load() == 0U); + CHECK(staged.counters.metadata_syncs.load() == 0U); + test::expect_cudf_tables_equal_on_stream( + table->select(kFixedWidthColumns), converted->get_table_view(), consumer); + } + + REQUIRE(converted != nullptr); + CHECK(converted->get_table_view().num_rows() == kRows); + CHECK(staged.counters.final_syncs.load() == 1U); + CHECK(staged.counters.rings.load() == 1U); + CHECK(staged.counters.last_ring_bytes.load() <= kLargestRing); + CHECK(host_peak_bytes(manager) <= host_before + kLargestRing); + CHECK(host_bytes_in_use(manager) == host_before); + + converted.reset(); + table.reset(); + producer.synchronize(); + consumer.synchronize(); +} + +TEST_CASE("gpu_to_gpu staged route honours its ring settings", + "[gpu-to-gpu-converter][staged-copy]") +{ + if (test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + memory_reservation_manager manager(one_gpu_spaces()); + auto* const space = manager.get_memory_space(Tier::GPU, kSourceDevice); + REQUIRE(space != nullptr); + + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{kSourceDevice}}; + rmm::cuda_stream producer{rmm::cuda_stream::flags::non_blocking}; + rmm::cuda_stream consumer{rmm::cuda_stream::flags::non_blocking}; + auto const mr = space->get_default_allocator(); + + constexpr cudf::size_type kRows = 200'000; + staged_registry staged{{.chunk_bytes = 1 * kMiB, .depth = 2}}; + + auto converted = [&] { + gpu_table_representation source{test::make_mixed_table(kRows, producer, mr), *space, producer}; + return staged.registry.convert(source, space, consumer); + }(); + + // Two slots of one 1 MiB block carry the whole table. + CHECK(staged.counters.last_ring_bytes.load() == 2 * kMiB); + CHECK(staged.counters.final_syncs.load() == 1U); + + auto expected = test::make_mixed_table(kRows, consumer, mr); + test::expect_cudf_tables_equal_on_stream(expected->view(), converted->get_table_view(), consumer); + + expected.reset(); + converted.reset(); + producer.synchronize(); + consumer.synchronize(); +} + +TEST_CASE("gpu_to_gpu staged route builds tables without rows on the host", + "[gpu-to-gpu-converter][staged-copy]") +{ + if (test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + memory_reservation_manager manager(one_gpu_spaces()); + auto* const space = manager.get_memory_space(Tier::GPU, kSourceDevice); + REQUIRE(space != nullptr); + + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{kSourceDevice}}; + rmm::cuda_stream producer{rmm::cuda_stream::flags::non_blocking}; + rmm::cuda_stream consumer{rmm::cuda_stream::flags::non_blocking}; + ::cuda::stream_ref const producer_ref = producer; + auto const mr = space->get_default_allocator(); + + auto owner = std::shared_ptr(test::make_mixed_table(1024, producer, mr)); + producer.synchronize(); + + staged_registry staged; + std::size_t const host_peak_before = host_peak_bytes(manager); + + // Nothing is read from the source, so nothing is waited for and no host memory is borrowed. + auto const check_built_on_host = [&](gpu_table_representation const& source) { + auto converted = staged.registry.convert(source, space, consumer); + REQUIRE(converted != nullptr); + auto const result = converted->get_table_view(); + CHECK(result.num_rows() == 0); + CHECK(result.num_columns() == source.get_table_view().num_columns()); + CHECK(cudf::have_same_types(source.get_table_view(), result)); + CHECK(staged.counters.rings.load() == 0U); + CHECK(staged.counters.readiness_syncs.load() == 0U); + CHECK(staged.counters.metadata_syncs.load() == 0U); + CHECK(staged.counters.final_syncs.load() == 0U); + CHECK(host_peak_bytes(manager) == host_peak_before); + }; + + SECTION("a table without rows whose writer is unknown") + { + gpu_table_representation source{ + cudf::empty_like(owner->view()), *space, ::cuda::stream_ref{cudaStream_t{nullptr}}}; + check_built_on_host(source); + } + + SECTION("views without rows that start past row 0 and keep their children") + { + auto const views = cudf::slice(owner->view(), std::vector{5, 5}, producer_ref); + gpu_table_representation source{ + views.front(), std::shared_ptr{owner}, std::size_t{0}, *space, producer}; + check_built_on_host(source); + } + + SECTION("a table without columns") + { + gpu_table_representation source{std::make_unique(), *space, producer}; + check_built_on_host(source); + } + + owner.reset(); + producer.synchronize(); + consumer.synchronize(); +} + +TEST_CASE("gpu_to_gpu staged route refuses sliced columns it cannot copy whole", + "[gpu-to-gpu-converter][staged-copy]") +{ + if (test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + memory_reservation_manager manager(one_gpu_spaces()); + auto* const space = manager.get_memory_space(Tier::GPU, kSourceDevice); + REQUIRE(space != nullptr); + + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{kSourceDevice}}; + rmm::cuda_stream producer{rmm::cuda_stream::flags::non_blocking}; + rmm::cuda_stream consumer{rmm::cuda_stream::flags::non_blocking}; + ::cuda::stream_ref const producer_ref = producer; + auto const mr = space->get_default_allocator(); + + constexpr cudf::size_type kRows = 1024; + auto owner = std::shared_ptr(test::make_mixed_table(kRows, producer, mr)); + + staged_registry staged; + std::size_t const host_before = host_bytes_in_use(manager); + + auto const convert_view = [&](cudf::table_view const& view) { + gpu_table_representation source{ + view, std::shared_ptr{owner}, std::size_t{0}, *space, producer}; + return staged.registry.convert(source, space, consumer); + }; + + SECTION("a view that starts past row 0, whatever its type") + { + auto const tail = + cudf::slice(owner->view(), std::vector{1, kRows}, producer_ref).front(); + CHECK_THROWS_AS(convert_view(tail), std::invalid_argument); + CHECK_THROWS_AS(convert_view(tail.select(kFixedWidthColumns)), std::invalid_argument); + CHECK(staged.counters.rings.load() == 0U); + } + + SECTION("a view that ends early over a column with children") + { + auto const head = + cudf::slice(owner->view(), std::vector{0, kRows - 1}, producer_ref).front(); + for (cudf::size_type column = 6; column < head.num_columns(); ++column) { + CAPTURE(column); + CHECK_THROWS_AS(convert_view(head.select({column})), std::invalid_argument); + } + CHECK(staged.counters.rings.load() == 0U); + } + + SECTION("a view that ends early over fixed-width columns is copied") + { + auto const head = cudf::slice(owner->view().select(kFixedWidthColumns), + std::vector{0, kRows - 1}, + producer_ref) + .front(); + auto converted = convert_view(head); + REQUIRE(converted != nullptr); + test::expect_cudf_tables_equal_on_stream(head, converted->get_table_view(), consumer); + } + + CHECK(host_bytes_in_use(manager) == host_before); + + owner.reset(); + producer.synchronize(); + consumer.synchronize(); +} + +TEST_CASE("gpu_to_gpu staged route runs concurrent conversions within one GPU", + "[gpu-to-gpu-converter][staged-copy]") +{ + if (test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + memory_reservation_manager manager(one_gpu_spaces()); + + // Four conversions at a time share the device's stream pool and the HOST pool. + check_concurrent_staged_conversions(manager, + {{kSourceDevice, kSourceDevice}, + {kSourceDevice, kSourceDevice}, + {kSourceDevice, kSourceDevice}, + {kSourceDevice, kSourceDevice}}, + 50'000, + 3); +} + +TEST_CASE("gpu_to_gpu staged route runs concurrent conversions in opposite directions", + "[gpu-to-gpu-converter][staged-copy][multi-device]") +{ + if (test::cuda_device_count() < 2) { SKIP("requires at least two GPUs"); } + + reservation_manager_configurator builder; + builder.set_gpu_ids({kSourceDevice, kTargetDevice}) + .set_gpu_usage_limit(1 * kGiB) + .use_numa_id_as_host_id() + .set_per_numa_region_capacity(256 * kMiB) + .set_host_portability(true); + memory_reservation_manager manager(builder.build()); + + // Each device is the source of two conversions and the target of two others at once. + check_concurrent_staged_conversions(manager, + {{kSourceDevice, kTargetDevice}, + {kTargetDevice, kSourceDevice}, + {kSourceDevice, kTargetDevice}, + {kTargetDevice, kSourceDevice}}, + 50'000, + 3); +} + +TEST_CASE("gpu_to_gpu staged route copes with a HOST pool smaller than its ring", + "[gpu-to-gpu-converter][staged-copy]") +{ + if (test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + // Half of what the default ring of four 4 MiB chunks asks for. + constexpr std::size_t kHostCapacity = 8 * kMiB; + memory_reservation_manager manager(one_gpu_spaces(kHostCapacity)); + auto* const space = manager.get_memory_space(Tier::GPU, kSourceDevice); + REQUIRE(space != nullptr); + + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{kSourceDevice}}; + rmm::cuda_stream producer{rmm::cuda_stream::flags::non_blocking}; + rmm::cuda_stream consumer{rmm::cuda_stream::flags::non_blocking}; + auto const mr = space->get_default_allocator(); + + constexpr cudf::size_type kRows = 200'000; + staged_registry staged; + gpu_table_representation source{test::make_mixed_table(kRows, producer, mr), *space, producer}; + auto expected = test::make_mixed_table(kRows, consumer, mr); + + SECTION("it converts through a smaller ring") + { + auto converted = staged.registry.convert(source, space, consumer); + REQUIRE(converted != nullptr); + CHECK(staged.counters.last_ring_bytes.load() <= kHostCapacity); + CHECK(staged.counters.last_ring_bytes.load() > 0U); + CHECK(host_bytes_in_use(manager) == 0U); + test::expect_cudf_tables_equal_on_stream( + expected->view(), converted->get_table_view(), consumer); + } + + SECTION("it fails with out-of-memory when no block is left, and works again afterwards") + { + using held_blocks = fixed_size_host_memory_resource::fixed_multiple_blocks_allocation; + std::vector held; + for (memory_space const* host_space : manager.get_memory_spaces_for_tier(Tier::HOST)) { + auto* const pool = host_space->get_memory_resource_of(); + held.push_back(pool->allocate_multiple_blocks(pool->get_available_memory())); + } + std::size_t const host_held = host_bytes_in_use(manager); + + CHECK_THROWS_AS(staged.registry.convert(source, space, consumer), + rmm::out_of_memory); + CHECK(staged.counters.rings.load() == 0U); + CHECK(host_bytes_in_use(manager) == host_held); + + held.clear(); + auto converted = staged.registry.convert(source, space, consumer); + REQUIRE(converted != nullptr); + CHECK(host_bytes_in_use(manager) == 0U); + test::expect_cudf_tables_equal_on_stream( + expected->view(), converted->get_table_view(), consumer); + } + + expected.reset(); + producer.synchronize(); + consumer.synchronize(); +} + +TEST_CASE("staged copy settings reject values outside their range", + "[gpu-to-gpu-converter][staged-copy]") +{ + CHECK(staged_copy_settings{}.chunk_bytes() == std::size_t{4} << 20); + CHECK(staged_copy_settings{}.ring_depth() == 4U); + + CHECK_NOTHROW(staged_copy_settings{1, 1}); + CHECK_NOTHROW(staged_copy_settings{staged_copy_settings::max_chunk_bytes, + staged_copy_settings::max_ring_depth}); + CHECK_THROWS_AS((staged_copy_settings{0, 4}), std::invalid_argument); + CHECK_THROWS_AS((staged_copy_settings{4 * kMiB, 0}), std::invalid_argument); + CHECK_THROWS_AS((staged_copy_settings{staged_copy_settings::max_chunk_bytes + 1, 4}), + std::invalid_argument); + CHECK_THROWS_AS((staged_copy_settings{4 * kMiB, staged_copy_settings::max_ring_depth + 1}), + std::invalid_argument); + + representation_converter_registry registry; + CHECK_NOTHROW(register_builtin_converters(registry, staged_copy_settings{1 * kMiB, 2})); + CHECK(registry.has_converter()); +} + } // namespace cucascade diff --git a/test/memory/test_staged_copy.cpp b/test/memory/test_staged_copy.cpp new file mode 100644 index 00000000..a243eab9 --- /dev/null +++ b/test/memory/test_staged_copy.cpp @@ -0,0 +1,738 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test Tags: + * [staged-copy] - copies between GPUs through pinned host memory + * [multi-device] - needs two GPUs and skips otherwise + * + * Each test that needs a HOST pool creates its own and checks that it is the only one registered, + * so that the accounting it asserts on is the accounting the copy used. + */ + +#include "utils/cuda_device_utils.hpp" +#include "utils/cuda_test_utils.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace cucascade::memory; +using namespace cucascade::memory::detail; + +constexpr std::size_t kKiB = 1024; +constexpr std::size_t kMiB = 1024 * kKiB; +constexpr std::size_t kBlock = 64 * kKiB; + +// A HOST space with small blocks, so a few hundred KiB already cross many block and slot +// boundaries. It is registered for the NUMA node of @p destination_device and is the most recent +// registration there, which makes its pool the one a staged copy to that device selects. +std::unique_ptr make_host_space(std::size_t capacity_bytes, + int destination_device = 0) +{ + host_memory_space_config config; + config.numa_id = gpu_numa_node(destination_device); + config.memory_capacity = capacity_bytes; + config.block_size = kBlock; + config.pool_size = 64; + config.initial_number_pools = 1; + config.make_portable = true; + auto space = std::make_unique(config); + + auto* const pool = space->get_memory_resource_of(); + REQUIRE(pool != nullptr); + REQUIRE(pool->get_block_size() == kBlock); + REQUIRE(find_host_pool(config.numa_id) == pool); + return space; +} + +rmm::cuda_stream make_stream(int device) +{ + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{device}}; + return rmm::cuda_stream{rmm::cuda_stream::flags::non_blocking}; +} + +std::vector pattern(std::size_t bytes, std::uint32_t seed) +{ + std::vector values(bytes); + std::uint32_t state = seed * 2654435761U + 12345U; + for (auto& value : values) { + state = state * 1664525U + 1013904223U; + value = static_cast(state >> 24); + } + return values; +} + +// Device memory owned by the test, independent of any memory resource under test. +class device_bytes { + public: + device_bytes(int device, std::size_t bytes) : _device(device), _bytes(bytes) + { + if (bytes == 0) { return; } + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{device}}; + REQUIRE(cudaMalloc(&_data, bytes) == cudaSuccess); + REQUIRE(cudaMemset(_data, 0, bytes) == cudaSuccess); + } + + ~device_bytes() + { + if (_data == nullptr) { return; } + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{_device}}; + (void)cudaFree(_data); + } + + device_bytes(device_bytes&& other) noexcept + : _device(other._device), _bytes(other._bytes), _data(std::exchange(other._data, nullptr)) + { + } + device_bytes(device_bytes const&) = delete; + device_bytes& operator=(device_bytes const&) = delete; + device_bytes& operator=(device_bytes&&) = delete; + + [[nodiscard]] void* data() const noexcept { return _data; } + [[nodiscard]] std::size_t size() const noexcept { return _bytes; } + + void write(std::vector const& values) + { + REQUIRE(values.size() == _bytes); + if (_bytes == 0) { return; } + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{_device}}; + REQUIRE(cudaMemcpy(_data, values.data(), _bytes, cudaMemcpyHostToDevice) == cudaSuccess); + } + + // Write @p values at byte @p offset. + void write_at(std::size_t offset, std::vector const& values) + { + REQUIRE(offset + values.size() <= _bytes); + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{_device}}; + REQUIRE(cudaMemcpy(static_cast(_data) + offset, + values.data(), + values.size(), + cudaMemcpyHostToDevice) == cudaSuccess); + } + + // Read @p bytes starting at byte @p offset. + [[nodiscard]] std::vector read_at(std::size_t offset, std::size_t bytes) const + { + REQUIRE(offset + bytes <= _bytes); + std::vector values(bytes); + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{_device}}; + REQUIRE(cudaMemcpy(values.data(), + static_cast(_data) + offset, + bytes, + cudaMemcpyDeviceToHost) == cudaSuccess); + return values; + } + + [[nodiscard]] std::vector read() const + { + std::vector values(_bytes); + if (_bytes == 0) { return values; } + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{_device}}; + REQUIRE(cudaMemcpy(values.data(), _data, _bytes, cudaMemcpyDeviceToHost) == cudaSuccess); + return values; + } + + private: + int _device; + std::size_t _bytes; + void* _data{nullptr}; +}; + +// Source buffers with known bytes on one device and zeroed buffers of the same sizes on another. +class payload { + public: + payload(int source_device, int destination_device, std::vector const& sizes) + { + _sources.reserve(sizes.size()); + _destinations.reserve(sizes.size()); + for (std::size_t i = 0; i < sizes.size(); ++i) { + _expected.push_back(pattern(sizes[i], static_cast(i + 1))); + _sources.emplace_back(source_device, sizes[i]); + _sources.back().write(_expected.back()); + _destinations.emplace_back(destination_device, sizes[i]); + } + } + + [[nodiscard]] std::vector segments() const + { + std::vector result; + for (std::size_t i = 0; i < _sources.size(); ++i) { + result.push_back({_sources[i].data(), _destinations[i].data(), _sources[i].size()}); + } + return result; + } + + [[nodiscard]] std::size_t bytes() const + { + std::size_t total = 0; + for (auto const& source : _sources) { + total += source.size(); + } + return total; + } + + // Every destination byte equals its source byte. + void check_arrived() const + { + for (std::size_t i = 0; i < _destinations.size(); ++i) { + auto const actual = _destinations[i].read(); + auto const first_difference = + std::mismatch(actual.begin(), actual.end(), _expected[i].begin()).first - actual.begin(); + // Compared outside the assertion so that a failure does not print the buffers. + bool const arrived = actual == _expected[i]; + CAPTURE(i, actual.size(), first_difference); + CHECK(arrived); + } + } + + private: + std::vector> _expected; + std::vector _sources; + std::vector _destinations; +}; + +std::size_t ring_bytes(staged_copy_ring_geometry const& geometry, std::size_t block_bytes) +{ + return geometry.slots * geometry.blocks_per_slot * block_bytes; +} + +} // namespace + +TEST_CASE("staged copy ring is sized to the payload", "[staged-copy]") +{ + staged_copy_ring_settings const defaults{}; + + SECTION("documented examples for 1 MiB blocks") + { + CHECK(plan_staged_copy_ring(64 * kKiB, kMiB, defaults) == staged_copy_ring_geometry{1, 1}); + CHECK(plan_staged_copy_ring(4 * kMiB, kMiB, defaults) == staged_copy_ring_geometry{4, 1}); + CHECK(plan_staged_copy_ring(4 * kMiB + 1, kMiB, defaults) == staged_copy_ring_geometry{4, 2}); + CHECK(plan_staged_copy_ring(6 * kMiB, kMiB, defaults) == staged_copy_ring_geometry{4, 2}); + CHECK(plan_staged_copy_ring(1024 * kMiB, kMiB, defaults) == staged_copy_ring_geometry{4, 4}); + } + + SECTION("a ring holds a chunk per slot and never exceeds its bounds") + { + auto const block = GENERATE(std::size_t{4096}, std::size_t{64 * kKiB}, std::size_t{kMiB}); + auto const settings = + GENERATE(staged_copy_ring_settings{}, + staged_copy_ring_settings{.chunk_bytes = 1, .depth = 1}, + staged_copy_ring_settings{.chunk_bytes = 3 * kMiB + 5, .depth = 7}, + staged_copy_ring_settings{.chunk_bytes = staged_copy_ring_settings::max_chunk_bytes, + .depth = staged_copy_ring_settings::max_depth}); + auto const payload_bytes = GENERATE(std::size_t{1}, + std::size_t{4095}, + std::size_t{kMiB - 1}, + std::size_t{kMiB}, + std::size_t{kMiB + 1}, + std::size_t{4 * kMiB}, + std::size_t{4 * kMiB + 1}, + std::size_t{37 * kMiB + 3}, + std::size_t{5} << 32); + CAPTURE(block, settings.chunk_bytes, settings.depth, payload_bytes); + + auto const geometry = plan_staged_copy_ring(payload_bytes, block, settings); + auto const round_up = [block](std::size_t bytes) { + return (bytes + block - 1) / block * block; + }; + + CHECK(geometry.slots >= 1); + CHECK(geometry.slots <= settings.depth); + CHECK(geometry.blocks_per_slot >= 1); + CHECK(geometry.blocks_per_slot * block <= round_up(settings.chunk_bytes)); + CHECK(ring_bytes(geometry, block) <= round_up(payload_bytes) + (geometry.slots - 1) * block); + // Either the whole payload fits in the ring, or every slot holds a full chunk. + CHECK((ring_bytes(geometry, block) >= payload_bytes || + geometry.blocks_per_slot * block == round_up(settings.chunk_bytes))); + } + + SECTION("invalid arguments are rejected") + { + using settings = staged_copy_ring_settings; + CHECK_THROWS_AS(plan_staged_copy_ring(0, kMiB, defaults), std::invalid_argument); + CHECK_THROWS_AS(plan_staged_copy_ring(1, 0, defaults), std::invalid_argument); + CHECK_THROWS_AS(plan_staged_copy_ring(1, kMiB, settings{.chunk_bytes = 0, .depth = 4}), + std::invalid_argument); + CHECK_THROWS_AS(plan_staged_copy_ring(1, kMiB, settings{.chunk_bytes = kMiB, .depth = 0}), + std::invalid_argument); + CHECK_THROWS_AS(plan_staged_copy_ring( + 1, kMiB, settings{.chunk_bytes = settings::max_chunk_bytes + 1, .depth = 4}), + std::invalid_argument); + CHECK_THROWS_AS(plan_staged_copy_ring( + 1, kMiB, settings{.chunk_bytes = kMiB, .depth = settings::max_depth + 1}), + std::invalid_argument); + } +} + +TEST_CASE("staged copy ring shrinks slot size first, then slot count", "[staged-copy]") +{ + staged_copy_ring_geometry geometry{4, 4}; + REQUIRE(shrink_staged_copy_ring(geometry)); + CHECK(geometry == staged_copy_ring_geometry{4, 2}); + REQUIRE(shrink_staged_copy_ring(geometry)); + CHECK(geometry == staged_copy_ring_geometry{4, 1}); + REQUIRE(shrink_staged_copy_ring(geometry)); + CHECK(geometry == staged_copy_ring_geometry{2, 1}); + REQUIRE(shrink_staged_copy_ring(geometry)); + CHECK(geometry == staged_copy_ring_geometry{1, 1}); + CHECK_FALSE(shrink_staged_copy_ring(geometry)); + CHECK(geometry == staged_copy_ring_geometry{1, 1}); + + staged_copy_ring_geometry odd{3, 5}; + REQUIRE(shrink_staged_copy_ring(odd)); + CHECK(odd == staged_copy_ring_geometry{3, 2}); + REQUIRE(shrink_staged_copy_ring(odd)); + CHECK(odd == staged_copy_ring_geometry{3, 1}); + REQUIRE(shrink_staged_copy_ring(odd)); + CHECK(odd == staged_copy_ring_geometry{2, 1}); + + staged_copy_ring_geometry single{1, 4}; + REQUIRE(shrink_staged_copy_ring(single)); + CHECK(single == staged_copy_ring_geometry{1, 2}); + REQUIRE(shrink_staged_copy_ring(single)); + CHECK(single == staged_copy_ring_geometry{1, 1}); + CHECK_FALSE(shrink_staged_copy_ring(single)); +} + +TEST_CASE("staged copy moves segments of any size within one GPU", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + auto host = make_host_space(64 * kMiB); + auto* const pool = host->get_memory_resource_of(); + auto source = make_stream(0); + auto destination = make_stream(0); + + // Sizes on both sides of a block boundary, an empty segment in the middle and one segment + // spanning several blocks; the bytes after it share a slot with its tail. + payload data{0, 0, {1, 3, kBlock - 1, kBlock, 0, kBlock + 1, 5 * kBlock + 7, 2}}; + auto const segments = data.segments(); + + std::size_t const in_use_before = pool->get_total_allocated_bytes(); + staged_copy_instrumentation counters; + staged_copy(segments, {0, source}, {0, destination}, staged_copy_ring_settings{}, &counters); + + data.check_arrived(); + CHECK(counters.final_syncs.load() == 1U); + CHECK(counters.rings.load() == 1U); + CHECK(counters.abandoned_rings.load() == 0U); + CHECK( + counters.last_ring_bytes.load() == + ring_bytes(plan_staged_copy_ring(data.bytes(), kBlock, staged_copy_ring_settings{}), kBlock)); + CHECK(pool->get_peak_total_allocated_bytes() >= in_use_before + counters.last_ring_bytes.load()); + CHECK(pool->get_total_allocated_bytes() == in_use_before); +} + +TEST_CASE("staged copy reuses its slots for a payload larger than the ring", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + auto host = make_host_space(64 * kMiB); + auto* const pool = host->get_memory_resource_of(); + auto source = make_stream(0); + auto destination = make_stream(0); + + // Two slots of two blocks each carry 37 blocks and a few bytes: nine and a bit trips round. + staged_copy_ring_settings const settings{.chunk_bytes = 2 * kBlock, .depth = 2}; + payload data{0, 0, {11 * kBlock + 5, 3, 26 * kBlock + 3}}; + auto const segments = data.segments(); + + std::size_t const in_use_before = pool->get_total_allocated_bytes(); + staged_copy_instrumentation counters; + staged_copy(segments, {0, source}, {0, destination}, settings, &counters); + + data.check_arrived(); + CHECK(counters.last_ring_bytes.load() == 4 * kBlock); + CHECK(counters.final_syncs.load() == 1U); + CHECK(pool->get_total_allocated_bytes() == in_use_before); +} + +TEST_CASE("staged copy of no bytes takes no memory and does not wait", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + auto host = make_host_space(64 * kMiB); + auto* const pool = host->get_memory_resource_of(); + auto source = make_stream(0); + auto destination = make_stream(0); + + std::size_t const peak_before = pool->get_peak_total_allocated_bytes(); + staged_copy_instrumentation counters; + + staged_copy({}, {0, source}, {0, destination}, staged_copy_ring_settings{}, &counters); + std::vector const empty_segments{{nullptr, nullptr, 0}, + {nullptr, nullptr, 0}}; + staged_copy( + empty_segments, {0, source}, {0, destination}, staged_copy_ring_settings{}, &counters); + + CHECK(counters.rings.load() == 0U); + CHECK(counters.final_syncs.load() == 0U); + CHECK(pool->get_peak_total_allocated_bytes() == peak_before); +} + +TEST_CASE("staged copy keeps a slot until its destination leg has read it", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + auto host = make_host_space(64 * kMiB); + auto source = make_stream(0); + auto destination = make_stream(0); + REQUIRE(source.value() != destination.value()); + + // Two one-block slots and twelve chunks. The destination stream is parked before its first + // leg, so a source leg that did not wait for its slot would overwrite chunks 0 and 1 with later + // ones long before they are read. + staged_copy_ring_settings const settings{.chunk_bytes = kBlock, .depth = 2}; + payload data{0, 0, {12 * kBlock}}; + auto const segments = data.segments(); + + cucascade::test::stream_gate gate{destination.value()}; + staged_copy_instrumentation hooks; + hooks.before_leg = [&](staged_copy_leg leg, std::size_t chunk_index, ::cuda::stream_ref) { + if (leg == staged_copy_leg::DESTINATION && chunk_index == 0) { + gate.arm(); + gate.release_after(std::chrono::milliseconds{200}); + } + }; + + staged_copy(segments, {0, source}, {0, destination}, settings, &hooks); + + data.check_arrived(); + CHECK(hooks.last_ring_bytes.load() == 2 * kBlock); +} + +TEST_CASE("staged copy waits for its enqueued work when a leg fails", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + auto host = make_host_space(64 * kMiB); + auto* const pool = host->get_memory_resource_of(); + auto source = make_stream(0); + auto destination = make_stream(0); + + staged_copy_ring_settings const settings{.chunk_bytes = kBlock, .depth = 2}; + payload data{0, 0, {12 * kBlock}}; + auto const segments = data.segments(); + + auto const failing_leg = GENERATE(staged_copy_leg::SOURCE, staged_copy_leg::DESTINATION); + staged_copy_instrumentation hooks; + hooks.before_leg = [&](staged_copy_leg leg, std::size_t chunk_index, ::cuda::stream_ref) { + if (leg == failing_leg && chunk_index == 5) { throw std::runtime_error{"injected"}; } + }; + + std::size_t const in_use_before = pool->get_total_allocated_bytes(); + + SECTION("the ring is given back once the work is known to be complete") + { + CHECK_THROWS_WITH(staged_copy(segments, {0, source}, {0, destination}, settings, &hooks), + "injected"); + CHECK(hooks.abandoned_rings.load() == 0U); + CHECK(pool->get_total_allocated_bytes() == in_use_before); + + // The streams and the pool are usable afterwards. + hooks.before_leg = nullptr; + staged_copy(segments, {0, source}, {0, destination}, settings, &hooks); + data.check_arrived(); + CHECK(pool->get_total_allocated_bytes() == in_use_before); + } + + SECTION("the ring is abandoned when that cannot be confirmed") + { + hooks.fail_cleanup = true; + CHECK_THROWS_WITH(staged_copy(segments, {0, source}, {0, destination}, settings, &hooks), + "injected"); + CHECK(hooks.abandoned_rings.load() == 1U); + CHECK(hooks.last_ring_bytes.load() == 2 * kBlock); + CHECK(pool->get_total_allocated_bytes() == in_use_before + 2 * kBlock); + } +} + +TEST_CASE("staged copy moves a segment larger than 2 GiB", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + constexpr std::size_t kBytes = (std::size_t{2} << 30) + 3 * kMiB + 5; + { + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{0}}; + std::size_t free_bytes = 0; + std::size_t total_bytes = 0; + REQUIRE(cudaMemGetInfo(&free_bytes, &total_bytes) == cudaSuccess); + if (free_bytes < 3 * kBytes) { SKIP("requires three times the segment in free device memory"); } + } + + auto host = make_host_space(64 * kMiB); + auto* const pool = host->get_memory_resource_of(); + auto source = make_stream(0); + auto destination = make_stream(0); + + // Both buffers start zeroed. Known bytes go where an offset could go wrong: the start, the + // first chunk boundary, the first trip round the default ring, 512 MiB and 2 GiB (where a + // 32-bit count of bits and of bytes overflow), and the tail. + device_bytes from{0, kBytes}; + device_bytes to{0, kBytes}; + constexpr std::size_t kWindow = 128 * kKiB; + std::vector const windows{0, + 4 * kMiB - kWindow / 2, + 16 * kMiB - kWindow / 2, + (std::size_t{1} << 29) - kWindow / 2, + (std::size_t{1} << 31) - kWindow / 2, + kBytes - kWindow}; + for (std::size_t i = 0; i < windows.size(); ++i) { + from.write_at(windows[i], pattern(kWindow, static_cast(100 + i))); + } + + std::vector const segments{{from.data(), to.data(), kBytes}}; + staged_copy_instrumentation counters; + staged_copy(segments, {0, source}, {0, destination}, staged_copy_ring_settings{}, &counters); + + CHECK(counters.final_syncs.load() == 1U); + CHECK(counters.last_ring_bytes.load() == 16 * kMiB); + CHECK(pool->get_total_allocated_bytes() == 0U); + for (std::size_t i = 0; i < windows.size(); ++i) { + CAPTURE(i, windows[i]); + bool const arrived = + to.read_at(windows[i], kWindow) == pattern(kWindow, static_cast(100 + i)); + CHECK(arrived); + } + // Between the windows the source is zero, and so must the destination be: nothing was written + // twice or to the wrong place. + for (std::size_t offset : {std::size_t{1} * kMiB, std::size_t{700} * kMiB, kBytes - 2 * kMiB}) { + CAPTURE(offset); + bool const untouched = to.read_at(offset, kWindow) == std::vector(kWindow); + CHECK(untouched); + } +} + +TEST_CASE("staged copy takes a smaller ring when the pool is short", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + // Room for three blocks: rings of 16, 8 and 4 blocks do not fit, one of 2 does. + auto host = make_host_space(3 * kBlock); + auto* const pool = host->get_memory_resource_of(); + auto source = make_stream(0); + auto destination = make_stream(0); + + staged_copy_ring_settings const settings{.chunk_bytes = 4 * kBlock, .depth = 4}; + payload data{0, 0, {40 * kBlock + 9}}; + auto const segments = data.segments(); + staged_copy_instrumentation counters; + + SECTION("it copies through the ring that fits") + { + staged_copy(segments, {0, source}, {0, destination}, settings, &counters); + data.check_arrived(); + CHECK(counters.last_ring_bytes.load() == 2 * kBlock); + CHECK(pool->get_total_allocated_bytes() == 0U); + } + + SECTION("it fails only when not even one block is left") + { + auto const held = pool->allocate_multiple_blocks(3 * kBlock); + CHECK_THROWS_AS(staged_copy(segments, {0, source}, {0, destination}, settings, &counters), + rmm::out_of_memory); + CHECK(counters.rings.load() == 0U); + CHECK(pool->get_total_allocated_bytes() == 3 * kBlock); + } +} + +TEST_CASE("staged copy without a HOST pool pins memory for the call", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + if (find_host_pool(gpu_numa_node(0)) != nullptr) { SKIP("a HOST pool is registered"); } + + auto source = make_stream(0); + auto destination = make_stream(0); + + staged_copy_ring_settings const settings{.chunk_bytes = kMiB, .depth = 2}; + payload data{0, 0, {5 * kMiB + 3, 17}}; + auto const segments = data.segments(); + staged_copy_instrumentation counters; + + staged_copy(segments, {0, source}, {0, destination}, settings, &counters); + + data.check_arrived(); + CHECK(counters.last_ring_bytes.load() == 2 * kMiB); + CHECK(counters.abandoned_rings.load() == 0U); +} + +TEST_CASE("staged copy rejects segments it cannot copy", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + auto host = make_host_space(64 * kMiB); + auto source = make_stream(0); + auto destination = make_stream(0); + payload data{0, 0, {kBlock}}; + auto segments = data.segments(); + staged_copy_instrumentation counters; + + SECTION("a null source") + { + segments[0].source = nullptr; + CHECK_THROWS_AS(staged_copy(segments, {0, source}, {0, destination}, {}, &counters), + std::invalid_argument); + } + SECTION("a null destination") + { + segments[0].destination = nullptr; + CHECK_THROWS_AS(staged_copy(segments, {0, source}, {0, destination}, {}, &counters), + std::invalid_argument); + } + SECTION("settings out of range") + { + CHECK_THROWS_AS(staged_copy(segments, + {0, source}, + {0, destination}, + staged_copy_ring_settings{.chunk_bytes = kBlock, .depth = 0}, + &counters), + std::invalid_argument); + } + CHECK(counters.rings.load() == 0U); +} + +TEST_CASE("staged copy moves bytes between two GPUs in both directions", + "[staged-copy][multi-device]") +{ + if (cucascade::test::cuda_device_count() < 2) { SKIP("requires at least two GPUs"); } + + auto const [source_device, destination_device] = + GENERATE(std::pair{0, 1}, std::pair{1, 0}); + CAPTURE(source_device, destination_device); + + auto host = make_host_space(64 * kMiB, destination_device); + auto* const pool = host->get_memory_resource_of(); + + auto source = make_stream(source_device); + auto destination = make_stream(destination_device); + + staged_copy_ring_settings const settings{.chunk_bytes = 3 * kBlock, .depth = 3}; + payload data{source_device, destination_device, {7, 20 * kBlock + 1, kBlock - 1, 9 * kBlock}}; + auto const segments = data.segments(); + staged_copy_instrumentation counters; + + // Neither leg's device is the caller's. + int const caller_device = destination_device; + rmm::cuda_set_device_raii const caller{rmm::cuda_device_id{caller_device}}; + + SECTION("each leg runs on a stream of its own device") + { + staged_copy( + segments, {source_device, source}, {destination_device, destination}, settings, &counters); + + CHECK(rmm::get_current_cuda_device().value() == caller_device); + data.check_arrived(); + CHECK(counters.final_syncs.load() == 1U); + CHECK(pool->get_total_allocated_bytes() == 0U); + } + + SECTION("a stream of the other device is rejected before anything is enqueued") + { + CHECK_THROWS_AS(staged_copy(segments, + {source_device, destination}, + {destination_device, destination}, + settings, + &counters), + std::invalid_argument); + CHECK_THROWS_AS( + staged_copy( + segments, {source_device, source}, {destination_device, source}, settings, &counters), + std::invalid_argument); + CHECK(counters.rings.load() == 0U); + CHECK(rmm::get_current_cuda_device().value() == caller_device); + } +} + +TEST_CASE("device integers of both widths are read with one wait", "[staged-copy]") +{ + if (cucascade::test::cuda_device_count() < 1) { SKIP("requires a GPU"); } + + auto stream = make_stream(0); + rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{0}}; + auto const scratch_mr = rmm::mr::get_current_device_resource_ref(); + staged_copy_instrumentation counters; + + SECTION("no probes, no device work") + { + CHECK(read_device_integers({}, {0, stream}, scratch_mr, &counters).empty()); + CHECK(counters.metadata_syncs.load() == 0U); + } + + SECTION("values keep their sign and their order") + { + std::vector const narrow{7, -3, 2'000'000'000}; + std::vector const wide{-1, 5'000'000'000LL, 42}; + device_bytes narrow_device{0, narrow.size() * sizeof(std::int32_t)}; + device_bytes wide_device{0, wide.size() * sizeof(std::int64_t)}; + REQUIRE(cudaMemcpy( + narrow_device.data(), narrow.data(), narrow_device.size(), cudaMemcpyHostToDevice) == + cudaSuccess); + REQUIRE( + cudaMemcpy(wide_device.data(), wide.data(), wide_device.size(), cudaMemcpyHostToDevice) == + cudaSuccess); + + auto const* const narrow_values = static_cast(narrow_device.data()); + auto const* const wide_values = static_cast(wide_device.data()); + std::vector const probes{{wide_values + 1, sizeof(std::int64_t)}, + {narrow_values + 1, sizeof(std::int32_t)}, + {narrow_values + 2, sizeof(std::int32_t)}, + {wide_values, sizeof(std::int64_t)}, + {narrow_values, sizeof(std::int32_t)}, + {wide_values + 2, sizeof(std::int64_t)}}; + + auto const values = read_device_integers(probes, {0, stream}, scratch_mr, &counters); + + CHECK(values == std::vector{5'000'000'000LL, -3, 2'000'000'000, -1, 7, 42}); + CHECK(counters.metadata_syncs.load() == 1U); + } + + SECTION("a probe needs an address and a width of 4 or 8") + { + std::int64_t const value = 0; + std::vector const no_address{{nullptr, sizeof(std::int64_t)}}; + std::vector const odd_width{{&value, 2}}; + CHECK_THROWS_AS(read_device_integers(no_address, {0, stream}, scratch_mr, &counters), + std::invalid_argument); + CHECK_THROWS_AS(read_device_integers(odd_width, {0, stream}, scratch_mr, &counters), + std::invalid_argument); + CHECK(counters.metadata_syncs.load() == 0U); + } +}