Skip to content
Closed
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,9 @@ if(NOT CUCASCADE_TOPOLOGY_ONLY AND CUCASCADE_BUILD_CUDF)
cudf::cudf)
target_include_directories(cucascade_cudf_objects
PUBLIC ${CUCASCADE_PUBLIC_INCLUDE_DIRS})
# Internal headers of the core library that the converters build on.
target_include_directories(cucascade_cudf_objects
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/memory)
target_compile_definitions(cucascade_cudf_objects
PUBLIC ${CUCASCADE_PUBLIC_COMPILE_DEFS})
target_compile_features(cucascade_cudf_objects PUBLIC cxx_std_20)
Expand Down
1 change: 1 addition & 0 deletions benchmark/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ add_executable(cucascade_benchmarks ${BENCHMARK_SOURCES})
target_include_directories(
cucascade_benchmarks
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../test
${PROJECT_SOURCE_DIR}/src/cudf ${PROJECT_SOURCE_DIR}/src/memory
${benchmark_SOURCE_DIR}/include)

# Apply warning flags to benchmark executable. Suppress -Wnull-dereference:
Expand Down
279 changes: 266 additions & 13 deletions benchmark/benchmark_gpu_to_gpu_converter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,13 @@
// The trailing wait on each result's writer event keeps the measurement valid if the converter
// stops blocking the host before the copy completes. BM_RawPeerCopy copies matching total bytes
// in one contiguous buffer per input, with a single host sync: a lower bound for the converter.
//
// BM_ConvertGpuToGpu needs peer access between the two devices. The Staged benchmarks force the
// route through pinned host memory instead, which is what device pairs without peer access take,
// so they run on any two GPUs. BM_ConvertGpuToGpuStagedPairs runs one such consumer per directed
// device pair at the same time and reports their combined throughput.

#include "builtin_converters_detail.hpp"
#include "cucascade/cudf/builtin_converters.hpp"
#include "cucascade/cudf/gpu_data_representation.hpp"
#include "cucascade/data/representation_converter.hpp"
Expand All @@ -41,6 +47,7 @@

#include <cucascade/error.hpp>
#include <cucascade/memory/common.hpp>
#include <cucascade/memory/fixed_size_host_memory_resource.hpp>

#include <cudf/table/table.hpp>
#include <cudf/types.hpp>
Expand All @@ -56,8 +63,10 @@
#include <algorithm>
#include <array>
#include <cstddef>
#include <future>
#include <iterator>
#include <memory>
#include <numeric>
#include <utility>
#include <vector>

Expand Down Expand Up @@ -93,12 +102,25 @@ constexpr std::size_t kInputBytes =
constexpr std::size_t kPeakBytesPerGpu = (kNumInputs + 1) * kInputBytes;
constexpr std::size_t kGpuCapacityPerDevice = kPeakBytesPerGpu + kPeakBytesPerGpu / 4;

// Host staging is unused (benchmarks skip without peer access); the host space still pins
// 4 x 128 x 1 MiB = 512 MiB at startup, so cap just above that.
// The host space pins 4 x 128 x 1 MiB = 512 MiB at startup, so cap just above that. A staged copy
// borrows host blocks for one column buffer at a time, a few MiB per consumer at kMaxRows.
constexpr std::size_t kHostCapacityPerRegion = 1 * kGiB;

static std::unique_ptr<memory_reservation_manager> g_manager;
static std::unique_ptr<representation_converter_registry> g_registry;
// Same converters, with every GPU -> GPU conversion forced through pinned host memory.
static std::unique_ptr<representation_converter_registry> g_staged_registry;

void make_registries()
{
g_registry = std::make_unique<representation_converter_registry>();
register_builtin_converters(*g_registry);

g_staged_registry = std::make_unique<representation_converter_registry>();
cucascade::detail::register_builtin_converters(
*g_staged_registry,
cucascade::detail::gpu_to_gpu_converter_options{.force_staged_route = true});
}

/**
* @brief Build a two-GPU manager with NUMA-local host spaces and peer-granted GPU pools.
Expand All @@ -116,12 +138,36 @@ void DoSetup([[maybe_unused]] const benchmark::State& state)
.set_per_numa_region_capacity(kHostCapacityPerRegion);
g_manager = std::make_unique<memory_reservation_manager>(builder.build());

g_registry = std::make_unique<representation_converter_registry>();
register_builtin_converters(*g_registry);
make_registries();
}

/**
* @brief Build a manager over every visible GPU, for the benchmark that runs several device pairs.
*
* Each GPU holds the inputs of one pair and the outputs of another, so it gets twice the capacity
* of the two-GPU setup.
*/
void DoSetupAllGpus([[maybe_unused]] const benchmark::State& state)
{
int const devices = test::cuda_device_count();
if (devices < 2) { return; }

std::vector<int> gpu_ids(static_cast<std::size_t>(devices));
std::iota(gpu_ids.begin(), gpu_ids.end(), 0);

reservation_manager_configurator builder;
builder.set_gpu_ids(gpu_ids)
.set_gpu_usage_limit(2 * kGpuCapacityPerDevice)
.use_numa_id_as_host_id()
.set_per_numa_region_capacity(kHostCapacityPerRegion);
g_manager = std::make_unique<memory_reservation_manager>(builder.build());

make_registries();
}

void DoTeardown([[maybe_unused]] const benchmark::State& state)
{
g_staged_registry.reset();
g_registry.reset();
g_manager.reset();
}
Expand Down Expand Up @@ -170,14 +216,42 @@ bool skip_without_peers(benchmark::State& state)
}

/**
* @brief Build @p count source representations on the source GPU, each carrying a writer event
* @brief Skip @p state unless two devices exist.
* @return True if the benchmark was skipped.
*/
bool skip_without_two_devices(benchmark::State& state)
{
if (test::cuda_device_count() < 2) {
state.SkipWithError("requires two CUDA devices");
return true;
}
return false;
}

/**
* @brief Peak MiB the HOST spaces have handed out since the manager was built.
*
* Staged copies borrow their host blocks there and nothing else in these benchmarks does.
*/
double peak_host_mib()
{
std::size_t bytes = 0;
for (memory_space const* space : g_manager->get_memory_spaces_for_tier(Tier::HOST)) {
bytes += space->get_memory_resource_of<Tier::HOST>()->get_peak_total_allocated_bytes();
}
return static_cast<double>(bytes) / static_cast<double>(kMiB);
}

/**
* @brief Build @p count source representations on @p src_device, each carrying a writer event
* recorded on @p producer, as a Sirius partition task would leave them.
*/
std::vector<std::unique_ptr<gpu_table_representation>> make_inputs(std::size_t count,
cudf::size_type rows,
::cuda::stream_ref producer)
::cuda::stream_ref producer,
int src_device = kSrcDeviceId)
{
memory_space* src_space = g_manager->get_memory_space(Tier::GPU, kSrcDeviceId);
memory_space* src_space = g_manager->get_memory_space(Tier::GPU, src_device);

std::vector<std::unique_ptr<gpu_table_representation>> inputs;
inputs.reserve(count);
Expand All @@ -204,13 +278,13 @@ void set_counters(benchmark::State& state,
}

/**
* @brief Sequentially convert N source-GPU partitions to the destination GPU on one thread.
* @brief Sequentially convert N source-GPU partitions to the destination GPU on one thread, with
* the converters of @p registry.
* @param state.range(0) Rows per input (fixed schema and partition count)
*/
void BM_ConvertGpuToGpu(benchmark::State& state)
void run_sequential_conversions(benchmark::State& state,
representation_converter_registry& registry)
{
if (skip_without_peers(state)) { return; }

cudf::size_type const rows = static_cast<cudf::size_type>(state.range(0));

rmm::cuda_stream producer = [] {
Expand All @@ -231,7 +305,7 @@ void BM_ConvertGpuToGpu(benchmark::State& state)

// Warm-up: first touch of the destination pool and the peer-route probe.
{
auto warm = g_registry->convert<gpu_table_representation>(*inputs[0], dst_space, consumer);
auto warm = registry.convert<gpu_table_representation>(*inputs[0], dst_space, consumer);
warm->wait_for_writer(consumer);
consumer.synchronize();
}
Expand All @@ -240,7 +314,7 @@ void BM_ConvertGpuToGpu(benchmark::State& state)
outputs.reserve(kNumInputs);
for (auto _ : state) {
for (auto const& input : inputs) {
outputs.push_back(g_registry->convert<gpu_table_representation>(*input, dst_space, consumer));
outputs.push_back(registry.convert<gpu_table_representation>(*input, dst_space, consumer));
}

// Consumer may read every output only after its writer event.
Expand All @@ -261,6 +335,161 @@ void BM_ConvertGpuToGpu(benchmark::State& state)
state.counters["rows"] = rows;
}

/**
* @brief The consumer fan-in over the peer route.
* @param state.range(0) Rows per input
*/
void BM_ConvertGpuToGpu(benchmark::State& state)
{
if (skip_without_peers(state)) { return; }

run_sequential_conversions(state, *g_registry);
}

/**
* @brief The same consumer fan-in with every conversion staged through pinned host memory.
* @param state.range(0) Rows per input
*/
void BM_ConvertGpuToGpuStaged(benchmark::State& state)
{
if (skip_without_two_devices(state)) { return; }

run_sequential_conversions(state, *g_staged_registry);
state.counters["staging_peak_MiB"] = peak_host_mib();
}

/// One directed device pair of BM_ConvertGpuToGpuStagedPairs, with its own streams and tables.
struct staged_pair {
int src_device;
int dst_device;
rmm::cuda_stream producer; ///< On src_device; wrote the inputs
rmm::cuda_stream consumer; ///< On dst_device; reads the outputs
std::vector<std::unique_ptr<gpu_table_representation>> inputs;
std::vector<std::unique_ptr<gpu_table_representation>> outputs;
};

/**
* @brief Directed pairs for @p devices GPUs, each device paired with the one half the count away.
*
* Both directions of a pair are adjacent, e.g. for 4 devices: 0>2, 2>0, 1>3, 3>1. On a machine
* with two GPUs behind each of two PCIe switches these are the pairs that cross switches.
*/
std::vector<std::pair<int, int>> directed_pairs(int devices)
{
std::vector<std::pair<int, int>> pairs;
int const half = devices / 2;
for (int low = 0; low < half; ++low) {
pairs.emplace_back(low, low + half);
pairs.emplace_back(low + half, low);
}
return pairs;
}

/**
* @brief Run the consumer fan-in on several directed device pairs at once, one thread per pair,
* with every conversion staged through pinned host memory.
* @param state.range(0) Rows per input
* @param state.range(1) Number of directed pairs running concurrently
*/
void BM_ConvertGpuToGpuStagedPairs(benchmark::State& state)
{
if (skip_without_two_devices(state)) { return; }

cudf::size_type const rows = static_cast<cudf::size_type>(state.range(0));
auto const num_pairs = static_cast<std::size_t>(state.range(1));

auto const candidates = directed_pairs(test::cuda_device_count());
if (candidates.size() < num_pairs) {
state.SkipWithError("not enough CUDA devices for this many pairs");
return;
}

auto const make_stream = [](int device) {
rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{device}};
return rmm::cuda_stream{};
};

std::vector<std::unique_ptr<staged_pair>> pairs;
pairs.reserve(num_pairs);
std::size_t bytes_per_iteration = 0;
for (std::size_t i = 0; i < num_pairs; ++i) {
auto const [src_device, dst_device] = candidates[i];
pairs.emplace_back(new staged_pair{
src_device, dst_device, make_stream(src_device), make_stream(dst_device), {}, {}});
staged_pair& pair = *pairs.back();

pair.inputs = make_inputs(kNumInputs, rows, pair.producer, src_device);
pair.outputs.reserve(kNumInputs);
{
rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{src_device}};
pair.producer.synchronize();
}
for (auto const& input : pair.inputs) {
bytes_per_iteration += input->get_size_in_bytes();
}
}

auto const convert_pair = [](staged_pair& pair) {
const memory_space* dst_space = g_manager->get_memory_space(Tier::GPU, pair.dst_device);
rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{pair.dst_device}};
for (auto const& input : pair.inputs) {
pair.outputs.push_back(
g_staged_registry->convert<gpu_table_representation>(*input, dst_space, pair.consumer));
}
// The consumer may read every output only after its writer event.
for (auto const& output : pair.outputs) {
output->wait_for_writer(pair.consumer);
}
pair.consumer.synchronize();
};

auto const free_outputs = [](staged_pair& pair) {
rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{pair.dst_device}};
pair.outputs.clear();
CUCASCADE_CUDA_TRY(cudaDeviceSynchronize());
};

// Warm-up: first touch of each destination pool.
for (auto const& pair : pairs) {
convert_pair(*pair);
free_outputs(*pair);
}

std::vector<std::future<void>> workers;
workers.reserve(num_pairs);
for (auto _ : state) {
for (auto const& pair : pairs) {
workers.push_back(std::async(std::launch::async, convert_pair, std::ref(*pair)));
}
for (auto& worker : workers) {
worker.get();
}

// Free outside the timed region.
state.PauseTiming();
workers.clear();
for (auto const& pair : pairs) {
free_outputs(*pair);
}
state.ResumeTiming();
}

set_counters(state,
bytes_per_iteration,
kNumInputs * num_pairs,
pairs.front()->inputs.front()->get_table_view().num_columns());
state.counters["rows"] = rows;
state.counters["pairs"] = static_cast<double>(num_pairs);
state.counters["staging_peak_MiB"] = peak_host_mib();

// Free each pair's inputs with their device current, before the streams go away.
for (auto const& pair : pairs) {
rmm::cuda_set_device_raii const guard{rmm::cuda_device_id{pair->src_device}};
pair->inputs.clear();
CUCASCADE_CUDA_TRY(cudaDeviceSynchronize());
}
}

/**
* @brief Lower bound: peer-copy the same buffers, all queued on one stream, one host sync.
* @param state.range(0) Rows per input, sized using the converter's actual table allocation
Expand Down Expand Up @@ -324,13 +553,37 @@ void gpu_to_gpu_args(benchmark::internal::Benchmark* b)
b->ArgName("rows");
}

void staged_pairs_args(benchmark::internal::Benchmark* b)
{
for (int64_t pairs : {1, 2, 4}) {
for (int64_t rows : kRowCounts) {
b->Args({rows, pairs});
}
}
b->ArgNames({"rows", "pairs"});
}

BENCHMARK(BM_ConvertGpuToGpu)
->Setup(DoSetup)
->Teardown(DoTeardown)
->Apply(gpu_to_gpu_args)
->Unit(benchmark::kMillisecond)
->UseRealTime();

BENCHMARK(BM_ConvertGpuToGpuStaged)
->Setup(DoSetup)
->Teardown(DoTeardown)
->Apply(gpu_to_gpu_args)
->Unit(benchmark::kMillisecond)
->UseRealTime();

BENCHMARK(BM_ConvertGpuToGpuStagedPairs)
->Setup(DoSetupAllGpus)
->Teardown(DoTeardown)
->Apply(staged_pairs_args)
->Unit(benchmark::kMillisecond)
->UseRealTime();

BENCHMARK(BM_RawPeerCopy)
->Setup(DoSetup)
->Teardown(DoTeardown)
Expand Down
Loading