Skip to content

Batch STRING-size readbacks during cross-GPU conversion #285

Description

@dhruv9vats

STRING reconstruction:

if (src.type().id() == cudf::type_id::STRING) {
if (src.num_children() < 1) {
// Empty / degenerate STRING column with no offsets child (cudf produces
// these for empty intermediate results). Use cudf::make_empty_column
// which builds the correct internal layout (offsets child of size 1
// holding [0], chars in parent's data buffer of size 0).
return cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING});
}
// Copy offsets as-is (preserve source's INT32 vs INT64 type). make_strings_column
// accepts either offset type.
auto offsets_col =
reconstruct_column_p2p_async(src.child(0), src_device, dst_device, stream, mr);
rmm::device_buffer chars_buf{};
if (src.size() > 0 && src.data<char>() != nullptr) {
// Read total chars size from offsets[size-1]. Reading from the *source* column
// here is unsafe: cudf::strings_column_view::chars_size() ultimately calls
// cudaMemcpyAsync(... cudaMemcpyDefault ...), which on cudaMallocAsync pool
// memory across devices silently returns success without copying bytes — the
// host buffer keeps its uninitialized stack value (often -1). Read instead
// from the target-side offsets buffer we just peer-copied; that's a same-device
// D→H read on the target stream and works reliably regardless of pool peer
// access semantics.
const cudf::column_view offsets_view = offsets_col->view();
const int last_idx = offsets_view.size() - 1;
int64_t chars_bytes = 0;
if (offsets_view.type().id() == cudf::type_id::INT32) {
int32_t value = 0;
CUCASCADE_CUDA_TRY(cudaMemcpyAsync(&value,
offsets_view.head<int32_t>() + last_idx,
sizeof(int32_t),
cudaMemcpyDeviceToHost,
stream.get()));
stream.sync();
chars_bytes = value;
} else {
int64_t value = 0;
CUCASCADE_CUDA_TRY(cudaMemcpyAsync(&value,
offsets_view.head<int64_t>() + last_idx,
sizeof(int64_t),
cudaMemcpyDeviceToHost,
stream.get()));
stream.sync();
chars_bytes = value;
}
if (chars_bytes > 0) {
const auto data_size_in_bytes = static_cast<std::size_t>(chars_bytes);
chars_buf = peer_copy_into_async(rmm::device_buffer(data_size_in_bytes, stream, mr),
src.data<char>(),
src_device,
data_size_in_bytes,
dst_device,
stream);
}
}
return cudf::make_strings_column(
src.size(), std::move(offsets_col), std::move(chars_buf), null_count, std::move(null_mask));

reads each terminal offset into pageable host memory and waits before allocating the character buffer. Nested columns:

if (src.type().id() == cudf::type_id::LIST) {
if (src.num_children() < 2) {
throw std::invalid_argument(
"reconstruct_column_p2p: LIST column must have two children (offsets, values)");
}
// Preserve source's offsets type — make_lists_column accepts INT32 or INT64.
auto offsets_col =
reconstruct_column_p2p_async(src.child(0), src_device, dst_device, stream, mr);
auto values_col =
reconstruct_column_p2p_async(src.child(1), src_device, dst_device, stream, mr);
return cudf::make_lists_column(
src.size(), std::move(offsets_col), std::move(values_col), null_count, std::move(null_mask));
}
if (src.type().id() == cudf::type_id::STRUCT) {
std::vector<std::unique_ptr<cudf::column>> fields;
fields.reserve(static_cast<std::size_t>(src.num_children()));
for (cudf::size_type i = 0; i < src.num_children(); ++i) {
fields.push_back(
reconstruct_column_p2p_async(src.child(i), src_device, dst_device, stream, mr));
}
// Construct directly to skip make_structs_column's superimpose_nulls kernel —
// our peer-copied null masks are already consistent (mirrors reconstruct_column()).
return std::make_unique<cudf::column>(cudf::data_type{cudf::type_id::STRUCT},
src.size(),
rmm::device_buffer{},
std::move(null_mask),
null_count,
std::move(fields));
}
if (src.type().id() == cudf::type_id::DICTIONARY32) {
if (src.num_children() < 2) {
throw std::invalid_argument(
"reconstruct_column_p2p: DICTIONARY32 column must have two children "
"(indices, keys)");
}
// cudf DICTIONARY32 child order: [0]=indices, [1]=keys.
// make_dictionary_column parameter order: (keys, indices, ...).
auto indices_col =
reconstruct_column_p2p_async(src.child(0), src_device, dst_device, stream, mr);
auto keys_col = reconstruct_column_p2p_async(src.child(1), src_device, dst_device, stream, mr);
return cudf::make_dictionary_column(
std::move(keys_col), std::move(indices_col), std::move(null_mask), null_count);
repeat this overhead.

Collect STRING-size requests recursively, submit all readbacks into one pinned buffer on a source-device stream, and wait once. Reconstruct using the collected sizes.

Current readback

From the INT32-offset-size branch:

if (offsets_view.type().id() == cudf::type_id::INT32) {
int32_t value = 0;
CUCASCADE_CUDA_TRY(cudaMemcpyAsync(&value,
offsets_view.head<int32_t>() + last_idx,
sizeof(int32_t),
cudaMemcpyDeviceToHost,
stream.get()));
stream.sync();
chars_bytes = value;

Proposed flow
Traverse column metadata, including nested STRINGs
Allocate pinned slots through cuDF's configured resource
Wait for the source writer on a source-device stream
Enqueue all terminal-offset D2H reads
Wait once
Reconstruct using collected sizes
Keep the final conversion wait

Integration sketch in cross-device conversion:

// Same-device case: clone via source's own clone() method.
if (source.get_device_id() == target_memory_space->get_device_id()) {
// A null stream handle represents the default stream of the current device: ensure it's the
// device attached to the representation, or clone() will throw.
rmm::cuda_set_device_raii guard{rmm::cuda_device_id{target_memory_space->get_device_id()}};
return source.clone(stream);
}
auto const src_device_id = gpu_source.get_device_id();
auto const dst_device_id = target_memory_space->get_device_id();
auto const copy_path = memory::probe_peer_dma_works(src_device_id, dst_device_id)
? "convert:gpu_to_gpu:peer_copy"
: "convert:gpu_to_gpu:host_staging";
nvtx_scope copy_path_range{copy_path};
rmm::cuda_set_device_raii target_guard{rmm::cuda_device_id{dst_device_id}};
// Target-bound stream from the target memory_space's stream pool. All peer copies
// and target-side allocations are issued on this stream so they observe in-order
// completion without explicit cross-stream events.
auto target_stream = target_memory_space->acquire_stream();
auto mr = target_memory_space->get_default_allocator();
std::unique_ptr<cudf::table> new_table{nullptr};
try {
// Order the peer copies below after the most recent writer of gpu_source. With an unknown
// writer this synchronizes the whole source device, which is correct but invisible to
// compute-sanitizer: its stream-ordered race tracking ignores host-side synchronization, so it
// may report races here that a recorded writer event avoids.
gpu_source.wait_for_writer(target_stream);
cudf::table_view const src_view = gpu_source.get_table_view();
std::vector<std::unique_ptr<cudf::column>> target_columns;
target_columns.reserve(static_cast<std::size_t>(src_view.num_columns()));
for (const cudf::column_view& col : src_view) {
target_columns.push_back(
reconstruct_column_p2p_async(col, src_device_id, dst_device_id, target_stream, mr));

:

auto string_sizes = read_string_sizes(gpu_source);
string_size_cursor cursor{
  std::span<const std::size_t>{string_sizes}};

// Each STRING consumes one entry before empty-column handling.
const auto chars_bytes = cursor.take();

Preserve source protection, destination writer dependencies, and the final completion wait

// Sync for the source's lifetime, not for readers of the result (the writer event recorded
// below covers those). The peer reads on target_stream are not ordered with the source's
// deallocation stream, so a caller freeing the source right after convert (e.g.
// mutable_data_batch::convert_to, which syncs only the caller's stream) would race them.
// TODO: explore establishing ordering for source's lifetime via events.
target_stream.sync();

Validate source-local reads against the documented pool-memory limitation

// Read total chars size from offsets[size-1]. Reading from the *source* column
// here is unsafe: cudf::strings_column_view::chars_size() ultimately calls
// cudaMemcpyAsync(... cudaMemcpyDefault ...), which on cudaMallocAsync pool
// memory across devices silently returns success without copying bytes — the
// host buffer keeps its uninitialized stack value (often -1). Read instead
// from the target-side offsets buffer we just peer-copied; that's a same-device
// D→H read on the target stream and works reliably regardless of pool peer
// access semantics.

For 27 nonempty STRING nodes on the direct-peer path with known writer events, explicit host waits should fall from 28 to 2 per conversion. Performance remains unmeasured.

Activity

  1. toxicteddy00077 commented on Nov 13, 2026

    @toxicteddy00077

    @dhruv9vats I think I can help. A possible approach is to have a single batched pinned-memory readback of all terminal offsets(including the nested STRINGs) with a single wait, consumed in traversal order, preserving current writer/completion order. I can open a PR if this is alright. Let me know if ive missed something

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions