Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 49 additions & 72 deletions vortex-cuda/kernels/src/decimal_cast.cu
Comment thread
0ax1 marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -6,64 +6,43 @@
#include <stdint.h>
#include <type_traits>

// Arrow decimal schemas fix the physical values buffer width:
// - Decimal32: 4 bytes per value.
// - Decimal64: 8 bytes per value.
// - Decimal128: 16 bytes per value.
// - Decimal256: 32 bytes per value.
// Widen decimal values buffers to a wider physical width. Two callers use these kernels:
// - The Arrow export casts to the width that the Arrow schema fixes: 4 bytes for Decimal32,
// 8 bytes for Decimal64, 16 bytes for Decimal128, and 32 bytes for Decimal256.
// - The Widen executor casts a narrow decimal child to the canonical width of its decimal
// dtype, from 2 to 32 bytes.
//
// Vortex storage width can differ, so export casts to the schema-implied width.
// Rust-side export rejects narrowing casts because detecting overflow on-device
// would require synchronizing an overflow flag back to the host.
// Only widening pairs are instantiated. A narrowing cast can overflow, and detecting that on the
// device would require synchronizing an overflow flag back to the host, so the Rust side rejects
// narrowing before it loads a kernel.

// Low 64-bit conversion for Decimal32/64 outputs.
template <typename Input>
__device__ __forceinline__ int64_t decimal_to_i64(Input value) {
if constexpr (std::is_same_v<Input, int128_t>) {
return value.lo;
} else if constexpr (std::is_same_v<Input, int256_t>) {
return value.parts[0];
} else {
return static_cast<int64_t>(value);
}
}

// 128-bit conversion for Decimal128 outputs.
template <typename Input>
__device__ __forceinline__ int128_t decimal_to_i128(Input value) {
if constexpr (std::is_same_v<Input, int128_t>) {
return value;
} else if constexpr (std::is_same_v<Input, int256_t>) {
return int128_t {value.parts[0], value.parts[1]};
} else {
const int64_t lo = static_cast<int64_t>(value);
const int64_t hi = value < 0 ? -1 : 0;
return int128_t {lo, hi};
}
}

// Convert one value to the Arrow schema's physical width.
// Sign-extend one value to the wider output width in a single step, with no intermediate width.
// Native outputs use one `static_cast`. The 128-bit and 256-bit outputs keep the input's words
// and fill the upper words with its sign.
template <typename Output, typename Input>
__device__ __forceinline__ Output decimal_cast_value(Input value) {
if constexpr (std::is_same_v<Output, int32_t>) {
return static_cast<int32_t>(decimal_to_i64(value));
} else if constexpr (std::is_same_v<Output, int64_t>) {
return decimal_to_i64(value);
} else if constexpr (std::is_same_v<Output, int128_t>) {
return decimal_to_i128(value);
static_assert(sizeof(Input) < sizeof(Output), "decimal casts on the GPU only widen");

if constexpr (std::is_same_v<Output, int128_t>) {
static_assert(std::is_integral_v<Input> && std::is_signed_v<Input>);
const int64_t lo = static_cast<int64_t>(value);
return int128_t {lo, lo < 0 ? -1 : 0};
} else if constexpr (std::is_same_v<Output, int256_t> && std::is_same_v<Input, int128_t>) {
const int64_t sign = value.hi < 0 ? -1 : 0;
return int256_t {{value.lo, value.hi, sign, sign}};
} else if constexpr (std::is_same_v<Output, int256_t>) {
static_assert(std::is_integral_v<Input> && std::is_signed_v<Input>);
const int64_t lo = static_cast<int64_t>(value);
const int64_t sign = lo < 0 ? -1 : 0;
return int256_t {{lo, sign, sign, sign}};
} else {
static_assert(std::is_same_v<Output, int256_t>);
if constexpr (std::is_same_v<Input, int256_t>) {
return value;
} else {
const int128_t value128 = decimal_to_i128(value);
const int64_t sign = value128.hi < 0 ? -1 : 0;
return int256_t {{value128.lo, value128.hi, sign, sign}};
}
static_assert(std::is_integral_v<Input> && std::is_signed_v<Input>);
static_assert(std::is_integral_v<Output> && std::is_signed_v<Output>);
return static_cast<Output>(value);
}
}

// Cast a contiguous values buffer to the Arrow schema's physical width.
// Cast a contiguous values buffer to the output physical width.
template <typename Input, typename Output>
__device__ void
decimal_cast_device(const Input *__restrict input, Output *__restrict output, uint64_t array_len) {
Expand All @@ -76,28 +55,26 @@ decimal_cast_device(const Input *__restrict input, Output *__restrict output, ui
}
}

// Generate Decimal32/64/128/256 cast kernels for one input storage type.
#define GENERATE_DECIMAL_CAST_KERNELS(input_suffix, InputType) \
extern "C" __global__ void decimal_cast_##input_suffix##_i32(const InputType *__restrict input, \
int32_t *__restrict output, \
uint64_t array_len) { \
decimal_cast_device(input, output, array_len); \
} \
extern "C" __global__ void decimal_cast_##input_suffix##_i64(const InputType *__restrict input, \
int64_t *__restrict output, \
uint64_t array_len) { \
decimal_cast_device(input, output, array_len); \
} \
extern "C" __global__ void decimal_cast_##input_suffix##_i128(const InputType *__restrict input, \
int128_t *__restrict output, \
uint64_t array_len) { \
decimal_cast_device(input, output, array_len); \
} \
extern "C" __global__ void decimal_cast_##input_suffix##_i256(const InputType *__restrict input, \
int256_t *__restrict output, \
uint64_t array_len) { \
#define GENERATE_DECIMAL_CAST(input_suffix, InputType, output_suffix, OutputType) \
extern "C" __global__ void decimal_cast_##input_suffix##_##output_suffix( \
const InputType *__restrict input, \
OutputType *__restrict output, \
uint64_t array_len) { \
decimal_cast_device(input, output, array_len); \
}

FOR_EACH_SIGNED_INT(GENERATE_DECIMAL_CAST_KERNELS)
FOR_EACH_LARGE_DECIMAL(GENERATE_DECIMAL_CAST_KERNELS)
GENERATE_DECIMAL_CAST(i8, int8_t, i16, int16_t)
GENERATE_DECIMAL_CAST(i8, int8_t, i32, int32_t)
GENERATE_DECIMAL_CAST(i8, int8_t, i64, int64_t)
GENERATE_DECIMAL_CAST(i8, int8_t, i128, int128_t)
GENERATE_DECIMAL_CAST(i8, int8_t, i256, int256_t)
GENERATE_DECIMAL_CAST(i16, int16_t, i32, int32_t)
GENERATE_DECIMAL_CAST(i16, int16_t, i64, int64_t)
GENERATE_DECIMAL_CAST(i16, int16_t, i128, int128_t)
GENERATE_DECIMAL_CAST(i16, int16_t, i256, int256_t)
GENERATE_DECIMAL_CAST(i32, int32_t, i64, int64_t)
GENERATE_DECIMAL_CAST(i32, int32_t, i128, int128_t)
GENERATE_DECIMAL_CAST(i32, int32_t, i256, int256_t)
GENERATE_DECIMAL_CAST(i64, int64_t, i128, int128_t)
GENERATE_DECIMAL_CAST(i64, int64_t, i256, int256_t)
GENERATE_DECIMAL_CAST(i128, int128_t, i256, int256_t)
36 changes: 36 additions & 0 deletions vortex-cuda/kernels/src/widen.cu
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

#include "scalar_kernel.cuh"
#include "types.cuh"
#include <type_traits>

template <typename InputT, typename OutputT>
struct WidenOp {
static_assert(sizeof(InputT) < sizeof(OutputT));
static_assert(std::is_signed_v<InputT> == std::is_signed_v<OutputT>);

__device__ OutputT operator()(InputT value) const {
return static_cast<OutputT>(value);
}
};

#define GENERATE_WIDEN(input_suffix, InputT, output_suffix, OutputT) \
extern "C" __global__ void widen_##input_suffix##_##output_suffix(const InputT *__restrict input, \
OutputT *__restrict output, \
uint64_t array_len) { \
scalar_kernel(input, output, array_len, WidenOp<InputT, OutputT> {}); \
}

GENERATE_WIDEN(i8, int8_t, i16, int16_t)
GENERATE_WIDEN(i8, int8_t, i32, int32_t)
GENERATE_WIDEN(i8, int8_t, i64, int64_t)
GENERATE_WIDEN(i16, int16_t, i32, int32_t)
GENERATE_WIDEN(i16, int16_t, i64, int64_t)
GENERATE_WIDEN(i32, int32_t, i64, int64_t)
GENERATE_WIDEN(u8, uint8_t, u16, uint16_t)
GENERATE_WIDEN(u8, uint8_t, u32, uint32_t)
GENERATE_WIDEN(u8, uint8_t, u64, uint64_t)
GENERATE_WIDEN(u16, uint16_t, u32, uint32_t)
GENERATE_WIDEN(u16, uint16_t, u64, uint64_t)
GENERATE_WIDEN(u32, uint32_t, u64, uint64_t)
71 changes: 3 additions & 68 deletions vortex-cuda/src/arrow/canonical.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,13 @@ use vortex::array::arrays::varbinview::VarBinViewDataParts;
use vortex::array::buffer::BufferHandle;
use vortex::array::buffer::DeviceBuffer;
use vortex::array::builtins::ArrayBuiltins;
use vortex::array::match_each_decimal_value_type;
use vortex::array::optimizer::ArrayOptimizer;
use vortex::array::validity::Validity;
use vortex::buffer::Buffer;
use vortex::dtype::DType;
use vortex::dtype::DecimalType;
use vortex::dtype::NativeDecimalType;
use vortex::dtype::Nullability;
use vortex::dtype::PType;
use vortex::dtype::i256;
use vortex::encodings::fsst::FSST;
use vortex::error::VortexResult;
use vortex::error::vortex_bail;
Expand All @@ -62,7 +59,6 @@ use vortex::error::vortex_ensure_eq;
use vortex::error::vortex_err;
use vortex::extension::datetime::AnyTemporal;
use vortex_cuda_driver::CudaSlice;
use vortex_cuda_driver::DeviceRepr;
use vortex_cuda_driver::LaunchConfig;
use vortex_cuda_driver::PushKernelArg;
use vortex_onpair::OnPair;
Expand Down Expand Up @@ -92,6 +88,7 @@ use crate::executor::execute_validity_cuda;
use crate::kernel::DecodedVarBin;
use crate::kernel::decode_fsst_varbin;
use crate::kernel::decode_onpair_varbin;
use crate::kernel::widen_decimal_values;
use crate::stream::zero_padding;

const BITMAP_THREADS_PER_BLOCK: u32 = 256;
Expand Down Expand Up @@ -475,73 +472,11 @@ async fn export_decimal_values(
"cannot export decimal values from {values_type} storage to Arrow {target_type}: narrowing would require a device-to-host overflow check",
);
}
let values = ctx.ensure_on_device(values).await?;
if values_type == target_type {
return Ok(values);
return ctx.ensure_on_device(values).await;
}

match_each_decimal_value_type!(values_type, |S| {
export_decimal_values_from::<S>(values, target_type, len, ctx).await
})
}

/// Dispatch from a concrete storage type `S` to the Arrow-required output width.
async fn export_decimal_values_from<S>(
values: BufferHandle,
target_type: DecimalType,
len: usize,
ctx: &mut CudaExecutionCtx,
) -> VortexResult<BufferHandle>
where
S: NativeDecimalType + DeviceRepr,
{
match target_type {
DecimalType::I32 => decimal_cast::<S, i32>(values, len, ctx).await,
DecimalType::I64 => decimal_cast::<S, i64>(values, len, ctx).await,
DecimalType::I128 => decimal_cast::<S, i128>(values, len, ctx).await,
DecimalType::I256 => decimal_cast::<S, i256>(values, len, ctx).await,
target_type => {
vortex_bail!("cannot export DecimalArray as Arrow decimal value type {target_type}")
}
}
}

/// Launches the CUDA kernel that casts from Vortex storage type `S` to Arrow output type `D`.
///
/// The caller must ensure this is not a narrowing cast.
async fn decimal_cast<S, D>(
values: BufferHandle,
len: usize,
ctx: &mut CudaExecutionCtx,
) -> VortexResult<BufferHandle>
where
S: NativeDecimalType + DeviceRepr,
D: NativeDecimalType + DeviceRepr,
{
if len == 0 {
return ctx
.ensure_on_device(BufferHandle::new_host(
Buffer::<D>::empty().into_byte_buffer(),
))
.await;
}

let mut output_buffer = ctx.device_alloc::<D>(len)?;

let values_view = values.cuda_view::<S>()?;
let len_u64 = len as u64;
let cuda_function = ctx.load_function_with_suffixes(
"decimal_cast",
&[&S::DECIMAL_TYPE.to_string(), &D::DECIMAL_TYPE.to_string()],
)?;

ctx.launch_kernel(&cuda_function, len, |args| {
args.arg(&values_view).arg(&mut output_buffer).arg(&len_u64);
})?;

Ok(BufferHandle::new_device(Arc::new(CudaDeviceBuffer::new(
output_buffer,
)?)))
widen_decimal_values(values, values_type, target_type, len, ctx).await
}

/// Export Vortex binary views as an Arrow Device array with `Utf8View`/`BinaryView` layout.
Expand Down
2 changes: 2 additions & 0 deletions vortex-cuda/src/dynamic_dispatch/plan_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool {
_ => false,
};
}
// TODO(connor): Add Widen fusion after fixing sign extension of signed narrow BitPacked and
// FoR children in dynamic_dispatch.cu. Those paths currently zero-extend to the output width.
(id == FoR.id() && array.as_::<FoR>().references().is_global())
|| id == ZigZag.id()
|| id == Primitive.id()
Expand Down
Loading
Loading