Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions be/src/runtime/types.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,14 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>& types, int* idx)
DCHECK(scalar_type.__isset.len);
len = scalar_type.len;
} else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 ||
type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256 || type == TYPE_DATETIMEV2) {
type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256) {
DCHECK(scalar_type.__isset.precision);
DCHECK(scalar_type.__isset.scale);
precision = scalar_type.precision;
scale = scalar_type.scale;
} else if (type == TYPE_DATETIMEV2) {
DCHECK(scalar_type.__isset.scale);
scale = scalar_type.scale;
} else if (type == TYPE_TIMEV2) {
if (scalar_type.__isset.scale) {
scale = scalar_type.scale;
Expand Down Expand Up @@ -157,11 +160,14 @@ void TypeDescriptor::to_thrift(TTypeDesc* thrift_type) const {
// DCHECK_NE(len, -1);
scalar_type.__set_len(len);
} else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 ||
type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256 || type == TYPE_DATETIMEV2) {
type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256) {
DCHECK_NE(precision, -1);
DCHECK_NE(scale, -1);
scalar_type.__set_precision(precision);
scalar_type.__set_scale(scale);
} else if (type == TYPE_DATETIMEV2) {
DCHECK_NE(scale, -1);
scalar_type.__set_scale(scale);
}
}
}
Expand All @@ -174,11 +180,14 @@ void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const {
if (type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_HLL || type == TYPE_STRING) {
scalar_type->set_len(len);
} else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 ||
type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256 || type == TYPE_DATETIMEV2) {
type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256) {
DCHECK_NE(precision, -1);
DCHECK_NE(scale, -1);
scalar_type->set_precision(precision);
scalar_type->set_scale(scale);
} else if (type == TYPE_DATETIMEV2) {
DCHECK_NE(scale, -1);
scalar_type->set_scale(scale);
} else if (type == TYPE_ARRAY) {
node->set_type(TTypeNodeType::ARRAY);
node->set_contains_null(contains_nulls[0]);
Expand Down Expand Up @@ -224,11 +233,14 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
DCHECK(scalar_type.has_len());
len = scalar_type.len();
} else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 ||
type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256 || type == TYPE_DATETIMEV2) {
type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256) {
DCHECK(scalar_type.has_precision());
DCHECK(scalar_type.has_scale());
precision = scalar_type.precision();
scale = scalar_type.scale();
} else if (type == TYPE_DATETIMEV2) {
DCHECK(scalar_type.has_scale());
scale = scalar_type.scale();
} else if (type == TYPE_STRING) {
if (scalar_type.has_len()) {
len = scalar_type.len();
Expand Down
3 changes: 2 additions & 1 deletion be/src/runtime/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ struct TypeDescriptor {

/// Only set if type == TYPE_DECIMAL
int precision;
/// Only set if type == TYPE_DECIMAL or type = TYPE_DATETIMEV2
int scale;

std::vector<TypeDescriptor> children;
Expand All @@ -69,11 +70,11 @@ struct TypeDescriptor {

// explicit TypeDescriptor(PrimitiveType type) :
TypeDescriptor(PrimitiveType type) : type(type), len(-1), precision(-1), scale(-1) {
// TODO, should not initialize default values, force initialization by parameters or external.
if (type == TYPE_DECIMALV2) {
precision = 27;
scale = 9;
} else if (type == TYPE_DATETIMEV2) {
precision = 18;
scale = 6;
}
}
Expand Down
272 changes: 2 additions & 270 deletions be/src/util/arrow/block_convertor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,7 @@ class Array;

namespace doris {

// Convert Block to an Arrow::Array
// We should keep this function to keep compatible with arrow's type visitor
// Now we inherit TypeVisitor to use default Visit implementation
class FromBlockConverter : public arrow::TypeVisitor {
class FromBlockConverter {
public:
FromBlockConverter(const vectorized::Block& block, const std::shared_ptr<arrow::Schema>& schema,
arrow::MemoryPool* pool, const cctz::time_zone& timezone_obj)
Expand All @@ -79,276 +76,11 @@ class FromBlockConverter : public arrow::TypeVisitor {
_cur_field_idx(-1),
_timezone_obj(timezone_obj) {}

~FromBlockConverter() override = default;

// Use base class function
using arrow::TypeVisitor::Visit;

#define PRIMITIVE_VISIT(TYPE) \
arrow::Status Visit(const arrow::TYPE& type) override { return _visit(type); }

PRIMITIVE_VISIT(Int8Type)
PRIMITIVE_VISIT(Int16Type)
PRIMITIVE_VISIT(Int32Type)
PRIMITIVE_VISIT(Int64Type)
PRIMITIVE_VISIT(FloatType)
PRIMITIVE_VISIT(DoubleType)

#undef PRIMITIVE_VISIT

// process string-transformable field
arrow::Status Visit(const arrow::StringType& type) override {
auto& builder = assert_cast<arrow::StringBuilder&>(*_cur_builder);
size_t start = _cur_start;
size_t num_rows = _cur_rows;
ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
for (size_t i = start; i < start + num_rows; ++i) {
bool is_null = _cur_col->is_null_at(i);
if (is_null) {
ARROW_RETURN_NOT_OK(builder.AppendNull());
continue;
}
const auto& data_ref = _cur_col->get_data_at(i);
vectorized::TypeIndex type_idx = vectorized::remove_nullable(_cur_type)->get_type_id();
switch (type_idx) {
case vectorized::TypeIndex::String:
case vectorized::TypeIndex::FixedString:
case vectorized::TypeIndex::HLL: {
if (data_ref.size == 0) {
// 0x01 is a magic num, not useful actually, just for present ""
//char* tmp_val = reinterpret_cast<char*>(0x01);
ARROW_RETURN_NOT_OK(builder.Append(""));
} else {
ARROW_RETURN_NOT_OK(builder.Append(data_ref.data, data_ref.size));
}
break;
}
case vectorized::TypeIndex::Date:
case vectorized::TypeIndex::DateTime: {
char buf[64];
const VecDateTimeValue* time_val = (const VecDateTimeValue*)(data_ref.data);
int len = time_val->to_buffer(buf);
ARROW_RETURN_NOT_OK(builder.Append(buf, len));
break;
}
case vectorized::TypeIndex::DateV2: {
char buf[64];
const DateV2Value<DateV2ValueType>* time_val =
(const DateV2Value<DateV2ValueType>*)(data_ref.data);
int len = time_val->to_buffer(buf);
ARROW_RETURN_NOT_OK(builder.Append(buf, len));
break;
}
case vectorized::TypeIndex::DateTimeV2: {
char buf[64];
const DateV2Value<DateTimeV2ValueType>* time_val =
(const DateV2Value<DateTimeV2ValueType>*)(data_ref.data);
int len = time_val->to_buffer(buf);
ARROW_RETURN_NOT_OK(builder.Append(buf, len));
break;
}
case vectorized::TypeIndex::Int128: {
auto string_temp = LargeIntValue::to_string(
reinterpret_cast<const PackedInt128*>(data_ref.data)->value);
ARROW_RETURN_NOT_OK(builder.Append(string_temp.data(), string_temp.size()));
break;
}
case vectorized::TypeIndex::JSONB: {
std::string string_temp =
JsonbToJson::jsonb_to_json_string(data_ref.data, data_ref.size);
ARROW_RETURN_NOT_OK(builder.Append(string_temp.data(), string_temp.size()));
break;
}
default: {
LOG(WARNING) << "can't convert this type = " << vectorized::getTypeName(type_idx)
<< " to arrow type";
return arrow::Status::TypeError("unsupported column type");
}
}
}
return arrow::Status::OK();
}

// process doris Decimal
arrow::Status Visit(const arrow::Decimal128Type& type) override {
auto& builder = assert_cast<arrow::Decimal128Builder&>(*_cur_builder);
size_t start = _cur_start;
size_t num_rows = _cur_rows;
if (auto* decimalv2_column = vectorized::check_and_get_column<
vectorized::ColumnDecimal<vectorized::Decimal128V2>>(
*vectorized::remove_nullable(_cur_col))) {
std::shared_ptr<arrow::DataType> s_decimal_ptr =
std::make_shared<arrow::Decimal128Type>(27, 9);
ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
for (size_t i = start; i < start + num_rows; ++i) {
bool is_null = _cur_col->is_null_at(i);
if (is_null) {
ARROW_RETURN_NOT_OK(builder.AppendNull());
continue;
}
const auto& data_ref = decimalv2_column->get_data_at(i);
const PackedInt128* p_value = reinterpret_cast<const PackedInt128*>(data_ref.data);
int64_t high = (p_value->value) >> 64;
uint64 low = p_value->value;
arrow::Decimal128 value(high, low);
ARROW_RETURN_NOT_OK(builder.Append(value));
}
return arrow::Status::OK();
} else if (auto* decimal128_column = vectorized::check_and_get_column<
vectorized::ColumnDecimal<vectorized::Decimal128V3>>(
*vectorized::remove_nullable(_cur_col))) {
std::shared_ptr<arrow::DataType> s_decimal_ptr =
std::make_shared<arrow::Decimal128Type>(38, decimal128_column->get_scale());
ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
for (size_t i = start; i < start + num_rows; ++i) {
bool is_null = _cur_col->is_null_at(i);
if (is_null) {
ARROW_RETURN_NOT_OK(builder.AppendNull());
continue;
}
const auto& data_ref = decimal128_column->get_data_at(i);
const PackedInt128* p_value = reinterpret_cast<const PackedInt128*>(data_ref.data);
int64_t high = (p_value->value) >> 64;
uint64 low = p_value->value;
arrow::Decimal128 value(high, low);
ARROW_RETURN_NOT_OK(builder.Append(value));
}
return arrow::Status::OK();
} else if (auto* decimal32_column = vectorized::check_and_get_column<
vectorized::ColumnDecimal<vectorized::Decimal32>>(
*vectorized::remove_nullable(_cur_col))) {
std::shared_ptr<arrow::DataType> s_decimal_ptr =
std::make_shared<arrow::Decimal128Type>(8, decimal32_column->get_scale());
ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
for (size_t i = start; i < start + num_rows; ++i) {
bool is_null = _cur_col->is_null_at(i);
if (is_null) {
ARROW_RETURN_NOT_OK(builder.AppendNull());
continue;
}
const auto& data_ref = decimal32_column->get_data_at(i);
const int32_t* p_value = reinterpret_cast<const int32_t*>(data_ref.data);
int64_t high = *p_value > 0 ? 0 : 1UL << 63;
arrow::Decimal128 value(high, *p_value > 0 ? *p_value : -*p_value);
ARROW_RETURN_NOT_OK(builder.Append(value));
}
return arrow::Status::OK();
} else if (auto* decimal64_column = vectorized::check_and_get_column<
vectorized::ColumnDecimal<vectorized::Decimal64>>(
*vectorized::remove_nullable(_cur_col))) {
std::shared_ptr<arrow::DataType> s_decimal_ptr =
std::make_shared<arrow::Decimal128Type>(18, decimal64_column->get_scale());
ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
for (size_t i = start; i < start + num_rows; ++i) {
bool is_null = _cur_col->is_null_at(i);
if (is_null) {
ARROW_RETURN_NOT_OK(builder.AppendNull());
continue;
}
const auto& data_ref = decimal64_column->get_data_at(i);
const int64_t* p_value = reinterpret_cast<const int64_t*>(data_ref.data);
int64_t high = *p_value > 0 ? 0 : 1UL << 63;
arrow::Decimal128 value(high, *p_value > 0 ? *p_value : -*p_value);
ARROW_RETURN_NOT_OK(builder.Append(value));
}
return arrow::Status::OK();
} else {
return arrow::Status::TypeError("Unsupported column:" + _cur_col->get_name());
}
}
// process boolean
arrow::Status Visit(const arrow::BooleanType& type) override {
auto& builder = assert_cast<arrow::BooleanBuilder&>(*_cur_builder);
size_t start = _cur_start;
size_t num_rows = _cur_rows;
ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
for (size_t i = start; i < start + num_rows; ++i) {
bool is_null = _cur_col->is_null_at(i);
if (is_null) {
ARROW_RETURN_NOT_OK(builder.AppendNull());
continue;
}
const auto& data_ref = _cur_col->get_data_at(i);
ARROW_RETURN_NOT_OK(builder.Append(*(const bool*)data_ref.data));
}
return arrow::Status::OK();
}

// process array type
arrow::Status Visit(const arrow::ListType& type) override {
auto& builder = assert_cast<arrow::ListBuilder&>(*_cur_builder);
auto orignal_col = _cur_col;
size_t start = _cur_start;
size_t num_rows = _cur_rows;

const vectorized::ColumnArray* array_column = nullptr;
if (orignal_col->is_nullable()) {
auto nullable_column =
assert_cast<const vectorized::ColumnNullable*>(orignal_col.get());
array_column = assert_cast<const vectorized::ColumnArray*>(
&nullable_column->get_nested_column());
} else {
array_column = assert_cast<const vectorized::ColumnArray*>(orignal_col.get());
}
const auto& offsets = array_column->get_offsets();
vectorized::ColumnPtr nested_column = array_column->get_data_ptr();

// set current col/type/builder to nested
_cur_col = nested_column;
if (_cur_type->is_nullable()) {
auto nullable_type = assert_cast<const vectorized::DataTypeNullable*>(_cur_type.get());
_cur_type = assert_cast<const vectorized::DataTypeArray*>(
nullable_type->get_nested_type().get())
->get_nested_type();
} else {
_cur_type = assert_cast<const vectorized::DataTypeArray*>(_cur_type.get())
->get_nested_type();
}
_cur_builder = builder.value_builder();

ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
for (size_t i = start; i < start + num_rows; ++i) {
bool is_null = orignal_col->is_null_at(i);
if (is_null) {
ARROW_RETURN_NOT_OK(builder.AppendNull());
continue;
}
// append array elements in row i
ARROW_RETURN_NOT_OK(builder.Append());
_cur_start = offsets[i - 1];
_cur_rows = offsets[i] - offsets[i - 1];
ARROW_RETURN_NOT_OK(arrow::VisitTypeInline(*type.value_type(), this));
}

return arrow::Status::OK();
}
~FromBlockConverter() = default;

Status convert(std::shared_ptr<arrow::RecordBatch>* out);

private:
template <typename T>
arrow::Status _visit(const T& type) {
auto& builder = assert_cast<arrow::NumericBuilder<T>&>(*_cur_builder);
size_t start = _cur_start;
size_t num_rows = _cur_rows;
ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
if (_cur_col->is_nullable()) {
for (size_t i = start; i < start + num_rows; ++i) {
bool is_null = _cur_col->is_null_at(i);
if (is_null) {
ARROW_RETURN_NOT_OK(builder.AppendNull());
continue;
}
const auto& data_ref = _cur_col->get_data_at(i);
ARROW_RETURN_NOT_OK(builder.Append(*(const typename T::c_type*)data_ref.data));
}
} else {
ARROW_RETURN_NOT_OK(builder.AppendValues(
(const typename T::c_type*)_cur_col->get_data_at(start).data, num_rows));
}
return arrow::Status::OK();
}

const vectorized::Block& _block;
const std::shared_ptr<arrow::Schema>& _schema;
arrow::MemoryPool* _pool;
Expand Down
2 changes: 0 additions & 2 deletions be/src/util/arrow/row_batch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,6 @@ Status convert_to_arrow_type(const TypeDescriptor& type, std::shared_ptr<arrow::
}
break;
case TYPE_DECIMALV2:
*result = std::make_shared<arrow::Decimal128Type>(27, 9);
break;
case TYPE_DECIMAL32:
case TYPE_DECIMAL64:
case TYPE_DECIMAL128I:
Expand Down
1 change: 1 addition & 0 deletions be/src/vec/columns/column_array.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ void ColumnArray::update_crcs_with_value(uint32_t* __restrict hash, PrimitiveTyp
}

void ColumnArray::insert(const Field& x) {
DCHECK_EQ(x.get_type(), Field::Types::Array);
if (x.is_null()) {
get_data().insert(Null());
get_offsets().push_back(get_offsets().back() + 1);
Expand Down
1 change: 1 addition & 0 deletions be/src/vec/columns/column_map.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ void ColumnMap::insert_data(const char*, size_t) {
}

void ColumnMap::insert(const Field& x) {
DCHECK_EQ(x.get_type(), Field::Types::Map);
const auto& map = doris::vectorized::get<const Map&>(x);
CHECK_EQ(map.size(), 2);
const auto& k_f = doris::vectorized::get<const Array&>(map[0]);
Expand Down
Loading