Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions be/src/cloud/cloud_index_change_compaction.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
#include "gen_cpp/cloud.pb.h"
#include "storage/compaction/compaction.h"

namespace doris {
Expand Down
4 changes: 4 additions & 0 deletions be/src/cloud/cloud_schema_change_job.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@

namespace doris {

namespace cloud {
class TabletJobInfoPB;
}

class CloudSchemaChangeJob {
public:
CloudSchemaChangeJob(CloudStorageEngine& cloud_storage_engine, std::string job_id,
Expand Down
1 change: 1 addition & 0 deletions be/src/cloud/cloud_tablet.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include "storage/tablet/base_tablet.h"

namespace doris {
class TTabletInfo;

class CloudStorageEngine;

Expand Down
1 change: 1 addition & 0 deletions be/src/common/signal_handler.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#pragma once

#include <butil/macros.h> // ARRAYSIZE_UNSAFE
#include <gen_cpp/types.pb.h>
#include <glog/logging.h>

#include <boost/stacktrace.hpp>
Expand Down
170 changes: 170 additions & 0 deletions be/src/core/data_type_serde/data_type_number_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <arrow/builder.h>
#include <arrow/util/float16.h>
#include <gen_cpp/types.pb.h>

#include <bit>
#include <cmath>
Expand Down Expand Up @@ -2005,6 +2006,175 @@ Status DataTypeNumberSerDe<T>::read_column_from_orc(IColumn& column,
return DataTypeSerDe::read_column_from_orc(column, view);
}

template <PrimitiveType T>
Status DataTypeNumberSerDe<T>::read_column_from_pb(IColumn& column, const PValues& arg) const {
auto old_column_size = column.size();
if constexpr (T == TYPE_BOOLEAN) {
column.resize(old_column_size + arg.uint32_value_size());
auto& data = assert_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.uint32_value_size(); ++i) {
data[old_column_size + i] =
cast_set<typename PrimitiveTypeTraits<T>::CppType, uint32_t, false>(
arg.uint32_value(i));
}
} else if constexpr (T == TYPE_DATEV2 || T == TYPE_IPV4) {
column.resize(old_column_size + arg.uint32_value_size());
auto& data = assert_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.uint32_value_size(); ++i) {
data[old_column_size + i] = arg.uint32_value(i);
}
} else if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT) {
column.resize(old_column_size + arg.int32_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.int32_value_size(); ++i) {
data[old_column_size + i] =
cast_set<typename PrimitiveTypeTraits<T>::CppType, int32_t, false>(
arg.int32_value(i));
}
} else if constexpr (T == TYPE_INT) {
column.resize(old_column_size + arg.int32_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.int32_value_size(); ++i) {
data[old_column_size + i] = arg.int32_value(i);
}
} else if constexpr (T == TYPE_DATETIMEV2) {
column.resize(old_column_size + arg.uint64_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.uint64_value_size(); ++i) {
data[old_column_size + i] =
binary_cast<UInt64, DateV2Value<DateTimeV2ValueType>>(arg.uint64_value(i));
}
} else if constexpr (T == TYPE_TIMESTAMPTZ) {
column.resize(old_column_size + arg.uint64_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.uint64_value_size(); ++i) {
data[old_column_size + i] = binary_cast<UInt64, TimestampTzValue>(arg.uint64_value(i));
}
} else if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) {
column.resize(old_column_size + arg.int64_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.int64_value_size(); ++i) {
data[old_column_size + i] = binary_cast<Int64, VecDateTimeValue>(arg.int64_value(i));
}
} else if constexpr (T == TYPE_BIGINT) {
column.resize(old_column_size + arg.int64_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.int64_value_size(); ++i) {
data[old_column_size + i] = arg.int64_value(i);
}
} else if constexpr (T == TYPE_TIMESTAMP_NS) {
column.resize(old_column_size + arg.int64_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.int64_value_size(); ++i) {
data[old_column_size + i] = TimeStampNsValue(arg.int64_value(i));
}
} else if constexpr (T == TYPE_FLOAT) {
column.resize(old_column_size + arg.float_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.float_value_size(); ++i) {
data[old_column_size + i] = arg.float_value(i);
}
} else if constexpr (T == TYPE_DOUBLE || T == TYPE_TIMEV2) {
column.resize(old_column_size + arg.double_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.double_value_size(); ++i) {
data[old_column_size + i] = arg.double_value(i);
}
} else if constexpr (T == TYPE_LARGEINT) {
column.resize(old_column_size + arg.bytes_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.bytes_value_size(); ++i) {
data[old_column_size + i] = *(int128_t*)(arg.bytes_value(i).c_str());
}
} else {
return Status::NotSupported("unknown ColumnType for reading from pb");
}
return Status::OK();
}

template <PrimitiveType T>
Status DataTypeNumberSerDe<T>::write_column_to_pb(const IColumn& column, PValues& result,
int64_t start, int64_t end) const {
auto row_count = cast_set<int>(end - start);
auto* ptype = result.mutable_type();
const auto* col = assert_cast<const ColumnType*>(&column);
if constexpr (T == TYPE_LARGEINT) {
ptype->set_id(PGenericType::INT128);
result.mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
StringRef single_data = col->get_data_at(row_num);
result.add_bytes_value(single_data.data, single_data.size);
}
return Status::OK();
}
auto& data = col->get_data();
if constexpr (T == TYPE_BOOLEAN) {
ptype->set_id(PGenericType::UINT8);
auto* values = result.mutable_uint32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (T == TYPE_DATEV2) {
ptype->set_id(PGenericType::UINT32);
auto* values = result.mutable_uint32_value();
values->Reserve(row_count);
values->Add((uint32_t*)data.begin() + start, (uint32_t*)data.begin() + end);
} else if constexpr (T == TYPE_IPV4) {
ptype->set_id(PGenericType::UINT32);
auto* values = result.mutable_uint32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) {
ptype->set_id(PGenericType::UINT64);
auto* values = result.mutable_uint64_value();
values->Reserve(row_count);
values->Add((uint64_t*)data.begin() + start, (uint64_t*)data.begin() + end);
} else if constexpr (T == TYPE_TINYINT) {
ptype->set_id(PGenericType::INT8);
auto* values = result.mutable_int32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (T == TYPE_SMALLINT) {
ptype->set_id(PGenericType::INT16);
auto* values = result.mutable_int32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (T == TYPE_INT) {
ptype->set_id(PGenericType::INT32);
auto* values = result.mutable_int32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) {
ptype->set_id(PGenericType::INT64);
auto* values = result.mutable_int64_value();
values->Reserve(row_count);
values->Add((int64_t*)data.begin() + start, (int64_t*)data.begin() + end);
} else if constexpr (T == TYPE_BIGINT) {
ptype->set_id(PGenericType::INT64);
auto* values = result.mutable_int64_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (T == TYPE_TIMESTAMP_NS) {
ptype->set_id(PGenericType::INT64);
auto* values = result.mutable_int64_value();
values->Reserve(row_count);
values->Add(reinterpret_cast<const int64_t*>(data.begin()) + start,
reinterpret_cast<const int64_t*>(data.begin()) + end);
} else if constexpr (T == TYPE_FLOAT) {
ptype->set_id(PGenericType::FLOAT);
auto* values = result.mutable_float_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (T == TYPE_DOUBLE || T == TYPE_TIMEV2) {
ptype->set_id(PGenericType::DOUBLE);
auto* values = result.mutable_double_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else {
return Status::NotSupported("unknown ColumnType for writing to pb");
}
return Status::OK();
}

/// Explicit template instantiations - to avoid code bloat in headers.
template class DataTypeNumberSerDe<TYPE_BOOLEAN>;
template class DataTypeNumberSerDe<TYPE_TINYINT>;
Expand Down
Loading
Loading