Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "core/data_type_serde/arrow_validation.h"
#include "core/data_type_serde/decoded_column_view.h"
#include "core/data_type_serde/orc_serde_utils.h"
#include "core/data_type_serde/orc_timestamp_decode_context.h"
#include "core/data_type_serde/parquet_decode_source.h"
#include "core/data_type_serde/parquet_timestamp.h"
#include "core/types.h"
Expand All @@ -52,8 +53,7 @@ static const int64_t micro_to_nano_second = 1000;

namespace {

Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view,
const cctz::time_zone& timezone) {
Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view) {
const auto* orc_batch = dynamic_cast<const ::orc::TimestampVectorBatch*>(orc_view.batch);
if (orc_batch == nullptr) {
return Status::InternalError("Unexpected ORC timestamp batch type {}",
Expand All @@ -72,9 +72,10 @@ Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColum
}
auto& value =
reinterpret_cast<DateV2Value<DateTimeV2ValueType>&>(data[old_data_size + row]);
const int64_t seconds = orc_batch->data[source_row];
orc_serde_utils::RoundedOrcTimestamp timestamp;
auto status = orc_serde_utils::round_orc_timestamp_to_microseconds(
orc_batch->data[source_row], orc_batch->nanoseconds[source_row], &timestamp);
seconds, orc_batch->nanoseconds[source_row], &timestamp);
if (!status.ok()) {
data.resize(old_data_size);
return status;
Expand All @@ -84,8 +85,9 @@ Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColum
// Instant carry precedes timezone conversion; plain TIMESTAMP carry stays in civil time
// so rounding cannot jump across a daylight-saving gap or fold.
status = orc_serde_utils::orc_timestamp_to_datetime(
is_timestamp_instant ? timestamp.seconds : orc_batch->data[source_row],
timestamp.microseconds, timezone, !is_timestamp_instant && timestamp.carry, &value);
is_timestamp_instant ? timestamp.seconds : seconds, timestamp.microseconds,
orc_view.timestamp_context->timezone_for(*orc_view.file_type, seconds),
!is_timestamp_instant && timestamp.carry, &value);
if (!status.ok()) {
data.resize(old_data_size);
return status;
Expand Down Expand Up @@ -1045,11 +1047,11 @@ Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column,
DORIS_CHECK(view.batch != nullptr);
const auto kind = view.file_type->getKind();
DORIS_CHECK(kind == ::orc::TypeKind::TIMESTAMP || kind == ::orc::TypeKind::TIMESTAMP_INSTANT);
DORIS_CHECK(view.timezone != nullptr);
DORIS_CHECK(view.timestamp_context != nullptr);
if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) {
return Status::OK();
}
return decode_timestamp_orc_values(column, view, *view.timezone);
return decode_timestamp_orc_values(column, view);
}

} // namespace doris
4 changes: 3 additions & 1 deletion be/src/core/data_type_serde/data_type_serde.h
Original file line number Diff line number Diff line change
Expand Up @@ -118,13 +118,15 @@ struct FieldInfo {
int precision = 0;
};

struct OrcTimestampDecodeContext;

struct OrcDecodedColumnView {
const orc::Type* file_type = nullptr;
const orc::Type* selected_type = nullptr;
const orc::ColumnVectorBatch* batch = nullptr;
size_t rows = 0;
const std::vector<size_t>* selected_rows = nullptr;
const cctz::time_zone* timezone = nullptr;
const OrcTimestampDecodeContext* timestamp_context = nullptr;
bool enable_mapping_timestamp_tz = false;
};

Expand Down
185 changes: 184 additions & 1 deletion be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@
#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
#include "core/data_type_serde/arrow_validation.h"
#include "core/data_type_serde/decoded_column_view.h"
#include "core/data_type_serde/orc_serde_utils.h"
#include "core/data_type_serde/orc_timestamp_decode_context.h"
#include "core/data_type_serde/parquet_timestamp.h"
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_timestamp_ns.h"
#include "util/mysql_row_buffer.h"
Expand All @@ -37,6 +41,56 @@ namespace doris {

namespace {

class TimestampNsParquetConsumer final : public ParquetFixedValueConsumer {
public:
TimestampNsParquetConsumer(IColumn& column, const ParquetDecodeContext& context,
ParquetMaterializationState* state = nullptr)
: _data(assert_cast<ColumnTimeStampNs&>(column).get_data()),
_context(context),
_state(state) {}

Status consume(const uint8_t* values, size_t count, size_t width) override {
const auto old_size = _data.size();
for (size_t i = 0; i < count; ++i) {
int64_t nanos;
Status status;
if (_context.physical_type == ParquetPhysicalType::INT96) {
DORIS_CHECK_EQ(width, sizeof(ParquetInt96Timestamp));
status = parquet_int96_timestamp_nanos(
unaligned_load<ParquetInt96Timestamp>(values + i * width), &nanos);
} else if (_context.physical_type == ParquetPhysicalType::INT64) {
DORIS_CHECK_EQ(width, sizeof(int64_t));
status = parquet_timestamp_nanos(
_context.time_unit, unaligned_load<int64_t>(values + i * width), &nanos);
} else {
return Status::NotSupported("TIMESTAMP_NS expects Parquet INT64 or INT96");
}
if (!status.ok()) {
if (_state != nullptr && _state->mark_conversion_failure(_data.size())) {
_data.emplace_back();
continue;
}
_data.resize(old_size);
return status;
}
_data.emplace_back(nanos);
}
return Status::OK();
}

private:
ColumnTimeStampNs::Container& _data;
const ParquetDecodeContext& _context;
ParquetMaterializationState* _state;
};

class RejectTimestampNsBinaryConsumer final : public ParquetBinaryValueConsumer {
public:
Status consume(const StringRef*, size_t) override {
return Status::NotSupported("Binary Parquet values cannot be materialized as TIMESTAMP_NS");
}
};

Status get_nanos_per_arrow_timestamp_unit(arrow::TimeUnit::type unit, int64_t* nanos_per_unit) {
switch (unit) {
case arrow::TimeUnit::SECOND:
Expand Down Expand Up @@ -345,6 +399,113 @@ Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb_vector(
source_null_map);
}

Status DataTypeTimeStampNsSerDe::read_parquet_dictionary(
IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const {
TimestampNsParquetConsumer consumer(column, context);
RejectTimestampNsBinaryConsumer binary;
return source.decode_dictionary(consumer, binary);
}

Status DataTypeTimeStampNsSerDe::read_column_from_parquet(
IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context,
size_t num_values, ParquetMaterializationState& state) const {
TimestampNsParquetConsumer consumer(column, context, &state);
if (context.encoding != ParquetValueEncoding::DICTIONARY) {
return source.decode_fixed_values(num_values, consumer);
}
if (state.dictionary_generation != source.dictionary_generation()) {
state.typed_dictionary = column.clone_empty();
auto* null_map = state.begin_dictionary_conversion(source.dictionary_size());
TimestampNsParquetConsumer dictionary(*state.typed_dictionary, context, &state);
RejectTimestampNsBinaryConsumer binary;
auto status = source.decode_dictionary(dictionary, binary);
state.end_dictionary_conversion(null_map);
RETURN_IF_ERROR(status);
DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size());
state.dictionary_generation = source.dictionary_generation();
}
return state.materialize_dictionary(column, source, num_values);
}

Status DataTypeTimeStampNsSerDe::read_column_from_decoded_values(
IColumn& column, const DecodedColumnView& view) const {
if (view.value_kind != DecodedValueKind::INT64 && view.value_kind != DecodedValueKind::INT96) {
return Status::NotSupported("TIMESTAMP_NS expects decoded INT64 or INT96 values");
}
if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) {
return Status::Corruption("Decoded TIMESTAMP_NS value buffer is null");
}
auto unit = ParquetTimeUnit::MICROS;
if (view.time_unit == DecodedTimeUnit::MILLIS) {
unit = ParquetTimeUnit::MILLIS;
} else if (view.time_unit == DecodedTimeUnit::NANOS) {
unit = ParquetTimeUnit::NANOS;
}
auto& data = assert_cast<ColumnTimeStampNs&>(column).get_data();
const auto old_size = data.size();
for (int64_t row = 0; row < view.row_count; ++row) {
if (decoded_column_view_row_is_null(view, row)) {
data.emplace_back();
continue;
}
int64_t nanos;
auto status =
view.value_kind == DecodedValueKind::INT96
? parquet_int96_timestamp_nanos(
unaligned_load<ParquetInt96Timestamp>(
view.values + row * sizeof(ParquetInt96Timestamp)),
&nanos)
: parquet_timestamp_nanos(
unit,
unaligned_load<int64_t>(view.values + row * sizeof(int64_t)),
&nanos);
if (!status.ok()) {
if (decoded_column_view_can_null_on_conversion_failure(view)) {
decoded_column_view_insert_null_on_conversion_failure(column, view, row);
continue;
}
data.resize(old_size);
return status;
}
data.emplace_back(nanos);
}
return Status::OK();
}

Status DataTypeTimeStampNsSerDe::read_column_from_orc(IColumn& column,
const OrcDecodedColumnView& view) const {
const auto& batch = assert_cast<const orc::TimestampVectorBatch&>(*view.batch);
auto& data = assert_cast<ColumnTimeStampNs&>(column).get_data();
const auto old_size = data.size();
const auto rows = orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows);
DORIS_CHECK(view.file_type != nullptr);
DORIS_CHECK(view.timestamp_context != nullptr);
for (size_t row = 0; row < rows; ++row) {
const auto source = orc_serde_utils::orc_source_row_at(row, view.selected_rows);
if (orc_serde_utils::orc_row_is_null(batch, source)) {
data.emplace_back();
continue;
}
const auto nanos = batch.nanoseconds[source];
if (nanos < 0 || nanos >= TimeStampNsValue::NANOS_PER_SECOND) {
data.resize(old_size);
return Status::DataQualityError("Invalid ORC timestamp nanoseconds: {}", nanos);
}
DateV2Value<DateTimeV2ValueType> civil;
const int64_t seconds = batch.data[source];
const auto timezone = view.timestamp_context->timezone_for(*view.file_type, seconds);
auto status = orc_serde_utils::orc_timestamp_to_datetime(seconds, nanos / 1000, timezone,
Comment thread
Gabriel39 marked this conversation as resolved.
Resolved
false, &civil);
TimeStampNsValue value;
if (!status.ok() || !value.from_datetime(civil, nanos % 1000)) {
data.resize(old_size);
return Status::DataQualityError("ORC timestamp is outside the TIMESTAMP_NS range");
}
data.push_back(value);
}
return Status::OK();
}

Status DataTypeTimeStampNsSerDe::write_column_to_arrow(const IColumn& column,
const NullMap* null_map,
arrow::ArrayBuilder* array_builder,
Expand Down Expand Up @@ -438,7 +599,29 @@ Status DataTypeTimeStampNsSerDe::write_column_to_orc(const std::string& timezone
orc::ColumnVectorBatch* orc_col_batch,
int64_t start, int64_t end, Arena& arena,
const FormatOptions& options) const {
return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_orc");
const auto& data = assert_cast<const ColumnTimeStampNs&>(column).get_data();
auto& batch = assert_cast<orc::TimestampVectorBatch&>(*orc_col_batch);
for (int64_t row = start; row < end; ++row) {
if (batch.notNull[row] == 0) {
continue;
}
int64_t seconds;
auto civil = data[row].to_datetime();
if (!civil.unix_timestamp(&seconds, timezone)) {
return Status::DataQualityError("Cannot convert TIMESTAMP_NS to ORC timestamp");
}
const auto nanos = data[row].nanosecond();
// Match the other timestamp writers: ORC-645 aliases these fractions across the epoch.
if (seconds == -1 && nanos >= 1000000) {
return Status::NotSupported(
"ORC cannot represent pre-epoch timestamp fractions in [-0.999, 0) "
"seconds without data loss; use Parquet for these values");
}
batch.data[row] = seconds;
batch.nanoseconds[row] = nanos;
}
batch.numElements = end - start;
return Status::OK();
}

void DataTypeTimeStampNsSerDe::write_one_cell_to_binary(const IColumn& src_column,
Expand Down
9 changes: 9 additions & 0 deletions be/src/core/data_type_serde/data_type_timestamp_ns_serde.h
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,15 @@ class DataTypeTimeStampNsSerDe final
ColumnNullable& column_to, const ColumnString& column_from, CastParameters& cast_params,
const NullMap::value_type* source_null_map = nullptr) const override;

Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source,
const ParquetDecodeContext& context, size_t num_values,
ParquetMaterializationState& state) const override;
Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source,
const ParquetDecodeContext& context) const override;
Status read_column_from_decoded_values(IColumn& column,
const DecodedColumnView& view) const override;
Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override;

Status write_column_to_arrow(const IColumn& column, const NullMap* null_map,
arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
const cctz::time_zone& ctz) const override;
Expand Down
18 changes: 17 additions & 1 deletion be/src/core/data_type_serde/orc_serde_utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,23 @@
#include "common/cast_set.h"
#include "common/check.h"
#include "core/column/column_array.h"
#include "core/data_type_serde/orc_timestamp_decode_context.h"

namespace doris::orc_serde_utils {

bool has_nanosecond_timestamp(const ::orc::Type& type) {
if (type.getKind() == ::orc::TIMESTAMP && type.hasAttributeKey("iceberg.timestamp-unit") &&
type.getAttributeValue("iceberg.timestamp-unit") == "NANOS") {
return true;
}
for (uint64_t child = 0; child < type.getSubtypeCount(); ++child) {
if (has_nanosecond_timestamp(*type.getSubtype(child))) {
return true;
}
}
return false;
}

size_t orc_decode_row_count(size_t rows, const std::vector<size_t>* selected_rows) {
if (selected_rows == nullptr) {
return rows;
Expand Down Expand Up @@ -93,7 +107,9 @@ DecodedColumnView make_orc_decoded_view(const OrcDecodedColumnView& orc_view,
DecodedColumnView view;
view.value_kind = value_kind;
view.row_count = cast_set<int64_t>(orc_decode_row_count(orc_view.rows, orc_view.selected_rows));
view.timezone = orc_view.timezone;
if (orc_view.timestamp_context != nullptr) {
view.timezone = &orc_view.timestamp_context->session_timezone;
}
return view;
}

Expand Down
3 changes: 3 additions & 0 deletions be/src/core/data_type_serde/orc_serde_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#pragma once

#include <cstring>
#include <orc/Type.hh>
#include <orc/Vector.hh>
#include <vector>

Expand All @@ -29,6 +30,8 @@
namespace doris {
namespace orc_serde_utils {

bool has_nanosecond_timestamp(const ::orc::Type& type);

size_t orc_decode_row_count(size_t rows, const std::vector<size_t>* selected_rows);
size_t orc_source_row_at(size_t row, const std::vector<size_t>* selected_rows);
bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row);
Expand Down
38 changes: 38 additions & 0 deletions be/src/core/data_type_serde/orc_timestamp_decode_context.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "core/data_type_serde/orc_timestamp_decode_context.h"

#include <orc/Type.hh>

#include "common/check.h"

namespace doris {

cctz::time_zone OrcTimestampDecodeContext::timezone_for(const orc::Type& file_type,
int64_t seconds) const {
const auto kind = file_type.getKind();
DORIS_CHECK(kind == orc::TIMESTAMP || kind == orc::TIMESTAMP_INSTANT);
// Plain timestamps reconstruct the SDK reader's civil value. Instant-to-DATETIME conversion
// uses the session zone, which must not follow a plain sibling's stripe writer timezone.
if (kind == orc::TIMESTAMP && writer_timezone) {
return writer_timezone(seconds);
}
return kind == orc::TIMESTAMP ? reader_timezone : session_timezone;
}

} // namespace doris
Loading
Loading