Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions gensrc/thrift/DataSinks.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ enum TDataSinkType {
MULTI_CAST_DATA_STREAM_SINK,
GROUP_COMMIT_OLAP_TABLE_SINK, // deprecated
GROUP_COMMIT_BLOCK_SINK,
HIVE_TABLE_SINK,
}

enum TResultSinkType {
Expand Down Expand Up @@ -101,7 +102,7 @@ enum TParquetRepetitionType {
struct TParquetSchema {
1: optional TParquetRepetitionType schema_repetition_type
2: optional TParquetDataType schema_data_type
3: optional string schema_column_name
3: optional string schema_column_name
4: optional TParquetDataLogicalType schema_data_logical_type
}

Expand Down Expand Up @@ -282,6 +283,7 @@ struct TOlapTableSink {
struct THiveLocationParams {
1: optional string write_path
2: optional string target_path
3: optional Types.TFileType file_type
}

struct TSortedColumn {
Expand All @@ -300,11 +302,16 @@ struct THiveBucket {
4: optional list<TSortedColumn> sorted_by
}

enum THiveCompressionType {
SNAPPY = 3,
LZ4 = 4,
ZLIB = 6,
ZSTD = 7,
enum THiveColumnType {
PARTITION_KEY = 0,
REGULAR = 1,
SYNTHESIZED = 2
}

struct THiveColumn {
1: optional string name
2: optional Types.TTypeDesc data_type
3: optional THiveColumnType column_type
}

struct THivePartition {
Expand All @@ -316,13 +323,14 @@ struct THivePartition {
struct THiveTableSink {
1: optional string db_name
2: optional string table_name
3: optional list<string> data_column_names
4: optional list<string> partition_column_names
5: optional list<THivePartition> partitions
6: optional list<THiveBucket> buckets
7: optional PlanNodes.TFileFormatType file_format
8: optional THiveCompressionType compression_type
9: optional THiveLocationParams location
3: optional list<THiveColumn> columns
4: optional list<THivePartition> partitions
5: optional THiveBucket bucket_info
6: optional PlanNodes.TFileFormatType file_format
7: optional PlanNodes.TFileCompressType compression_type
8: optional THiveLocationParams location
9: optional map<string, string> hadoop_config
10: optional bool overwrite
}

enum TUpdateMode {
Expand Down
3 changes: 3 additions & 0 deletions gensrc/thrift/FrontendService.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ include "Exprs.thrift"
include "RuntimeProfile.thrift"
include "MasterService.thrift"
include "AgentService.thrift"
include "DataSinks.thrift"

// These are supporting structs for JniFrontend.java, which serves as the glue
// between our C++ execution environment and the Java frontend.
Expand Down Expand Up @@ -481,6 +482,8 @@ struct TReportExecStatusParams {
24: optional TQueryStatistics query_statistics // deprecated

25: optional TReportWorkloadRuntimeStatusParams report_workload_runtime_status

26: optional list<DataSinks.THivePartitionUpdate> hive_partition_updates
}

struct TFeResult {
Expand Down
8 changes: 7 additions & 1 deletion gensrc/thrift/Partitions.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,13 @@ enum TPartitionType {
BUCKET_SHFFULE_HASH_PARTITIONED,

// used for shuffle data by parititon and tablet
TABLET_SINK_SHUFFLE_PARTITIONED
TABLET_SINK_SHUFFLE_PARTITIONED,

// used for shuffle data by hive parititon
TABLE_SINK_HASH_PARTITIONED,

// used for hive unparititoned table
TABLE_SINK_RANDOM_PARTITIONED
}

enum TDistributionType {
Expand Down
4 changes: 3 additions & 1 deletion gensrc/thrift/PlanNodes.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,9 @@ enum TFileCompressType {
DEFLATE,
LZOP,
LZ4BLOCK,
SNAPPYBLOCK
SNAPPYBLOCK,
ZLIB,
ZSTD
}

struct THdfsConf {
Expand Down