Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions be/cmake/thirdparty.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -189,10 +189,8 @@ if (OS_MACOSX)
add_thirdparty(intl)
endif()

# Only used on x86 or x86_64
if ("${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86" OR "${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86_64")
add_thirdparty(deflate)
endif()
# The pinned ORC decoder uses libdeflate on ARM too; omitting it leaves unresolved decoder symbols.
add_thirdparty(deflate)

add_thirdparty(icuuc LIB64)
add_thirdparty(icui18n LIB64)
Expand Down
9 changes: 7 additions & 2 deletions be/src/exec/es/es_scan_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@ namespace doris {

// hits.hits._id used for obtain ES document `_id`
const std::string SOURCE_SCROLL_SEARCH_FILTER_PATH =
"filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id";
"filter_path=_scroll_id,hits.hits._source,hits.hits._index,hits.total,hits.hits._id";
// hits.hits._score used for processing field not exists in one batch
const std::string DOCVALUE_SCROLL_SEARCH_FILTER_PATH =
"filter_path=_scroll_id,hits.total,hits.hits._score,hits.hits.fields";
"filter_path=_scroll_id,hits.total,hits.hits._index,hits.hits._score,hits.hits.fields";

const std::string REQUEST_PREFERENCE_PREFIX = "&preference=_shards:";
const std::string REQUEST_SEARCH_SCROLL_PATH = "/_search/scroll";
Expand Down Expand Up @@ -72,6 +72,10 @@ ESScanReader::ESScanReader(const std::string& target,
std::istringstream(props.at(KEY_HTTP_SSL_ENABLED)) >> std::boolalpha >> _use_ssl_client;
}

// The same numeric string can mean a compact calendar date or an epoch on different backings.
if (const auto formats = props.find("date_format_context"); formats != props.end()) {
_date_format_context = formats->second;
}
std::string batch_size_str = props.at(KEY_BATCH_SIZE);
_batch_size = atoi(batch_size_str.c_str());
std::string filter_path =
Expand Down Expand Up @@ -216,6 +220,7 @@ Status ESScanReader::get_next(bool* scan_eos, std::unique_ptr<ScrollParser>& scr
}

scroll_parser.reset(new ScrollParser(_doc_value_mode));
RETURN_IF_ERROR(scroll_parser->set_date_format_context(_date_format_context));
VLOG_CRITICAL << "get_next request ES, returned response: " << response;
Status status = scroll_parser->parse(response, _exactly_once);
if (!status.ok()) {
Expand Down
1 change: 1 addition & 0 deletions be/src/exec/es/es_scan_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ class ESScanReader {
std::string _type;
// push down filter
std::string _query;
std::string _date_format_context;
// Elasticsearch shards to fetch document
std::string _shards;
// whether use ssl client
Expand Down
Loading
Loading