Repository navigation
Description
Describe the bug
using stream load to load one row into a table but core dump happends.
core dump as below:
#0 0x000000000211b0ee in std::default_deletedoris::segment_v2::PageBuilder::operator() (this=0x3f43b198, __ptr=0x3dd1bf40)
at /usr/local/include/c++/7.2.0/bits/unique_ptr.h:78
#1 0x000000000211a385 in std::unique_ptr<doris::segment_v2::PageBuilder, std::default_deletedoris::segment_v2::PageBuilder >::~unique_ptr (
this=0x3f43b198, __in_chrg=) at /usr/local/include/c++/7.2.0/bits/unique_ptr.h:268
#2 0x00000000021160d7 in doris::segment_v2::ColumnWriter::~ColumnWriter (this=0x3f43b140, __in_chrg=)
at /wangbo/doris//be/src/olap/rowset/segment_v2/column_writer.cpp:84
#3 0x00000000020c5a72 in std::default_deletedoris::segment_v2::ColumnWriter::operator() (this=0x3f55f990, __ptr=0x3f43b140)
at /usr/local/include/c++/7.2.0/bits/unique_ptr.h:78
#4 0x00000000020c5157 in std::unique_ptr<doris::segment_v2::ColumnWriter, std::default_deletedoris::segment_v2::ColumnWriter >::~unique_ptr (
this=0x3f55f990, __in_chrg=) at /usr/local/include/c++/7.2.0/bits/unique_ptr.h:268
#5 0x00000000020c6b86 in std::_Destroy<std::unique_ptr<doris::segment_v2::ColumnWriter, std::default_deletedoris::segment_v2::ColumnWriter > > (
__pointer=0x3f55f990) at /usr/local/include/c++/7.2.0/bits/stl_construct.h:98
#6 0x00000000020c6701 in std::_Destroy_aux::__destroy<std::unique_ptr<doris::segment_v2::ColumnWriter, std::default_deletedoris::segment_v2::ColumnWriter >> (__first=0x3f55f990, __last=0x3f55fa18) at /usr/local/include/c++/7.2.0/bits/stl_construct.h:108
#7 0x00000000020c6044 in std::_Destroy<std::unique_ptr<doris::segment_v2::ColumnWriter, std::default_deletedoris::segment_v2::ColumnWriter >> (
__first=0x3f55f800, __last=0x3f55fa18) at /usr/local/include/c++/7.2.0/bits/stl_construct.h:137
#8 0x00000000020c5887 in std::_Destroy<std::unique_ptr<doris::segment_v2::ColumnWriter, std::default_deletedoris::segment_v2::ColumnWriter >*, std::unique_ptr<doris::segment_v2::ColumnWriter, std::default_deletedoris::segment_v2::ColumnWriter > > (__first=0x3f55f800, __last=0x3f55fa18)
at /usr/local/include/c++/7.2.0/bits/stl_construct.h:206
#9 0x00000000020c4fd5 in std::vector<std::unique_ptr<doris::segment_v2::ColumnWriter, std::default_deletedoris::segment_v2::ColumnWriter >, std::allocator<std::unique_ptr<doris::segment_v2::ColumnWriter, std::default_deletedoris::segment_v2::ColumnWriter > > >::~vector (this=0x3b099080,
__in_chrg=) at /usr/local/include/c++/7.2.0/bits/stl_vector.h:434
#10 0x00000000020c2784 in doris::segment_v2::SegmentWriter::~SegmentWriter (this=0x3b098fc0, __in_chrg=)
at /wangbo/doris//be/src/olap/rowset/segment_v2/segment_writer.h:49
#11 0x000000000196279a in std::default_deletedoris::segment_v2::SegmentWriter::operator() (this=0x3d9ee218, __ptr=0x3b098fc0)
at /usr/local/include/c++/7.2.0/bits/unique_ptr.h:78
#12 0x0000000001962679 in std::unique_ptr<doris::segment_v2::SegmentWriter, std::default_deletedoris::segment_v2::SegmentWriter >::reset (
this=0x3d9ee218, __p=0x3b098fc0) at /usr/local/include/c++/7.2.0/bits/unique_ptr.h:376
#13 0x000000000195f3a8 in doris::BetaRowsetWriter::_flush_segment_writer (this=0x3d9ee120)
at /wangbo/doris//be/src/olap/rowset/beta_rowset_writer.cpp:183
#14 0x000000000195ed63 in doris::BetaRowsetWriter::flush (this=0x3d9ee120)
at /wangbo/doris//be/src/olap/rowset/beta_rowset_writer.cpp:126
#15 0x0000000001923474 in doris::MemTable::flush (this=0x3d9ec330) at /wangbo/doris//be/src/olap/memtable.cpp:94
#16 0x000000000189ffaf in doris::MemTableFlushExecutor::_flush_memtable (this=0x84299d0, queue_idx=0)
at /wangbo/doris//be/src/olap/memtable_flush_executor.cpp:145
#17 0x00000000018ac016 in boost::_mfi::mf1<void, doris::MemTableFlushExecutor, int>::operator() (this=0x8c95820, p=0x84299d0, a1=0)
at /wangbo/doris//thirdparty/installed/include/boost/bind/mem_fn_template.hpp:165
To Reproduce
code commit:9c2d149c3654686ab45abad1c72ffcc8d3ba5d1b
envirenment:
1fe node,3be node
table desc:
CREATE TABLE `stream_load_test` (
`partition_date` date null COMMENT "",
`b_int_1` bigint(20) null COMMENT "",
`b_int_2` bigint(20) null COMMENT "",
`varchar_1` varchar(6) null COMMENT "",
`b_int_3` bigint(20) null COMMENT "",
`b_int_4` bigint(20) null COMMENT "",
`varchar_2` varchar(100) null COMMENT "",
`varchar_3` varchar(100) null COMMENT "",
`b_int_5` bigint(20) null COMMENT "",
`b_int_6` bigint(20) null COMMENT "",
`b_int_7` bigint(20) null COMMENT "",
`b_int_8` bigint(20) null COMMENT "",
`b_int_9` bigint(20) null COMMENT "",
`b_int_10` bigint(20) null COMMENT "",
`b_int_11` bigint(20) null COMMENT "",
`b_int_12` bigint(20) null COMMENT "",
`b_int_13` bigint(20) null COMMENT "",
`varchar_4` varchar(100) null COMMENT "",
`varchar_5` varchar(100) null COMMENT "",
`varchar_6` varchar(100) null COMMENT "",
`varchar_7` varchar(100) null COMMENT "",
`varchar_8` varchar(100) null COMMENT "",
`varchar_9` varchar(100) null COMMENT "",
`varchar_10` varchar(100) null COMMENT "",
`varchar_11` varchar(100) null COMMENT "",
`varchar_12` varchar(100) null COMMENT "",
`varchar_13` varchar(100) null COMMENT "",
`varchar_14` varchar(100) null COMMENT "",
`varchar_15` varchar(100) null COMMENT "",
`varchar_16` varchar(100) null COMMENT "",
`varchar_17` varchar(100) null COMMENT "",
`varchar_18` varchar(100) null COMMENT "",
`varchar_19` varchar(100) null COMMENT "",
`decimal_1` decimal(20, 4) null COMMENT "",
`decimal_2` decimal(20, 4) null COMMENT "",
`decimal_3` decimal(20, 4) null COMMENT "",
`decimal_4` decimal(20, 4) null COMMENT "",
`b_int_14` bigint(20) null COMMENT "",
`b_int_15` bigint(20) null COMMENT "",
`varchar_20` varchar(100) null COMMENT "",
`varchar_21` varchar(50) null COMMENT "",
`int_1` int(11) null COMMENT "",
`varchar_22` varchar(50) null COMMENT "",
`varchar_23` varchar(50) null COMMENT "",
`b_int_16` bigint(20) null COMMENT "",
`d_sum_1` double SUM null COMMENT "",
`d_sum_2` double SUM null COMMENT "",
`d_sum_3` double SUM null COMMENT "",
`d_sum_4` double SUM null COMMENT "",
`d_sum_5` double SUM null COMMENT "",
`d_sum_6` double SUM null COMMENT "",
`d_sum_7` double SUM null COMMENT "",
`dc_sum_1` decimal(20, 4) SUM null COMMENT "",
`d_sum_8` double SUM null COMMENT "",
`dc_sum_2` decimal(20, 4) SUM null COMMENT "",
`d_sum_9` double SUM null COMMENT "",
`d_sum_10` double SUM null COMMENT "",
`d_sum_11` double SUM null COMMENT "",
`d_sum_12` double SUM null COMMENT "",
`d_sum_13` double SUM null COMMENT "",
`d_sum_14` double SUM null COMMENT "",
`dc_sum_3` decimal(20, 4) SUM null COMMENT "",
`d_sum_15` double SUM null COMMENT "",
`d_sum_16` double SUM null COMMENT "",
`d_sum_17` double SUM null COMMENT "",
`d_sum_18` double SUM null COMMENT "",
`d_sum_19` double SUM null COMMENT ""
) ENGINE=OLAP
AGGREGATE KEY(`partition_date`,`b_int_1` ,`b_int_2` ,`varchar_1`,`b_int_3` ,`b_int_4` ,`varchar_2`,`varchar_3`,`b_int_5` ,`b_int_6` ,`b_int_7`,`b_int_8` ,`b_int_9` ,`b_int_10` ,`b_int_11` ,`b_int_12` ,`b_int_13` ,`varchar_4` ,`varchar_5` ,`varchar_6` ,`varchar_7` ,`varchar_8` ,`varchar_9` ,`varchar_10`,`varchar_11`,`varchar_12`,`varchar_13`,`varchar_14`,`varchar_15`,`varchar_16`,`varchar_17`,`varchar_18`,`varchar_19`,`decimal_1`,`decimal_2`,`decimal_3`,`decimal_4`,`b_int_14`,`b_int_15`,`varchar_20`,`varchar_21`,`int_1`,`varchar_22`,`varchar_23`,`b_int_16`)
PARTITION BY RANGE(`partition_date`)
(PARTITION p201905 VALUES LESS THAN ('2019-06-01'))
DISTRIBUTED BY HASH(`b_int_2`) BUCKETS 100
PROPERTIES (
"storage_type" = "COLUMN",
"replication_num" = "1"
);input data:
2019-05-10 5 52062434 201905 109 109 mt voucher 1080183 100855 569 100912 70 100855 \N \N \N 字符串1 字符串2 字符串3 字符串4 字符串5 字符串6 字符串7 字符串8 richarhao 字符串9 字符串10 字符串10 字符串11 \N \N \N 0.1667 0.0000 0.0000 0.0000 \N \N \N \N 0 50110212_voucher_mt 50110212_voucher_mt_52062434 52062434 0.0 0.0 0.0 0.0 0.0 0.9 0.0833 0.0000 7.5 0.0000 0.0 0.0 0.0 0.0 0.0 0.0 0.0000 0.0 0.0 0.0 0.0 0.0
Additional context
It seems that the table desc effects, I test create a tablet with a less columns,stream load is success;
See core dumped be's log,the log stops at doing DeltaWriter's close,some DeltaWriter has finished
Activity
wangbo commentedon Oct 23, 2019
wangbo commentedon Oct 23, 2019
use ASAN to build code and reproduce the case,stack as above;
In thread T653,Page try to deconstruct slice's data,but failed,the address is 0x60e0005e6ba8;
the 0x60e0005e6ba8 is allocate in a new BitshuffleBuilder;
The address transfer chain as below:
BitshuffleBuilder.fastring.data() => BitshuffleBuilder.finish => ColumnWriter.pages.data.data => delete ColumnWriter.pages.data.data
the key point is:if faststring didn't GrowArray,the fastring.data() is not a array when using new to allocate;
So I use faststring.release to instead of fastring.data() in all PageBuilder, then the core dump is fixed
(apache#2037)
(apache#2037)
Segment v2 stream load core dump(#2037) (#2075)