From 48d47d26a4f18e180fa5602ad0aa89cc7af234a4 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 14 Sep 2023 14:29:35 +0000 Subject: [PATCH 01/45] Add simple unit test for full sorting join --- .../Transforms/MergeJoinTransform.cpp | 71 +++++++++----- .../Transforms/MergeJoinTransform.h | 30 ++++-- .../tests/gtest_merge_join_algorithm.cpp | 95 +++++++++++++++++++ 3 files changed, 166 insertions(+), 30 deletions(-) create mode 100644 src/Processors/tests/gtest_merge_join_algorithm.cpp diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 159a3244fe9..a9fd7978249 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -270,35 +270,45 @@ bool FullMergeJoinCursor::fullyCompleted() const } MergeJoinAlgorithm::MergeJoinAlgorithm( - JoinPtr table_join_, + JoinKind kind_, + JoinStrictness strictness_, + const TableJoin::JoinOnClause & on_clause_, const Blocks & input_headers, size_t max_block_size_) - : table_join(table_join_) + : kind(kind_) + , strictness(strictness_) , max_block_size(max_block_size_) , log(getLogger("MergeJoinAlgorithm")) { if (input_headers.size() != 2) throw Exception(ErrorCodes::LOGICAL_ERROR, "MergeJoinAlgorithm requires exactly two inputs"); - auto strictness = table_join->getTableJoin().strictness(); if (strictness != JoinStrictness::Any && strictness != JoinStrictness::All) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for strictness {}", strictness); - auto kind = table_join->getTableJoin().kind(); if (!isInner(kind) && !isLeft(kind) && !isRight(kind) && !isFull(kind)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for kind {}", kind); - const auto & join_on = table_join->getTableJoin().getOnlyClause(); - - if (join_on.on_filter_condition_left || join_on.on_filter_condition_right) + if (on_clause_.on_filter_condition_left || on_clause_.on_filter_condition_right) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ON filter conditions"); cursors = { - createCursor(input_headers[0], join_on.key_names_left), - createCursor(input_headers[1], join_on.key_names_right) + createCursor(input_headers[0], on_clause_.key_names_left), + createCursor(input_headers[1], on_clause_.key_names_right) }; - for (const auto & [left_key, right_key] : table_join->getTableJoin().leftToRightKeyRemap()) +MergeJoinAlgorithm::MergeJoinAlgorithm( + JoinPtr join_ptr, + const Blocks & input_headers, + size_t max_block_size_) + : MergeJoinAlgorithm( + join_ptr->getTableJoin().kind(), + join_ptr->getTableJoin().strictness(), + join_ptr->getTableJoin().getOnlyClause(), + input_headers, + max_block_size_) +{ + for (const auto & [left_key, right_key] : join_ptr->getTableJoin().leftToRightKeyRemap()) { size_t left_idx = input_headers[0].getPositionByName(left_key); size_t right_idx = input_headers[1].getPositionByName(right_key); @@ -398,7 +408,7 @@ struct AllJoinImpl size_t lnum = nextDistinct(left_cursor.cursor); size_t rnum = nextDistinct(right_cursor.cursor); - bool all_fit_in_block = std::max(left_map.size(), right_map.size()) + lnum * rnum <= max_block_size; + bool all_fit_in_block = !max_block_size || std::max(left_map.size(), right_map.size()) + lnum * rnum <= max_block_size; bool have_all_ranges = left_cursor.cursor.isValid() && right_cursor.cursor.isValid(); if (all_fit_in_block && have_all_ranges) { @@ -498,7 +508,7 @@ std::optional MergeJoinAlgorithm::handleAllJoinState } size_t total_rows = 0; - while (total_rows < max_block_size) + while (!max_block_size || total_rows < max_block_size) { const auto & left_range = all_join_state->getLeft(); const auto & right_range = all_join_state->getRight(); @@ -523,7 +533,7 @@ std::optional MergeJoinAlgorithm::handleAllJoinState return {}; } -MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) +MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin() { PaddedPODArray idx_map[2]; @@ -671,8 +681,6 @@ std::optional MergeJoinAlgorithm::handleAnyJoinState if (any_join_state.empty()) return {}; - auto kind = table_join->getTableJoin().kind(); - Chunk result; for (size_t source_num = 0; source_num < 2; ++source_num) @@ -717,7 +725,7 @@ std::optional MergeJoinAlgorithm::handleAnyJoinState return {}; } -MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin(JoinKind kind) +MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin() { if (auto result = handleAnyJoinState()) return std::move(*result); @@ -804,8 +812,6 @@ Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num) IMergingAlgorithm::Status MergeJoinAlgorithm::merge() { - auto kind = table_join->getTableJoin().kind(); - if (!cursors[0]->cursor.isValid() && !cursors[0]->fullyCompleted()) return Status(0); @@ -849,13 +855,11 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() } } - auto strictness = table_join->getTableJoin().strictness(); - if (strictness == JoinStrictness::Any) - return anyJoin(kind); + return anyJoin(); if (strictness == JoinStrictness::All) - return allJoin(kind); + return allJoin(); throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported strictness '{}'", strictness); } @@ -874,9 +878,30 @@ MergeJoinTransform::MergeJoinTransform( /* always_read_till_end_= */ false, /* empty_chunk_on_finish_= */ true, table_join, input_headers, max_block_size) +<<<<<<< HEAD , log(getLogger("MergeJoinTransform")) +======= +>>>>>>> b4a16f38320 (Add simple unit test for full sorting join) +{ +} + +MergeJoinTransform::MergeJoinTransform( + JoinKind kind_, + JoinStrictness strictness_, + const TableJoin::JoinOnClause & on_clause_, + const Blocks & input_headers, + const Block & output_header, + size_t max_block_size, + UInt64 limit_hint_) + : IMergingTransform( + input_headers, + output_header, + /* have_all_inputs_= */ true, + limit_hint_, + /* always_read_till_end_= */ false, + /* empty_chunk_on_finish_= */ true, + kind_, strictness_, on_clause_, input_headers, max_block_size) { - LOG_TRACE(log, "Use MergeJoinTransform"); } void MergeJoinTransform::onFinish() diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index cf9331abd59..0b0efa33722 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -19,6 +20,7 @@ #include #include #include +#include namespace Poco { class Logger; } @@ -227,7 +229,13 @@ private: class MergeJoinAlgorithm final : public IMergingAlgorithm { public: - explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_); + MergeJoinAlgorithm(JoinKind kind_, + JoinStrictness strictness_, + const TableJoin::JoinOnClause & on_clause_, + const Blocks & input_headers, + size_t max_block_size_); + + MergeJoinAlgorithm(JoinPtr join_ptr, const Blocks & input_headers, size_t max_block_size_); const char * getName() const override { return "MergeJoinAlgorithm"; } void initialize(Inputs inputs) override; @@ -238,10 +246,10 @@ public: private: std::optional handleAnyJoinState(); - Status anyJoin(JoinKind kind); + Status anyJoin(); std::optional handleAllJoinState(); - Status allJoin(JoinKind kind); + Status allJoin(); Chunk createBlockWithDefaults(size_t source_num); Chunk createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const; @@ -251,11 +259,12 @@ private: std::array cursors; - /// Keep some state to make connection between data in different blocks + /// Keep some state to make handle data from different blocks AnyJoinState any_join_state; std::unique_ptr all_join_state; - JoinPtr table_join; + JoinKind kind; + JoinStrictness strictness; size_t max_block_size; int null_direction_hint = 1; @@ -285,12 +294,19 @@ public: size_t max_block_size, UInt64 limit_hint = 0); + MergeJoinTransform( + JoinKind kind_, + JoinStrictness strictness_, + const TableJoin::JoinOnClause & on_clause_, + const Blocks & input_headers, + const Block & output_header, + size_t max_block_size, + UInt64 limit_hint_ = 0); + String getName() const override { return "MergeJoinTransform"; } protected: void onFinish() override; - - LoggerPtr log; }; } diff --git a/src/Processors/tests/gtest_merge_join_algorithm.cpp b/src/Processors/tests/gtest_merge_join_algorithm.cpp new file mode 100644 index 00000000000..9a8b70efc17 --- /dev/null +++ b/src/Processors/tests/gtest_merge_join_algorithm.cpp @@ -0,0 +1,95 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + + +using namespace DB; + + +QueryPipeline buildJoinPipeline(std::shared_ptr left_source, std::shared_ptr right_source) +{ + Blocks inputs; + inputs.emplace_back(left_source->getPort().getHeader()); + inputs.emplace_back(right_source->getPort().getHeader()); + Block out_header = { + ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared(), "t1.x"), + ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared(), "t2.x"), + }; + + TableJoin::JoinOnClause on_clause; + on_clause.key_names_left = {"x"}; + on_clause.key_names_right = {"x"}; + auto joining = std::make_shared( + JoinKind::Inner, + JoinStrictness::All, + on_clause, + inputs, out_header, /* max_block_size = */ 0); + + chassert(joining->getInputs().size() == 2); + + connect(left_source->getPort(), joining->getInputs().front()); + connect(right_source->getPort(), joining->getInputs().back()); + + auto * output_port = &joining->getOutputPort(); + + auto processors = std::make_shared(); + processors->emplace_back(std::move(left_source)); + processors->emplace_back(std::move(right_source)); + processors->emplace_back(std::move(joining)); + + QueryPipeline pipeline(QueryPlanResourceHolder{}, processors, output_port); + return pipeline; +} + + +std::shared_ptr createSourceWithSingleValue(size_t rows_per_chunk, size_t total_chunks) +{ + Block header = { + ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "x") + }; + + Chunks chunks; + + for (size_t i = 0; i < total_chunks; ++i) + { + auto col = ColumnUInt64::create(rows_per_chunk, 1); + chunks.emplace_back(Columns{std::move(col)}, rows_per_chunk); + } + + return std::make_shared(std::move(header), std::move(chunks)); +} + +TEST(FullSortingJoin, Simple) +try +{ + auto left_source = createSourceWithSingleValue(3, 10); + auto right_source = createSourceWithSingleValue(2, 15); + + auto pipeline = buildJoinPipeline(left_source, right_source); + PullingPipelineExecutor executor(pipeline); + + Block block; + + size_t total_result_rows = 0; + while (executor.pull(block)) + { + total_result_rows += block.rows(); + } + ASSERT_EQ(total_result_rows, 3 * 10 * 2 * 15); +} +catch (Exception & e) +{ + std::cout << e.getStackTraceString() << std::endl; + throw; +} From 2412f8521985c8d31322ed04baa502c4e7543ef6 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 18 Sep 2023 16:14:02 +0000 Subject: [PATCH 02/45] wip full sorting asof join --- .../Transforms/MergeJoinTransform.cpp | 138 ++++++++- .../Transforms/MergeJoinTransform.h | 38 ++- .../tests/gtest_full_sorting_join.cpp | 287 ++++++++++++++++++ .../tests/gtest_merge_join_algorithm.cpp | 95 ------ 4 files changed, 446 insertions(+), 112 deletions(-) create mode 100644 src/Processors/tests/gtest_full_sorting_join.cpp delete mode 100644 src/Processors/tests/gtest_merge_join_algorithm.cpp diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index a9fd7978249..8370e548fcb 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -34,13 +34,15 @@ namespace ErrorCodes namespace { -FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns) +constexpr UInt64 DEFAULT_VALUE_INDEX = std::numeric_limits::max(); + +FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns, JoinStrictness strictness) { SortDescription desc; desc.reserve(columns.size()); for (const auto & name : columns) desc.emplace_back(name); - return std::make_unique(materializeBlock(block), desc); + return std::make_unique(materializeBlock(block), desc, strictness == JoinStrictness::Asof); } template @@ -90,9 +92,10 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, size_t lpos, const SortCursorImpl & rhs, size_t rpos, + size_t key_length, int null_direction_hint) { - for (size_t i = 0; i < lhs.sort_columns_size; ++i) + for (size_t i = 0; i < key_length; ++i) { /// TODO(@vdimir): use nullableCompareAt only if there's nullable columns int cmp = nullableCompareAt(*lhs.sort_columns[i], *rhs.sort_columns[i], lpos, rpos, null_direction_hint); @@ -104,13 +107,18 @@ int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, size_t lpos, int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImpl & rhs, int null_direction_hint) { - return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow(), null_direction_hint); + return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow(), lhs.sort_columns_size, null_direction_hint); +} + +int compareAsofCursors(const FullMergeJoinCursor & lhs, const FullMergeJoinCursor & rhs) +{ + return nullableCompareAt(lhs.getAsofColumn(), rhs.getAsofColumn(), lhs->getRow(), rhs->getRow()); } bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint) { /// The last row of left cursor is less than the current row of the right cursor. - int cmp = compareCursors(lhs, lhs.rows - 1, rhs, rhs.getRow(), null_direction_hint); + int cmp = compareCursors(lhs, lhs.rows - 1, rhs, rhs.getRow(), lhs.sort_columns_size, null_direction_hint); return cmp < 0; } @@ -222,11 +230,11 @@ Chunk getRowFromChunk(const Chunk & chunk, size_t pos) return result; } -void inline addRange(PaddedPODArray & left_map, size_t start, size_t end) +void inline addRange(PaddedPODArray & values, UInt64 start, UInt64 end) { assert(end > start); - for (size_t i = start; i < end; ++i) - left_map.push_back(i); + for (UInt64 i = start; i < end; ++i) + values.push_back(i); } void inline addMany(PaddedPODArray & left_or_right_map, size_t idx, size_t num) @@ -235,6 +243,11 @@ void inline addMany(PaddedPODArray & left_or_right_map, size_t idx, size left_or_right_map.push_back(idx); } +void inline addMany(PaddedPODArray & values, UInt64 value, size_t num) +{ + values.resize_fill(values.size() + num, value); +} + } const Chunk & FullMergeJoinCursor::getCurrent() const @@ -283,9 +296,15 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( if (input_headers.size() != 2) throw Exception(ErrorCodes::LOGICAL_ERROR, "MergeJoinAlgorithm requires exactly two inputs"); - if (strictness != JoinStrictness::Any && strictness != JoinStrictness::All) + if (strictness != JoinStrictness::Any && strictness != JoinStrictness::All && strictness != JoinStrictness::Asof) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for strictness {}", strictness); + if (strictness == JoinStrictness::Asof) + { + if (kind != JoinKind::Left && kind != JoinKind::Inner) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not implement ASOF {} join", kind); + } + if (!isInner(kind) && !isLeft(kind) && !isRight(kind) && !isFull(kind)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for kind {}", kind); @@ -293,8 +312,8 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ON filter conditions"); cursors = { - createCursor(input_headers[0], on_clause_.key_names_left), - createCursor(input_headers[1], on_clause_.key_names_right) + createCursor(input_headers[0], on_clause_.key_names_left, strictness), + createCursor(input_headers[1], on_clause_.key_names_right, strictness), }; MergeJoinAlgorithm::MergeJoinAlgorithm( @@ -313,6 +332,8 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( size_t left_idx = input_headers[0].getPositionByName(left_key); size_t right_idx = input_headers[1].getPositionByName(right_key); left_to_right_key_remap[left_idx] = right_idx; + if (strictness == JoinStrictness::Asof) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ASOF joins USING"); } const auto *smjPtr = typeid_cast(table_join.get()); @@ -321,6 +342,19 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( null_direction_hint = smjPtr->getNullDirection(); } + if (strictness == JoinStrictness::Asof) + setAsofInequality(join_ptr->getTableJoin().getAsofInequality()); +} + +void MergeJoinAlgorithm::setAsofInequality(ASOFJoinInequality asof_inequality_) +{ + if (strictness != JoinStrictness::Asof) + throw Exception(ErrorCodes::LOGICAL_ERROR, "setAsofInequality is only supported for ASOF joins"); + + if (asof_inequality_ == ASOFJoinInequality::None) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ASOF inequality cannot be None"); + + asof_inequality = asof_inequality_; } void MergeJoinAlgorithm::logElapsed(double seconds) @@ -770,6 +804,81 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin() return Status(std::move(result)); } + +MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() +{ + auto & left_cursor = *cursors[0]; + if (!left_cursor->isValid()) + return Status(0); + + auto & right_cursor = *cursors[1]; + if (!right_cursor->isValid()) + return Status(1); + + PaddedPODArray left_map; + PaddedPODArray right_map; + + while (left_cursor->isValid() && right_cursor->isValid()) + { + auto lpos = left_cursor->getRow(); + auto rpos = right_cursor->getRow(); + auto cmp = compareCursors(*left_cursor, *right_cursor); + if (cmp == 0) + { + auto asof_cmp = compareAsofCursors(left_cursor, right_cursor); + if ((asof_inequality == ASOFJoinInequality::Less && asof_cmp <= -1) + || (asof_inequality == ASOFJoinInequality::LessOrEquals && asof_cmp <= 0)) + { + /// First row in right table that is greater (or equal) than current row in left table + /// matches asof join condition the best + left_map.push_back(lpos); + right_map.push_back(rpos); + left_cursor->next(); + continue; + } + + if (asof_inequality == ASOFJoinInequality::Less || asof_inequality == ASOFJoinInequality::LessOrEquals) + { + /// Asof condition is not (yet) satisfied, skip row in right table + right_cursor->next(); + continue; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "TODO: implement ASOF equality join"); + } + else if (cmp < 0) + { + /// no matches for rows in left table, just pass them through + size_t num = nextDistinct(*left_cursor); + if (isLeft(kind)) + { + /// return them with default values at right side + addRange(left_map, lpos, lpos + num); + addMany(right_map, DEFAULT_VALUE_INDEX, num); + } + } + else + { + /// skip rows in right table until we find match for current row in left table + nextDistinct(*right_cursor); + } + } + + chassert(left_map.size() == right_map.size()); + Chunk result; + { + Columns lcols = indexColumns(left_cursor.getCurrent().getColumns(), left_map); + for (auto & col : lcols) + result.addColumn(std::move(col)); + + Columns rcols = indexColumns(right_cursor.getCurrent().getColumns(), right_map); + for (auto & col : rcols) + result.addColumn(std::move(col)); + } + UNUSED(asof_inequality); + return Status(std::move(result)); +} + + /// if `source_num == 0` get data from left cursor and fill defaults at right /// otherwise - vice versa Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const @@ -861,6 +970,9 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() if (strictness == JoinStrictness::All) return allJoin(); + if (strictness == JoinStrictness::Asof) + return asofJoin(); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported strictness '{}'", strictness); } @@ -878,10 +990,6 @@ MergeJoinTransform::MergeJoinTransform( /* always_read_till_end_= */ false, /* empty_chunk_on_finish_= */ true, table_join, input_headers, max_block_size) -<<<<<<< HEAD - , log(getLogger("MergeJoinTransform")) -======= ->>>>>>> b4a16f38320 (Add simple unit test for full sorting join) { } diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 0b0efa33722..3ee01e57992 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -195,10 +195,27 @@ private: class FullMergeJoinCursor : boost::noncopyable { public: - explicit FullMergeJoinCursor(const Block & sample_block_, const SortDescription & description_) + FullMergeJoinCursor( + const Block & sample_block_, + const SortDescription & description_, + bool is_asof = false) : sample_block(sample_block_.cloneEmpty()) , desc(description_) { + if (desc.size() == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty sort description for FullMergeJoinCursor"); + + if (is_asof) + { + /// For ASOF join prefix of sort description is used for equality comparison + /// and the last column is used for inequality comparison and is handled separately + + auto asof_column_description = desc.back(); + desc.pop_back(); + + chassert(asof_column_description.direction == 1 && asof_column_description.nulls_direction == 1); + asof_column_position = sample_block.getPositionByName(asof_column_description.column_name); + } } bool fullyCompleted() const; @@ -209,17 +226,27 @@ public: SortCursorImpl * operator-> () { return &cursor; } const SortCursorImpl * operator-> () const { return &cursor; } + SortCursorImpl & operator* () { return cursor; } + const SortCursorImpl & operator* () const { return cursor; } + SortCursorImpl cursor; const Block & sampleBlock() const { return sample_block; } Columns sampleColumns() const { return sample_block.getColumns(); } + const IColumn & getAsofColumn() const + { + return *cursor.all_columns[asof_column_position]; + } + private: Block sample_block; SortDescription desc; Chunk current_chunk; bool recieved_all_blocks = false; + + size_t asof_column_position; }; /* @@ -242,8 +269,9 @@ public: void consume(Input & input, size_t source_num) override; Status merge() override; - void logElapsed(double seconds); + void setAsofInequality(ASOFJoinInequality asof_inequality_); + void logElapsed(double seconds); private: std::optional handleAnyJoinState(); Status anyJoin(); @@ -251,13 +279,17 @@ private: std::optional handleAllJoinState(); Status allJoin(); + Status asofJoin(); + Chunk createBlockWithDefaults(size_t source_num); Chunk createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const; + /// For `USING` join key columns should have values from right side instead of defaults std::unordered_map left_to_right_key_remap; std::array cursors; + ASOFJoinInequality asof_inequality = ASOFJoinInequality::None; /// Keep some state to make handle data from different blocks AnyJoinState any_join_state; @@ -305,6 +337,8 @@ public: String getName() const override { return "MergeJoinTransform"; } + void setAsofInequality(ASOFJoinInequality asof_inequality_) { algorithm.setAsofInequality(asof_inequality_); } + protected: void onFinish() override; }; diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp new file mode 100644 index 00000000000..888e280b55f --- /dev/null +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -0,0 +1,287 @@ +#include + + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + + +using namespace DB; + +UInt64 getAndPrintRandomSeed() +{ + UInt64 seed = randomSeed(); + std::cerr << "TEST_RANDOM_SEED: " << seed << std::endl; + return seed; +} + +static UInt64 TEST_RANDOM_SEED = getAndPrintRandomSeed(); + +static pcg64 rng(TEST_RANDOM_SEED); + + +QueryPipeline buildJoinPipeline( + std::shared_ptr left_source, + std::shared_ptr right_source, + size_t key_length = 1, + JoinKind kind = JoinKind::Inner, + JoinStrictness strictness = JoinStrictness::All, + ASOFJoinInequality asof_inequality = ASOFJoinInequality::None) +{ + Blocks inputs; + inputs.emplace_back(left_source->getPort().getHeader()); + inputs.emplace_back(right_source->getPort().getHeader()); + + Block out_header; + for (const auto & input : inputs) + { + for (ColumnWithTypeAndName column : input) + { + if (&input == &inputs.front()) + column.name = "t1." + column.name; + else + column.name = "t2." + column.name; + out_header.insert(column); + } + } + + TableJoin::JoinOnClause on_clause; + for (size_t i = 0; i < key_length; ++i) + { + on_clause.key_names_left.emplace_back(inputs[0].getByPosition(i).name); + on_clause.key_names_right.emplace_back(inputs[1].getByPosition(i).name); + } + + auto joining = std::make_shared( + kind, + strictness, + on_clause, + inputs, out_header, /* max_block_size = */ 0); + + if (asof_inequality != ASOFJoinInequality::None) + joining->setAsofInequality(asof_inequality); + + chassert(joining->getInputs().size() == 2); + + connect(left_source->getPort(), joining->getInputs().front()); + connect(right_source->getPort(), joining->getInputs().back()); + + auto * output_port = &joining->getOutputPort(); + + auto processors = std::make_shared(); + processors->emplace_back(std::move(left_source)); + processors->emplace_back(std::move(right_source)); + processors->emplace_back(std::move(joining)); + + QueryPipeline pipeline(QueryPlanResourceHolder{}, processors, output_port); + return pipeline; +} + + +std::shared_ptr oneColumnSource(const std::vector> & values) +{ + Block header = { ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "x") }; + Chunks chunks; + for (const auto & chunk_values : values) + { + auto column = ColumnUInt64::create(); + for (auto n : chunk_values) + column->insertValue(n); + chunks.emplace_back(Chunk(Columns{std::move(column)}, chunk_values.size())); + } + return std::make_shared(header, std::move(chunks)); +} + + +TEST(FullSortingJoin, Simple) +try +{ + auto left_source = oneColumnSource({ {1, 2, 3, 4, 5} }); + auto right_source = oneColumnSource({ {1}, {2}, {3}, {4}, {5} }); + + auto pipeline = buildJoinPipeline(left_source, right_source); + PullingPipelineExecutor executor(pipeline); + + Block block; + + size_t total_result_rows = 0; + while (executor.pull(block)) + total_result_rows += block.rows(); + + ASSERT_EQ(total_result_rows, 5); +} +catch (Exception & e) +{ + std::cout << e.getStackTraceString() << std::endl; + throw; +} + +std::shared_ptr sourceFromRows( + const Block & header, const std::vector> & values, double break_prob = 0.0) +{ + Chunks chunks; + auto columns = header.cloneEmptyColumns(); + + std::uniform_real_distribution<> prob_dis(0.0, 1.0); + + + for (auto row : values) + { + if (!columns.empty() && (row.empty() || prob_dis(rng) < break_prob)) + { + size_t rows = columns.front()->size(); + chunks.emplace_back(std::move(columns), rows); + columns = header.cloneEmptyColumns(); + continue; + } + + for (size_t i = 0; i < columns.size(); ++i) + columns[i]->insert(row[i]); + } + + if (!columns.empty()) + chunks.emplace_back(std::move(columns), columns.front()->size()); + + return std::make_shared(header, std::move(chunks)); +} + + +std::vector> getValuesFromBlock(const Block & block, const Names & names) +{ + std::vector> result; + for (size_t i = 0; i < block.rows(); ++i) + { + auto & row = result.emplace_back(); + for (const auto & name : names) + block.getByName(name).column->get(i, row.emplace_back()); + } + return result; +} + + +Block executePipeline(QueryPipeline & pipeline) +{ + PullingPipelineExecutor executor(pipeline); + + Blocks result_blocks; + while (true) + { + Block block; + bool is_ok = executor.pull(block); + if (!is_ok) + break; + result_blocks.emplace_back(std::move(block)); + } + + return concatenateBlocks(result_blocks); +} + +TEST(FullSortingJoin, Asof) +try +{ + const std::vector chunk_break = {}; + + auto left_source = sourceFromRows({ + ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "key"), + ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "t"), + }, { + {"AMZN", 3}, + {"AMZN", 4}, + {"AMZN", 6}, + }); + + auto right_source = sourceFromRows({ + ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "key"), + ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "t"), + ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "value"), + }, { + {"AAPL", 1, 97}, + chunk_break, + {"AAPL", 2, 98}, + {"AAPL", 3, 99}, + {"AMZN", 1, 100}, + {"AMZN", 2, 110}, + chunk_break, + {"AMZN", 4, 130}, + {"AMZN", 5, 140}, + }); + + auto pipeline = buildJoinPipeline( + left_source, right_source, /* key_length = */ 2, + JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals); + + Block result_block = executePipeline(pipeline); + auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); + ASSERT_EQ(values.size(), 2); + ASSERT_EQ(values[0], (std::vector{"AMZN", 3u, 4u, 130u})); + ASSERT_EQ(values[1], (std::vector{"AMZN", 4u, 4u, 130u})); +} +catch (Exception & e) +{ + std::cout << e.getStackTraceString() << std::endl; + throw; +} + + +TEST(FullSortingJoin, AsofOnlyColumn) +try +{ + const std::vector chunk_break = {}; + + auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} }); + + UInt64 p = std::uniform_int_distribution<>(0, 2)(rng); + double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0); + + auto right_source = sourceFromRows({ + ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "t"), + ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "value"), + }, { + {1, 101}, + {2, 102}, + {4, 104}, + {5, 105}, + {11, 111}, + {15, 115}, + }, + break_prob); + + auto pipeline = buildJoinPipeline( + left_source, right_source, /* key_length = */ 1, + JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals); + + Block result_block = executePipeline(pipeline); + + ASSERT_EQ( + assert_cast(result_block.getByName("t1.x").column.get())->getData(), + (ColumnUInt64::Container{3, 3, 3, 3, 3, 5, 5, 6, 9, 9, 10}) + ); + + ASSERT_EQ( + assert_cast(result_block.getByName("t2.t").column.get())->getData(), + (ColumnUInt64::Container{4, 4, 4, 4, 4, 5, 5, 11, 11, 11, 15}) + ); + + ASSERT_EQ( + assert_cast(result_block.getByName("t2.value").column.get())->getData(), + (ColumnUInt64::Container{104, 104, 104, 104, 104, 105, 105, 111, 111, 111, 115}) + ); +} +catch (Exception & e) +{ + std::cout << e.getStackTraceString() << std::endl; + throw; +} diff --git a/src/Processors/tests/gtest_merge_join_algorithm.cpp b/src/Processors/tests/gtest_merge_join_algorithm.cpp deleted file mode 100644 index 9a8b70efc17..00000000000 --- a/src/Processors/tests/gtest_merge_join_algorithm.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - - -using namespace DB; - - -QueryPipeline buildJoinPipeline(std::shared_ptr left_source, std::shared_ptr right_source) -{ - Blocks inputs; - inputs.emplace_back(left_source->getPort().getHeader()); - inputs.emplace_back(right_source->getPort().getHeader()); - Block out_header = { - ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared(), "t1.x"), - ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared(), "t2.x"), - }; - - TableJoin::JoinOnClause on_clause; - on_clause.key_names_left = {"x"}; - on_clause.key_names_right = {"x"}; - auto joining = std::make_shared( - JoinKind::Inner, - JoinStrictness::All, - on_clause, - inputs, out_header, /* max_block_size = */ 0); - - chassert(joining->getInputs().size() == 2); - - connect(left_source->getPort(), joining->getInputs().front()); - connect(right_source->getPort(), joining->getInputs().back()); - - auto * output_port = &joining->getOutputPort(); - - auto processors = std::make_shared(); - processors->emplace_back(std::move(left_source)); - processors->emplace_back(std::move(right_source)); - processors->emplace_back(std::move(joining)); - - QueryPipeline pipeline(QueryPlanResourceHolder{}, processors, output_port); - return pipeline; -} - - -std::shared_ptr createSourceWithSingleValue(size_t rows_per_chunk, size_t total_chunks) -{ - Block header = { - ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "x") - }; - - Chunks chunks; - - for (size_t i = 0; i < total_chunks; ++i) - { - auto col = ColumnUInt64::create(rows_per_chunk, 1); - chunks.emplace_back(Columns{std::move(col)}, rows_per_chunk); - } - - return std::make_shared(std::move(header), std::move(chunks)); -} - -TEST(FullSortingJoin, Simple) -try -{ - auto left_source = createSourceWithSingleValue(3, 10); - auto right_source = createSourceWithSingleValue(2, 15); - - auto pipeline = buildJoinPipeline(left_source, right_source); - PullingPipelineExecutor executor(pipeline); - - Block block; - - size_t total_result_rows = 0; - while (executor.pull(block)) - { - total_result_rows += block.rows(); - } - ASSERT_EQ(total_result_rows, 3 * 10 * 2 * 15); -} -catch (Exception & e) -{ - std::cout << e.getStackTraceString() << std::endl; - throw; -} From 4079f25a3865f23f23cec9c43aefe241b58e7972 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 20 Sep 2023 12:33:29 +0000 Subject: [PATCH 03/45] fix test --- .../tests/gtest_full_sorting_join.cpp | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 888e280b55f..b3b462ee1e3 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -134,26 +134,37 @@ std::shared_ptr sourceFromRows( { Chunks chunks; auto columns = header.cloneEmptyColumns(); + chassert(!columns.empty()); std::uniform_real_distribution<> prob_dis(0.0, 1.0); - - for (auto row : values) + for (const auto & row : values) { - if (!columns.empty() && (row.empty() || prob_dis(rng) < break_prob)) + if (!columns.front()->empty() && (row.empty() || prob_dis(rng) < break_prob)) { size_t rows = columns.front()->size(); chunks.emplace_back(std::move(columns), rows); columns = header.cloneEmptyColumns(); - continue; + if (row.empty()) + continue; } + chassert(row.size() == columns.size()); for (size_t i = 0; i < columns.size(); ++i) columns[i]->insert(row[i]); } - if (!columns.empty()) - chunks.emplace_back(std::move(columns), columns.front()->size()); + if (!columns.front()->empty()) + { + size_t rows = columns.front()->size(); + chunks.emplace_back(std::move(columns), rows); + } + + /// Check that code above is correct. + size_t total_result_rows = 0; + for (const auto & chunk : chunks) + total_result_rows += chunk.getNumRows(); + chassert(total_result_rows == values.size()); return std::make_shared(header, std::move(chunks)); } @@ -225,9 +236,11 @@ try Block result_block = executePipeline(pipeline); auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); - ASSERT_EQ(values.size(), 2); - ASSERT_EQ(values[0], (std::vector{"AMZN", 3u, 4u, 130u})); - ASSERT_EQ(values[1], (std::vector{"AMZN", 4u, 4u, 130u})); + + ASSERT_EQ(values, (std::vector>{ + {"AMZN", 3u, 4u, 130u}, + {"AMZN", 4u, 4u, 130u}, + })); } catch (Exception & e) { @@ -272,12 +285,12 @@ try ASSERT_EQ( assert_cast(result_block.getByName("t2.t").column.get())->getData(), - (ColumnUInt64::Container{4, 4, 4, 4, 4, 5, 5, 11, 11, 11, 15}) + (ColumnUInt64::Container{4, 4, 4, 4, 4, 5, 5, 11, 11, 11, 11}) ); ASSERT_EQ( assert_cast(result_block.getByName("t2.value").column.get())->getData(), - (ColumnUInt64::Container{104, 104, 104, 104, 104, 105, 105, 111, 111, 111, 115}) + (ColumnUInt64::Container{104, 104, 104, 104, 104, 105, 105, 111, 111, 111, 111}) ); } catch (Exception & e) From 984d94e5f10e7e1c3cc953e1f5394c153e8fdedc Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 20 Sep 2023 15:46:19 +0000 Subject: [PATCH 04/45] upd gtest_full_sorting_join --- .../tests/gtest_full_sorting_join.cpp | 154 +++++++++--------- 1 file changed, 76 insertions(+), 78 deletions(-) diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index b3b462ee1e3..4d7ce25a7e8 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -93,7 +93,7 @@ QueryPipeline buildJoinPipeline( std::shared_ptr oneColumnSource(const std::vector> & values) { - Block header = { ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "x") }; + Block header = { ColumnWithTypeAndName(std::make_shared(), "x") }; Chunks chunks; for (const auto & chunk_values : values) { @@ -129,45 +129,46 @@ catch (Exception & e) throw; } -std::shared_ptr sourceFromRows( - const Block & header, const std::vector> & values, double break_prob = 0.0) +class SourceChunksBuilder { +public: + explicit SourceChunksBuilder(const Block & header_) + : header(header_) + { + current_chunk = header.cloneEmptyColumns(); + chassert(!current_chunk.empty()); + } + + SourceChunksBuilder & addRow(const std::vector & row) + { + chassert(row.size() == current_chunk.size()); + for (size_t i = 0; i < current_chunk.size(); ++i) + current_chunk[i]->insert(row[i]); + return *this; + } + + SourceChunksBuilder & addChunk() + { + if (current_chunk.front()->empty()) + return *this; + + size_t rows = current_chunk.front()->size(); + chunks.emplace_back(std::move(current_chunk), rows); + current_chunk = header.cloneEmptyColumns(); + return *this; + } + + std::shared_ptr build() + { + addChunk(); + return std::make_shared(header, std::move(chunks)); + } + +private: + Block header; Chunks chunks; - auto columns = header.cloneEmptyColumns(); - chassert(!columns.empty()); - - std::uniform_real_distribution<> prob_dis(0.0, 1.0); - - for (const auto & row : values) - { - if (!columns.front()->empty() && (row.empty() || prob_dis(rng) < break_prob)) - { - size_t rows = columns.front()->size(); - chunks.emplace_back(std::move(columns), rows); - columns = header.cloneEmptyColumns(); - if (row.empty()) - continue; - } - - chassert(row.size() == columns.size()); - for (size_t i = 0; i < columns.size(); ++i) - columns[i]->insert(row[i]); - } - - if (!columns.front()->empty()) - { - size_t rows = columns.front()->size(); - chunks.emplace_back(std::move(columns), rows); - } - - /// Check that code above is correct. - size_t total_result_rows = 0; - for (const auto & chunk : chunks) - total_result_rows += chunk.getNumRows(); - chassert(total_result_rows == values.size()); - - return std::make_shared(header, std::move(chunks)); -} + MutableColumns current_chunk; +}; std::vector> getValuesFromBlock(const Block & block, const Names & names) @@ -203,32 +204,30 @@ Block executePipeline(QueryPipeline & pipeline) TEST(FullSortingJoin, Asof) try { - const std::vector chunk_break = {}; + auto left_source = SourceChunksBuilder({ + {std::make_shared(), "key"}, + {std::make_shared(), "t"}, + }) + .addRow({"AMZN", 3}) + .addRow({"AMZN", 4}) + .addRow({"AMZN", 6}) + .build(); - auto left_source = sourceFromRows({ - ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "key"), - ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "t"), - }, { - {"AMZN", 3}, - {"AMZN", 4}, - {"AMZN", 6}, - }); - - auto right_source = sourceFromRows({ - ColumnWithTypeAndName(ColumnString::create(), std::make_shared(), "key"), - ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "t"), - ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "value"), - }, { - {"AAPL", 1, 97}, - chunk_break, - {"AAPL", 2, 98}, - {"AAPL", 3, 99}, - {"AMZN", 1, 100}, - {"AMZN", 2, 110}, - chunk_break, - {"AMZN", 4, 130}, - {"AMZN", 5, 140}, - }); + auto right_source = SourceChunksBuilder({ + {std::make_shared(), "key"}, + {std::make_shared(), "t"}, + {std::make_shared(), "value"}, + }) + .addRow({"AAPL", 1, 97}) + .addChunk() + .addRow({"AAPL", 2, 98}) + .addRow({"AAPL", 3, 99}) + .addRow({"AMZN", 1, 100}) + .addRow({"AMZN", 2, 110}) + .addChunk() + .addRow({"AMZN", 4, 130}) + .addRow({"AMZN", 5, 140}) + .build(); auto pipeline = buildJoinPipeline( left_source, right_source, /* key_length = */ 2, @@ -252,25 +251,24 @@ catch (Exception & e) TEST(FullSortingJoin, AsofOnlyColumn) try { - const std::vector chunk_break = {}; - auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} }); UInt64 p = std::uniform_int_distribution<>(0, 2)(rng); - double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0); - auto right_source = sourceFromRows({ - ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "t"), - ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "value"), - }, { - {1, 101}, - {2, 102}, - {4, 104}, - {5, 105}, - {11, 111}, - {15, 115}, - }, - break_prob); + SourceChunksBuilder right_source_builder({ + {std::make_shared(), "t"}, + {std::make_shared(), "value"}, + }); + + double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0); + std::uniform_real_distribution<> prob_dis(0.0, 1.0); + for (const auto & row : std::vector>{ {1, 101}, {2, 102}, {4, 104}, {5, 105}, {11, 111}, {15, 115} }) + { + right_source_builder.addRow(row); + if (prob_dis(rng) < break_prob) + right_source_builder.addChunk(); + } + auto right_source = right_source_builder.build(); auto pipeline = buildJoinPipeline( left_source, right_source, /* key_length = */ 1, From 6330b466aa326e40eb7abca2699554c98241b342 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 21 Sep 2023 11:19:20 +0000 Subject: [PATCH 05/45] Add randomized test FullSortingJoin.AsofGeneratedTestData --- src/Common/getRandomASCIIString.cpp | 7 +- src/Common/getRandomASCIIString.h | 3 + .../Transforms/MergeJoinTransform.cpp | 2 +- .../tests/gtest_full_sorting_join.cpp | 121 +++++++++++++++--- 4 files changed, 116 insertions(+), 17 deletions(-) diff --git a/src/Common/getRandomASCIIString.cpp b/src/Common/getRandomASCIIString.cpp index 594b4cd3228..a295277b453 100644 --- a/src/Common/getRandomASCIIString.cpp +++ b/src/Common/getRandomASCIIString.cpp @@ -6,12 +6,17 @@ namespace DB { String getRandomASCIIString(size_t length) +{ + return getRandomASCIIString(length, thread_local_rng); +} + +String getRandomASCIIString(size_t length, pcg64 & rng) { std::uniform_int_distribution distribution('a', 'z'); String res; res.resize(length); for (auto & c : res) - c = distribution(thread_local_rng); + c = distribution(rng); return res; } diff --git a/src/Common/getRandomASCIIString.h b/src/Common/getRandomASCIIString.h index 627d2700ce3..19e1ff7120e 100644 --- a/src/Common/getRandomASCIIString.h +++ b/src/Common/getRandomASCIIString.h @@ -2,11 +2,14 @@ #include +#include + namespace DB { /// Slow random string. Useful for random names and things like this. Not for generating data. String getRandomASCIIString(size_t length); +String getRandomASCIIString(size_t length, pcg64 & rng); } diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 8370e548fcb..bfde5892289 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -874,7 +874,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() for (auto & col : rcols) result.addColumn(std::move(col)); } - UNUSED(asof_inequality); + return Status(std::move(result)); } diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 4d7ce25a7e8..741e945bfdb 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -1,30 +1,31 @@ #include - -#include #include #include -#include -#include -#include -#include -#include -#include - #include +#include +#include + #include + #include +#include +#include +#include +#include +#include #include +#include using namespace DB; UInt64 getAndPrintRandomSeed() { UInt64 seed = randomSeed(); - std::cerr << "TEST_RANDOM_SEED: " << seed << std::endl; + std::cerr << __FILE__ << "::" << "TEST_RANDOM_SEED = " << seed << "ull" << std::endl; return seed; } @@ -132,6 +133,8 @@ catch (Exception & e) class SourceChunksBuilder { public: + double break_prob = 0.0; + explicit SourceChunksBuilder(const Block & header_) : header(header_) { @@ -144,6 +147,10 @@ public: chassert(row.size() == current_chunk.size()); for (size_t i = 0; i < current_chunk.size(); ++i) current_chunk[i]->insert(row[i]); + + if (break_prob > 0.0 && std::uniform_real_distribution<>(0.0, 1.0)(rng) < break_prob) + addChunk(); + return *this; } @@ -184,7 +191,7 @@ std::vector> getValuesFromBlock(const Block & block, const Na } -Block executePipeline(QueryPipeline & pipeline) +Block executePipeline(QueryPipeline && pipeline) { PullingPipelineExecutor executor(pipeline); @@ -233,7 +240,7 @@ try left_source, right_source, /* key_length = */ 2, JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals); - Block result_block = executePipeline(pipeline); + Block result_block = executePipeline(std::move(pipeline)); auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); ASSERT_EQ(values, (std::vector>{ @@ -253,13 +260,12 @@ try { auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} }); - UInt64 p = std::uniform_int_distribution<>(0, 2)(rng); - SourceChunksBuilder right_source_builder({ {std::make_shared(), "t"}, {std::make_shared(), "value"}, }); + UInt64 p = std::uniform_int_distribution<>(0, 2)(rng); double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0); std::uniform_real_distribution<> prob_dis(0.0, 1.0); for (const auto & row : std::vector>{ {1, 101}, {2, 102}, {4, 104}, {5, 105}, {11, 111}, {15, 115} }) @@ -274,7 +280,7 @@ try left_source, right_source, /* key_length = */ 1, JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals); - Block result_block = executePipeline(pipeline); + Block result_block = executePipeline(std::move(pipeline)); ASSERT_EQ( assert_cast(result_block.getByName("t1.x").column.get())->getData(), @@ -296,3 +302,88 @@ catch (Exception & e) std::cout << e.getStackTraceString() << std::endl; throw; } + +TEST(FullSortingJoin, AsofGeneratedTestData) +try +{ + auto left_source_builder = SourceChunksBuilder({ + {std::make_shared(), "k1"}, + {std::make_shared(), "k2"}, + {std::make_shared(), "t"}, + {std::make_shared(), "attr"}, + }); + + auto right_source_builder = SourceChunksBuilder({ + {std::make_shared(), "k1"}, + {std::make_shared(), "k2"}, + {std::make_shared(), "t"}, + {std::make_shared(), "attr"}, + }); + + /// uniform_int_distribution to have 0.0 and 1.0 probabilities + left_source_builder.break_prob = std::uniform_int_distribution<>(0, 5)(rng) / 5.0; + right_source_builder.break_prob = std::uniform_int_distribution<>(0, 5)(rng) / 5.0; + + auto get_next_key = [](UInt64 & k1, String & k2) + { + size_t str_len = std::uniform_int_distribution<>(1, 10)(rng); + String new_k2 = getRandomASCIIString(str_len, rng); + if (new_k2.compare(k2) <= 0) + ++k1; + k2 = new_k2; + }; + + ColumnUInt64::Container expected; + + UInt64 k1 = 0; + String k2 = "asdfg"; + auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); + for (size_t key_num = 0; key_num < key_num_total; ++key_num) + { + UInt64 left_t = 0; + size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng); + for (size_t i = 0; i < num_left_rows; ++i) + { + left_t += std::uniform_int_distribution<>(1, 10)(rng); + + left_source_builder.addRow({k1, k2, left_t, 10 * left_t}); + expected.push_back(10 * left_t); + + auto num_matches = 1 + std::poisson_distribution<>(4)(rng); + + size_t right_t = left_t; + for (size_t j = 0; j < num_matches; ++j) + { + right_t += std::uniform_int_distribution<>(0, 3)(rng); + right_source_builder.addRow({k1, k2, right_t, j == 0 ? 100 * left_t : 0}); + } + /// next left_t should be greater than right_t not to match with previous rows + left_t = right_t; + } + + /// generate some rows with greater left_t to check that they are not matched + num_left_rows = std::uniform_int_distribution<>(1, 100)(rng); + for (size_t i = 0; i < num_left_rows; ++i) + { + left_t += std::uniform_int_distribution<>(1, 10)(rng); + left_source_builder.addRow({k1, k2, left_t, 10 * left_t}); + } + + get_next_key(k1, k2); + } + + Block result_block = executePipeline(buildJoinPipeline( + left_source_builder.build(), right_source_builder.build(), + /* key_length = */ 3, + JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals)); + + ASSERT_EQ(assert_cast(block.getByName("t1.attr").column.get())->getData(), expected); + + for (auto & e : expected) + e = 10 * e; + ASSERT_EQ(assert_cast(block.getByName("t2.attr").column.get())->getData(), expected); +} +catch (Exception & e) { + std::cout << e.getStackTraceString() << std::endl; + throw; +} From da4f35556100847d527f194cee4b2f99b50bfa58 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 21 Sep 2023 11:38:37 +0000 Subject: [PATCH 06/45] upd FullSortingJoin.AsofGeneratedTestData --- .../tests/gtest_full_sorting_join.cpp | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 741e945bfdb..49c9f58b1d5 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -208,6 +208,14 @@ Block executePipeline(QueryPipeline && pipeline) return concatenateBlocks(result_blocks); } +template +void checkColumn(const typename ColumnVector::Container & expected, const Block & block, const std::string & name) +{ + const auto & actual = assert_cast *>(block.getByName(name).column.get())->getData(); + EXPECT_EQ(actual.size(), expected.size()); + ASSERT_EQ(actual, expected); +} + TEST(FullSortingJoin, Asof) try { @@ -306,18 +314,27 @@ catch (Exception & e) TEST(FullSortingJoin, AsofGeneratedTestData) try { + std::vector join_kinds = {JoinKind::Inner, JoinKind::Left}; + auto join_kind = join_kinds[std::uniform_int_distribution(0, join_kinds.size() - 1)(rng)]; + + std::vector asof_inequalities = { + ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals, + // ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals, + }; + auto asof_inequality = asof_inequalities[std::uniform_int_distribution(0, asof_inequalities.size() - 1)(rng)]; + auto left_source_builder = SourceChunksBuilder({ {std::make_shared(), "k1"}, {std::make_shared(), "k2"}, {std::make_shared(), "t"}, - {std::make_shared(), "attr"}, + {std::make_shared(), "attr"}, }); auto right_source_builder = SourceChunksBuilder({ {std::make_shared(), "k1"}, {std::make_shared(), "k2"}, {std::make_shared(), "t"}, - {std::make_shared(), "attr"}, + {std::make_shared(), "attr"}, }); /// uniform_int_distribution to have 0.0 and 1.0 probabilities @@ -333,14 +350,14 @@ try k2 = new_k2; }; - ColumnUInt64::Container expected; + ColumnInt64::Container expected; UInt64 k1 = 0; String k2 = "asdfg"; auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); for (size_t key_num = 0; key_num < key_num_total; ++key_num) { - UInt64 left_t = 0; + Int64 left_t = 0; size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng); for (size_t i = 0; i < num_left_rows; ++i) { @@ -351,11 +368,22 @@ try auto num_matches = 1 + std::poisson_distribution<>(4)(rng); - size_t right_t = left_t; + auto right_t = left_t; for (size_t j = 0; j < num_matches; ++j) { - right_t += std::uniform_int_distribution<>(0, 3)(rng); - right_source_builder.addRow({k1, k2, right_t, j == 0 ? 100 * left_t : 0}); + int min_step = 1; + if (asof_inequality == ASOFJoinInequality::LessOrEquals || asof_inequality == ASOFJoinInequality::GreaterOrEquals) + min_step = 0; + right_t += std::uniform_int_distribution<>(min_step, 3)(rng); + + bool is_match = false; + + if (asof_inequality == ASOFJoinInequality::LessOrEquals || asof_inequality == ASOFJoinInequality::Less) + is_match = j == 0; + else if (asof_inequality == ASOFJoinInequality::GreaterOrEquals || asof_inequality == ASOFJoinInequality::Greater) + is_match = j == num_matches - 1; + + right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * left_t : -1}); } /// next left_t should be greater than right_t not to match with previous rows left_t = right_t; @@ -366,7 +394,10 @@ try for (size_t i = 0; i < num_left_rows; ++i) { left_t += std::uniform_int_distribution<>(1, 10)(rng); - left_source_builder.addRow({k1, k2, left_t, 10 * left_t}); + left_source_builder.addRow({k1, k2, left_t, -10 * left_t}); + + if (join_kind == JoinKind::Left) + expected.push_back(-10 * left_t); } get_next_key(k1, k2); @@ -375,13 +406,14 @@ try Block result_block = executePipeline(buildJoinPipeline( left_source_builder.build(), right_source_builder.build(), /* key_length = */ 3, - JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals)); + join_kind, JoinStrictness::Asof, asof_inequality)); - ASSERT_EQ(assert_cast(block.getByName("t1.attr").column.get())->getData(), expected); + checkColumn(expected, result_block, "t1.attr"); for (auto & e : expected) - e = 10 * e; - ASSERT_EQ(assert_cast(block.getByName("t2.attr").column.get())->getData(), expected); + e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr + + checkColumn(expected, result_block, "t2.attr"); } catch (Exception & e) { std::cout << e.getStackTraceString() << std::endl; From 97e3ee6e661620187bf115d948115936e36cdaf4 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 21 Sep 2023 11:40:41 +0000 Subject: [PATCH 07/45] upd gtest_full_sorting_join --- .../tests/gtest_full_sorting_join.cpp | 92 +++++++++++-------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 49c9f58b1d5..bbd321a78d7 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -142,7 +142,7 @@ public: chassert(!current_chunk.empty()); } - SourceChunksBuilder & addRow(const std::vector & row) + void addRow(const std::vector & row) { chassert(row.size() == current_chunk.size()); for (size_t i = 0; i < current_chunk.size(); ++i) @@ -150,25 +150,29 @@ public: if (break_prob > 0.0 && std::uniform_real_distribution<>(0.0, 1.0)(rng) < break_prob) addChunk(); - - return *this; } - SourceChunksBuilder & addChunk() + void addChunk() { if (current_chunk.front()->empty()) - return *this; + return; size_t rows = current_chunk.front()->size(); chunks.emplace_back(std::move(current_chunk), rows); current_chunk = header.cloneEmptyColumns(); - return *this; + return; } std::shared_ptr build() { addChunk(); - return std::make_shared(header, std::move(chunks)); + + /// copy chunk to allow reusing same builder + Chunks chunks_copy; + chunks_copy.reserve(chunks.size()); + for (const auto & chunk : chunks) + chunks_copy.emplace_back(chunk.clone()); + return std::make_shared(header, std::move(chunks_copy)); } private: @@ -219,42 +223,54 @@ void checkColumn(const typename ColumnVector::Container & expected, const Blo TEST(FullSortingJoin, Asof) try { - auto left_source = SourceChunksBuilder({ + SourceChunksBuilder left_source({ {std::make_shared(), "key"}, {std::make_shared(), "t"}, - }) - .addRow({"AMZN", 3}) - .addRow({"AMZN", 4}) - .addRow({"AMZN", 6}) - .build(); + }); - auto right_source = SourceChunksBuilder({ + left_source.addRow({"AMZN", 3}); + left_source.addRow({"AMZN", 4}); + left_source.addRow({"AMZN", 6}); + + SourceChunksBuilder right_source({ {std::make_shared(), "key"}, {std::make_shared(), "t"}, {std::make_shared(), "value"}, - }) - .addRow({"AAPL", 1, 97}) - .addChunk() - .addRow({"AAPL", 2, 98}) - .addRow({"AAPL", 3, 99}) - .addRow({"AMZN", 1, 100}) - .addRow({"AMZN", 2, 110}) - .addChunk() - .addRow({"AMZN", 4, 130}) - .addRow({"AMZN", 5, 140}) - .build(); + }); + right_source.addRow({"AAPL", 1, 97}); + right_source.addChunk(); + right_source.addRow({"AAPL", 2, 98}); + right_source.addRow({"AAPL", 3, 99}); + right_source.addRow({"AMZN", 1, 100}); + right_source.addRow({"AMZN", 2, 110}); + right_source.addChunk(); + right_source.addRow({"AMZN", 4, 130}); + right_source.addRow({"AMZN", 5, 140}); - auto pipeline = buildJoinPipeline( - left_source, right_source, /* key_length = */ 2, - JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals); + { + Block result_block = executePipeline(buildJoinPipeline( + left_source.build(), right_source.build(), /* key_length = */ 2, + JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals)); + auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); - Block result_block = executePipeline(std::move(pipeline)); - auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); + ASSERT_EQ(values, (std::vector>{ + {"AMZN", 3u, 4u, 130u}, + {"AMZN", 4u, 4u, 130u}, + })); + } - ASSERT_EQ(values, (std::vector>{ - {"AMZN", 3u, 4u, 130u}, - {"AMZN", 4u, 4u, 130u}, - })); + { + Block result_block = executePipeline(buildJoinPipeline( + left_source.build(), right_source.build(), /* key_length = */ 2, + JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals)); + auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); + + ASSERT_EQ(values, (std::vector>{ + {"AMZN", 3u, 2u, 110u}, + {"AMZN", 4u, 4u, 130u}, + {"AMZN", 6u, 5u, 140u}, + })); + } } catch (Exception & e) { @@ -314,23 +330,23 @@ catch (Exception & e) TEST(FullSortingJoin, AsofGeneratedTestData) try { - std::vector join_kinds = {JoinKind::Inner, JoinKind::Left}; + std::array join_kinds{JoinKind::Inner, JoinKind::Left}; auto join_kind = join_kinds[std::uniform_int_distribution(0, join_kinds.size() - 1)(rng)]; - std::vector asof_inequalities = { + std::array asof_inequalities{ ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals, // ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals, }; auto asof_inequality = asof_inequalities[std::uniform_int_distribution(0, asof_inequalities.size() - 1)(rng)]; - auto left_source_builder = SourceChunksBuilder({ + SourceChunksBuilder left_source_builder({ {std::make_shared(), "k1"}, {std::make_shared(), "k2"}, {std::make_shared(), "t"}, {std::make_shared(), "attr"}, }); - auto right_source_builder = SourceChunksBuilder({ + SourceChunksBuilder right_source_builder({ {std::make_shared(), "k1"}, {std::make_shared(), "k2"}, {std::make_shared(), "t"}, From 4a1a7d4c6278246e097cf0583b401aa8ac3775fa Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 25 Sep 2023 10:54:08 +0000 Subject: [PATCH 08/45] Add randomized test FullSortingJoin.Any --- .../Transforms/MergeJoinTransform.cpp | 28 ++- .../Transforms/MergeJoinTransform.h | 5 + .../tests/gtest_full_sorting_join.cpp | 181 ++++++++++++++---- 3 files changed, 166 insertions(+), 48 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index bfde5892289..f5e277ea8c8 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -630,7 +630,7 @@ struct AnyJoinImpl FullMergeJoinCursor & right_cursor, PaddedPODArray & left_map, PaddedPODArray & right_map, - AnyJoinState & state, + AnyJoinState & any_join_state, int null_direction_hint) { assert(enabled); @@ -691,21 +691,21 @@ struct AnyJoinImpl } } - /// Remember index of last joined row to propagate it to next block + /// Remember last joined row to propagate it to next block - state.setValue({}); + any_join_state.setValue({}); if (!left_cursor->isValid()) { - state.set(0, left_cursor.cursor); + any_join_state.set(0, left_cursor.cursor); if (cmp == 0 && isLeft(kind)) - state.setValue(getRowFromChunk(right_cursor.getCurrent(), rpos)); + any_join_state.setValue(getRowFromChunk(right_cursor.getCurrent(), rpos)); } if (!right_cursor->isValid()) { - state.set(1, right_cursor.cursor); + any_join_state.set(1, right_cursor.cursor); if (cmp == 0 && isRight(kind)) - state.setValue(getRowFromChunk(left_cursor.getCurrent(), lpos)); + any_join_state.setValue(getRowFromChunk(left_cursor.getCurrent(), lpos)); } } }; @@ -720,7 +720,6 @@ std::optional MergeJoinAlgorithm::handleAnyJoinState for (size_t source_num = 0; source_num < 2; ++source_num) { auto & current = *cursors[source_num]; - auto & state = any_join_state; if (any_join_state.keys[source_num].equals(current.cursor)) { size_t start_pos = current->getRow(); @@ -728,25 +727,22 @@ std::optional MergeJoinAlgorithm::handleAnyJoinState if (length && isLeft(kind) && source_num == 0) { - if (state.value) - result = copyChunkResized(current.getCurrent(), state.value, start_pos, length); + if (any_join_state.value) + result = copyChunkResized(current.getCurrent(), any_join_state.value, start_pos, length); else result = createBlockWithDefaults(source_num, start_pos, length); } if (length && isRight(kind) && source_num == 1) { - if (state.value) - result = copyChunkResized(state.value, current.getCurrent(), start_pos, length); + if (any_join_state.value) + result = copyChunkResized(any_join_state.value, current.getCurrent(), start_pos, length); else result = createBlockWithDefaults(source_num, start_pos, length); } - /// We've found row with other key, no need to skip more rows with current key if (current->isValid()) - { - state.keys[source_num].reset(); - } + any_join_state.keys[source_num].reset(); } else { diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 3ee01e57992..375c9ebd3cc 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -85,6 +85,11 @@ public: keys[source_num] = JoinKeyRow(cursor, cursor.rows - 1); } + void reset(size_t source_num) + { + keys[source_num].reset(); + } + void setValue(Chunk value_) { value = std::move(value_); } bool empty() const { return keys[0].row.empty() && keys[1].row.empty(); } diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index bbd321a78d7..1dc410bd6b8 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -213,13 +213,142 @@ Block executePipeline(QueryPipeline && pipeline) } template -void checkColumn(const typename ColumnVector::Container & expected, const Block & block, const std::string & name) +void assertColumnVectorEq(const typename ColumnVector::Container & expected, const Block & block, const std::string & name) { const auto & actual = assert_cast *>(block.getByName(name).column.get())->getData(); EXPECT_EQ(actual.size(), expected.size()); ASSERT_EQ(actual, expected); } +template +void assertColumnEq(const IColumn & expected, const Block & block, const std::string & name) +{ + const ColumnPtr & actual = block.getByName(name).column; + ASSERT_TRUE(checkColumn(*actual)); + ASSERT_TRUE(checkColumn(expected)); + EXPECT_EQ(actual->size(), expected.size()); + + auto dump_val = [](const IColumn & col, size_t i) -> String + { + Field value; + col.get(i, value); + return value.dump(); + }; + + size_t num_rows = std::min(actual->size(), expected.size()); + for (size_t i = 0; i < num_rows; ++i) + ASSERT_EQ(actual->compareAt(i, i, expected, 1), 0) << dump_val(*actual, i) << " != " << dump_val(expected, i) << " at row " << i; +} + +template +T getRandomFrom(const std::initializer_list & opts) +{ + std::vector options(opts.begin(), opts.end()); + size_t idx = std::uniform_int_distribution(0, options.size() - 1)(rng); + return options[idx]; +} + +/// Used to have accurate 0.0 and 1.0 probabilities +double getRandomDoubleQuantized(size_t quants = 5) +{ + return std::uniform_int_distribution(0, quants)(rng) / static_cast(quants); +} + +void generateNextKey(UInt64 & k1, String & k2) +{ + size_t str_len = std::uniform_int_distribution<>(1, 10)(rng); + String new_k2 = getRandomASCIIString(str_len, rng); + if (new_k2.compare(k2) <= 0) + ++k1; + k2 = new_k2; +} + +TEST(FullSortingJoin, Any) +try +{ + JoinKind kind = getRandomFrom({JoinKind::Inner, JoinKind::Left, JoinKind::Right}); + + SourceChunksBuilder left_source({ + {std::make_shared(), "k1"}, + {std::make_shared(), "k2"}, + {std::make_shared(), "attr"}, + }); + + SourceChunksBuilder right_source({ + {std::make_shared(), "k1"}, + {std::make_shared(), "k2"}, + {std::make_shared(), "attr"}, + }); + + left_source.break_prob = getRandomDoubleQuantized(); + right_source.break_prob = getRandomDoubleQuantized(); + + size_t num_keys = std::uniform_int_distribution<>(100, 1000)(rng); + + auto expected_left = ColumnString::create(); + auto expected_right = ColumnString::create(); + + UInt64 k1 = 0; + String k2 = ""; + + auto get_attr = [&](const String & side, size_t idx) -> String + { + return toString(k1) + "_" + k2 + "_" + side + "_" + toString(idx); + }; + + for (size_t i = 0; i < num_keys; ++i) + { + generateNextKey(k1, k2); + + /// Key is present in left, right or both tables. Both tables is more probable. + size_t key_presence = std::uniform_int_distribution<>(0, 10)(rng); + + size_t num_rows_left = key_presence == 0 ? 0 : std::uniform_int_distribution<>(1, 10)(rng); + for (size_t j = 0; j < num_rows_left; ++j) + left_source.addRow({k1, k2, get_attr("left", j)}); + + size_t num_rows_right = key_presence == 1 ? 0 : std::uniform_int_distribution<>(1, 10)(rng); + for (size_t j = 0; j < num_rows_right; ++j) + right_source.addRow({k1, k2, get_attr("right", j)}); + + String left_attr = num_rows_left ? get_attr("left", 0) : ""; + String right_attr = num_rows_right ? get_attr("right", 0) : ""; + + if (kind == JoinKind::Inner && num_rows_left && num_rows_right) + { + expected_left->insert(left_attr); + expected_right->insert(right_attr); + } + else if (kind == JoinKind::Left) + { + for (size_t j = 0; j < num_rows_left; ++j) + { + expected_left->insert(get_attr("left", j)); + expected_right->insert(right_attr); + } + } + else if (kind == JoinKind::Right) + { + for (size_t j = 0; j < num_rows_right; ++j) + { + expected_left->insert(left_attr); + expected_right->insert(get_attr("right", j)); + } + } + } + + Block result_block = executePipeline(buildJoinPipeline( + left_source.build(), right_source.build(), /* key_length = */ 2, + kind, JoinStrictness::Any)); + assertColumnEq(*expected_left, result_block, "t1.attr"); + assertColumnEq(*expected_right, result_block, "t2.attr"); +} +catch (Exception & e) +{ + std::cout << e.getStackTraceString() << std::endl; + throw; +} + TEST(FullSortingJoin, Asof) try { @@ -259,18 +388,18 @@ try })); } - { - Block result_block = executePipeline(buildJoinPipeline( - left_source.build(), right_source.build(), /* key_length = */ 2, - JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals)); - auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); + // { + // Block result_block = executePipeline(buildJoinPipeline( + // left_source.build(), right_source.build(), /* key_length = */ 2, + // JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals)); + // auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); - ASSERT_EQ(values, (std::vector>{ - {"AMZN", 3u, 2u, 110u}, - {"AMZN", 4u, 4u, 130u}, - {"AMZN", 6u, 5u, 140u}, - })); - } + // ASSERT_EQ(values, (std::vector>{ + // {"AMZN", 3u, 2u, 110u}, + // {"AMZN", 4u, 4u, 130u}, + // {"AMZN", 6u, 5u, 140u}, + // })); + // } } catch (Exception & e) { @@ -330,14 +459,12 @@ catch (Exception & e) TEST(FullSortingJoin, AsofGeneratedTestData) try { - std::array join_kinds{JoinKind::Inner, JoinKind::Left}; - auto join_kind = join_kinds[std::uniform_int_distribution(0, join_kinds.size() - 1)(rng)]; + auto join_kind = getRandomFrom({JoinKind::Inner, JoinKind::Left}); - std::array asof_inequalities{ + auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals, // ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals, - }; - auto asof_inequality = asof_inequalities[std::uniform_int_distribution(0, asof_inequalities.size() - 1)(rng)]; + }); SourceChunksBuilder left_source_builder({ {std::make_shared(), "k1"}, @@ -353,18 +480,8 @@ try {std::make_shared(), "attr"}, }); - /// uniform_int_distribution to have 0.0 and 1.0 probabilities - left_source_builder.break_prob = std::uniform_int_distribution<>(0, 5)(rng) / 5.0; - right_source_builder.break_prob = std::uniform_int_distribution<>(0, 5)(rng) / 5.0; - - auto get_next_key = [](UInt64 & k1, String & k2) - { - size_t str_len = std::uniform_int_distribution<>(1, 10)(rng); - String new_k2 = getRandomASCIIString(str_len, rng); - if (new_k2.compare(k2) <= 0) - ++k1; - k2 = new_k2; - }; + left_source_builder.break_prob = getRandomDoubleQuantized(); + right_source_builder.break_prob = getRandomDoubleQuantized(); ColumnInt64::Container expected; @@ -416,7 +533,7 @@ try expected.push_back(-10 * left_t); } - get_next_key(k1, k2); + generateNextKey(k1, k2); } Block result_block = executePipeline(buildJoinPipeline( @@ -424,12 +541,12 @@ try /* key_length = */ 3, join_kind, JoinStrictness::Asof, asof_inequality)); - checkColumn(expected, result_block, "t1.attr"); + assertColumnVectorEq(expected, result_block, "t1.attr"); for (auto & e : expected) e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr - checkColumn(expected, result_block, "t2.attr"); + assertColumnVectorEq(expected, result_block, "t2.attr"); } catch (Exception & e) { std::cout << e.getStackTraceString() << std::endl; From 7e0c2d7bcb88f6d8575fc1f0e288e94f13456f87 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 26 Sep 2023 12:25:12 +0000 Subject: [PATCH 09/45] wip full sorting asof join --- .../Transforms/MergeJoinTransform.cpp | 300 +++++++++++++-- .../Transforms/MergeJoinTransform.h | 89 ++--- .../tests/gtest_full_sorting_join.cpp | 346 ++++++++++++++---- .../02276_full_sort_join_unsupported.sql | 2 +- 4 files changed, 575 insertions(+), 162 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index f5e277ea8c8..638f2f1cb10 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -34,8 +34,6 @@ namespace ErrorCodes namespace { -constexpr UInt64 DEFAULT_VALUE_INDEX = std::numeric_limits::max(); - FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns, JoinStrictness strictness) { SortDescription desc; @@ -112,7 +110,7 @@ int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImp int compareAsofCursors(const FullMergeJoinCursor & lhs, const FullMergeJoinCursor & rhs) { - return nullableCompareAt(lhs.getAsofColumn(), rhs.getAsofColumn(), lhs->getRow(), rhs->getRow()); + return nullableCompareAt(*lhs.getAsofColumn(), *rhs.getAsofColumn(), lhs->getRow(), rhs->getRow()); } bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint) @@ -250,6 +248,87 @@ void inline addMany(PaddedPODArray & values, UInt64 value, size_t num) } +JoinKeyRow::JoinKeyRow(const FullMergeJoinCursor & cursor, size_t pos) +{ + row.reserve(cursor->sort_columns.size()); + for (const auto & col : cursor->sort_columns) + { + auto new_col = col->cloneEmpty(); + new_col->insertFrom(*col, pos); + row.push_back(std::move(new_col)); + } + if (auto asof_column = cursor.getAsofColumn()) + { + auto new_col = asof_column->cloneEmpty(); + new_col->insertFrom(*asof_column, pos); + row.push_back(std::move(new_col)); + } +} + +void JoinKeyRow::reset() +{ + row.clear(); +} + +bool JoinKeyRow::equals(const FullMergeJoinCursor & cursor) const +{ + if (row.empty()) + return false; + + assert(this->row.size() == cursor->sort_columns_size); + for (size_t i = 0; i < cursor->sort_columns_size; ++i) + { + int cmp = this->row[i]->compareAt(0, cursor->getRow(), *(cursor->sort_columns[i]), cursor->desc[i].nulls_direction); + if (cmp != 0) + return false; + } + return true; +} + +bool JoinKeyRow::asofMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) const +{ + if (!equals(cursor)) + return false; + int cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *row.back(), 1); + return (asof_inequality == ASOFJoinInequality::Less && cmp < 0) + || (asof_inequality == ASOFJoinInequality::LessOrEquals && cmp <= 0) + || (asof_inequality == ASOFJoinInequality::Greater && cmp > 0) + || (asof_inequality == ASOFJoinInequality::GreaterOrEquals && cmp >= 0); +} + +void AnyJoinState::set(size_t source_num, const FullMergeJoinCursor & cursor) +{ + assert(cursor->rows); + keys[source_num] = JoinKeyRow(cursor, cursor->rows - 1); +} + +void AnyJoinState::reset(size_t source_num) +{ + keys[source_num].reset(); + value.clear(); +} + +void AnyJoinState::setValue(Chunk value_) +{ + value = std::move(value_); +} + +bool AnyJoinState::empty() const { return keys[0].row.empty() && keys[1].row.empty(); } + + +void AsofJoinState::set(const FullMergeJoinCursor & rcursor, size_t rpos) +{ + key = JoinKeyRow(rcursor, rpos); + value = rcursor.getCurrent().clone(); + value_row = rpos; +} + +void AsofJoinState::reset() +{ + key.reset(); + value.clear(); +} + const Chunk & FullMergeJoinCursor::getCurrent() const { return current_chunk; @@ -282,6 +361,31 @@ bool FullMergeJoinCursor::fullyCompleted() const return !cursor.isValid() && recieved_all_blocks; } +String FullMergeJoinCursor::dump() const +{ + Strings row_dump; + if (cursor.isValid()) + { + Field val; + for (size_t i = 0; i < cursor.sort_columns_size; ++i) + { + cursor.sort_columns[i]->get(cursor.getRow(), val); + row_dump.push_back(val.dump()); + } + + if (auto * asof_column = getAsofColumn()) + { + asof_column->get(cursor.getRow(), val); + row_dump.push_back(val.dump()); + } + } + + return fmt::format("<{}/{}{}>[{}]", + cursor.getRow(), cursor.rows, + recieved_all_blocks ? "(finished)" : "", + fmt::join(row_dump, ", ")); +} + MergeJoinAlgorithm::MergeJoinAlgorithm( JoinKind kind_, JoinStrictness strictness_, @@ -456,7 +560,7 @@ struct AllJoinImpl else { assert(state == nullptr); - state = std::make_unique(left_cursor.cursor, lpos, right_cursor.cursor, rpos); + state = std::make_unique(left_cursor, lpos, right_cursor, rpos); state->addRange(0, left_cursor.getCurrent().clone(), lpos, lnum); state->addRange(1, right_cursor.getCurrent().clone(), rpos, rnum); return; @@ -501,6 +605,17 @@ void dispatchKind(JoinKind kind, Args && ... args) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); } +MutableColumns MergeJoinAlgorithm::getEmptyResultColumns() const +{ + MutableColumns result_cols; + for (size_t i = 0; i < 2; ++i) + { + for (const auto & col : cursors[i]->sampleColumns()) + result_cols.push_back(col->cloneEmpty()); + } + return result_cols; +} + std::optional MergeJoinAlgorithm::handleAllJoinState() { if (all_join_state && all_join_state->finished()) @@ -514,7 +629,7 @@ std::optional MergeJoinAlgorithm::handleAllJoinState /// Accumulate blocks with same key in all_join_state for (size_t i = 0; i < 2; ++i) { - if (cursors[i]->cursor.isValid() && all_join_state->keys[i].equals(cursors[i]->cursor)) + if (cursors[i]->cursor.isValid() && all_join_state->keys[i].equals(*cursors[i])) { size_t pos = cursors[i]->cursor.getRow(); size_t num = nextDistinct(cursors[i]->cursor); @@ -534,12 +649,7 @@ std::optional MergeJoinAlgorithm::handleAllJoinState stat.max_blocks_loaded = std::max(stat.max_blocks_loaded, all_join_state->blocksStored()); /// join all rows with current key - MutableColumns result_cols; - for (size_t i = 0; i < 2; ++i) - { - for (const auto & col : cursors[i]->sampleColumns()) - result_cols.push_back(col->cloneEmpty()); - } + MutableColumns result_cols = getEmptyResultColumns(); size_t total_rows = 0; while (!max_block_size || total_rows < max_block_size) @@ -567,6 +677,52 @@ std::optional MergeJoinAlgorithm::handleAllJoinState return {}; } +std::optional MergeJoinAlgorithm::handleAsofJoinState() +{ + if (strictness != JoinStrictness::Asof) + return {}; + + if (!cursors[1]->fullyCompleted()) + return {}; + + auto & left_cursor = *cursors[0]; + size_t lpos = left_cursor->getRow(); + const auto & left_columns = left_cursor.getCurrent().getColumns(); + + MutableColumns result_cols = getEmptyResultColumns(); + + while (left_cursor->isValid() && asof_join_state.hasMatch(left_cursor, asof_inequality)) + { + size_t i = 0; + for (const auto & col : left_columns) + result_cols[i++]->insertFrom(*col, lpos); + for (const auto & col : asof_join_state.value.getColumns()) + result_cols[i++]->insertFrom(*col, asof_join_state.value_row); + chassert(i == result_cols.size()); + left_cursor->next(); + } + + while (isLeft(kind) && left_cursor->isValid()) + { + /// return row with default values at right side + size_t i = 0; + for (const auto & col : left_columns) + result_cols[i++]->insertFrom(*col, lpos); + for (; i < result_cols.size(); ++i) + result_cols[i]->insertDefault(); + chassert(i == result_cols.size()); + + left_cursor->next(); + } + + size_t result_rows = result_cols.empty() ? 0 : result_cols.front()->size(); + if (result_rows) + return Status(Chunk(std::move(result_cols), result_rows)); + + return {}; +} + + MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin() { PaddedPODArray idx_map[2]; @@ -696,14 +852,14 @@ struct AnyJoinImpl any_join_state.setValue({}); if (!left_cursor->isValid()) { - any_join_state.set(0, left_cursor.cursor); + any_join_state.set(0, left_cursor); if (cmp == 0 && isLeft(kind)) any_join_state.setValue(getRowFromChunk(right_cursor.getCurrent(), rpos)); } if (!right_cursor->isValid()) { - any_join_state.set(1, right_cursor.cursor); + any_join_state.set(1, right_cursor); if (cmp == 0 && isRight(kind)) any_join_state.setValue(getRowFromChunk(left_cursor.getCurrent(), lpos)); } @@ -720,7 +876,7 @@ std::optional MergeJoinAlgorithm::handleAnyJoinState for (size_t source_num = 0; source_num < 2; ++source_num) { auto & current = *cursors[source_num]; - if (any_join_state.keys[source_num].equals(current.cursor)) + if (any_join_state.keys[source_num].equals(current)) { size_t start_pos = current->getRow(); size_t length = nextDistinct(current.cursor); @@ -811,24 +967,35 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() if (!right_cursor->isValid()) return Status(1); - PaddedPODArray left_map; - PaddedPODArray right_map; + const auto & left_columns = left_cursor.getCurrent().getColumns(); + const auto & right_columns = right_cursor.getCurrent().getColumns(); + + MutableColumns result_cols = getEmptyResultColumns(); while (left_cursor->isValid() && right_cursor->isValid()) { auto lpos = left_cursor->getRow(); auto rpos = right_cursor->getRow(); auto cmp = compareCursors(*left_cursor, *right_cursor); + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ({}) <=> ({}) -> {}", __FILE__, __LINE__, left_cursor.dump(), right_cursor.dump(), cmp); + if (cmp == 0) { auto asof_cmp = compareAsofCursors(left_cursor, right_cursor); + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ({}) <=> ({}) -> asof {}", __FILE__, __LINE__, left_cursor.dump(), right_cursor.dump(), asof_cmp); + if ((asof_inequality == ASOFJoinInequality::Less && asof_cmp <= -1) || (asof_inequality == ASOFJoinInequality::LessOrEquals && asof_cmp <= 0)) { /// First row in right table that is greater (or equal) than current row in left table /// matches asof join condition the best - left_map.push_back(lpos); - right_map.push_back(rpos); + size_t i = 0; + for (const auto & col : left_columns) + result_cols[i++]->insertFrom(*col, lpos); + for (const auto & col : right_columns) + result_cols[i++]->insertFrom(*col, rpos); + chassert(i == result_cols.size()); + left_cursor->next(); continue; } @@ -839,39 +1006,99 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() right_cursor->next(); continue; } + + if ((asof_inequality == ASOFJoinInequality::Greater && asof_cmp >= 1) + || (asof_inequality == ASOFJoinInequality::GreaterOrEquals && asof_cmp >= 0)) + { + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + /// condition is satisfied, remember this row and move next to try to find better match + asof_join_state.set(right_cursor, rpos); + right_cursor->next(); + continue; + } + + if (asof_inequality == ASOFJoinInequality::Greater || asof_inequality == ASOFJoinInequality::GreaterOrEquals) + { + /// Asof condition is not satisfied anymore, use last matched row from right table + if (asof_join_state.hasMatch(left_cursor, asof_inequality)) + { + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + size_t i = 0; + for (const auto & col : left_columns) + result_cols[i++]->insertFrom(*col, lpos); + for (const auto & col : asof_join_state.value.getColumns()) + result_cols[i++]->insertFrom(*col, asof_join_state.value_row); + chassert(i == result_cols.size()); + } + else + { + asof_join_state.reset(); + if (isLeft(kind)) + { + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + + /// return row with default values at right side + size_t i = 0; + for (const auto & col : left_columns) + result_cols[i++]->insertFrom(*col, lpos); + for (; i < result_cols.size(); ++i) + result_cols[i]->insertDefault(); + chassert(i == result_cols.size()); + } + } + left_cursor->next(); + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + continue; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "TODO: implement ASOF equality join"); } else if (cmp < 0) { + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + if (asof_join_state.hasMatch(left_cursor, asof_inequality)) + { + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + + size_t i = 0; + for (const auto & col : left_columns) + result_cols[i++]->insertFrom(*col, lpos); + for (const auto & col : asof_join_state.value.getColumns()) + result_cols[i++]->insertFrom(*col, asof_join_state.value_row); + chassert(i == result_cols.size()); + left_cursor->next(); + continue; + } + else + { + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + asof_join_state.reset(); + } + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + /// no matches for rows in left table, just pass them through size_t num = nextDistinct(*left_cursor); - if (isLeft(kind)) + if (isLeft(kind) && num) { /// return them with default values at right side - addRange(left_map, lpos, lpos + num); - addMany(right_map, DEFAULT_VALUE_INDEX, num); + size_t i = 0; + for (const auto & col : left_columns) + result_cols[i++]->insertRangeFrom(*col, lpos, num); + for (; i < result_cols.size(); ++i) + result_cols[i]->insertManyDefaults(num); + chassert(i == result_cols.size()); } } else { + // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); + /// skip rows in right table until we find match for current row in left table nextDistinct(*right_cursor); } } - - chassert(left_map.size() == right_map.size()); - Chunk result; - { - Columns lcols = indexColumns(left_cursor.getCurrent().getColumns(), left_map); - for (auto & col : lcols) - result.addColumn(std::move(col)); - - Columns rcols = indexColumns(right_cursor.getCurrent().getColumns(), right_map); - for (auto & col : rcols) - result.addColumn(std::move(col)); - } - - return Status(std::move(result)); + size_t num_rows = result_cols.empty() ? 0 : result_cols.front()->size(); + return Status(Chunk(std::move(result_cols), num_rows)); } @@ -929,6 +1156,9 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() return std::move(*result); } + if (auto result = handleAsofJoinState()) + return std::move(*result); + if (cursors[0]->fullyCompleted() || cursors[1]->fullyCompleted()) { if (!cursors[0]->fullyCompleted() && isLeftOrFull(kind)) diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 375c9ebd3cc..dbdda0b166b 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -37,62 +37,28 @@ using FullMergeJoinCursorPtr = std::unique_ptr; /// Used instead of storing previous block struct JoinKeyRow { - std::vector row; - JoinKeyRow() = default; - explicit JoinKeyRow(const SortCursorImpl & impl_, size_t pos) - { - row.reserve(impl_.sort_columns.size()); - for (const auto & col : impl_.sort_columns) - { - auto new_col = col->cloneEmpty(); - new_col->insertFrom(*col, pos); - row.push_back(std::move(new_col)); - } - } + JoinKeyRow(const FullMergeJoinCursor & cursor, size_t pos); - void reset() - { - row.clear(); - } + bool equals(const FullMergeJoinCursor & cursor) const; + bool asofMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) const; - bool equals(const SortCursorImpl & impl) const - { - if (row.empty()) - return false; + void reset(); - assert(this->row.size() == impl.sort_columns_size); - for (size_t i = 0; i < impl.sort_columns_size; ++i) - { - int cmp = this->row[i]->compareAt(0, impl.getRow(), *impl.sort_columns[i], impl.desc[i].nulls_direction); - if (cmp != 0) - return false; - } - return true; - } + std::vector row; }; /// Remembers previous key if it was joined in previous block class AnyJoinState : boost::noncopyable { public: - AnyJoinState() = default; + void set(size_t source_num, const FullMergeJoinCursor & cursor); + void setValue(Chunk value_); - void set(size_t source_num, const SortCursorImpl & cursor) - { - assert(cursor.rows); - keys[source_num] = JoinKeyRow(cursor, cursor.rows - 1); - } + void reset(size_t source_num); - void reset(size_t source_num) - { - keys[source_num].reset(); - } - - void setValue(Chunk value_) { value = std::move(value_); } - - bool empty() const { return keys[0].row.empty() && keys[1].row.empty(); } + bool empty() const; /// current keys JoinKeyRow keys[2]; @@ -125,8 +91,8 @@ public: Chunk chunk; }; - AllJoinState(const SortCursorImpl & lcursor, size_t lpos, - const SortCursorImpl & rcursor, size_t rpos) + AllJoinState(const FullMergeJoinCursor & lcursor, size_t lpos, + const FullMergeJoinCursor & rcursor, size_t rpos) : keys{JoinKeyRow(lcursor, lpos), JoinKeyRow(rcursor, rpos)} { } @@ -194,6 +160,25 @@ private: size_t ridx = 0; }; + +class AsofJoinState : boost::noncopyable +{ +public: + void set(const FullMergeJoinCursor & rcursor, size_t rpos); + void reset(); + + bool hasMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) + { + if (value.empty()) + return false; + return key.asofMatch(cursor, asof_inequality); + } + + JoinKeyRow key; + Chunk value; + size_t value_row = 0; +}; + /* * Wrapper for SortCursorImpl */ @@ -239,11 +224,15 @@ public: const Block & sampleBlock() const { return sample_block; } Columns sampleColumns() const { return sample_block.getColumns(); } - const IColumn & getAsofColumn() const + const IColumn * getAsofColumn() const { - return *cursor.all_columns[asof_column_position]; + if (!asof_column_position) + return nullptr; + return cursor.all_columns[*asof_column_position]; } + String dump() const; + private: Block sample_block; SortDescription desc; @@ -251,7 +240,7 @@ private: Chunk current_chunk; bool recieved_all_blocks = false; - size_t asof_column_position; + std::optional asof_column_position; }; /* @@ -284,12 +273,13 @@ private: std::optional handleAllJoinState(); Status allJoin(); + std::optional handleAsofJoinState(); Status asofJoin(); + MutableColumns getEmptyResultColumns() const; Chunk createBlockWithDefaults(size_t source_num); Chunk createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const; - /// For `USING` join key columns should have values from right side instead of defaults std::unordered_map left_to_right_key_remap; @@ -299,6 +289,7 @@ private: /// Keep some state to make handle data from different blocks AnyJoinState any_join_state; std::unique_ptr all_join_state; + AsofJoinState asof_join_state; JoinKind kind; JoinStrictness strictness; diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 1dc410bd6b8..4e0727779b7 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -18,19 +18,53 @@ #include #include +#include +#include + + #include using namespace DB; +namespace +{ + +[[ maybe_unused ]] +String dumpBlock(std::shared_ptr source) +{ + WriteBufferFromOwnString buf; + { + Block header = source->getPort().getHeader(); + QueryPipeline pipeline(source); + auto format = std::make_shared(buf, header, FormatSettings{}, false); + pipeline.complete(std::move(format)); + + CompletedPipelineExecutor executor(pipeline); + executor.execute(); + } + return buf.str(); +} + +[[ maybe_unused ]] +String dumpBlock(const Block & block) +{ + Block header = block.cloneEmpty(); + Chunk data(block.getColumns(), block.rows()); + auto source = std::make_shared(header, std::move(data)); + return dumpBlock(std::move(source)); +} + UInt64 getAndPrintRandomSeed() { UInt64 seed = randomSeed(); - std::cerr << __FILE__ << "::" << "TEST_RANDOM_SEED = " << seed << "ull" << std::endl; + if (const char * random_seed = std::getenv("TEST_RANDOM_SEED")) // NOLINT(concurrency-mt-unsafe) + seed = std::stoull(random_seed); + + std::cerr << __FILE__ << " :: " << "TEST_RANDOM_SEED=" << seed << std::endl; return seed; } static UInt64 TEST_RANDOM_SEED = getAndPrintRandomSeed(); - static pcg64 rng(TEST_RANDOM_SEED); @@ -94,42 +128,29 @@ QueryPipeline buildJoinPipeline( std::shared_ptr oneColumnSource(const std::vector> & values) { - Block header = { ColumnWithTypeAndName(std::make_shared(), "x") }; + Block header = { + ColumnWithTypeAndName(std::make_shared(), "key"), + ColumnWithTypeAndName(std::make_shared(), "idx"), + }; + + UInt64 idx = 0; Chunks chunks; for (const auto & chunk_values : values) { - auto column = ColumnUInt64::create(); + auto key_column = ColumnUInt64::create(); + auto idx_column = ColumnUInt64::create(); + for (auto n : chunk_values) - column->insertValue(n); - chunks.emplace_back(Chunk(Columns{std::move(column)}, chunk_values.size())); + { + key_column->insertValue(n); + idx_column->insertValue(idx); + ++idx; + } + chunks.emplace_back(Chunk(Columns{std::move(key_column), std::move(idx_column)}, chunk_values.size())); } return std::make_shared(header, std::move(chunks)); } - -TEST(FullSortingJoin, Simple) -try -{ - auto left_source = oneColumnSource({ {1, 2, 3, 4, 5} }); - auto right_source = oneColumnSource({ {1}, {2}, {3}, {4}, {5} }); - - auto pipeline = buildJoinPipeline(left_source, right_source); - PullingPipelineExecutor executor(pipeline); - - Block block; - - size_t total_result_rows = 0; - while (executor.pull(block)) - total_result_rows += block.rows(); - - ASSERT_EQ(total_result_rows, 5); -} -catch (Exception & e) -{ - std::cout << e.getStackTraceString() << std::endl; - throw; -} - class SourceChunksBuilder { public: @@ -163,7 +184,7 @@ public: return; } - std::shared_ptr build() + std::shared_ptr getSource() { addChunk(); @@ -215,9 +236,11 @@ Block executePipeline(QueryPipeline && pipeline) template void assertColumnVectorEq(const typename ColumnVector::Container & expected, const Block & block, const std::string & name) { - const auto & actual = assert_cast *>(block.getByName(name).column.get())->getData(); - EXPECT_EQ(actual.size(), expected.size()); - ASSERT_EQ(actual, expected); + const auto * actual = typeid_cast *>(block.getByName(name).column.get()); + ASSERT_TRUE(actual) << "unexpected column type: " << block.getByName(name).column->dumpStructure() << "expected: " << typeid(ColumnVector).name(); + + EXPECT_EQ(actual->getData().size(), expected.size()); + ASSERT_EQ(actual->getData(), expected) << "column name: " << name; } template @@ -263,7 +286,90 @@ void generateNextKey(UInt64 & k1, String & k2) k2 = new_k2; } -TEST(FullSortingJoin, Any) +bool isStrict(ASOFJoinInequality inequality) +{ + return inequality == ASOFJoinInequality::Less || inequality == ASOFJoinInequality::Greater; +} + +} + +TEST(FullSortingJoin, AllAnyOneKey) +try +{ + { + SCOPED_TRACE("Inner All"); + Block result = executePipeline(buildJoinPipeline( + oneColumnSource({ {1, 2, 3, 4, 5} }), + oneColumnSource({ {1}, {2}, {3}, {4}, {5} }), + 1, JoinKind::Inner, JoinStrictness::All)); + + assertColumnVectorEq(ColumnUInt64::Container({0, 1, 2, 3, 4}), result, "t1.idx"); + assertColumnVectorEq(ColumnUInt64::Container({0, 1, 2, 3, 4}), result, "t2.idx"); + } + { + SCOPED_TRACE("Inner Any"); + Block result = executePipeline(buildJoinPipeline( + oneColumnSource({ {1, 2, 3, 4, 5} }), + oneColumnSource({ {1}, {2}, {3}, {4}, {5} }), + 1, JoinKind::Inner, JoinStrictness::Any)); + assertColumnVectorEq(ColumnUInt64::Container({0, 1, 2, 3, 4}), result, "t1.idx"); + assertColumnVectorEq(ColumnUInt64::Container({0, 1, 2, 3, 4}), result, "t2.idx"); + } + { + SCOPED_TRACE("Inner All"); + Block result = executePipeline(buildJoinPipeline( + oneColumnSource({ {2, 2, 2}, {2, 3}, {3, 5} }), + oneColumnSource({ {1, 1, 1}, {2, 2}, {3, 4} }), + 1, JoinKind::Inner, JoinStrictness::All)); + assertColumnVectorEq(ColumnUInt64::Container({0, 1, 2, 0, 1, 2, 3, 3, 4, 5}), result, "t1.idx"); + assertColumnVectorEq(ColumnUInt64::Container({3, 3, 3, 4, 4, 4, 3, 4, 5, 5}), result, "t2.idx"); + } + { + SCOPED_TRACE("Inner Any"); + Block result = executePipeline(buildJoinPipeline( + oneColumnSource({ {2, 2, 2}, {2, 3}, {3, 5} }), + oneColumnSource({ {1, 1, 1}, {2, 2}, {3, 4} }), + 1, JoinKind::Inner, JoinStrictness::Any)); + assertColumnVectorEq(ColumnUInt64::Container({0, 4}), result, "t1.idx"); + assertColumnVectorEq(ColumnUInt64::Container({3, 5}), result, "t2.idx"); + } + { + SCOPED_TRACE("Inner Any"); + Block result = executePipeline(buildJoinPipeline( + oneColumnSource({ {2, 2, 2, 2}, {3}, {3, 5} }), + oneColumnSource({ {1, 1, 1, 2}, {2}, {3, 4} }), + 1, JoinKind::Inner, JoinStrictness::Any)); + assertColumnVectorEq(ColumnUInt64::Container({0, 4}), result, "t1.idx"); + assertColumnVectorEq(ColumnUInt64::Container({3, 5}), result, "t2.idx"); + } + { + + SCOPED_TRACE("Left Any"); + Block result = executePipeline(buildJoinPipeline( + oneColumnSource({ {2, 2, 2}, {2, 3}, {3, 5} }), + oneColumnSource({ {1, 1, 1}, {2, 2}, {3, 4} }), + 1, JoinKind::Left, JoinStrictness::Any)); + assertColumnVectorEq(ColumnUInt64::Container({0, 1, 2, 3, 4, 5, 6}), result, "t1.idx"); + assertColumnVectorEq(ColumnUInt64::Container({3, 3, 3, 3, 5, 5, 0}), result, "t2.idx"); + } + { + SCOPED_TRACE("Left Any"); + Block result = executePipeline(buildJoinPipeline( + oneColumnSource({ {2, 2, 2, 2}, {3}, {3, 5} }), + oneColumnSource({ {1, 1, 1, 2}, {2}, {3, 4} }), + 1, JoinKind::Left, JoinStrictness::Any)); + assertColumnVectorEq(ColumnUInt64::Container({0, 1, 2, 3, 4, 5, 6}), result, "t1.idx"); + assertColumnVectorEq(ColumnUInt64::Container({3, 3, 3, 3, 5, 5, 0}), result, "t2.idx"); + } +} +catch (Exception & e) +{ + std::cout << e.getStackTraceString() << std::endl; + throw; +} + + +TEST(FullSortingJoin, AnyRandomized) try { JoinKind kind = getRandomFrom({JoinKind::Inner, JoinKind::Left, JoinKind::Right}); @@ -288,7 +394,7 @@ try auto expected_left = ColumnString::create(); auto expected_right = ColumnString::create(); - UInt64 k1 = 0; + UInt64 k1 = 1; String k2 = ""; auto get_attr = [&](const String & side, size_t idx) -> String @@ -338,7 +444,7 @@ try } Block result_block = executePipeline(buildJoinPipeline( - left_source.build(), right_source.build(), /* key_length = */ 2, + left_source.getSource(), right_source.getSource(), /* key_length = */ 2, kind, JoinStrictness::Any)); assertColumnEq(*expected_left, result_block, "t1.attr"); assertColumnEq(*expected_right, result_block, "t2.attr"); @@ -356,10 +462,10 @@ try {std::make_shared(), "key"}, {std::make_shared(), "t"}, }); - left_source.addRow({"AMZN", 3}); left_source.addRow({"AMZN", 4}); left_source.addRow({"AMZN", 6}); + left_source.addRow({"SBUX", 10}); SourceChunksBuilder right_source({ {std::make_shared(), "key"}, @@ -371,14 +477,19 @@ try right_source.addRow({"AAPL", 2, 98}); right_source.addRow({"AAPL", 3, 99}); right_source.addRow({"AMZN", 1, 100}); + right_source.addRow({"AMZN", 2, 0}); + right_source.addChunk(); right_source.addRow({"AMZN", 2, 110}); right_source.addChunk(); right_source.addRow({"AMZN", 4, 130}); right_source.addRow({"AMZN", 5, 140}); + right_source.addRow({"SBUX", 8, 180}); + right_source.addChunk(); + right_source.addRow({"SBUX", 9, 190}); { Block result_block = executePipeline(buildJoinPipeline( - left_source.build(), right_source.build(), /* key_length = */ 2, + left_source.getSource(), right_source.getSource(), /* key_length = */ 2, JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals)); auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); @@ -388,18 +499,19 @@ try })); } - // { - // Block result_block = executePipeline(buildJoinPipeline( - // left_source.build(), right_source.build(), /* key_length = */ 2, - // JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals)); - // auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); + { + Block result_block = executePipeline(buildJoinPipeline( + left_source.getSource(), right_source.getSource(), /* key_length = */ 2, + JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals)); + auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"}); - // ASSERT_EQ(values, (std::vector>{ - // {"AMZN", 3u, 2u, 110u}, - // {"AMZN", 4u, 4u, 130u}, - // {"AMZN", 6u, 5u, 140u}, - // })); - // } + ASSERT_EQ(values, (std::vector>{ + {"AMZN", 3u, 2u, 110u}, + {"AMZN", 4u, 4u, 130u}, + {"AMZN", 6u, 5u, 140u}, + {"SBUX", 10u, 9u, 190u}, + })); + } } catch (Exception & e) { @@ -418,8 +530,7 @@ try {std::make_shared(), "value"}, }); - UInt64 p = std::uniform_int_distribution<>(0, 2)(rng); - double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0); + double break_prob = getRandomDoubleQuantized(2); std::uniform_real_distribution<> prob_dis(0.0, 1.0); for (const auto & row : std::vector>{ {1, 101}, {2, 102}, {4, 104}, {5, 105}, {11, 111}, {15, 115} }) { @@ -427,7 +538,7 @@ try if (prob_dis(rng) < break_prob) right_source_builder.addChunk(); } - auto right_source = right_source_builder.build(); + auto right_source = right_source_builder.getSource(); auto pipeline = buildJoinPipeline( left_source, right_source, /* key_length = */ 1, @@ -436,7 +547,7 @@ try Block result_block = executePipeline(std::move(pipeline)); ASSERT_EQ( - assert_cast(result_block.getByName("t1.x").column.get())->getData(), + assert_cast(result_block.getByName("t1.key").column.get())->getData(), (ColumnUInt64::Container{3, 3, 3, 3, 3, 5, 5, 6, 9, 9, 10}) ); @@ -456,15 +567,14 @@ catch (Exception & e) throw; } -TEST(FullSortingJoin, AsofGeneratedTestData) +TEST(FullSortingJoin, AsofLessGeneratedTestData) try { - auto join_kind = getRandomFrom({JoinKind::Inner, JoinKind::Left}); + auto join_kind = getRandomFrom({ JoinKind::Inner, JoinKind::Left }); - auto asof_inequality = getRandomFrom({ - ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals, - // ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals, - }); + auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals }); + + SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality)); SourceChunksBuilder left_source_builder({ {std::make_shared(), "k1"}, @@ -485,11 +595,13 @@ try ColumnInt64::Container expected; - UInt64 k1 = 0; - String k2 = "asdfg"; + UInt64 k1 = 1; + String k2 = ""; auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); for (size_t key_num = 0; key_num < key_num_total; ++key_num) { + generateNextKey(k1, k2); + Int64 left_t = 0; size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng); for (size_t i = 0; i < num_left_rows; ++i) @@ -504,18 +616,10 @@ try auto right_t = left_t; for (size_t j = 0; j < num_matches; ++j) { - int min_step = 1; - if (asof_inequality == ASOFJoinInequality::LessOrEquals || asof_inequality == ASOFJoinInequality::GreaterOrEquals) - min_step = 0; + int min_step = isStrict(asof_inequality) ? 1 : 0; right_t += std::uniform_int_distribution<>(min_step, 3)(rng); - bool is_match = false; - - if (asof_inequality == ASOFJoinInequality::LessOrEquals || asof_inequality == ASOFJoinInequality::Less) - is_match = j == 0; - else if (asof_inequality == ASOFJoinInequality::GreaterOrEquals || asof_inequality == ASOFJoinInequality::Greater) - is_match = j == num_matches - 1; - + bool is_match = j == 0; right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * left_t : -1}); } /// next left_t should be greater than right_t not to match with previous rows @@ -523,7 +627,7 @@ try } /// generate some rows with greater left_t to check that they are not matched - num_left_rows = std::uniform_int_distribution<>(1, 100)(rng); + num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 100)(rng) : 0; for (size_t i = 0; i < num_left_rows; ++i) { left_t += std::uniform_int_distribution<>(1, 10)(rng); @@ -532,12 +636,10 @@ try if (join_kind == JoinKind::Left) expected.push_back(-10 * left_t); } - - generateNextKey(k1, k2); } Block result_block = executePipeline(buildJoinPipeline( - left_source_builder.build(), right_source_builder.build(), + left_source_builder.getSource(), right_source_builder.getSource(), /* key_length = */ 3, join_kind, JoinStrictness::Asof, asof_inequality)); @@ -548,7 +650,97 @@ try assertColumnVectorEq(expected, result_block, "t2.attr"); } -catch (Exception & e) { +catch (Exception & e) +{ + std::cout << e.getStackTraceString() << std::endl; + throw; +} + +TEST(FullSortingJoin, AsofGreaterGeneratedTestData) +try +{ + auto join_kind = getRandomFrom({ JoinKind::Inner, JoinKind::Left }); + + auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals }); + + SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality)); + + SourceChunksBuilder left_source_builder({ + {std::make_shared(), "k1"}, + {std::make_shared(), "k2"}, + {std::make_shared(), "t"}, + {std::make_shared(), "attr"}, + }); + + SourceChunksBuilder right_source_builder({ + {std::make_shared(), "k1"}, + {std::make_shared(), "k2"}, + {std::make_shared(), "t"}, + {std::make_shared(), "attr"}, + }); + + left_source_builder.break_prob = getRandomDoubleQuantized(); + right_source_builder.break_prob = getRandomDoubleQuantized(); + + ColumnInt64::Container expected; + + UInt64 k1 = 1; + String k2 = ""; + UInt64 left_t = 0; + + auto key_num_total = std::uniform_int_distribution<>(1, 100)(rng); + for (size_t key_num = 0; key_num < key_num_total; ++key_num) + { + generateNextKey(k1, k2); + + /// generate some rows with smaller left_t to check that they are not matched + size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 10)(rng) : 0; + for (size_t i = 0; i < num_left_rows; ++i) + { + left_t += std::uniform_int_distribution<>(1, 10)(rng); + left_source_builder.addRow({k1, k2, left_t, -10 * left_t}); + + if (join_kind == JoinKind::Left) + expected.push_back(-10 * left_t); + } + + if (std::bernoulli_distribution(0.1)(rng)) + continue; + + size_t num_right_matches = std::uniform_int_distribution<>(1, 10)(rng); + auto right_t = left_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 0 : 1, 10)(rng); + for (size_t j = 0; j < num_right_matches; ++j) + { + right_t += std::uniform_int_distribution<>(0, 3)(rng); + bool is_match = j == num_right_matches - 1; + right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * right_t : -1}); + } + + /// next left_t should be greater than (or equals) right_t to match with previous rows + left_t = right_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 1 : 0, 10)(rng); + size_t num_left_matches = std::uniform_int_distribution<>(1, 10)(rng); + for (size_t j = 0; j < num_left_matches; ++j) + { + left_t += std::uniform_int_distribution<>(0, 3)(rng); + left_source_builder.addRow({k1, k2, left_t, 10 * right_t}); + expected.push_back(10 * right_t); + } + } + + Block result_block = executePipeline(buildJoinPipeline( + left_source_builder.getSource(), right_source_builder.getSource(), + /* key_length = */ 3, + join_kind, JoinStrictness::Asof, asof_inequality)); + + assertColumnVectorEq(expected, result_block, "t1.attr"); + + for (auto & e : expected) + e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr + + assertColumnVectorEq(expected, result_block, "t2.attr"); +} +catch (Exception & e) +{ std::cout << e.getStackTraceString() << std::endl; throw; } diff --git a/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql b/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql index a4e60ff54dd..03936107563 100644 --- a/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql +++ b/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql @@ -19,7 +19,7 @@ SELECT * FROM t1 ANTI JOIN t2 ON t1.key = t2.key; -- { serverError NOT_IMPLEMENT SELECT * FROM t1 SEMI JOIN t2 ON t1.key = t2.key; -- { serverError NOT_IMPLEMENTED } -SELECT * FROM t1 ASOF JOIN t2 ON t1.key = t2.key AND t1.val > t2.val; -- { serverError NOT_IMPLEMENTED } +-- SELECT * FROM t1 ASOF JOIN t2 ON t1.key = t2.key AND t1.val > t2.val; -- { serverError NOT_IMPLEMENTED } SELECT * FROM t1 ANY JOIN t2 ON t1.key = t2.key SETTINGS any_join_distinct_right_table_keys = 1; -- { serverError NOT_IMPLEMENTED } From b6b55cfb187ddcbb6316d905c60d9be481bfb1a4 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 27 Sep 2023 09:05:50 +0000 Subject: [PATCH 10/45] fix asof join --- src/Processors/Transforms/MergeJoinTransform.cpp | 2 +- src/Processors/tests/gtest_full_sorting_join.cpp | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 638f2f1cb10..ad564557c36 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -1171,7 +1171,7 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() } /// check if blocks are not intersecting at all - if (int cmp = totallyCompare(cursors[0]->cursor, cursors[1]->cursor, null_direction_hint); cmp != 0) + if (int cmp = totallyCompare(cursors[0]->cursor, cursors[1]->cursor, null_direction_hint); cmp != 0 && strictness != JoinStrictness::Asof) { if (cmp < 0) { diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 4e0727779b7..2ecf7805df4 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -30,13 +30,13 @@ namespace { [[ maybe_unused ]] -String dumpBlock(std::shared_ptr source) +String dumpBlockSource(std::shared_ptr source, bool mono_block = false) { WriteBufferFromOwnString buf; { Block header = source->getPort().getHeader(); QueryPipeline pipeline(source); - auto format = std::make_shared(buf, header, FormatSettings{}, false); + auto format = std::make_shared(buf, header, FormatSettings{}, mono_block); pipeline.complete(std::move(format)); CompletedPipelineExecutor executor(pipeline); @@ -51,7 +51,7 @@ String dumpBlock(const Block & block) Block header = block.cloneEmpty(); Chunk data(block.getColumns(), block.rows()); auto source = std::make_shared(header, std::move(data)); - return dumpBlock(std::move(source)); + return dumpBlockSource(std::move(source)); } UInt64 getAndPrintRandomSeed() @@ -732,6 +732,10 @@ try /* key_length = */ 3, join_kind, JoinStrictness::Asof, asof_inequality)); + // std::cerr << "============ left ============" << std::endl << dumpBlockSource(left_source_builder.getSource()) << std::endl; + // std::cerr << "============ right ============" << std::endl << dumpBlockSource(right_source_builder.getSource()) << std::endl; + // std::cerr << "============ result ============" << std::endl << dumpBlock(result_block) << std::endl; + assertColumnVectorEq(expected, result_block, "t1.attr"); for (auto & e : expected) From d015a023bb85c72c2747cebed12d065d5aad1159 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 27 Sep 2023 09:56:30 +0000 Subject: [PATCH 11/45] unique rng seed per each test in gtest_full_sorting_join --- .../tests/gtest_full_sorting_join.cpp | 122 ++++++++++-------- 1 file changed, 69 insertions(+), 53 deletions(-) diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 2ecf7805df4..2bd3357eff9 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -54,20 +54,6 @@ String dumpBlock(const Block & block) return dumpBlockSource(std::move(source)); } -UInt64 getAndPrintRandomSeed() -{ - UInt64 seed = randomSeed(); - if (const char * random_seed = std::getenv("TEST_RANDOM_SEED")) // NOLINT(concurrency-mt-unsafe) - seed = std::stoull(random_seed); - - std::cerr << __FILE__ << " :: " << "TEST_RANDOM_SEED=" << seed << std::endl; - return seed; -} - -static UInt64 TEST_RANDOM_SEED = getAndPrintRandomSeed(); -static pcg64 rng(TEST_RANDOM_SEED); - - QueryPipeline buildJoinPipeline( std::shared_ptr left_source, std::shared_ptr right_source, @@ -154,7 +140,6 @@ std::shared_ptr oneColumnSource(const std::vector> class SourceChunksBuilder { public: - double break_prob = 0.0; explicit SourceChunksBuilder(const Block & header_) : header(header_) @@ -163,13 +148,20 @@ public: chassert(!current_chunk.empty()); } + void setBreakProbability(pcg64 & rng_) + { + /// random probability with possibility to have exact 0.0 and 1.0 values + break_prob = std::uniform_int_distribution(0, 5)(rng_) / static_cast(5); + rng = &rng_; + } + void addRow(const std::vector & row) { chassert(row.size() == current_chunk.size()); for (size_t i = 0; i < current_chunk.size(); ++i) current_chunk[i]->insert(row[i]); - if (break_prob > 0.0 && std::uniform_real_distribution<>(0.0, 1.0)(rng) < break_prob) + if (rng && std::uniform_real_distribution<>(0.0, 1.0)(*rng) < break_prob) addChunk(); } @@ -200,6 +192,9 @@ private: Block header; Chunks chunks; MutableColumns current_chunk; + + pcg64 * rng = nullptr; + double break_prob = 0.0; }; @@ -239,8 +234,20 @@ void assertColumnVectorEq(const typename ColumnVector::Container & expected, const auto * actual = typeid_cast *>(block.getByName(name).column.get()); ASSERT_TRUE(actual) << "unexpected column type: " << block.getByName(name).column->dumpStructure() << "expected: " << typeid(ColumnVector).name(); + auto get_first_diff = [&]() -> String + { + const auto & actual_data = actual->getData(); + size_t num_rows = std::min(expected.size(), actual_data.size()); + for (size_t i = 0; i < num_rows; ++i) + { + if (expected[i] != actual_data[i]) + return fmt::format(", expected: {}, actual: {} at row {}", expected[i], actual_data[i], i); + } + return ""; + }; + EXPECT_EQ(actual->getData().size(), expected.size()); - ASSERT_EQ(actual->getData(), expected) << "column name: " << name; + ASSERT_EQ(actual->getData(), expected) << "column name: " << name << get_first_diff(); } template @@ -264,20 +271,14 @@ void assertColumnEq(const IColumn & expected, const Block & block, const std::st } template -T getRandomFrom(const std::initializer_list & opts) +T getRandomFrom(pcg64 & rng, const std::initializer_list & opts) { std::vector options(opts.begin(), opts.end()); size_t idx = std::uniform_int_distribution(0, options.size() - 1)(rng); return options[idx]; } -/// Used to have accurate 0.0 and 1.0 probabilities -double getRandomDoubleQuantized(size_t quants = 5) -{ - return std::uniform_int_distribution(0, quants)(rng) / static_cast(quants); -} - -void generateNextKey(UInt64 & k1, String & k2) +void generateNextKey(pcg64 & rng, UInt64 & k1, String & k2) { size_t str_len = std::uniform_int_distribution<>(1, 10)(rng); String new_k2 = getRandomASCIIString(str_len, rng); @@ -293,6 +294,28 @@ bool isStrict(ASOFJoinInequality inequality) } +class FullSortingJoinRandomized : public ::testing::Test +{ +public: + FullSortingJoinRandomized() = default; + + void SetUp() override + { + UInt64 seed = randomSeed(); + if (const char * random_seed = std::getenv("TEST_RANDOM_SEED")) // NOLINT(concurrency-mt-unsafe) + seed = std::stoull(random_seed); + + std::cerr << "TEST_RANDOM_SEED=" << seed << std::endl; + rng = pcg64(seed); + } + + void TearDown() override + { + } + + pcg64 rng; +}; + TEST(FullSortingJoin, AllAnyOneKey) try { @@ -369,10 +392,10 @@ catch (Exception & e) } -TEST(FullSortingJoin, AnyRandomized) +TEST_F(FullSortingJoinRandomized, Any) try { - JoinKind kind = getRandomFrom({JoinKind::Inner, JoinKind::Left, JoinKind::Right}); + JoinKind kind = getRandomFrom(rng, {JoinKind::Inner, JoinKind::Left, JoinKind::Right}); SourceChunksBuilder left_source({ {std::make_shared(), "k1"}, @@ -386,8 +409,8 @@ try {std::make_shared(), "attr"}, }); - left_source.break_prob = getRandomDoubleQuantized(); - right_source.break_prob = getRandomDoubleQuantized(); + left_source.setBreakProbability(rng); + right_source.setBreakProbability(rng); size_t num_keys = std::uniform_int_distribution<>(100, 1000)(rng); @@ -404,7 +427,7 @@ try for (size_t i = 0; i < num_keys; ++i) { - generateNextKey(k1, k2); + generateNextKey(rng, k1, k2); /// Key is present in left, right or both tables. Both tables is more probable. size_t key_presence = std::uniform_int_distribution<>(0, 10)(rng); @@ -520,7 +543,7 @@ catch (Exception & e) } -TEST(FullSortingJoin, AsofOnlyColumn) +TEST_F(FullSortingJoinRandomized, AsofOnlyColumn) try { auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} }); @@ -530,14 +553,11 @@ try {std::make_shared(), "value"}, }); - double break_prob = getRandomDoubleQuantized(2); - std::uniform_real_distribution<> prob_dis(0.0, 1.0); + right_source_builder.setBreakProbability(rng); + for (const auto & row : std::vector>{ {1, 101}, {2, 102}, {4, 104}, {5, 105}, {11, 111}, {15, 115} }) - { right_source_builder.addRow(row); - if (prob_dis(rng) < break_prob) - right_source_builder.addChunk(); - } + auto right_source = right_source_builder.getSource(); auto pipeline = buildJoinPipeline( @@ -567,12 +587,12 @@ catch (Exception & e) throw; } -TEST(FullSortingJoin, AsofLessGeneratedTestData) +TEST_F(FullSortingJoinRandomized, AsofLessGeneratedTestData) try { - auto join_kind = getRandomFrom({ JoinKind::Inner, JoinKind::Left }); + auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left }); - auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals }); + auto asof_inequality = getRandomFrom(rng, { ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals }); SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality)); @@ -590,8 +610,8 @@ try {std::make_shared(), "attr"}, }); - left_source_builder.break_prob = getRandomDoubleQuantized(); - right_source_builder.break_prob = getRandomDoubleQuantized(); + left_source_builder.setBreakProbability(rng); + right_source_builder.setBreakProbability(rng); ColumnInt64::Container expected; @@ -600,7 +620,7 @@ try auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); for (size_t key_num = 0; key_num < key_num_total; ++key_num) { - generateNextKey(k1, k2); + generateNextKey(rng, k1, k2); Int64 left_t = 0; size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng); @@ -656,12 +676,12 @@ catch (Exception & e) throw; } -TEST(FullSortingJoin, AsofGreaterGeneratedTestData) +TEST_F(FullSortingJoinRandomized, AsofGreaterGeneratedTestData) try { - auto join_kind = getRandomFrom({ JoinKind::Inner, JoinKind::Left }); + auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left }); - auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals }); + auto asof_inequality = getRandomFrom(rng, { ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals }); SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality)); @@ -679,8 +699,8 @@ try {std::make_shared(), "attr"}, }); - left_source_builder.break_prob = getRandomDoubleQuantized(); - right_source_builder.break_prob = getRandomDoubleQuantized(); + left_source_builder.setBreakProbability(rng); + right_source_builder.setBreakProbability(rng); ColumnInt64::Container expected; @@ -691,7 +711,7 @@ try auto key_num_total = std::uniform_int_distribution<>(1, 100)(rng); for (size_t key_num = 0; key_num < key_num_total; ++key_num) { - generateNextKey(k1, k2); + generateNextKey(rng, k1, k2); /// generate some rows with smaller left_t to check that they are not matched size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 10)(rng) : 0; @@ -732,10 +752,6 @@ try /* key_length = */ 3, join_kind, JoinStrictness::Asof, asof_inequality)); - // std::cerr << "============ left ============" << std::endl << dumpBlockSource(left_source_builder.getSource()) << std::endl; - // std::cerr << "============ right ============" << std::endl << dumpBlockSource(right_source_builder.getSource()) << std::endl; - // std::cerr << "============ result ============" << std::endl << dumpBlock(result_block) << std::endl; - assertColumnVectorEq(expected, result_block, "t1.attr"); for (auto & e : expected) From 3e9090cbebb3cdb97f4dbb4b9d03c7e9d9e242b1 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 27 Sep 2023 11:14:37 +0000 Subject: [PATCH 12/45] fix --- .../sql-reference/statements/select/join.md | 5 ++- .../Transforms/MergeJoinTransform.cpp | 41 ++++++++++++------- .../Transforms/MergeJoinTransform.h | 23 +---------- .../tests/gtest_full_sorting_join.cpp | 36 +++++++++------- 4 files changed, 52 insertions(+), 53 deletions(-) diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md index 34c6016235a..96d9d26977d 100644 --- a/docs/en/sql-reference/statements/select/join.md +++ b/docs/en/sql-reference/statements/select/join.md @@ -297,7 +297,7 @@ Algorithm requires the special column in tables. This column: - Must contain an ordered sequence. - Can be one of the following types: [Int, UInt](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md), [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md), [Decimal](../../../sql-reference/data-types/decimal.md). -- Can’t be the only column in the `JOIN` clause. +- For `hash` join algorithm it can’t be the only column in the `JOIN` clause. Syntax `ASOF JOIN ... ON`: @@ -337,7 +337,8 @@ For example, consider the following tables: `ASOF JOIN` can take the timestamp of a user event from `table_1` and find an event in `table_2` where the timestamp is closest to the timestamp of the event from `table_1` corresponding to the closest match condition. Equal timestamp values are the closest if available. Here, the `user_id` column can be used for joining on equality and the `ev_time` column can be used for joining on the closest match. In our example, `event_1_1` can be joined with `event_2_1` and `event_1_2` can be joined with `event_2_3`, but `event_2_2` can’t be joined. :::note -`ASOF` join is **not** supported in the [Join](../../../engines/table-engines/special/join.md) table engine. +`ASOF JOIN` is supported only by `hash` and `full_sorting_merge` join algorithms. +It's **not** supported in the [Join](../../../engines/table-engines/special/join.md) table engine. ::: ## PASTE JOIN Usage diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index ad564557c36..0d9eab248d8 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -329,6 +329,26 @@ void AsofJoinState::reset() value.clear(); } +FullMergeJoinCursor::FullMergeJoinCursor(const Block & sample_block_, const SortDescription & description_, bool is_asof) + : sample_block(sample_block_.cloneEmpty()) + , desc(description_) +{ + if (desc.size() == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty sort description for FullMergeJoinCursor"); + + if (is_asof) + { + /// For ASOF join prefix of sort description is used for equality comparison + /// and the last column is used for inequality comparison and is handled separately + + auto asof_column_description = desc.back(); + desc.pop_back(); + + chassert(asof_column_description.direction == 1 && asof_column_description.nulls_direction == 1); + asof_column_position = sample_block.getPositionByName(asof_column_description.column_name); + } +} + const Chunk & FullMergeJoinCursor::getCurrent() const { return current_chunk; @@ -686,7 +706,6 @@ std::optional MergeJoinAlgorithm::handleAsofJoinStat return {}; auto & left_cursor = *cursors[0]; - size_t lpos = left_cursor->getRow(); const auto & left_columns = left_cursor.getCurrent().getColumns(); MutableColumns result_cols = getEmptyResultColumns(); @@ -695,7 +714,7 @@ std::optional MergeJoinAlgorithm::handleAsofJoinStat { size_t i = 0; for (const auto & col : left_columns) - result_cols[i++]->insertFrom(*col, lpos); + result_cols[i++]->insertFrom(*col, left_cursor->getRow()); for (const auto & col : asof_join_state.value.getColumns()) result_cols[i++]->insertFrom(*col, asof_join_state.value_row); chassert(i == result_cols.size()); @@ -707,7 +726,7 @@ std::optional MergeJoinAlgorithm::handleAsofJoinStat /// return row with default values at right side size_t i = 0; for (const auto & col : left_columns) - result_cols[i++]->insertFrom(*col, lpos); + result_cols[i++]->insertFrom(*col, left_cursor->getRow()); for (; i < result_cols.size(); ++i) result_cols[i]->insertDefault(); chassert(i == result_cols.size()); @@ -977,12 +996,10 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() auto lpos = left_cursor->getRow(); auto rpos = right_cursor->getRow(); auto cmp = compareCursors(*left_cursor, *right_cursor); - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ({}) <=> ({}) -> {}", __FILE__, __LINE__, left_cursor.dump(), right_cursor.dump(), cmp); if (cmp == 0) { auto asof_cmp = compareAsofCursors(left_cursor, right_cursor); - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ({}) <=> ({}) -> asof {}", __FILE__, __LINE__, left_cursor.dump(), right_cursor.dump(), asof_cmp); if ((asof_inequality == ASOFJoinInequality::Less && asof_cmp <= -1) || (asof_inequality == ASOFJoinInequality::LessOrEquals && asof_cmp <= 0)) @@ -1010,7 +1027,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() if ((asof_inequality == ASOFJoinInequality::Greater && asof_cmp >= 1) || (asof_inequality == ASOFJoinInequality::GreaterOrEquals && asof_cmp >= 0)) { - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); /// condition is satisfied, remember this row and move next to try to find better match asof_join_state.set(right_cursor, rpos); right_cursor->next(); @@ -1022,7 +1038,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() /// Asof condition is not satisfied anymore, use last matched row from right table if (asof_join_state.hasMatch(left_cursor, asof_inequality)) { - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); size_t i = 0; for (const auto & col : left_columns) result_cols[i++]->insertFrom(*col, lpos); @@ -1035,7 +1050,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() asof_join_state.reset(); if (isLeft(kind)) { - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); /// return row with default values at right side size_t i = 0; @@ -1047,7 +1061,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() } } left_cursor->next(); - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); continue; } @@ -1055,10 +1068,8 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() } else if (cmp < 0) { - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); if (asof_join_state.hasMatch(left_cursor, asof_inequality)) { - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); size_t i = 0; for (const auto & col : left_columns) @@ -1071,13 +1082,12 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() } else { - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); asof_join_state.reset(); } - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); /// no matches for rows in left table, just pass them through size_t num = nextDistinct(*left_cursor); + if (isLeft(kind) && num) { /// return them with default values at right side @@ -1091,7 +1101,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() } else { - // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__); /// skip rows in right table until we find match for current row in left table nextDistinct(*right_cursor); @@ -1106,6 +1115,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() /// otherwise - vice versa Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const { + ColumnRawPtrs cols; { const auto & columns_left = source_num == 0 ? cursors[0]->getCurrent().getColumns() : cursors[0]->sampleColumns(); @@ -1128,7 +1138,6 @@ Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num, size_t star cols.push_back(col.get()); } } - Chunk result_chunk; copyColumnsResized(cols, start, num_rows, result_chunk); return result_chunk; @@ -1144,6 +1153,7 @@ Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num) IMergingAlgorithm::Status MergeJoinAlgorithm::merge() { + if (!cursors[0]->cursor.isValid() && !cursors[0]->fullyCompleted()) return Status(0); @@ -1161,6 +1171,7 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() if (cursors[0]->fullyCompleted() || cursors[1]->fullyCompleted()) { + if (!cursors[0]->fullyCompleted() && isLeftOrFull(kind)) return Status(createBlockWithDefaults(0)); diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index dbdda0b166b..c8ba857781e 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -185,28 +185,7 @@ public: class FullMergeJoinCursor : boost::noncopyable { public: - FullMergeJoinCursor( - const Block & sample_block_, - const SortDescription & description_, - bool is_asof = false) - : sample_block(sample_block_.cloneEmpty()) - , desc(description_) - { - if (desc.size() == 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty sort description for FullMergeJoinCursor"); - - if (is_asof) - { - /// For ASOF join prefix of sort description is used for equality comparison - /// and the last column is used for inequality comparison and is handled separately - - auto asof_column_description = desc.back(); - desc.pop_back(); - - chassert(asof_column_description.direction == 1 && asof_column_description.nulls_direction == 1); - asof_column_position = sample_block.getPositionByName(asof_column_description.column_name); - } - } + FullMergeJoinCursor(const Block & sample_block_, const SortDescription & description_, bool is_asof = false); bool fullyCompleted() const; void setChunk(Chunk && chunk); diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 2bd3357eff9..e3423aa0386 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -294,18 +295,25 @@ bool isStrict(ASOFJoinInequality inequality) } -class FullSortingJoinRandomized : public ::testing::Test +class FullSortingJoinTest : public ::testing::Test { public: - FullSortingJoinRandomized() = default; + FullSortingJoinTest() = default; void SetUp() override { + Poco::AutoPtr channel(new Poco::ConsoleChannel(std::cerr)); + Poco::Logger::root().setChannel(channel); + if (const char * test_log_level = std::getenv("TEST_LOG_LEVEL")) // NOLINT(concurrency-mt-unsafe) + Poco::Logger::root().setLevel(test_log_level); + else + Poco::Logger::root().setLevel("none"); + + UInt64 seed = randomSeed(); if (const char * random_seed = std::getenv("TEST_RANDOM_SEED")) // NOLINT(concurrency-mt-unsafe) seed = std::stoull(random_seed); - - std::cerr << "TEST_RANDOM_SEED=" << seed << std::endl; + std::cout << "TEST_RANDOM_SEED=" << seed << std::endl; rng = pcg64(seed); } @@ -316,7 +324,7 @@ public: pcg64 rng; }; -TEST(FullSortingJoin, AllAnyOneKey) +TEST_F(FullSortingJoinTest, AllAnyOneKey) try { { @@ -392,7 +400,7 @@ catch (Exception & e) } -TEST_F(FullSortingJoinRandomized, Any) +TEST_F(FullSortingJoinTest, AnySimple) try { JoinKind kind = getRandomFrom(rng, {JoinKind::Inner, JoinKind::Left, JoinKind::Right}); @@ -478,7 +486,7 @@ catch (Exception & e) throw; } -TEST(FullSortingJoin, Asof) +TEST_F(FullSortingJoinTest, AsofSimple) try { SourceChunksBuilder left_source({ @@ -543,7 +551,7 @@ catch (Exception & e) } -TEST_F(FullSortingJoinRandomized, AsofOnlyColumn) +TEST_F(FullSortingJoinTest, AsofOnlyColumn) try { auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} }); @@ -587,7 +595,7 @@ catch (Exception & e) throw; } -TEST_F(FullSortingJoinRandomized, AsofLessGeneratedTestData) +TEST_F(FullSortingJoinTest, AsofLessGeneratedTestData) try { auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left }); @@ -676,7 +684,7 @@ catch (Exception & e) throw; } -TEST_F(FullSortingJoinRandomized, AsofGreaterGeneratedTestData) +TEST_F(FullSortingJoinTest, AsofGreaterGeneratedTestData) try { auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left }); @@ -708,13 +716,13 @@ try String k2 = ""; UInt64 left_t = 0; - auto key_num_total = std::uniform_int_distribution<>(1, 100)(rng); + auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); for (size_t key_num = 0; key_num < key_num_total; ++key_num) { generateNextKey(rng, k1, k2); /// generate some rows with smaller left_t to check that they are not matched - size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 10)(rng) : 0; + size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 100)(rng) : 0; for (size_t i = 0; i < num_left_rows; ++i) { left_t += std::uniform_int_distribution<>(1, 10)(rng); @@ -737,8 +745,8 @@ try } /// next left_t should be greater than (or equals) right_t to match with previous rows - left_t = right_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 1 : 0, 10)(rng); - size_t num_left_matches = std::uniform_int_distribution<>(1, 10)(rng); + left_t = right_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 1 : 0, 100)(rng); + size_t num_left_matches = std::uniform_int_distribution<>(1, 100)(rng); for (size_t j = 0; j < num_left_matches; ++j) { left_t += std::uniform_int_distribution<>(0, 3)(rng); From a8690947c705339113266c1ec8db05695a36e472 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 28 Sep 2023 12:50:24 +0000 Subject: [PATCH 13/45] enable stateless tessts for full sorting asof join --- src/Interpreters/InterpreterSelectQuery.cpp | 7 ++- src/Planner/PlannerJoinTree.cpp | 6 ++- .../Transforms/MergeJoinTransform.cpp | 2 - .../00927_asof_join_correct_bt.reference | 15 +++++++ .../00927_asof_join_correct_bt.sql | 30 ++++++++----- .../00927_asof_join_long.reference | 1 + .../0_stateless/00927_asof_join_long.sql | 17 ++++++- .../00927_asof_join_noninclusive.reference | 29 ++++++++++++ .../00927_asof_join_noninclusive.sql | 5 ++- .../00927_asof_join_other_types.reference | 45 +++++++++++++++++++ .../00927_asof_join_other_types.sh | 27 ----------- .../00927_asof_join_other_types.sql.j2 | 27 +++++++++++ .../0_stateless/00927_asof_joins.reference | 15 +++++++ .../queries/0_stateless/00927_asof_joins.sql | 8 +++- .../0_stateless/00976_asof_join_on.reference | 38 ++++++++++++++++ ..._join_on.sql => 00976_asof_join_on.sql.j2} | 8 ++++ 16 files changed, 231 insertions(+), 49 deletions(-) delete mode 100755 tests/queries/0_stateless/00927_asof_join_other_types.sh create mode 100755 tests/queries/0_stateless/00927_asof_join_other_types.sql.j2 rename tests/queries/0_stateless/{00976_asof_join_on.sql => 00976_asof_join_on.sql.j2} (90%) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index b72399df2c1..a0b53ad42d1 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1726,7 +1726,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

getCurrentDataStream().header, join_clause.key_names_right); - if (settings.max_rows_in_set_to_optimize_join > 0 && kind_allows_filtering && has_non_const_keys) + if (settings.max_rows_in_set_to_optimize_join > 0 && join_type_allows_filtering && has_non_const_keys) { auto * left_set = add_create_set(query_plan, join_clause.key_names_left, JoinTableSide::Left); auto * right_set = add_create_set(*joined_plan, join_clause.key_names_right, JoinTableSide::Right); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index aa46f65d2d4..63acf194139 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1484,7 +1484,9 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ { const auto & join_clause = table_join->getOnlyClause(); - bool kind_allows_filtering = isInner(join_kind) || isLeft(join_kind) || isRight(join_kind); + bool join_type_allows_filtering = (join_strictness == JoinStrictness::All || join_strictness == JoinStrictness::Any) + && (isInner(join_kind) || isLeft(join_kind) || isRight(join_kind)); + auto has_non_const = [](const Block & block, const auto & keys) { @@ -1504,7 +1506,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ bool has_non_const_keys = has_non_const(left_plan.getCurrentDataStream().header, join_clause.key_names_left) && has_non_const(right_plan.getCurrentDataStream().header, join_clause.key_names_right); - if (settings.max_rows_in_set_to_optimize_join > 0 && kind_allows_filtering && has_non_const_keys) + if (settings.max_rows_in_set_to_optimize_join > 0 && join_type_allows_filtering && has_non_const_keys) { auto * left_set = add_create_set(left_plan, join_clause.key_names_left, JoinTableSide::Left); auto * right_set = add_create_set(right_plan, join_clause.key_names_right, JoinTableSide::Right); diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 0d9eab248d8..df56ffc2871 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -456,8 +456,6 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( size_t left_idx = input_headers[0].getPositionByName(left_key); size_t right_idx = input_headers[1].getPositionByName(right_key); left_to_right_key_remap[left_idx] = right_idx; - if (strictness == JoinStrictness::Asof) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ASOF joins USING"); } const auto *smjPtr = typeid_cast(table_join.get()); diff --git a/tests/queries/0_stateless/00927_asof_join_correct_bt.reference b/tests/queries/0_stateless/00927_asof_join_correct_bt.reference index bb199d0159a..a398f9604fd 100644 --- a/tests/queries/0_stateless/00927_asof_join_correct_bt.reference +++ b/tests/queries/0_stateless/00927_asof_join_correct_bt.reference @@ -13,3 +13,18 @@ 1 103 3 2 102 1 1 104 4 4 104 1 1 105 5 4 104 1 +1 101 1 0 0 0 +1 102 2 2 102 1 +1 103 3 2 102 1 +1 104 4 4 104 1 +1 105 5 4 104 1 +1 101 1 0 0 0 +1 102 2 2 102 1 +1 103 3 2 102 1 +1 104 4 4 104 1 +1 105 5 4 104 1 +1 101 1 0 0 0 +1 102 2 2 102 1 +1 103 3 2 102 1 +1 104 4 4 104 1 +1 105 5 4 104 1 diff --git a/tests/queries/0_stateless/00927_asof_join_correct_bt.sql b/tests/queries/0_stateless/00927_asof_join_correct_bt.sql index 281a81d51c0..761d6bacde6 100644 --- a/tests/queries/0_stateless/00927_asof_join_correct_bt.sql +++ b/tests/queries/0_stateless/00927_asof_join_correct_bt.sql @@ -4,20 +4,26 @@ DROP TABLE IF EXISTS B; CREATE TABLE A(k UInt32, t UInt32, a UInt64) ENGINE = MergeTree() ORDER BY (k, t); INSERT INTO A(k,t,a) VALUES (1,101,1),(1,102,2),(1,103,3),(1,104,4),(1,105,5); -CREATE TABLE B(k UInt32, t UInt32, b UInt64) ENGINE = MergeTree() ORDER BY (k, t); -INSERT INTO B(k,t,b) VALUES (1,102,2), (1,104,4); -SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t); -DROP TABLE B; +CREATE TABLE B1(k UInt32, t UInt32, b UInt64) ENGINE = MergeTree() ORDER BY (k, t); +INSERT INTO B1(k,t,b) VALUES (1,102,2), (1,104,4); +CREATE TABLE B2(t UInt32, k UInt32, b UInt64) ENGINE = MergeTree() ORDER BY (k, t); +INSERT INTO B2(k,t,b) VALUES (1,102,2), (1,104,4); -CREATE TABLE B(t UInt32, k UInt32, b UInt64) ENGINE = MergeTree() ORDER BY (k, t); -INSERT INTO B(k,t,b) VALUES (1,102,2), (1,104,4); -SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t); -DROP TABLE B; +CREATE TABLE B3(k UInt32, b UInt64, t UInt32) ENGINE = MergeTree() ORDER BY (k, t); +INSERT INTO B3(k,t,b) VALUES (1,102,2), (1,104,4); -CREATE TABLE B(k UInt32, b UInt64, t UInt32) ENGINE = MergeTree() ORDER BY (k, t); -INSERT INTO B(k,t,b) VALUES (1,102,2), (1,104,4); -SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t); -DROP TABLE B; +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t); +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t); +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t); + +SET join_algorithm = 'full_sorting_merge'; +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t); +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t); +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t); + +DROP TABLE B1; +DROP TABLE B2; +DROP TABLE B3; DROP TABLE A; diff --git a/tests/queries/0_stateless/00927_asof_join_long.reference b/tests/queries/0_stateless/00927_asof_join_long.reference index d4f015c68e4..ec40d2bc463 100644 --- a/tests/queries/0_stateless/00927_asof_join_long.reference +++ b/tests/queries/0_stateless/00927_asof_join_long.reference @@ -1 +1,2 @@ 3000000 +3000000 diff --git a/tests/queries/0_stateless/00927_asof_join_long.sql b/tests/queries/0_stateless/00927_asof_join_long.sql index c03a06d48d4..7a73875e93e 100644 --- a/tests/queries/0_stateless/00927_asof_join_long.sql +++ b/tests/queries/0_stateless/00927_asof_join_long.sql @@ -2,15 +2,28 @@ DROP TABLE IF EXISTS tvs; +-- to use different algorithms for in subquery +SET allow_experimental_analyzer = 1; + CREATE TABLE tvs(k UInt32, t UInt32, tv UInt64) ENGINE = Memory; INSERT INTO tvs(k,t,tv) SELECT k, t, t FROM (SELECT toUInt32(number) AS k FROM numbers(1000)) keys -CROSS JOIN (SELECT toUInt32(number * 3) as t FROM numbers(10000)) tv_times; +CROSS JOIN (SELECT toUInt32(number * 3) as t FROM numbers(10000)) tv_times +SETTINGS join_algorithm = 'hash'; SELECT SUM(trades.price - tvs.tv) FROM (SELECT k, t, t as price FROM (SELECT toUInt32(number) AS k FROM numbers(1000)) keys - CROSS JOIN (SELECT toUInt32(number * 10) AS t FROM numbers(3000)) trade_times) trades + CROSS JOIN (SELECT toUInt32(number * 10) AS t FROM numbers(3000)) trade_times + SETTINGS join_algorithm = 'hash') trades ASOF LEFT JOIN tvs USING(k,t); +SELECT SUM(trades.price - tvs.tv) FROM +(SELECT k, t, t as price + FROM (SELECT toUInt32(number) AS k FROM numbers(1000)) keys + CROSS JOIN (SELECT toUInt32(number * 10) AS t FROM numbers(3000)) trade_times + SETTINGS join_algorithm = 'hash') trades +ASOF LEFT JOIN tvs USING(k,t) +SETTINGS join_algorithm = 'full_sorting_merge'; + DROP TABLE tvs; diff --git a/tests/queries/0_stateless/00927_asof_join_noninclusive.reference b/tests/queries/0_stateless/00927_asof_join_noninclusive.reference index fe2844a2a43..d856372fb4a 100644 --- a/tests/queries/0_stateless/00927_asof_join_noninclusive.reference +++ b/tests/queries/0_stateless/00927_asof_join_noninclusive.reference @@ -27,3 +27,32 @@ 2 1970-01-01 00:00:03 3 3 1970-01-01 00:00:03 2 2 1970-01-01 00:00:04 4 3 1970-01-01 00:00:03 2 2 1970-01-01 00:00:05 5 3 1970-01-01 00:00:03 2 +1 1970-01-01 00:00:01 1 0 1970-01-01 00:00:00 0 +1 1970-01-01 00:00:02 2 2 1970-01-01 00:00:02 1 +1 1970-01-01 00:00:03 3 2 1970-01-01 00:00:02 1 +1 1970-01-01 00:00:04 4 4 1970-01-01 00:00:04 1 +1 1970-01-01 00:00:05 5 4 1970-01-01 00:00:04 1 +2 1970-01-01 00:00:01 1 0 1970-01-01 00:00:00 0 +2 1970-01-01 00:00:02 2 0 1970-01-01 00:00:00 0 +2 1970-01-01 00:00:03 3 3 1970-01-01 00:00:03 2 +2 1970-01-01 00:00:04 4 3 1970-01-01 00:00:03 2 +2 1970-01-01 00:00:05 5 3 1970-01-01 00:00:03 2 +3 1970-01-01 00:00:01 1 0 1970-01-01 00:00:00 0 +3 1970-01-01 00:00:02 2 0 1970-01-01 00:00:00 0 +3 1970-01-01 00:00:03 3 0 1970-01-01 00:00:00 0 +3 1970-01-01 00:00:04 4 0 1970-01-01 00:00:00 0 +3 1970-01-01 00:00:05 5 0 1970-01-01 00:00:00 0 +1 1970-01-01 00:00:02 2 2 1970-01-01 00:00:02 1 +1 1970-01-01 00:00:03 3 2 1970-01-01 00:00:02 1 +1 1970-01-01 00:00:04 4 4 1970-01-01 00:00:04 1 +1 1970-01-01 00:00:05 5 4 1970-01-01 00:00:04 1 +2 1970-01-01 00:00:03 3 3 1970-01-01 00:00:03 2 +2 1970-01-01 00:00:04 4 3 1970-01-01 00:00:03 2 +2 1970-01-01 00:00:05 5 3 1970-01-01 00:00:03 2 +1 1970-01-01 00:00:02 2 2 1970-01-01 00:00:02 1 +1 1970-01-01 00:00:03 3 2 1970-01-01 00:00:02 1 +1 1970-01-01 00:00:04 4 4 1970-01-01 00:00:04 1 +1 1970-01-01 00:00:05 5 4 1970-01-01 00:00:04 1 +2 1970-01-01 00:00:03 3 3 1970-01-01 00:00:03 2 +2 1970-01-01 00:00:04 4 3 1970-01-01 00:00:03 2 +2 1970-01-01 00:00:05 5 3 1970-01-01 00:00:03 2 diff --git a/tests/queries/0_stateless/00927_asof_join_noninclusive.sql b/tests/queries/0_stateless/00927_asof_join_noninclusive.sql index 5f15f3b593d..3cc99df4462 100644 --- a/tests/queries/0_stateless/00927_asof_join_noninclusive.sql +++ b/tests/queries/0_stateless/00927_asof_join_noninclusive.sql @@ -11,9 +11,12 @@ INSERT INTO B(k,t,b) VALUES (1,2,2),(1,4,4); INSERT INTO B(k,t,b) VALUES (2,3,3); SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t); - SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF INNER JOIN B ON A.k == B.k AND A.t >= B.t ORDER BY (A.k, A.t); +SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF JOIN B USING(k,t) ORDER BY (A.k, A.t); +SET join_algorithm = 'full_sorting_merge'; +SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t); +SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF INNER JOIN B ON A.k == B.k AND A.t >= B.t ORDER BY (A.k, A.t); SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF JOIN B USING(k,t) ORDER BY (A.k, A.t); DROP TABLE A; diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.reference b/tests/queries/0_stateless/00927_asof_join_other_types.reference index 80c85ec1ae3..ddbc24ff925 100644 --- a/tests/queries/0_stateless/00927_asof_join_other_types.reference +++ b/tests/queries/0_stateless/00927_asof_join_other_types.reference @@ -1,27 +1,72 @@ +- 2 1 1 0 2 3 3 3 2 5 5 3 +- 2 1 1 0 2 3 3 3 2 5 5 3 +- 2 1 1 0 2 3 3 3 2 5 5 3 +- 2 1 1 0 2 3 3 3 2 5 5 3 +- +2 1 1 0 +2 3 3 3 +2 5 5 3 +- +2 1 1 0 +2 3 3 3 +2 5 5 3 +- +2 1 1 0 +2 3 3 3 +2 5 5 3 +- +2 1 1 0 +2 3 3 3 +2 5 5 3 +- 2 1970-01-01 02:00:01 1 0 2 1970-01-01 02:00:03 3 3 2 1970-01-01 02:00:05 5 3 +- +2 1970-01-01 02:00:01 1 0 +2 1970-01-01 02:00:03 3 3 +2 1970-01-01 02:00:05 5 3 +- 2 1 1 0 2 3 3 3 2 5 5 3 +- 2 1 1 0 2 3 3 3 2 5 5 3 +- 2 1 1 0 2 3 3 3 2 5 5 3 +- +2 1 1 0 +2 3 3 3 +2 5 5 3 +- +2 1 1 0 +2 3 3 3 +2 5 5 3 +- +2 1 1 0 +2 3 3 3 +2 5 5 3 +- +2 1970-01-01 02:00:00.001 1 0 +2 1970-01-01 02:00:00.003 3 3 +2 1970-01-01 02:00:00.005 5 3 +- 2 1970-01-01 02:00:00.001 1 0 2 1970-01-01 02:00:00.003 3 3 2 1970-01-01 02:00:00.005 5 3 diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.sh b/tests/queries/0_stateless/00927_asof_join_other_types.sh deleted file mode 100755 index 10173a3e43f..00000000000 --- a/tests/queries/0_stateless/00927_asof_join_other_types.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -for typename in "UInt32" "UInt64" "Float64" "Float32" "DateTime('Asia/Istanbul')" "Decimal32(5)" "Decimal64(5)" "Decimal128(5)" "DateTime64(3, 'Asia/Istanbul')" -do - $CLICKHOUSE_CLIENT -mn <= B.t ORDER BY (A.a, A.t); SELECT count() FROM A ASOF LEFT JOIN B ON A.a == B.b AND B.t <= A.t; SELECT A.a, A.t, B.b, B.t FROM A ASOF INNER JOIN B ON B.t <= A.t AND A.a == B.b ORDER BY (A.a, A.t); @@ -28,5 +34,7 @@ ASOF INNER JOIN (SELECT * FROM B UNION ALL SELECT 1, 3) AS B ON B.t <= A.t AND A WHERE B.t != 3 ORDER BY (A.a, A.t) ; +{% endfor %} + DROP TABLE A; DROP TABLE B; From 256ad60115ddd1eb8c8a5597478e057926fe30f6 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 28 Sep 2023 12:52:20 +0000 Subject: [PATCH 14/45] fix style, clang tidy --- src/Planner/PlannerJoinTree.cpp | 8 +-- .../Transforms/MergeJoinTransform.cpp | 59 +++++++++------- .../Transforms/MergeJoinTransform.h | 2 +- .../tests/gtest_full_sorting_join.cpp | 27 +------- .../00927_asof_join_other_types.sql.j2 | 0 .../01116_asof_join_dolbyzerr.reference | 3 + .../0_stateless/01116_asof_join_dolbyzerr.sql | 14 ++++ .../02240_asof_join_biginteger.reference | 4 ++ .../02240_asof_join_biginteger.sql | 8 +++ ...n_in_left_table_clause_asof_join.reference | 1 + ...unction_in_left_table_clause_asof_join.sql | 12 ++++ .../03143_asof_join_ddb_long.reference | 2 + .../0_stateless/03143_asof_join_ddb_long.sql | 48 +++++++++++++ .../03144_asof_join_ddb_doubles.reference | 58 ++++++++++++++++ .../03144_asof_join_ddb_doubles.sql | 64 +++++++++++++++++ ...03145_asof_join_ddb_inequalities.reference | 68 +++++++++++++++++++ .../03145_asof_join_ddb_inequalities.sql | 63 +++++++++++++++++ .../03146_asof_join_ddb_merge_long.reference | 2 + .../03146_asof_join_ddb_merge_long.sql.j2 | 37 ++++++++++ 19 files changed, 424 insertions(+), 56 deletions(-) mode change 100755 => 100644 tests/queries/0_stateless/00927_asof_join_other_types.sql.j2 create mode 100644 tests/queries/0_stateless/03143_asof_join_ddb_long.reference create mode 100644 tests/queries/0_stateless/03143_asof_join_ddb_long.sql create mode 100644 tests/queries/0_stateless/03144_asof_join_ddb_doubles.reference create mode 100644 tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql create mode 100644 tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference create mode 100644 tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql create mode 100644 tests/queries/0_stateless/03146_asof_join_ddb_merge_long.reference create mode 100644 tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 63acf194139..a9d9e11f458 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -75,7 +75,6 @@ namespace ErrorCodes extern const int INVALID_JOIN_ON_EXPRESSION; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; - extern const int SYNTAX_ERROR; extern const int ACCESS_DENIED; extern const int PARAMETER_OUT_OF_BOUND; extern const int TOO_MANY_COLUMNS; @@ -1357,12 +1356,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ { if (!join_clause.hasASOF()) throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION, - "JOIN {} no inequality in ASOF JOIN ON section.", - join_node.formatASTForErrorMessage()); - - if (table_join_clause.key_names_left.size() <= 1) - throw Exception(ErrorCodes::SYNTAX_ERROR, - "JOIN {} ASOF join needs at least one equi-join column", + "JOIN {} no inequality in ASOF JOIN ON section", join_node.formatASTForErrorMessage()); } diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index df56ffc2871..26e1ebb0b60 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -43,6 +43,13 @@ FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns, return std::make_unique(materializeBlock(block), desc, strictness == JoinStrictness::Asof); } +bool isNullAt(const IColumn & column, size_t row) +{ + if (const auto * nullable = checkAndGetColumn(column)) + return nullable->isNullAt(row); + return false; +} + template int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, size_t lhs_pos, size_t rhs_pos, int null_direction_hint) { @@ -54,7 +61,7 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, if (left_nullable && right_nullable) { int res = left_nullable->compareAt(lhs_pos, rhs_pos, right_column, null_direction_hint); - if (res) + if (res != 0) return res; /// NULL != NULL case @@ -108,9 +115,9 @@ int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImp return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow(), lhs.sort_columns_size, null_direction_hint); } -int compareAsofCursors(const FullMergeJoinCursor & lhs, const FullMergeJoinCursor & rhs) +int compareAsofCursors(const FullMergeJoinCursor & lhs, const FullMergeJoinCursor & rhs, int null_direction_hint) { - return nullableCompareAt(*lhs.getAsofColumn(), *rhs.getAsofColumn(), lhs->getRow(), rhs->getRow()); + return nullableCompareAt(*lhs.getAsofColumn(), *rhs.getAsofColumn(), lhs->getRow(), rhs->getRow(), null_direction_hint); } bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint) @@ -235,12 +242,6 @@ void inline addRange(PaddedPODArray & values, UInt64 start, UInt64 end) values.push_back(i); } -void inline addMany(PaddedPODArray & left_or_right_map, size_t idx, size_t num) -{ - for (size_t i = 0; i < num; ++i) - left_or_right_map.push_back(idx); -} - void inline addMany(PaddedPODArray & values, UInt64 value, size_t num) { values.resize_fill(values.size() + num, value); @@ -257,7 +258,7 @@ JoinKeyRow::JoinKeyRow(const FullMergeJoinCursor & cursor, size_t pos) new_col->insertFrom(*col, pos); row.push_back(std::move(new_col)); } - if (auto asof_column = cursor.getAsofColumn()) + if (const auto * asof_column = cursor.getAsofColumn()) { auto new_col = asof_column->cloneEmpty(); new_col->insertFrom(*asof_column, pos); @@ -275,10 +276,10 @@ bool JoinKeyRow::equals(const FullMergeJoinCursor & cursor) const if (row.empty()) return false; - assert(this->row.size() == cursor->sort_columns_size); for (size_t i = 0; i < cursor->sort_columns_size; ++i) { - int cmp = this->row[i]->compareAt(0, cursor->getRow(), *(cursor->sort_columns[i]), cursor->desc[i].nulls_direction); + // int cmp = this->row[i]->compareAt(0, cursor->getRow(), *(cursor->sort_columns[i]), cursor->desc[i].nulls_direction); + int cmp = nullableCompareAt(*this->row[i], *cursor->sort_columns[i], 0, cursor->getRow(), cursor->desc[i].nulls_direction); if (cmp != 0) return false; } @@ -287,9 +288,16 @@ bool JoinKeyRow::equals(const FullMergeJoinCursor & cursor) const bool JoinKeyRow::asofMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) const { + chassert(this->row.size() == cursor->sort_columns_size + 1); if (!equals(cursor)) return false; - int cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *row.back(), 1); + + const auto & asof_row = row.back(); + if (isNullAt(*asof_row, 0) || isNullAt(*cursor.getAsofColumn(), cursor->getRow())) + return false; + + int cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *asof_row, 1); + return (asof_inequality == ASOFJoinInequality::Less && cmp < 0) || (asof_inequality == ASOFJoinInequality::LessOrEquals && cmp <= 0) || (asof_inequality == ASOFJoinInequality::Greater && cmp > 0) @@ -333,7 +341,7 @@ FullMergeJoinCursor::FullMergeJoinCursor(const Block & sample_block_, const Sort : sample_block(sample_block_.cloneEmpty()) , desc(description_) { - if (desc.size() == 0) + if (desc.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty sort description for FullMergeJoinCursor"); if (is_asof) @@ -393,7 +401,7 @@ String FullMergeJoinCursor::dump() const row_dump.push_back(val.dump()); } - if (auto * asof_column = getAsofColumn()) + if (const auto * asof_column = getAsofColumn()) { asof_column->get(cursor.getRow(), val); row_dump.push_back(val.dump()); @@ -436,9 +444,10 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ON filter conditions"); cursors = { - createCursor(input_headers[0], on_clause_.key_names_left, strictness), - createCursor(input_headers[1], on_clause_.key_names_right, strictness), + createCursor(input_headers[0], on_clause_.key_names_left, strictness), + createCursor(input_headers[1], on_clause_.key_names_right, strictness), }; +} MergeJoinAlgorithm::MergeJoinAlgorithm( JoinPtr join_ptr, @@ -458,7 +467,7 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( left_to_right_key_remap[left_idx] = right_idx; } - const auto *smjPtr = typeid_cast(table_join.get()); + const auto *smjPtr = typeid_cast(join_ptr.get()); if (smjPtr) { null_direction_hint = smjPtr->getNullDirection(); @@ -993,11 +1002,18 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() { auto lpos = left_cursor->getRow(); auto rpos = right_cursor->getRow(); - auto cmp = compareCursors(*left_cursor, *right_cursor); + auto cmp = compareCursors(*left_cursor, *right_cursor, null_direction_hint); + if (cmp == 0) + { + if (isNullAt(*left_cursor.getAsofColumn(), lpos)) + cmp = -1; + if (isNullAt(*right_cursor.getAsofColumn(), rpos)) + cmp = 1; + } if (cmp == 0) { - auto asof_cmp = compareAsofCursors(left_cursor, right_cursor); + auto asof_cmp = compareAsofCursors(left_cursor, right_cursor, null_direction_hint); if ((asof_inequality == ASOFJoinInequality::Less && asof_cmp <= -1) || (asof_inequality == ASOFJoinInequality::LessOrEquals && asof_cmp <= 0)) @@ -1048,7 +1064,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() asof_join_state.reset(); if (isLeft(kind)) { - /// return row with default values at right side size_t i = 0; for (const auto & col : left_columns) @@ -1068,7 +1083,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() { if (asof_join_state.hasMatch(left_cursor, asof_inequality)) { - size_t i = 0; for (const auto & col : left_columns) result_cols[i++]->insertFrom(*col, lpos); @@ -1099,7 +1113,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin() } else { - /// skip rows in right table until we find match for current row in left table nextDistinct(*right_cursor); } diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index c8ba857781e..15bf13381b8 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -167,7 +167,7 @@ public: void set(const FullMergeJoinCursor & rcursor, size_t rpos); void reset(); - bool hasMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) + bool hasMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) const { if (value.empty()) return false; diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index e3423aa0386..a3fda006eb8 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -30,31 +32,6 @@ using namespace DB; namespace { -[[ maybe_unused ]] -String dumpBlockSource(std::shared_ptr source, bool mono_block = false) -{ - WriteBufferFromOwnString buf; - { - Block header = source->getPort().getHeader(); - QueryPipeline pipeline(source); - auto format = std::make_shared(buf, header, FormatSettings{}, mono_block); - pipeline.complete(std::move(format)); - - CompletedPipelineExecutor executor(pipeline); - executor.execute(); - } - return buf.str(); -} - -[[ maybe_unused ]] -String dumpBlock(const Block & block) -{ - Block header = block.cloneEmpty(); - Chunk data(block.getColumns(), block.rows()); - auto source = std::make_shared(header, std::move(data)); - return dumpBlockSource(std::move(source)); -} - QueryPipeline buildJoinPipeline( std::shared_ptr left_source, std::shared_ptr right_source, diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.sql.j2 b/tests/queries/0_stateless/00927_asof_join_other_types.sql.j2 old mode 100755 new mode 100644 diff --git a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference index 1055a67ea5b..0aa1a85f19d 100644 --- a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference +++ b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference @@ -1,3 +1,6 @@ v1 o1 ['s2','s1'] v1 o2 ['s4'] v2 o3 ['s5','s3'] +v1 o1 ['s2','s1'] +v1 o2 ['s4'] +v2 o3 ['s5','s3'] diff --git a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql index 8a94b6ddd24..652cb35cf2a 100644 --- a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql +++ b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql @@ -16,3 +16,17 @@ GROUP BY ORDER BY visitorId ASC, orderId ASC; + +SELECT + visitorId, + orderId, + groupUniqArray(sessionId) +FROM sessions +ASOF INNER JOIN orders ON (sessions.visitorId = orders.visitorId) AND (sessions.date <= orders.date) +GROUP BY + visitorId, + orderId +ORDER BY + visitorId ASC, + orderId ASC +SETTINGS join_algorithm = 'full_sorting_merge'; diff --git a/tests/queries/0_stateless/02240_asof_join_biginteger.reference b/tests/queries/0_stateless/02240_asof_join_biginteger.reference index cac55eec430..f7eb4d74375 100644 --- a/tests/queries/0_stateless/02240_asof_join_biginteger.reference +++ b/tests/queries/0_stateless/02240_asof_join_biginteger.reference @@ -2,3 +2,7 @@ 0 340282366920938463463374607431768211457 0 18446744073709551617 0 340282366920938463463374607431768211457 +0 18446744073709551617 +0 340282366920938463463374607431768211457 +0 18446744073709551617 +0 340282366920938463463374607431768211457 diff --git a/tests/queries/0_stateless/02240_asof_join_biginteger.sql b/tests/queries/0_stateless/02240_asof_join_biginteger.sql index 6dc5b00f116..a5c1faae4ea 100644 --- a/tests/queries/0_stateless/02240_asof_join_biginteger.sql +++ b/tests/queries/0_stateless/02240_asof_join_biginteger.sql @@ -3,3 +3,11 @@ select * from (select 0 as k, toInt256('340282366920938463463374607431768211457' select * from (select 0 as k, toUInt128('18446744073709551617') as v) t1 asof join (select 0 as k, toUInt128('18446744073709551616') as v) t2 using(k, v); select * from (select 0 as k, toUInt256('340282366920938463463374607431768211457') as v) t1 asof join (select 0 as k, toUInt256('340282366920938463463374607431768211456') as v) t2 using(k, v); + +SET join_algorithm = 'full_sorting_merge'; + +select * from (select 0 as k, toInt128('18446744073709551617') as v) t1 asof join (select 0 as k, toInt128('18446744073709551616') as v) t2 using(k, v); +select * from (select 0 as k, toInt256('340282366920938463463374607431768211457') as v) t1 asof join (select 0 as k, toInt256('340282366920938463463374607431768211456') as v) t2 using(k, v); + +select * from (select 0 as k, toUInt128('18446744073709551617') as v) t1 asof join (select 0 as k, toUInt128('18446744073709551616') as v) t2 using(k, v); +select * from (select 0 as k, toUInt256('340282366920938463463374607431768211457') as v) t1 asof join (select 0 as k, toUInt256('340282366920938463463374607431768211456') as v) t2 using(k, v); diff --git a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference index d00491fd7e5..6ed281c757a 100644 --- a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference +++ b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference @@ -1 +1,2 @@ 1 +1 diff --git a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql index 13dfb5debe7..6aa70a379c1 100644 --- a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql +++ b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql @@ -6,3 +6,15 @@ ASOF LEFT JOIN ( select 1 as session_id, 4 as id ) as visitors ON visitors.session_id <= sessions.id AND arrayFirst(a -> a, arrayMap((a) -> a, sessions.arr)) = visitors.id +; + +select count(*) +from ( + select 1 as id, [1, 2, 3] as arr +) as sessions +ASOF LEFT JOIN ( + select 1 as session_id, 4 as id +) as visitors +ON visitors.session_id <= sessions.id AND arrayFirst(a -> a, arrayMap((a) -> a, sessions.arr)) = visitors.id +SETTINGS join_algorithm = 'full_sorting_merge' +; diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.reference b/tests/queries/0_stateless/03143_asof_join_ddb_long.reference new file mode 100644 index 00000000000..2850a8aba98 --- /dev/null +++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.reference @@ -0,0 +1,2 @@ +49999983751397 10000032 +49999983751397 10000032 diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql new file mode 100644 index 00000000000..c421702bb00 --- /dev/null +++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql @@ -0,0 +1,48 @@ +-- Tags: long + +DROP TABLE IF EXISTS build; +DROP TABLE IF EXISTS skewed_probe; + +CREATE TABLE build ENGINE = MergeTree ORDER BY (key, begin) +AS + SELECT + toDateTime('1990-03-21 13:00:00') + INTERVAL number MINUTE AS begin, + number % 4 AS key, + number AS value + FROM numbers(0, 10000000); + +CREATE TABLE skewed_probe ENGINE = MergeTree ORDER BY (key, begin) +AS + SELECT + toDateTime('1990-04-21 13:00:01') + INTERVAL number MINUTE AS begin, + 0 AS key + FROM numbers(0, 5) + UNION ALL + SELECT + toDateTime('1990-05-21 13:00:01') + INTERVAL number MINUTE AS begin, + 1 AS key + FROM numbers(0, 10) + UNION ALL + SELECT + toDateTime('1990-06-21 13:00:01') + INTERVAL number MINUTE AS begin, + 2 AS key + FROM numbers(0, 20) + UNION ALL + SELECT + toDateTime('1990-03-21 13:00:01') + INTERVAL number MINUTE AS begin, + 3 AS key + FROM numbers(0, 10000000); + + +SELECT SUM(value), COUNT(*) +FROM skewed_probe +ASOF JOIN build +USING (key, begin) +; + +SELECT SUM(value), COUNT(*) +FROM skewed_probe +ASOF JOIN build +USING (key, begin) +SETTINGS join_algorithm = 'full_sorting_merge' +; diff --git a/tests/queries/0_stateless/03144_asof_join_ddb_doubles.reference b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.reference new file mode 100644 index 00000000000..f130f0a3f3b --- /dev/null +++ b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.reference @@ -0,0 +1,58 @@ +1 0 +2 0 +3 1 +4 1 +5 1 +6 2 +7 2 +8 3 +9 3 +0 0 +1 0 +2 0 +3 1 +4 1 +5 1 +6 2 +7 2 +8 3 +9 3 +1 1 0 +1 2 0 +1 3 1 +1 4 1 +1 5 1 +1 6 2 +1 7 2 +1 8 3 +1 9 3 +2 0 10 +2 1 10 +2 2 10 +2 3 10 +2 4 10 +2 5 10 +2 6 10 +2 7 20 +2 8 20 +2 9 20 +1 0 0 +1 1 0 +1 2 0 +1 3 1 +1 4 1 +1 5 1 +1 6 2 +1 7 2 +1 8 3 +1 9 3 +2 0 10 +2 1 10 +2 2 10 +2 3 10 +2 4 10 +2 5 10 +2 6 10 +2 7 20 +2 8 20 +2 9 20 diff --git a/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql new file mode 100644 index 00000000000..ef16ced3082 --- /dev/null +++ b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql @@ -0,0 +1,64 @@ +SET join_algorithm = 'full_sorting_merge'; + +DROP TABLE IF EXISTS events0; + +CREATE TABLE events0 ( + begin Float64, + value Int32 +) ENGINE = MergeTree ORDER BY begin; + +INSERT INTO events0 VALUES (1.0, 0), (3.0, 1), (6.0, 2), (8.0, 3); + +SELECT p.ts, e.value +FROM + (SELECT number :: Float64 AS ts FROM numbers(10)) p +ASOF JOIN events0 e +ON p.ts >= e.begin +ORDER BY p.ts ASC; + +SELECT p.ts, e.value +FROM + (SELECT number :: Float64 AS ts FROM numbers(10)) p +ASOF LEFT JOIN events0 e +ON p.ts >= e.begin +ORDER BY p.ts ASC +-- SETTINGS join_use_nulls = 1 +; + +DROP TABLE IF EXISTS events0; + +DROP TABLE IF EXISTS events; +DROP TABLE IF EXISTS probes; + +CREATE TABLE events ( + key Int32, + begin Float64, + value Int32 +) ENGINE = MergeTree ORDER BY (key, begin); + +INSERT INTO events VALUES (1, 1.0, 0), (1, 3.0, 1), (1, 6.0, 2), (1, 8.0, 3), (2, 0.0, 10), (2, 7.0, 20), (2, 11.0, 30); + +CREATE TABLE probes ( + key Int32, + ts Float64 +) ENGINE = MergeTree ORDER BY (key, ts) AS +SELECT + key.number, + ts.number +FROM + numbers(1, 2) as key, + numbers(10) as ts +SETTINGS join_algorithm = 'hash'; + +SELECT p.key, p.ts, e.value +FROM probes p +ASOF JOIN events e +ON p.key = e.key AND p.ts >= e.begin +ORDER BY p.key, p.ts ASC; + +SELECT p.key, p.ts, e.value +FROM probes p +ASOF LEFT JOIN events e +ON p.key = e.key AND p.ts >= e.begin +ORDER BY p.key, p.ts ASC NULLS FIRST; + diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference new file mode 100644 index 00000000000..73c4f7dfe25 --- /dev/null +++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference @@ -0,0 +1,68 @@ +2023-03-21 12:00:00 1970-01-01 00:00:00 -1 +2023-03-21 13:00:00 1970-01-01 00:00:00 -1 +2023-03-21 14:00:00 2023-03-21 13:00:00 0 +2023-03-21 15:00:00 2023-03-21 14:00:00 1 +2023-03-21 16:00:00 2023-03-21 15:00:00 2 +2023-03-21 17:00:00 2023-03-21 16:00:00 3 +2023-03-21 18:00:00 2023-03-21 16:00:00 3 +2023-03-21 19:00:00 2023-03-21 16:00:00 3 +2023-03-21 20:00:00 2023-03-21 16:00:00 3 +2023-03-21 21:00:00 2023-03-21 16:00:00 3 +2027-10-18 11:03:27 2023-03-21 16:00:00 3 +2023-03-21 12:00:00 1970-01-01 00:00:00 -1 +2023-03-21 13:00:00 1970-01-01 00:00:00 -1 +2023-03-21 14:00:00 2023-03-21 13:00:00 0 +2023-03-21 15:00:00 2023-03-21 14:00:00 1 +2023-03-21 16:00:00 2023-03-21 15:00:00 2 +2023-03-21 17:00:00 2023-03-21 16:00:00 3 +2023-03-21 18:00:00 2023-03-21 16:00:00 3 +2023-03-21 19:00:00 2023-03-21 16:00:00 3 +2023-03-21 20:00:00 2023-03-21 16:00:00 3 +2023-03-21 21:00:00 2023-03-21 16:00:00 3 +2027-10-18 11:03:27 2023-03-21 16:00:00 3 +\N \N \N +2023-03-21 12:00:00 2023-03-21 13:00:00 0 +2023-03-21 13:00:00 2023-03-21 13:00:00 0 +2023-03-21 14:00:00 2023-03-21 14:00:00 1 +2023-03-21 15:00:00 2023-03-21 15:00:00 2 +2023-03-21 16:00:00 2023-03-21 16:00:00 3 +2023-03-21 17:00:00 2027-10-18 11:03:27 9 +2023-03-21 18:00:00 2027-10-18 11:03:27 9 +2023-03-21 19:00:00 2027-10-18 11:03:27 9 +2023-03-21 20:00:00 2027-10-18 11:03:27 9 +2023-03-21 21:00:00 2027-10-18 11:03:27 9 +2027-10-18 11:03:27 2027-10-18 11:03:27 9 +2023-03-21 12:00:00 2023-03-21 13:00:00 0 +2023-03-21 13:00:00 2023-03-21 13:00:00 0 +2023-03-21 14:00:00 2023-03-21 14:00:00 1 +2023-03-21 15:00:00 2023-03-21 15:00:00 2 +2023-03-21 16:00:00 2023-03-21 16:00:00 3 +2023-03-21 17:00:00 2027-10-18 11:03:27 9 +2023-03-21 18:00:00 2027-10-18 11:03:27 9 +2023-03-21 19:00:00 2027-10-18 11:03:27 9 +2023-03-21 20:00:00 2027-10-18 11:03:27 9 +2023-03-21 21:00:00 2027-10-18 11:03:27 9 +2027-10-18 11:03:27 2027-10-18 11:03:27 9 +\N \N \N +2023-03-21 12:00:00 2023-03-21 13:00:00 0 +2023-03-21 13:00:00 2023-03-21 14:00:00 1 +2023-03-21 14:00:00 2023-03-21 15:00:00 2 +2023-03-21 15:00:00 2023-03-21 16:00:00 3 +2023-03-21 16:00:00 2027-10-18 11:03:27 9 +2023-03-21 17:00:00 2027-10-18 11:03:27 9 +2023-03-21 18:00:00 2027-10-18 11:03:27 9 +2023-03-21 19:00:00 2027-10-18 11:03:27 9 +2023-03-21 20:00:00 2027-10-18 11:03:27 9 +2023-03-21 21:00:00 2027-10-18 11:03:27 9 +2023-03-21 12:00:00 2023-03-21 13:00:00 0 +2023-03-21 13:00:00 2023-03-21 14:00:00 1 +2023-03-21 14:00:00 2023-03-21 15:00:00 2 +2023-03-21 15:00:00 2023-03-21 16:00:00 3 +2023-03-21 16:00:00 2027-10-18 11:03:27 9 +2023-03-21 17:00:00 2027-10-18 11:03:27 9 +2023-03-21 18:00:00 2027-10-18 11:03:27 9 +2023-03-21 19:00:00 2027-10-18 11:03:27 9 +2023-03-21 20:00:00 2027-10-18 11:03:27 9 +2023-03-21 21:00:00 2027-10-18 11:03:27 9 +2027-10-18 11:03:27 \N \N +\N \N \N diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql new file mode 100644 index 00000000000..69de17541c1 --- /dev/null +++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql @@ -0,0 +1,63 @@ +DROP TABLE IF EXISTS events0; +DROP TABLE IF EXISTS probe0; + +SET join_algorithm = 'full_sorting_merge'; + +CREATE TABLE events0 ( + begin Nullable(DateTime('UTC')), + value Int32 +) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO events0 SELECT toDateTime('2023-03-21 13:00:00', 'UTC') + INTERVAL number HOUR, number FROM numbers(4); +INSERT INTO events0 VALUES (NULL, -10),('0000-01-01 00:00:00', -1), ('9999-12-31 23:59:59', 9); + +CREATE TABLE probe0 ( + begin Nullable(DateTime('UTC')) +) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO probe0 SELECT toDateTime('2023-03-21 12:00:00', 'UTC') + INTERVAl number HOUR FROM numbers(10); +INSERT INTO probe0 VALUES (NULL),('9999-12-31 23:59:59'); + +SET join_use_nulls = 1; + +SELECT p.begin, e.begin, e.value +FROM probe0 p +ASOF JOIN events0 e +ON p.begin > e.begin +ORDER BY p.begin ASC; + +SELECT p.begin, e.begin, e.value +FROM probe0 p +ASOF LEFT JOIN events0 e +ON p.begin > e.begin +ORDER BY p.begin ASC +; + +SELECT p.begin, e.begin, e.value +FROM probe0 p +ASOF JOIN events0 e +ON p.begin <= e.begin +ORDER BY p.begin ASC; + +SELECT p.begin, e.begin, e.value +FROM probe0 p +ASOF LEFT JOIN events0 e +ON p.begin <= e.begin +ORDER BY p.begin ASC; + +SELECT p.begin, e.begin, e.value +FROM probe0 p +ASOF JOIN events0 e +ON p.begin < e.begin +ORDER BY p.begin ASC +; + +SELECT p.begin, e.begin, e.value +FROM probe0 p +ASOF LEFT JOIN events0 e +ON p.begin < e.begin +ORDER BY p.begin ASC; + + +DROP TABLE IF EXISTS events0; +DROP TABLE IF EXISTS probe0; diff --git a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.reference b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.reference new file mode 100644 index 00000000000..ca481c7fff0 --- /dev/null +++ b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.reference @@ -0,0 +1,2 @@ +26790 1488 +26790 1488 diff --git a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 new file mode 100644 index 00000000000..551bac0cc06 --- /dev/null +++ b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 @@ -0,0 +1,37 @@ +SET allow_experimental_analyzer=1; + +SET session_timezone = 'UTC'; + +{% for join_algorithm in ['default', 'full_sorting_merge'] -%} + +SET join_algorithm = '{{ join_algorithm }}'; + +-- TODO: support enable for full_sorting_merge +-- SET join_use_nulls = 1; + +WITH build AS ( + SELECT + tk.number AS k, + toDateTime('2021-01-01 00:00:00') + INTERVAL i.number SECONDS AS t, + i.number % 37 AS v + FROM numbers(3000000) AS i + CROSS JOIN numbers(2) AS tk + SETTINGS join_algorithm = 'hash', join_use_nulls = 0 +), +probe AS ( + SELECT + tk.number AS k, + toDateTime('2021-01-01 00:00:30') + INTERVAL tt.number HOUR AS t + FROM numbers(2) AS tk + CROSS JOIN numbers(toUInt32((toDateTime('2021-02-01 00:00:30') - toDateTime('2021-01-01 00:00:30')) / 3600)) AS tt + SETTINGS join_algorithm = 'hash', join_use_nulls = 0 +) +SELECT + SUM(v) AS v, + COUNT(*) AS n +FROM probe +ASOF LEFT JOIN build +USING (k, t) +; + +{% endfor -%} From 29494d0bc6cc1b0f82a887a06f0643ea4880e681 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 1 May 2024 15:08:50 +0000 Subject: [PATCH 15/45] add asof_join_ddb tests --- .../03144_asof_join_ddb_doubles.sql | 1 + .../03145_asof_join_ddb_inequalities.sql | 1 + .../03147_asof_join_ddb_missing.reference | 10 + .../03147_asof_join_ddb_missing.sql | 186 ++++++++++++++++++ .../03148_asof_join_ddb_subquery.reference | 4 + .../03148_asof_join_ddb_subquery.sql | 29 +++ .../03149_asof_join_ddb_timestamps.reference | 56 ++++++ .../03149_asof_join_ddb_timestamps.sql | 95 +++++++++ 8 files changed, 382 insertions(+) create mode 100644 tests/queries/0_stateless/03147_asof_join_ddb_missing.reference create mode 100644 tests/queries/0_stateless/03147_asof_join_ddb_missing.sql create mode 100644 tests/queries/0_stateless/03148_asof_join_ddb_subquery.reference create mode 100644 tests/queries/0_stateless/03148_asof_join_ddb_subquery.sql create mode 100644 tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference create mode 100644 tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql diff --git a/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql index ef16ced3082..87aece14628 100644 --- a/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql +++ b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql @@ -1,4 +1,5 @@ SET join_algorithm = 'full_sorting_merge'; +SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS events0; diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql index 69de17541c1..ce4badbd597 100644 --- a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql +++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS events0; DROP TABLE IF EXISTS probe0; +SET allow_experimental_analyzer = 1; SET join_algorithm = 'full_sorting_merge'; CREATE TABLE events0 ( diff --git a/tests/queries/0_stateless/03147_asof_join_ddb_missing.reference b/tests/queries/0_stateless/03147_asof_join_ddb_missing.reference new file mode 100644 index 00000000000..11eb84463f4 --- /dev/null +++ b/tests/queries/0_stateless/03147_asof_join_ddb_missing.reference @@ -0,0 +1,10 @@ +108 +108 27 +513 +1218 +3528 +14553 +121275 +1495503 +12462525 +1249625025 diff --git a/tests/queries/0_stateless/03147_asof_join_ddb_missing.sql b/tests/queries/0_stateless/03147_asof_join_ddb_missing.sql new file mode 100644 index 00000000000..95a5f8ab3ff --- /dev/null +++ b/tests/queries/0_stateless/03147_asof_join_ddb_missing.sql @@ -0,0 +1,186 @@ +SET allow_experimental_analyzer=1; + +SET session_timezone = 'UTC'; +SET joined_subquery_requires_alias = 0; +SET allow_experimental_analyzer = 1; +SET join_algorithm = 'full_sorting_merge'; + +-- # 10 dates, 5 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(10), (SELECT number AS k FROM numbers(5)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); + +-- # Coverage: Missing right side bin +WITH build AS ( + SELECT + k * 2 AS k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(10), (SELECT number AS k FROM numbers(5)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + intDiv(k, 2) AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v), COUNT(*) +FROM probe ASOF JOIN build USING (k, t); + +-- # 20 dates, 5 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(20), (SELECT number AS k FROM numbers(5)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); + +-- # 30 dates, 5 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(30), (SELECT number AS k FROM numbers(5)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); + +-- # 50 dates, 5 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(50), (SELECT number AS k FROM numbers(5)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); + +-- # 100 dates, 5 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(100), (SELECT number AS k FROM numbers(5)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); + +-- # 100 dates, 50 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(100), (SELECT number AS k FROM numbers(50)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); + +-- # 1000 dates, 5 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(1000), (SELECT number AS k FROM numbers(5)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); + +-- # 1000 dates, 50 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(1000), (SELECT number AS k FROM numbers(50)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); + +-- # 10000 dates, 50 keys +WITH build AS ( + SELECT + k, + toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t, + number AS v + FROM numbers(10000), (SELECT number AS k FROM numbers(50)) + SETTINGS join_algorithm = 'default' +), +probe AS ( + SELECT + k * 2 AS k, + t - INTERVAL 30 SECOND AS t + FROM build +) +SELECT SUM(v) +FROM probe ASOF JOIN build USING (k, t); diff --git a/tests/queries/0_stateless/03148_asof_join_ddb_subquery.reference b/tests/queries/0_stateless/03148_asof_join_ddb_subquery.reference new file mode 100644 index 00000000000..387a4a8f249 --- /dev/null +++ b/tests/queries/0_stateless/03148_asof_join_ddb_subquery.reference @@ -0,0 +1,4 @@ +1 1 +3 1 +6 1 +8 1 diff --git a/tests/queries/0_stateless/03148_asof_join_ddb_subquery.sql b/tests/queries/0_stateless/03148_asof_join_ddb_subquery.sql new file mode 100644 index 00000000000..2ddf0f09b1e --- /dev/null +++ b/tests/queries/0_stateless/03148_asof_join_ddb_subquery.sql @@ -0,0 +1,29 @@ +DROP TABLE IF EXISTS events; +CREATE TABLE events (begin Float64, value Int32) ENGINE = MergeTree() ORDER BY begin; + +INSERT INTO events VALUES (1, 0), (3, 1), (6, 2), (8, 3); + +SET allow_experimental_analyzer = 1; +SET join_algorithm = 'full_sorting_merge'; +SET joined_subquery_requires_alias = 0; + +SELECT + begin, + value IN ( + SELECT e1.value + FROM ( + SELECT * + FROM events e1 + WHERE e1.value = events.value + ) AS e1 + ASOF JOIN ( + SELECT number :: Float64 AS begin + FROM numbers(10) + WHERE number >= 1 AND number < 10 + ) + USING (begin) + ) +FROM events +ORDER BY begin ASC; + +DROP TABLE IF EXISTS events; diff --git a/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference new file mode 100644 index 00000000000..7cfc85d23a5 --- /dev/null +++ b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference @@ -0,0 +1,56 @@ +2023-03-21 13:00:00 0 +2023-03-21 14:00:00 1 +2023-03-21 15:00:00 2 +2023-03-21 16:00:00 3 +2023-03-21 17:00:00 3 +2023-03-21 18:00:00 3 +2023-03-21 19:00:00 3 +2023-03-21 20:00:00 3 +2023-03-21 21:00:00 3 +2106-02-07 06:28:15 9 +2023-03-21 13:00:00 0 +2023-03-21 14:00:00 1 +2023-03-21 15:00:00 2 +2023-03-21 16:00:00 3 +2023-03-21 17:00:00 3 +2023-03-21 18:00:00 3 +2023-03-21 19:00:00 3 +2023-03-21 20:00:00 3 +2023-03-21 21:00:00 3 +2106-02-07 06:28:15 9 +2023-03-21 12:00:00 \N +2023-03-21 13:00:00 0 +2023-03-21 14:00:00 1 +2023-03-21 15:00:00 2 +2023-03-21 16:00:00 3 +2023-03-21 17:00:00 3 +2023-03-21 18:00:00 3 +2023-03-21 19:00:00 3 +2023-03-21 20:00:00 3 +2023-03-21 21:00:00 3 +2106-02-07 06:28:15 9 +\N \N +2023-03-21 12:00:00 0 +2023-03-21 13:00:00 0 +2023-03-21 14:00:00 1 +2023-03-21 15:00:00 2 +2023-03-21 16:00:00 3 +2023-03-21 17:00:00 3 +2023-03-21 18:00:00 3 +2023-03-21 19:00:00 3 +2023-03-21 20:00:00 3 +2023-03-21 21:00:00 3 +2106-02-07 06:28:15 9 +\N 0 +2023-03-21 12:00:00 \N +2023-03-21 13:00:00 \N +2023-03-21 14:00:00 \N +2023-03-21 15:00:00 \N +2023-03-21 16:00:00 \N +2023-03-21 17:00:00 \N +2023-03-21 18:00:00 \N +2023-03-21 19:00:00 \N +2023-03-21 20:00:00 \N +2023-03-21 21:00:00 \N +2106-02-07 06:28:15 \N +\N \N diff --git a/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql new file mode 100644 index 00000000000..ff4518a3775 --- /dev/null +++ b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql @@ -0,0 +1,95 @@ +DROP TABLE IF EXISTS events0; +DROP TABLE IF EXISTS probe0; + +SET session_timezone = 'UTC'; +SET allow_experimental_analyzer = 1; +SET join_algorithm = 'full_sorting_merge'; +SET join_use_nulls = 1; + +CREATE TABLE events0 +ENGINE = MergeTree() +ORDER BY COALESCE(begin, toDateTime('9999-12-31 23:59:59')) +AS +SELECT + toNullable(toDateTime('2023-03-21 13:00:00') + INTERVAL number HOUR) AS begin, + number AS value +FROM numbers(4); + +INSERT INTO events0 VALUES (NULL, -1), (toDateTime('9999-12-31 23:59:59'), 9); + +CREATE TABLE probe0 +ENGINE = MergeTree() +ORDER BY COALESCE(begin, toDateTime('9999-12-31 23:59:59')) +AS +SELECT + toNullable(toDateTime('2023-03-21 12:00:00') + INTERVAL number HOUR) AS begin +FROM numbers(10); + +INSERT INTO probe0 VALUES (NULL), (toDateTime('9999-12-31 23:59:59')); + +SELECT + p.begin, + e.value +FROM + probe0 p + ASOF JOIN events0 e ON p.begin >= e.begin +ORDER BY p.begin ASC; + +SELECT + p.begin, + e.value +FROM + probe0 p + ASOF JOIN events0 e USING (begin) +ORDER BY p.begin ASC +SETTINGS join_use_nulls = 0 +; + +SELECT + p.begin, + e.value +FROM + probe0 p + ASOF LEFT JOIN events0 e ON p.begin >= e.begin +ORDER BY p.begin ASC; + +SELECT + p.begin, + e.value +FROM + probe0 p + ASOF LEFT JOIN events0 e USING (begin) +ORDER BY p.begin ASC +SETTINGS join_use_nulls = 0 +; + +SELECT + p.begin, + e.value +FROM + probe0 p + ASOF RIGHT JOIN events0 e ON p.begin >= e.begin +ORDER BY e.begin ASC; -- { serverError NOT_IMPLEMENTED} + +SELECT + p.begin, + e.value +FROM + probe0 p + ASOF RIGHT JOIN events0 e USING (begin) +ORDER BY e.begin ASC; -- { serverError NOT_IMPLEMENTED} + + +SELECT + p.begin, + e.value +FROM + probe0 p + ASOF LEFT JOIN ( + SELECT * FROM events0 WHERE log(value + 5) > 10 + ) e ON p.begin >= e.begin +ORDER BY p.begin ASC; + + +DROP TABLE IF EXISTS events0; +DROP TABLE IF EXISTS probe0; From d37f03201aaac06b1a175ac0b177c17d945dddbb Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 7 May 2024 10:07:43 +0000 Subject: [PATCH 16/45] fix --- src/Processors/Transforms/MergeJoinTransform.cpp | 2 -- tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 26e1ebb0b60..9e6904f0613 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -1173,9 +1173,7 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() if (auto result = handleAllJoinState()) - { return std::move(*result); - } if (auto result = handleAsofJoinState()) return std::move(*result); diff --git a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 index 551bac0cc06..44c54ae2a39 100644 --- a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 +++ b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 @@ -1,3 +1,5 @@ +-- Tags: long + SET allow_experimental_analyzer=1; SET session_timezone = 'UTC'; From 558b73aba4f89db4a2b3e5ed0754749caddd5dfb Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 7 May 2024 14:20:40 +0000 Subject: [PATCH 17/45] t --- tests/queries/0_stateless/03143_asof_join_ddb_long.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql index c421702bb00..a41b667e6c8 100644 --- a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql +++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql @@ -1,5 +1,7 @@ -- Tags: long +-- https://s3.amazonaws.com/clickhouse-test-reports/55051/07f288862c56b0a98379a07101062689b0460788/stateless_tests_flaky_check__asan_.html + DROP TABLE IF EXISTS build; DROP TABLE IF EXISTS skewed_probe; From f710a67fb2e72361801f0ae9a45ccbd07e2c7f30 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 22 May 2024 12:27:04 +0000 Subject: [PATCH 18/45] set timezone in 03143_asof_join_ddb_long --- src/Processors/Transforms/MergeJoinTransform.cpp | 4 ++-- tests/queries/0_stateless/03143_asof_join_ddb_long.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 9e6904f0613..38b63a856f6 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -45,8 +45,8 @@ FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns, bool isNullAt(const IColumn & column, size_t row) { - if (const auto * nullable = checkAndGetColumn(column)) - return nullable->isNullAt(row); + if (const auto * nullable_column = checkAndGetColumn(&column)) + return nullable_column->isNullAt(row); return false; } diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql index a41b667e6c8..17a67511030 100644 --- a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql +++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql @@ -1,10 +1,10 @@ -- Tags: long --- https://s3.amazonaws.com/clickhouse-test-reports/55051/07f288862c56b0a98379a07101062689b0460788/stateless_tests_flaky_check__asan_.html - DROP TABLE IF EXISTS build; DROP TABLE IF EXISTS skewed_probe; +SET session_timezone = 'UTC'; + CREATE TABLE build ENGINE = MergeTree ORDER BY (key, begin) AS SELECT From 11f4ed75fd18a853e70c94f10596d5b3b09a35dd Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 5 Jun 2024 12:26:05 +0200 Subject: [PATCH 19/45] fix build --- src/Processors/tests/gtest_full_sorting_join.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index a3fda006eb8..7294a1b381a 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -151,7 +151,6 @@ public: size_t rows = current_chunk.front()->size(); chunks.emplace_back(std::move(current_chunk), rows); current_chunk = header.cloneEmptyColumns(); - return; } std::shared_ptr getSource() @@ -403,7 +402,7 @@ try auto expected_right = ColumnString::create(); UInt64 k1 = 1; - String k2 = ""; + String k2; auto get_attr = [&](const String & side, size_t idx) -> String { @@ -601,7 +600,7 @@ try ColumnInt64::Container expected; UInt64 k1 = 1; - String k2 = ""; + String k2; auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); for (size_t key_num = 0; key_num < key_num_total; ++key_num) { @@ -690,7 +689,7 @@ try ColumnInt64::Container expected; UInt64 k1 = 1; - String k2 = ""; + String k2; UInt64 left_t = 0; auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); From a84dd6b7710da54e89722318755d4e4e70984e2b Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 11 Jul 2024 10:37:00 +0000 Subject: [PATCH 20/45] Fix some review comments --- .../Transforms/MergeJoinTransform.cpp | 2 +- .../tests/gtest_full_sorting_join.cpp | 48 +++++++++++++------ .../00927_asof_join_correct_bt.reference | 8 ++++ .../00927_asof_join_correct_bt.sql | 3 ++ .../02276_full_sort_join_unsupported.sql | 2 - ...03145_asof_join_ddb_inequalities.reference | 5 ++ .../03145_asof_join_ddb_inequalities.sql | 12 +++-- .../03146_asof_join_ddb_merge_long.sql.j2 | 2 +- .../03149_asof_join_ddb_timestamps.sql | 2 +- 9 files changed, 59 insertions(+), 25 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 38a59cd6d9a..3b69ddaec06 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -43,7 +43,7 @@ FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns, return std::make_unique(block, desc, strictness == JoinStrictness::Asof); } -bool isNullAt(const IColumn & column, size_t row) +bool ALWAYS_INLINE isNullAt(const IColumn & column, size_t row) { if (const auto * nullable_column = checkAndGetColumn(&column)) return nullable_column->isNullAt(row); diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp index 7294a1b381a..f678d7984e8 100644 --- a/src/Processors/tests/gtest_full_sorting_join.cpp +++ b/src/Processors/tests/gtest_full_sorting_join.cpp @@ -484,7 +484,7 @@ try right_source.addRow({"AAPL", 2, 98}); right_source.addRow({"AAPL", 3, 99}); right_source.addRow({"AMZN", 1, 100}); - right_source.addRow({"AMZN", 2, 0}); + right_source.addRow({"AMZN", 2, 110}); right_source.addChunk(); right_source.addRow({"AMZN", 2, 110}); right_source.addChunk(); @@ -574,12 +574,15 @@ catch (Exception & e) TEST_F(FullSortingJoinTest, AsofLessGeneratedTestData) try { - auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left }); + /// Generate data random and build expected result at the same time. + /// Test specific combinations of join kind and inequality per each run + auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left }); auto asof_inequality = getRandomFrom(rng, { ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals }); SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality)); + /// Key is complex, `k1, k2` for equality and `t` for asof SourceChunksBuilder left_source_builder({ {std::make_shared(), "k1"}, {std::make_shared(), "k2"}, @@ -594,9 +597,11 @@ try {std::make_shared(), "attr"}, }); + /// How small generated block should be left_source_builder.setBreakProbability(rng); right_source_builder.setBreakProbability(rng); + /// We are going to generate sorted data and remember expected result ColumnInt64::Container expected; UInt64 k1 = 1; @@ -604,29 +609,34 @@ try auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); for (size_t key_num = 0; key_num < key_num_total; ++key_num) { + /// Generate new key greater than previous generateNextKey(rng, k1, k2); Int64 left_t = 0; + /// Generate several rows for the key size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng); for (size_t i = 0; i < num_left_rows; ++i) { + /// t is strictly greater than previous left_t += std::uniform_int_distribution<>(1, 10)(rng); - left_source_builder.addRow({k1, k2, left_t, 10 * left_t}); - expected.push_back(10 * left_t); + auto left_arrtibute_value = 10 * left_t; + left_source_builder.addRow({k1, k2, left_t, left_arrtibute_value}); + expected.push_back(left_arrtibute_value); auto num_matches = 1 + std::poisson_distribution<>(4)(rng); - + /// Generate several matches in the right table auto right_t = left_t; for (size_t j = 0; j < num_matches; ++j) { int min_step = isStrict(asof_inequality) ? 1 : 0; right_t += std::uniform_int_distribution<>(min_step, 3)(rng); + /// First row should match bool is_match = j == 0; - right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * left_t : -1}); + right_source_builder.addRow({k1, k2, right_t, is_match ? 10 * left_arrtibute_value : -1}); } - /// next left_t should be greater than right_t not to match with previous rows + /// Next left_t should be greater than right_t not to match with previous rows left_t = right_t; } @@ -650,7 +660,9 @@ try assertColumnVectorEq(expected, result_block, "t1.attr"); for (auto & e : expected) - e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr + /// Non matched rows from left table have negative attr + /// Value if attribute in right table is 10 times greater than in left table + e = e < 0 ? 0 : 10 * e; assertColumnVectorEq(expected, result_block, "t2.attr"); } @@ -663,8 +675,10 @@ catch (Exception & e) TEST_F(FullSortingJoinTest, AsofGreaterGeneratedTestData) try { - auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left }); + /// Generate data random and build expected result at the same time. + /// Test specific combinations of join kind and inequality per each run + auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left }); auto asof_inequality = getRandomFrom(rng, { ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals }); SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality)); @@ -695,9 +709,10 @@ try auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng); for (size_t key_num = 0; key_num < key_num_total; ++key_num) { + /// Generate new key greater than previous generateNextKey(rng, k1, k2); - /// generate some rows with smaller left_t to check that they are not matched + /// Generate some rows with smaller left_t to check that they are not matched size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 100)(rng) : 0; for (size_t i = 0; i < num_left_rows; ++i) { @@ -713,21 +728,22 @@ try size_t num_right_matches = std::uniform_int_distribution<>(1, 10)(rng); auto right_t = left_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 0 : 1, 10)(rng); + auto attribute_value = 10 * right_t; for (size_t j = 0; j < num_right_matches; ++j) { right_t += std::uniform_int_distribution<>(0, 3)(rng); bool is_match = j == num_right_matches - 1; - right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * right_t : -1}); + right_source_builder.addRow({k1, k2, right_t, is_match ? 10 * attribute_value : -1}); } - /// next left_t should be greater than (or equals) right_t to match with previous rows + /// Next left_t should be greater than (or equals) right_t to match with previous rows left_t = right_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 1 : 0, 100)(rng); size_t num_left_matches = std::uniform_int_distribution<>(1, 100)(rng); for (size_t j = 0; j < num_left_matches; ++j) { left_t += std::uniform_int_distribution<>(0, 3)(rng); - left_source_builder.addRow({k1, k2, left_t, 10 * right_t}); - expected.push_back(10 * right_t); + left_source_builder.addRow({k1, k2, left_t, attribute_value}); + expected.push_back(attribute_value); } } @@ -739,7 +755,9 @@ try assertColumnVectorEq(expected, result_block, "t1.attr"); for (auto & e : expected) - e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr + /// Non matched rows from left table have negative attr + /// Value if attribute in right table is 10 times greater than in left table + e = e < 0 ? 0 : 10 * e; assertColumnVectorEq(expected, result_block, "t2.attr"); } diff --git a/tests/queries/0_stateless/00927_asof_join_correct_bt.reference b/tests/queries/0_stateless/00927_asof_join_correct_bt.reference index a398f9604fd..28c48d2e290 100644 --- a/tests/queries/0_stateless/00927_asof_join_correct_bt.reference +++ b/tests/queries/0_stateless/00927_asof_join_correct_bt.reference @@ -1,28 +1,36 @@ +-- { echoOn } +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t); 1 101 1 0 0 0 1 102 2 2 102 1 1 103 3 2 102 1 1 104 4 4 104 1 1 105 5 4 104 1 +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t); 1 101 1 0 0 0 1 102 2 2 102 1 1 103 3 2 102 1 1 104 4 4 104 1 1 105 5 4 104 1 +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t); 1 101 1 0 0 0 1 102 2 2 102 1 1 103 3 2 102 1 1 104 4 4 104 1 1 105 5 4 104 1 +SET join_algorithm = 'full_sorting_merge'; +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t); 1 101 1 0 0 0 1 102 2 2 102 1 1 103 3 2 102 1 1 104 4 4 104 1 1 105 5 4 104 1 +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t); 1 101 1 0 0 0 1 102 2 2 102 1 1 103 3 2 102 1 1 104 4 4 104 1 1 105 5 4 104 1 +SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t); 1 101 1 0 0 0 1 102 2 2 102 1 1 103 3 2 102 1 diff --git a/tests/queries/0_stateless/00927_asof_join_correct_bt.sql b/tests/queries/0_stateless/00927_asof_join_correct_bt.sql index 761d6bacde6..d796b62d3b3 100644 --- a/tests/queries/0_stateless/00927_asof_join_correct_bt.sql +++ b/tests/queries/0_stateless/00927_asof_join_correct_bt.sql @@ -13,6 +13,7 @@ INSERT INTO B2(k,t,b) VALUES (1,102,2), (1,104,4); CREATE TABLE B3(k UInt32, b UInt64, t UInt32) ENGINE = MergeTree() ORDER BY (k, t); INSERT INTO B3(k,t,b) VALUES (1,102,2), (1,104,4); +-- { echoOn } SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t); SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t); SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t); @@ -22,6 +23,8 @@ SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t); SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t); +-- { echoOff } + DROP TABLE B1; DROP TABLE B2; DROP TABLE B3; diff --git a/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql b/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql index 03936107563..0b10101d8f2 100644 --- a/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql +++ b/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql @@ -19,8 +19,6 @@ SELECT * FROM t1 ANTI JOIN t2 ON t1.key = t2.key; -- { serverError NOT_IMPLEMENT SELECT * FROM t1 SEMI JOIN t2 ON t1.key = t2.key; -- { serverError NOT_IMPLEMENTED } --- SELECT * FROM t1 ASOF JOIN t2 ON t1.key = t2.key AND t1.val > t2.val; -- { serverError NOT_IMPLEMENTED } - SELECT * FROM t1 ANY JOIN t2 ON t1.key = t2.key SETTINGS any_join_distinct_right_table_keys = 1; -- { serverError NOT_IMPLEMENTED } SELECT * FROM t1 JOIN t2 USING (key) SETTINGS join_use_nulls = 1; -- { serverError NOT_IMPLEMENTED } diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference index 73c4f7dfe25..4aac918c98c 100644 --- a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference +++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference @@ -1,3 +1,4 @@ +- 2023-03-21 12:00:00 1970-01-01 00:00:00 -1 2023-03-21 13:00:00 1970-01-01 00:00:00 -1 2023-03-21 14:00:00 2023-03-21 13:00:00 0 @@ -9,6 +10,7 @@ 2023-03-21 20:00:00 2023-03-21 16:00:00 3 2023-03-21 21:00:00 2023-03-21 16:00:00 3 2027-10-18 11:03:27 2023-03-21 16:00:00 3 +- 2023-03-21 12:00:00 1970-01-01 00:00:00 -1 2023-03-21 13:00:00 1970-01-01 00:00:00 -1 2023-03-21 14:00:00 2023-03-21 13:00:00 0 @@ -32,6 +34,7 @@ 2023-03-21 20:00:00 2027-10-18 11:03:27 9 2023-03-21 21:00:00 2027-10-18 11:03:27 9 2027-10-18 11:03:27 2027-10-18 11:03:27 9 +- 2023-03-21 12:00:00 2023-03-21 13:00:00 0 2023-03-21 13:00:00 2023-03-21 13:00:00 0 2023-03-21 14:00:00 2023-03-21 14:00:00 1 @@ -44,6 +47,7 @@ 2023-03-21 21:00:00 2027-10-18 11:03:27 9 2027-10-18 11:03:27 2027-10-18 11:03:27 9 \N \N \N +- 2023-03-21 12:00:00 2023-03-21 13:00:00 0 2023-03-21 13:00:00 2023-03-21 14:00:00 1 2023-03-21 14:00:00 2023-03-21 15:00:00 2 @@ -54,6 +58,7 @@ 2023-03-21 19:00:00 2027-10-18 11:03:27 9 2023-03-21 20:00:00 2027-10-18 11:03:27 9 2023-03-21 21:00:00 2027-10-18 11:03:27 9 +- 2023-03-21 12:00:00 2023-03-21 13:00:00 0 2023-03-21 13:00:00 2023-03-21 14:00:00 1 2023-03-21 14:00:00 2023-03-21 15:00:00 2 diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql index ce4badbd597..d67aa254bd6 100644 --- a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql +++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql @@ -21,18 +21,19 @@ INSERT INTO probe0 VALUES (NULL),('9999-12-31 23:59:59'); SET join_use_nulls = 1; +SELECT '-'; SELECT p.begin, e.begin, e.value FROM probe0 p ASOF JOIN events0 e ON p.begin > e.begin ORDER BY p.begin ASC; +SELECT '-'; SELECT p.begin, e.begin, e.value FROM probe0 p ASOF LEFT JOIN events0 e ON p.begin > e.begin -ORDER BY p.begin ASC -; +ORDER BY p.begin ASC; SELECT p.begin, e.begin, e.value FROM probe0 p @@ -40,25 +41,26 @@ ASOF JOIN events0 e ON p.begin <= e.begin ORDER BY p.begin ASC; +SELECT '-'; SELECT p.begin, e.begin, e.value FROM probe0 p ASOF LEFT JOIN events0 e ON p.begin <= e.begin ORDER BY p.begin ASC; +SELECT '-'; SELECT p.begin, e.begin, e.value FROM probe0 p ASOF JOIN events0 e ON p.begin < e.begin -ORDER BY p.begin ASC -; +ORDER BY p.begin ASC; +SELECT '-'; SELECT p.begin, e.begin, e.value FROM probe0 p ASOF LEFT JOIN events0 e ON p.begin < e.begin ORDER BY p.begin ASC; - DROP TABLE IF EXISTS events0; DROP TABLE IF EXISTS probe0; diff --git a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 index 44c54ae2a39..49ba70c471e 100644 --- a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 +++ b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 @@ -8,7 +8,7 @@ SET session_timezone = 'UTC'; SET join_algorithm = '{{ join_algorithm }}'; --- TODO: support enable for full_sorting_merge +-- TODO: enable once USING and `join_use_nulls` is supported by `full_sorting_merge` -- SET join_use_nulls = 1; WITH build AS ( diff --git a/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql index ff4518a3775..cd83d62dc70 100644 --- a/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql +++ b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql @@ -87,7 +87,7 @@ FROM probe0 p ASOF LEFT JOIN ( SELECT * FROM events0 WHERE log(value + 5) > 10 - ) e ON p.begin >= e.begin + ) e ON p.begin + INTERVAL 2 HOUR >= e.begin + INTERVAL 1 HOUR ORDER BY p.begin ASC; From 85714e7d7e18a9d91dfe385b658e1ae1ddedb2ea Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 18 Jul 2024 11:31:23 +0000 Subject: [PATCH 21/45] fix asof join on nulls --- .../Transforms/MergeJoinTransform.cpp | 10 ++- .../0_stateless/00976_asof_join_on.reference | 78 ++++++++++++++++++- .../0_stateless/00976_asof_join_on.sql.j2 | 7 +- 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 3b69ddaec06..7bdb3d4f30f 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -257,8 +257,14 @@ JoinKeyRow::JoinKeyRow(const FullMergeJoinCursor & cursor, size_t pos) new_col->insertFrom(*col, pos); row.push_back(std::move(new_col)); } - if (const auto * asof_column = cursor.getAsofColumn()) + if (const IColumn * asof_column = cursor.getAsofColumn()) { + if (const auto * nullable_asof_column = checkAndGetColumn(asof_column)) + { + /// We save matched column, and since NULL do not match anything, we can't use it as a key + chassert(!nullable_asof_column->isNullAt(pos)); + asof_column = nullable_asof_column->getNestedColumnPtr().get(); + } auto new_col = asof_column->cloneEmpty(); new_col->insertFrom(*asof_column, pos); row.push_back(std::move(new_col)); @@ -1174,7 +1180,6 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() if (!cursors[1]->cursor.isValid() && !cursors[1]->fullyCompleted()) return Status(1); - if (auto result = handleAllJoinState()) return std::move(*result); @@ -1183,7 +1188,6 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() if (cursors[0]->fullyCompleted() || cursors[1]->fullyCompleted()) { - if (!cursors[0]->fullyCompleted() && isLeftOrFull(kind)) return Status(createBlockWithDefaults(0)); diff --git a/tests/queries/0_stateless/00976_asof_join_on.reference b/tests/queries/0_stateless/00976_asof_join_on.reference index e13745bed9d..80af4287d3c 100644 --- a/tests/queries/0_stateless/00976_asof_join_on.reference +++ b/tests/queries/0_stateless/00976_asof_join_on.reference @@ -1,4 +1,4 @@ -- default - +- default / join_use_nulls = 0 - 1 1 0 0 1 2 1 2 1 3 1 2 @@ -35,7 +35,7 @@ 2 1 2 3 2 2 2 3 1 2 1 2 -- full_sorting_merge - +- full_sorting_merge / join_use_nulls = 0 - 1 1 0 0 1 2 1 2 1 3 1 2 @@ -72,3 +72,77 @@ 2 1 2 3 2 2 2 3 1 2 1 2 +- default / join_use_nulls = 1 - +1 1 \N \N +1 2 1 2 +1 3 1 2 +2 1 \N \N +2 2 \N \N +2 3 2 3 +3 1 \N \N +3 2 \N \N +3 3 \N \N +9 +1 2 1 2 +1 3 1 2 +2 3 2 3 +- +1 1 1 2 +1 2 1 2 +1 3 1 4 +2 1 2 3 +2 2 2 3 +2 3 2 3 +- +1 1 1 2 +1 2 1 2 +1 3 1 4 +2 1 2 3 +2 2 2 3 +2 3 2 3 +- +1 3 1 2 +- +1 1 1 2 +1 2 1 4 +1 3 1 4 +2 1 2 3 +2 2 2 3 +1 2 1 2 +- full_sorting_merge / join_use_nulls = 1 - +1 1 \N \N +1 2 1 2 +1 3 1 2 +2 1 \N \N +2 2 \N \N +2 3 2 3 +3 1 \N \N +3 2 \N \N +3 3 \N \N +9 +1 2 1 2 +1 3 1 2 +2 3 2 3 +- +1 1 1 2 +1 2 1 2 +1 3 1 4 +2 1 2 3 +2 2 2 3 +2 3 2 3 +- +1 1 1 2 +1 2 1 2 +1 3 1 4 +2 1 2 3 +2 2 2 3 +2 3 2 3 +- +1 3 1 2 +- +1 1 1 2 +1 2 1 4 +1 3 1 4 +2 1 2 3 +2 2 2 3 +1 2 1 2 diff --git a/tests/queries/0_stateless/00976_asof_join_on.sql.j2 b/tests/queries/0_stateless/00976_asof_join_on.sql.j2 index aecf472a36c..ea642366a05 100644 --- a/tests/queries/0_stateless/00976_asof_join_on.sql.j2 +++ b/tests/queries/0_stateless/00976_asof_join_on.sql.j2 @@ -7,11 +7,13 @@ CREATE TABLE B(b UInt32, t UInt32) ENGINE = Memory; INSERT INTO A (a,t) VALUES (1,1),(1,2),(1,3), (2,1),(2,2),(2,3), (3,1),(3,2),(3,3); INSERT INTO B (b,t) VALUES (1,2),(1,4),(2,3); +{% for join_use_nulls in [0, 1] -%} {% for join_algorithm in ['default', 'full_sorting_merge'] -%} SET join_algorithm = '{{ join_algorithm }}'; -SELECT '- {{ join_algorithm }} -'; +SELECT '- {{ join_algorithm }} / join_use_nulls = {{ join_use_nulls }} -'; +set join_use_nulls = {{ join_use_nulls }}; SELECT A.a, A.t, B.b, B.t FROM A ASOF LEFT JOIN B ON A.a == B.b AND A.t >= B.t ORDER BY (A.a, A.t); SELECT count() FROM A ASOF LEFT JOIN B ON A.a == B.b AND B.t <= A.t; @@ -34,7 +36,8 @@ ASOF INNER JOIN (SELECT * FROM B UNION ALL SELECT 1, 3) AS B ON B.t <= A.t AND A WHERE B.t != 3 ORDER BY (A.a, A.t) ; -{% endfor %} +{% endfor -%} +{% endfor -%} DROP TABLE A; DROP TABLE B; From ab2522b17020f1fb31b760f0594784cd58468797 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 18 Jul 2024 13:17:34 +0000 Subject: [PATCH 22/45] fix asof join on nulls --- src/Processors/Transforms/MergeJoinTransform.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 7bdb3d4f30f..e96a75d277b 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -301,7 +301,11 @@ bool JoinKeyRow::asofMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequalit if (isNullAt(*asof_row, 0) || isNullAt(*cursor.getAsofColumn(), cursor->getRow())) return false; - int cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *asof_row, 1); + int cmp = 0; + if (const auto * nullable_column = checkAndGetColumn(cursor.getAsofColumn())) + cmp = nullable_column->getNestedColumn().compareAt(cursor->getRow(), 0, *asof_row, 1); + else + cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *asof_row, 1); return (asof_inequality == ASOFJoinInequality::Less && cmp < 0) || (asof_inequality == ASOFJoinInequality::LessOrEquals && cmp <= 0) From 093b13329c4206b4ecd604fb373d8d60cf345a1f Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 18 Jul 2024 15:46:05 +0000 Subject: [PATCH 23/45] One more check in JOIN ON ... IS NULL --- .../Passes/LogicalExpressionOptimizerPass.cpp | 46 +++++++++++++++++-- ...11_join_on_nullsafe_optimization.reference | 4 ++ .../02911_join_on_nullsafe_optimization.sql | 1 + 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 854697bca9f..7e54b5a4b42 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -67,6 +67,41 @@ QueryTreeNodePtr findEqualsFunction(const QueryTreeNodes & nodes) return nullptr; } +/// Checks if the node is combination of isNull and notEquals functions of two the same arguments +bool matchIsNullOfTwoArgs(const QueryTreeNodes & nodes, QueryTreeNodePtr & lhs, QueryTreeNodePtr & rhs) +{ + QueryTreeNodePtrWithHashSet all_arguments; + for (const auto & node : nodes) + { + const auto * func_node = node->as(); + if (!func_node) + return false; + + const auto & arguments = func_node->getArguments().getNodes(); + if (func_node->getFunctionName() == "isNull" && arguments.size() == 1) + all_arguments.insert(QueryTreeNodePtrWithHash(arguments[0])); + else if (func_node->getFunctionName() == "notEquals" && arguments.size() == 2) + { + if (arguments[0]->isEqual(*arguments[1])) + return false; + all_arguments.insert(QueryTreeNodePtrWithHash(arguments[0])); + all_arguments.insert(QueryTreeNodePtrWithHash(arguments[1])); + } + else + return false; + + if (all_arguments.size() > 2) + return false; + } + + if (all_arguments.size() != 2) + return false; + + lhs = all_arguments.begin()->node; + rhs = std::next(all_arguments.begin())->node; + return true; +} + bool isBooleanConstant(const QueryTreeNodePtr & node, bool expected_value) { const auto * constant_node = node->as(); @@ -212,11 +247,14 @@ private: else if (func_name == "and") { const auto & and_arguments = argument_function->getArguments().getNodes(); - bool all_are_is_null = and_arguments.size() == 2 && isNodeFunction(and_arguments[0], "isNull") && isNodeFunction(and_arguments[1], "isNull"); - if (all_are_is_null) + + QueryTreeNodePtr is_null_lhs_arg; + QueryTreeNodePtr is_null_rhs_arg; + if (matchIsNullOfTwoArgs(and_arguments, is_null_lhs_arg, is_null_rhs_arg)) { - is_null_argument_to_indices[getFunctionArgument(and_arguments.front(), 0)].push_back(or_operands.size() - 1); - is_null_argument_to_indices[getFunctionArgument(and_arguments.back(), 0)].push_back(or_operands.size() - 1); + is_null_argument_to_indices[is_null_lhs_arg].push_back(or_operands.size() - 1); + is_null_argument_to_indices[is_null_rhs_arg].push_back(or_operands.size() - 1); + continue; } /// Expression `a = b AND (a IS NOT NULL) AND true AND (b IS NOT NULL)` we can be replaced with `a = b` diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference index 4eb7e74446d..31a1cda18e7 100644 --- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference +++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference @@ -35,6 +35,10 @@ SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS N 2 2 2 2 3 3 3 33 \N \N \N \N +SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST; +2 2 2 2 +3 3 3 33 +\N \N \N \N -- aliases defined in the join condition are valid -- FIXME(@vdimir) broken query formatting for the following queries: -- SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST; diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql index f7813e2a1b4..f739259caf9 100644 --- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql +++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql @@ -34,6 +34,7 @@ SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) A SELECT x = y OR (x IS NULL AND y IS NULL) FROM t1 ORDER BY x NULLS LAST; SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST; +SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST; -- aliases defined in the join condition are valid -- FIXME(@vdimir) broken query formatting for the following queries: From f19de4effdf1e9d15acab69cecd882d7fd9b156b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 19 Jul 2024 03:38:05 +0200 Subject: [PATCH 24/45] Fix bad tests long_select_and_alter, CC @alesapin --- .../0_stateless/01338_long_select_and_alter.reference | 2 +- tests/queries/0_stateless/01338_long_select_and_alter.sh | 6 +++--- .../01338_long_select_and_alter_zookeeper.reference | 2 +- .../0_stateless/01338_long_select_and_alter_zookeeper.sh | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.reference b/tests/queries/0_stateless/01338_long_select_and_alter.reference index c2678e7052e..921730b17ce 100644 --- a/tests/queries/0_stateless/01338_long_select_and_alter.reference +++ b/tests/queries/0_stateless/01338_long_select_and_alter.reference @@ -1,3 +1,3 @@ -10 5 +10 CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.sh b/tests/queries/0_stateless/01338_long_select_and_alter.sh index 2b0709162a3..2659e5c16cf 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter.sh @@ -13,15 +13,15 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number) $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query "SELECT count(distinct concat(value, '_')) FROM alter_mt WHERE not sleepEachRow(2)" & -# to be sure that select took all required locks +# To be sure that select took all required locks for better test sensitivity, although it isn't guaranteed (then the test will also succeed). sleep 2 $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64" -$CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt" - wait +$CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt" + $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE alter_mt" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference index b4ed8efab63..9c5ad0fa468 100644 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference @@ -1,3 +1,3 @@ -10 5 +10 CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh index 41e0a12f369..6eb795408f4 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh @@ -13,15 +13,15 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number) $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query "SELECT count(distinct concat(value, '_')) FROM alter_mt WHERE not sleepEachRow(2)" & -# to be sure that select took all required locks +# To be sure that select took all required locks for better test sensitivity, although it isn't guaranteed (then the test will also succeed). sleep 2 $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64" -$CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt" - wait +$CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt" + $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE alter_mt" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" From 0cab22fd16caf260306e1b1feb77d2ddcced5205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 19 Jul 2024 16:44:56 +0200 Subject: [PATCH 25/45] Attempt to fix flakiness of 01194_http_query_id --- tests/queries/0_stateless/01194_http_query_id.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01194_http_query_id.sh b/tests/queries/0_stateless/01194_http_query_id.sh index 5aebdc10dfc..fac17cca3c6 100755 --- a/tests/queries/0_stateless/01194_http_query_id.sh +++ b/tests/queries/0_stateless/01194_http_query_id.sh @@ -4,14 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?session_id=test_01194" rnd=$RANDOM +url="${CLICKHOUSE_URL}&session_id=test_01194_$RANDOM" ${CLICKHOUSE_CURL} -sS "$url&query=SELECT+'test_01194',$rnd,1" > /dev/null ${CLICKHOUSE_CURL} -sS "$url&query=SELECT+'test_01194',$rnd,2" > /dev/null ${CLICKHOUSE_CURL} -sS "$url" --data "SELECT 'test_01194',$rnd,3" > /dev/null ${CLICKHOUSE_CURL} -sS "$url" --data "SELECT 'test_01194',$rnd,4" > /dev/null -${CLICKHOUSE_CURL} -sS "$url" --data "SYSTEM FLUSH LOGS" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CURL} -sS "$url&query=SELECT+count(DISTINCT+query_id)+FROM+system.query_log+WHERE+current_database+LIKE+currentDatabase()+AND+query+LIKE+'SELECT+''test_01194'',$rnd%25'" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data "SELECT count(DISTINCT query_id) FROM system.query_log WHERE current_database = currentDatabase() AND event_date >= yesterday() AND query LIKE 'SELECT ''test_01194'',$rnd%'" From c948103dee50bf5bddeff9af485d4df2acc8b0f7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 19 Jul 2024 22:23:50 +0200 Subject: [PATCH 26/45] Better tests --- .../0_stateless/01338_long_select_and_alter.reference | 2 +- tests/queries/0_stateless/01338_long_select_and_alter.sh | 6 +++--- .../01338_long_select_and_alter_zookeeper.reference | 2 +- .../0_stateless/01338_long_select_and_alter_zookeeper.sh | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.reference b/tests/queries/0_stateless/01338_long_select_and_alter.reference index 921730b17ce..276d6bcc29d 100644 --- a/tests/queries/0_stateless/01338_long_select_and_alter.reference +++ b/tests/queries/0_stateless/01338_long_select_and_alter.reference @@ -1,3 +1,3 @@ 5 -10 +5 CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.sh b/tests/queries/0_stateless/01338_long_select_and_alter.sh index 2659e5c16cf..08609546ff5 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter.sh @@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" $CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=MergeTree() ORDER BY key" -$CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number) FROM numbers(5)" +$CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number - 1 AS x, toString(x) FROM numbers(5)" $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query "SELECT count(distinct concat(value, '_')) FROM alter_mt WHERE not sleepEachRow(2)" & @@ -18,10 +18,10 @@ sleep 2 $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64" -wait - $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt" +wait + $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE alter_mt" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference index 9c5ad0fa468..aab1b93f6bd 100644 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference @@ -1,3 +1,3 @@ 5 -10 +5 CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh index 6eb795408f4..def6d2ab127 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh @@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" $CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_mt', '1') ORDER BY key" -$CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number) FROM numbers(5)" +$CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number - 1 AS x, toString(x) FROM numbers(5)" $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query "SELECT count(distinct concat(value, '_')) FROM alter_mt WHERE not sleepEachRow(2)" & @@ -18,10 +18,10 @@ sleep 2 $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64" -wait - $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt" +wait + $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE alter_mt" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" From 8a67713e63bbaec8bf820bd6813affefe305d2cc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 20 Jul 2024 01:09:45 +0200 Subject: [PATCH 27/45] Fix error --- tests/queries/0_stateless/01338_long_select_and_alter.sh | 4 ++-- .../0_stateless/01338_long_select_and_alter_zookeeper.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.sh b/tests/queries/0_stateless/01338_long_select_and_alter.sh index 08609546ff5..5d2759ac884 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" -$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=MergeTree() ORDER BY key" +$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key Int64, value String) ENGINE=MergeTree() ORDER BY key" $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number - 1 AS x, toString(x) FROM numbers(5)" @@ -16,7 +16,7 @@ $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query # To be sure that select took all required locks for better test sensitivity, although it isn't guaranteed (then the test will also succeed). sleep 2 -$CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64" +$CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value Int64" $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt" diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh index def6d2ab127..593a96a7cc8 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" -$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_mt', '1') ORDER BY key" +$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key Int64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_mt', '1') ORDER BY key" $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number - 1 AS x, toString(x) FROM numbers(5)" @@ -16,7 +16,7 @@ $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query # To be sure that select took all required locks for better test sensitivity, although it isn't guaranteed (then the test will also succeed). sleep 2 -$CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64" +$CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value Int64" $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt" From c786e6eb49152d7992ee3ee5c76300b7e4712309 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 20 Jul 2024 21:27:35 +0200 Subject: [PATCH 28/45] Fix tests --- tests/queries/0_stateless/01338_long_select_and_alter.reference | 2 +- .../0_stateless/01338_long_select_and_alter_zookeeper.reference | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.reference b/tests/queries/0_stateless/01338_long_select_and_alter.reference index 276d6bcc29d..027109252e1 100644 --- a/tests/queries/0_stateless/01338_long_select_and_alter.reference +++ b/tests/queries/0_stateless/01338_long_select_and_alter.reference @@ -1,3 +1,3 @@ 5 5 -CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_mt\n(\n `key` Int64,\n `value` Int64\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference index aab1b93f6bd..65e638bc3a4 100644 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference @@ -1,3 +1,3 @@ 5 5 -CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_mt\n(\n `key` Int64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 From 5ff125e37ef35ce5b451e54482e92d7663e25bdb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Jul 2024 10:32:54 +0200 Subject: [PATCH 29/45] Miscellaneous --- src/Common/CurrentMetrics.cpp | 8 ++++---- src/Databases/DatabaseLazy.cpp | 4 ++-- src/Databases/DatabasesCommon.cpp | 8 ++++---- src/Parsers/ExpressionListParsers.cpp | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 7c97e73f278..1011ab12d15 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -234,10 +234,10 @@ M(PartsCommitted, "Deprecated. See PartsActive.") \ M(PartsPreActive, "The part is in data_parts, but not used for SELECTs.") \ M(PartsActive, "Active data part, used by current and upcoming SELECTs.") \ - M(AttachedDatabase, "Active database, used by current and upcoming SELECTs.") \ - M(AttachedTable, "Active table, used by current and upcoming SELECTs.") \ - M(AttachedView, "Active view, used by current and upcoming SELECTs.") \ - M(AttachedDictionary, "Active dictionary, used by current and upcoming SELECTs.") \ + M(AttachedDatabase, "Active databases.") \ + M(AttachedTable, "Active tables.") \ + M(AttachedView, "Active views.") \ + M(AttachedDictionary, "Active dictionaries.") \ M(PartsOutdated, "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.") \ M(PartsDeleting, "Not active data part with identity refcounter, it is deleting right now by a cleaner.") \ M(PartsDeleteOnDestroy, "Part was moved to another disk and should be deleted in own destructor.") \ diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 5017c9b25cb..ca30ee6db15 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -195,7 +195,7 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n snapshot_detached_tables.erase(table_name); } - CurrentMetrics::add(CurrentMetrics::AttachedTable, 1); + CurrentMetrics::add(CurrentMetrics::AttachedTable); } StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & table_name) @@ -221,7 +221,7 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta .metadata_path = getObjectMetadataPath(table_name), .is_permanently = false}); - CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1); + CurrentMetrics::sub(CurrentMetrics::AttachedTable); } return res; } diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index fe0baf30e57..6ccaf811764 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -289,8 +289,8 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n tables.erase(it); table_storage->is_detached = true; - if (table_storage->isSystemStorage() == false) - CurrentMetrics::sub(getAttachedCounterForStorage(table_storage), 1); + if (!table_storage->isSystemStorage() && database_name != DatabaseCatalog::SYSTEM_DATABASE) + CurrentMetrics::sub(getAttachedCounterForStorage(table_storage)); auto table_id = table_storage->getStorageID(); if (table_id.hasUUID()) @@ -334,8 +334,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c /// non-Atomic database the is_detached is set to true before RENAME. table->is_detached = false; - if (table->isSystemStorage() == false && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE) - CurrentMetrics::add(getAttachedCounterForStorage(table), 1); + if (!table->isSystemStorage() && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE) + CurrentMetrics::add(getAttachedCounterForStorage(table)); } void DatabaseWithOwnTablesBase::shutdown() diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index fff8383e7b3..f97c042e91e 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -2743,7 +2743,7 @@ Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & po /// 'AND' can be both boolean function and part of the '... BETWEEN ... AND ...' operator if (op.function_name == "and" && layers.back()->between_counter) { - layers.back()->between_counter--; + --layers.back()->between_counter; op = finish_between_operator; } From 7f03b189667161de61e33523586a594694b02071 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Jul 2024 10:49:53 +0200 Subject: [PATCH 30/45] Fix two terrible bugs --- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ea10ad59db4..8c1a3cb4dff 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1606,7 +1606,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, UInt64 table_count = CurrentMetrics::get(CurrentMetrics::AttachedTable); if (table_count >= table_num_limit) throw Exception(ErrorCodes::TOO_MANY_TABLES, - "Too many tables in the Clickhouse. " + "Too many tables. " "The limit (setting 'max_table_num_to_throw') is set to {}, current number of tables is {}", table_num_limit, table_count); } From a6f1c46448a17f294cc6047dfc3fef3802b40abf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Jul 2024 11:08:38 +0200 Subject: [PATCH 31/45] Miscellaneous --- src/Interpreters/InterpreterCreateQuery.cpp | 9 ++++---- .../test_table_db_num_limit/test.py | 22 +++++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 8c1a3cb4dff..2a4a02597fe 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include @@ -82,13 +81,13 @@ #include #include -#include #include #include #include #include + namespace CurrentMetrics { extern const Metric AttachedTable; @@ -166,8 +165,8 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (db_count >= db_num_limit) throw Exception(ErrorCodes::TOO_MANY_DATABASES, - "Too many databases in the Clickhouse. " - "The limit (setting 'max_database_num_to_throw') is set to {}, current number of databases is {}", + "Too many databases. " + "The limit (server configuration parameter `max_database_num_to_throw`) is set to {}, the current number of databases is {}", db_num_limit, db_count); } @@ -1607,7 +1606,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, if (table_count >= table_num_limit) throw Exception(ErrorCodes::TOO_MANY_TABLES, "Too many tables. " - "The limit (setting 'max_table_num_to_throw') is set to {}, current number of tables is {}", + "The limit (server configuration parameter `max_table_num_to_throw`) is set to {}, the current number of tables is {}", table_num_limit, table_count); } diff --git a/tests/integration/test_table_db_num_limit/test.py b/tests/integration/test_table_db_num_limit/test.py index aa8030b077c..56403d165b2 100644 --- a/tests/integration/test_table_db_num_limit/test.py +++ b/tests/integration/test_table_db_num_limit/test.py @@ -4,8 +4,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance( - "node1", main_configs=["config/config.xml"], with_zookeeper=True +node = cluster.add_instance( + "node", main_configs=["config/config.xml"] ) @@ -22,22 +22,26 @@ def started_cluster(): def test_table_db_limit(started_cluster): for i in range(10): - node1.query("create database db{}".format(i)) + node.query("create database db{}".format(i)) with pytest.raises(QueryRuntimeException) as exp_info: - node1.query("create database db_exp".format(i)) + node.query("create database db_exp".format(i)) assert "TOO_MANY_DATABASES" in str(exp_info) for i in range(10): - node1.query("create table t{} (a Int32) Engine = Log".format(i)) + node.query("create table t{} (a Int32) Engine = Log".format(i)) + + # This checks that system tables are not accounted in the number of tables. + node.query("system flush logs") - node1.query("system flush logs") for i in range(10): - node1.query("drop table t{}".format(i)) + node.query("drop table t{}".format(i)) + for i in range(10): - node1.query("create table t{} (a Int32) Engine = Log".format(i)) + node.query("create table t{} (a Int32) Engine = Log".format(i)) with pytest.raises(QueryRuntimeException) as exp_info: - node1.query("create table default.tx (a Int32) Engine = Log") + node.query("create table default.tx (a Int32) Engine = Log") + assert "TOO_MANY_TABLES" in str(exp_info) From efd5ff9f4c63c82d77cfda43af786ad3b7a4b8e0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 21 Jul 2024 09:15:13 +0000 Subject: [PATCH 32/45] Automatic style fix --- tests/integration/test_table_db_num_limit/test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_table_db_num_limit/test.py b/tests/integration/test_table_db_num_limit/test.py index 56403d165b2..a7bb04905f2 100644 --- a/tests/integration/test_table_db_num_limit/test.py +++ b/tests/integration/test_table_db_num_limit/test.py @@ -4,9 +4,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance( - "node", main_configs=["config/config.xml"] -) +node = cluster.add_instance("node", main_configs=["config/config.xml"]) @pytest.fixture(scope="module") From 7ed9ab2338f7de5a2495aad267cc688799c5ff18 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Jul 2024 11:29:12 +0200 Subject: [PATCH 33/45] Remove wrong logic from InterpreterCreateQuery --- src/Interpreters/InterpreterCreateQuery.cpp | 12 ++++++------ tests/integration/test_table_db_num_limit/test.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 2a4a02597fe..7e0b6eb4193 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -146,21 +146,21 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) } auto db_num_limit = getContext()->getGlobalContext()->getServerSettings().max_database_num_to_throw; - if (db_num_limit > 0) + if (db_num_limit > 0 && !internal) { size_t db_count = DatabaseCatalog::instance().getDatabases().size(); - std::vector system_databases = { + std::initializer_list system_databases = + { DatabaseCatalog::TEMPORARY_DATABASE, DatabaseCatalog::SYSTEM_DATABASE, DatabaseCatalog::INFORMATION_SCHEMA, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE, - DatabaseCatalog::DEFAULT_DATABASE }; for (const auto & system_database : system_databases) { - if (db_count > 0 && DatabaseCatalog::instance().isDatabaseExist(system_database)) - db_count--; + if (db_count > 0 && DatabaseCatalog::instance().isDatabaseExist(std::string(system_database))) + --db_count; } if (db_count >= db_num_limit) @@ -1600,7 +1600,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, } UInt64 table_num_limit = getContext()->getGlobalContext()->getServerSettings().max_table_num_to_throw; - if (table_num_limit > 0 && create.getDatabase() != DatabaseCatalog::SYSTEM_DATABASE) + if (table_num_limit > 0 && !internal) { UInt64 table_count = CurrentMetrics::get(CurrentMetrics::AttachedTable); if (table_count >= table_num_limit) diff --git a/tests/integration/test_table_db_num_limit/test.py b/tests/integration/test_table_db_num_limit/test.py index 56403d165b2..aa6404c385c 100644 --- a/tests/integration/test_table_db_num_limit/test.py +++ b/tests/integration/test_table_db_num_limit/test.py @@ -21,7 +21,8 @@ def started_cluster(): def test_table_db_limit(started_cluster): - for i in range(10): + # By the way, default database already exists. + for i in range(9): node.query("create database db{}".format(i)) with pytest.raises(QueryRuntimeException) as exp_info: From 433ac55d13276e7a0b26ded07f27f17726703dab Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Sun, 21 Jul 2024 11:41:42 +0200 Subject: [PATCH 34/45] Correctly handle failure --- src/Common/Allocator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/Allocator.cpp b/src/Common/Allocator.cpp index 7f2241ab4c0..1b43e746c69 100644 --- a/src/Common/Allocator.cpp +++ b/src/Common/Allocator.cpp @@ -206,7 +206,7 @@ void * Allocator::realloc(void * buf, size_t old_size, } else { - [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(old_size); + [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(new_size); ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed); } } @@ -239,7 +239,7 @@ void * Allocator::realloc(void * buf, size_t old_size, void * new_buf = ::realloc(buf, new_size); if (nullptr == new_buf) { - [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(old_size); + [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(new_size); throw DB::ErrnoException( DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Allocator: Cannot realloc from {} to {}", From 0c2c027af63fcbababffbe3a39ed2631884e1938 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Jul 2024 12:30:20 +0200 Subject: [PATCH 35/45] Remove bad tests @azat --- ...2_part_log_rmt_fetch_merge_error.reference | 10 ----- .../03002_part_log_rmt_fetch_merge_error.sql | 35 ---------------- ..._part_log_rmt_fetch_mutate_error.reference | 10 ----- .../03002_part_log_rmt_fetch_mutate_error.sql | 41 ------------------- 4 files changed, 96 deletions(-) delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference deleted file mode 100644 index b19d389d8d0..00000000000 --- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference +++ /dev/null @@ -1,10 +0,0 @@ -before -rmt_master NewPart 0 1 -rmt_master MergeParts 0 1 -rmt_slave MergeParts 1 0 -rmt_slave DownloadPart 0 1 -after -rmt_master NewPart 0 1 -rmt_master MergeParts 0 1 -rmt_slave MergeParts 1 0 -rmt_slave DownloadPart 0 2 diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql deleted file mode 100644 index 548a8e5570a..00000000000 --- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql +++ /dev/null @@ -1,35 +0,0 @@ --- Tags: no-replicated-database, no-parallel, no-shared-merge-tree --- SMT: The merge process is completely different from RMT - -drop table if exists rmt_master; -drop table if exists rmt_slave; - -create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by key settings always_fetch_merged_part=0; --- always_fetch_merged_part=1, consider this table as a "slave" -create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by key settings always_fetch_merged_part=1; - -insert into rmt_master values (1); - -system sync replica rmt_master; -system sync replica rmt_slave; -system stop replicated sends rmt_master; -optimize table rmt_master final settings alter_sync=1, optimize_throw_if_noop=1; - -select sleep(3) format Null; - -system flush logs; -select 'before'; -select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3; - -system start replicated sends rmt_master; --- sleep few seconds to try rmt_slave to fetch the part and reflect this error --- in system.part_log -select sleep(3) format Null; -system sync replica rmt_slave; - -system flush logs; -select 'after'; -select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3; - -drop table rmt_master; -drop table rmt_slave; diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference deleted file mode 100644 index aac9e7527d1..00000000000 --- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference +++ /dev/null @@ -1,10 +0,0 @@ -before -rmt_master NewPart 0 1 -rmt_master MutatePart 0 1 -rmt_slave DownloadPart 0 1 -rmt_slave MutatePart 1 0 -after -rmt_master NewPart 0 1 -rmt_master MutatePart 0 1 -rmt_slave DownloadPart 0 2 -rmt_slave MutatePart 1 0 diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql deleted file mode 100644 index d8b5ebb3148..00000000000 --- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql +++ /dev/null @@ -1,41 +0,0 @@ --- Tags: no-replicated-database, no-parallel, no-shared-merge-tree --- SMT: The merge process is completely different from RMT - -drop table if exists rmt_master; -drop table if exists rmt_slave; - -create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by tuple() settings always_fetch_merged_part=0, old_parts_lifetime=600; --- prefer_fetch_merged_part_*_threshold=0, consider this table as a "slave" -create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by tuple() settings prefer_fetch_merged_part_time_threshold=0, prefer_fetch_merged_part_size_threshold=0, old_parts_lifetime=600; - -insert into rmt_master values (1); - -system sync replica rmt_master; -system sync replica rmt_slave; -system stop replicated sends rmt_master; -system stop pulling replication log rmt_slave; -alter table rmt_master update key=key+100 where 1 settings alter_sync=1; - --- first we need to make the rmt_master execute mutation so that it will have --- the part, and rmt_slave will consider it instead of performing mutation on --- it's own, otherwise prefer_fetch_merged_part_*_threshold will be simply ignored -select sleep(3) format Null; -system start pulling replication log rmt_slave; --- and sleep few more seconds to try rmt_slave to fetch the part and reflect --- this error in system.part_log -select sleep(3) format Null; - -system flush logs; -select 'before'; -select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3; - -system start replicated sends rmt_master; -select sleep(3) format Null; -system sync replica rmt_slave; - -system flush logs; -select 'after'; -select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3; - -drop table rmt_master; -drop table rmt_slave; From dd9fe61d1a973a7fa528259b507218cf264548fe Mon Sep 17 00:00:00 2001 From: Max K Date: Sun, 21 Jul 2024 17:44:32 +0200 Subject: [PATCH 36/45] CI: New Release workflow updates and fixes --- .github/actions/release/action.yml | 27 +++++------- tests/ci/artifactory.py | 56 ++++++++++++++----------- tests/ci/ci_utils.py | 3 +- tests/ci/create_release.py | 66 +++++++++++++++++++----------- 4 files changed, 87 insertions(+), 65 deletions(-) diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml index 99ec02662f6..c3897682a33 100644 --- a/.github/actions/release/action.yml +++ b/.github/actions/release/action.yml @@ -62,8 +62,8 @@ runs: if: ${{ inputs.type == 'patch' }} shell: bash run: | - python3 ./tests/ci/create_release.py --set-progress-started --progress "update ChangeLog" - [ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1 + git checkout master + python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security" echo "List versions" ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv echo "Update docker version" @@ -96,17 +96,13 @@ runs: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }} ### Changelog category (leave one): - Not for changelog (changelog entry is not required) - - name: Reset changes if Dry-run - if: ${{ inputs.dry-run }} + - name: Complete previous steps and Restore git state + if: ${{ inputs.type == 'patch' }} shell: bash run: | - git reset --hard HEAD - - name: Checkout back to GITHUB_REF - shell: bash - run: | - git checkout "$GITHUB_REF_NAME" - # set current progress to OK python3 ./tests/ci/create_release.py --set-progress-completed + git reset --hard HEAD + git checkout "$GITHUB_REF_NAME" - name: Create GH Release shell: bash if: ${{ inputs.type == 'patch' }} @@ -146,24 +142,23 @@ runs: if: ${{ inputs.type == 'patch' }} shell: bash run: | - python3 ./tests/ci/create_release.py --set-progress-started --progress "docker server release" cd "./tests/ci" + python3 ./create_release.py --set-progress-started --progress "docker server release" export CHECK_NAME="Docker server image" python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} - python3 ./tests/ci/create_release.py --set-progress-completed + python3 ./create_release.py --set-progress-completed - name: Docker clickhouse/clickhouse-keeper building if: ${{ inputs.type == 'patch' }} shell: bash run: | - python3 ./tests/ci/create_release.py --set-progress-started --progress "docker keeper release" cd "./tests/ci" + python3 ./create_release.py --set-progress-started --progress "docker keeper release" export CHECK_NAME="Docker keeper image" python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} - python3 ./tests/ci/create_release.py --set-progress-completed - - name: Set Release progress completed + python3 ./create_release.py --set-progress-completed + - name: Set current Release progress to Completed with OK shell: bash run: | - # If we here - set completed status, to post proper Slack OK or FAIL message in the next step python3 ./tests/ci/create_release.py --set-progress-started --progress "completed" python3 ./tests/ci/create_release.py --set-progress-completed - name: Post Slack Message diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py index 98a0345c6bd..86dcaf79854 100644 --- a/tests/ci/artifactory.py +++ b/tests/ci/artifactory.py @@ -43,7 +43,6 @@ class R2MountPoint: self.bucket_name = self._PROD_BUCKET_NAME self.aux_mount_options = "" - self.async_mount = False if self.app == MountPointApp.S3FS: self.cache_dir = "/home/ubuntu/s3fs_cache" # self.aux_mount_options += "-o nomodtime " if self.NOMODTIME else "" not for s3fs @@ -57,7 +56,6 @@ class R2MountPoint: self.mount_cmd = f"s3fs {self.bucket_name} {self.MOUNT_POINT} -o url={self.API_ENDPOINT} -o use_path_request_style -o umask=0000 -o nomultipart -o logfile={self.LOG_FILE} {self.aux_mount_options}" elif self.app == MountPointApp.RCLONE: # run rclone mount process asynchronously, otherwise subprocess.run(daemonized command) will not return - self.async_mount = True self.cache_dir = "/home/ubuntu/rclone_cache" self.aux_mount_options += "--no-modtime " if self.NOMODTIME else "" self.aux_mount_options += "-v " if self.DEBUG else "" # -vv too verbose @@ -85,10 +83,12 @@ class R2MountPoint: Shell.run(_UNMOUNT_CMD) Shell.run(_MKDIR_CMD) Shell.run(_MKDIR_FOR_CACHE) - # didn't manage to use simple run() and not block or fail - Shell.run_as_daemon(self.mount_cmd) - if self.async_mount: - time.sleep(3) + if self.app == MountPointApp.S3FS: + Shell.run(self.mount_cmd, check=True) + else: + # didn't manage to use simple run() and without blocking or failure + Shell.run_as_daemon(self.mount_cmd) + time.sleep(3) Shell.run(_TEST_MOUNT_CMD, check=True) @classmethod @@ -107,6 +107,7 @@ class DebianArtifactory: _PROD_REPO_URL = "https://packages.clickhouse.com/deb" def __init__(self, release_info: ReleaseInfo, dry_run: bool): + self.release_info = release_info self.codename = release_info.codename self.version = release_info.version if dry_run: @@ -154,9 +155,8 @@ class DebianArtifactory: print("Running test command:") print(f" {cmd}") Shell.run(cmd, check=True) - release_info = ReleaseInfo.from_file() - release_info.debian_command = debian_command - release_info.dump() + self.release_info.debian_command = debian_command + self.release_info.dump() def _copy_if_not_exists(src: Path, dst: Path) -> Path: @@ -177,6 +177,7 @@ class RpmArtifactory: _SIGN_KEY = "885E2BDCF96B0B45ABF058453E4AD4719DDE9A38" def __init__(self, release_info: ReleaseInfo, dry_run: bool): + self.release_info = release_info self.codename = release_info.codename self.version = release_info.version if dry_run: @@ -230,9 +231,8 @@ class RpmArtifactory: print("Running test command:") print(f" {cmd}") Shell.run(cmd, check=True) - release_info = ReleaseInfo.from_file() - release_info.rpm_command = rpm_command - release_info.dump() + self.release_info.rpm_command = rpm_command + self.release_info.dump() class TgzArtifactory: @@ -240,6 +240,7 @@ class TgzArtifactory: _PROD_REPO_URL = "https://packages.clickhouse.com/tgz" def __init__(self, release_info: ReleaseInfo, dry_run: bool): + self.release_info = release_info self.codename = release_info.codename self.version = release_info.version if dry_run: @@ -290,9 +291,8 @@ class TgzArtifactory: expected_checksum == actual_checksum ), f"[{actual_checksum} != {expected_checksum}]" Shell.run("rm /tmp/tmp.tgz*") - release_info = ReleaseInfo.from_file() - release_info.tgz_command = cmd - release_info.dump() + self.release_info.tgz_command = cmd + self.release_info.dump() def parse_args() -> argparse.Namespace: @@ -340,9 +340,7 @@ def parse_args() -> argparse.Namespace: if __name__ == "__main__": args = parse_args() - assert args.dry_run - release_info = ReleaseInfo.from_file() """ Use S3FS. RCLONE has some errors with r2 remote which I didn't figure out how to resolve: ERROR : IO error: NotImplemented: versionId not implemented @@ -350,26 +348,38 @@ if __name__ == "__main__": """ mp = R2MountPoint(MountPointApp.S3FS, dry_run=args.dry_run) if args.export_debian: - with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_DEB) as _: + with ReleaseContextManager( + release_progress=ReleaseProgress.EXPORT_DEB + ) as release_info: mp.init() DebianArtifactory(release_info, dry_run=args.dry_run).export_packages() mp.teardown() if args.export_rpm: - with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_RPM) as _: + with ReleaseContextManager( + release_progress=ReleaseProgress.EXPORT_RPM + ) as release_info: mp.init() RpmArtifactory(release_info, dry_run=args.dry_run).export_packages() mp.teardown() if args.export_tgz: - with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_TGZ) as _: + with ReleaseContextManager( + release_progress=ReleaseProgress.EXPORT_TGZ + ) as release_info: mp.init() TgzArtifactory(release_info, dry_run=args.dry_run).export_packages() mp.teardown() if args.test_debian: - with ReleaseContextManager(release_progress=ReleaseProgress.TEST_DEB) as _: + with ReleaseContextManager( + release_progress=ReleaseProgress.TEST_DEB + ) as release_info: DebianArtifactory(release_info, dry_run=args.dry_run).test_packages() if args.test_tgz: - with ReleaseContextManager(release_progress=ReleaseProgress.TEST_TGZ) as _: + with ReleaseContextManager( + release_progress=ReleaseProgress.TEST_TGZ + ) as release_info: TgzArtifactory(release_info, dry_run=args.dry_run).test_packages() if args.test_rpm: - with ReleaseContextManager(release_progress=ReleaseProgress.TEST_RPM) as _: + with ReleaseContextManager( + release_progress=ReleaseProgress.TEST_RPM + ) as release_info: RpmArtifactory(release_info, dry_run=args.dry_run).test_packages() diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py index efbf014cd52..9a1b12af310 100644 --- a/tests/ci/ci_utils.py +++ b/tests/ci/ci_utils.py @@ -182,10 +182,11 @@ class Shell: check=False, ) if result.returncode == 0: + print(f"stdout: {result.stdout.strip()}") res = result.stdout else: print( - f"ERROR: stdout {result.stdout.strip()}, stderr {result.stderr.strip()}" + f"ERROR: stdout: {result.stdout.strip()}, stderr: {result.stderr.strip()}" ) if check: assert result.returncode == 0 diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py index 4347cfebb54..a0b4083b673 100755 --- a/tests/ci/create_release.py +++ b/tests/ci/create_release.py @@ -43,6 +43,7 @@ class ReleaseProgress: TEST_TGZ = "test TGZ packages" TEST_RPM = "test RPM packages" TEST_DEB = "test DEB packages" + COMPLETED = "completed" class ReleaseProgressDescription: @@ -108,6 +109,12 @@ class ReleaseInfo: release_progress: str = "" progress_description: str = "" + def is_patch(self): + return self.release_branch != "master" + + def is_new_release_branch(self): + return self.release_branch == "master" + @staticmethod def from_file() -> "ReleaseInfo": with open(RELEASE_INFO_FILE, "r", encoding="utf-8") as json_file: @@ -126,12 +133,12 @@ class ReleaseInfo: release_tag = None previous_release_tag = None previous_release_sha = None - codename = None + codename = "" assert release_type in ("patch", "new") if release_type == "new": # check commit_ref is right and on a right branch Shell.run( - f"git merge-base --is-ancestor origin/{commit_ref} origin/master", + f"git merge-base --is-ancestor {commit_ref} origin/master", check=True, ) with checkout(commit_ref): @@ -146,9 +153,6 @@ class ReleaseInfo: git.latest_tag == expected_prev_tag ), f"BUG: latest tag [{git.latest_tag}], expected [{expected_prev_tag}]" release_tag = version.describe - codename = ( - VersionType.STABLE - ) # dummy value (artifactory won't be updated for new release) previous_release_tag = expected_prev_tag previous_release_sha = Shell.run_strict( f"git rev-parse {previous_release_tag}" @@ -205,7 +209,7 @@ class ReleaseInfo: and commit_sha and release_tag and version - and codename in ("lts", "stable") + and (codename in ("lts", "stable") or release_type == "new") ) self.release_branch = release_branch @@ -320,24 +324,27 @@ class ReleaseInfo: Shell.run( f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'" ) - self.version_bump_pr = GHActions.get_pr_url_by_branch( - repo=GITHUB_REPOSITORY, branch=branch_upd_version_contributors - ) + self.version_bump_pr = "dry-run" + else: + self.version_bump_pr = GHActions.get_pr_url_by_branch( + repo=GITHUB_REPOSITORY, branch=branch_upd_version_contributors + ) def update_release_info(self, dry_run: bool) -> "ReleaseInfo": - branch = f"auto/{release_info.release_tag}" - if not dry_run: - url = GHActions.get_pr_url_by_branch(repo=GITHUB_REPOSITORY, branch=branch) - else: - url = "dry-run" - - print(f"ChangeLog PR url [{url}]") - self.changelog_pr = url - print(f"Release url [{url}]") - self.release_url = ( - f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}" - ) - self.docker_command = f"docker run --rm clickhouse/clickhouse:{self.release_branch} clickhouse --version" + if self.release_branch != "master": + branch = f"auto/{release_info.release_tag}" + if not dry_run: + url = GHActions.get_pr_url_by_branch( + repo=GITHUB_REPOSITORY, branch=branch + ) + else: + url = "dry-run" + print(f"ChangeLog PR url [{url}]") + self.changelog_pr = url + print(f"Release url [{url}]") + self.release_url = f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}" + if self.release_progress == ReleaseProgress.COMPLETED: + self.docker_command = f"docker run --rm clickhouse/clickhouse:{self.version} clickhouse --version" self.dump() return self @@ -712,13 +719,22 @@ if __name__ == "__main__": if args.post_status: release_info = ReleaseInfo.from_file() release_info.update_release_info(dry_run=args.dry_run) - if release_info.debian_command: + if release_info.is_new_release_branch(): + title = "New release branch" + else: + title = "New release" + if ( + release_info.progress_description == ReleaseProgressDescription.OK + and release_info.release_progress == ReleaseProgress.COMPLETED + ): + title = "Completed: " + title CIBuddy(dry_run=args.dry_run).post_done( - f"New release issued", dataclasses.asdict(release_info) + title, dataclasses.asdict(release_info) ) else: + title = "Failed: " + title CIBuddy(dry_run=args.dry_run).post_critical( - f"Failed to issue new release", dataclasses.asdict(release_info) + title, dataclasses.asdict(release_info) ) if args.set_progress_started: From 989476d5234bffd223988bf8aa88e2021e999574 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Jul 2024 06:41:52 +0200 Subject: [PATCH 37/45] Make test `01592_long_window_functions1` lighter --- .../0_stateless/01592_long_window_functions1.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01592_long_window_functions1.sql b/tests/queries/0_stateless/01592_long_window_functions1.sql index d2d32e24eaa..671245599cc 100644 --- a/tests/queries/0_stateless/01592_long_window_functions1.sql +++ b/tests/queries/0_stateless/01592_long_window_functions1.sql @@ -8,14 +8,14 @@ drop table if exists stack; set max_insert_threads = 4; create table stack(item_id Int64, brand_id Int64, rack_id Int64, dt DateTime, expiration_dt DateTime, quantity UInt64) -Engine = MergeTree -partition by toYYYYMM(dt) +Engine = MergeTree +partition by toYYYYMM(dt) order by (brand_id, toStartOfHour(dt)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; -insert into stack -select number%99991, number%11, number%1111, toDateTime('2020-01-01 00:00:00')+number/100, +insert into stack +select number%99991, number%11, number%1111, toDateTime('2020-01-01 00:00:00')+number/100, toDateTime('2020-02-01 00:00:00')+number/10, intDiv(number,100)+1 -from numbers_mt(10000000); +from numbers_mt(1000000); select '---- arrays ----'; @@ -32,8 +32,8 @@ select '---- window f ----'; select cityHash64( toString( groupArray (tuple(*) ) )) from ( select brand_id, rack_id, quantity from ( select brand_id, rack_id, quantity, row_number() over (partition by brand_id, rack_id order by quantity) rn - from stack ) as t0 - where rn <= 2 + from stack ) as t0 + where rn <= 2 order by brand_id, rack_id, quantity ) t; From ed02246e0c2019f9ee661e9636f166b955a672de Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Jul 2024 08:46:41 +0200 Subject: [PATCH 38/45] What if I will change this test? --- tests/integration/test_ssl_cert_authentication/test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 756a1e1996c..3af88759e82 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -43,15 +43,10 @@ def started_cluster(): config = """ - none - + strict {certificateFile} {privateKeyFile} {caConfig} - - - AcceptCertificateHandler - """ From 3d4604a64ad01c70c80924ea11154514bab4e424 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Jul 2024 08:47:08 +0200 Subject: [PATCH 39/45] Revert "What if I will change this test?" This reverts commit ed02246e0c2019f9ee661e9636f166b955a672de. --- tests/integration/test_ssl_cert_authentication/test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 3af88759e82..756a1e1996c 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -43,10 +43,15 @@ def started_cluster(): config = """ - strict + none + {certificateFile} {privateKeyFile} {caConfig} + + + AcceptCertificateHandler + """ From 1a3559fbc3c5257c4e0f5ec16eda3d09d8ebcca0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Jul 2024 08:46:41 +0200 Subject: [PATCH 40/45] What if I will change this test? --- tests/integration/test_ssl_cert_authentication/test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 756a1e1996c..3af88759e82 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -43,15 +43,10 @@ def started_cluster(): config = """ - none - + strict {certificateFile} {privateKeyFile} {caConfig} - - - AcceptCertificateHandler - """ From 0f327869132940b3bae932730cb2ce2f9c394163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 22 Jul 2024 10:46:42 +0200 Subject: [PATCH 41/45] Better random and queries --- tests/queries/0_stateless/01194_http_query_id.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01194_http_query_id.sh b/tests/queries/0_stateless/01194_http_query_id.sh index fac17cca3c6..42321112185 100755 --- a/tests/queries/0_stateless/01194_http_query_id.sh +++ b/tests/queries/0_stateless/01194_http_query_id.sh @@ -4,14 +4,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -rnd=$RANDOM -url="${CLICKHOUSE_URL}&session_id=test_01194_$RANDOM" +rnd="$CLICKHOUSE_DATABASE" +url="${CLICKHOUSE_URL}&session_id=test_01194_${CLICKHOUSE_DATABASE}" ${CLICKHOUSE_CURL} -sS "$url&query=SELECT+'test_01194',$rnd,1" > /dev/null ${CLICKHOUSE_CURL} -sS "$url&query=SELECT+'test_01194',$rnd,2" > /dev/null ${CLICKHOUSE_CURL} -sS "$url" --data "SELECT 'test_01194',$rnd,3" > /dev/null ${CLICKHOUSE_CURL} -sS "$url" --data "SELECT 'test_01194',$rnd,4" > /dev/null -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data "SYSTEM FLUSH LOGS" +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data "SELECT count(DISTINCT query_id) FROM system.query_log WHERE current_database = currentDatabase() AND event_date >= yesterday() AND query LIKE 'SELECT ''test_01194'',$rnd%'" +$CLICKHOUSE_CLIENT -q " + SELECT + count(DISTINCT query_id) + FROM system.query_log + WHERE + current_database = currentDatabase() + AND event_date >= yesterday() + AND query LIKE 'SELECT ''test_01194'',$rnd%' + AND query_id != queryID()" From 255dcec501e7506291cee44f5c9eb30b5eec7e99 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 22 Jul 2024 11:10:34 +0200 Subject: [PATCH 42/45] Fix 02241_join_rocksdb_bs --- .../02241_join_rocksdb_bs.reference | 56 ------------------- .../0_stateless/02241_join_rocksdb_bs.sql.j2 | 20 +++---- 2 files changed, 7 insertions(+), 69 deletions(-) diff --git a/tests/queries/0_stateless/02241_join_rocksdb_bs.reference b/tests/queries/0_stateless/02241_join_rocksdb_bs.reference index 8416a2991c1..4dff9ef38ef 100644 --- a/tests/queries/0_stateless/02241_join_rocksdb_bs.reference +++ b/tests/queries/0_stateless/02241_join_rocksdb_bs.reference @@ -10,59 +10,3 @@ 1 1 1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 diff --git a/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 b/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 index 4ab98201eed..e5703f99d62 100644 --- a/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 +++ b/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 @@ -1,4 +1,4 @@ --- Tags: use-rocksdb, long, no-s3-storage, no-random-settings, no-random-merge-tree-settings +-- Tags: use-rocksdb, long, no-s3-storage SET join_algorithm = 'direct'; @@ -13,27 +13,21 @@ INSERT INTO rdb_{{ table_size }} SELECT (sipHash64(number) % {{ table_size }}) as key, ('val' || toString(key)) AS value FROM numbers_mt({{ table_size }}); -{% for block_size in [10, 11, 128, 129, 65505, 65506, 70000] -%} - -{% if block_size * 5000 > table_size -%} - -SET max_block_size = {{ block_size }}; - {% for right_size in [table_size // 2, table_size + table_size // 4 + 1] -%} SELECT count() == (SELECT count() FROM rdb_{{ table_size }} WHERE key < {{ right_size }}) FROM (SELECT number as k FROM numbers_mt({{ right_size }})) as t1 INNER JOIN rdb_{{ table_size }} as rdb -ON rdb.key == t1.k; +ON rdb.key == t1.k +{% if table_size < 100 %}SETTINGS max_block_size = 1{% endif -%} +; SELECT count() == {{ right_size }} and countIf(value != '') == (SELECT count() FROM rdb_{{ table_size }} WHERE key < {{ right_size }}) FROM (SELECT number as k FROM numbers_mt({{ right_size }})) as t1 LEFT JOIN rdb_{{ table_size }} as rdb -ON rdb.key == t1.k; - -{% endfor -%} - -{% endif -%} +ON rdb.key == t1.k +{% if table_size < 100 %}SETTINGS max_block_size = 1{% endif -%} +; {% endfor -%} {% endfor -%} From e708219f6aecfe9934827f453665e03142ad5112 Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 22 Jul 2024 13:01:27 +0200 Subject: [PATCH 43/45] CI: Print instance info in runner's init script --- tests/ci/worker/init_runner.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index d6cdb6d9c57..1bfeeb38c15 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -50,7 +50,7 @@ set -uo pipefail # set accordingly to a runner role # #################################### -echo "Running init v1" +echo "Running init v1.1" export DEBIAN_FRONTEND=noninteractive export RUNNER_HOME=/home/ubuntu/actions-runner @@ -66,6 +66,11 @@ bash /usr/local/share/scripts/init-network.sh RUNNER_TYPE=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='github:runner-type'].Value" --output text) LABELS="self-hosted,Linux,$(uname -m),$RUNNER_TYPE" export LABELS +echo "Instance Labels: $LABELS" + +LIFE_CYCLE=$(curl -s --fail http://169.254.169.254/latest/meta-data/instance-life-cycle) +export LIFE_CYCLE +echo "Instance lifecycle: $LIFE_CYCLE" # Refresh CloudWatch agent config aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json @@ -124,10 +129,6 @@ terminate_decrease_and_exit() { declare -f terminate_and_exit >> /tmp/actions-hooks/common.sh check_spot_instance_is_old() { - # This function should be executed ONLY BETWEEN runnings. - # It's unsafe to execute while the runner is working! - local LIFE_CYCLE - LIFE_CYCLE=$(curl -s --fail http://169.254.169.254/latest/meta-data/instance-life-cycle) if [ "$LIFE_CYCLE" == "spot" ]; then local UPTIME UPTIME=$(< /proc/uptime) From cd700c59b1d0b894a7a8461a33d994732b7864f0 Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 22 Jul 2024 13:17:25 +0200 Subject: [PATCH 44/45] minor fix --- tests/ci/auto_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/auto_release.py b/tests/ci/auto_release.py index 39ab3156c80..f2386fe207f 100644 --- a/tests/ci/auto_release.py +++ b/tests/ci/auto_release.py @@ -191,7 +191,7 @@ def main(): title=f"Auto Release Status for {release_info.release_branch}", body=release_info.to_dict(), ) - if args.post_auto_release_complete: + elif args.post_auto_release_complete: assert args.wf_status, "--wf-status Required with --post-auto-release-complete" if args.wf_status != SUCCESS: CIBuddy(dry_run=False).post_job_error( From 69ad8feb9078421aca99709ffb839e2f3b923427 Mon Sep 17 00:00:00 2001 From: Max K Date: Mon, 22 Jul 2024 13:38:10 +0200 Subject: [PATCH 45/45] add instance type --- tests/ci/worker/init_runner.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index 1bfeeb38c15..5177e112edd 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -72,6 +72,9 @@ LIFE_CYCLE=$(curl -s --fail http://169.254.169.254/latest/meta-data/instance-lif export LIFE_CYCLE echo "Instance lifecycle: $LIFE_CYCLE" +INSTANCE_TYPE=$(ec2metadata --instance-type) +echo "Instance type: $INSTANCE_TYPE" + # Refresh CloudWatch agent config aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json systemctl restart amazon-cloudwatch-agent.service