2018-11-02 18:53:23 +00:00
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include <Core/Names.h>
|
|
|
|
#include <Core/NamesAndTypes.h>
|
2020-07-16 22:01:08 +00:00
|
|
|
#include <Core/SettingsEnums.h>
|
2019-09-02 19:58:45 +00:00
|
|
|
#include <Parsers/ASTTablesInSelectQuery.h>
|
2019-09-09 19:43:37 +00:00
|
|
|
#include <Interpreters/IJoin.h>
|
2021-02-09 13:17:42 +00:00
|
|
|
#include <Interpreters/join_common.h>
|
2019-10-11 17:56:26 +00:00
|
|
|
#include <Interpreters/asof.h>
|
2021-10-15 20:18:20 +00:00
|
|
|
#include <QueryPipeline/SizeLimits.h>
|
2021-02-09 13:17:42 +00:00
|
|
|
#include <DataTypes/getLeastSupertype.h>
|
2020-04-08 18:59:52 +00:00
|
|
|
#include <Storages/IStorage_fwd.h>
|
2021-09-09 09:47:08 +00:00
|
|
|
#include <Common/Exception.h>
|
2021-09-15 11:16:10 +00:00
|
|
|
#include <Parsers/IAST_fwd.h>
|
|
|
|
|
|
|
|
#include <cstddef>
|
|
|
|
#include <unordered_map>
|
2018-11-02 18:53:23 +00:00
|
|
|
|
|
|
|
#include <utility>
|
|
|
|
#include <memory>
|
2021-10-02 07:13:14 +00:00
|
|
|
#include <base/types.h>
|
|
|
|
#include <base/logger_useful.h>
|
2021-06-11 14:47:30 +00:00
|
|
|
|
2018-11-02 18:53:23 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
class Context;
|
|
|
|
class ASTSelectQuery;
|
2019-02-13 19:00:52 +00:00
|
|
|
struct DatabaseAndTableWithAlias;
|
2019-09-02 19:58:45 +00:00
|
|
|
class Block;
|
2020-04-08 18:59:52 +00:00
|
|
|
class DictionaryReader;
|
2021-06-29 09:22:53 +00:00
|
|
|
class StorageJoin;
|
|
|
|
class StorageDictionary;
|
2020-09-08 11:06:36 +00:00
|
|
|
|
2020-09-08 11:07:26 +00:00
|
|
|
struct ColumnWithTypeAndName;
|
2020-09-08 11:06:36 +00:00
|
|
|
using ColumnsWithTypeAndName = std::vector<ColumnWithTypeAndName>;
|
2019-09-02 19:58:45 +00:00
|
|
|
|
2019-09-09 19:43:37 +00:00
|
|
|
struct Settings;
|
2018-11-02 18:53:23 +00:00
|
|
|
|
2020-07-08 14:25:23 +00:00
|
|
|
class IVolume;
|
|
|
|
using VolumePtr = std::shared_ptr<IVolume>;
|
2020-01-19 14:26:28 +00:00
|
|
|
|
2021-07-21 17:03:33 +00:00
|
|
|
enum class JoinTableSide
|
|
|
|
{
|
|
|
|
Left,
|
|
|
|
Right
|
|
|
|
};
|
|
|
|
|
2020-04-07 09:48:47 +00:00
|
|
|
class TableJoin
|
2018-11-02 18:53:23 +00:00
|
|
|
{
|
2021-02-09 13:17:42 +00:00
|
|
|
|
|
|
|
public:
|
|
|
|
using NameToTypeMap = std::unordered_map<String, DataTypePtr>;
|
|
|
|
|
2021-09-06 10:59:18 +00:00
|
|
|
/// Corresponds to one disjunct
|
|
|
|
struct JoinOnClause
|
|
|
|
{
|
|
|
|
Names key_names_left;
|
|
|
|
Names key_names_right; /// Duplicating right key names are qualified.
|
|
|
|
|
|
|
|
ASTPtr on_filter_condition_left;
|
|
|
|
ASTPtr on_filter_condition_right;
|
|
|
|
|
|
|
|
JoinOnClause() = default;
|
|
|
|
|
|
|
|
std::pair<String, String> condColumnNames() const
|
|
|
|
{
|
|
|
|
std::pair<String, String> res;
|
|
|
|
if (on_filter_condition_left)
|
|
|
|
res.first = on_filter_condition_left->getColumnName();
|
|
|
|
if (on_filter_condition_right)
|
|
|
|
res.second = on_filter_condition_right->getColumnName();
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t keysCount() const
|
|
|
|
{
|
|
|
|
assert(key_names_left.size() == key_names_right.size());
|
|
|
|
return key_names_right.size();
|
|
|
|
}
|
2021-09-09 09:47:08 +00:00
|
|
|
|
|
|
|
String formatDebug() const
|
|
|
|
{
|
|
|
|
return fmt::format("Left keys: [{}] Right keys [{}] Condition columns: '{}', '{}'",
|
|
|
|
fmt::join(key_names_left, ", "), fmt::join(key_names_right, ", "),
|
|
|
|
condColumnNames().first, condColumnNames().second);
|
|
|
|
}
|
2021-09-06 10:59:18 +00:00
|
|
|
};
|
|
|
|
|
2021-09-15 11:16:10 +00:00
|
|
|
using Clauses = std::vector<JoinOnClause>;
|
|
|
|
|
2021-02-09 13:17:42 +00:00
|
|
|
private:
|
2018-11-02 18:53:23 +00:00
|
|
|
/** Query of the form `SELECT expr(x) AS k FROM t1 ANY LEFT JOIN (SELECT expr(x) AS k FROM t2) USING k`
|
|
|
|
* The join is made by column k.
|
|
|
|
* During the JOIN,
|
|
|
|
* - in the "right" table, it will be available by alias `k`, since `Project` action for the subquery was executed.
|
|
|
|
* - in the "left" table, it will be accessible by the name `expr(x)`, since `Project` action has not been executed yet.
|
|
|
|
* You must remember both of these options.
|
|
|
|
*
|
|
|
|
* Query of the form `SELECT ... from t1 ANY LEFT JOIN (SELECT ... from t2) ON expr(t1 columns) = expr(t2 columns)`
|
|
|
|
* to the subquery will be added expression `expr(t2 columns)`.
|
|
|
|
* It's possible to use name `expr(t2 columns)`.
|
|
|
|
*/
|
2019-07-30 18:39:37 +00:00
|
|
|
|
2020-07-22 17:13:05 +00:00
|
|
|
friend class TreeRewriter;
|
2019-07-30 18:39:37 +00:00
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
const SizeLimits size_limits;
|
|
|
|
const size_t default_max_bytes = 0;
|
|
|
|
const bool join_use_nulls = false;
|
|
|
|
const size_t max_joined_block_rows = 0;
|
|
|
|
JoinAlgorithm join_algorithm = JoinAlgorithm::AUTO;
|
|
|
|
const bool partial_merge_join_optimizations = false;
|
|
|
|
const size_t partial_merge_join_rows_in_right_blocks = 0;
|
|
|
|
const size_t partial_merge_join_left_table_buffer_bytes = 0;
|
|
|
|
const size_t max_files_to_merge = 0;
|
|
|
|
const String temporary_files_codec = "LZ4";
|
2019-09-09 19:43:37 +00:00
|
|
|
|
2021-09-19 21:41:11 +00:00
|
|
|
/// the limit has no technical reasons, it supposed to improve safety
|
|
|
|
const size_t MAX_DISJUNCTS = 16;
|
|
|
|
|
2021-09-06 10:59:18 +00:00
|
|
|
ASTs key_asts_left;
|
|
|
|
ASTs key_asts_right;
|
2021-09-02 11:40:04 +00:00
|
|
|
|
2021-09-15 11:16:10 +00:00
|
|
|
Clauses clauses;
|
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
ASTTableJoin table_join;
|
2021-03-05 14:41:39 +00:00
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
ASOF::Inequality asof_inequality = ASOF::Inequality::GreaterOrEquals;
|
2018-11-02 18:53:23 +00:00
|
|
|
|
|
|
|
/// All columns which can be read from joined table. Duplicating names are qualified.
|
2019-07-30 18:39:37 +00:00
|
|
|
NamesAndTypesList columns_from_joined_table;
|
2021-02-09 13:17:42 +00:00
|
|
|
/// Columns will be added to block by JOIN.
|
2021-06-28 13:44:19 +00:00
|
|
|
/// It's a subset of columns_from_joined_table
|
|
|
|
/// Note: without corrected Nullability or type, see correctedColumnsAddedByJoin
|
2019-09-02 19:58:45 +00:00
|
|
|
NamesAndTypesList columns_added_by_join;
|
2021-02-09 13:17:42 +00:00
|
|
|
|
|
|
|
/// Target type to convert key columns before join
|
|
|
|
NameToTypeMap left_type_map;
|
|
|
|
NameToTypeMap right_type_map;
|
2019-09-02 19:58:45 +00:00
|
|
|
|
2019-07-30 18:39:37 +00:00
|
|
|
/// Name -> original name. Names are the same as in columns_from_joined_table list.
|
|
|
|
std::unordered_map<String, String> original_names;
|
2021-02-01 13:53:54 +00:00
|
|
|
/// Original name -> name. Only renamed columns.
|
2019-07-30 18:39:37 +00:00
|
|
|
std::unordered_map<String, String> renames;
|
|
|
|
|
2020-07-08 14:25:23 +00:00
|
|
|
VolumePtr tmp_volume;
|
2019-10-15 16:31:49 +00:00
|
|
|
|
2021-06-29 09:22:53 +00:00
|
|
|
std::shared_ptr<StorageJoin> right_storage_join;
|
|
|
|
|
|
|
|
std::shared_ptr<StorageDictionary> right_storage_dictionary;
|
|
|
|
std::shared_ptr<DictionaryReader> dictionary_reader;
|
|
|
|
|
2021-02-18 11:49:32 +00:00
|
|
|
Names requiredJoinedNames() const;
|
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
/// Create converting actions and change key column names if required
|
|
|
|
ActionsDAGPtr applyKeyConvertToTable(
|
2021-09-06 10:59:18 +00:00
|
|
|
const ColumnsWithTypeAndName & cols_src, const NameToTypeMap & type_mapping, NameToNameMap & key_column_rename) const;
|
2021-03-05 14:34:43 +00:00
|
|
|
|
2021-09-10 14:52:44 +00:00
|
|
|
void addKey(const String & left_name, const String & right_name, const ASTPtr & left_ast, const ASTPtr & right_ast = nullptr);
|
2021-09-06 10:59:18 +00:00
|
|
|
|
2021-09-09 09:47:08 +00:00
|
|
|
void assertHasOneOnExpr() const;
|
2021-06-25 12:03:10 +00:00
|
|
|
|
2021-09-15 11:16:10 +00:00
|
|
|
/// Calculates common supertypes for corresponding join key columns.
|
|
|
|
template <typename LeftNamesAndTypes, typename RightNamesAndTypes>
|
|
|
|
bool inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right);
|
|
|
|
|
|
|
|
NamesAndTypesList correctedColumnsAddedByJoin() const;
|
|
|
|
|
2021-09-06 10:59:18 +00:00
|
|
|
public:
|
2021-09-24 16:21:05 +00:00
|
|
|
TableJoin() = default;
|
2021-09-06 10:59:18 +00:00
|
|
|
|
2021-06-25 12:03:10 +00:00
|
|
|
TableJoin(const Settings & settings, VolumePtr tmp_volume_);
|
2021-03-05 14:34:43 +00:00
|
|
|
|
|
|
|
/// for StorageJoin
|
|
|
|
TableJoin(SizeLimits limits, bool use_nulls, ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness,
|
2021-09-02 11:40:04 +00:00
|
|
|
const Names & key_names_right)
|
2021-03-05 14:34:43 +00:00
|
|
|
: size_limits(limits)
|
|
|
|
, default_max_bytes(0)
|
|
|
|
, join_use_nulls(use_nulls)
|
|
|
|
, join_algorithm(JoinAlgorithm::HASH)
|
|
|
|
{
|
2021-09-10 14:52:44 +00:00
|
|
|
clauses.emplace_back().key_names_right = key_names_right;
|
2021-03-05 14:34:43 +00:00
|
|
|
table_join.kind = kind;
|
|
|
|
table_join.strictness = strictness;
|
|
|
|
}
|
2019-09-09 19:43:37 +00:00
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
ASTTableJoin::Kind kind() const { return table_join.kind; }
|
|
|
|
ASTTableJoin::Strictness strictness() const { return table_join.strictness; }
|
2020-02-02 16:12:48 +00:00
|
|
|
bool sameStrictnessAndKind(ASTTableJoin::Strictness, ASTTableJoin::Kind) const;
|
2021-03-05 14:34:43 +00:00
|
|
|
const SizeLimits & sizeLimits() const { return size_limits; }
|
|
|
|
VolumePtr getTemporaryVolume() { return tmp_volume; }
|
|
|
|
bool allowMergeJoin() const;
|
|
|
|
bool preferMergeJoin() const { return join_algorithm == JoinAlgorithm::PREFER_PARTIAL_MERGE; }
|
|
|
|
bool forceMergeJoin() const { return join_algorithm == JoinAlgorithm::PARTIAL_MERGE; }
|
2021-04-20 09:52:52 +00:00
|
|
|
bool forceHashJoin() const
|
|
|
|
{
|
|
|
|
/// HashJoin always used for DictJoin
|
|
|
|
return dictionary_reader || join_algorithm == JoinAlgorithm::HASH;
|
|
|
|
}
|
2021-03-05 14:34:43 +00:00
|
|
|
|
|
|
|
bool forceNullableRight() const { return join_use_nulls && isLeftOrFull(table_join.kind); }
|
|
|
|
bool forceNullableLeft() const { return join_use_nulls && isRightOrFull(table_join.kind); }
|
|
|
|
size_t defaultMaxBytes() const { return default_max_bytes; }
|
|
|
|
size_t maxJoinedBlockRows() const { return max_joined_block_rows; }
|
|
|
|
size_t maxRowsInRightBlock() const { return partial_merge_join_rows_in_right_blocks; }
|
|
|
|
size_t maxBytesInLeftBuffer() const { return partial_merge_join_left_table_buffer_bytes; }
|
|
|
|
size_t maxFilesToMerge() const { return max_files_to_merge; }
|
|
|
|
const String & temporaryFilesCodec() const { return temporary_files_codec; }
|
|
|
|
bool enablePartialMergeJoinOptimizations() const { return partial_merge_join_optimizations; }
|
2020-07-10 18:10:06 +00:00
|
|
|
bool needStreamWithNonJoinedRows() const;
|
2019-09-09 19:43:37 +00:00
|
|
|
|
2021-09-06 10:59:18 +00:00
|
|
|
bool oneDisjunct() const;
|
|
|
|
|
|
|
|
JoinOnClause & getOnlyClause() { assertHasOneOnExpr(); return clauses[0]; }
|
|
|
|
const JoinOnClause & getOnlyClause() const { assertHasOneOnExpr(); return clauses[0]; }
|
|
|
|
|
|
|
|
std::vector<JoinOnClause> & getClauses() { return clauses; }
|
|
|
|
const std::vector<JoinOnClause> & getClauses() const { return clauses; }
|
|
|
|
|
|
|
|
Names getAllNames(JoinTableSide side) const;
|
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
void resetCollected();
|
2019-02-13 15:18:02 +00:00
|
|
|
void addUsingKey(const ASTPtr & ast);
|
2021-09-27 12:57:26 +00:00
|
|
|
|
2021-09-24 16:21:05 +00:00
|
|
|
void addDisjunct();
|
2021-09-27 12:57:26 +00:00
|
|
|
|
2019-02-13 15:18:02 +00:00
|
|
|
void addOnKeys(ASTPtr & left_table_ast, ASTPtr & right_table_ast);
|
2019-02-06 16:44:47 +00:00
|
|
|
|
2021-07-21 17:03:33 +00:00
|
|
|
/* Conditions for left/right table from JOIN ON section.
|
|
|
|
*
|
|
|
|
* Conditions for left and right tables stored separately and united with 'and' function into one column.
|
|
|
|
* For example for query:
|
|
|
|
* SELECT ... JOIN ... ON t1.id == t2.id AND expr11(t1) AND expr21(t2) AND expr12(t1) AND expr22(t2)
|
|
|
|
*
|
|
|
|
* We will build two new ASTs: `expr11(t1) AND expr12(t1)`, `expr21(t2) AND expr22(t2)`
|
|
|
|
* Such columns will be added and calculated for left and right tables respectively.
|
|
|
|
* Only rows where conditions are met (where new columns have non-zero value) will be joined.
|
|
|
|
*
|
|
|
|
* NOTE: non-equi condition containing columns from different tables (like `... ON t1.id = t2.id AND t1.val > t2.val)
|
|
|
|
* doesn't supported yet, it can be added later.
|
|
|
|
*/
|
|
|
|
void addJoinCondition(const ASTPtr & ast, bool is_left);
|
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
bool hasUsing() const { return table_join.using_expression_list != nullptr; }
|
|
|
|
bool hasOn() const { return table_join.on_expression != nullptr; }
|
2019-01-30 15:51:39 +00:00
|
|
|
|
2019-09-04 16:20:02 +00:00
|
|
|
NamesWithAliases getNamesWithAliases(const NameSet & required_columns) const;
|
2020-03-08 23:48:08 +00:00
|
|
|
NamesWithAliases getRequiredColumns(const Block & sample, const Names & action_required_columns) const;
|
2018-11-02 18:53:23 +00:00
|
|
|
|
2019-07-30 18:39:37 +00:00
|
|
|
void deduplicateAndQualifyColumnNames(const NameSet & left_table_columns, const String & right_table_prefix);
|
2019-05-13 18:58:15 +00:00
|
|
|
size_t rightKeyInclusion(const String & name) const;
|
2019-09-11 15:57:09 +00:00
|
|
|
NameSet requiredRightKeys() const;
|
2019-09-02 19:58:45 +00:00
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
bool leftBecomeNullable(const DataTypePtr & column_type) const;
|
|
|
|
bool rightBecomeNullable(const DataTypePtr & column_type) const;
|
2019-09-02 19:58:45 +00:00
|
|
|
void addJoinedColumn(const NameAndTypePair & joined_column);
|
2021-02-18 11:49:32 +00:00
|
|
|
|
2021-06-28 13:44:19 +00:00
|
|
|
void addJoinedColumnsAndCorrectTypes(NamesAndTypesList & left_columns, bool correct_nullability);
|
2021-02-18 16:43:41 +00:00
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
/// Calculate converting actions, rename key columns in required
|
|
|
|
/// For `USING` join we will convert key columns inplace and affect into types in the result table
|
|
|
|
/// For `JOIN ON` we will create new columns with converted keys to join by.
|
2021-06-29 09:52:19 +00:00
|
|
|
std::pair<ActionsDAGPtr, ActionsDAGPtr>
|
|
|
|
createConvertingActions(const ColumnsWithTypeAndName & left_sample_columns, const ColumnsWithTypeAndName & right_sample_columns);
|
2019-09-02 19:58:45 +00:00
|
|
|
|
2021-03-05 14:34:43 +00:00
|
|
|
void setAsofInequality(ASOF::Inequality inequality) { asof_inequality = inequality; }
|
|
|
|
ASOF::Inequality getAsofInequality() { return asof_inequality; }
|
2019-10-11 17:56:26 +00:00
|
|
|
|
2019-09-02 19:58:45 +00:00
|
|
|
ASTPtr leftKeysList() const;
|
|
|
|
ASTPtr rightKeysList() const; /// For ON syntax only
|
|
|
|
|
|
|
|
const NamesAndTypesList & columnsFromJoinedTable() const { return columns_from_joined_table; }
|
2021-06-28 13:44:19 +00:00
|
|
|
|
2021-02-18 11:49:32 +00:00
|
|
|
Names columnsAddedByJoin() const
|
|
|
|
{
|
|
|
|
Names res;
|
|
|
|
for (const auto & col : columns_added_by_join)
|
|
|
|
res.push_back(col.name);
|
|
|
|
return res;
|
|
|
|
}
|
2021-03-05 14:34:43 +00:00
|
|
|
|
|
|
|
/// StorageJoin overrides key names (cause of different names qualification)
|
2021-09-09 09:47:08 +00:00
|
|
|
void setRightKeys(const Names & keys) { getOnlyClause().key_names_right = keys; }
|
2021-03-05 14:34:43 +00:00
|
|
|
|
|
|
|
Block getRequiredRightKeys(const Block & right_table_keys, std::vector<String> & keys_sources) const;
|
2021-04-29 14:30:02 +00:00
|
|
|
|
|
|
|
String renamedRightColumnName(const String & name) const;
|
2021-08-06 14:15:11 +00:00
|
|
|
std::unordered_map<String, String> leftToRightKeyRemap() const;
|
2021-06-29 09:22:53 +00:00
|
|
|
|
|
|
|
void setStorageJoin(std::shared_ptr<StorageJoin> storage);
|
|
|
|
void setStorageJoin(std::shared_ptr<StorageDictionary> storage);
|
|
|
|
|
2021-06-29 09:52:19 +00:00
|
|
|
std::shared_ptr<StorageJoin> getStorageJoin() { return right_storage_join; }
|
2021-06-29 09:22:53 +00:00
|
|
|
|
|
|
|
bool tryInitDictJoin(const Block & sample_block, ContextPtr context);
|
|
|
|
|
2021-06-29 09:52:19 +00:00
|
|
|
bool isSpecialStorage() const { return right_storage_dictionary || right_storage_join; }
|
|
|
|
const DictionaryReader * getDictionaryReader() const { return dictionary_reader.get(); }
|
2018-11-02 18:53:23 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|