From f24bedd2b8e98d44aa8a5b49c22891cbf78e0e51 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 12 May 2019 14:36:02 +0300 Subject: [PATCH 001/357] Added constraints to parsers --- dbms/src/Parsers/ASTConstraintDeclaration.h | 38 ++++++++++++++ dbms/src/Parsers/ASTCreateQuery.cpp | 12 +++++ dbms/src/Parsers/ASTCreateQuery.h | 1 + dbms/src/Parsers/ParserCreateQuery.cpp | 55 ++++++++++++++++++--- dbms/src/Parsers/ParserCreateQuery.h | 22 +++++++-- 5 files changed, 118 insertions(+), 10 deletions(-) create mode 100644 dbms/src/Parsers/ASTConstraintDeclaration.h diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.h b/dbms/src/Parsers/ASTConstraintDeclaration.h new file mode 100644 index 00000000000..cfa4f8f98ab --- /dev/null +++ b/dbms/src/Parsers/ASTConstraintDeclaration.h @@ -0,0 +1,38 @@ +#pragma once + +#include + +namespace DB +{ + +/** name CHECK logical_expr + */ +class ASTConstraintDeclaration : public IAST { +public: + String name; + IAST *expr; + + String getID(char) const override { return "Constraint"; } + + ASTPtr clone() const override { + auto res = std::make_shared(); + + res->name = name; + + if (expr) + res->set(res->expr, expr->clone()); + + return res; + } + + void formatImpl(const FormatSettings &s, FormatState &state, FormatStateStacked frame) const override { + frame.need_parens = false; + std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); + + s.ostr << s.nl_or_ws << indent_str; + s.ostr << backQuoteIfNeed(name); + s.ostr << (s.hilite ? hilite_keyword : "") << " CHECK " << (s.hilite ? hilite_none : ""); + expr->formatImpl(s, state, frame); + } +}; +} diff --git a/dbms/src/Parsers/ASTCreateQuery.cpp b/dbms/src/Parsers/ASTCreateQuery.cpp index e99c543f5ec..b60eceb5167 100644 --- a/dbms/src/Parsers/ASTCreateQuery.cpp +++ b/dbms/src/Parsers/ASTCreateQuery.cpp @@ -128,6 +128,8 @@ ASTPtr ASTColumns::clone() const res->set(res->columns, columns->clone()); if (indices) res->set(res->indices, indices->clone()); + if (constraints) + res->set(res->constraints, constraints->clone()); return res; } @@ -156,6 +158,16 @@ void ASTColumns::formatImpl(const FormatSettings & s, FormatState & state, Forma list.children.push_back(elem); } } + if (constraints) + { + for (const auto & constraint : constraints->children) + { + auto elem = std::make_shared(); + elem->prefix = "CONSTRAINT"; + elem->set(elem->elem, constraint->clone()); + list.children.push_back(elem); + } + } if (!list.children.empty()) list.formatImpl(s, state, frame); diff --git a/dbms/src/Parsers/ASTCreateQuery.h b/dbms/src/Parsers/ASTCreateQuery.h index 2755e1a3d78..b6948c19146 100644 --- a/dbms/src/Parsers/ASTCreateQuery.h +++ b/dbms/src/Parsers/ASTCreateQuery.h @@ -36,6 +36,7 @@ class ASTColumns : public IAST public: ASTExpressionList * columns = nullptr; ASTExpressionList * indices = nullptr; + ASTExpressionList * constraints = nullptr; String getID(char) const override { return "Columns definition"; } diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index fd6665a5a2c..9f584dbbf8c 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -137,12 +138,41 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return true; } +bool ParserConstraintDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_check("CHECK"); -bool ParserColumnAndIndexDeclaraion::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + ParserIdentifier name_p; + ParserLogicalOrExpression expression_p; + + ASTPtr name; + ASTPtr expr; + + if (!name_p.parse(pos, name, expected)) + return false; + + if (!s_check.ignore(pos, expected)) + return false; + + if (!expression_p.parse(pos, expr, expected)) + return false; + + auto constraint = std::make_shared(); + constraint->name = name->as().name; + constraint->set(constraint->expr, expr); + node = constraint; + + return true; +} + + +bool ParserTablePropertyDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_index("INDEX"); + ParserKeyword s_constraint("CONSTRAINT"); ParserIndexDeclaration index_p; + ParserConstraintDeclaration constraint_p; ParserColumnDeclaration column_p; ASTPtr new_node = nullptr; @@ -152,6 +182,11 @@ bool ParserColumnAndIndexDeclaraion::parseImpl(Pos & pos, ASTPtr & node, Expecte if (!index_p.parse(pos, new_node, expected)) return false; } + else if (s_constraint.ignore(pos, expected)) + { + if (!constraint_p.parse(pos, new_node, expected)) + return false; + } else { if (!column_p.parse(pos, new_node, expected)) @@ -168,16 +203,18 @@ bool ParserIndexDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & .parse(pos, node, expected); } - -bool ParserColumnsOrIndicesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr list; - if (!ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + if (!ParserList( + std::make_unique(), + std::make_unique(TokenType::Comma), false) .parse(pos, list, expected)) return false; ASTPtr columns = std::make_shared(); ASTPtr indices = std::make_shared(); + ASTPtr constraints = std::make_shared(); for (const auto & elem : list->children) { @@ -185,6 +222,8 @@ bool ParserColumnsOrIndicesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, columns->children.push_back(elem); else if (elem->as()) indices->children.push_back(elem); + else if (elem->as()) + constraints->children.push_back(elem); else return false; } @@ -195,6 +234,8 @@ bool ParserColumnsOrIndicesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, res->set(res->columns, columns); if (!indices->children.empty()) res->set(res->indices, indices); + if (!constraints->children.empty()) + res->set(res->constraints, constraints); node = res; @@ -317,7 +358,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserToken s_rparen(TokenType::ClosingRoundBracket); ParserStorage storage_p; ParserIdentifier name_p; - ParserColumnsOrIndicesDeclarationList columns_or_indices_p; + ParserTablePropertiesDeclarationList table_properties_p; ParserSelectWithUnionQuery select_p; ASTPtr database; @@ -391,7 +432,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) /// List of columns. if (s_lparen.ignore(pos, expected)) { - if (!columns_or_indices_p.parse(pos, columns_list, expected)) + if (!table_properties_p.parse(pos, columns_list, expected)) return false; if (!s_rparen.ignore(pos, expected)) @@ -498,7 +539,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) /// Optional - a list of columns can be specified. It must fully comply with SELECT. if (s_lparen.ignore(pos, expected)) { - if (!columns_or_indices_p.parse(pos, columns_list, expected)) + if (!table_properties_p.parse(pos, columns_list, expected)) return false; if (!s_rparen.ignore(pos, expected)) diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index bd3c8f671f0..42583d8dd19 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -244,11 +244,20 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserConstraintDeclaration : public IParserBase +{ +public: + ParserConstraintDeclaration() {} -class ParserColumnAndIndexDeclaraion : public IParserBase +protected: + const char * getName() const override { return "constraint declaration"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +class ParserTablePropertyDeclaration : public IParserBase { protected: - const char * getName() const override { return "column or index declaration"; } + const char * getName() const override { return "table propery (column, index, constraint) declaration"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -260,8 +269,15 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserConstraintDeclarationList : public IParserBase +{ +protected: + const char * getName() const override { return "constraint declaration list"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; -class ParserColumnsOrIndicesDeclarationList : public IParserBase + +class ParserTablePropertiesDeclarationList : public IParserBase { protected: const char * getName() const override { return "columns or indices declaration list"; } From 502d86bd022bc53f8f4f9452cd4d1201e849aa4c Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 07:08:03 +0300 Subject: [PATCH 002/357] Added constraints to InterpreterCreateQuery and storages --- .../Interpreters/InterpreterCreateQuery.cpp | 23 +++++++++++ .../src/Interpreters/InterpreterCreateQuery.h | 2 + dbms/src/Storages/ConstraintsDescription.cpp | 39 +++++++++++++++++++ dbms/src/Storages/ConstraintsDescription.h | 23 +++++++++++ dbms/src/Storages/ITableDeclaration.cpp | 5 +++ dbms/src/Storages/ITableDeclaration.h | 5 +++ dbms/src/Storages/StorageMergeTree.h | 3 ++ 7 files changed, 100 insertions(+) create mode 100644 dbms/src/Storages/ConstraintsDescription.cpp create mode 100644 dbms/src/Storages/ConstraintsDescription.h diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 4bc35f1e378..e4e5a8e7f83 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -252,6 +252,16 @@ ASTPtr InterpreterCreateQuery::formatIndices(const IndicesDescription & indices) return res; } +ASTPtr InterpreterCreateQuery::formatConstraints(const ConstraintsDescription & constraints) +{ + auto res = std::make_shared(); + + for (const auto & constraint : constraints.constraints) + res->children.push_back(constraint->clone()); + + return res; +} + ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns_ast, const Context & context) { /// First, deduce implicit types. @@ -370,6 +380,8 @@ ColumnsDescription InterpreterCreateQuery::setColumns( { ColumnsDescription columns; IndicesDescription indices; + ConstraintsDescription constraints; + if (create.columns_list) { @@ -379,11 +391,16 @@ ColumnsDescription InterpreterCreateQuery::setColumns( for (const auto & index : create.columns_list->indices->children) indices.indices.push_back( std::dynamic_pointer_cast(index->clone())); + if (create.columns_list->constraints) + for (const auto & constraint : create.columns_list->constraints->children) + constraints.constraints.push_back( + std::dynamic_pointer_cast(constraint->clone())); } else if (!create.as_table.empty()) { columns = as_storage->getColumns(); indices = as_storage->getIndicesDescription(); + constraints = as_storage->getConstraintsDescription(); } else if (create.select) { @@ -395,6 +412,7 @@ ColumnsDescription InterpreterCreateQuery::setColumns( /// Even if query has list of columns, canonicalize it (unfold Nested columns). ASTPtr new_columns = formatColumns(columns); ASTPtr new_indices = formatIndices(indices); + ASTPtr new_constraints = formatConstraints(constraints); if (!create.columns_list) { @@ -412,6 +430,11 @@ ColumnsDescription InterpreterCreateQuery::setColumns( else if (new_indices) create.columns_list->set(create.columns_list->indices, new_indices); + if (new_constraints && create.columns_list->constraints) + create.columns_list->replace(create.columns_list->constraints, new_constraints); + else if (new_constraints) + create.columns_list->set(create.columns_list->constraints, new_constraints); + /// Check for duplicates std::set all_columns; for (const auto & column : columns) diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.h b/dbms/src/Interpreters/InterpreterCreateQuery.h index 2f124e7df9b..a7886f644ad 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.h +++ b/dbms/src/Interpreters/InterpreterCreateQuery.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -31,6 +32,7 @@ public: static ASTPtr formatColumns(const ColumnsDescription & columns); static ASTPtr formatIndices(const IndicesDescription & indices); + static ASTPtr formatConstraints(const ConstraintsDescription & constraints); void setDatabaseLoadingThreadpool(ThreadPool & thread_pool_) { diff --git a/dbms/src/Storages/ConstraintsDescription.cpp b/dbms/src/Storages/ConstraintsDescription.cpp new file mode 100644 index 00000000000..042ee06ff59 --- /dev/null +++ b/dbms/src/Storages/ConstraintsDescription.cpp @@ -0,0 +1,39 @@ +#include + +#include +#include +#include +#include + + +namespace DB +{ + +String ConstraintsDescription::toString() const +{ + if (constraints.empty()) + return {}; + + ASTExpressionList list; + for (const auto & constraint : constraints) + list.children.push_back(constraint); + + return serializeAST(list, true); +} + +ConstraintsDescription ConstraintsDescription::parse(const String & str) +{ + if (str.empty()) + return {}; + + ConstraintsDescription res; + ParserConstraintDeclarationList parser; + ASTPtr list = parseQuery(parser, str, 0); + + for (const auto & constraint : list->children) + res.constraints.push_back(std::dynamic_pointer_cast(constraint)); + + return res; +} + +} diff --git a/dbms/src/Storages/ConstraintsDescription.h b/dbms/src/Storages/ConstraintsDescription.h new file mode 100644 index 00000000000..c2954d94428 --- /dev/null +++ b/dbms/src/Storages/ConstraintsDescription.h @@ -0,0 +1,23 @@ +#pragma once + +#include + + +namespace DB +{ + +using ConstraintsASTs = std::vector>; + +struct ConstraintsDescription +{ + ConstraintsASTs constraints; + + ConstraintsDescription() = default; + + bool empty() const { return constraints.empty(); } + String toString() const; + + static ConstraintsDescription parse(const String & str); +}; + +} diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index c9385c24cbe..b24072f0de8 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -36,6 +36,11 @@ void ITableDeclaration::setIndicesDescription(IndicesDescription indices_) indices = std::move(indices_); } +void ITableDeclaration::setConstraintsDescription(ConstraintsDescription constraints_) +{ + constraints = std::move(constraints_); +} + bool ITableDeclaration::hasColumn(const String & column_name) const { diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h index e2ac3b1d6c4..e1e6d88a68f 100644 --- a/dbms/src/Storages/ITableDeclaration.h +++ b/dbms/src/Storages/ITableDeclaration.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB @@ -19,6 +20,9 @@ public: virtual const IndicesDescription & getIndicesDescription() const { return indices; } virtual void setIndicesDescription(IndicesDescription indices_); + virtual const ConstraintsDescription & getConstraintsDescription() const { return constraints; } + virtual void setConstraintsDescription(ConstraintsDescription constraints_); + /// NOTE: These methods should include virtual columns, but should NOT include ALIAS columns /// (they are treated separately). virtual NameAndTypePair getColumn(const String & column_name) const; @@ -57,6 +61,7 @@ public: private: ColumnsDescription columns; IndicesDescription indices; + ConstraintsDescription constraints; }; } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index c5ea2d8c3a0..0b4b00b7cb7 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -46,6 +46,9 @@ public: virtual const IndicesDescription & getIndicesDescription() const override { return data.getIndicesDescription(); } virtual void setIndicesDescription(IndicesDescription indices_) override { data.setIndicesDescription(std::move(indices_)); } + virtual const ConstraintsDescription & getConstraintsDescription() const override { return data.getConstraintsDescription(); } + virtual void setConstraintsDescription(ConstraintsDescription constraints_) override { data.setConstraintsDescription(constraints_; )} + NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); } bool hasColumn(const String & column_name) const override { return data.hasColumn(column_name); } From 2f8864a6ea96f124af79914fd351fe998d47b538 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 07:14:13 +0300 Subject: [PATCH 003/357] Some style fixes --- dbms/src/Parsers/ASTConstraintDeclaration.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.h b/dbms/src/Parsers/ASTConstraintDeclaration.h index cfa4f8f98ab..d72358be498 100644 --- a/dbms/src/Parsers/ASTConstraintDeclaration.h +++ b/dbms/src/Parsers/ASTConstraintDeclaration.h @@ -7,14 +7,16 @@ namespace DB /** name CHECK logical_expr */ -class ASTConstraintDeclaration : public IAST { +class ASTConstraintDeclaration : public IAST +{ public: String name; IAST *expr; String getID(char) const override { return "Constraint"; } - ASTPtr clone() const override { + ASTPtr clone() const override + { auto res = std::make_shared(); res->name = name; @@ -25,7 +27,8 @@ public: return res; } - void formatImpl(const FormatSettings &s, FormatState &state, FormatStateStacked frame) const override { + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override + { frame.need_parens = false; std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); From 72d5a4634c10d4d1e846867c22da3061fe20417e Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 07:27:09 +0300 Subject: [PATCH 004/357] Syntax fix --- dbms/src/Storages/StorageMergeTree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 0b4b00b7cb7..184db1a858b 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -47,7 +47,7 @@ public: virtual void setIndicesDescription(IndicesDescription indices_) override { data.setIndicesDescription(std::move(indices_)); } virtual const ConstraintsDescription & getConstraintsDescription() const override { return data.getConstraintsDescription(); } - virtual void setConstraintsDescription(ConstraintsDescription constraints_) override { data.setConstraintsDescription(constraints_; )} + virtual void setConstraintsDescription(ConstraintsDescription constraints_) override { data.setConstraintsDescription(constraints_); } NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); } bool hasColumn(const String & column_name) const override { return data.hasColumn(column_name); } From 07abed6d31b2a024527f34601c457de40bdcc220 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 08:05:19 +0300 Subject: [PATCH 005/357] Renamed constraints getter and setter --- dbms/src/Interpreters/InterpreterCreateQuery.cpp | 2 +- dbms/src/Storages/ITableDeclaration.cpp | 2 +- dbms/src/Storages/ITableDeclaration.h | 4 ++-- dbms/src/Storages/StorageMergeTree.h | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index e9027f473e5..15968d58ac5 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -400,7 +400,7 @@ ColumnsDescription InterpreterCreateQuery::setColumns( { columns = as_storage->getColumns(); indices = as_storage->getIndices(); - constraints = as_storage->getConstraintsDescription(); + constraints = as_storage->getConstraints(); } else if (create.select) { diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index cb9ba9daf9a..47d24300452 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -46,7 +46,7 @@ void ITableDeclaration::setIndices(IndicesDescription indices_) indices = std::move(indices_); } -void ITableDeclaration::setConstraintsDescription(ConstraintsDescription constraints_) +void ITableDeclaration::setConstraints(ConstraintsDescription constraints_) { constraints = std::move(constraints_); } diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h index 6d50f8acc0f..52db660b19b 100644 --- a/dbms/src/Storages/ITableDeclaration.h +++ b/dbms/src/Storages/ITableDeclaration.h @@ -20,8 +20,8 @@ public: const IndicesDescription & getIndices() const; void setIndices(IndicesDescription indices_); - virtual const ConstraintsDescription & getConstraintsDescription() const { return constraints; } - virtual void setConstraintsDescription(ConstraintsDescription constraints_); + virtual const ConstraintsDescription & getConstraints() const { return constraints; } + virtual void setConstraints(ConstraintsDescription constraints_); /// NOTE: These methods should include virtual columns, but should NOT include ALIAS columns /// (they are treated separately). diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 2c42bbd35c5..18386d78b8c 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -32,8 +32,8 @@ public: std::string getDatabaseName() const override { return database_name; } bool supportsIndexForIn() const override { return true; } - virtual const ConstraintsDescription & getConstraintsDescription() const override { return data.getConstraintsDescription(); } - virtual void setConstraintsDescription(ConstraintsDescription constraints_) override { data.setConstraintsDescription(constraints_); } + virtual const ConstraintsDescription & getConstraints() const override { return data.getConstraints(); } + virtual void setConstraints(ConstraintsDescription constraints_) override { data.setConstraints(constraints_); } BlockInputStreams read( const Names & column_names, From 7919a62198ce97493f1f151914ea69765423037c Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 09:21:29 +0300 Subject: [PATCH 006/357] Removed constraints getter and setter from StorageMergeTree --- dbms/src/Storages/StorageMergeTree.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 18386d78b8c..b5156ce7137 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -32,8 +32,6 @@ public: std::string getDatabaseName() const override { return database_name; } bool supportsIndexForIn() const override { return true; } - virtual const ConstraintsDescription & getConstraints() const override { return data.getConstraints(); } - virtual void setConstraints(ConstraintsDescription constraints_) override { data.setConstraints(constraints_); } BlockInputStreams read( const Names & column_names, From 1f5715b985125d8820d06c5a23a98d290d278cdb Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 18 May 2019 08:16:33 +0300 Subject: [PATCH 007/357] Removed double whitespace --- dbms/src/Parsers/ParserCreateQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index 9f584dbbf8c..76150b95b07 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -143,7 +143,7 @@ bool ParserConstraintDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & ParserKeyword s_check("CHECK"); ParserIdentifier name_p; - ParserLogicalOrExpression expression_p; + ParserLogicalOrExpression expression_p; ASTPtr name; ASTPtr expr; From ce9389660853759cb7566c7bf2a932894ccd06de Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 18 May 2019 11:05:52 +0300 Subject: [PATCH 008/357] Added constraints description to MergeTree and related storages --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 ++ dbms/src/Storages/MergeTree/MergeTreeData.h | 1 + .../Storages/MergeTree/registerStorageMergeTree.cpp | 10 +++++++--- dbms/src/Storages/StorageMergeTree.cpp | 3 ++- dbms/src/Storages/StorageMergeTree.h | 1 + dbms/src/Storages/StorageReplicatedMergeTree.cpp | 3 ++- dbms/src/Storages/StorageReplicatedMergeTree.h | 1 + 7 files changed, 16 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ea51159d9ba..4f09547a578 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -93,6 +93,7 @@ MergeTreeData::MergeTreeData( const String & database_, const String & table_, const String & full_path_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, Context & context_, const String & date_column_name, const ASTPtr & partition_by_ast_, @@ -121,6 +122,7 @@ MergeTreeData::MergeTreeData( data_parts_by_state_and_info(data_parts_indexes.get()) { setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_); + setConstraints(constraints_); /// NOTE: using the same columns list as is read when performing actual merges. merging_params.check(getColumns().getAllPhysical()); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index fecddb28540..294588721ae 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -332,6 +332,7 @@ public: const String & full_path_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, Context & context_, const String & date_column_name, const ASTPtr & partition_by_ast_, diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index b23a2eedc0e..b255f16c327 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -574,6 +574,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) ASTPtr sample_by_ast; ASTPtr ttl_table_ast; IndicesDescription indices_description; + ConstraintsDescription constraints_description; MergeTreeSettings storage_settings = args.context.getMergeTreeSettings(); if (is_extended_storage_def) @@ -602,7 +603,10 @@ static StoragePtr create(const StorageFactory::Arguments & args) indices_description.indices.push_back( std::dynamic_pointer_cast(index->clone())); - + if (args.query.columns_list && args.query.columns_list->constraints) + for (const auto & constraint : args.query.columns_list->constraints->children) + constraints_description.constraints.push_back( + std::dynamic_pointer_cast(constraint->clone())); storage_settings.loadFromQuery(*args.storage_def); } else @@ -639,14 +643,14 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (replicated) return StorageReplicatedMergeTree::create( zookeeper_path, replica_name, args.attach, args.data_path, args.database_name, args.table_name, - args.columns, indices_description, + args.columns, indices_description, constraints_description, args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, storage_settings, args.has_force_restore_data_flag); else return StorageMergeTree::create( args.data_path, args.database_name, args.table_name, args.columns, indices_description, - args.attach, args.context, date_column_name, partition_by_ast, order_by_ast, + constraints_description, args.attach, args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, storage_settings, args.has_force_restore_data_flag); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 8feb2d1fe81..12e2ffc77af 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -53,6 +53,7 @@ StorageMergeTree::StorageMergeTree( const String & table_name_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, bool attach, Context & context_, const String & date_column_name, @@ -66,7 +67,7 @@ StorageMergeTree::StorageMergeTree( bool has_force_restore_data_flag) : MergeTreeData(database_name_, table_name_, path_ + escapeForFileName(table_name_) + '/', - columns_, indices_, + columns_, indices_, constraints_, context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_, sample_by_ast_, ttl_table_ast_, merging_params_, settings_, false, attach), diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index b5156ce7137..74e0d85bfb8 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -139,6 +139,7 @@ protected: const String & table_name_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, bool attach, Context & context_, const String & date_column_name, diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index daa1b0d10e2..8d467418f12 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -199,6 +199,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const String & table_name_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, Context & context_, const String & date_column_name, const ASTPtr & partition_by_ast_, @@ -211,7 +212,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( bool has_force_restore_data_flag) : MergeTreeData(database_name_, table_name_, path_ + escapeForFileName(table_name_) + '/', - columns_, indices_, + columns_, indices_, constraints_, context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_, sample_by_ast_, ttl_table_ast_, merging_params_, settings_, true, attach, diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index eba0511e15e..cd7d043a54a 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -513,6 +513,7 @@ protected: const String & path_, const String & database_name_, const String & name_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, Context & context_, const String & date_column_name, const ASTPtr & partition_by_ast_, From 300ec160f40fffb48bf160fe82a11781819ba30e Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 08:27:00 +0300 Subject: [PATCH 009/357] Constraints MVP --- .../CheckConstraintsBlockOutputStream.cpp | 43 +++++++++++++++++ .../CheckConstraintsBlockOutputStream.h | 48 +++++++++++++++++++ .../Interpreters/InterpreterInsertQuery.cpp | 4 ++ dbms/src/Storages/ConstraintsDescription.h | 14 +++++- 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp create mode 100644 dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp new file mode 100644 index 00000000000..99f9f9bc90d --- /dev/null +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -0,0 +1,43 @@ +#include + + +namespace DB +{ + +void CheckConstraintsBlockOutputStream::write(const Block & block) +{ + for (auto & constraint_expr: expressions) + if (!checkConstraintOnBlock(block, constraint_expr)) + throw Exception("Some constraints are not satisfied", ErrorCodes::QUERY_WAS_CANCELLED); + output->write(block); +} + +void CheckConstraintsBlockOutputStream::flush() +{ + output->flush(); +} + +void CheckConstraintsBlockOutputStream::writePrefix() +{ + output->writePrefix(); +} + +void CheckConstraintsBlockOutputStream::writeSuffix() +{ + output->writeSuffix(); +} + +bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) +{ + Block res = block; + constraint->execute(res); + assert(block.columns() == res.columns() - 1); + ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); + size_t column_size = res_column.column->size(); + for (size_t i = 0; i < column_size; ++i) + if (!res_column.column->getBool(i)) + return false; + return true; +} + +} diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h new file mode 100644 index 00000000000..623eccc8172 --- /dev/null +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CONSTRAINTS_ARE_NOT_SATISFIED; +} + +class CheckConstraintsBlockOutputStream : public IBlockOutputStream +{ +public: + CheckConstraintsBlockOutputStream( + const BlockOutputStreamPtr & output_, + const Block & header_, + const ConstraintsDescription & constraints_, + const Context & context_) + : output(output_), + header(header_), + constraints(constraints_), + expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())), + context(context_) + { } + + Block getHeader() const override { return header; } + void write(const Block & block) override; + + void flush() override; + + void writePrefix() override; + void writeSuffix() override; + + bool checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint); + +private: + BlockOutputStreamPtr output; + Block header; + const ConstraintsDescription constraints; + const ConstraintsExpressions expressions; + const Context & context; +}; +} diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index e4391f52247..fa6df1599ea 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -117,6 +118,9 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared( out, query_sample_block, table->getSampleBlock(), table->getColumns().getDefaults(), context); + out = std::make_shared( + out, query_sample_block, table->getConstraints(), context); + auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); out = std::move(out_wrapper); diff --git a/dbms/src/Storages/ConstraintsDescription.h b/dbms/src/Storages/ConstraintsDescription.h index c2954d94428..fbb0f5167fc 100644 --- a/dbms/src/Storages/ConstraintsDescription.h +++ b/dbms/src/Storages/ConstraintsDescription.h @@ -1,12 +1,13 @@ #pragma once #include - +#include namespace DB { using ConstraintsASTs = std::vector>; +using ConstraintsExpressions = std::vector; struct ConstraintsDescription { @@ -18,6 +19,17 @@ struct ConstraintsDescription String toString() const; static ConstraintsDescription parse(const String & str); + + ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const { + ConstraintsExpressions res; + res.reserve(constraints.size()); + for (const auto & constraint : constraints) { + ASTPtr expr = constraint->expr->clone(); + auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); + res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); + } + return res; + } }; } From 60a5b94ba47ee4050dbb5d38822ceb2d15a479f2 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 08:44:31 +0300 Subject: [PATCH 010/357] Style fix --- dbms/src/Storages/ConstraintsDescription.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/ConstraintsDescription.h b/dbms/src/Storages/ConstraintsDescription.h index fbb0f5167fc..0f565379204 100644 --- a/dbms/src/Storages/ConstraintsDescription.h +++ b/dbms/src/Storages/ConstraintsDescription.h @@ -20,10 +20,12 @@ struct ConstraintsDescription static ConstraintsDescription parse(const String & str); - ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const { + ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const + { ConstraintsExpressions res; res.reserve(constraints.size()); - for (const auto & constraint : constraints) { + for (const auto & constraint : constraints) + { ASTPtr expr = constraint->expr->clone(); auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); From 19d099f90a147fc385721a525a39249bfb682304 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 09:03:18 +0300 Subject: [PATCH 011/357] Removed ITableDeclaration --- dbms/src/Storages/ITableDeclaration.cpp | 0 dbms/src/Storages/ITableDeclaration.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dbms/src/Storages/ITableDeclaration.cpp delete mode 100644 dbms/src/Storages/ITableDeclaration.h diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h deleted file mode 100644 index e69de29bb2d..00000000000 From 773849a43d9ab25df77c38e00de5d6b914c0b0f2 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 09:08:25 +0300 Subject: [PATCH 012/357] Added constraints getter and setter to IStorage --- dbms/src/Storages/IStorage.cpp | 10 ++++++++++ dbms/src/Storages/IStorage.h | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 06320cc1f30..7c19fd94aea 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -47,6 +47,16 @@ void IStorage::setIndices(IndicesDescription indices_) indices = std::move(indices_); } +const ConstraintsDescription & IStorage::getConstraints() const +{ + return constraints; +} + +void IStorage::setConstraints(ConstraintsDescription constraints_) +{ + constraints = std::move(constraints_); +} + NameAndTypePair IStorage::getColumn(const String & column_name) const { /// By default, we assume that there are no virtual columns in the storage. diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index f18592ebce5..b01244ba111 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -88,6 +89,9 @@ public: /// thread-unsafe part. lockStructure must be acquired const IndicesDescription & getIndices() const; void setIndices(IndicesDescription indices_); + const ConstraintsDescription & getConstraints() const; + void setConstraints(ConstraintsDescription constraints_); + /// NOTE: these methods should include virtual columns, /// but should NOT include ALIAS columns (they are treated separately). virtual NameAndTypePair getColumn(const String & column_name) const; @@ -115,6 +119,7 @@ public: /// thread-unsafe part. lockStructure must be acquired private: ColumnsDescription columns; IndicesDescription indices; + ConstraintsDescription constraints; public: /// Acquire this lock if you need the table structure to remain constant during the execution of From d1492fc05d534034d1a0ed1322a9e86f1020cc62 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 10:17:06 +0300 Subject: [PATCH 013/357] Removed context from CheckConstraintsBlockOutputStream --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h | 7 ++----- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index 623eccc8172..e1e15f8e454 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -19,13 +19,11 @@ public: CheckConstraintsBlockOutputStream( const BlockOutputStreamPtr & output_, const Block & header_, - const ConstraintsDescription & constraints_, - const Context & context_) + const ConstraintsDescription & constraints_) : output(output_), header(header_), constraints(constraints_), - expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())), - context(context_) + expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())) { } Block getHeader() const override { return header; } @@ -43,6 +41,5 @@ private: Block header; const ConstraintsDescription constraints; const ConstraintsExpressions expressions; - const Context & context; }; } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index fa6df1599ea..d47dd978b3a 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -118,8 +118,7 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared( out, query_sample_block, table->getSampleBlock(), table->getColumns().getDefaults(), context); - out = std::make_shared( - out, query_sample_block, table->getConstraints(), context); + out = std::make_shared(out, query_sample_block, table->getConstraints()); auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); From 9e6625441c936a4fef236c34cefa0b0654645364 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 10:19:44 +0300 Subject: [PATCH 014/357] Returned context to CheckConstraintsBlockOutputStream constructor --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h | 3 ++- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index e1e15f8e454..16b240eb758 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -19,7 +19,8 @@ public: CheckConstraintsBlockOutputStream( const BlockOutputStreamPtr & output_, const Block & header_, - const ConstraintsDescription & constraints_) + const ConstraintsDescription & constraints_, + const Context & context_) : output(output_), header(header_), constraints(constraints_), diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index d47dd978b3a..fa6df1599ea 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -118,7 +118,8 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared( out, query_sample_block, table->getSampleBlock(), table->getColumns().getDefaults(), context); - out = std::make_shared(out, query_sample_block, table->getConstraints()); + out = std::make_shared( + out, query_sample_block, table->getConstraints(), context); auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); From c926cf65627dfbbc724b8b469c254399e6fb8486 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 25 May 2019 17:07:45 +0300 Subject: [PATCH 015/357] Minor review fixes --- dbms/src/Parsers/ASTConstraintDeclaration.cpp | 28 +++++++++++++++++++ dbms/src/Parsers/ASTConstraintDeclaration.h | 25 ++--------------- dbms/src/Parsers/ParserCreateQuery.cpp | 6 ++++ dbms/src/Parsers/ParserCreateQuery.h | 3 -- dbms/src/Storages/ConstraintsDescription.cpp | 15 ++++++++++ dbms/src/Storages/ConstraintsDescription.h | 13 +-------- 6 files changed, 53 insertions(+), 37 deletions(-) create mode 100644 dbms/src/Parsers/ASTConstraintDeclaration.cpp diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.cpp b/dbms/src/Parsers/ASTConstraintDeclaration.cpp new file mode 100644 index 00000000000..834ac81891b --- /dev/null +++ b/dbms/src/Parsers/ASTConstraintDeclaration.cpp @@ -0,0 +1,28 @@ +#include + +namespace DB { + +ASTPtr ASTConstraintDeclaration::clone() const +{ + auto res = std::make_shared(); + + res->name = name; + + if (expr) + res->set(res->expr, expr->clone()); + + return res; +} + +void ASTConstraintDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +{ + frame.need_parens = false; + std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); + + s.ostr << s.nl_or_ws << indent_str; + s.ostr << backQuoteIfNeed(name); + s.ostr << (s.hilite ? hilite_keyword : "") << " CHECK " << (s.hilite ? hilite_none : ""); + expr->formatImpl(s, state, frame); +} + +} \ No newline at end of file diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.h b/dbms/src/Parsers/ASTConstraintDeclaration.h index d72358be498..3a8ad75f54b 100644 --- a/dbms/src/Parsers/ASTConstraintDeclaration.h +++ b/dbms/src/Parsers/ASTConstraintDeclaration.h @@ -11,31 +11,12 @@ class ASTConstraintDeclaration : public IAST { public: String name; - IAST *expr; + IAST * expr; String getID(char) const override { return "Constraint"; } - ASTPtr clone() const override - { - auto res = std::make_shared(); + ASTPtr clone() const override; - res->name = name; - - if (expr) - res->set(res->expr, expr->clone()); - - return res; - } - - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override - { - frame.need_parens = false; - std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); - - s.ostr << s.nl_or_ws << indent_str; - s.ostr << backQuoteIfNeed(name); - s.ostr << (s.hilite ? hilite_keyword : "") << " CHECK " << (s.hilite ? hilite_none : ""); - expr->formatImpl(s, state, frame); - } + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index 76150b95b07..c828cd0d780 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -203,6 +203,12 @@ bool ParserIndexDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & .parse(pos, node, expected); } +bool ParserConstraintDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + .parse(pos, node, expected); +} + bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr list; diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index 42583d8dd19..bc921773605 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -246,9 +246,6 @@ protected: class ParserConstraintDeclaration : public IParserBase { -public: - ParserConstraintDeclaration() {} - protected: const char * getName() const override { return "constraint declaration"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; diff --git a/dbms/src/Storages/ConstraintsDescription.cpp b/dbms/src/Storages/ConstraintsDescription.cpp index 042ee06ff59..ad0cd76733a 100644 --- a/dbms/src/Storages/ConstraintsDescription.cpp +++ b/dbms/src/Storages/ConstraintsDescription.cpp @@ -36,4 +36,19 @@ ConstraintsDescription ConstraintsDescription::parse(const String & str) return res; } +ConstraintsExpressions ConstraintsDescription::getExpressions(const DB::Context & context, + const DB::NamesAndTypesList & source_columns_) const +{ + ConstraintsExpressions res; + res.reserve(constraints.size()); + for (const auto & constraint : constraints) + { + // SyntaxAnalyzer::analyze has query as non-const argument so to avoid accidental query changes we clone it + ASTPtr expr = constraint->expr->clone(); + auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); + res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); + } + return res; +} + } diff --git a/dbms/src/Storages/ConstraintsDescription.h b/dbms/src/Storages/ConstraintsDescription.h index 0f565379204..3ced0e8ddc9 100644 --- a/dbms/src/Storages/ConstraintsDescription.h +++ b/dbms/src/Storages/ConstraintsDescription.h @@ -20,18 +20,7 @@ struct ConstraintsDescription static ConstraintsDescription parse(const String & str); - ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const - { - ConstraintsExpressions res; - res.reserve(constraints.size()); - for (const auto & constraint : constraints) - { - ASTPtr expr = constraint->expr->clone(); - auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); - res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); - } - return res; - } + ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const; }; } From 94db808cd7f73c9ad5ec3dfcd2f348061c38b8e3 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 25 May 2019 18:54:49 +0300 Subject: [PATCH 016/357] Basic test for constraints parsers and failures --- .../0_stateless/00951_basic_constraints.sh | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00951_basic_constraints.sh diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh new file mode 100644 index 00000000000..43bf274a82b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh @@ -0,0 +1,45 @@ +exec 2>&1 + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT b_constraint CHECK b > 0 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw and exception +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" + +# Test two constraints on one table +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT b_constraint CHECK b > 10, + CONSTRAINT a_constraint CHECK a < 10 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must throw an exception +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw an exception +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (7, 18), (0, 11);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" \ No newline at end of file From a3535c69b23aff266dcbdc8ed856024efff7fa69 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 25 May 2019 18:57:35 +0300 Subject: [PATCH 017/357] Minor style fix --- dbms/src/Parsers/ASTConstraintDeclaration.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.cpp b/dbms/src/Parsers/ASTConstraintDeclaration.cpp index 834ac81891b..a1b063fc44a 100644 --- a/dbms/src/Parsers/ASTConstraintDeclaration.cpp +++ b/dbms/src/Parsers/ASTConstraintDeclaration.cpp @@ -1,6 +1,7 @@ #include -namespace DB { +namespace DB +{ ASTPtr ASTConstraintDeclaration::clone() const { @@ -25,4 +26,4 @@ void ASTConstraintDeclaration::formatImpl(const FormatSettings & s, FormatState expr->formatImpl(s, state, frame); } -} \ No newline at end of file +} From c185f5741c116f6038979e56f220e8f2b632ab2a Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 25 May 2019 21:13:43 +0300 Subject: [PATCH 018/357] Fixed test --- dbms/tests/queries/0_stateless/00951_basic_constraints.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh index 43bf274a82b..cf622293688 100644 --- a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + exec 2>&1 $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" From b0e3315ac7855bae85dbb8adc21b826e971f6ce0 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 26 May 2019 08:32:22 +0300 Subject: [PATCH 019/357] Added tests reference --- .../0_stateless/00951_basic_constraints.reference | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00951_basic_constraints.reference diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.reference b/dbms/tests/queries/0_stateless/00951_basic_constraints.reference new file mode 100644 index 00000000000..28bf65f8d48 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00951_basic_constraints.reference @@ -0,0 +1,10 @@ +1 2 +Received exception from server (version 19.8.1): +Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +1 2 +Received exception from server (version 19.8.1): +Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +Received exception from server (version 19.8.1): +Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +0 11 +7 18 From f87bb846586a848d5c24a793d8474783008f4cd8 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 26 May 2019 11:43:45 +0300 Subject: [PATCH 020/357] Added curdir to 00951 test --- dbms/tests/queries/0_stateless/00951_basic_constraints.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh index cf622293688..49bf6771ab4 100644 --- a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + exec 2>&1 $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" From 94e7521beae08dafcc700bf8c9991f786a6b39ec Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 26 May 2019 12:37:35 +0300 Subject: [PATCH 021/357] Changed number of tests --- ...ic_constraints.reference => 00952_basic_constraints.reference} | 0 .../{00951_basic_constraints.sh => 00952_basic_constraints.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename dbms/tests/queries/0_stateless/{00951_basic_constraints.reference => 00952_basic_constraints.reference} (100%) rename dbms/tests/queries/0_stateless/{00951_basic_constraints.sh => 00952_basic_constraints.sh} (100%) diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.reference b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00951_basic_constraints.reference rename to dbms/tests/queries/0_stateless/00952_basic_constraints.reference diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh similarity index 100% rename from dbms/tests/queries/0_stateless/00951_basic_constraints.sh rename to dbms/tests/queries/0_stateless/00952_basic_constraints.sh From af50b1ff5cef7bea4c35d25067c914174acc7b89 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 26 May 2019 14:08:37 +0300 Subject: [PATCH 022/357] Made 00952 test executable --- dbms/tests/queries/0_stateless/00952_basic_constraints.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 dbms/tests/queries/0_stateless/00952_basic_constraints.sh diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh old mode 100644 new mode 100755 From 1aaab0745969277b9e9fab643d10832a571ff469 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 27 May 2019 02:57:18 +0300 Subject: [PATCH 023/357] Fixed exception handling in 00952 test --- .../CheckConstraintsBlockOutputStream.cpp | 2 +- .../0_stateless/00952_basic_constraints.reference | 9 +++------ .../queries/0_stateless/00952_basic_constraints.sh | 14 +++++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 99f9f9bc90d..1cc271e5578 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -8,7 +8,7 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { for (auto & constraint_expr: expressions) if (!checkConstraintOnBlock(block, constraint_expr)) - throw Exception("Some constraints are not satisfied", ErrorCodes::QUERY_WAS_CANCELLED); + throw Exception{"Some constraints are not satisfied", ErrorCodes::QUERY_WAS_CANCELLED}; output->write(block); } diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference index 28bf65f8d48..1bede18351d 100644 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference @@ -1,10 +1,7 @@ 1 2 -Received exception from server (version 19.8.1): -Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +Exception ok 1 2 -Received exception from server (version 19.8.1): -Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. -Received exception from server (version 19.8.1): -Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +Exception ok +Exception ok 0 11 7 18 diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index 49bf6771ab4..b214982da5e 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -3,8 +3,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -exec 2>&1 - $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints @@ -20,7 +18,9 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" +EXCEPTION_TEXT="Some constraints are not satisfied" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" $CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" @@ -36,11 +36,15 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +EXCEPTION_TEXT="Some constraints are not satisfied" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" +EXCEPTION_TEXT="Some constraints are not satisfied" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must succeed From ff6cdaeb9846ae63ddba399e215809f0be641793 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 27 May 2019 09:30:18 +0300 Subject: [PATCH 024/357] Removed word "exception" from test reference --- .../queries/0_stateless/00952_basic_constraints.reference | 6 +++--- dbms/tests/queries/0_stateless/00952_basic_constraints.sh | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference index 1bede18351d..4d98efd8939 100644 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference @@ -1,7 +1,7 @@ 1 2 -Exception ok +ok 1 2 -Exception ok -Exception ok +ok +ok 0 11 7 18 diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index b214982da5e..93fa16ce4af 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -3,6 +3,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +EXCEPTION_SUCCESS_TEXT=ok + $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints @@ -20,7 +22,7 @@ $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception EXCEPTION_TEXT="Some constraints are not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" 2>&1 \ - | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" $CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" @@ -38,13 +40,13 @@ ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception EXCEPTION_TEXT="Some constraints are not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ - | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception EXCEPTION_TEXT="Some constraints are not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" 2>&1 \ - | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must succeed From e7293486bd39103a25d4d51ca4e8e8597c1c11ce Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 2 Jun 2019 17:41:12 +0300 Subject: [PATCH 025/357] Added ALTER TABLE support to constraints (ADD CONSTRAINT, DROP CONSTRAINT) --- dbms/src/Databases/DatabaseDictionary.cpp | 1 + dbms/src/Databases/DatabaseDictionary.h | 1 + dbms/src/Databases/DatabaseMemory.cpp | 1 + dbms/src/Databases/DatabaseMemory.h | 1 + dbms/src/Databases/DatabaseOrdinary.cpp | 7 ++ dbms/src/Databases/DatabaseOrdinary.h | 1 + dbms/src/Databases/IDatabase.h | 2 + dbms/src/Parsers/ASTAlterQuery.cpp | 11 +++ dbms/src/Parsers/ASTAlterQuery.h | 13 ++- dbms/src/Parsers/ParserAlterQuery.cpp | 25 ++++++ dbms/src/Storages/AlterCommands.cpp | 80 ++++++++++++++++++- dbms/src/Storages/AlterCommands.h | 17 +++- dbms/src/Storages/IStorage.cpp | 3 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 13 +-- dbms/src/Storages/MergeTree/MergeTreeData.h | 3 +- .../ReplicatedMergeTreeTableMetadata.cpp | 16 ++++ .../ReplicatedMergeTreeTableMetadata.h | 6 +- dbms/src/Storages/StorageBuffer.cpp | 3 +- dbms/src/Storages/StorageDistributed.cpp | 3 +- dbms/src/Storages/StorageMerge.cpp | 3 +- dbms/src/Storages/StorageMergeTree.cpp | 16 ++-- dbms/src/Storages/StorageNull.cpp | 3 +- .../Storages/StorageReplicatedMergeTree.cpp | 14 +++- .../00953_constraints_operations.reference | 4 + .../00953_constraints_operations.sh | 40 ++++++++++ 25 files changed, 255 insertions(+), 32 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00953_constraints_operations.reference create mode 100755 dbms/tests/queries/0_stateless/00953_constraints_operations.sh diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 195dcea5287..8add0fa8911 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -131,6 +131,7 @@ void DatabaseDictionary::alterTable( const String &, const ColumnsDescription &, const IndicesDescription &, + const ConstraintsDescription &, const ASTModifier &) { throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index 3bff84c36b8..d0ea33c6e4e 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -72,6 +72,7 @@ public: const String & name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 3eea0bc666a..c53309ca6c1 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -54,6 +54,7 @@ void DatabaseMemory::alterTable( const String &, const ColumnsDescription &, const IndicesDescription &, + const ConstraintsDescription &, const ASTModifier &) { throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index fe7cc783ba3..dc770373360 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -49,6 +49,7 @@ public: const String & name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 9fa7d1b1196..1ef67dfd9f4 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -516,6 +516,7 @@ void DatabaseOrdinary::alterTable( const String & table_name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & storage_modifier) { /// Read the definition of the table and replace the necessary parts with new ones. @@ -538,6 +539,7 @@ void DatabaseOrdinary::alterTable( ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns); ASTPtr new_indices = InterpreterCreateQuery::formatIndices(indices); + ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(constraints); ast_create_query.columns_list->replace(ast_create_query.columns_list->columns, new_columns); @@ -546,6 +548,11 @@ void DatabaseOrdinary::alterTable( else ast_create_query.columns_list->set(ast_create_query.columns_list->indices, new_indices); + if (ast_create_query.columns_list->constraints) + ast_create_query.columns_list->replace(ast_create_query.columns_list->constraints, new_constraints); + else + ast_create_query.columns_list->set(ast_create_query.columns_list->constraints, new_constraints); + if (storage_modifier) storage_modifier(*ast_create_query.storage); diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index 887bf101d62..2ed1a426d64 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -43,6 +43,7 @@ public: const String & name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index d53de1dfcb5..37ee1e676e9 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -114,6 +115,7 @@ public: const String & name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & engine_modifier) = 0; /// Returns time of table's metadata change, 0 if there is no corresponding metadata file. diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index e614f64d208..a85890d3cd0 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -105,6 +105,17 @@ void ASTAlterCommand::formatImpl( << "DROP INDEX " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); index->formatImpl(settings, state, frame); } + else if (type == ASTAlterCommand::ADD_CONSTRAINT) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ADD CONSTRAINT" << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); + constraint_decl->formatImpl(settings, state, frame); + } + else if (type == ASTAlterCommand::DROP_CONSTRAINT) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str + << "DROP CONSTRAINT " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); + constraint->formatImpl(settings, state, frame); + } else if (type == ASTAlterCommand::DROP_PARTITION) { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << (detach ? "DETACH" : "DROP") << " PARTITION " diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 2c4b3ddbaf1..2ca88f3145d 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -32,6 +32,9 @@ public: ADD_INDEX, DROP_INDEX, + ADD_CONSTRAINT, + DROP_CONSTRAINT, + DROP_PARTITION, ATTACH_PARTITION, REPLACE_PARTITION, @@ -69,7 +72,15 @@ public: /** The ADD INDEX query stores the name of the index following AFTER. * The DROP INDEX query stores the name for deletion. */ - ASTPtr index; + ASTPtr index; + + /** The ADD CONSTRAINT query stores the ConstraintDeclaration there. + */ + ASTPtr constraint_decl; + + /** The DROP CONSTRAINT query stores the name for deletion. + */ + ASTPtr constraint; /** Used in DROP PARTITION and ATTACH PARTITION FROM queries. * The value or ID of the partition is stored here. diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 98891bbdf5f..e4220fda868 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -32,6 +32,9 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_add_index("ADD INDEX"); ParserKeyword s_drop_index("DROP INDEX"); + ParserKeyword s_add_constraint("ADD CONSTRAINT"); + ParserKeyword s_drop_constraint("DROP CONSTRAINT"); + ParserKeyword s_attach_partition("ATTACH PARTITION"); ParserKeyword s_detach_partition("DETACH PARTITION"); ParserKeyword s_drop_partition("DROP PARTITION"); @@ -57,6 +60,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserStringLiteral parser_string_literal; ParserCompoundColumnDeclaration parser_col_decl; ParserIndexDeclaration parser_idx_decl; + ParserConstraintDeclaration parser_constraint_decl; ParserCompoundColumnDeclaration parser_modify_col_decl(false); ParserPartition parser_partition; ParserExpression parser_exp_elem; @@ -125,6 +129,27 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->type = ASTAlterCommand::DROP_INDEX; command->detach = false; } + else if (s_add_constraint.ignore(pos, expected)) + { + if (s_if_not_exists.ignore(pos, expected)) + command->if_not_exists = true; + + if (!parser_constraint_decl.parse(pos, command->constraint_decl, expected)) + return false; + + command->type = ASTAlterCommand::ADD_CONSTRAINT; + } + else if (s_drop_constraint.ignore(pos, expected)) + { + if (s_if_exists.ignore(pos, expected)) + command->if_exists = true; + + if (!parser_name.parse(pos, command->constraint, expected)) + return false; + + command->type = ASTAlterCommand::DROP_CONSTRAINT; + command->detach = false; + } else if (s_clear_column.ignore(pos, expected)) { if (s_if_exists.ignore(pos, expected)) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 88f3e909f49..22095ec4ae3 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -165,6 +166,32 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } + else if (command_ast->type == ASTAlterCommand::ADD_CONSTRAINT) + { + AlterCommand command; + command.constraint_decl = command_ast->constraint_decl; + command.type = AlterCommand::ADD_CONSTRAINT; + + const auto & ast_constraint_decl = command_ast->constraint_decl->as(); + + command.constraint_name = ast_constraint_decl.name; + + command.if_not_exists = command_ast->if_not_exists; + + return command; + } + else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) + { + if (command_ast->clear_column) + throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); + + AlterCommand command; + command.type = AlterCommand::DROP_CONSTRAINT; + command.constraint_name = command_ast->constraint->as().name; + command.if_exists = command_ast->if_exists; + + return command; + } else if (command_ast->type == ASTAlterCommand::MODIFY_TTL) { AlterCommand command; @@ -178,7 +205,8 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const + ConstraintsDescription & constraints_description, + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const { if (type == ADD_COLUMN) { @@ -298,6 +326,44 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri indices_description.indices.erase(erase_it); } + else if (type == ADD_CONSTRAINT) + { + if (std::any_of( + constraints_description.constraints.cbegin(), + constraints_description.constraints.cend(), + [this](const ASTPtr & constraint_ast) + { + return constraint_ast->as().name == constraint_name; + })) + { + if (if_not_exists) + return; + throw Exception("Cannot add constraint " + constraint_name + ": constraint with this name already exists", + ErrorCodes::ILLEGAL_COLUMN); + } + + auto insert_it = constraints_description.constraints.end(); + + constraints_description.constraints.emplace(insert_it, std::dynamic_pointer_cast(constraint_decl)); + } + else if (type == DROP_CONSTRAINT) + { + auto erase_it = std::find_if( + constraints_description.constraints.begin(), + constraints_description.constraints.end(), + [this](const ASTPtr & constraint_ast) + { + return constraint_ast->as().name == constraint_name; + }); + + if (erase_it == constraints_description.constraints.end()) { + if (if_exists) + return; + throw Exception("Wrong constraint name. Cannot find constraint `" + constraint_name + "` to drop.", + ErrorCodes::LOGICAL_ERROR); + } + constraints_description.constraints.erase(erase_it); + } else if (type == MODIFY_TTL) { ttl_table_ast = ttl; @@ -317,20 +383,23 @@ bool AlterCommand::isMutable() const } void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const + ConstraintsDescription & constraints_description, + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const { auto new_columns_description = columns_description; auto new_indices_description = indices_description; + auto new_constraints_description = constraints_description; auto new_order_by_ast = order_by_ast; auto new_primary_key_ast = primary_key_ast; auto new_ttl_table_ast = ttl_table_ast; for (const AlterCommand & command : *this) if (!command.ignore) - command.apply(new_columns_description, new_indices_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + command.apply(new_columns_description, new_indices_description, new_constraints_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); columns_description = std::move(new_columns_description); indices_description = std::move(new_indices_description); + constraints_description = std::move(new_constraints_description); order_by_ast = std::move(new_order_by_ast); primary_key_ast = std::move(new_primary_key_ast); ttl_table_ast = std::move(new_ttl_table_ast); @@ -518,10 +587,11 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const { auto out_columns_description = columns_description; IndicesDescription indices_description; + ConstraintsDescription constraints_description; ASTPtr out_order_by; ASTPtr out_primary_key; ASTPtr out_ttl_table; - apply(out_columns_description, indices_description, out_order_by, out_primary_key, out_ttl_table); + apply(out_columns_description, indices_description, constraints_description, out_order_by, out_primary_key, out_ttl_table); if (out_order_by) throw Exception("Storage doesn't support modifying ORDER BY expression", ErrorCodes::NOT_IMPLEMENTED); @@ -529,6 +599,8 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const throw Exception("Storage doesn't support modifying PRIMARY KEY expression", ErrorCodes::NOT_IMPLEMENTED); if (!indices_description.indices.empty()) throw Exception("Storage doesn't support modifying indices", ErrorCodes::NOT_IMPLEMENTED); + if (!constraints_description.constraints.empty()) + throw Exception("Storage doesn't support modifying constraints", ErrorCodes::NOT_IMPLEMENTED); if (out_ttl_table) throw Exception("Storage doesn't support modifying TTL expression", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index 4905b80f92f..1dfd46f9617 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -25,6 +26,8 @@ struct AlterCommand MODIFY_ORDER_BY, ADD_INDEX, DROP_INDEX, + ADD_CONSTRAINT, + DROP_CONSTRAINT, MODIFY_TTL, UKNOWN_TYPE, }; @@ -62,6 +65,12 @@ struct AlterCommand /// For ADD/DROP INDEX String index_name; + // For ADD CONSTRAINT + ASTPtr constraint_decl; + + // For ADD/DROP CONSTRAINT + String constraint_name; + /// For MODIFY TTL ASTPtr ttl; @@ -84,7 +93,8 @@ struct AlterCommand static std::optional parse(const ASTAlterCommand * command); void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; + ConstraintsDescription & constraints_description, + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; /// Checks that not only metadata touched by that command bool isMutable() const; @@ -95,8 +105,9 @@ class Context; class AlterCommands : public std::vector { public: - void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, ASTPtr & order_by_ast, - ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; + void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, + ConstraintsDescription & constraints_description, + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; /// For storages that don't support MODIFY_ORDER_BY. void apply(ColumnsDescription & columns_description) const; diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 7c19fd94aea..07f52749d2d 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -346,8 +346,9 @@ void IStorage::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 8427982efd7..144c6591ed9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -121,7 +121,7 @@ MergeTreeData::MergeTreeData( data_parts_by_info(data_parts_indexes.get()), data_parts_by_state_and_info(data_parts_indexes.get()) { - setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_); + setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_); setConstraints(constraints_); /// NOTE: using the same columns list as is read when performing actual merges. @@ -255,7 +255,8 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam void MergeTreeData::setPrimaryKeyIndicesAndColumns( const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, - const ColumnsDescription & new_columns, const IndicesDescription & indices_description, bool only_check) + const ColumnsDescription & new_columns, const IndicesDescription & indices_description, + const ConstraintsDescription & constraints_description, bool only_check) { if (!new_order_by_ast) throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -425,6 +426,8 @@ void MergeTreeData::setPrimaryKeyIndicesAndColumns( setIndices(indices_description); skip_indices = std::move(new_indices); + setConstraints(constraints_description); + primary_key_and_skip_indices_expr = new_indices_with_primary_key_expr; sorting_key_and_skip_indices_expr = new_indices_with_sorting_key_expr; } @@ -1180,11 +1183,11 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c /// Check that needed transformations can be applied to the list of columns without considering type conversions. auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - commands.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); - + commands.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); if (getIndices().empty() && !new_indices.empty() && !context.getSettingsRef().allow_experimental_data_skipping_indices) throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \ @@ -1267,7 +1270,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c } setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, - new_columns, new_indices, /* only_check = */ true); + new_columns, new_indices, new_constraints, /* only_check = */ true); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast, /* only_check = */ true); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index d589bb77013..bfcfbaba9a1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -778,7 +778,8 @@ protected: void setPrimaryKeyIndicesAndColumns(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, const ColumnsDescription & new_columns, - const IndicesDescription & indices_description, bool only_check = false); + const IndicesDescription & indices_description, + const ConstraintsDescription & constraints_description, bool only_check = false); void initPartitionKey(); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index b122785c5fd..8549264e2c2 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -46,6 +46,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr partition_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.partition_by_ast)); skip_indices = data.getIndices().toString(); + constraints = data.getConstraints().toString(); index_granularity_bytes = data.index_granularity_info.index_granularity_bytes; ttl_table = formattedAST(data.ttl_table_ast); } @@ -229,6 +230,21 @@ ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTabl ErrorCodes::METADATA_MISMATCH); } + if (constraints != from_zk.constraints) + { + if (allow_alter) + { + diff.constraints_changed = true; + diff.new_constraints = from_zk.constraints; + } + else + throw Exception( + "Existing table metadata in ZooKeeper differs in constraints." + " Stored in ZooKeeper: " + from_zk.constraints + + ", local: " + constraints, + ErrorCodes::METADATA_MISMATCH); + } + if (index_granularity_bytes != from_zk.index_granularity_bytes) throw Exception("Existing table metadata in ZooKeeper differs in index granularity bytes." " Stored in ZooKeeper: " + DB::toString(from_zk.index_granularity_bytes) + diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h index 55cfdb1494d..b28a7306e33 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h @@ -26,6 +26,7 @@ struct ReplicatedMergeTreeTableMetadata String partition_key; String sorting_key; String skip_indices; + String constraints; UInt64 index_granularity_bytes; String ttl_table; @@ -46,10 +47,13 @@ struct ReplicatedMergeTreeTableMetadata bool skip_indices_changed = false; String new_skip_indices; + bool constraints_changed = false; + String new_constraints; + bool ttl_table_changed = false; String new_ttl_table; - bool empty() const { return !sorting_key_changed && !skip_indices_changed; } + bool empty() const { return !sorting_key_changed && !skip_indices_changed && !constraints_changed; } }; Diff checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, bool allow_alter) const; diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 24e1ecef2e3..5a4409600d8 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -701,8 +701,9 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_ auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 4440d2b96ee..42e769019d6 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -346,8 +346,9 @@ void StorageDistributed::alter( auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); + context.getDatabase(database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 1fbde64fd85..9f3331a033e 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -417,8 +417,9 @@ void StorageMerge::alter( auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); setColumns(new_columns); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 01049bfbf76..10709cd8181 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -242,8 +242,9 @@ void StorageMergeTree::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); return; } @@ -252,15 +253,14 @@ void StorageMergeTree::alter( auto merge_blocker = merger_mutator.actions_blocker.cancel(); lockNewDataStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - checkAlter(params, context); - auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + params.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); auto transactions = prepareAlterTransactions(new_columns, new_indices, context); @@ -279,11 +279,10 @@ void StorageMergeTree::alter( if (new_ttl_table_ast.get() != ttl_table_ast.get()) storage_ast.set(storage_ast.ttl_table, new_ttl_table_ast); }; - - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, storage_modifier); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier); /// Reinitialize primary key because primary key column types might have changed. - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices); + setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast); @@ -834,10 +833,11 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr ignored_order_by_ast; ASTPtr ignored_primary_key_ast; ASTPtr ignored_ttl_table_ast; - alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); + alter_command.apply(new_columns, new_indices, new_constraints, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); auto columns_for_parts = new_columns.getAllPhysical(); for (const auto & part : parts) diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index 1762c8372f5..73cb0243c11 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -38,8 +38,9 @@ void StorageNull::alter( ColumnsDescription new_columns = getColumns(); IndicesDescription new_indices = getIndices(); + ConstraintsDescription new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index aaf8f18b65b..250a9dc96e8 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -422,6 +422,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_order_by_ast = order_by_ast; auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr new_ttl_table_ast = ttl_table_ast; IDatabase::ASTModifier storage_modifier; if (!metadata_diff.empty()) @@ -451,6 +452,9 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column if (metadata_diff.skip_indices_changed) new_indices = IndicesDescription::parse(metadata_diff.new_skip_indices); + if (metadata_diff.constraints_changed) + new_constraints = ConstraintsDescription::parse(metadata_diff.new_constraints); + if (metadata_diff.ttl_table_changed) { ParserExpression parser; @@ -476,11 +480,11 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column }; } - global_context.getDatabase(database_name)->alterTable(global_context, table_name, new_columns, new_indices, storage_modifier); + global_context.getDatabase(database_name)->alterTable(global_context, table_name, new_columns, new_indices, new_constraints, storage_modifier); /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices); + setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast); } @@ -1507,10 +1511,11 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr ignored_order_by_ast; ASTPtr ignored_primary_key_ast; ASTPtr ignored_ttl_table_ast; - alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); + alter_command.apply(new_columns, new_indices, new_constraints, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); size_t modified_parts = 0; auto parts = getDataParts(); @@ -3114,10 +3119,11 @@ void StorageReplicatedMergeTree::alter( ColumnsDescription new_columns = getColumns(); IndicesDescription new_indices = getIndices(); + ConstraintsDescription new_constraints = getConstraints(); ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + params.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); String new_columns_str = new_columns.toString(); if (new_columns_str != getColumns().toString()) diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.reference b/dbms/tests/queries/0_stateless/00953_constraints_operations.reference new file mode 100644 index 00000000000..5713da9fef5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.reference @@ -0,0 +1,4 @@ +1 2 +ok +1 2 +ok diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh new file mode 100755 index 00000000000..917719c3e46 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +EXCEPTION_SUCCESS_TEXT=ok + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT b_constraint CHECK b > 0 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw and exception +EXCEPTION_TEXT="Some constraints are not satisfied" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE test_constraints DROP CONSTRAINT b_constraint;" + +# This one must suceed now +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE test_constraints ADD CONSTRAINT b_constraint CHECK b > 10;" + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 10);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 11);" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" \ No newline at end of file From 33f0ebd8ab83ef61990a66f7fb04d424f25cff5f Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 2 Jun 2019 18:08:28 +0300 Subject: [PATCH 026/357] Brace style fix --- dbms/src/Storages/AlterCommands.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 22095ec4ae3..0d20847727a 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -356,7 +356,8 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri return constraint_ast->as().name == constraint_name; }); - if (erase_it == constraints_description.constraints.end()) { + if (erase_it == constraints_description.constraints.end()) + { if (if_exists) return; throw Exception("Wrong constraint name. Cannot find constraint `" + constraint_name + "` to drop.", From b63623d0146965191cc5dcec728d25ec7d84ab46 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Wed, 5 Jun 2019 10:33:34 +0300 Subject: [PATCH 027/357] Extended constraint exception with constraint name and expression --- .../DataStreams/CheckConstraintsBlockOutputStream.cpp | 10 +++++++--- .../queries/0_stateless/00952_basic_constraints.sh | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 1cc271e5578..da77e4a1c2e 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -1,14 +1,18 @@ #include - +#include namespace DB { void CheckConstraintsBlockOutputStream::write(const Block & block) { - for (auto & constraint_expr: expressions) + for (size_t i = 0; i < expressions.size(); ++i) + { + auto constraint_expr = expressions[i]; if (!checkConstraintOnBlock(block, constraint_expr)) - throw Exception{"Some constraints are not satisfied", ErrorCodes::QUERY_WAS_CANCELLED}; + throw Exception{"Constraint " + constraints.constraints[i]->name + " is not satisfied at, constraint expression: " + + serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::LOGICAL_ERROR}; + } output->write(block); } diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index 93fa16ce4af..1d2a46dae61 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh EXCEPTION_SUCCESS_TEXT=ok - +$CLICKHOUSE_CLIENT --query="CREATE DATABASE IF NOT EXISTS test;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints @@ -20,7 +20,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Some constraints are not satisfied" +EXCEPTION_TEXT="Constraint b_constraint is not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" @@ -38,13 +38,13 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception -EXCEPTION_TEXT="Some constraints are not satisfied" +EXCEPTION_TEXT="Constraint b_constraint is not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception -EXCEPTION_TEXT="Some constraints are not satisfied" +EXCEPTION_TEXT="Constraint a_constraint is not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" From f413b1e346270e732988826e84e548857dc8174e Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Wed, 5 Jun 2019 11:05:46 +0300 Subject: [PATCH 028/357] Implemented memoryIsByte, replaced memoryIsZero with it, implemented memory check in CheckConstraintsBlockOutputStream --- dbms/src/Columns/ColumnsCommon.cpp | 43 ++++--------------- dbms/src/Columns/ColumnsCommon.h | 2 +- .../CheckConstraintsBlockOutputStream.cpp | 21 ++++++++- .../CheckConstraintsBlockOutputStream.h | 2 + 4 files changed, 32 insertions(+), 36 deletions(-) diff --git a/dbms/src/Columns/ColumnsCommon.cpp b/dbms/src/Columns/ColumnsCommon.cpp index 6ad3d0907ab..0745a3d5b9f 100644 --- a/dbms/src/Columns/ColumnsCommon.cpp +++ b/dbms/src/Columns/ColumnsCommon.cpp @@ -61,43 +61,18 @@ std::vector countColumnsSizeInSelector(IColumn::ColumnIndex num_columns, return counts; } -/** clang 4 generates better code than gcc 6. - * And both gcc and clang could not vectorize trivial loop by bytes automatically. - */ -bool memoryIsZero(const void * data, size_t size) +bool memoryIsByte(const void * data, size_t size, uint8_t byte) { - const Int8 * pos = reinterpret_cast(data); - const Int8 * end = pos + size; - -#ifdef __SSE2__ - const __m128 zero16 = _mm_setzero_ps(); - const Int8 * end64 = pos + size / 64 * 64; - - for (; pos < end64; pos += 64) - if (_mm_movemask_ps(_mm_cmpneq_ps( - _mm_loadu_ps(reinterpret_cast(pos)), - zero16)) - | _mm_movemask_ps(_mm_cmpneq_ps( - _mm_loadu_ps(reinterpret_cast(pos + 16)), - zero16)) - | _mm_movemask_ps(_mm_cmpneq_ps( - _mm_loadu_ps(reinterpret_cast(pos + 32)), - zero16)) - | _mm_movemask_ps(_mm_cmpneq_ps( - _mm_loadu_ps(reinterpret_cast(pos + 48)), - zero16))) - return false; - - /// TODO Add duff device for tail? -#endif - - for (; pos < end; ++pos) - if (*pos) - return false; - - return true; + if (size == 0) + return true; + auto ptr = reinterpret_cast(data); + return *ptr == byte && memcmp(ptr, ptr + 1, size - 1) == 0; } +bool memoryIsZero(const void * data, size_t size) +{ + return memoryIsByte(data, size, 0x0); +} namespace ErrorCodes { diff --git a/dbms/src/Columns/ColumnsCommon.h b/dbms/src/Columns/ColumnsCommon.h index 0b14b76ad39..46c6c0e9df3 100644 --- a/dbms/src/Columns/ColumnsCommon.h +++ b/dbms/src/Columns/ColumnsCommon.h @@ -22,7 +22,7 @@ std::vector countColumnsSizeInSelector(IColumn::ColumnIndex num_columns, /// Returns true, if the memory contains only zeros. bool memoryIsZero(const void * data, size_t size); - +bool memoryIsByte(const void * data, size_t size, uint8_t byte); /// The general implementation of `filter` function for ColumnArray and ColumnString. template diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index da77e4a1c2e..ac432694d83 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -1,5 +1,8 @@ #include +#include +#include #include +#include namespace DB { @@ -31,17 +34,33 @@ void CheckConstraintsBlockOutputStream::writeSuffix() output->writeSuffix(); } -bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) +bool CheckConstraintsBlockOutputStream::checkImplMemory(const Block & block, const ExpressionActionsPtr & constraint) +{ + Block res = block; + constraint->execute(res); + assert(block.columns() == res.columns() - 1); + ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); + auto res_column_uint8 = checkAndGetColumn(res_column.column.get()); + return memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1); +} + +bool CheckConstraintsBlockOutputStream::checkImplBool(const Block & block, const ExpressionActionsPtr & constraint) { Block res = block; constraint->execute(res); assert(block.columns() == res.columns() - 1); ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); size_t column_size = res_column.column->size(); + // std::cerr << "Sizes of constraints: " << res_column.column->size() << ' ' << res_column.column->get << '\n'; for (size_t i = 0; i < column_size; ++i) if (!res_column.column->getBool(i)) return false; return true; } +bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) +{ + return checkImplMemory(block, constraint); +} + } diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index 16b240eb758..6ea42cf44af 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -35,6 +35,8 @@ public: void writePrefix() override; void writeSuffix() override; + bool checkImplMemory(const Block & block, const ExpressionActionsPtr & constraint); + bool checkImplBool(const Block & block, const ExpressionActionsPtr & constraint); bool checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint); private: From bba3b33bdca89da53a8ac1707234b02959e8486c Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Wed, 5 Jun 2019 15:17:53 +0300 Subject: [PATCH 029/357] Fixed exception text in 00953_constraints_operations --- dbms/tests/queries/0_stateless/00953_constraints_operations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh index 917719c3e46..f0fc5b71fbf 100755 --- a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh @@ -20,7 +20,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Some constraints are not satisfied" +EXCEPTION_TEXT="Constraint b_constraint is not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" From 375e4640867c0318b47fbab3149a35a53d0d46bd Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Thu, 6 Jun 2019 00:25:48 +0300 Subject: [PATCH 030/357] Fixed exception text --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index ac432694d83..75067b6afa7 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -13,7 +13,7 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { auto constraint_expr = expressions[i]; if (!checkConstraintOnBlock(block, constraint_expr)) - throw Exception{"Constraint " + constraints.constraints[i]->name + " is not satisfied at, constraint expression: " + + throw Exception{"Constraint " + constraints.constraints[i]->name + " is not satisfied, constraint expression: " + serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::LOGICAL_ERROR}; } output->write(block); From 76772d1de095ff3c41174a0a914d544162d60c02 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Thu, 6 Jun 2019 01:25:57 +0300 Subject: [PATCH 031/357] Documentation on constraints (RU, EN) --- docs/en/query_language/alter.md | 16 ++++++++++++++++ docs/en/query_language/create.md | 20 ++++++++++++++++++++ docs/en/query_language/insert_into.md | 3 +++ docs/ru/query_language/alter.md | 16 ++++++++++++++++ docs/ru/query_language/create.md | 20 ++++++++++++++++++++ docs/ru/query_language/insert_into.md | 4 ++++ 6 files changed, 79 insertions(+) diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md index 85941987ce9..fc42fc636e7 100644 --- a/docs/en/query_language/alter.md +++ b/docs/en/query_language/alter.md @@ -166,6 +166,22 @@ are available: These commands are lightweight in a sense that they only change metadata or remove files. Also, they are replicated (syncing indices metadata through ZooKeeper). +### Manipulations with constraints + +See more on [constraints](create.md#constraints) + +Constraints could be added or deleted using following syntax: +``` +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +Queries will add or remove metadata about constraints from table so they are processed immediately. + +Constraint check *will not be executed* on existing table if it was added. For now, we recommend to create new table and use `INSERT SELECT` query to fill new table. + +All changes on distributed tables are broadcasting to ZooKeeper so will be applied on other replicas. + ### Manipulations With Partitions and Parts {#alter_manipulations-with-partitions} The following operations with [partitions](../operations/table_engines/custom_partitioning_key.md) are available: diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md index 9ebd50839d3..573388195e3 100644 --- a/docs/en/query_language/create.md +++ b/docs/en/query_language/create.md @@ -80,6 +80,26 @@ If you add a new column to a table but later change its default expression, the It is not possible to set default values for elements in nested data structures. +### Constraints {#constraints} + +WARNING: This feature is experimental. Correct work is not guaranteed on non-MergeTree family engines. + +Along with columns descriptions constraints could be defined: + +``sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` could by any boolean expression. If constraints are defined for the table, each of them will be checked for every row in `INSERT` query. If any constraint is not satisfied — server will raise an exception with constraint name and checking expression. + +Adding large amount of constraints can negatively affect performance of big `INSERT` queries. + ### TTL expression Can be specified only for MergeTree-family tables. An expression for setting storage time for values. It must depends on `Date` or `DateTime` column and has one `Date` or `DateTime` column as a result. Example: diff --git a/docs/en/query_language/insert_into.md b/docs/en/query_language/insert_into.md index 914c3b2917f..c0cb9f8c3b1 100644 --- a/docs/en/query_language/insert_into.md +++ b/docs/en/query_language/insert_into.md @@ -40,6 +40,9 @@ INSERT INTO t FORMAT TabSeparated You can insert data separately from the query by using the command-line client or the HTTP interface. For more information, see the section "[Interfaces](../interfaces/index.md#interfaces)". +### Constraints + +If table has [constraints](create.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. ### Inserting The Results of `SELECT` {#insert_query_insert-select} diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md index 2367386172a..5e847abce4d 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/query_language/alter.md @@ -165,6 +165,22 @@ ALTER TABLE [db].name DROP INDEX name Запрос на изменение индексов реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. +### Манипуляции с ограничениями (constraints) + +Про ограничения подробнее написано [тут](create.md#constraints). + +Добавить или удалить ограничение можно с помощью запросов +``` +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +Запросы выполняют добавление или удаление метаданных об ограничениях таблицы `[db].name`, поэтому выполняются мнгновенно. + +Если ограничение появилось для непустой таблицы, то *проверка ограничения вызвана не будет*. Если же важно добавить ограничение на существующую таблицу, то рекомендуется создать новую таблицу с нужным ограничением и выполнить `INSERT SELECT` запрос для перекачки данных из одной таблицы в другую. + +Запрос на изменение ограничений так же, как и с индексами, реплицируется через ZooKeeper. + ### Манипуляции с партициями и кусками {#alter_manipulations-with-partitions} Для работы с [партициями](../operations/table_engines/custom_partitioning_key.md) доступны следующие операции: diff --git a/docs/ru/query_language/create.md b/docs/ru/query_language/create.md index ee6dc3c7820..0b842e22e47 100644 --- a/docs/ru/query_language/create.md +++ b/docs/ru/query_language/create.md @@ -80,6 +80,26 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... Отсутствует возможность задать значения по умолчанию для элементов вложенных структур данных. +### Ограничения (constraints) {#constraints} + +WARNING: Находится в экспериментальном режиме, поддержано в MergeTree (работоспособность на других типах движков таблиц не гарантируется). + +Наряду с объявлением столбцов можно объявить ограчения на значения в столбцах таблицы: + +```sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` может быть любым булевым выражением, состоящим из операторов сравнения или функций. При наличии одного или нескольких ограничений в момент вставки данных выражения ограничений будут проверяться на истинность для каждой вставляемой строки данных. В случае, если в теле INSERT запроса придут некорректные данные — клиентов будет выкинуто исключение с нарушенным ограничением. + +Добавление большого числа ограничений может негативно повлиять на производительность объёмных `INSERT` запросов. + ### Выражение для TTL Может быть указано только для таблиц семейства MergeTree. Выражение для указания времени хранения значений. Оно должно зависеть от стобца типа `Date` или `DateTime` и в качестве результата вычислять столбец типа `Date` или `DateTime`. Пример: diff --git a/docs/ru/query_language/insert_into.md b/docs/ru/query_language/insert_into.md index 356b720e157..454339ebcdb 100644 --- a/docs/ru/query_language/insert_into.md +++ b/docs/ru/query_language/insert_into.md @@ -40,6 +40,10 @@ INSERT INTO t FORMAT TabSeparated С помощью консольного клиента или HTTP интерфейса можно вставлять данные отдельно от запроса. Как это сделать, читайте в разделе "[Интерфейсы](../interfaces/index.md#interfaces)". +### Ограничения (constraints) + +Если в таблице объявлены [ограничения](create.md#constraints), то их выполнимость будет проверена для каждой вставляемой строки. Если для хотя бы одной строки ограничения не будут выполнены, запрос будет остановлен. + ### Вставка результатов `SELECT` {#insert_query_insert-select} ``` sql From bb78012cf91e21d7394d001eba6480918ab4fb01 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 30 Jun 2019 12:09:06 +0200 Subject: [PATCH 032/357] Removed unused method and assertions --- .../CheckConstraintsBlockOutputStream.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 75067b6afa7..ec4a7bd45b8 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -38,26 +38,11 @@ bool CheckConstraintsBlockOutputStream::checkImplMemory(const Block & block, con { Block res = block; constraint->execute(res); - assert(block.columns() == res.columns() - 1); ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); auto res_column_uint8 = checkAndGetColumn(res_column.column.get()); return memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1); } -bool CheckConstraintsBlockOutputStream::checkImplBool(const Block & block, const ExpressionActionsPtr & constraint) -{ - Block res = block; - constraint->execute(res); - assert(block.columns() == res.columns() - 1); - ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); - size_t column_size = res_column.column->size(); - // std::cerr << "Sizes of constraints: " << res_column.column->size() << ' ' << res_column.column->get << '\n'; - for (size_t i = 0; i < column_size; ++i) - if (!res_column.column->getBool(i)) - return false; - return true; -} - bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) { return checkImplMemory(block, constraint); From ea4d42c3aca78eb6d114ea3703042649cc0db110 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 30 Jun 2019 12:09:58 +0200 Subject: [PATCH 033/357] Merged DROP_(COLUMN|INDEX|CONSTRAINT) AST parsing into one block --- dbms/src/Storages/AlterCommands.cpp | 39 ++++++++++------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 80c01a0028e..1ce70723238 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -74,17 +74,6 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } - else if (command_ast->type == ASTAlterCommand::DROP_COLUMN && !command_ast->partition) - { - if (command_ast->clear_column) - throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); - - AlterCommand command; - command.type = AlterCommand::DROP_COLUMN; - command.column_name = *getIdentifierName(command_ast->column); - command.if_exists = command_ast->if_exists; - return command; - } else if (command_ast->type == ASTAlterCommand::MODIFY_COLUMN) { AlterCommand command; @@ -154,18 +143,6 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } - else if (command_ast->type == ASTAlterCommand::DROP_INDEX) - { - if (command_ast->clear_column) - throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); - - AlterCommand command; - command.type = AlterCommand::DROP_INDEX; - command.index_name = command_ast->index->as().name; - command.if_exists = command_ast->if_exists; - - return command; - } else if (command_ast->type == ASTAlterCommand::ADD_CONSTRAINT) { AlterCommand command; @@ -180,15 +157,25 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } - else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) + else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT + || command_ast->type == ASTAlterCommand::DROP_INDEX + || (command_ast->type == ASTAlterCommand::DROP_COLUMN && !command_ast->partition)) { if (command_ast->clear_column) throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; - command.type = AlterCommand::DROP_CONSTRAINT; - command.constraint_name = command_ast->constraint->as().name; command.if_exists = command_ast->if_exists; + if (command_ast->type == ASTAlterCommand::DROP_INDEX) { + command.type = AlterCommand::DROP_INDEX; + command.index_name = command_ast->index->as().name; + } else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) { + command.type = AlterCommand::DROP_CONSTRAINT; + command.constraint_name = command_ast->constraint->as().name; + } else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) { + command.type = AlterCommand::DROP_COLUMN; + command.column_name = *getIdentifierName(command_ast->column); + } return command; } From 06e92e14b828c1457410d04aa8682acbda11d182 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 5 Jul 2019 10:10:15 +0300 Subject: [PATCH 034/357] alterTable fix in MySQL --- dbms/src/Databases/DatabaseMySQL.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Databases/DatabaseMySQL.h b/dbms/src/Databases/DatabaseMySQL.h index 7ce836d6a64..01a8c1df0d2 100644 --- a/dbms/src/Databases/DatabaseMySQL.h +++ b/dbms/src/Databases/DatabaseMySQL.h @@ -71,7 +71,7 @@ public: throw Exception("MySQL database engine does not support create table.", ErrorCodes::NOT_IMPLEMENTED); } - void alterTable(const Context &, const String &, const ColumnsDescription &, const IndicesDescription &, const ASTModifier &) override + void alterTable(const Context &, const String &, const ColumnsDescription &, const IndicesDescription &, const ConstraintsDescription &, const ASTModifier &) override { throw Exception("MySQL database engine does not support alter table.", ErrorCodes::NOT_IMPLEMENTED); } From 430400c3c5206c383a1e0870f00a77f5923147a3 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 5 Jul 2019 10:16:34 +0300 Subject: [PATCH 035/357] Style fix --- dbms/src/Storages/AlterCommands.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 8b3713ae31f..c90d5ca2c25 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -166,13 +166,16 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ AlterCommand command; command.if_exists = command_ast->if_exists; - if (command_ast->type == ASTAlterCommand::DROP_INDEX) { + if (command_ast->type == ASTAlterCommand::DROP_INDEX) + { command.type = AlterCommand::DROP_INDEX; command.index_name = command_ast->index->as().name; - } else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) { + } else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) + { command.type = AlterCommand::DROP_CONSTRAINT; command.constraint_name = command_ast->constraint->as().name; - } else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) { + } else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) + { command.type = AlterCommand::DROP_COLUMN; command.column_name = *getIdentifierName(command_ast->column); } From 3757aa9d020046d0a825a27546b43066ced2ffec Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 6 Jul 2019 23:30:48 +0300 Subject: [PATCH 036/357] More style fix --- dbms/src/Storages/AlterCommands.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index c90d5ca2c25..b250452c683 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -170,11 +170,13 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ { command.type = AlterCommand::DROP_INDEX; command.index_name = command_ast->index->as().name; - } else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) + } + else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) { command.type = AlterCommand::DROP_CONSTRAINT; command.constraint_name = command_ast->constraint->as().name; - } else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) + } + else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) { command.type = AlterCommand::DROP_COLUMN; command.column_name = *getIdentifierName(command_ast->column); From c1d91222d50e368529e0c6675b4a6a6bb6681103 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 13 Jul 2019 13:42:52 +0300 Subject: [PATCH 037/357] Ranamed setPrimaryKeyIndicesAndColumns to setProperties --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 6 +++--- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 2 +- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 3c30e7ce778..c2aa0a61855 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -121,7 +121,7 @@ MergeTreeData::MergeTreeData( data_parts_by_info(data_parts_indexes.get()), data_parts_by_state_and_info(data_parts_indexes.get()) { - setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_); + setProperties(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_); setConstraints(constraints_); /// NOTE: using the same columns list as is read when performing actual merges. @@ -232,7 +232,7 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam } -void MergeTreeData::setPrimaryKeyIndicesAndColumns( +void MergeTreeData::setProperties( const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, const ColumnsDescription & new_columns, const IndicesDescription & indices_description, const ConstraintsDescription & constraints_description, bool only_check) @@ -1262,7 +1262,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c } } - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, + setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints, /* only_check = */ true); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast, /* only_check = */ true); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 7572e9f2856..d4848462df0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -757,7 +757,7 @@ protected: /// The same for clearOldTemporaryDirectories. std::mutex clear_old_temporary_directories_mutex; - void setPrimaryKeyIndicesAndColumns(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, + void setProperties(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, const ColumnsDescription & new_columns, const IndicesDescription & indices_description, const ConstraintsDescription & constraints_description, bool only_check = false); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 7cf8facb5f2..b7adc8c5bff 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -282,7 +282,7 @@ void StorageMergeTree::alter( context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier); /// Reinitialize primary key because primary key column types might have changed. - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); + setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index f1f4a4049f8..ae0ccfaaf1e 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -484,7 +484,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); + setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast); } From 3c2172b750989b72d9290615aeb669a6b90dc096 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 22 Jul 2019 14:23:11 +0300 Subject: [PATCH 038/357] parse and interpret query --- dbms/src/Core/Settings.h | 3 ++- dbms/src/Interpreters/InterpreterAlterQuery.cpp | 7 +++++++ dbms/src/Parsers/ASTAlterQuery.cpp | 6 ++++++ dbms/src/Parsers/ASTAlterQuery.h | 3 ++- dbms/src/Parsers/ParserAlterQuery.cpp | 17 +++++++++++++++++ dbms/src/Storages/PartitionCommands.cpp | 11 +++++++++++ dbms/src/Storages/PartitionCommands.h | 3 ++- dbms/src/Storages/StorageMergeTree.cpp | 4 ++++ .../src/Storages/StorageReplicatedMergeTree.cpp | 4 ++++ 9 files changed, 55 insertions(+), 3 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index b1182cae9bf..4bb76039cab 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -336,7 +336,8 @@ struct Settings : public SettingsCollection \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ - M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") + M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ + M(SettingBool, allow_drop_detached_part, false, "Allow ALTER TABLE ... DROP DETACHED PART ... queries") DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 8751ff067b1..074fbb7d4c2 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int ILLEGAL_COLUMN; + extern const int SUPPORT_IS_DISABLED; } @@ -53,7 +54,13 @@ BlockIO InterpreterAlterQuery::execute() if (auto alter_command = AlterCommand::parse(command_ast)) alter_commands.emplace_back(std::move(*alter_command)); else if (auto partition_command = PartitionCommand::parse(command_ast)) + { + if (partition_command->type == PartitionCommand::DROP_DETACHED_PARTITION + && !context.getSettingsRef().allow_drop_detached_part) + throw DB::Exception("Cannot execute query: DROP DETACHED PART is disabled " + "(see allow_drop_detached setting)", ErrorCodes::SUPPORT_IS_DISABLED); partition_commands.emplace_back(std::move(*partition_command)); + } else if (auto mut_command = MutationCommand::parse(command_ast)) mutation_commands.emplace_back(std::move(*mut_command)); else diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index c7cd100b415..6d87156a19b 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -118,6 +118,12 @@ void ASTAlterCommand::formatImpl( << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } + else if (type == ASTAlterCommand::DROP_DETACHED_PARTITION) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DROP DETACHED" << (part ? " PART " : " PARTITION ") + << (settings.hilite ? hilite_none : ""); + partition->formatImpl(settings, state, frame); + } else if (type == ASTAlterCommand::ATTACH_PARTITION) { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ATTACH " diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 2c4b3ddbaf1..d6a54812960 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -33,6 +33,7 @@ public: DROP_INDEX, DROP_PARTITION, + DROP_DETACHED_PARTITION, ATTACH_PARTITION, REPLACE_PARTITION, FETCH_PARTITION, @@ -90,7 +91,7 @@ public: bool detach = false; /// true for DETACH PARTITION - bool part = false; /// true for ATTACH PART + bool part = false; /// true for ATTACH PART and DROP DETACHED PART bool clear_column = false; /// for CLEAR COLUMN (do not drop column from metadata) diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 98891bbdf5f..75c6f6291a8 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -35,6 +35,8 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_attach_partition("ATTACH PARTITION"); ParserKeyword s_detach_partition("DETACH PARTITION"); ParserKeyword s_drop_partition("DROP PARTITION"); + ParserKeyword s_drop_detached_partition("DROP DETACHED PARTITION"); + ParserKeyword s_drop_detached_part("DROP DETACHED PART"); ParserKeyword s_attach_part("ATTACH PART"); ParserKeyword s_fetch_partition("FETCH PARTITION"); ParserKeyword s_replace_partition("REPLACE PARTITION"); @@ -87,6 +89,21 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->type = ASTAlterCommand::DROP_PARTITION; } + else if (s_drop_detached_partition.ignore(pos, expected)) + { + if (!parser_partition.parse(pos, command->partition, expected)) + return false; + + command->type = ASTAlterCommand::DROP_DETACHED_PARTITION; + } + else if (s_drop_detached_part.ignore(pos, expected)) + { + if (!parser_string_literal.parse(pos, command->partition, expected)) + return false; + + command->type = ASTAlterCommand::DROP_DETACHED_PARTITION; + command->part = true; + } else if (s_drop_column.ignore(pos, expected)) { if (s_if_exists.ignore(pos, expected)) diff --git a/dbms/src/Storages/PartitionCommands.cpp b/dbms/src/Storages/PartitionCommands.cpp index f6aaee4c70e..bab3f6ced24 100644 --- a/dbms/src/Storages/PartitionCommands.cpp +++ b/dbms/src/Storages/PartitionCommands.cpp @@ -23,6 +23,17 @@ std::optional PartitionCommand::parse(const ASTAlterCommand * res.detach = command_ast->detach; return res; } + else if (command_ast->type == ASTAlterCommand::DROP_DETACHED_PARTITION) + { + if (!command_ast->part) // TODO + throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + + PartitionCommand res; + res.type = DROP_DETACHED_PARTITION; + res.partition = command_ast->partition; + res.part = command_ast->part; + return res; + } else if (command_ast->type == ASTAlterCommand::ATTACH_PARTITION) { PartitionCommand res; diff --git a/dbms/src/Storages/PartitionCommands.h b/dbms/src/Storages/PartitionCommands.h index 1f66c3f0c30..cb71a02548c 100644 --- a/dbms/src/Storages/PartitionCommands.h +++ b/dbms/src/Storages/PartitionCommands.h @@ -21,6 +21,7 @@ struct PartitionCommand ATTACH_PARTITION, CLEAR_COLUMN, DROP_PARTITION, + DROP_DETACHED_PARTITION, FETCH_PARTITION, FREEZE_ALL_PARTITIONS, FREEZE_PARTITION, @@ -35,7 +36,7 @@ struct PartitionCommand /// true for DETACH PARTITION. bool detach = false; - /// true for ATTACH PART (and false for PARTITION) + /// true for ATTACH PART and DROP DETACHED PART (and false for PARTITION) bool part = false; /// For ATTACH PARTITION partition FROM db.table diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index d021866487c..6e527c0c6c1 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -920,6 +920,10 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma dropPartition(command.partition, command.detach, context); break; + case PartitionCommand::DROP_DETACHED_PARTITION: + // TODO + throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + case PartitionCommand::ATTACH_PARTITION: attachPartition(command.partition, command.part, context); break; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index b51da168192..5f91c304e98 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3348,6 +3348,10 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part dropPartition(query, command.partition, command.detach, query_context); break; + case PartitionCommand::DROP_DETACHED_PARTITION: + // TODO + throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + case PartitionCommand::ATTACH_PARTITION: attachPartition(command.partition, command.part, query_context); break; From ad787938f5fa247901da7003c5717fed6a838445 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Jul 2019 22:43:33 +0300 Subject: [PATCH 039/357] better detached part name parsing --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 17 +------------- .../Storages/MergeTree/MergeTreePartInfo.cpp | 22 +++++++++++++++++++ .../Storages/MergeTree/MergeTreePartInfo.h | 6 +++++ .../System/StorageSystemDetachedParts.cpp | 1 + 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b32470f9f77..6a7b6d5405e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2579,22 +2579,7 @@ MergeTreeData::getDetachedParts() const res.emplace_back(); auto & part = res.back(); - /// First, try to parse as . - if (MergeTreePartInfo::tryParsePartName(dir_name, &part, format_version)) - continue; - - /// Next, as _. Use entire name as prefix if it fails. - part.prefix = dir_name; - const auto first_separator = dir_name.find_first_of('_'); - if (first_separator == String::npos) - continue; - - const auto part_name = dir_name.substr(first_separator + 1, - dir_name.size() - first_separator - 1); - if (!MergeTreePartInfo::tryParsePartName(part_name, &part, format_version)) - continue; - - part.prefix = dir_name.substr(0, first_separator); + DetachedPartInfo::tryParseDetachedPartName(dir_name, &part, format_version); } return res; } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 19f77448110..732cc3436f4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -188,4 +188,26 @@ String MergeTreePartInfo::getPartNameV0(DayNum left_date, DayNum right_date) con return wb.str(); } +bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, + MergeTreeDataFormatVersion format_version) +{ + /// First, try to parse as . + if (MergeTreePartInfo::tryParsePartName(dir_name, part_info, format_version)) + return part_info->valid_name = true; + + /// Next, as _. Use entire name as prefix if it fails. + part_info->prefix = dir_name; + const auto first_separator = dir_name.find_first_of('_'); + if (first_separator == String::npos) + return part_info->valid_name = false; + + // TODO what if contains '_'? + const auto part_name = dir_name.substr(first_separator + 1, + dir_name.size() - first_separator - 1); + if (!MergeTreePartInfo::tryParsePartName(part_name, part_info, format_version)) + return part_info->valid_name = false; + + part_info->prefix = dir_name.substr(0, first_separator); + return part_info->valid_name = true; +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index e80664c3dd9..2a168086a1c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -93,6 +93,12 @@ struct MergeTreePartInfo struct DetachedPartInfo : public MergeTreePartInfo { String prefix; + + /// If false, prefix contains full directory name and MergeTreePartInfo may be in invalid state + /// (directory name was not successfully parsed). + bool valid_name; + + static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, MergeTreeDataFormatVersion format_version); }; } diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index 9ae6f7b607a..9f33a60b84a 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -28,6 +28,7 @@ public: protected: explicit StorageSystemDetachedParts() { + // TODO add column "directory_name" or "is_valid_name" setColumns(ColumnsDescription{{ {"database", std::make_shared()}, {"table", std::make_shared()}, From 6e4aabbb1a4074d1a979190b063ffbe959894ebc Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Jul 2019 23:18:18 +0300 Subject: [PATCH 040/357] draft for StorageMergeTree --- dbms/src/Storages/StorageMergeTree.cpp | 33 ++++++++++++++++++++++++-- dbms/src/Storages/StorageMergeTree.h | 1 + 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 6e527c0c6c1..bd09588981b 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -921,8 +921,8 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma break; case PartitionCommand::DROP_DETACHED_PARTITION: - // TODO - throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + dropDetached(command.partition, command.part, context); + break; case PartitionCommand::ATTACH_PARTITION: attachPartition(command.partition, command.part, context); @@ -993,6 +993,34 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } +void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const Context & /*context*/) +{ + if (!part) // TODO + throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); + + String part_id = partition->as().value.safeGet(); + Poco::Path part_path(part_id); + const bool file_zero_depth = part_path.isFile() && part_path.depth() == 0 && part_path.getFileName() != ".."; + const bool dir_zero_depth = part_path.isDirectory() && part_path.depth() == 1 && part_path.directory(0) != ".."; + const bool zero_depth = file_zero_depth || dir_zero_depth; + if (!part_path.isRelative() || !zero_depth) + throw DB::Exception("Part name must contain exactly one path component: name of detached part", ErrorCodes::INCORRECT_FILE_NAME); + + part_id = part_path.isFile() ? part_path.getFileName() : part_path.directory(0); + Poco::Path base_dir(full_path + "detached"); + Poco::File detached_part_dir(Poco::Path(base_dir, part_id)); + if (!detached_part_dir.exists()) + throw DB::Exception("Detached part \"" + part_id + "\" not found" , ErrorCodes::INCORRECT_FILE_NAME); + + DetachedPartInfo info; + DetachedPartInfo::tryParseDetachedPartName(part_id, &info, format_version); + MergeTreeDataPart detached_part(*this, part_id, info); + detached_part.relative_path = "detached/" + part_id; + + // TODO make sure it's ok + detached_part.remove(); +} + void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) { // TODO: should get some locks to prevent race with 'alter … modify column' @@ -1039,6 +1067,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par LOG_DEBUG(log, "Checking data"); MutableDataPartPtr part = loadPartAndFixMetadata(source_path); + // TODO fix race with DROP DETACHED PARTITION LOG_INFO(log, "Attaching part " << source_part_name << " from " << source_path); renameTempPartAndAdd(part, &increment); diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 0de9618d915..fa2561e4ab2 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -120,6 +120,7 @@ private: // Partition helpers void dropPartition(const ASTPtr & partition, bool detach, const Context & context); + void dropDetached(const ASTPtr & partition, bool part, const Context & context); void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); From 8a00ce5ff1c413cfa22185795682ccf084cdb533 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 24 Jul 2019 15:56:39 +0300 Subject: [PATCH 041/357] Intermediate step --- dbms/src/Common/ErrorCodes.cpp | 1 + dbms/src/Parsers/ASTAlterQuery.cpp | 10 +++++ dbms/src/Parsers/ASTAlterQuery.h | 6 ++- dbms/src/Storages/AlterCommands.cpp | 42 +++++++++++++++---- dbms/src/Storages/AlterCommands.h | 16 ++++--- dbms/src/Storages/IStorage.cpp | 8 +++- dbms/src/Storages/IStorage.h | 5 ++- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 11 ++++- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 +- dbms/src/Storages/StorageBuffer.cpp | 2 +- dbms/src/Storages/StorageDistributed.cpp | 2 +- dbms/src/Storages/StorageMerge.cpp | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 29 +++++++++++-- dbms/src/Storages/StorageMergeTree.h | 3 ++ dbms/src/Storages/StorageNull.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 6 ++- 16 files changed, 121 insertions(+), 26 deletions(-) diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index c472a336d73..d0a5edab1d3 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -437,6 +437,7 @@ namespace ErrorCodes extern const int CANNOT_CREATE_TIMER = 460; extern const int CANNOT_SET_TIMER_PERIOD = 461; extern const int CANNOT_DELETE_TIMER = 462; + extern const int SETTINGS_ARE_NOT_SUPPORTED = 463; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index c7cd100b415..52aa95d65e5 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -45,6 +45,11 @@ ASTPtr ASTAlterCommand::clone() const res->ttl = ttl->clone(); res->children.push_back(res->ttl); } + if (settings_changes) + { + res->settings_changes = settings_changes->clone(); + res->children.push_back(res->settings_changes); + } return res; } @@ -184,6 +189,11 @@ void ASTAlterCommand::formatImpl( settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY TTL " << (settings.hilite ? hilite_none : ""); ttl->formatImpl(settings, state, frame); } + else if (type == ASTAlterCommand::MODIFY_SETTING) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY SETTING " << (settings.hilite ? hilite_none : ""); + settings_changes->formatImpl(settings, state, frame); + } else throw Exception("Unexpected type of ALTER", ErrorCodes::UNEXPECTED_AST_STRUCTURE); } diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 2c4b3ddbaf1..96ec8d91255 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -28,6 +28,7 @@ public: COMMENT_COLUMN, MODIFY_ORDER_BY, MODIFY_TTL, + MODIFY_SETTING, ADD_INDEX, DROP_INDEX, @@ -69,7 +70,7 @@ public: /** The ADD INDEX query stores the name of the index following AFTER. * The DROP INDEX query stores the name for deletion. */ - ASTPtr index; + ASTPtr index; /** Used in DROP PARTITION and ATTACH PARTITION FROM queries. * The value or ID of the partition is stored here. @@ -88,6 +89,9 @@ public: /// For MODIFY TTL query ASTPtr ttl; + /// FOR MODIFY_SETTING + ASTPtr settings_changes; + bool detach = false; /// true for DETACH PARTITION bool part = false; /// true for ATTACH PART diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 7814f1a6ba0..4b733f86d22 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int UNKNOWN_SETTING; } @@ -172,13 +174,20 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ command.ttl = command_ast->ttl; return command; } + else if (command_ast->type == ASTAlterCommand::MODIFY_SETTING) + { + AlterCommand command; + command.type = AlterCommand::MODIFY_SETTING; + command.settings_changes = command_ast->settings_changes->as().changes; + return command; + } else return {}; } void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast, SettingsChanges & changes) const { if (type == ADD_COLUMN) { @@ -306,38 +315,43 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri { ttl_table_ast = ttl; } + else if (type == MODIFY_SETTING) + { + changes.insert(changes.begin(), settings_changes.begin(), settings_changes.end()); + } else throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); } bool AlterCommand::isMutable() const { - if (type == COMMENT_COLUMN) + if (type == COMMENT_COLUMN || type == MODIFY_SETTING) return false; if (type == MODIFY_COLUMN) return data_type.get() || default_expression; - // TODO: возможно, здесь нужно дополнить return true; } void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast, SettingsChanges & changes) const { auto new_columns_description = columns_description; auto new_indices_description = indices_description; auto new_order_by_ast = order_by_ast; auto new_primary_key_ast = primary_key_ast; auto new_ttl_table_ast = ttl_table_ast; + auto new_changes = changes; for (const AlterCommand & command : *this) if (!command.ignore) - command.apply(new_columns_description, new_indices_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + command.apply(new_columns_description, new_indices_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, changes); columns_description = std::move(new_columns_description); indices_description = std::move(new_indices_description); order_by_ast = std::move(new_order_by_ast); primary_key_ast = std::move(new_primary_key_ast); ttl_table_ast = std::move(new_ttl_table_ast); + changes = std::move(new_changes); } void AlterCommands::validate(const IStorage & table, const Context & context) @@ -453,6 +467,16 @@ void AlterCommands::validate(const IStorage & table, const Context & context) throw Exception{"Wrong column name. Cannot find column " + command.column_name + " to comment", ErrorCodes::ILLEGAL_COLUMN}; } } + else if (command.type == AlterCommand::MODIFY_SETTING) + { + for (const auto & change : command.settings_changes) + { + if (!table.hasSetting(change.name)) + { + throw Exception{"Storage '" + table.getName() + "' doesn't have setting '" + change.name + "'", ErrorCodes::UNKNOWN_SETTING}; + } + } + } } /** Existing defaulted columns may require default expression extensions with a type conversion, @@ -518,14 +542,15 @@ void AlterCommands::validate(const IStorage & table, const Context & context) } } -void AlterCommands::apply(ColumnsDescription & columns_description) const +void AlterCommands::applyForColumnsOnly(ColumnsDescription & columns_description) const { auto out_columns_description = columns_description; IndicesDescription indices_description; ASTPtr out_order_by; ASTPtr out_primary_key; ASTPtr out_ttl_table; - apply(out_columns_description, indices_description, out_order_by, out_primary_key, out_ttl_table); + SettingsChanges out_changes; + apply(out_columns_description, indices_description, out_order_by, out_primary_key, out_ttl_table, out_changes); if (out_order_by) throw Exception("Storage doesn't support modifying ORDER BY expression", ErrorCodes::NOT_IMPLEMENTED); @@ -535,6 +560,9 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const throw Exception("Storage doesn't support modifying indices", ErrorCodes::NOT_IMPLEMENTED); if (out_ttl_table) throw Exception("Storage doesn't support modifying TTL expression", ErrorCodes::NOT_IMPLEMENTED); + if (!out_changes.empty()) + throw Exception("Storage doesn't support modifying settings", ErrorCodes::NOT_IMPLEMENTED); + columns_description = std::move(out_columns_description); } diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index 4905b80f92f..6daafe5cd7f 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -27,6 +28,7 @@ struct AlterCommand DROP_INDEX, MODIFY_TTL, UKNOWN_TYPE, + MODIFY_SETTING, }; Type type = UKNOWN_TYPE; @@ -71,6 +73,9 @@ struct AlterCommand /// For ADD and MODIFY CompressionCodecPtr codec; + /// For MODIFY SETTING + SettingsChanges settings_changes; + AlterCommand() = default; AlterCommand(const Type type, const String & column_name, const DataTypePtr & data_type, const ColumnDefaultKind default_kind, const ASTPtr & default_expression, @@ -84,7 +89,7 @@ struct AlterCommand static std::optional parse(const ASTAlterCommand * command); void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast, SettingsChanges & changes) const; /// Checks that not only metadata touched by that command bool isMutable() const; @@ -95,11 +100,12 @@ class Context; class AlterCommands : public std::vector { public: - void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, ASTPtr & order_by_ast, - ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; - /// For storages that don't support MODIFY_ORDER_BY. - void apply(ColumnsDescription & columns_description) const; + void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, ASTPtr & order_by_ast, + ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast, SettingsChanges & changes) const; + + /// Used for primiteive table engines, where only columns metadata can be changed + void applyForColumnsOnly(ColumnsDescription & columns_description) const; void validate(const IStorage & table, const Context & context); bool isMutable() const; diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 687ca970311..0fd1994f5ce 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -18,6 +18,7 @@ namespace ErrorCodes extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int TYPE_MISMATCH; + extern const int SETTINGS_ARE_NOT_SUPPORTED; } IStorage::IStorage(ColumnsDescription columns_) @@ -292,6 +293,11 @@ bool IStorage::isVirtualColumn(const String & column_name) const return getColumns().get(column_name).is_virtual; } +bool IStorage::hasSetting(const String & /* setting_name */) const +{ + throw Exception("Storage '" + getName() + "' doesn't support settings.", ErrorCodes::SETTINGS_ARE_NOT_SUPPORTED); +} + TableStructureReadLockHolder IStorage::lockStructureForShare(bool will_add_new_data, const String & query_id) { TableStructureReadLockHolder result; @@ -362,7 +368,7 @@ void IStorage::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); - params.apply(new_columns); + params.applyForColumnsOnly(new_columns); context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 477c4456b17..f269dce220f 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -129,6 +130,9 @@ public: /// thread-unsafe part. lockStructure must be acquired /// If |need_all| is set, then checks that all the columns of the table are in the block. void check(const Block & block, bool need_all = false) const; + /// Check storage has setting + virtual bool hasSetting(const String & setting_name) const; + protected: /// still thread-unsafe part. void setColumns(ColumnsDescription columns_); /// sets only real columns, possibly overwrites virtual ones. void setIndices(IndicesDescription indices_); @@ -136,7 +140,6 @@ protected: /// still thread-unsafe part. /// Returns whether the column is virtual - by default all columns are real. /// Initially reserved virtual column name may be shadowed by real column. virtual bool isVirtualColumn(const String & column_name) const; - private: ColumnsDescription columns; /// combined real and virtual columns const ColumnsDescription virtuals = {}; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b32470f9f77..25e4246a414 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -87,6 +87,7 @@ namespace ErrorCodes extern const int CANNOT_MUNMAP; extern const int CANNOT_MREMAP; extern const int BAD_TTL_EXPRESSION; + extern const int UNKNOWN_SETTING; } @@ -1174,7 +1175,8 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - commands.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + SettingsChanges new_changes; + commands.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes); if (getIndices().empty() && !new_indices.empty() && !context.getSettingsRef().allow_experimental_data_skipping_indices) @@ -1262,6 +1264,13 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast, /* only_check = */ true); + for (const auto & setting : new_changes) + { + if (!hasSetting(setting.name)) + throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting.name + "'", ErrorCodes::UNKNOWN_SETTING}; + + } + /// Check that type conversions are possible. ExpressionActionsPtr unused_expression; NameToNameMap unused_map; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 29962382749..33200500667 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -629,7 +629,7 @@ public: String sampling_expr_column_name; Names columns_required_for_sampling; - const MergeTreeSettings settings; + MergeTreeSettings settings; /// Limiting parallel sends per one table, used in DataPartsExchange std::atomic_uint current_table_sends {0}; diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 3c334b2a48b..6a857675018 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -701,7 +701,7 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_ auto new_columns = getColumns(); auto new_indices = getIndices(); - params.apply(new_columns); + params.applyForColumnsOnly(new_columns); context.getDatabase(database_name_)->alterTable(context, table_name_, new_columns, new_indices, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 27ceb1f45db..e8aed6d5702 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -350,7 +350,7 @@ void StorageDistributed::alter( auto new_columns = getColumns(); auto new_indices = getIndices(); - params.apply(new_columns); + params.applyForColumnsOnly(new_columns); context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 4c029fab677..a0454185307 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -401,7 +401,7 @@ void StorageMerge::alter( auto new_columns = getColumns(); auto new_indices = getIndices(); - params.apply(new_columns); + params.applyForColumnsOnly(new_columns); context.getDatabase(database_name_)->alterTable(context, table_name_, new_columns, new_indices, {}); setColumns(new_columns); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 0536423101d..d8360cb61e4 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int CANNOT_ASSIGN_OPTIMIZE; extern const int INCOMPATIBLE_COLUMNS; + extern const int UNKNOWN_SETTING; } namespace ActionLocks @@ -245,9 +247,21 @@ void StorageMergeTree::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); - params.apply(new_columns); + ASTPtr new_order_by_ast = order_by_ast; + ASTPtr new_primary_key_ast = primary_key_ast; + ASTPtr new_ttl_table_ast = ttl_table_ast; + SettingsChanges new_changes; + params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes); + IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) + { + auto & storage_ast = ast.as(); + if (!new_changes.empty()) + storage_ast.settings->changes.insert(storage_ast.settings->changes.begin(), new_changes.begin(), new_changes.end()); + }; + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); setColumns(std::move(new_columns)); + settings.applyChanges(new_changes); return; } @@ -263,7 +277,8 @@ void StorageMergeTree::alter( ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + SettingsChanges new_changes; + params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes); auto transactions = prepareAlterTransactions(new_columns, new_indices, context); @@ -844,7 +859,9 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi ASTPtr ignored_order_by_ast; ASTPtr ignored_primary_key_ast; ASTPtr ignored_ttl_table_ast; - alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); + SettingsChanges ignore_settings_changes; + + alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast, ignore_settings_changes); auto columns_for_parts = new_columns.getAllPhysical(); for (const auto & part : parts) @@ -1009,6 +1026,12 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } + +bool StorageMergeTree::hasSetting(const String & setting_name) const +{ + return settings.findIndex(setting_name) != MergeTreeSettings::npos; +} + void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) { // TODO: should get some locks to prevent race with 'alter … modify column' diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 0de9618d915..e2d24ceecdd 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -128,7 +128,10 @@ private: friend class MergeTreeData; friend struct CurrentlyMergingPartsTagger; + bool hasSetting(const String & setting_name) const override; + protected: + /** Attach the table with the appropriate name, along the appropriate path (with / at the end), * (correctness of names and paths are not checked) * consisting of the specified columns. diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index f6de6e87e37..f5b463ff972 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -38,7 +38,7 @@ void StorageNull::alter( ColumnsDescription new_columns = getColumns(); IndicesDescription new_indices = getIndices(); - params.apply(new_columns); + params.applyForColumnsOnly(new_columns); context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 934d651fead..87ab8815c57 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1510,7 +1510,8 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & ASTPtr ignored_order_by_ast; ASTPtr ignored_primary_key_ast; ASTPtr ignored_ttl_table_ast; - alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); + SettingsChanges ignored_changes; + alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast, ignored_changes); size_t modified_parts = 0; auto parts = getDataParts(); @@ -3119,7 +3120,8 @@ void StorageReplicatedMergeTree::alter( ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + SettingsChanges new_changes; + params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes); String new_columns_str = new_columns.toString(); if (new_columns_str != getColumns().toString()) From f7e0d17490ddace60010895d0609e01ff9717dd9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 24 Jul 2019 18:22:16 +0300 Subject: [PATCH 042/357] Fix some bugs --- dbms/src/Parsers/ParserAlterQuery.cpp | 9 +++++++++ dbms/src/Parsers/ParserAlterQuery.h | 1 + dbms/src/Storages/AlterCommands.cpp | 4 ++-- dbms/src/Storages/StorageMergeTree.cpp | 4 ++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 98891bbdf5f..b0ff5bf6c8c 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_comment_column("COMMENT COLUMN"); ParserKeyword s_modify_order_by("MODIFY ORDER BY"); ParserKeyword s_modify_ttl("MODIFY TTL"); + ParserKeyword s_modify_setting("MODIFY SETTING"); ParserKeyword s_add_index("ADD INDEX"); ParserKeyword s_drop_index("DROP INDEX"); @@ -63,6 +65,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserList parser_assignment_list( std::make_unique(), std::make_unique(TokenType::Comma), /* allow_empty = */ false); + ParserSetQuery parser_settings(true); if (s_add_column.ignore(pos, expected)) { @@ -289,6 +292,12 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected return false; command->type = ASTAlterCommand::MODIFY_TTL; } + else if (s_modify_setting.ignore(pos, expected)) + { + if (!parser_settings.parse(pos, command->settings_changes, expected)) + return false; + command->type = ASTAlterCommand::MODIFY_SETTING; + } else return false; diff --git a/dbms/src/Parsers/ParserAlterQuery.h b/dbms/src/Parsers/ParserAlterQuery.h index 282a4277e17..49d10eac34b 100644 --- a/dbms/src/Parsers/ParserAlterQuery.h +++ b/dbms/src/Parsers/ParserAlterQuery.h @@ -13,6 +13,7 @@ namespace DB * [CLEAR COLUMN [IF EXISTS] col_to_clear [IN PARTITION partition],] * [MODIFY COLUMN [IF EXISTS] col_to_modify type, ...] * [MODIFY PRIMARY KEY (a, b, c...)] + * [MODIFY SETTING setting_name=setting_value, ...] * [COMMENT COLUMN [IF EXISTS] col_name string] * [DROP|DETACH|ATTACH PARTITION|PART partition, ...] * [FETCH PARTITION partition FROM ...] diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 4b733f86d22..789b66b271a 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -317,7 +317,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri } else if (type == MODIFY_SETTING) { - changes.insert(changes.begin(), settings_changes.begin(), settings_changes.end()); + changes.insert(changes.end(), settings_changes.begin(), settings_changes.end()); } else throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); @@ -344,7 +344,7 @@ void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescr for (const AlterCommand & command : *this) if (!command.ignore) - command.apply(new_columns_description, new_indices_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, changes); + command.apply(new_columns_description, new_indices_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes); columns_description = std::move(new_columns_description); indices_description = std::move(new_indices_description); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index d8360cb61e4..f39735b1595 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -256,10 +256,10 @@ void StorageMergeTree::alter( { auto & storage_ast = ast.as(); if (!new_changes.empty()) - storage_ast.settings->changes.insert(storage_ast.settings->changes.begin(), new_changes.begin(), new_changes.end()); + storage_ast.settings->changes.insert(storage_ast.settings->changes.end(), new_changes.begin(), new_changes.end()); }; - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, storage_modifier); setColumns(std::move(new_columns)); settings.applyChanges(new_changes); return; From 856c8ef0e8dedf6a0166673c2a1b292b4648c655 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 24 Jul 2019 18:53:41 +0300 Subject: [PATCH 043/357] test for bug in ATTACH PART --- .../0_stateless/00974_attach_active_part.reference | 5 +++++ .../0_stateless/00974_attach_active_part.sh | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00974_attach_active_part.reference create mode 100755 dbms/tests/queries/0_stateless/00974_attach_active_part.sh diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference new file mode 100644 index 00000000000..3a90499810c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference @@ -0,0 +1,5 @@ +0_1_1_0 +1_2_2_0 +2_3_3_0 +3_4_4_0 +16 \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh new file mode 100755 index 00000000000..a3b2505f197 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_bug"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_bug (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; +$CLICKHOUSE_CLIENT --query="INSERT INTO attach_bug SELECT number FROM system.numbers LIMIT 16"; +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_bug ATTACH PART '../1_2_2_0'" 2> /dev/null; # | grep "" +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_bug' ORDER BY name FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT count() FROM attach_bug FORMAT TSV"; # will fail +$CLICKHOUSE_CLIENT --query="DROP TABLE attach_bug"; + + From 1805ab5736ec6c934af98dbda8c50a00fb6bb165 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Jul 2019 13:46:07 +0300 Subject: [PATCH 044/357] attach --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 8 ++- dbms/src/Storages/MergeTree/MergeTreeData.h | 1 + .../Storages/MergeTree/MergeTreePartInfo.cpp | 9 +++ .../Storages/MergeTree/MergeTreePartInfo.h | 2 + dbms/src/Storages/StorageMergeTree.cpp | 63 +++++++++++++------ dbms/src/Storages/StorageMergeTree.h | 1 + 6 files changed, 62 insertions(+), 22 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 6a7b6d5405e..94e42c34d0f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2336,6 +2336,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const St { MutableDataPartPtr part = std::make_shared(*this, Poco::Path(relative_path).getFileName()); part->relative_path = relative_path; + loadPartAndFixMetadata(part); + return part; +} + +void MergeTreeData::loadPartAndFixMetadata(MutableDataPartPtr part) +{ String full_part_path = part->getFullPath(); /// Earlier the list of columns was written incorrectly. Delete it and re-create. @@ -2357,8 +2363,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const St Poco::File(full_part_path + "checksums.txt.tmp").renameTo(full_part_path + "checksums.txt"); } - - return part; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 29962382749..cec3651652b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -533,6 +533,7 @@ public: /// Check that the part is not broken and calculate the checksums for it if they are not present. MutableDataPartPtr loadPartAndFixMetadata(const String & relative_path); + void loadPartAndFixMetadata(MutableDataPartPtr part); /** Create local backup (snapshot) for parts with specified prefix. * Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number, diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 732cc3436f4..45a0e1d488c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -210,4 +210,13 @@ bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, Detache part_info->prefix = dir_name.substr(0, first_separator); return part_info->valid_name = true; } + +String DetachedPartInfo::fullDirName() const +{ + if (!valid_name) + return prefix; + if (prefix.empty()) + return getPartName(); + return prefix + "_" + fullDirName(); +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index 2a168086a1c..7d0fb446ee3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -98,6 +98,8 @@ struct DetachedPartInfo : public MergeTreePartInfo /// (directory name was not successfully parsed). bool valid_name; + String fullDirName() const; + static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, MergeTreeDataFormatVersion format_version); }; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index bd09588981b..2f437e6d46b 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -36,6 +36,7 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int CANNOT_ASSIGN_OPTIMIZE; extern const int INCOMPATIBLE_COLUMNS; + extern const int BAD_DATA_PART_NAME; } namespace ActionLocks @@ -999,25 +1000,13 @@ void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const C throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); String part_id = partition->as().value.safeGet(); - Poco::Path part_path(part_id); - const bool file_zero_depth = part_path.isFile() && part_path.depth() == 0 && part_path.getFileName() != ".."; - const bool dir_zero_depth = part_path.isDirectory() && part_path.depth() == 1 && part_path.directory(0) != ".."; - const bool zero_depth = file_zero_depth || dir_zero_depth; - if (!part_path.isRelative() || !zero_depth) - throw DB::Exception("Part name must contain exactly one path component: name of detached part", ErrorCodes::INCORRECT_FILE_NAME); - - part_id = part_path.isFile() ? part_path.getFileName() : part_path.directory(0); - Poco::Path base_dir(full_path + "detached"); - Poco::File detached_part_dir(Poco::Path(base_dir, part_id)); - if (!detached_part_dir.exists()) - throw DB::Exception("Detached part \"" + part_id + "\" not found" , ErrorCodes::INCORRECT_FILE_NAME); + validateDetachedPartName(part_id); DetachedPartInfo info; DetachedPartInfo::tryParseDetachedPartName(part_id, &info, format_version); MergeTreeDataPart detached_part(*this, part_id, info); detached_part.relative_path = "detached/" + part_id; - // TODO make sure it's ok detached_part.remove(); } @@ -1038,6 +1027,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par Strings parts; if (attach_part) { + validateDetachedPartName(partition_id); parts.push_back(partition_id); } else @@ -1048,6 +1038,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par { const String & name = it.name(); MergeTreePartInfo part_info; + /// Parts with prefix in name (e.g. attaching_1_3_3_0, delete_tmp_1_3_3_0) will be ignored if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version) || part_info.partition_id != partition_id) { @@ -1062,16 +1053,38 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par for (const auto & source_part_name : parts) { - String source_path = source_dir + source_part_name; + MutableDataPartPtr part; + try + { + part = std::make_shared(*this, source_part_name); + part->relative_path = "detached/" + source_part_name; + part->renameTo("detached/attaching_" + source_part_name, false); - LOG_DEBUG(log, "Checking data"); - MutableDataPartPtr part = loadPartAndFixMetadata(source_path); + LOG_DEBUG(log, "Checking data in " << part->relative_path); + loadPartAndFixMetadata(part); - // TODO fix race with DROP DETACHED PARTITION - LOG_INFO(log, "Attaching part " << source_part_name << " from " << source_path); - renameTempPartAndAdd(part, &increment); + LOG_INFO(log, "Attaching part " << source_part_name << " from " << part->relative_path); + renameTempPartAndAdd(part, &increment); - LOG_INFO(log, "Finished attaching part"); + LOG_INFO(log, "Finished attaching part"); + } + catch (...) + { + tryLogCurrentException(log, String(__PRETTY_FUNCTION__) + ": cannot attach part " + source_part_name); + + if (part->relative_path == "detached/attaching_" + source_part_name) + { + try + { + part->renameTo("detached/" + source_part_name, false); + } + catch (...) + { + tryLogCurrentException(log, __PRETTY_FUNCTION__); + } + } + + } } /// New parts with other data may appear in place of deleted parts. @@ -1150,6 +1163,16 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } } +void StorageMergeTree::validateDetachedPartName(const String & name) const +{ + if (name.find('/') != std::string::npos || name == "." || name == "..") + throw DB::Exception("Invalid part name", ErrorCodes::INCORRECT_FILE_NAME); + + Poco::File detached_part_dir(full_path + "detached/" + name); + if (!detached_part_dir.exists()) + throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); +} + ActionLock StorageMergeTree::getActionLock(StorageActionBlockType action_type) { if (action_type == ActionLocks::PartsMerge) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index fa2561e4ab2..42061894a8e 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -124,6 +124,7 @@ private: void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); + void validateDetachedPartName(const String & name) const; friend class MergeTreeBlockOutputStream; friend class MergeTreeData; From 6ac950c6dc710fed7e1daf0a4bbdb09d3c0feb86 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Jul 2019 14:09:01 +0300 Subject: [PATCH 045/357] test attach active part fails --- .../queries/0_stateless/00974_attach_active_part.reference | 3 ++- dbms/tests/queries/0_stateless/00974_attach_active_part.sh | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference index 3a90499810c..fc0fce0a541 100644 --- a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference +++ b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference @@ -1,5 +1,6 @@ +OK 0_1_1_0 1_2_2_0 2_3_3_0 3_4_4_0 -16 \ No newline at end of file +16 diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh index a3b2505f197..32e2b21608f 100755 --- a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh +++ b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh @@ -6,9 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_bug"; $CLICKHOUSE_CLIENT --query="CREATE TABLE attach_bug (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; $CLICKHOUSE_CLIENT --query="INSERT INTO attach_bug SELECT number FROM system.numbers LIMIT 16"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_bug ATTACH PART '../1_2_2_0'" 2> /dev/null; # | grep "" +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_bug ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' $CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_bug' ORDER BY name FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="SElECT count() FROM attach_bug FORMAT TSV"; # will fail +$CLICKHOUSE_CLIENT --query="SElECT count() FROM attach_bug FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE attach_bug"; - - From e4212bfe593f3e28b42603c4dfeffcb2f60702ce Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Jul 2019 14:57:16 +0300 Subject: [PATCH 046/357] add full part name to detached_parts --- dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp | 2 +- dbms/src/Storages/System/StorageSystemDetachedParts.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 45a0e1d488c..de7150e4cea 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -217,6 +217,6 @@ String DetachedPartInfo::fullDirName() const return prefix; if (prefix.empty()) return getPartName(); - return prefix + "_" + fullDirName(); + return prefix + "_" + getPartName(); } } diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index 9f33a60b84a..9b32f1fb29b 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -28,7 +28,6 @@ public: protected: explicit StorageSystemDetachedParts() { - // TODO add column "directory_name" or "is_valid_name" setColumns(ColumnsDescription{{ {"database", std::make_shared()}, {"table", std::make_shared()}, @@ -37,7 +36,8 @@ protected: {"reason", std::make_shared()}, {"min_block_number", std::make_shared()}, {"max_block_number", std::make_shared()}, - {"level", std::make_shared()} + {"level", std::make_shared()}, + {"directory_name", std::make_shared()} }}); } @@ -63,12 +63,13 @@ protected: int i = 0; columns[i++]->insert(info.database); columns[i++]->insert(info.table); - columns[i++]->insert(p.partition_id); - columns[i++]->insert(p.getPartName()); + columns[i++]->insert(p.valid_name ? p.partition_id : ""); + columns[i++]->insert(p.valid_name ? p.getPartName() : ""); columns[i++]->insert(p.prefix); columns[i++]->insert(p.min_block); columns[i++]->insert(p.max_block); columns[i++]->insert(p.level); + columns[i++]->insert(p.fullDirName()); } } From 3ba26aba43e30aa92fed80fb6f842dc3eb064ef1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Jul 2019 19:28:08 +0300 Subject: [PATCH 047/357] tests --- .../Storages/MergeTree/MergeTreeDataPart.cpp | 9 +++- .../Storages/MergeTree/MergeTreeDataPart.h | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 6 +-- .../00974_attach_active_part.reference | 6 --- .../0_stateless/00974_attach_active_part.sh | 12 ------ .../00974_attach_invalid_parts.reference | 17 ++++++++ .../0_stateless/00974_attach_invalid_parts.sh | 41 +++++++++++++++++++ .../0_stateless/00975_drop_detached.reference | 2 + .../0_stateless/00975_drop_detached.sh | 30 ++++++++++++++ 9 files changed, 102 insertions(+), 23 deletions(-) delete mode 100644 dbms/tests/queries/0_stateless/00974_attach_active_part.reference delete mode 100755 dbms/tests/queries/0_stateless/00974_attach_active_part.sh create mode 100644 dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference create mode 100755 dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh create mode 100644 dbms/tests/queries/0_stateless/00975_drop_detached.reference create mode 100755 dbms/tests/queries/0_stateless/00975_drop_detached.sh diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 7b8be970e1d..865aaf80ed1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -347,7 +347,7 @@ UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) return res; } -void MergeTreeDataPart::remove() const +void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const { if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); @@ -398,6 +398,13 @@ void MergeTreeDataPart::remove() const return; } + if (force_recursive) + { + /// Part is not loaded (we don't know which files are there), so remove dir recursively. + to_dir.remove(true); + return; + } + try { /// Remove each expected file in directory, then remove directory itself. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h index f41ea8af424..98af00c071a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h @@ -241,7 +241,7 @@ struct MergeTreeDataPart /// Calculate the total size of the entire directory with all the files static UInt64 calculateTotalSizeOnDisk(const String & from); - void remove() const; + void remove(bool force_recursive = false) const; /// Makes checks and move part to new directory /// Changes only relative_dir_name, you need to update other metadata (name, is_temp) explicitly diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 2f437e6d46b..0c1503347c0 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -1007,7 +1007,7 @@ void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const C MergeTreeDataPart detached_part(*this, part_id, info); detached_part.relative_path = "detached/" + part_id; - detached_part.remove(); + detached_part.remove(true); } void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) @@ -1070,9 +1070,9 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par } catch (...) { - tryLogCurrentException(log, String(__PRETTY_FUNCTION__) + ": cannot attach part " + source_part_name); + LOG_INFO(log, "Cannot attach part " << source_part_name << " :" << getCurrentExceptionMessage(false)); - if (part->relative_path == "detached/attaching_" + source_part_name) + if (part && part->relative_path == "detached/attaching_" + source_part_name) { try { diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference deleted file mode 100644 index fc0fce0a541..00000000000 --- a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference +++ /dev/null @@ -1,6 +0,0 @@ -OK -0_1_1_0 -1_2_2_0 -2_3_3_0 -3_4_4_0 -16 diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh deleted file mode 100755 index 32e2b21608f..00000000000 --- a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_bug"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_bug (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; -$CLICKHOUSE_CLIENT --query="INSERT INTO attach_bug SELECT number FROM system.numbers LIMIT 16"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_bug ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_bug' ORDER BY name FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="SElECT count() FROM attach_bug FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="DROP TABLE attach_bug"; diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference new file mode 100644 index 00000000000..d44f46779ca --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference @@ -0,0 +1,17 @@ +=== cannot attach active === +OK +0_1_1_0 +1_2_2_0 +2_3_3_0 +3_4_4_0 +16 120 +=== attach all valid parts === +0_5_5_0 +0_6_6_0 +1_2_2_0 +1_4_4_0 +16 120 +=== detached === +0_5_5_0 +delete_tmp_0_7_7 +attaching_0_6_6 diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh new file mode 100755 index 00000000000..89a6be183d2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` +cur_db=`${CLICKHOUSE_CLIENT} --query "SELECT currentDatabase()"` + +echo '=== cannot attach active ==='; +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_active"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; +$CLICKHOUSE_CLIENT --query="INSERT INTO attach_active SELECT number FROM system.numbers LIMIT 16"; +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_active ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_active' AND database='${cur_db}' ORDER BY name FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_active FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="DROP TABLE attach_active"; + + + +echo '=== attach all valid parts ==='; +$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_partitions"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_partitions (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; +$CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8"; +$CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; + +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; +mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part +cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ +cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/delete_tmp_0_7_7_0/ +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; + +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_partitions FORMAT TSV"; +echo '=== detached ==='; +$CLICKHOUSE_CLIENT --query="SELECT directory_name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' FORMAT TSV"; + +$CLICKHOUSE_CLIENT --query="DROP TABLE attach_partitions"; +$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.reference b/dbms/tests/queries/0_stateless/00975_drop_detached.reference new file mode 100644 index 00000000000..40732c908ab --- /dev/null +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.reference @@ -0,0 +1,2 @@ +OK +0_3_3_0 diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh new file mode 100755 index 00000000000..9f831560bdc --- /dev/null +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` +cur_db=`${CLICKHOUSE_CLIENT} --query "SELECT currentDatabase()"` + +$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS drop_detached"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; +$CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8"; +$CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; + +$CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; +mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ +mkdir $ch_dir/data/$cur_db/drop_detached/detached/delete_tmp_0_7_7_0/ +mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ + +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'delete_tmp_0_7_7_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" + +$CLICKHOUSE_CLIENT --query="SElECT directory_name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; +$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From c5a42a69481d2ae11c379ec8a7c325a18cb27138 Mon Sep 17 00:00:00 2001 From: Sergi Vladykin Date: Sun, 28 Jul 2019 13:30:46 +0300 Subject: [PATCH 048/357] TTL: Only drop altogether the expired parts and not partially prune them. Though in regular merges eviction of expired values still must happen. --- dbms/src/DataStreams/TTLBlockInputStream.cpp | 25 ++++++++----------- dbms/src/Storages/MergeTree/MergeSelector.h | 5 +++- .../MergeTree/MergeTreeDataMergerMutator.cpp | 7 ++++-- .../MergeTree/MergeTreeDataPartTTLInfo.cpp | 8 +++--- .../MergeTree/MergeTreeDataPartTTLInfo.h | 10 +++++--- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../Storages/MergeTree/MergeTreeSettings.h | 1 + .../Storages/MergeTree/TTLMergeSelector.cpp | 14 ++++++++--- .../src/Storages/MergeTree/TTLMergeSelector.h | 3 ++- 9 files changed, 45 insertions(+), 30 deletions(-) diff --git a/dbms/src/DataStreams/TTLBlockInputStream.cpp b/dbms/src/DataStreams/TTLBlockInputStream.cpp index 11999b894aa..0ad72876ea9 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.cpp +++ b/dbms/src/DataStreams/TTLBlockInputStream.cpp @@ -63,22 +63,19 @@ TTLBlockInputStream::TTLBlockInputStream( Block TTLBlockInputStream::readImpl() { + /// Skip all data if table ttl is expired for part + if (storage.hasTableTTL() && old_ttl_infos.table_ttl.max <= current_time) + { + rows_removed = data_part->rows_count; + return {}; + } + Block block = children.at(0)->read(); if (!block) return block; - if (storage.hasTableTTL()) - { - /// Skip all data if table ttl is expired for part - if (old_ttl_infos.table_ttl.max <= current_time) - { - rows_removed = data_part->rows_count; - return {}; - } - - if (old_ttl_infos.table_ttl.min <= current_time) - removeRowsWithExpiredTableTTL(block); - } + if (storage.hasTableTTL() && old_ttl_infos.table_ttl.min <= current_time) + removeRowsWithExpiredTableTTL(block); removeValuesWithExpiredColumnTTL(block); @@ -88,9 +85,9 @@ Block TTLBlockInputStream::readImpl() void TTLBlockInputStream::readSuffixImpl() { for (const auto & elem : new_ttl_infos.columns_ttl) - new_ttl_infos.updatePartMinTTL(elem.second.min); + new_ttl_infos.updatePartMinMaxTTL(elem.second.min, elem.second.max); - new_ttl_infos.updatePartMinTTL(new_ttl_infos.table_ttl.min); + new_ttl_infos.updatePartMinMaxTTL(new_ttl_infos.table_ttl.min, new_ttl_infos.table_ttl.max); data_part->ttl_infos = std::move(new_ttl_infos); data_part->empty_columns = std::move(empty_columns); diff --git a/dbms/src/Storages/MergeTree/MergeSelector.h b/dbms/src/Storages/MergeTree/MergeSelector.h index b01ef1d1d43..3c3cd8190ac 100644 --- a/dbms/src/Storages/MergeTree/MergeSelector.h +++ b/dbms/src/Storages/MergeTree/MergeSelector.h @@ -40,8 +40,11 @@ public: /// Opaque pointer to avoid dependencies (it is not possible to do forward declaration of typedef). const void * data; - /// Minimal time, when we need to delete some data from this part + /// Minimal time, when we need to delete some data from this part. time_t min_ttl; + + /// Maximum time, when we will need to drop this part altogether because all rows in it are expired. + time_t max_ttl; }; /// Parts are belong to partitions. Only parts within same partition could be merged. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index e30bbdcda0b..727459291b5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -200,8 +200,11 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( part_info.level = part->info.level; part_info.data = ∂ part_info.min_ttl = part->ttl_infos.part_min_ttl; + part_info.max_ttl = part->ttl_infos.part_max_ttl; - if (part_info.min_ttl && part_info.min_ttl <= current_time) + time_t ttl = data.settings.ttl_only_drop_parts ? part_info.max_ttl : part_info.min_ttl; + + if (ttl && ttl <= current_time) has_part_with_expired_ttl = true; partitions.back().emplace_back(part_info); @@ -228,7 +231,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( /// NOTE Could allow selection of different merge strategy. if (can_merge_with_ttl && has_part_with_expired_ttl) { - merge_selector = std::make_unique(current_time); + merge_selector = std::make_unique(current_time, data.settings.ttl_only_drop_parts); last_merge_with_ttl = current_time; } else diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp index d3fe3231e05..39665f03c84 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp @@ -12,11 +12,11 @@ void MergeTreeDataPartTTLInfos::update(const MergeTreeDataPartTTLInfos & other_i for (const auto & [name, ttl_info] : other_infos.columns_ttl) { columns_ttl[name].update(ttl_info); - updatePartMinTTL(ttl_info.min); + updatePartMinMaxTTL(ttl_info.min, ttl_info.max); } table_ttl.update(other_infos.table_ttl); - updatePartMinTTL(table_ttl.min); + updatePartMinMaxTTL(table_ttl.min, table_ttl.max); } void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) @@ -37,7 +37,7 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) String name = col["name"].getString(); columns_ttl.emplace(name, ttl_info); - updatePartMinTTL(ttl_info.min); + updatePartMinMaxTTL(ttl_info.min, ttl_info.max); } } if (json.has("table")) @@ -46,7 +46,7 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) table_ttl.min = table["min"].getUInt(); table_ttl.max = table["max"].getUInt(); - updatePartMinTTL(table_ttl.min); + updatePartMinMaxTTL(table_ttl.min, table_ttl.max); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h index ff181aa29a4..71a7c9f602f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h @@ -36,15 +36,19 @@ struct MergeTreeDataPartTTLInfos std::unordered_map columns_ttl; MergeTreeDataPartTTLInfo table_ttl; time_t part_min_ttl = 0; + time_t part_max_ttl = 0; void read(ReadBuffer & in); void write(WriteBuffer & out) const; void update(const MergeTreeDataPartTTLInfos & other_infos); - void updatePartMinTTL(time_t time) + void updatePartMinMaxTTL(time_t time_min, time_t time_max) { - if (time && (!part_min_ttl || time < part_min_ttl)) - part_min_ttl = time; + if (time_min && (!part_min_ttl || time_min < part_min_ttl)) + part_min_ttl = time_min; + + if (time_max && (!part_max_ttl || time_max > part_max_ttl)) + part_max_ttl = time_max; } }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index d7a48171499..8515348a4b5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -99,7 +99,7 @@ void updateTTL(const MergeTreeData::TTLEntry & ttl_entry, MergeTreeDataPart::TTL else throw Exception("Unexpected type of result ttl column", ErrorCodes::LOGICAL_ERROR); - ttl_infos.updatePartMinTTL(ttl_info.min); + ttl_infos.updatePartMinMaxTTL(ttl_info.min, ttl_info.max); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 9bd58e77f9c..457208cd0c3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -79,6 +79,7 @@ struct MergeTreeSettings : public SettingsCollection M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024, "Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled).") \ M(SettingUInt64, index_granularity_bytes, 10 * 1024 * 1024, "Approximate amount of bytes in single granule (0 - disabled).") \ M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "Minimal time in seconds, when merge with TTL can be repeated.") \ + M(SettingBool, ttl_only_drop_parts, false, "Only drop altogether the expired parts and not partially prune them.") \ M(SettingBool, write_final_mark, 1, "Write final mark after end of column (0 - disabled, do nothing if index_granularity_bytes=0)") \ M(SettingBool, enable_mixed_granularity_parts, 0, "Enable parts with adaptive and non adaptive granularity") diff --git a/dbms/src/Storages/MergeTree/TTLMergeSelector.cpp b/dbms/src/Storages/MergeTree/TTLMergeSelector.cpp index 3669bd1e2cc..0ba341fca64 100644 --- a/dbms/src/Storages/MergeTree/TTLMergeSelector.cpp +++ b/dbms/src/Storages/MergeTree/TTLMergeSelector.cpp @@ -20,9 +20,11 @@ IMergeSelector::PartsInPartition TTLMergeSelector::select( { for (auto it = partitions[i].begin(); it != partitions[i].end(); ++it) { - if (it->min_ttl && (partition_to_merge_index == -1 || it->min_ttl < partition_to_merge_min_ttl)) + time_t ttl = only_drop_parts ? it->max_ttl : it->min_ttl; + + if (ttl && (partition_to_merge_index == -1 || ttl < partition_to_merge_min_ttl)) { - partition_to_merge_min_ttl = it->min_ttl; + partition_to_merge_min_ttl = ttl; partition_to_merge_index = i; best_begin = it; } @@ -38,7 +40,9 @@ IMergeSelector::PartsInPartition TTLMergeSelector::select( while (true) { - if (!best_begin->min_ttl || best_begin->min_ttl > current_time + time_t ttl = only_drop_parts ? best_begin->max_ttl : best_begin->min_ttl; + + if (!ttl || ttl > current_time || (max_total_size_to_merge && total_size > max_total_size_to_merge)) { ++best_begin; @@ -54,7 +58,9 @@ IMergeSelector::PartsInPartition TTLMergeSelector::select( while (best_end != best_partition.end()) { - if (!best_end->min_ttl || best_end->min_ttl > current_time + time_t ttl = only_drop_parts ? best_end->max_ttl : best_end->min_ttl; + + if (!ttl || ttl > current_time || (max_total_size_to_merge && total_size > max_total_size_to_merge)) break; diff --git a/dbms/src/Storages/MergeTree/TTLMergeSelector.h b/dbms/src/Storages/MergeTree/TTLMergeSelector.h index 3c035d03a99..2f03d5b9feb 100644 --- a/dbms/src/Storages/MergeTree/TTLMergeSelector.h +++ b/dbms/src/Storages/MergeTree/TTLMergeSelector.h @@ -14,13 +14,14 @@ namespace DB class TTLMergeSelector : public IMergeSelector { public: - explicit TTLMergeSelector(time_t current_time_) : current_time(current_time_) {} + explicit TTLMergeSelector(time_t current_time_, bool only_drop_parts_) : current_time(current_time_), only_drop_parts(only_drop_parts_) {} PartsInPartition select( const Partitions & partitions, const size_t max_total_size_to_merge) override; private: time_t current_time; + bool only_drop_parts; }; } From 8f4883b0d2d84f324c530068116dabc6f5c05146 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 28 Jul 2019 15:33:40 +0300 Subject: [PATCH 049/357] Better constraint exception --- .../CheckConstraintsBlockOutputStream.cpp | 46 +++++++++++++++---- .../CheckConstraintsBlockOutputStream.h | 16 ++++--- .../Interpreters/InterpreterInsertQuery.cpp | 2 +- .../0_stateless/00952_basic_constraints.sh | 14 +++--- .../00953_constraints_operations.sh | 2 +- 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index ec4a7bd45b8..cb9b8871a68 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -12,11 +12,23 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) for (size_t i = 0; i < expressions.size(); ++i) { auto constraint_expr = expressions[i]; - if (!checkConstraintOnBlock(block, constraint_expr)) - throw Exception{"Constraint " + constraints.constraints[i]->name + " is not satisfied, constraint expression: " + - serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::LOGICAL_ERROR}; + auto res_column_uint8 = executeOnBlock(block, constraint_expr); + if (!memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1)) + { + auto indices_wrong = findAllWrong(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize()); + std::string indices_str = "{"; + for (size_t j = 0; j < indices_wrong.size(); ++j) { + indices_str += std::to_string(indices_wrong[j]); + indices_str += (j != indices_wrong.size() - 1) ? ", " : "}"; + } + + throw Exception{"Violated constraint " + constraints.constraints[i]->name + + " in table " + table + " at indices " + indices_str + ", constraint expression: " + + serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::LOGICAL_ERROR}; + } } output->write(block); + rows_written += block.rows(); } void CheckConstraintsBlockOutputStream::flush() @@ -34,18 +46,34 @@ void CheckConstraintsBlockOutputStream::writeSuffix() output->writeSuffix(); } -bool CheckConstraintsBlockOutputStream::checkImplMemory(const Block & block, const ExpressionActionsPtr & constraint) +const ColumnUInt8 *CheckConstraintsBlockOutputStream::executeOnBlock( + const Block & block, + const ExpressionActionsPtr & constraint) { Block res = block; + constraint->execute(res); ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); - auto res_column_uint8 = checkAndGetColumn(res_column.column.get()); - return memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1); + return checkAndGetColumn(res_column.column.get()); } -bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) +std::vector CheckConstraintsBlockOutputStream::findAllWrong(const void *data, size_t size) { - return checkImplMemory(block, constraint); -} + std::vector res; + if (size == 0) + return res; + + auto ptr = reinterpret_cast(data); + + for (size_t i = 0; i < size; ++i) + { + if (*(ptr + i) == 0x0) + { + res.push_back(i); + } + } + + return res; +} } diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index 6ea42cf44af..ac2e7e974a1 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -17,14 +17,17 @@ class CheckConstraintsBlockOutputStream : public IBlockOutputStream { public: CheckConstraintsBlockOutputStream( + const String & table_, const BlockOutputStreamPtr & output_, const Block & header_, const ConstraintsDescription & constraints_, const Context & context_) - : output(output_), + : table(table_), + output(output_), header(header_), constraints(constraints_), - expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())) + expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())), + rows_written(0) { } Block getHeader() const override { return header; } @@ -35,14 +38,15 @@ public: void writePrefix() override; void writeSuffix() override; - bool checkImplMemory(const Block & block, const ExpressionActionsPtr & constraint); - bool checkImplBool(const Block & block, const ExpressionActionsPtr & constraint); - bool checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint); - private: + const ColumnUInt8* executeOnBlock(const Block & block, const ExpressionActionsPtr & constraint); + std::vector findAllWrong(const void *data, size_t size); + + String table; BlockOutputStreamPtr output; Block header; const ConstraintsDescription constraints; const ConstraintsExpressions expressions; + size_t rows_written; }; } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 9c0cc31cb8e..8454df97f08 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -119,7 +119,7 @@ BlockIO InterpreterInsertQuery::execute() out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); out = std::make_shared( - out, query_sample_block, table->getConstraints(), context); + query.table, out, query_sample_block, table->getConstraints(), context); auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index 1d2a46dae61..b6aa28c46bf 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -20,8 +20,8 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Constraint b_constraint is not satisfied" -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" 2>&1 \ +EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices {1, 3}" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0), (3, 4), (6, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" @@ -32,20 +32,20 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints ( a UInt32, b UInt32, - CONSTRAINT b_constraint CHECK b > 10, - CONSTRAINT a_constraint CHECK a < 10 + CONSTRAINT a_constraint CHECK a < 10, + CONSTRAINT b_constraint CHECK b > 10 ) ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception -EXCEPTION_TEXT="Constraint b_constraint is not satisfied" +EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices {0}" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception -EXCEPTION_TEXT="Constraint a_constraint is not satisfied" -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" 2>&1 \ +EXCEPTION_TEXT="Violated constraint a_constraint in table test_constraints at indices {1}" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11), (9, 11), (8, 12);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh index f0fc5b71fbf..8a563a21e02 100755 --- a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh @@ -20,7 +20,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Constraint b_constraint is not satisfied" +EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" From 915092515c41b5b83f0bbd9cb6d2294c5b56b0e7 Mon Sep 17 00:00:00 2001 From: Sergi Vladykin Date: Mon, 29 Jul 2019 11:20:11 +0300 Subject: [PATCH 050/357] revert the performance bug fix --- dbms/src/DataStreams/TTLBlockInputStream.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/DataStreams/TTLBlockInputStream.cpp b/dbms/src/DataStreams/TTLBlockInputStream.cpp index 0ad72876ea9..78d8d25e8bc 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.cpp +++ b/dbms/src/DataStreams/TTLBlockInputStream.cpp @@ -63,6 +63,10 @@ TTLBlockInputStream::TTLBlockInputStream( Block TTLBlockInputStream::readImpl() { + Block block = children.at(0)->read(); + if (!block) + return block; + /// Skip all data if table ttl is expired for part if (storage.hasTableTTL() && old_ttl_infos.table_ttl.max <= current_time) { @@ -70,10 +74,6 @@ Block TTLBlockInputStream::readImpl() return {}; } - Block block = children.at(0)->read(); - if (!block) - return block; - if (storage.hasTableTTL() && old_ttl_infos.table_ttl.min <= current_time) removeRowsWithExpiredTableTTL(block); From 0fc47fbbe4b70b5ffc254024798d8f9ed45b0418 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 26 Jul 2019 23:04:45 +0300 Subject: [PATCH 051/357] fixes --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 14 +++++++++- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 ++ .../Storages/MergeTree/MergeTreeDataPart.cpp | 8 ++++-- .../Storages/MergeTree/MergeTreePartInfo.cpp | 28 ++++++++----------- .../Storages/MergeTree/MergeTreePartInfo.h | 9 +++--- dbms/src/Storages/StorageMergeTree.cpp | 15 ++-------- dbms/src/Storages/StorageMergeTree.h | 1 - .../Storages/StorageReplicatedMergeTree.cpp | 2 ++ .../System/StorageSystemDetachedParts.cpp | 25 ++++++++--------- .../00974_attach_invalid_parts.reference | 4 +-- .../0_stateless/00974_attach_invalid_parts.sh | 11 +++++--- .../0_stateless/00975_drop_detached.sh | 13 +++++---- 12 files changed, 69 insertions(+), 63 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 94e42c34d0f..9d12a9ee6ea 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -87,6 +87,8 @@ namespace ErrorCodes extern const int CANNOT_MUNMAP; extern const int CANNOT_MREMAP; extern const int BAD_TTL_EXPRESSION; + extern const int INCORRECT_FILE_NAME; + extern const int BAD_DATA_PART_NAME; } @@ -2583,11 +2585,21 @@ MergeTreeData::getDetachedParts() const res.emplace_back(); auto & part = res.back(); - DetachedPartInfo::tryParseDetachedPartName(dir_name, &part, format_version); + DetachedPartInfo::tryParseDetachedPartName(dir_name, part, format_version); } return res; } +void MergeTreeData::validateDetachedPartName(const String & name) const +{ + if (name.find('/') != std::string::npos || name == "." || name == "..") + throw DB::Exception("Invalid part name", ErrorCodes::INCORRECT_FILE_NAME); + + Poco::File detached_part_dir(full_path + "detached/" + name); + if (!detached_part_dir.exists()) + throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); +} + MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const { DataParts res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index cec3651652b..2333135d53e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -389,6 +389,8 @@ public: /// Returns all detached parts std::vector getDetachedParts() const; + void validateDetachedPartName(const String & name) const; + /// Returns Committed parts DataParts getDataParts() const; DataPartsVector getDataPartsVector() const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 865aaf80ed1..24bc5cd2463 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -356,16 +356,18 @@ void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const * - rename directory to temporary name; * - remove it recursive. * - * For temporary name we use "delete_tmp_" prefix. + * For temporary name we use "detached/deleting_" prefix. * - * NOTE: We cannot use "tmp_delete_" prefix, because there is a second thread, + * NOTE: We cannot use "tmp_*" prefix, because there is a second thread, * that calls "clearOldTemporaryDirectories" and removes all directories, that begin with "tmp_" and are old enough. * But when we removing data part, it can be old enough. And rename doesn't change mtime. * And a race condition can happen that will lead to "File not found" error here. + * We move directory to detached/, because if an attempt to remove directory after renaming failed for some reason + * there would be no way to remove directory from storage.full_path (except manually). */ String from = storage.full_path + relative_path; - String to = storage.full_path + "delete_tmp_" + name; + String to = storage.full_path + getRelativePathForDetachedPart("deleting_"); Poco::File from_dir{from}; Poco::File to_dir{to}; diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index de7150e4cea..a9e31a988b3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -188,35 +188,29 @@ String MergeTreePartInfo::getPartNameV0(DayNum left_date, DayNum right_date) con return wb.str(); } -bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, +bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version) { + part_info.dir_name = dir_name; + /// First, try to parse as . - if (MergeTreePartInfo::tryParsePartName(dir_name, part_info, format_version)) - return part_info->valid_name = true; + if (MergeTreePartInfo::tryParsePartName(dir_name, &part_info, format_version)) + return part_info.valid_name = true; /// Next, as _. Use entire name as prefix if it fails. - part_info->prefix = dir_name; + part_info.prefix = dir_name; const auto first_separator = dir_name.find_first_of('_'); if (first_separator == String::npos) - return part_info->valid_name = false; + return part_info.valid_name = false; // TODO what if contains '_'? const auto part_name = dir_name.substr(first_separator + 1, dir_name.size() - first_separator - 1); - if (!MergeTreePartInfo::tryParsePartName(part_name, part_info, format_version)) - return part_info->valid_name = false; + if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) + return part_info.valid_name = false; - part_info->prefix = dir_name.substr(0, first_separator); - return part_info->valid_name = true; + part_info.prefix = dir_name.substr(0, first_separator); + return part_info.valid_name = true; } -String DetachedPartInfo::fullDirName() const -{ - if (!valid_name) - return prefix; - if (prefix.empty()) - return getPartName(); - return prefix + "_" + getPartName(); -} } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index 7d0fb446ee3..25cf46ad46d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -92,15 +92,14 @@ struct MergeTreePartInfo /// addition to the above fields. struct DetachedPartInfo : public MergeTreePartInfo { + /// Suddenly, name of detached part may contain suffix (such as _tryN), which is ignored by MergeTreePartInfo::tryParsePartName(...) + String dir_name; String prefix; - /// If false, prefix contains full directory name and MergeTreePartInfo may be in invalid state - /// (directory name was not successfully parsed). + /// If false, MergeTreePartInfo is in invalid state (directory name was not successfully parsed). bool valid_name; - String fullDirName() const; - - static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, MergeTreeDataFormatVersion format_version); + static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version); }; } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 0c1503347c0..ad4d0cd933f 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -36,7 +36,6 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int CANNOT_ASSIGN_OPTIMIZE; extern const int INCOMPATIBLE_COLUMNS; - extern const int BAD_DATA_PART_NAME; } namespace ActionLocks @@ -1003,7 +1002,7 @@ void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const C validateDetachedPartName(part_id); DetachedPartInfo info; - DetachedPartInfo::tryParseDetachedPartName(part_id, &info, format_version); + DetachedPartInfo::tryParseDetachedPartName(part_id, info, format_version); MergeTreeDataPart detached_part(*this, part_id, info); detached_part.relative_path = "detached/" + part_id; @@ -1038,7 +1037,8 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par { const String & name = it.name(); MergeTreePartInfo part_info; - /// Parts with prefix in name (e.g. attaching_1_3_3_0, delete_tmp_1_3_3_0) will be ignored + /// Parts with prefix in name (e.g. attaching_1_3_3_0, deleting_1_3_3_0) will be ignored + // TODO what if name contains "_tryN" suffix? if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version) || part_info.partition_id != partition_id) { @@ -1163,15 +1163,6 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } } -void StorageMergeTree::validateDetachedPartName(const String & name) const -{ - if (name.find('/') != std::string::npos || name == "." || name == "..") - throw DB::Exception("Invalid part name", ErrorCodes::INCORRECT_FILE_NAME); - - Poco::File detached_part_dir(full_path + "detached/" + name); - if (!detached_part_dir.exists()) - throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); -} ActionLock StorageMergeTree::getActionLock(StorageActionBlockType action_type) { diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 42061894a8e..fa2561e4ab2 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -124,7 +124,6 @@ private: void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); - void validateDetachedPartName(const String & name) const; friend class MergeTreeBlockOutputStream; friend class MergeTreeData; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 5f91c304e98..192384602eb 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3554,6 +3554,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool Strings parts; if (attach_part) { + validateDetachedPartName(partition_id); parts.push_back(partition_id); } else @@ -3566,6 +3567,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool { String name = it.name(); MergeTreePartInfo part_info; + // TODO what if name contains "_tryN" suffix? if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version)) continue; if (part_info.partition_id != partition_id) diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index 9b32f1fb29b..e27c7945670 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -31,13 +32,12 @@ protected: setColumns(ColumnsDescription{{ {"database", std::make_shared()}, {"table", std::make_shared()}, - {"partition_id", std::make_shared()}, + {"partition_id", std::make_shared(std::make_shared())}, {"name", std::make_shared()}, - {"reason", std::make_shared()}, - {"min_block_number", std::make_shared()}, - {"max_block_number", std::make_shared()}, - {"level", std::make_shared()}, - {"directory_name", std::make_shared()} + {"reason", std::make_shared(std::make_shared())}, + {"min_block_number", std::make_shared(std::make_shared())}, + {"max_block_number", std::make_shared(std::make_shared())}, + {"level", std::make_shared(std::make_shared())} }}); } @@ -63,13 +63,12 @@ protected: int i = 0; columns[i++]->insert(info.database); columns[i++]->insert(info.table); - columns[i++]->insert(p.valid_name ? p.partition_id : ""); - columns[i++]->insert(p.valid_name ? p.getPartName() : ""); - columns[i++]->insert(p.prefix); - columns[i++]->insert(p.min_block); - columns[i++]->insert(p.max_block); - columns[i++]->insert(p.level); - columns[i++]->insert(p.fullDirName()); + columns[i++]->insert(p.valid_name ? p.partition_id : Field()); + columns[i++]->insert(p.dir_name); + columns[i++]->insert(p.valid_name ? p.prefix : Field()); + columns[i++]->insert(p.valid_name ? p.min_block : Field()); + columns[i++]->insert(p.valid_name ? p.max_block : Field()); + columns[i++]->insert(p.valid_name ? p.level : Field()); } } diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference index d44f46779ca..42a04fe5666 100644 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference @@ -13,5 +13,5 @@ OK 16 120 === detached === 0_5_5_0 -delete_tmp_0_7_7 -attaching_0_6_6 +deleting_0_7_7_0 +attaching_0_6_6_0 diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh index 89a6be183d2..4e9efa64ad1 100755 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh @@ -27,15 +27,18 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM sys $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; -mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part -cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ -cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/delete_tmp_0_7_7_0/ +sudo -n mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ + mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part +sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ + cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ +sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ + cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; $CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_partitions FORMAT TSV"; echo '=== detached ==='; -$CLICKHOUSE_CLIENT --query="SELECT directory_name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE attach_partitions"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh index 9f831560bdc..3a5e920da75 100755 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.sh +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.sh @@ -15,16 +15,19 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system. $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; -mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ -mkdir $ch_dir/data/$cur_db/drop_detached/detached/delete_tmp_0_7_7_0/ -mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ +sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ + mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ +sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ + mkdir $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ +sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ + mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'delete_tmp_0_7_7_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" -$CLICKHOUSE_CLIENT --query="SElECT directory_name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From 86831fe500c5dbf1124c7cecaf832f502700d5ba Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 29 Jul 2019 14:03:50 +0300 Subject: [PATCH 052/357] Add ability to create immutable settings --- .../performance-test/PerformanceTestInfo.cpp | 2 +- dbms/src/Common/ErrorCodes.cpp | 1 + dbms/src/Core/Settings.h | 2 +- dbms/src/Core/SettingsCommon.h | 78 +++++++++++++++++-- dbms/src/Storages/Kafka/KafkaSettings.cpp | 2 +- dbms/src/Storages/Kafka/KafkaSettings.h | 22 +++--- .../Storages/MergeTree/MergeTreeSettings.cpp | 2 +- .../Storages/MergeTree/MergeTreeSettings.h | 4 +- dbms/src/Storages/StorageMergeTree.cpp | 2 +- 9 files changed, 89 insertions(+), 26 deletions(-) diff --git a/dbms/programs/performance-test/PerformanceTestInfo.cpp b/dbms/programs/performance-test/PerformanceTestInfo.cpp index f016b257c5f..aa8c05f2d44 100644 --- a/dbms/programs/performance-test/PerformanceTestInfo.cpp +++ b/dbms/programs/performance-test/PerformanceTestInfo.cpp @@ -96,7 +96,7 @@ void PerformanceTestInfo::applySettings(XMLConfigurationPtr config) } extractSettings(config, "settings", config_settings, settings_to_apply); - settings.applyChanges(settings_to_apply); + settings.loadFromChanges(settings_to_apply); if (settings_contain("average_rows_speed_precision")) TestStats::avg_rows_speed_precision = diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 35213e3064f..7b955e47938 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -439,6 +439,7 @@ namespace ErrorCodes extern const int CANNOT_DELETE_TIMER = 462; extern const int CANNOT_FCNTL = 463; extern const int SETTINGS_ARE_NOT_SUPPORTED = 464; + extern const int IMMUTABLE_SETTING = 465; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 60f2599c73f..7501e4324a7 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -42,7 +42,7 @@ struct Settings : public SettingsCollection * but we are not going to do it, because settings is used everywhere as static struct fields. */ -#define LIST_OF_SETTINGS(M) \ +#define LIST_OF_SETTINGS(M, IM) \ M(SettingUInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.") \ M(SettingUInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.") \ M(SettingUInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading") \ diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index 08064096133..e9dce9c9dcc 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -17,6 +17,10 @@ class Field; class ReadBuffer; class WriteBuffer; +namespace ErrorCodes +{ + extern const int IMMUTABLE_SETTING; +} /** One setting for any type. * Stores a value within itself, as well as a flag - whether the value was changed. @@ -317,6 +321,7 @@ private: size_t offset_of_changed; StringRef name; StringRef description; + bool immutable; GetStringFunction get_string; GetFieldFunction get_field; SetStringFunction set_string; @@ -396,6 +401,7 @@ public: const_reference(const const_reference & src) = default; const StringRef & getName() const { return member->name; } const StringRef & getDescription() const { return member->description; } + bool isImmutable() const { return member->immutable; } bool isChanged() const { return member->isChanged(*collection); } Field getValue() const { return member->get_field(*collection); } String getValueAsString() const { return member->get_string(*collection); } @@ -497,6 +503,40 @@ public: void set(size_t index, const String & value) { (*this)[index].setValue(value); } void set(const String & name, const String & value) { (*this)[name].setValue(value); } + /// Updates setting's value. + void update(size_t index, const Field & value) + { + auto ref = (*this)[index]; + if (ref.isImmutable()) + throw Exception("Cannot modify immutable setting '" + ref.getName().toString() + "'", ErrorCodes::IMMUTABLE_SETTING); + ref.setValue(value); + } + + void update(const String & name, const Field & value) + { + auto ref = (*this)[name]; + if (ref.isImmutable()) + throw Exception("Cannot modify immutable setting '" + ref.getName().toString() + "'", ErrorCodes::IMMUTABLE_SETTING); + ref.setValue(value); + } + + /// Updates setting's value. Read value in text form from string (for example, from configuration file or from URL parameter). + void update(size_t index, const String & value) + { + auto ref = (*this)[index]; + if (ref.isImmutable()) + throw Exception("Cannot modify immutable setting '" + ref.getName().toString() + "'", ErrorCodes::IMMUTABLE_SETTING); + (*this)[index].setValue(value); + } + + void update(const String & name, const String & value) + { + auto ref = (*this)[name]; + if (ref.isImmutable()) + throw Exception("Cannot modify immutable setting '" + ref.getName().toString() + "'", ErrorCodes::IMMUTABLE_SETTING); + (*this)[name].setValue(value); + } + /// Returns value of a setting. Field get(size_t index) const { return (*this)[index].getValue(); } Field get(const String & name) const { return (*this)[name].getValue(); } @@ -561,17 +601,30 @@ public: } /// Applies changes to the settings. - void applyChange(const SettingChange & change) + void loadFromChange(const SettingChange & change) { set(change.name, change.value); } - void applyChanges(const SettingsChanges & changes) + void loadFromChanges(const SettingsChanges & changes) { for (const SettingChange & change : changes) - applyChange(change); + loadFromChange(change); } + /// Applies changes to the settings. + void updateFromChange(const SettingChange & change) + { + update(change.name, change.value); + } + + void updateFromChanges(const SettingsChanges & changes) + { + for (const SettingChange & change : changes) + updateFromChange(change); + } + + void copyChangesFrom(const Derived & src) { for (const auto & member : members()) @@ -615,7 +668,7 @@ public: }; #define DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS_MACRO) \ - LIST_OF_SETTINGS_MACRO(DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_) + LIST_OF_SETTINGS_MACRO(DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_, DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_) #define IMPLEMENT_SETTINGS_COLLECTION(DERIVED_CLASS_NAME, LIST_OF_SETTINGS_MACRO) \ @@ -625,9 +678,9 @@ public: using Derived = DERIVED_CLASS_NAME; \ struct Functions \ { \ - LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \ + LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_, IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \ }; \ - LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_) \ + LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_, IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_) \ } @@ -645,13 +698,22 @@ public: static Field NAME##_castValueWithoutApplying(const Field & value) { TYPE temp{DEFAULT}; temp.set(value); return temp.toField(); } -#define IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ +#define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ static_assert(std::is_same_v().NAME.changed), bool>); \ add({offsetof(Derived, NAME.changed), \ - StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), \ + StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ &Functions::NAME##_castValueWithoutApplying }); +#define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ + static_assert(std::is_same_v().NAME.changed), bool>); \ + add({offsetof(Derived, NAME.changed), \ + StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ + &Functions::NAME##_getString, &Functions::NAME##_getField, \ + &Functions::NAME##_setString, &Functions::NAME##_setField, \ + &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ + &Functions::NAME##_castValueWithoutApplying }); + } diff --git a/dbms/src/Storages/Kafka/KafkaSettings.cpp b/dbms/src/Storages/Kafka/KafkaSettings.cpp index d08282a9794..b08d45780bb 100644 --- a/dbms/src/Storages/Kafka/KafkaSettings.cpp +++ b/dbms/src/Storages/Kafka/KafkaSettings.cpp @@ -22,7 +22,7 @@ void KafkaSettings::loadFromQuery(ASTStorage & storage_def) { try { - applyChanges(storage_def.settings->changes); + loadFromChanges(storage_def.settings->changes); } catch (Exception & e) { diff --git a/dbms/src/Storages/Kafka/KafkaSettings.h b/dbms/src/Storages/Kafka/KafkaSettings.h index f3642d66803..366ec715bf0 100644 --- a/dbms/src/Storages/Kafka/KafkaSettings.h +++ b/dbms/src/Storages/Kafka/KafkaSettings.h @@ -14,17 +14,17 @@ class ASTStorage; struct KafkaSettings : public SettingsCollection { -#define LIST_OF_KAFKA_SETTINGS(M) \ - M(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ - M(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ - M(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \ - M(SettingString, kafka_format, "", "The message format for Kafka engine.") \ - M(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.") \ - M(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine") \ - M(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.") \ - M(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \ - M(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block") \ - M(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block") +#define LIST_OF_KAFKA_SETTINGS(M, IM) \ + IM(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ + IM(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ + IM(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \ + IM(SettingString, kafka_format, "", "The message format for Kafka engine.") \ + IM(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.") \ + IM(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine") \ + IM(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.") \ + IM(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \ + IM(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block") \ + IM(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block") DECLARE_SETTINGS_COLLECTION(LIST_OF_KAFKA_SETTINGS) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index e3600b6ac4a..562f2d8f402 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -46,7 +46,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) { try { - applyChanges(storage_def.settings->changes); + loadFromChanges(storage_def.settings->changes); } catch (Exception & e) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 9bd58e77f9c..c982bef324f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -24,8 +24,8 @@ class ASTStorage; struct MergeTreeSettings : public SettingsCollection { -#define LIST_OF_MERGE_TREE_SETTINGS(M) \ - M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \ +#define LIST_OF_MERGE_TREE_SETTINGS(M, IM) \ + IM(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \ \ /** Merge settings. */ \ M(SettingUInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024, "Maximum in total size of parts to merge, when there are maximum free threads in background pool (or entries in replication queue).") \ diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index f39735b1595..41e2b04895a 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -261,7 +261,7 @@ void StorageMergeTree::alter( context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, storage_modifier); setColumns(std::move(new_columns)); - settings.applyChanges(new_changes); + settings.updateFromChanges(new_changes); return; } From a3ebe3153537170e344dc8766d48a2630e63146c Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 29 Jul 2019 13:05:43 +0300 Subject: [PATCH 053/357] Brace style fix --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index cb9b8871a68..5adf344cf0b 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -17,7 +17,8 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { auto indices_wrong = findAllWrong(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize()); std::string indices_str = "{"; - for (size_t j = 0; j < indices_wrong.size(); ++j) { + for (size_t j = 0; j < indices_wrong.size(); ++j) + { indices_str += std::to_string(indices_wrong[j]); indices_str += (j != indices_wrong.size() - 1) ? ", " : "}"; } From 2f33df1b2ef01ba19db8d465400f08a4d532e060 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 30 Jul 2019 20:24:40 +0300 Subject: [PATCH 054/357] rename all parts before attaching --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 40 ++++++++++++ dbms/src/Storages/MergeTree/MergeTreeData.h | 15 +++++ .../Storages/MergeTree/MergeTreeDataPart.cpp | 24 ++----- .../Storages/MergeTree/MergeTreeDataPart.h | 2 +- .../Storages/MergeTree/MergeTreePartInfo.cpp | 1 + dbms/src/Storages/StorageMergeTree.cpp | 65 ++++++------------- dbms/src/Storages/StorageMergeTree.h | 1 - .../Storages/StorageReplicatedMergeTree.cpp | 6 +- .../00974_attach_invalid_parts.reference | 17 +++-- .../0_stateless/00974_attach_invalid_parts.sh | 21 ++++-- .../0_stateless/00975_drop_detached.reference | 6 +- .../0_stateless/00975_drop_detached.sh | 20 +++--- 12 files changed, 130 insertions(+), 88 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 9d12a9ee6ea..4b13ebaa99f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1718,6 +1718,29 @@ MergeTreeData::AlterDataPartTransaction::~AlterDataPartTransaction() } } +void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const String & new_name) +{ + Poco::File(base_dir + old_name).renameTo(base_dir + new_name); + old_and_new_names.push_back({old_name, new_name}); +} + +MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename() +{ + for (const auto & names : old_and_new_names) + { + if (names.first.empty()) + continue; + try + { + Poco::File(base_dir + names.second).renameTo(base_dir + names.first); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } +} + MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( const MergeTreePartInfo & new_part_info, @@ -2600,6 +2623,23 @@ void MergeTreeData::validateDetachedPartName(const String & name) const throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); } +void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Context &) +{ + if (!part) // TODO + throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); + + String part_id = partition->as().value.safeGet(); + validateDetachedPartName(part_id); + if (startsWith(part_id, "attaching_") || startsWith(part_id, "deleting_")) + throw DB::Exception("Cannot drop part " + part_id + ": " + "most likely it is used by another DROP or ATTACH query.", ErrorCodes::BAD_DATA_PART_NAME); + + PartsTemporaryRename renamed_parts(full_path + "detached/"); + renamed_parts.addPart(part_id, "deleting_" + part_id); + Poco::File(renamed_parts.base_dir + renamed_parts.old_and_new_names.front().second).remove(true); + renamed_parts.old_and_new_names.front().first.clear(); +} + MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const { DataParts res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 099591a97e4..62cebf32f76 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -249,6 +249,19 @@ public: using AlterDataPartTransactionPtr = std::unique_ptr; + struct PartsTemporaryRename : private boost::noncopyable + { + PartsTemporaryRename(const String & base_dir_) : base_dir(base_dir_) {} + + /// Renames part from old_name to new_name + void addPart(const String & old_name, const String & new_name); + + /// Renames all added parts from new_name to old_name if old name is not empty + ~PartsTemporaryRename(); + + String base_dir; + std::vector> old_and_new_names; + }; /// Parameters for various modes. struct MergingParams @@ -392,6 +405,8 @@ public: void validateDetachedPartName(const String & name) const; + void dropDetached(const ASTPtr & partition, bool part, const Context & context); + /// Returns Committed parts DataParts getDataParts() const; DataPartsVector getDataPartsVector() const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index a0888732495..fa2847aa301 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -142,10 +142,7 @@ MergeTreeDataPart::MergeTreeDataPart(MergeTreeData & storage_, const String & na { } -MergeTreeDataPart::MergeTreeDataPart( - const MergeTreeData & storage_, - const String & name_, - const MergeTreePartInfo & info_) +MergeTreeDataPart::MergeTreeDataPart(const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_) : storage(storage_) , name(name_) , info(info_) @@ -350,7 +347,7 @@ UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) return res; } -void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const +void MergeTreeDataPart::remove() const { if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); @@ -359,18 +356,18 @@ void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const * - rename directory to temporary name; * - remove it recursive. * - * For temporary name we use "detached/deleting_" prefix. + * For temporary name we use "delete_tmp_" prefix. * - * NOTE: We cannot use "tmp_*" prefix, because there is a second thread, + * NOTE: We cannot use "tmp_delete_" prefix, because there is a second thread, * that calls "clearOldTemporaryDirectories" and removes all directories, that begin with "tmp_" and are old enough. * But when we removing data part, it can be old enough. And rename doesn't change mtime. * And a race condition can happen that will lead to "File not found" error here. - * We move directory to detached/, because if an attempt to remove directory after renaming failed for some reason - * there would be no way to remove directory from storage.full_path (except manually). */ + // TODO directory delete_tmp_ is never removed if server crashes before returning from this function + String from = storage.full_path + relative_path; - String to = storage.full_path + getRelativePathForDetachedPart("deleting_"); + String to = storage.full_path + "delete_tmp_" + name; Poco::File from_dir{from}; Poco::File to_dir{to}; @@ -403,13 +400,6 @@ void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const return; } - if (force_recursive) - { - /// Part is not loaded (we don't know which files are there), so remove dir recursively. - to_dir.remove(true); - return; - } - try { /// Remove each expected file in directory, then remove directory itself. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h index 98af00c071a..f41ea8af424 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h @@ -241,7 +241,7 @@ struct MergeTreeDataPart /// Calculate the total size of the entire directory with all the files static UInt64 calculateTotalSizeOnDisk(const String & from); - void remove(bool force_recursive = false) const; + void remove() const; /// Makes checks and move part to new directory /// Changes only relative_dir_name, you need to update other metadata (name, is_temp) explicitly diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index a9e31a988b3..449ea143e17 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -194,6 +194,7 @@ bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, Detache part_info.dir_name = dir_name; /// First, try to parse as . + // TODO what if tryParsePartName will parse prefix as partition_id? if (MergeTreePartInfo::tryParsePartName(dir_name, &part_info, format_version)) return part_info.valid_name = true; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 98b3ae6ad6c..db5632c3fe9 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -994,6 +994,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons /// TODO: should we include PreComitted parts like in Replicated case? auto parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); + // TODO should we throw an exception if parts_to_remove is empty? removePartsFromWorkingSet(parts_to_remove, true); if (detach) @@ -1013,22 +1014,6 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } -void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const Context & /*context*/) -{ - if (!part) // TODO - throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); - - String part_id = partition->as().value.safeGet(); - validateDetachedPartName(part_id); - - DetachedPartInfo info; - DetachedPartInfo::tryParseDetachedPartName(part_id, info, format_version); - MergeTreeDataPart detached_part(*this, part_id, info); - detached_part.relative_path = "detached/" + part_id; - - detached_part.remove(true); -} - void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) { // TODO: should get some locks to prevent race with 'alter … modify column' @@ -1069,42 +1054,30 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par } LOG_DEBUG(log, active_parts.size() << " of them are active"); parts = active_parts.getParts(); + + // TODO should we rename inactive parts? (see StorageReplicatedMergeTree::attachPartition) } + PartsTemporaryRename renamed_parts(full_path + source_dir); for (const auto & source_part_name : parts) + renamed_parts.addPart(source_part_name, "attaching_" + source_part_name); + + std::vector loaded_parts; + for (const auto & part_names : renamed_parts.old_and_new_names) { - MutableDataPartPtr part; - try - { - part = std::make_shared(*this, source_part_name); - part->relative_path = "detached/" + source_part_name; - part->renameTo("detached/attaching_" + source_part_name, false); - - LOG_DEBUG(log, "Checking data in " << part->relative_path); + LOG_DEBUG(log, "Checking data in " << part_names.second); + MutableDataPartPtr part = std::make_shared(*this, part_names.first); + part->relative_path = source_dir + part_names.second; loadPartAndFixMetadata(part); + loaded_parts.push_back(part); + } - LOG_INFO(log, "Attaching part " << source_part_name << " from " << part->relative_path); - renameTempPartAndAdd(part, &increment); - - LOG_INFO(log, "Finished attaching part"); - } - catch (...) - { - LOG_INFO(log, "Cannot attach part " << source_part_name << " :" << getCurrentExceptionMessage(false)); - - if (part && part->relative_path == "detached/attaching_" + source_part_name) - { - try - { - part->renameTo("detached/" + source_part_name, false); - } - catch (...) - { - tryLogCurrentException(log, __PRETTY_FUNCTION__); - } - } - - } + for (size_t i = 0; i < loaded_parts.size(); ++i) + { + LOG_INFO(log, "Attaching part " << loaded_parts[i]->name << " from " << renamed_parts.old_and_new_names[i].second); + renameTempPartAndAdd(loaded_parts[i], &increment); + renamed_parts.old_and_new_names[i].first.clear(); + LOG_INFO(log, "Finished attaching part"); } /// New parts with other data may appear in place of deleted parts. diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index fa2561e4ab2..0de9618d915 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -120,7 +120,6 @@ private: // Partition helpers void dropPartition(const ASTPtr & partition, bool detach, const Context & context); - void dropDetached(const ASTPtr & partition, bool part, const Context & context); void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 531678decc3..67577dee2b6 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3353,8 +3353,8 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part break; case PartitionCommand::DROP_DETACHED_PARTITION: - // TODO - throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + dropDetached(command.partition, command.part, query_context); + break; case PartitionCommand::ATTACH_PARTITION: attachPartition(command.partition, command.part, query_context); @@ -3601,6 +3601,8 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool loaded_parts.push_back(loadPartAndFixMetadata(source_dir + part)); } + // TODO fix race with DROP DETACHED + ReplicatedMergeTreeBlockOutputStream output(*this, 0, 0, 0, false); /// TODO Allow to use quorum here. for (auto & part : loaded_parts) { diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference index 42a04fe5666..f30fc160dfb 100644 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference @@ -1,17 +1,26 @@ === cannot attach active === -OK +OK1 0_1_1_0 1_2_2_0 2_3_3_0 3_4_4_0 16 120 -=== attach all valid parts === +=== check all parts before attaching === +OK2 +1_2_2_0 +1_4_4_0 +=== detached === +0_1_1_0 +0_3_3_0 +0_5_5_0 +attaching_0_6_6_0 +deleting_0_7_7_0 +=== attach === 0_5_5_0 0_6_6_0 1_2_2_0 1_4_4_0 16 120 === detached === -0_5_5_0 -deleting_0_7_7_0 attaching_0_6_6_0 +deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh index 4e9efa64ad1..a4afbe8f817 100755 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh @@ -12,14 +12,13 @@ echo '=== cannot attach active ==='; $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_active"; $CLICKHOUSE_CLIENT --query="CREATE TABLE attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; $CLICKHOUSE_CLIENT --query="INSERT INTO attach_active SELECT number FROM system.numbers LIMIT 16"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_active ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_active ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' $CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_active' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_active FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE attach_active"; -echo '=== attach all valid parts ==='; $CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_partitions"; $CLICKHOUSE_CLIENT --query="CREATE TABLE attach_partitions (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; @@ -27,18 +26,28 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM sys $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; -sudo -n mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ - mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; +echo '=== check all parts before attaching ==='; +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0" 2>&1 | grep "No columns in part 0_5_5_0" > /dev/null && echo 'OK2'; +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; +echo '=== detached ==='; +$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; + +echo '=== attach ==='; +sudo -n rm -r $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ + rm -r $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; $CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_partitions FORMAT TSV"; + echo '=== detached ==='; -$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE attach_partitions"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.reference b/dbms/tests/queries/0_stateless/00975_drop_detached.reference index 40732c908ab..2a355138980 100644 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.reference +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.reference @@ -1,2 +1,6 @@ -OK +OK1 +OK2 +OK3 0_3_3_0 +attaching_0_6_6_0 +deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh index 3a5e920da75..71c0b5681fd 100755 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.sh +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.sh @@ -15,19 +15,19 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system. $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; -sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ - mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ -sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ - mkdir $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ -sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ - mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK2' +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK3' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From c6717e0d3f977e23aa9b778d00bcf64456e893cb Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 30 Jul 2019 22:11:15 +0300 Subject: [PATCH 055/357] refactor attachPartition --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 71 +++++++++++++++++++ dbms/src/Storages/MergeTree/MergeTreeData.h | 3 + dbms/src/Storages/StorageMergeTree.cpp | 55 +------------- .../Storages/StorageReplicatedMergeTree.cpp | 68 ++---------------- 4 files changed, 83 insertions(+), 114 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 4b13ebaa99f..11ad7835b51 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2640,6 +2640,77 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Cont renamed_parts.old_and_new_names.front().first.clear(); } +MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part, + const Context & context, PartsTemporaryRename & renamed_parts) +{ + String partition_id; + + if (attach_part) + partition_id = partition->as().value.safeGet(); + else + partition_id = getPartitionIDFromQuery(partition, context); + + String source_dir = "detached/"; + + /// Let's compose a list of parts that should be added. + Strings parts; + if (attach_part) + { + validateDetachedPartName(partition_id); + parts.push_back(partition_id); + } + else + { + LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); + ActiveDataPartSet active_parts(format_version); + + std::set part_names; + for (Poco::DirectoryIterator it = Poco::DirectoryIterator(full_path + source_dir); it != Poco::DirectoryIterator(); ++it) + { + String name = it.name(); + MergeTreePartInfo part_info; + // TODO what if name contains "_tryN" suffix? + if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version)) + continue; + if (part_info.partition_id != partition_id) + continue; + LOG_DEBUG(log, "Found part " << name); + active_parts.add(name); + part_names.insert(name); + } + LOG_DEBUG(log, active_parts.size() << " of them are active"); + parts = active_parts.getParts(); + + /// Inactive parts rename so they can not be attached in case of repeated ATTACH. + for (const auto & name : part_names) + { + // TODO maybe use PartsTemporaryRename here? + String containing_part = active_parts.getContainingPart(name); + if (!containing_part.empty() && containing_part != name) + Poco::File(full_path + source_dir + name).renameTo(full_path + source_dir + "inactive_" + name); + } + } + + /// Try to rename all parts before attaching to prevent race with DROP DETACHED and another ATTACH. + for (const auto & source_part_name : parts) + renamed_parts.addPart(source_part_name, "attaching_" + source_part_name); + + /// Synchronously check that added parts exist and are not broken. We will write checksums.txt if it does not exist. + LOG_DEBUG(log, "Checking parts"); + MutableDataPartsVector loaded_parts; + loaded_parts.reserve(parts.size()); + for (const auto & part_names : renamed_parts.old_and_new_names) + { + LOG_DEBUG(log, "Checking part " << part_names.second); + MutableDataPartPtr part = std::make_shared(*this, part_names.first); + part->relative_path = source_dir + part_names.second; + loadPartAndFixMetadata(part); + loaded_parts.push_back(part); + } + + return loaded_parts; +} + MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const { DataParts res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 62cebf32f76..3592164fed5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -407,6 +407,9 @@ public: void dropDetached(const ASTPtr & partition, bool part, const Context & context); + MutableDataPartsVector tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part, + const Context & context, PartsTemporaryRename & renamed_parts); + /// Returns Committed parts DataParts getDataParts() const; DataPartsVector getDataPartsVector() const; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index db5632c3fe9..3464255e1b8 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -1018,59 +1018,8 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par { // TODO: should get some locks to prevent race with 'alter … modify column' - String partition_id; - - if (attach_part) - partition_id = partition->as().value.safeGet(); - else - partition_id = getPartitionIDFromQuery(partition, context); - - String source_dir = "detached/"; - - /// Let's make a list of parts to add. - Strings parts; - if (attach_part) - { - validateDetachedPartName(partition_id); - parts.push_back(partition_id); - } - else - { - LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); - ActiveDataPartSet active_parts(format_version); - for (Poco::DirectoryIterator it = Poco::DirectoryIterator(full_path + source_dir); it != Poco::DirectoryIterator(); ++it) - { - const String & name = it.name(); - MergeTreePartInfo part_info; - /// Parts with prefix in name (e.g. attaching_1_3_3_0, deleting_1_3_3_0) will be ignored - // TODO what if name contains "_tryN" suffix? - if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version) - || part_info.partition_id != partition_id) - { - continue; - } - LOG_DEBUG(log, "Found part " << name); - active_parts.add(name); - } - LOG_DEBUG(log, active_parts.size() << " of them are active"); - parts = active_parts.getParts(); - - // TODO should we rename inactive parts? (see StorageReplicatedMergeTree::attachPartition) - } - - PartsTemporaryRename renamed_parts(full_path + source_dir); - for (const auto & source_part_name : parts) - renamed_parts.addPart(source_part_name, "attaching_" + source_part_name); - - std::vector loaded_parts; - for (const auto & part_names : renamed_parts.old_and_new_names) - { - LOG_DEBUG(log, "Checking data in " << part_names.second); - MutableDataPartPtr part = std::make_shared(*this, part_names.first); - part->relative_path = source_dir + part_names.second; - loadPartAndFixMetadata(part); - loaded_parts.push_back(part); - } + PartsTemporaryRename renamed_parts(full_path + "detached/"); + MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, context, renamed_parts); for (size_t i = 0; i < loaded_parts.size(); ++i) { diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 67577dee2b6..7e192d77a33 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3545,70 +3545,16 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool assertNotReadonly(); - String partition_id; - - if (attach_part) - partition_id = partition->as().value.safeGet(); - else - partition_id = getPartitionIDFromQuery(partition, query_context); - - String source_dir = "detached/"; - - /// Let's compose a list of parts that should be added. - Strings parts; - if (attach_part) - { - validateDetachedPartName(partition_id); - parts.push_back(partition_id); - } - else - { - LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); - ActiveDataPartSet active_parts(format_version); - - std::set part_names; - for (Poco::DirectoryIterator it = Poco::DirectoryIterator(full_path + source_dir); it != Poco::DirectoryIterator(); ++it) - { - String name = it.name(); - MergeTreePartInfo part_info; - // TODO what if name contains "_tryN" suffix? - if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version)) - continue; - if (part_info.partition_id != partition_id) - continue; - LOG_DEBUG(log, "Found part " << name); - active_parts.add(name); - part_names.insert(name); - } - LOG_DEBUG(log, active_parts.size() << " of them are active"); - parts = active_parts.getParts(); - - /// Inactive parts rename so they can not be attached in case of repeated ATTACH. - for (const auto & name : part_names) - { - String containing_part = active_parts.getContainingPart(name); - if (!containing_part.empty() && containing_part != name) - Poco::File(full_path + source_dir + name).renameTo(full_path + source_dir + "inactive_" + name); - } - } - - /// Synchronously check that added parts exist and are not broken. We will write checksums.txt if it does not exist. - LOG_DEBUG(log, "Checking parts"); - std::vector loaded_parts; - for (const String & part : parts) - { - LOG_DEBUG(log, "Checking part " << part); - loaded_parts.push_back(loadPartAndFixMetadata(source_dir + part)); - } - - // TODO fix race with DROP DETACHED + PartsTemporaryRename renamed_parts(full_path + "detached/"); + MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, query_context, renamed_parts); ReplicatedMergeTreeBlockOutputStream output(*this, 0, 0, 0, false); /// TODO Allow to use quorum here. - for (auto & part : loaded_parts) + for (size_t i = 0; i < loaded_parts.size(); ++i) { - String old_name = part->name; - output.writeExistingPart(part); - LOG_DEBUG(log, "Attached part " << old_name << " as " << part->name); + String old_name = loaded_parts[i]->name; + output.writeExistingPart(loaded_parts[i]); + renamed_parts.old_and_new_names[i].first.clear(); + LOG_DEBUG(log, "Attached part " << old_name << " as " << loaded_parts[i]->name); } } From f0836553d449368cac474e9ed9f60d3120ea79c4 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 31 Jul 2019 17:44:55 +0300 Subject: [PATCH 056/357] drop detached partition --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 94 +++++++++++++------ dbms/src/Storages/MergeTree/MergeTreeData.h | 10 +- .../Storages/MergeTree/MergeTreePartInfo.cpp | 2 +- .../Storages/MergeTree/MergeTreePartInfo.h | 3 +- dbms/src/Storages/PartitionCommands.cpp | 3 - dbms/src/Storages/StorageMergeTree.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 2 +- .../0_stateless/00974_attach_invalid_parts.sh | 8 +- .../0_stateless/00975_drop_detached.reference | 9 ++ .../0_stateless/00975_drop_detached.sh | 12 +++ 10 files changed, 103 insertions(+), 42 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 11ad7835b51..32cd3ad508e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1720,12 +1720,35 @@ MergeTreeData::AlterDataPartTransaction::~AlterDataPartTransaction() void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const String & new_name) { - Poco::File(base_dir + old_name).renameTo(base_dir + new_name); old_and_new_names.push_back({old_name, new_name}); } +void MergeTreeData::PartsTemporaryRename::tryRenameAll() +{ + renamed = true; + for (size_t i = 0; i < old_and_new_names.size(); ++i) + { + try + { + const auto & names = old_and_new_names[i]; + if (names.first.empty() || names.second.empty()) + throw DB::Exception("Empty part name. Most likely it's a bug.", ErrorCodes::INCORRECT_FILE_NAME); + Poco::File(base_dir + names.first).renameTo(base_dir + names.second); + } + catch (...) + { + old_and_new_names.resize(i); + LOG_WARNING(storage.log, "Cannot rename parts to perform operation on them: " << getCurrentExceptionMessage(false)); + throw; + } + } +} + MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename() { + // TODO what if server had crashed before this destructor was called? + if (!renamed) + return; for (const auto & names : old_and_new_names) { if (names.first.empty()) @@ -2621,46 +2644,60 @@ void MergeTreeData::validateDetachedPartName(const String & name) const Poco::File detached_part_dir(full_path + "detached/" + name); if (!detached_part_dir.exists()) throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); + + if (startsWith(name, "attaching_") || startsWith(name, "deleting_")) + throw DB::Exception("Cannot drop part " + name + ": " + "most likely it is used by another DROP or ATTACH query.", + ErrorCodes::BAD_DATA_PART_NAME); } -void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Context &) +void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Context & context) { - if (!part) // TODO - throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); + PartsTemporaryRename renamed_parts(*this, full_path + "detached/"); - String part_id = partition->as().value.safeGet(); - validateDetachedPartName(part_id); - if (startsWith(part_id, "attaching_") || startsWith(part_id, "deleting_")) - throw DB::Exception("Cannot drop part " + part_id + ": " - "most likely it is used by another DROP or ATTACH query.", ErrorCodes::BAD_DATA_PART_NAME); + if (part) + { + String part_name = partition->as().value.safeGet(); + validateDetachedPartName(part_name); + renamed_parts.addPart(part_name, "deleting_" + part_name); + } + else + { + String partition_id = getPartitionIDFromQuery(partition, context); + DetachedPartsInfo detached_parts = getDetachedParts(); + for (const auto & part_info : detached_parts) + if (part_info.valid_name && part_info.partition_id == partition_id + && part_info.prefix != "attaching" && part_info.prefix != "deleting") + renamed_parts.addPart(part_info.dir_name, "deleting_" + part_info.dir_name); + } - PartsTemporaryRename renamed_parts(full_path + "detached/"); - renamed_parts.addPart(part_id, "deleting_" + part_id); - Poco::File(renamed_parts.base_dir + renamed_parts.old_and_new_names.front().second).remove(true); - renamed_parts.old_and_new_names.front().first.clear(); + LOG_DEBUG(log, "Will drop " << renamed_parts.old_and_new_names.size() << " detached parts."); + + renamed_parts.tryRenameAll(); + + for (auto & names : renamed_parts.old_and_new_names) + { + Poco::File(renamed_parts.base_dir + names.second).remove(true); + LOG_DEBUG(log, "Dropped detached part " << names.first); + names.first.clear(); + } } MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part, const Context & context, PartsTemporaryRename & renamed_parts) { - String partition_id; - - if (attach_part) - partition_id = partition->as().value.safeGet(); - else - partition_id = getPartitionIDFromQuery(partition, context); - String source_dir = "detached/"; /// Let's compose a list of parts that should be added. - Strings parts; if (attach_part) { - validateDetachedPartName(partition_id); - parts.push_back(partition_id); + String part_id = partition->as().value.safeGet(); + validateDetachedPartName(part_id); + renamed_parts.addPart(part_id, "attaching_" + part_id); } else { + String partition_id = getPartitionIDFromQuery(partition, context); LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); ActiveDataPartSet active_parts(format_version); @@ -2670,6 +2707,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const String name = it.name(); MergeTreePartInfo part_info; // TODO what if name contains "_tryN" suffix? + /// Parts with prefix in name (e.g. attaching_1_3_3_0, deleting_1_3_3_0) will be ignored if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version)) continue; if (part_info.partition_id != partition_id) @@ -2679,26 +2717,26 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const part_names.insert(name); } LOG_DEBUG(log, active_parts.size() << " of them are active"); - parts = active_parts.getParts(); /// Inactive parts rename so they can not be attached in case of repeated ATTACH. for (const auto & name : part_names) { - // TODO maybe use PartsTemporaryRename here? String containing_part = active_parts.getContainingPart(name); if (!containing_part.empty() && containing_part != name) + // TODO maybe use PartsTemporaryRename here? Poco::File(full_path + source_dir + name).renameTo(full_path + source_dir + "inactive_" + name); + else + renamed_parts.addPart(name, "attaching_" + name); } } /// Try to rename all parts before attaching to prevent race with DROP DETACHED and another ATTACH. - for (const auto & source_part_name : parts) - renamed_parts.addPart(source_part_name, "attaching_" + source_part_name); + renamed_parts.tryRenameAll(); /// Synchronously check that added parts exist and are not broken. We will write checksums.txt if it does not exist. LOG_DEBUG(log, "Checking parts"); MutableDataPartsVector loaded_parts; - loaded_parts.reserve(parts.size()); + loaded_parts.reserve(renamed_parts.old_and_new_names.size()); for (const auto & part_names : renamed_parts.old_and_new_names) { LOG_DEBUG(log, "Checking part " << part_names.second); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 3592164fed5..9f5d0961d27 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -251,16 +251,20 @@ public: struct PartsTemporaryRename : private boost::noncopyable { - PartsTemporaryRename(const String & base_dir_) : base_dir(base_dir_) {} + PartsTemporaryRename(const MergeTreeData & storage_, const String & base_dir_) : storage(storage_), base_dir(base_dir_) {} + + void addPart(const String & old_name, const String & new_name); /// Renames part from old_name to new_name - void addPart(const String & old_name, const String & new_name); + void tryRenameAll(); /// Renames all added parts from new_name to old_name if old name is not empty ~PartsTemporaryRename(); + const MergeTreeData & storage; String base_dir; std::vector> old_and_new_names; + bool renamed = false; }; /// Parameters for various modes. @@ -401,7 +405,7 @@ public: DataPartsVector getAllDataPartsVector(DataPartStateVector * out_states = nullptr) const; /// Returns all detached parts - std::vector getDetachedParts() const; + DetachedPartsInfo getDetachedParts() const; void validateDetachedPartName(const String & name) const; diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 449ea143e17..3ee330b6d1a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -194,7 +194,7 @@ bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, Detache part_info.dir_name = dir_name; /// First, try to parse as . - // TODO what if tryParsePartName will parse prefix as partition_id? + // TODO what if tryParsePartName will parse prefix as partition_id? It can happen if dir_name doesn't contain mutation number at the end if (MergeTreePartInfo::tryParsePartName(dir_name, &part_info, format_version)) return part_info.valid_name = true; diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index 25cf46ad46d..9fe0fbab533 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -92,7 +92,6 @@ struct MergeTreePartInfo /// addition to the above fields. struct DetachedPartInfo : public MergeTreePartInfo { - /// Suddenly, name of detached part may contain suffix (such as _tryN), which is ignored by MergeTreePartInfo::tryParsePartName(...) String dir_name; String prefix; @@ -102,4 +101,6 @@ struct DetachedPartInfo : public MergeTreePartInfo static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version); }; +using DetachedPartsInfo = std::vector; + } diff --git a/dbms/src/Storages/PartitionCommands.cpp b/dbms/src/Storages/PartitionCommands.cpp index bab3f6ced24..0537482dbc1 100644 --- a/dbms/src/Storages/PartitionCommands.cpp +++ b/dbms/src/Storages/PartitionCommands.cpp @@ -25,9 +25,6 @@ std::optional PartitionCommand::parse(const ASTAlterCommand * } else if (command_ast->type == ASTAlterCommand::DROP_DETACHED_PARTITION) { - if (!command_ast->part) // TODO - throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); - PartitionCommand res; res.type = DROP_DETACHED_PARTITION; res.partition = command_ast->partition; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 3464255e1b8..c2ee4854c39 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -1018,7 +1018,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par { // TODO: should get some locks to prevent race with 'alter … modify column' - PartsTemporaryRename renamed_parts(full_path + "detached/"); + PartsTemporaryRename renamed_parts(*this, full_path + "detached/"); MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, context, renamed_parts); for (size_t i = 0; i < loaded_parts.size(); ++i) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 7e192d77a33..5109d9f7e54 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3545,7 +3545,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool assertNotReadonly(); - PartsTemporaryRename renamed_parts(full_path + "detached/"); + PartsTemporaryRename renamed_parts(*this, full_path + "detached/"); MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, query_context, renamed_parts); ReplicatedMergeTreeBlockOutputStream output(*this, 0, 0, 0, false); /// TODO Allow to use quorum here. diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh index a4afbe8f817..db45cfe7f21 100755 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh @@ -28,10 +28,10 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM sys $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part -sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ - cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ -sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ - cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ +sudo -n cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ + cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ +sudo -n cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ + cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ echo '=== check all parts before attaching ==='; $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0" 2>&1 | grep "No columns in part 0_5_5_0" > /dev/null && echo 'OK2'; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.reference b/dbms/tests/queries/0_stateless/00975_drop_detached.reference index 2a355138980..414ac4b1927 100644 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.reference +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.reference @@ -1,6 +1,15 @@ +=== validate part name === OK1 OK2 OK3 +=== drop detached part === +0_3_3_0 +1_2_2_0 +1_4_4_0 +attaching_0_6_6_0 +deleting_0_7_7_0 +prefix_1_2_2_0_0 +=== drop detached partition === 0_3_3_0 attaching_0_6_6_0 deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh index 71c0b5681fd..8da831b019a 100755 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.sh +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.sh @@ -15,19 +15,31 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system. $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; +$CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 1"; sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0_0/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0_0/ +#sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0/ 2>/dev/null || \ +# mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0/ +echo '=== validate part name ===' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK2' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK3' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" +echo '=== drop detached part ===' $CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; + +echo '=== drop detached partition ===' +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PARTITION 1" +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; + $CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From 662364b7206ee374752889ccf17f25b7fecb49b6 Mon Sep 17 00:00:00 2001 From: Sergi Vladykin Date: Wed, 31 Jul 2019 22:36:03 +0300 Subject: [PATCH 057/357] performance bug fix: never read the children if we are not going to return any data --- dbms/src/DataStreams/TTLBlockInputStream.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/DataStreams/TTLBlockInputStream.cpp b/dbms/src/DataStreams/TTLBlockInputStream.cpp index 78d8d25e8bc..0ad72876ea9 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.cpp +++ b/dbms/src/DataStreams/TTLBlockInputStream.cpp @@ -63,10 +63,6 @@ TTLBlockInputStream::TTLBlockInputStream( Block TTLBlockInputStream::readImpl() { - Block block = children.at(0)->read(); - if (!block) - return block; - /// Skip all data if table ttl is expired for part if (storage.hasTableTTL() && old_ttl_infos.table_ttl.max <= current_time) { @@ -74,6 +70,10 @@ Block TTLBlockInputStream::readImpl() return {}; } + Block block = children.at(0)->read(); + if (!block) + return block; + if (storage.hasTableTTL() && old_ttl_infos.table_ttl.min <= current_time) removeRowsWithExpiredTableTTL(block); From cc3900817c82e2a153218961813826bdc422d746 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Mon, 5 Aug 2019 16:00:27 +0300 Subject: [PATCH 058/357] Clickhouse-benchmark comparison mode --- dbms/programs/benchmark/Benchmark.cpp | 213 ++++++++++++++++++-------- 1 file changed, 149 insertions(+), 64 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index c69e9a54feb..3d16974db4b 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -51,16 +51,29 @@ class Benchmark : public Poco::Util::Application { public: Benchmark(unsigned concurrency_, double delay_, - const String & host_, UInt16 port_, bool secure_, const String & default_database_, + const std::vector & hosts_, const std::vector & ports_, + bool secure_, const String & default_database_, const String & user_, const String & password_, const String & stage, bool randomize_, size_t max_iterations_, double max_time_, const String & json_path_, const Settings & settings_) : concurrency(concurrency_), delay(delay_), queue(concurrency), - connections(concurrency, host_, port_, default_database_, user_, password_, "benchmark", Protocol::Compression::Enable, secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable), randomize(randomize_), max_iterations(max_iterations_), max_time(max_time_), json_path(json_path_), settings(settings_), global_context(Context::createGlobal()), pool(concurrency) { + const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; + + connections.reserve(ports_.size()); + comparison_info_total.reserve(ports_.size()); + comparison_info_per_interval.reserve(ports_.size()); + + for (size_t i = 0; i < ports_.size(); ++i) + { + connections.emplace_back(std::make_shared(concurrency, hosts_[i], ports_[i], default_database_, user_, password_, "benchmark", Protocol::Compression::Enable, secure)); + comparison_info_per_interval.emplace_back(std::make_shared()); + comparison_info_total.emplace_back(std::make_shared()); + } + global_context.makeGlobalContext(); std::cerr << std::fixed << std::setprecision(3); @@ -101,18 +114,22 @@ public: } private: - using Query = std::string; + using Entry = ConnectionPool::Entry; + using EntryPtr = std::shared_ptr; + using EntryPtrs = std::vector; unsigned concurrency; double delay; + using Query = std::string; using Queries = std::vector; Queries queries; using Queue = ConcurrentBoundedQueue; Queue queue; - ConnectionPool connections; + ConnectionPoolPtrs connections; + bool randomize; size_t max_iterations; double max_time; @@ -160,15 +177,17 @@ private: } }; - Stats info_per_interval; - Stats info_total; + using MultiStats = std::vector>; + MultiStats comparison_info_per_interval; + MultiStats comparison_info_total; + + Stopwatch total_watch; Stopwatch delay_watch; std::mutex mutex; ThreadPool pool; - void readQueries() { ReadBufferFromFileDescriptor in(STDIN_FILENO); @@ -213,7 +232,7 @@ private: return false; } - if (max_time > 0 && info_total.watch.elapsedSeconds() >= max_time) + if (max_time > 0 && total_watch.elapsedSeconds() >= max_time) { std::cout << "Stopping launch of queries. Requested time limit is exhausted.\n"; return false; @@ -227,8 +246,8 @@ private: if (delay > 0 && delay_watch.elapsedSeconds() > delay) { - printNumberOfQueriesExecuted(info_total.queries); - report(info_per_interval); + printNumberOfQueriesExecuted(queries_executed); + report(comparison_info_per_interval); delay_watch.restart(); } } @@ -242,11 +261,21 @@ private: std::uniform_int_distribution distribution(0, queries.size() - 1); for (size_t i = 0; i < concurrency; ++i) - pool.schedule(std::bind(&Benchmark::thread, this, - connections.get(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings)))); + { + EntryPtrs connection_entries; + connection_entries.reserve(connections.size()); + + for (const auto & connection : connections) + connection_entries.emplace_back(std::make_shared(connection->get(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings)))); + + pool.schedule(std::bind(&Benchmark::thread, this, connection_entries)); + } InterruptListener interrupt_listener; - info_per_interval.watch.restart(); + + for (auto & connection_stats : comparison_info_per_interval) + connection_stats->watch.restart(); + delay_watch.restart(); /// Push queries into queue @@ -262,17 +291,20 @@ private: } pool.wait(); - info_total.watch.stop(); + total_watch.stop(); + + for (auto & connection_stats : comparison_info_total) + connection_stats->watch.stop(); if (!json_path.empty()) - reportJSON(info_total, json_path); + reportJSON(comparison_info_total, json_path); - printNumberOfQueriesExecuted(info_total.queries); - report(info_total); + printNumberOfQueriesExecuted(queries_executed); + report(comparison_info_total); } - void thread(ConnectionPool::Entry connection) + void thread(EntryPtrs & connection_entries) { Query query; @@ -296,8 +328,7 @@ private: if (shutdown || (max_iterations && queries_executed == max_iterations)) return; } - - execute(connection, query); + execute(connection_entries, query); ++queries_executed; } } @@ -309,20 +340,21 @@ private: } } - - void execute(ConnectionPool::Entry & connection, Query & query) + void execute(EntryPtrs & connection_entries, Query & query) { + size_t connection_index = rand() % connection_entries.size(); + Stopwatch watch; RemoteBlockInputStream stream( - *connection, + *(*connection_entries[connection_index]), query, {}, global_context, &settings, nullptr, Tables(), query_processing_stage); Progress progress; stream.setProgressCallback([&progress](const Progress & value) { progress.incrementPiecewiseAtomically(value); }); stream.readPrefix(); - while (Block block = stream.read()) - ; + while (Block block = stream.read()); + stream.readSuffix(); const BlockStreamProfileInfo & info = stream.getProfileInfo(); @@ -330,33 +362,48 @@ private: double seconds = watch.elapsedSeconds(); std::lock_guard lock(mutex); - info_per_interval.add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); - info_total.add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); + + comparison_info_per_interval[connection_index]->add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); + comparison_info_total[connection_index]->add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); } - - void report(Stats & info) + void report(MultiStats & infos) { std::lock_guard lock(mutex); - /// Avoid zeros, nans or exceptions - if (0 == info.queries) - return; + std::cerr << "\n"; + size_t info_counter = 1; + for (auto & info : infos) + { + /// Avoid zeros, nans or exceptions + if (0 == info->queries) + return; - double seconds = info.watch.elapsedSeconds(); + double seconds = info->watch.elapsedSeconds(); - std::cerr - << "\n" - << "QPS: " << (info.queries / seconds) << ", " - << "RPS: " << (info.read_rows / seconds) << ", " - << "MiB/s: " << (info.read_bytes / seconds / 1048576) << ", " - << "result RPS: " << (info.result_rows / seconds) << ", " - << "result MiB/s: " << (info.result_bytes / seconds / 1048576) << "." - << "\n"; + std::cerr + << "connection: " << info_counter++ << ", " + << "QPS: " << (info->queries / seconds) << ", " + << "RPS: " << (info->read_rows / seconds) << ", " + << "MiB/s: " << (info->read_bytes / seconds / 1048576) << ", " + << "result RPS: " << (info->result_rows / seconds) << ", " + << "result MiB/s: " << (info->result_bytes / seconds / 1048576) << "." + << "\n"; + } + std::cerr << "\n\t\t"; + for (size_t i = 1; i <= infos.size(); ++i) + std::cerr << "connection " << i << "\t"; + + std::cerr << "\n"; auto print_percentile = [&](double percent) { - std::cerr << percent << "%\t" << info.sampler.quantileInterpolated(percent / 100.0) << " sec." << std::endl; + std::cerr << percent << "%\t\t"; + for (auto & info : infos) + { + std::cerr << info->sampler.quantileInterpolated(percent / 100.0) << " sec." << "\t"; + } + std::cerr << "\n"; }; for (int percent = 0; percent <= 90; percent += 10) @@ -367,10 +414,11 @@ private: print_percentile(99.9); print_percentile(99.99); - info.clear(); + for (auto & info : infos) + info->clear(); } - void reportJSON(Stats & info, const std::string & filename) + void reportJSON(MultiStats & infos, const std::string & filename) { WriteBufferFromFile json_out(filename); @@ -381,36 +429,47 @@ private: json_out << double_quote << key << ": " << value << (with_comma ? ",\n" : "\n"); }; - auto print_percentile = [&json_out, &info](auto percent, bool with_comma = true) + auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true) { json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileInterpolated(percent / 100.0) << (with_comma ? ",\n" : "\n"); }; json_out << "{\n"; - json_out << double_quote << "statistics" << ": {\n"; + for (size_t i = 1; i <= infos.size(); ++i) + { + auto info = infos[i]; - double seconds = info.watch.elapsedSeconds(); - print_key_value("QPS", info.queries / seconds); - print_key_value("RPS", info.read_rows / seconds); - print_key_value("MiBPS", info.read_bytes / seconds); - print_key_value("RPS_result", info.result_rows / seconds); - print_key_value("MiBPS_result", info.result_bytes / seconds); - print_key_value("num_queries", info.queries.load(), false); + json_out << double_quote << "connection_" + toString(i) << ": {\n"; + json_out << double_quote << "statistics" << ": {\n"; - json_out << "},\n"; + double seconds = info->watch.elapsedSeconds(); + print_key_value("QPS", info->queries / seconds); + print_key_value("RPS", info->read_rows / seconds); + print_key_value("MiBPS", info->read_bytes / seconds); + print_key_value("RPS_result", info->result_rows / seconds); + print_key_value("MiBPS_result", info->result_bytes / seconds); + print_key_value("num_queries", info->queries.load(), false); - json_out << double_quote << "query_time_percentiles" << ": {\n"; + json_out << "},\n"; - for (int percent = 0; percent <= 90; percent += 10) - print_percentile(percent); + json_out << double_quote << "query_time_percentiles" << ": {\n"; - print_percentile(95); - print_percentile(99); - print_percentile(99.9); - print_percentile(99.99, false); + for (int percent = 0; percent <= 90; percent += 10) + print_percentile(*info, percent); - json_out << "}\n"; + print_percentile(*info, 95); + print_percentile(*info, 99); + print_percentile(*info, 99.9); + print_percentile(*info, 99.99, false); + + json_out << "}\n"; + + if (i == infos.size()) + json_out << "}\n"; + else + json_out << "},\n"; + } json_out << "}\n"; } @@ -450,7 +509,11 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ("randomize,r", value()->default_value(false), "randomize order of execution") ("json", value()->default_value(""), "write final report to specified file in JSON format") ("host,h", value()->default_value("localhost"), "") + ("host1,h1", value()->default_value("localhost"), "Is used for comparison mode") + ("host2,h2", value()->default_value("localhost"), "Is used for comparison mode") ("port", value()->default_value(9000), "") + ("port1", value()->default_value(9000), "Is used for comparison mode") + ("port2", value()->default_value(9000), "Is used for comparison mode") ("secure,s", "Use TLS connection") ("user", value()->default_value("default"), "") ("password", value()->default_value(""), "") @@ -472,6 +535,28 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) return 1; } + std::vector ports; + std::vector hosts; + + if (options["port1"].as() != options["port2"].as() + || options["host1"].as() != options["host2"].as()) + { + ports.reserve(2); + hosts.reserve(2); + hosts.emplace_back(options["host1"].as()); + hosts.emplace_back(options["host2"].as()); + ports.emplace_back(options["port1"].as()); + ports.emplace_back(options["port2"].as()); + } + else + { + ports.reserve(1); + hosts.reserve(1); + hosts.emplace_back(options["host"].as()); + ports.emplace_back(options["port"].as()); + } + + print_stacktrace = options.count("stacktrace"); UseSSL use_ssl; @@ -479,8 +564,8 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) Benchmark benchmark( options["concurrency"].as(), options["delay"].as(), - options["host"].as(), - options["port"].as(), + hosts, + ports, options.count("secure"), options["database"].as(), options["user"].as(), From 3ea83e35d7365b7b7b973c53a22479e085a33d66 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Mon, 5 Aug 2019 21:38:08 +0300 Subject: [PATCH 059/357] Multiple hosts and ports --- dbms/programs/benchmark/Benchmark.cpp | 52 ++++++++------------------- 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 3d16974db4b..d004faf7057 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -51,7 +51,7 @@ class Benchmark : public Poco::Util::Application { public: Benchmark(unsigned concurrency_, double delay_, - const std::vector & hosts_, const std::vector & ports_, + const std::vector & hosts_, const std::vector & ports_, bool secure_, const String & default_database_, const String & user_, const String & password_, const String & stage, bool randomize_, size_t max_iterations_, double max_time_, @@ -62,14 +62,18 @@ public: json_path(json_path_), settings(settings_), global_context(Context::createGlobal()), pool(concurrency) { const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; + size_t connections_cnt = std::max(ports_.size(), hosts_.size()); - connections.reserve(ports_.size()); - comparison_info_total.reserve(ports_.size()); - comparison_info_per_interval.reserve(ports_.size()); + connections.reserve(connections_cnt); + comparison_info_total.reserve(connections_cnt); + comparison_info_per_interval.reserve(connections_cnt); - for (size_t i = 0; i < ports_.size(); ++i) + for (size_t i = 0; i < connections_cnt; ++i) { - connections.emplace_back(std::make_shared(concurrency, hosts_[i], ports_[i], default_database_, user_, password_, "benchmark", Protocol::Compression::Enable, secure)); + UInt16 cur_port = i >= ports_.size() ? 9000 : ports_[i]; + std::string cur_host = i >= hosts_.size() ? "localhost" : hosts_[i]; + + connections.emplace_back(std::make_shared(concurrency, cur_host, cur_port, default_database_, user_, password_, "benchmark", Protocol::Compression::Enable, secure)); comparison_info_per_interval.emplace_back(std::make_shared()); comparison_info_total.emplace_back(std::make_shared()); } @@ -452,7 +456,6 @@ private: print_key_value("num_queries", info->queries.load(), false); json_out << "},\n"; - json_out << double_quote << "query_time_percentiles" << ": {\n"; for (int percent = 0; percent <= 90; percent += 10) @@ -470,7 +473,6 @@ private: else json_out << "},\n"; } - json_out << "}\n"; } @@ -508,12 +510,8 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") ("randomize,r", value()->default_value(false), "randomize order of execution") ("json", value()->default_value(""), "write final report to specified file in JSON format") - ("host,h", value()->default_value("localhost"), "") - ("host1,h1", value()->default_value("localhost"), "Is used for comparison mode") - ("host2,h2", value()->default_value("localhost"), "Is used for comparison mode") - ("port", value()->default_value(9000), "") - ("port1", value()->default_value(9000), "Is used for comparison mode") - ("port2", value()->default_value(9000), "Is used for comparison mode") + ("host,h", value>()->default_value(std::vector{"localhost"}, "localhost") ,"note that more than one host can be described") + ("port,p", value>()->default_value(std::vector{9000}, "9000") ,"note that more than one port can be described") ("secure,s", "Use TLS connection") ("user", value()->default_value("default"), "") ("password", value()->default_value(""), "") @@ -535,28 +533,6 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) return 1; } - std::vector ports; - std::vector hosts; - - if (options["port1"].as() != options["port2"].as() - || options["host1"].as() != options["host2"].as()) - { - ports.reserve(2); - hosts.reserve(2); - hosts.emplace_back(options["host1"].as()); - hosts.emplace_back(options["host2"].as()); - ports.emplace_back(options["port1"].as()); - ports.emplace_back(options["port2"].as()); - } - else - { - ports.reserve(1); - hosts.reserve(1); - hosts.emplace_back(options["host"].as()); - ports.emplace_back(options["port"].as()); - } - - print_stacktrace = options.count("stacktrace"); UseSSL use_ssl; @@ -564,8 +540,8 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) Benchmark benchmark( options["concurrency"].as(), options["delay"].as(), - hosts, - ports, + options["host"].as>(), + options["port"].as>(), options.count("secure"), options["database"].as(), options["user"].as(), From a5009c675948b4c742d1413f194dbe5ca3ea3f03 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Tue, 6 Aug 2019 13:54:35 +0300 Subject: [PATCH 060/357] --cumulative option added, and time statistics fixed --- dbms/programs/benchmark/Benchmark.cpp | 67 ++++++++++++--------------- 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index d004faf7057..63f58ae391f 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -52,13 +52,13 @@ class Benchmark : public Poco::Util::Application public: Benchmark(unsigned concurrency_, double delay_, const std::vector & hosts_, const std::vector & ports_, - bool secure_, const String & default_database_, + bool cumulative_, bool secure_, const String & default_database_, const String & user_, const String & password_, const String & stage, bool randomize_, size_t max_iterations_, double max_time_, const String & json_path_, const Settings & settings_) : - concurrency(concurrency_), delay(delay_), queue(concurrency), - randomize(randomize_), max_iterations(max_iterations_), max_time(max_time_), + concurrency(concurrency_), delay(delay_), queue(concurrency), randomize(randomize_), + cumulative(cumulative_), max_iterations(max_iterations_), max_time(max_time_), json_path(json_path_), settings(settings_), global_context(Context::createGlobal()), pool(concurrency) { const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; @@ -135,6 +135,7 @@ private: ConnectionPoolPtrs connections; bool randomize; + bool cumulative; size_t max_iterations; double max_time; String json_path; @@ -149,12 +150,12 @@ private: struct Stats { - Stopwatch watch; std::atomic queries{0}; size_t read_rows = 0; size_t read_bytes = 0; size_t result_rows = 0; size_t result_bytes = 0; + double work_time = 0; using Sampler = ReservoirSampler; Sampler sampler {1 << 16}; @@ -162,6 +163,7 @@ private: void add(double seconds, size_t read_rows_inc, size_t read_bytes_inc, size_t result_rows_inc, size_t result_bytes_inc) { ++queries; + work_time += seconds; read_rows += read_rows_inc; read_bytes += read_bytes_inc; result_rows += result_rows_inc; @@ -171,8 +173,8 @@ private: void clear() { - watch.restart(); queries = 0; + work_time = 0; read_rows = 0; read_bytes = 0; result_rows = 0; @@ -251,7 +253,7 @@ private: if (delay > 0 && delay_watch.elapsedSeconds() > delay) { printNumberOfQueriesExecuted(queries_executed); - report(comparison_info_per_interval); + cumulative ? report(comparison_info_total) : report(comparison_info_per_interval); delay_watch.restart(); } } @@ -276,10 +278,6 @@ private: } InterruptListener interrupt_listener; - - for (auto & connection_stats : comparison_info_per_interval) - connection_stats->watch.restart(); - delay_watch.restart(); /// Push queries into queue @@ -297,9 +295,6 @@ private: pool.wait(); total_watch.stop(); - for (auto & connection_stats : comparison_info_total) - connection_stats->watch.stop(); - if (!json_path.empty()) reportJSON(comparison_info_total, json_path); @@ -383,15 +378,14 @@ private: if (0 == info->queries) return; - double seconds = info->watch.elapsedSeconds(); - std::cerr - << "connection: " << info_counter++ << ", " - << "QPS: " << (info->queries / seconds) << ", " - << "RPS: " << (info->read_rows / seconds) << ", " - << "MiB/s: " << (info->read_bytes / seconds / 1048576) << ", " - << "result RPS: " << (info->result_rows / seconds) << ", " - << "result MiB/s: " << (info->result_bytes / seconds / 1048576) << "." + << "connection " << info_counter++ << ", " + << "queries " << info->queries << ", " + << "QPS: " << (info->queries / info->work_time) << ", " + << "RPS: " << (info->read_rows / info->work_time) << ", " + << "MiB/s: " << (info->read_bytes / info->work_time / 1048576) << ", " + << "result RPS: " << (info->result_rows / info->work_time) << ", " + << "result MiB/s: " << (info->result_bytes / info->work_time / 1048576) << "." << "\n"; } std::cerr << "\n\t\t"; @@ -418,8 +412,9 @@ private: print_percentile(99.9); print_percentile(99.99); - for (auto & info : infos) - info->clear(); + if (!cumulative) + for (auto & info : infos) + info->clear(); } void reportJSON(MultiStats & infos, const std::string & filename) @@ -442,17 +437,16 @@ private: for (size_t i = 1; i <= infos.size(); ++i) { - auto info = infos[i]; + auto info = infos[i - 1]; json_out << double_quote << "connection_" + toString(i) << ": {\n"; json_out << double_quote << "statistics" << ": {\n"; - double seconds = info->watch.elapsedSeconds(); - print_key_value("QPS", info->queries / seconds); - print_key_value("RPS", info->read_rows / seconds); - print_key_value("MiBPS", info->read_bytes / seconds); - print_key_value("RPS_result", info->result_rows / seconds); - print_key_value("MiBPS_result", info->result_bytes / seconds); + print_key_value("QPS", info->queries / info->work_time); + print_key_value("RPS", info->read_rows / info->work_time); + print_key_value("MiBPS", info->read_bytes / info->work_time); + print_key_value("RPS_result", info->result_rows / info->work_time); + print_key_value("MiBPS_result", info->result_bytes / info->work_time); print_key_value("num_queries", info->queries.load(), false); json_out << "},\n"; @@ -467,12 +461,9 @@ private: print_percentile(*info, 99.99, false); json_out << "}\n"; - - if (i == infos.size()) - json_out << "}\n"; - else - json_out << "},\n"; + json_out << (i == infos.size() ? "}\n" : "},\n"); } + json_out << "}\n"; } @@ -510,8 +501,9 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") ("randomize,r", value()->default_value(false), "randomize order of execution") ("json", value()->default_value(""), "write final report to specified file in JSON format") - ("host,h", value>()->default_value(std::vector{"localhost"}, "localhost") ,"note that more than one host can be described") - ("port,p", value>()->default_value(std::vector{9000}, "9000") ,"note that more than one port can be described") + ("host,h", value>()->default_value(std::vector{"localhost"}, "localhost"), "note that more than one host can be described") + ("port,p", value>()->default_value(std::vector{9000}, "9000"), "note that more than one port can be described") + ("cumulative", "prints cumulative data instead of data per interval") ("secure,s", "Use TLS connection") ("user", value()->default_value("default"), "") ("password", value()->default_value(""), "") @@ -542,6 +534,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) options["delay"].as(), options["host"].as>(), options["port"].as>(), + options.count("cumulative"), options.count("secure"), options["database"].as(), options["user"].as(), From 8e535a9cb0fe66b44de1be0f5537827aad8d7767 Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Mon, 8 Jul 2019 12:41:28 +0300 Subject: [PATCH 061/357] Implement nextInBlock function --- dbms/src/Functions/nextInBlock.cpp | 159 ++++++++++++++++++ .../registerFunctionsMiscellaneous.cpp | 2 + .../0_stateless/00957_next_in_block.reference | 12 ++ .../0_stateless/00957_next_in_block.sql | 22 +++ 4 files changed, 195 insertions(+) create mode 100644 dbms/src/Functions/nextInBlock.cpp create mode 100644 dbms/tests/queries/0_stateless/00957_next_in_block.reference create mode 100644 dbms/tests/queries/0_stateless/00957_next_in_block.sql diff --git a/dbms/src/Functions/nextInBlock.cpp b/dbms/src/Functions/nextInBlock.cpp new file mode 100644 index 00000000000..e672e539f25 --- /dev/null +++ b/dbms/src/Functions/nextInBlock.cpp @@ -0,0 +1,159 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +// Implements function, giving value for column in next row +// Example: +// | c1 | +// | 10 | +// | 20 | +// SELECT c1, nextInBlock(c1, 1) as c2: +// | c1 | c2 | +// | 10 | 20 | +// | 20 | 0 | +class FunctionNextInBlock : public IFunction +{ +public: + static constexpr auto name = "nextInBlock"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + /// Get the name of the function. + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isVariadic() const override { return true; } + + bool isDeterministic() const override { return false; } + + bool isDeterministicInScopeOfQuery() const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + size_t number_of_arguments = arguments.size(); + + if (number_of_arguments < 1 || number_of_arguments > 3) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(number_of_arguments) + + ", should be from 1 to 3", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + // second argument must be a positive, constant column + if (number_of_arguments == 2 && !isUnsignedInteger(arguments[1])) + throw Exception( + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + + " - should be positive integer", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + // check that default value has supertype with first argument + if (number_of_arguments == 3) + { + DataTypes types = {arguments[0], arguments[2]}; + try + { + return getLeastSupertype(types); + } + catch (const Exception &) + { + throw Exception( + "Illegal types of arguments (" + types[0]->getName() + ", " + types[1]->getName() + + ")" + " of function " + + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + + return arguments[0]; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + size_t offset_value = 1; + + if (arguments.size() > 1) + { + auto offset_column = block.getByPosition(arguments[1]); + if (!isColumnConst(*offset_column.column)) + throw Exception("Second argument of function " + getName() + " should be constant", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + Field offset_field = (*block.getByPosition(arguments[1]).column)[0]; + auto raw_value = safeGet(offset_field); + + if (raw_value == 0) + throw Exception( + "Second argument of function " + getName() + " should be positive integer, " + toString(raw_value) + " given", + ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + offset_value = raw_value; + } + + auto has_column_for_missing = arguments.size() == 3; + + DataTypes types = {block.getByPosition(arguments[0]).type}; + if (has_column_for_missing) + { + types.push_back(block.getByPosition(arguments[2]).type); + } + const DataTypePtr & result_type = getLeastSupertype(types); + + auto column = result_type->createColumn(); + column->reserve(input_rows_count); + + auto source_column = block.getByPosition(arguments[0]).column; + + for (size_t i = offset_value; i < input_rows_count; i++) + { + column->insertFrom(*source_column, i); + } + + if (has_column_for_missing) + { + auto default_values_column = block.getByPosition(arguments[2]).column; + size_t starting_pos = offset_value > input_rows_count ? 0 : input_rows_count - offset_value; + if (isColumnConst(*default_values_column)) + { + Field constant_value = (*default_values_column)[0]; + for (size_t i = starting_pos; i < input_rows_count; i++) + { + column->insert(constant_value); + } + } + else + { + for (size_t i = starting_pos; i < input_rows_count; i++) + { + column->insertFrom(*default_values_column, i); + } + } + } + else + { + for (size_t i = 0; i < std::min(offset_value, input_rows_count); i++) + { + column->insertDefault(); + } + } + + block.getByPosition(result).column = std::move(column); + } +}; + +void registerFunctionNextInBlock(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} \ No newline at end of file diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp index 6d201d65bd3..57ccfcd11c9 100644 --- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp @@ -17,6 +17,7 @@ void registerFunctionBlockSize(FunctionFactory &); void registerFunctionBlockNumber(FunctionFactory &); void registerFunctionRowNumberInBlock(FunctionFactory &); void registerFunctionRowNumberInAllBlocks(FunctionFactory &); +void registerFunctionNextInBlock(FunctionFactory &); void registerFunctionSleep(FunctionFactory &); void registerFunctionSleepEachRow(FunctionFactory &); void registerFunctionMaterialize(FunctionFactory &); @@ -67,6 +68,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionBlockNumber(factory); registerFunctionRowNumberInBlock(factory); registerFunctionRowNumberInAllBlocks(factory); + registerFunctionNextInBlock(factory); registerFunctionSleep(factory); registerFunctionSleepEachRow(factory); registerFunctionMaterialize(factory); diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.reference b/dbms/tests/queries/0_stateless/00957_next_in_block.reference new file mode 100644 index 00000000000..860ce6dc1ba --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_next_in_block.reference @@ -0,0 +1,12 @@ +0 1 +1 0 +0 2 +1 0 +2 0 +0 0 +1 0 +0 2 +1 3 +2 4 +3 1000 +4 1000 diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.sql b/dbms/tests/queries/0_stateless/00957_next_in_block.sql new file mode 100644 index 00000000000..7cbd932cf1a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_next_in_block.sql @@ -0,0 +1,22 @@ +-- no arguments +select nextInBlock(); -- { serverError 42 } +-- greater than 3 arguments +select nextInBlock(1,2,3,4); -- { serverError 42 } +-- zero offset value +select nextInBlock(dummy, 0); -- { serverError 69 } +-- negative offset value +select nextInBlock(dummy, -1); -- { serverError 43 } +-- non-constant offset value +select nextInBlock(dummy, dummy); -- { serverError 43 } +-- bad default value +select nextInBlock(dummy, 1, 'hello'); -- { serverError 43 } +-- single argument test +select number, nextInBlock(number) from numbers(2); +-- filling by column's default value +select number, nextInBlock(number, 2) from numbers(3); +-- offset is greater that block - should fill everything with defaults +select number, nextInBlock(number, 5) from numbers(2); +-- substitution by constant for missing values +select number, nextInBlock(number, 2, 1000) from numbers(5); +-- substitution by expression +-- select number, nextInBlock(number, 2, number % 2) from numbers(5); \ No newline at end of file From cfec857f2c5685aafc1aeaa061497baf9ab39c53 Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Mon, 8 Jul 2019 17:53:02 +0300 Subject: [PATCH 062/357] Add trailing newline --- dbms/src/Functions/nextInBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Functions/nextInBlock.cpp b/dbms/src/Functions/nextInBlock.cpp index e672e539f25..eeb33e28146 100644 --- a/dbms/src/Functions/nextInBlock.cpp +++ b/dbms/src/Functions/nextInBlock.cpp @@ -156,4 +156,4 @@ void registerFunctionNextInBlock(FunctionFactory & factory) factory.registerFunction(); } -} \ No newline at end of file +} From e12cb8da6df1efd5b21d4e4cd1207ed8e2a55185 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Aug 2019 15:52:08 +0300 Subject: [PATCH 063/357] Add some changes --- dbms/src/Core/SettingsCommon.h | 60 +++++++----------- dbms/src/Storages/AlterCommands.cpp | 34 ++++++++++ dbms/src/Storages/AlterCommands.h | 7 +++ .../00980_merge_alter_settings.reference | 2 + .../00980_merge_alter_settings.sql | 39 ++++++++++++ ...keeper_merge_tree_alter_settings.reference | 10 +++ ...80_zookeeper_merge_tree_alter_settings.sql | 62 +++++++++++++++++++ 7 files changed, 177 insertions(+), 37 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference create mode 100644 dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql create mode 100644 dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference create mode 100644 dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index e9dce9c9dcc..618fc00aa75 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -321,7 +321,7 @@ private: size_t offset_of_changed; StringRef name; StringRef description; - bool immutable; + bool updateable; GetStringFunction get_string; GetFieldFunction get_field; SetStringFunction set_string; @@ -401,7 +401,7 @@ public: const_reference(const const_reference & src) = default; const StringRef & getName() const { return member->name; } const StringRef & getDescription() const { return member->description; } - bool isImmutable() const { return member->immutable; } + bool isUpdateable() const { return member->updateable; } bool isChanged() const { return member->isChanged(*collection); } Field getValue() const { return member->get_field(*collection); } String getValueAsString() const { return member->get_string(*collection); } @@ -421,6 +421,18 @@ public: reference(const const_reference & src) : const_reference(src) {} void setValue(const Field & value) { this->member->set_field(*const_cast(this->collection), value); } void setValue(const String & value) { this->member->set_string(*const_cast(this->collection), value); } + void updateValue(const Field & value) + { + if (!this->member->updateable) + throw Exception("Setting '" + this->member->name.toString() + "' is restricted for updates.", ErrorCodes::IMMUTABLE_SETTING); + setValue(value); + } + void updateValue(const String & value) + { + if (!this->member->updateable) + throw Exception("Setting '" + this->member->name.toString() + "' is restricted for updates.", ErrorCodes::IMMUTABLE_SETTING); + setValue(value); + } }; /// Iterator to iterating through all the settings. @@ -503,39 +515,14 @@ public: void set(size_t index, const String & value) { (*this)[index].setValue(value); } void set(const String & name, const String & value) { (*this)[name].setValue(value); } - /// Updates setting's value. - void update(size_t index, const Field & value) - { - auto ref = (*this)[index]; - if (ref.isImmutable()) - throw Exception("Cannot modify immutable setting '" + ref.getName().toString() + "'", ErrorCodes::IMMUTABLE_SETTING); - ref.setValue(value); - } + /// Updates setting's value. Checks it' mutability. + void update(size_t index, const Field & value) { (*this)[index].updateValue(value); } - void update(const String & name, const Field & value) - { - auto ref = (*this)[name]; - if (ref.isImmutable()) - throw Exception("Cannot modify immutable setting '" + ref.getName().toString() + "'", ErrorCodes::IMMUTABLE_SETTING); - ref.setValue(value); - } + void update(const String & name, const Field & value) { (*this)[name].updateValue(value); } - /// Updates setting's value. Read value in text form from string (for example, from configuration file or from URL parameter). - void update(size_t index, const String & value) - { - auto ref = (*this)[index]; - if (ref.isImmutable()) - throw Exception("Cannot modify immutable setting '" + ref.getName().toString() + "'", ErrorCodes::IMMUTABLE_SETTING); - (*this)[index].setValue(value); - } + void update(size_t index, const String & value) { (*this)[index].updateValue(value); } - void update(const String & name, const String & value) - { - auto ref = (*this)[name]; - if (ref.isImmutable()) - throw Exception("Cannot modify immutable setting '" + ref.getName().toString() + "'", ErrorCodes::IMMUTABLE_SETTING); - (*this)[name].setValue(value); - } + void update(const String & name, const String & value) { (*this)[name].updateValue(value); } /// Returns value of a setting. Field get(size_t index) const { return (*this)[index].getValue(); } @@ -600,7 +587,7 @@ public: return found_changes; } - /// Applies changes to the settings. + /// Applies changes to the settings. Doesn't check settings mutability. void loadFromChange(const SettingChange & change) { set(change.name, change.value); @@ -612,7 +599,7 @@ public: loadFromChange(change); } - /// Applies changes to the settings. + /// Applies changes to the settings, checks settings mutability void updateFromChange(const SettingChange & change) { update(change.name, change.value); @@ -701,7 +688,7 @@ public: #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ static_assert(std::is_same_v().NAME.changed), bool>); \ add({offsetof(Derived, NAME.changed), \ - StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ + StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ @@ -710,10 +697,9 @@ public: #define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ static_assert(std::is_same_v().NAME.changed), bool>); \ add({offsetof(Derived, NAME.changed), \ - StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ + StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ &Functions::NAME##_castValueWithoutApplying }); - } diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 789b66b271a..16d6f0a0aa0 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -332,6 +332,11 @@ bool AlterCommand::isMutable() const return true; } +bool AlterCommand::isSettingsAlter() const +{ + return type == MODIFY_SETTING; +} + void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast, SettingsChanges & changes) const { @@ -567,6 +572,31 @@ void AlterCommands::applyForColumnsOnly(ColumnsDescription & columns_description columns_description = std::move(out_columns_description); } + +void AlterCommands::applyForSettingsOnly(SettingsChanges & changes) const +{ + ColumnsDescription out_columns_description; + IndicesDescription indices_description; + ASTPtr out_order_by; + ASTPtr out_primary_key; + ASTPtr out_ttl_table; + SettingsChanges out_changes; + apply(out_columns_description, indices_description, out_order_by, out_primary_key, out_ttl_table, out_changes); + + if (out_columns_description.begin() != out_columns_description.end()) + throw Exception("Alter modifying columns, but only settings change applied.", ErrorCodes::LOGICAL_ERROR); + if (out_order_by) + throw Exception("Alter modifying ORDER BY expression, but only settings change applied.", ErrorCodes::LOGICAL_ERROR); + if (out_primary_key) + throw Exception("Alter modifying PRIMARY KEY expression, but only settings change applied.", ErrorCodes::LOGICAL_ERROR); + if (!indices_description.indices.empty()) + throw Exception("Alter modifying indices, but only settings change applied.", ErrorCodes::NOT_IMPLEMENTED); + if (out_ttl_table) + throw Exception("Alter modifying TTL, but only settings change applied.", ErrorCodes::NOT_IMPLEMENTED); + + changes = std::move(out_changes); +} + bool AlterCommands::isMutable() const { for (const auto & param : *this) @@ -578,4 +608,8 @@ bool AlterCommands::isMutable() const return false; } +bool AlterCommands::isSettingsAlter() const +{ + return std::all_of(begin(), end(), [](const AlterCommand & c) { return c.isSettingsAlter(); }); +} } diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index fe05d014cba..9b59514020f 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -93,6 +93,9 @@ struct AlterCommand /// Checks that not only metadata touched by that command bool isMutable() const; + + /// checks that only settings changed by alter + bool isSettingsAlter() const; }; class Context; @@ -107,8 +110,12 @@ public: /// Used for primitive table engines, where only columns metadata can be changed void applyForColumnsOnly(ColumnsDescription & columns_description) const; + /// Apply alter commands only for settings. Exception will be thrown if any other part of table structure will be modified. + void applyForSettingsOnly(SettingsChanges & changes) const; + void validate(const IStorage & table, const Context & context); bool isMutable() const; + bool isSettingsAlter() const; }; } diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference new file mode 100644 index 00000000000..ad64346f90d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference @@ -0,0 +1,2 @@ +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096 +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql new file mode 100644 index 00000000000..52f5e4dd444 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql @@ -0,0 +1,39 @@ +DROP TABLE IF EXISTS log_for_alter; + +CREATE TABLE log_for_alter ( + id UInt64, + Data String +) ENGINE = Log(); + +ALTER TABLE log_for_alter MODIFY SETTING aaa=123; -- { serverError 468 } + +DROP TABLE IF EXISTS log_for_alter; + +DROP TABLE IF EXISTS table_for_alter; + +CREATE TABLE table_for_alter ( + id UInt64, + Data String +) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096; + +ALTER TABLE table_for_alter MODIFY SETTING index_granularity=555; -- { serverError 469 } + +SHOW CREATE TABLE table_for_alter; + +ALTER TABLE table_for_alter MODIFY SETTING parts_to_throw_insert = 1, parts_to_delay_insert = 1; + +SHOW CREATE TABLE table_for_alter; + +INSERT INTO table_for_alter VALUES (1, '1'); +INSERT INTO table_for_alter VALUES (2, '2'); -- { serverError 252 } + +DETACH TABLE table_for_alter; + +ATTACH TABLE table_for_alter; + +INSERT INTO table_for_alter VALUES (2, '2'); -- { serverError 252 } + +ALTER TABLE table_for_alter MODIFY SETTING xxx_yyy=124; -- { serverError 115 } + +DROP TABLE IF EXISTS table_for_alter; + diff --git a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference new file mode 100644 index 00000000000..e55bfadd538 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference @@ -0,0 +1,10 @@ +CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192 +4 +4 +4 +4 +6 +6 +CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 +CREATE TABLE default.replicated_table_for_alter2 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\') ORDER BY id SETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 diff --git a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql new file mode 100644 index 00000000000..9c7a7c33329 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql @@ -0,0 +1,62 @@ +DROP TABLE IF EXISTS replicated_table_for_alter1; +DROP TABLE IF EXISTS replicated_table_for_alter2; + +CREATE TABLE replicated_table_for_alter1 ( + id UInt64, + Data String +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/replicated_table_for_alter', '1') ORDER BY id; + +CREATE TABLE replicated_table_for_alter2 ( + id UInt64, + Data String +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/replicated_table_for_alter', '2') ORDER BY id; + +SHOW CREATE TABLE replicated_table_for_alter1; + +ALTER TABLE replicated_table_for_alter1 MODIFY SETTING index_granularity = 4096; -- { serverError 469 } + +SHOW CREATE TABLE replicated_table_for_alter1; + +INSERT INTO replicated_table_for_alter2 VALUES (1, '1'), (2, '2'); + +SYSTEM SYNC REPLICA replicated_table_for_alter1; + +ALTER TABLE replicated_table_for_alter1 MODIFY SETTING use_minimalistic_part_header_in_zookeeper = 1; + +INSERT INTO replicated_table_for_alter1 VALUES (3, '3'), (4, '4'); + +SYSTEM SYNC REPLICA replicated_table_for_alter2; + +SELECT COUNT() FROM replicated_table_for_alter1; +SELECT COUNT() FROM replicated_table_for_alter2; + +DETACH TABLE replicated_table_for_alter2; +ATTACH TABLE replicated_table_for_alter2; + +DETACH TABLE replicated_table_for_alter1; +ATTACH TABLE replicated_table_for_alter1; + +SELECT COUNT() FROM replicated_table_for_alter1; +SELECT COUNT() FROM replicated_table_for_alter2; + +ALTER TABLE replicated_table_for_alter2 MODIFY SETTING parts_to_throw_insert = 1, parts_to_delay_insert = 1; +INSERT INTO replicated_table_for_alter2 VALUES (3, '1'), (4, '2'); -- { serverError 252 } + +INSERT INTO replicated_table_for_alter1 VALUES (5, '5'), (6, '6'); + +SYSTEM SYNC REPLICA replicated_table_for_alter2; + +SELECT COUNT() FROM replicated_table_for_alter1; +SELECT COUNT() FROM replicated_table_for_alter2; + +DETACH TABLE replicated_table_for_alter2; +ATTACH TABLE replicated_table_for_alter2; + +DETACH TABLE replicated_table_for_alter1; +ATTACH TABLE replicated_table_for_alter1; + +SHOW CREATE TABLE replicated_table_for_alter1; +SHOW CREATE TABLE replicated_table_for_alter2; + +DROP TABLE IF EXISTS replicated_table_for_alter2; +DROP TABLE IF EXISTS replicated_table_for_alter1; From 02878c233a130d0a815da597a7747ed5c4080e6b Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Tue, 6 Aug 2019 15:54:33 +0300 Subject: [PATCH 064/357] True randomness --- dbms/programs/benchmark/Benchmark.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 63f58ae391f..61acc95ad9e 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -341,7 +341,10 @@ private: void execute(EntryPtrs & connection_entries, Query & query) { - size_t connection_index = rand() % connection_entries.size(); + /// Randomly choosing connection index + pcg64 generator(randomSeed()); + std::uniform_int_distribution distribution(0, connection_entries.size() - 1); + size_t connection_index = distribution(generator); Stopwatch watch; RemoteBlockInputStream stream( From 84fd4906cac21b2b63ac89242fba707b4d59eefb Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Aug 2019 16:04:29 +0300 Subject: [PATCH 065/357] Changes for engines --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 24 +++++++++++++++++++ dbms/src/Storages/MergeTree/MergeTreeData.h | 8 +++++++ dbms/src/Storages/StorageMergeTree.cpp | 15 ++++++------ dbms/src/Storages/StorageMergeTree.h | 2 -- .../Storages/StorageReplicatedMergeTree.cpp | 12 +++++++++- 5 files changed, 51 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index a99b6fbabeb..70c6c409c30 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -1583,6 +1584,29 @@ void MergeTreeData::alterDataPart( return; } +void MergeTreeData::alterSettings( + const SettingsChanges & new_changes, + const String & current_database_name, + const String & current_table_name, + const Context & context) +{ + settings.updateFromChanges(new_changes); + IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) + { + if (!new_changes.empty()) + { + auto & storage_ast = ast.as(); + storage_ast.settings->changes.insert(storage_ast.settings->changes.end(), new_changes.begin(), new_changes.end()); + } + }; + context.getDatabase(current_database_name)->alterTable(context, current_table_name, getColumns(), getIndices(), storage_modifier); +} + +bool MergeTreeData::hasSetting(const String & setting_name) const +{ + return settings.findIndex(setting_name) != MergeTreeSettings::npos; +} + void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part) { auto & empty_columns = data_part->empty_columns; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index cd95e5c7e58..d11074e1156 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -509,6 +509,14 @@ public: bool skip_sanity_checks, AlterDataPartTransactionPtr& transaction); + void alterSettings( + const SettingsChanges & new_changes, + const String & current_database_name, + const String & current_table_name, + const Context & context); + + bool hasSetting(const String & setting_name) const override; + /// Remove columns, that have been markedd as empty after zeroing values with expired ttl void removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index e823136e3bb..438b9468b69 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -246,12 +246,19 @@ void StorageMergeTree::alter( if (!params.isMutable()) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + SettingsChanges new_changes; + if (params.isSettingsAlter()) + { + params.applyForSettingsOnly(new_changes); + alterSettings(new_changes, current_database_name, current_table_name, context); + return; + } + auto new_columns = getColumns(); auto new_indices = getIndices(); ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - SettingsChanges new_changes; params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes); IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) { @@ -1031,12 +1038,6 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } - -bool StorageMergeTree::hasSetting(const String & setting_name) const -{ - return settings.findIndex(setting_name) != MergeTreeSettings::npos; -} - void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) { // TODO: should get some locks to prevent race with 'alter … modify column' diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index e2d24ceecdd..0df90604f67 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -128,8 +128,6 @@ private: friend class MergeTreeData; friend struct CurrentlyMergingPartsTagger; - bool hasSetting(const String & setting_name) const override; - protected: /** Attach the table with the appropriate name, along the appropriate path (with / at the end), diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 365f7e70143..23c49b05c1a 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3072,13 +3072,23 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p void StorageReplicatedMergeTree::alter( - const AlterCommands & params, const String & /*database_name*/, const String & /*table_name*/, + const AlterCommands & params, const String & current_database_name, const String & current_table_name, const Context & query_context, TableStructureWriteLockHolder & table_lock_holder) { assertNotReadonly(); LOG_DEBUG(log, "Doing ALTER"); + if (params.isSettingsAlter()) + { + LOG_DEBUG(log, "ALTER settings only"); + lockStructureExclusively(table_lock_holder, query_context.getCurrentQueryId()); + SettingsChanges new_changes; + params.applyForSettingsOnly(new_changes); + alterSettings(new_changes, current_database_name, current_table_name, query_context); + return; + } + /// Alter is done by modifying the metadata nodes in ZK that are shared between all replicas /// (/columns, /metadata). We set contents of the shared nodes to the new values and wait while /// replicas asynchronously apply changes (see ReplicatedMergeTreeAlterThread.cpp) and modify From 303c4e5a58347969da67efccfc7890e33619176d Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Aug 2019 16:06:09 +0300 Subject: [PATCH 066/357] Make index_granularity_bytes immutable --- dbms/src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index c982bef324f..3012b9e1694 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -77,7 +77,7 @@ struct MergeTreeSettings : public SettingsCollection M(SettingBool, use_minimalistic_part_header_in_zookeeper, false, "Store part header (checksums and columns) in a compact format and a single part znode instead of separate znodes (/columns and /checksums). This can dramatically reduce snapshot size in ZooKeeper. Before enabling check that all replicas support new format.") \ M(SettingUInt64, finished_mutations_to_keep, 100, "How many records about mutations that are done to keep. If zero, then keep all of them.") \ M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024, "Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled).") \ - M(SettingUInt64, index_granularity_bytes, 10 * 1024 * 1024, "Approximate amount of bytes in single granule (0 - disabled).") \ + IM(SettingUInt64, index_granularity_bytes, 10 * 1024 * 1024, "Approximate amount of bytes in single granule (0 - disabled).") \ M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "Minimal time in seconds, when merge with TTL can be repeated.") \ M(SettingBool, write_final_mark, 1, "Write final mark after end of column (0 - disabled, do nothing if index_granularity_bytes=0)") \ M(SettingBool, enable_mixed_granularity_parts, 0, "Enable parts with adaptive and non adaptive granularity") From 4f89f8f4dd8b05ee345abe45d9f198062c8aa5d5 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Tue, 6 Aug 2019 16:19:06 +0300 Subject: [PATCH 067/357] random generator is now one for thread --- dbms/programs/benchmark/Benchmark.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 61acc95ad9e..923745eb204 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -307,6 +307,10 @@ private: { Query query; + /// Randomly choosing connection index + pcg64 generator(randomSeed()); + std::uniform_int_distribution distribution(0, connection_entries.size() - 1); + try { /// In these threads we do not accept INT signal. @@ -327,7 +331,7 @@ private: if (shutdown || (max_iterations && queries_executed == max_iterations)) return; } - execute(connection_entries, query); + execute(connection_entries, query, distribution(generator)); ++queries_executed; } } @@ -339,13 +343,8 @@ private: } } - void execute(EntryPtrs & connection_entries, Query & query) + void execute(EntryPtrs & connection_entries, Query & query, size_t connection_index) { - /// Randomly choosing connection index - pcg64 generator(randomSeed()); - std::uniform_int_distribution distribution(0, connection_entries.size() - 1); - size_t connection_index = distribution(generator); - Stopwatch watch; RemoteBlockInputStream stream( *(*connection_entries[connection_index]), From ca29343f546122340b934ae3b340770e35011e3d Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Aug 2019 17:09:36 +0300 Subject: [PATCH 068/357] Uniq settings --- dbms/src/Core/SettingsCommon.h | 2 +- dbms/src/Storages/Kafka/KafkaSettings.h | 20 +++++++++---------- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 12 +++++++++-- dbms/src/Storages/MergeTree/MergeTreeData.h | 1 + .../Storages/MergeTree/MergeTreeSettings.cpp | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 10 +--------- .../00980_merge_alter_settings.reference | 3 +++ .../00980_merge_alter_settings.sql | 12 +++++++++++ 8 files changed, 39 insertions(+), 23 deletions(-) diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index 618fc00aa75..c079b875b9d 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -599,7 +599,7 @@ public: loadFromChange(change); } - /// Applies changes to the settings, checks settings mutability + /// Applies changes to the settings, checks settings mutability. void updateFromChange(const SettingChange & change) { update(change.name, change.value); diff --git a/dbms/src/Storages/Kafka/KafkaSettings.h b/dbms/src/Storages/Kafka/KafkaSettings.h index 366ec715bf0..fd8d9b3dc08 100644 --- a/dbms/src/Storages/Kafka/KafkaSettings.h +++ b/dbms/src/Storages/Kafka/KafkaSettings.h @@ -15,16 +15,16 @@ struct KafkaSettings : public SettingsCollection { #define LIST_OF_KAFKA_SETTINGS(M, IM) \ - IM(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ - IM(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ - IM(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \ - IM(SettingString, kafka_format, "", "The message format for Kafka engine.") \ - IM(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.") \ - IM(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine") \ - IM(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.") \ - IM(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \ - IM(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block") \ - IM(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block") + M(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ + M(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ + M(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \ + M(SettingString, kafka_format, "", "The message format for Kafka engine.") \ + M(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.") \ + M(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine") \ + M(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.") \ + M(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \ + M(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block") \ + M(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block") DECLARE_SETTINGS_COLLECTION(LIST_OF_KAFKA_SETTINGS) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 70c6c409c30..7c65b33d830 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1595,8 +1595,16 @@ void MergeTreeData::alterSettings( { if (!new_changes.empty()) { - auto & storage_ast = ast.as(); - storage_ast.settings->changes.insert(storage_ast.settings->changes.end(), new_changes.begin(), new_changes.end()); + auto & storage_changes = ast.as().settings->changes; + /// Make storage settings unique + for (const auto & change : new_changes) + { + auto finder = [&change] (const SettingChange & c) { return c.name == change.name;}; + if (auto it = std::find_if(storage_changes.begin(), storage_changes.end(), finder); it != storage_changes.end()) + it->value = change.value; + else + storage_changes.push_back(change); + } } }; context.getDatabase(current_database_name)->alterTable(context, current_table_name, getColumns(), getIndices(), storage_modifier); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index d11074e1156..f681baa4488 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -509,6 +509,7 @@ public: bool skip_sanity_checks, AlterDataPartTransactionPtr& transaction); + /// Performs ALTER of table settings void alterSettings( const SettingsChanges & new_changes, const String & current_database_name, diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index 562f2d8f402..9eee33554ab 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -67,7 +67,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) #define ADD_IF_ABSENT(NAME) \ if (std::find_if(changes.begin(), changes.end(), \ - [](const SettingChange & c) { return c.name == #NAME; }) \ + [](const SettingChange & c) { return c.name == #NAME; }) \ == changes.end()) \ changes.push_back(SettingChange{#NAME, NAME.value}); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 438b9468b69..9fe1458f5a7 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -260,16 +260,8 @@ void StorageMergeTree::alter( ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes); - IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) - { - auto & storage_ast = ast.as(); - if (!new_changes.empty()) - storage_ast.settings->changes.insert(storage_ast.settings->changes.end(), new_changes.begin(), new_changes.end()); - }; - - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, storage_modifier); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); setColumns(std::move(new_columns)); - settings.updateFromChanges(new_changes); return; } diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference index ad64346f90d..c7f912ddc79 100644 --- a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference @@ -1,2 +1,5 @@ CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096 CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100 +2 +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql index 52f5e4dd444..cb1ba315001 100644 --- a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql @@ -35,5 +35,17 @@ INSERT INTO table_for_alter VALUES (2, '2'); -- { serverError 252 } ALTER TABLE table_for_alter MODIFY SETTING xxx_yyy=124; -- { serverError 115 } +ALTER TABLE table_for_alter MODIFY SETTING parts_to_throw_insert = 100, parts_to_delay_insert = 100; + +INSERT INTO table_for_alter VALUES (2, '2'); + +SHOW CREATE TABLE table_for_alter; + +SELECT COUNT() FROM table_for_alter; + +ALTER TABLE table_for_alter MODIFY SETTING check_delay_period=10, check_delay_period=20, check_delay_period=30; + +SHOW CREATE TABLE table_for_alter; + DROP TABLE IF EXISTS table_for_alter; From e62101b8e86665956e605a934e7981b3e7550a63 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Aug 2019 19:29:31 +0300 Subject: [PATCH 069/357] Better comment --- dbms/src/Core/SettingsCommon.h | 8 ++++++-- dbms/src/Storages/IStorage.h | 3 +-- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- dbms/src/Storages/MergeTree/MergeTreeData.h | 7 +++++-- dbms/src/Storages/StorageMergeTree.cpp | 5 +++-- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 5 +++-- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index c079b875b9d..4b3f7020a6e 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -587,24 +587,28 @@ public: return found_changes; } - /// Applies changes to the settings. Doesn't check settings mutability. + /// Applies change to the settings. Doesn't check settings mutability. void loadFromChange(const SettingChange & change) { set(change.name, change.value); } + /// Applies changes to the settings. Should be used in initial settings loading. + /// (on table creation or loading from config) void loadFromChanges(const SettingsChanges & changes) { for (const SettingChange & change : changes) loadFromChange(change); } - /// Applies changes to the settings, checks settings mutability. + /// Applies change to the settings, checks settings mutability. void updateFromChange(const SettingChange & change) { update(change.name, change.value); } + /// Applies changes to the settings. Should be used for settigns update. + /// (ALTER MODIFY SETTINGS) void updateFromChanges(const SettingsChanges & changes) { for (const SettingChange & change : changes) diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 4babb7ded3e..79760bae5a8 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -130,7 +129,7 @@ public: /// thread-unsafe part. lockStructure must be acquired /// If |need_all| is set, then checks that all the columns of the table are in the block. void check(const Block & block, bool need_all = false) const; - /// Check storage has setting + /// Check storage has setting. Exception will be thrown if it doesn't support settings at all. virtual bool hasSetting(const String & setting_name) const; protected: /// still thread-unsafe part. diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 7c65b33d830..4f7452bd79f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1269,7 +1269,6 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c { if (!hasSetting(setting.name)) throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting.name + "'", ErrorCodes::UNKNOWN_SETTING}; - } /// Check that type conversions are possible. @@ -1588,7 +1587,8 @@ void MergeTreeData::alterSettings( const SettingsChanges & new_changes, const String & current_database_name, const String & current_table_name, - const Context & context) + const Context & context, + TableStructureWriteLockHolder & /* table_lock_holder */) { settings.updateFromChanges(new_changes); IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index f681baa4488..78c7df3bb48 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -509,13 +509,16 @@ public: bool skip_sanity_checks, AlterDataPartTransactionPtr& transaction); - /// Performs ALTER of table settings + /// Performs ALTER of table settings (MergeTreeSettings). Lightweight operation, affects metadata only. + /// Not atomic, have to be done with alter intention lock. void alterSettings( const SettingsChanges & new_changes, const String & current_database_name, const String & current_table_name, - const Context & context); + const Context & context, + TableStructureWriteLockHolder & table_lock_holder); + /// All MergeTreeData children have settings. bool hasSetting(const String & setting_name) const override; /// Remove columns, that have been markedd as empty after zeroing values with expired ttl diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 9fe1458f5a7..bebc6619509 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -245,15 +245,16 @@ void StorageMergeTree::alter( { if (!params.isMutable()) { - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); SettingsChanges new_changes; + /// We don't need to lock table structure exclusively to ALTER settings. if (params.isSettingsAlter()) { params.applyForSettingsOnly(new_changes); - alterSettings(new_changes, current_database_name, current_table_name, context); + alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); return; } + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); ASTPtr new_order_by_ast = order_by_ast; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 23c49b05c1a..e11ea304c61 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3081,11 +3081,12 @@ void StorageReplicatedMergeTree::alter( if (params.isSettingsAlter()) { + /// We don't replicate settings ALTER. It's local operation. + /// Also we don't upgrade alter lock to table structure lock. LOG_DEBUG(log, "ALTER settings only"); - lockStructureExclusively(table_lock_holder, query_context.getCurrentQueryId()); SettingsChanges new_changes; params.applyForSettingsOnly(new_changes); - alterSettings(new_changes, current_database_name, current_table_name, query_context); + alterSettings(new_changes, current_database_name, current_table_name, query_context, table_lock_holder); return; } From bca40a860890fb1dc94dc0f0aafd6aa5591b09f4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Aug 2019 19:34:27 +0300 Subject: [PATCH 070/357] Better name --- dbms/src/Storages/StorageMergeTree.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index bebc6619509..8f3897ba128 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -864,9 +864,9 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi ASTPtr ignored_order_by_ast; ASTPtr ignored_primary_key_ast; ASTPtr ignored_ttl_table_ast; - SettingsChanges ignore_settings_changes; + SettingsChanges ignored_settings_changes; - alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast, ignore_settings_changes); + alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast, ignored_settings_changes); auto columns_for_parts = new_columns.getAllPhysical(); for (const auto & part : parts) From 517730900157e84f02773165db9dbb66eea95074 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 7 Aug 2019 16:35:28 +0300 Subject: [PATCH 071/357] Add test for race condition --- .../00980_alter_settings_race.reference | 1 + .../0_stateless/00980_alter_settings_race.sh | 32 +++++++++++++++++++ .../00980_merge_alter_settings.sql | 2 +- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/00980_alter_settings_race.reference create mode 100755 dbms/tests/queries/0_stateless/00980_alter_settings_race.sh diff --git a/dbms/tests/queries/0_stateless/00980_alter_settings_race.reference b/dbms/tests/queries/0_stateless/00980_alter_settings_race.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_alter_settings_race.reference @@ -0,0 +1 @@ +1 diff --git a/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh b/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh new file mode 100755 index 00000000000..4a948841ed7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_for_concurrent_alter" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE table_for_concurrent_alter (id UInt64, Data String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096;"; + +n=0 +while [ "$n" -lt 100 ]; +do + n=$(( n + 1 )) + $CLICKHOUSE_CLIENT --query="INSERT INTO table_for_concurrent_alter VALUES(1, 'Hello')" > /dev/null 2> /dev/null & + $CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE table_for_concurrent_alter FINAL" > /dev/null 2> /dev/null & +done & + + +q=0 +while [ "$q" -lt 100 ]; +do + q=$(( q + 1 )) + counter=$(( 100 + q )) + $CLICKHOUSE_CLIENT --query="ALTER TABLE table_for_concurrent_alter MODIFY SETTING parts_to_throw_insert = $counter, parts_to_delay_insert = $counter, min_merge_bytes_to_use_direct_io = $counter" > /dev/null 2> /dev/null & +done & + +sleep 4 + +# we just test race conditions, not logic +$CLICKHOUSE_CLIENT --query "SELECT 1" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_for_concurrent_alter" diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql index cb1ba315001..866cf960710 100644 --- a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql @@ -35,7 +35,7 @@ INSERT INTO table_for_alter VALUES (2, '2'); -- { serverError 252 } ALTER TABLE table_for_alter MODIFY SETTING xxx_yyy=124; -- { serverError 115 } -ALTER TABLE table_for_alter MODIFY SETTING parts_to_throw_insert = 100, parts_to_delay_insert = 100; +ALTER TABLE table_for_alter MODIFY SETTING parts_to_throw_insert = 100, parts_to_delay_insert = 100; INSERT INTO table_for_alter VALUES (2, '2'); From 13e4581317126661fd0d7244976cf7bda854525d Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 7 Aug 2019 18:21:45 +0300 Subject: [PATCH 072/357] More general --- dbms/src/Storages/IStorage.cpp | 51 +++++++++++++++++-- dbms/src/Storages/IStorage.h | 13 +++++ dbms/src/Storages/Kafka/StorageKafka.cpp | 17 +++++++ dbms/src/Storages/Kafka/StorageKafka.h | 10 ++++ dbms/src/Storages/MergeTree/MergeTreeData.cpp | 22 ++------ dbms/src/Storages/MergeTree/MergeTreeData.h | 4 +- 6 files changed, 93 insertions(+), 24 deletions(-) diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 0fd1994f5ce..e748b72c769 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -1,6 +1,8 @@ #include #include +#include +#include #include #include @@ -19,6 +21,7 @@ namespace ErrorCodes extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int TYPE_MISMATCH; extern const int SETTINGS_ARE_NOT_SUPPORTED; + extern const int UNKNOWN_SETTING; } IStorage::IStorage(ColumnsDescription columns_) @@ -295,7 +298,9 @@ bool IStorage::isVirtualColumn(const String & column_name) const bool IStorage::hasSetting(const String & /* setting_name */) const { - throw Exception("Storage '" + getName() + "' doesn't support settings.", ErrorCodes::SETTINGS_ARE_NOT_SUPPORTED); + if (!supportsSettings()) + throw Exception("Storage '" + getName() + "' doesn't support settings.", ErrorCodes::SETTINGS_ARE_NOT_SUPPORTED); + return false; } TableStructureReadLockHolder IStorage::lockStructureForShare(bool will_add_new_data, const String & query_id) @@ -352,6 +357,39 @@ TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id) return result; } + +void IStorage::alterSettings( + const SettingsChanges & new_changes, + const String & current_database_name, + const String & current_table_name, + const Context & context, + TableStructureWriteLockHolder & /* table_lock_holder */) +{ + IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) + { + if (!new_changes.empty()) + { + auto & storage_changes = ast.as().settings->changes; + /// Make storage settings unique + for (const auto & change : new_changes) + { + if (hasSetting(change.name)) + { + auto finder = [&change] (const SettingChange & c) { return c.name == change.name;}; + if (auto it = std::find_if(storage_changes.begin(), storage_changes.end(), finder); it != storage_changes.end()) + it->value = change.value; + else + storage_changes.push_back(change); + } + else + throw Exception{"Storage '" + getName() + "' doesn't have setting '" + change.name + "'", ErrorCodes::UNKNOWN_SETTING}; + } + } + }; + context.getDatabase(current_database_name)->alterTable(context, current_table_name, getColumns(), getIndices(), storage_modifier); +} + + void IStorage::alter( const AlterCommands & params, const String & database_name, @@ -359,12 +397,17 @@ void IStorage::alter( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - for (const auto & param : params) + if (params.isSettingsAlter()) { - if (param.isMutable()) - throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); + SettingsChanges new_changes; + params.applyForSettingsOnly(new_changes); + alterSettings(new_changes, database_name, table_name, context, table_lock_holder); + return; } + if (params.isMutable()) + throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 79760bae5a8..7000eef76f6 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -95,6 +96,9 @@ public: /// Returns true if the storage supports deduplication of inserted data blocks. virtual bool supportsDeduplication() const { return false; } + /// Returns true if the storage supports settings. + virtual bool supportsSettings() const { return false; } + /// Optional size information of each physical column. /// Currently it's only used by the MergeTree family for query optimizations. using ColumnSizeByName = std::unordered_map; @@ -252,6 +256,15 @@ public: throw Exception("Partition operations are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } + /** ALTER table settings if possible. Otherwise throws exception. + */ + virtual void alterSettings( + const SettingsChanges & new_changes, + const String & current_database_name, + const String & current_table_name, + const Context & context, + TableStructureWriteLockHolder & table_lock_holder); + /** Perform any background work. For example, combining parts in a MergeTree type table. * Returns whether any work has been done. */ diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 20599c7e4f8..47f4fc8c610 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -40,6 +40,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int UNSUPPORTED_METHOD; } namespace @@ -388,6 +389,22 @@ bool StorageKafka::streamToViews() } +bool StorageKafka::hasSetting(const String & setting_name) const +{ + return KafkaSettings::findIndex(setting_name) != KafkaSettings::npos; +} + +void StorageKafka::alterSettings( + const SettingsChanges & /* new_changes */, + const String & /* current_database_name */, + const String & /* current_table_name */, + const Context & /* context */, + TableStructureWriteLockHolder & /* table_lock_holder */) +{ + throw Exception("Storage '" + getName() + "' doesn't support settings alter", ErrorCodes::UNSUPPORTED_METHOD); +} + + void registerStorageKafka(StorageFactory & factory) { factory.registerStorage("Kafka", [](const StorageFactory::Arguments & args) diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index f9b6609def5..38a63e8536e 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -24,6 +24,7 @@ public: std::string getName() const override { return "Kafka"; } std::string getTableName() const override { return table_name; } std::string getDatabaseName() const override { return database_name; } + bool supportsSettings() const override { return true; } void startup() override; void shutdown() override; @@ -50,6 +51,15 @@ public: const auto & getSchemaName() const { return schema_name; } const auto & skipBroken() const { return skip_broken; } + bool hasSetting(const String & setting_name) const override; + + void alterSettings( + const SettingsChanges & new_changes, + const String & current_database_name, + const String & current_table_name, + const Context & context, + TableStructureWriteLockHolder & table_lock_holder) override; + protected: StorageKafka( const std::string & table_name_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 4f7452bd79f..b0453b04c8a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1588,26 +1588,10 @@ void MergeTreeData::alterSettings( const String & current_database_name, const String & current_table_name, const Context & context, - TableStructureWriteLockHolder & /* table_lock_holder */) + TableStructureWriteLockHolder & table_lock_holder) { settings.updateFromChanges(new_changes); - IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) - { - if (!new_changes.empty()) - { - auto & storage_changes = ast.as().settings->changes; - /// Make storage settings unique - for (const auto & change : new_changes) - { - auto finder = [&change] (const SettingChange & c) { return c.name == change.name;}; - if (auto it = std::find_if(storage_changes.begin(), storage_changes.end(), finder); it != storage_changes.end()) - it->value = change.value; - else - storage_changes.push_back(change); - } - } - }; - context.getDatabase(current_database_name)->alterTable(context, current_table_name, getColumns(), getIndices(), storage_modifier); + IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); } bool MergeTreeData::hasSetting(const String & setting_name) const @@ -2245,7 +2229,7 @@ std::optional MergeTreeData::getMinPartDataVersion() const } -void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event *until) const +void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const { const size_t parts_count_in_total = getPartsCount(); if (parts_count_in_total >= settings.max_parts_in_total) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 78c7df3bb48..756c188c724 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -342,6 +342,8 @@ public: || merging_params.mode == MergingParams::VersionedCollapsing; } + bool supportsSettings() const override { return true; } + bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context &) const override; NameAndTypePair getColumn(const String & column_name) const override @@ -516,7 +518,7 @@ public: const String & current_database_name, const String & current_table_name, const Context & context, - TableStructureWriteLockHolder & table_lock_holder); + TableStructureWriteLockHolder & table_lock_holder) override; /// All MergeTreeData children have settings. bool hasSetting(const String & setting_name) const override; From 7409f1a3de81b5698364d9381ec113195b1cfdb3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 7 Aug 2019 18:33:10 +0300 Subject: [PATCH 073/357] More comments --- dbms/src/Core/Settings.h | 1 + dbms/src/Core/SettingsCommon.h | 3 +++ dbms/src/Storages/Kafka/KafkaSettings.h | 2 ++ dbms/src/Storages/MergeTree/MergeTreeSettings.h | 1 + 4 files changed, 7 insertions(+) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 0c5a4ceed60..54f6a14205b 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -42,6 +42,7 @@ struct Settings : public SettingsCollection * but we are not going to do it, because settings is used everywhere as static struct fields. */ +/// M (mutable) for normal settings, IM (immutable) for not updateable settings. #define LIST_OF_SETTINGS(M, IM) \ M(SettingUInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.") \ M(SettingUInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.") \ diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index 4b3f7020a6e..d4607e70904 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -321,6 +321,9 @@ private: size_t offset_of_changed; StringRef name; StringRef description; + /// Can be updated after first load for config/definition. + /// Non updatable settings can be isChanged, + /// if they were overwritten in config/definition. bool updateable; GetStringFunction get_string; GetFieldFunction get_field; diff --git a/dbms/src/Storages/Kafka/KafkaSettings.h b/dbms/src/Storages/Kafka/KafkaSettings.h index fd8d9b3dc08..e43ea7cd70e 100644 --- a/dbms/src/Storages/Kafka/KafkaSettings.h +++ b/dbms/src/Storages/Kafka/KafkaSettings.h @@ -14,6 +14,8 @@ class ASTStorage; struct KafkaSettings : public SettingsCollection { + +/// M (mutable) for normal settings, IM (immutable) for not updateable settings. #define LIST_OF_KAFKA_SETTINGS(M, IM) \ M(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ M(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 3012b9e1694..dc94ad5b11f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -24,6 +24,7 @@ class ASTStorage; struct MergeTreeSettings : public SettingsCollection { +/// M (mutable) for normal settings, IM (immutable) for not updateable settings. #define LIST_OF_MERGE_TREE_SETTINGS(M, IM) \ IM(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \ \ From 75c3ed967a9fe5fb780b0107b9887bf343d67a15 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 7 Aug 2019 19:13:28 +0300 Subject: [PATCH 074/357] Checking updatable for user settings --- dbms/src/Interpreters/Context.cpp | 11 +++++++++++ dbms/src/Interpreters/Context.h | 3 +++ dbms/src/Interpreters/InterpreterSetQuery.cpp | 4 ++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 992593d852c..2484342908f 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1126,6 +1126,17 @@ void Context::applySettingsChanges(const SettingsChanges & changes) applySettingChange(change); } +void Context::updateSettingsChanges(const SettingsChanges & changes) +{ + auto lock = getLock(); + for (const SettingChange & change : changes) + { + if (change.name == "profile") + setProfile(change.value.safeGet()); + else + settings.updateFromChange(change); + } +} void Context::checkSettingsConstraints(const SettingChange & change) { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 50b7ab3eba2..4f97922a513 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -281,6 +281,9 @@ public: void applySettingChange(const SettingChange & change); void applySettingsChanges(const SettingsChanges & changes); + /// Update checking that each setting is updatable + void updateSettingsChanges(const SettingsChanges & changes); + /// Checks the constraints. void checkSettingsConstraints(const SettingChange & change); void checkSettingsConstraints(const SettingsChanges & changes); diff --git a/dbms/src/Interpreters/InterpreterSetQuery.cpp b/dbms/src/Interpreters/InterpreterSetQuery.cpp index f92e9638822..ae982611e60 100644 --- a/dbms/src/Interpreters/InterpreterSetQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSetQuery.cpp @@ -10,7 +10,7 @@ BlockIO InterpreterSetQuery::execute() { const auto & ast = query_ptr->as(); context.checkSettingsConstraints(ast.changes); - context.getSessionContext().applySettingsChanges(ast.changes); + context.getSessionContext().updateSettingsChanges(ast.changes); return {}; } @@ -19,7 +19,7 @@ void InterpreterSetQuery::executeForCurrentContext() { const auto & ast = query_ptr->as(); context.checkSettingsConstraints(ast.changes); - context.applySettingsChanges(ast.changes); + context.updateSettingsChanges(ast.changes); } } From 2f36d80705d5a62c8efc15c623de20250cee1537 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 8 Aug 2019 18:51:17 +0300 Subject: [PATCH 075/357] move tests with sudo to integration tests --- .../integration/test_partition/__init__.py | 0 dbms/tests/integration/test_partition/test.py | 244 ++++++++++++++++++ .../0_stateless/00428_partition.reference | 54 ---- .../queries/0_stateless/00428_partition.sh | 60 ----- .../00974_attach_invalid_parts.reference | 26 -- .../0_stateless/00974_attach_invalid_parts.sh | 53 ---- .../0_stateless/00975_drop_detached.reference | 15 -- .../0_stateless/00975_drop_detached.sh | 45 ---- 8 files changed, 244 insertions(+), 253 deletions(-) create mode 100644 dbms/tests/integration/test_partition/__init__.py create mode 100644 dbms/tests/integration/test_partition/test.py delete mode 100644 dbms/tests/queries/0_stateless/00428_partition.reference delete mode 100755 dbms/tests/queries/0_stateless/00428_partition.sh delete mode 100644 dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference delete mode 100755 dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh delete mode 100644 dbms/tests/queries/0_stateless/00975_drop_detached.reference delete mode 100755 dbms/tests/queries/0_stateless/00975_drop_detached.sh diff --git a/dbms/tests/integration/test_partition/__init__.py b/dbms/tests/integration/test_partition/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_partition/test.py b/dbms/tests/integration/test_partition/test.py new file mode 100644 index 00000000000..59c48e5d9e9 --- /dev/null +++ b/dbms/tests/integration/test_partition/test.py @@ -0,0 +1,244 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') +q = instance.query +path_to_data = '/var/lib/clickhouse/' + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + q('CREATE DATABASE test') + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture +def partition_table_simple(started_cluster): + q("DROP TABLE IF EXISTS test.partition") + q("CREATE TABLE test.partition (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) " + "ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) " + "SETTINGS index_granularity=8192, index_granularity_bytes=0") + q("INSERT INTO test.partition ( x ) VALUES ( now() )") + q("INSERT INTO test.partition ( x ) VALUES ( now()+1 )") + + yield + + q('DROP TABLE test.partition') + + +def test_partition_simple(partition_table_simple): + q("ALTER TABLE test.partition DETACH PARTITION 197001") + q("ALTER TABLE test.partition ATTACH PARTITION 197001") + q("OPTIMIZE TABLE test.partition") + + +def exec_bash(cmd): + cmd = '/bin/bash -c "{}"'.format(cmd.replace('"', '\\"')) + return instance.exec_in_container(cmd) + + +def partition_complex_assert_columns_txt(): + path_to_parts = path_to_data + 'data/test/partition/' + parts = TSV(q("SELECT name FROM system.parts WHERE database='test' AND table='partition'")) + for part_name in parts.lines: + path_to_columns = path_to_parts + part_name + '/columns.txt' + # 2 header lines + 3 columns + assert exec_bash('cat {} | wc -l'.format(path_to_columns)) == u'5\n' + + +def partition_complex_assert_checksums(): + # Do `cd` for consistent output for reference + # Do not check increment.txt - it can be changed by other tests with FREEZE + cmd = 'cd ' + path_to_data + " && find shadow -type f -exec md5sum {} \\;" \ + " | grep partition" \ + " | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g'" \ + " | sort" \ + " | uniq" + + checksums = "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700102_2_2_0/k.bin\n" \ + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700201_1_1_0/v1.bin\n" \ + "13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition/19700102_2_2_0/minmax_p.idx\n" \ + "25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition/19700102_2_2_0/partition.dat\n" \ + "3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition/19700201_1_1_0/partition.dat\n" \ + "37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition/19700102_2_2_0/checksums.txt\n" \ + "38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition/19700102_2_2_0/v1.bin\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/k.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/p.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/v1.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/k.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/p.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/v1.mrk\n" \ + "55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition/19700201_1_1_0/primary.idx\n" \ + "5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition/19700201_1_1_0/checksums.txt\n" \ + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700102_2_2_0/columns.txt\n" \ + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700201_1_1_0/columns.txt\n" \ + "88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition/19700201_1_1_0/p.bin\n" \ + "9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition/19700102_2_2_0/primary.idx\n" \ + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700102_2_2_0/count.txt\n" \ + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700201_1_1_0/count.txt\n" \ + "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition/19700102_2_2_0/p.bin\n" \ + "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition/19700201_1_1_0/k.bin\n" \ + "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition/19700201_1_1_0/minmax_p.idx\n" + + assert TSV(exec_bash(cmd).replace(' ', '\t')) == TSV(checksums) + + +@pytest.fixture +def partition_table_complex(started_cluster): + q("DROP TABLE IF EXISTS test.partition") + q("CREATE TABLE test.partition (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) " + "ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0") + q("INSERT INTO test.partition (p, k) VALUES(toDate(31), 1)") + q("INSERT INTO test.partition (p, k) VALUES(toDate(1), 2)") + + yield + + q("DROP TABLE test.partition") + + +def test_partition_complex(partition_table_complex): + + partition_complex_assert_columns_txt() + + q("ALTER TABLE test.partition FREEZE") + + partition_complex_assert_checksums() + + q("ALTER TABLE test.partition DETACH PARTITION 197001") + q("ALTER TABLE test.partition ATTACH PARTITION 197001") + + partition_complex_assert_columns_txt() + + q("ALTER TABLE test.partition MODIFY COLUMN v1 Int8") + + # Check the backup hasn't changed + partition_complex_assert_checksums() + + q("OPTIMIZE TABLE test.partition") + + expected = TSV('31\t1\t2\n' + '1\t2\t3') + res = q("SELECT toUInt16(p), k, v1 FROM test.partition ORDER BY k") + assert(TSV(res) == expected) + + +@pytest.fixture +def cannot_attach_active_part_table(started_cluster): + q("DROP TABLE IF EXISTS test.attach_active") + q("CREATE TABLE test.attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n") + q("INSERT INTO test.attach_active SELECT number FROM system.numbers LIMIT 16") + + yield + + q("DROP TABLE test.attach_active") + + +def test_cannot_attach_active_part(cannot_attach_active_part_table): + error = instance.client.query_and_get_error("ALTER TABLE test.attach_active ATTACH PART '../1_2_2_0'") + print error + assert 0 <= error.find('Invalid part name') + + res = q("SElECT name FROM system.parts WHERE table='attach_active' AND database='test' ORDER BY name") + assert TSV(res) == TSV('0_1_1_0\n1_2_2_0\n2_3_3_0\n3_4_4_0') + assert TSV(q("SElECT count(), sum(n) FROM test.attach_active")) == TSV('16\t120') + + +@pytest.fixture +def attach_check_all_parts_table(started_cluster): + q("SYSTEM STOP MERGES") + q("DROP TABLE IF EXISTS test.attach_partition") + q("CREATE TABLE test.attach_partition (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n") + q("INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8") + q("INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8") + + yield + + q("DROP TABLE test.attach_partition") + q("SYSTEM START MERGES") + + +def test_attach_check_all_parts(attach_check_all_parts_table): + q("ALTER TABLE test.attach_partition DETACH PARTITION 0") + + path_to_detached = path_to_data + 'data/test/attach_partition/detached/' + exec_bash('mkdir {}'.format(path_to_detached + '0_5_5_0')) + exec_bash('cp -pr {} {}'.format(path_to_detached + '0_1_1_0', path_to_detached + 'attaching_0_6_6_0')) + exec_bash('cp -pr {} {}'.format(path_to_detached + '0_3_3_0', path_to_detached + 'deleting_0_7_7_0')) + + error = instance.client.query_and_get_error("ALTER TABLE test.attach_partition ATTACH PARTITION 0") + assert 0 <= error.find('No columns in part 0_5_5_0') + + parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(parts) == TSV('1_2_2_0\n1_4_4_0') + detached = q("SELECT name FROM system.detached_parts " + "WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_1_1_0\n0_3_3_0\n0_5_5_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + + exec_bash('rm -r {}'.format(path_to_detached + '0_5_5_0')) + + q("ALTER TABLE test.attach_partition ATTACH PARTITION 0") + parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") + expected = '0_5_5_0\n0_6_6_0\n1_2_2_0\n1_4_4_0' + assert TSV(parts) == TSV(expected) + assert TSV(q("SElECT count(), sum(n) FROM test.attach_partition")) == TSV('16\t120') + + detached = q("SELECT name FROM system.detached_parts " + "WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('attaching_0_6_6_0\ndeleting_0_7_7_0') + + +@pytest.fixture +def drop_detached_parts_table(started_cluster): + q("SYSTEM STOP MERGES") + q("DROP TABLE IF EXISTS test.drop_detached") + q("CREATE TABLE test.drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n") + q("INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8") + q("INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8") + + yield + + q("DROP TABLE test.drop_detached") + q("SYSTEM START MERGES") + + +def test_drop_detached_parts(drop_detached_parts_table): + s = {"allow_drop_detached_part": 1} + q("ALTER TABLE test.drop_detached DETACH PARTITION 0") + q("ALTER TABLE test.drop_detached DETACH PARTITION 1") + + path_to_detached = path_to_data + 'data/test/drop_detached/detached/' + exec_bash('mkdir {}'.format(path_to_detached + 'attaching_0_6_6_0')) + exec_bash('mkdir {}'.format(path_to_detached + 'deleting_0_7_7_0')) + exec_bash('mkdir {}'.format(path_to_detached + 'any_other_name')) + exec_bash('mkdir {}'.format(path_to_detached + 'prefix_1_2_2_0_0')) + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0'", settings=s) + assert 0 <= error.find('Invalid part name') + + q("ALTER TABLE test.drop_detached DROP DETACHED PART '0_1_1_0'", settings=s) + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0'", settings=s) + assert 0 <= error.find('Cannot drop part') + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0'", settings=s) + assert 0 <= error.find('Cannot drop part') + + q("ALTER TABLE test.drop_detached DROP DETACHED PART 'any_other_name'", settings=s) + + detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_3_3_0\n1_2_2_0\n1_4_4_0\nattaching_0_6_6_0\ndeleting_0_7_7_0\nprefix_1_2_2_0_0') + + q("ALTER TABLE test.drop_detached DROP DETACHED PARTITION 1", settings=s) + detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_3_3_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + diff --git a/dbms/tests/queries/0_stateless/00428_partition.reference b/dbms/tests/queries/0_stateless/00428_partition.reference deleted file mode 100644 index c777fd7a5c3..00000000000 --- a/dbms/tests/queries/0_stateless/00428_partition.reference +++ /dev/null @@ -1,54 +0,0 @@ -5 -5 -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700102_2_2_0/k.bin -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700201_1_1_0/v1.bin -13cae8e658e0ca4f75c56b1fc424e150 shadow/1/data/test/partition_428/19700102_2_2_0/minmax_p.idx -25daad3d9e60b45043a70c4ab7d3b1c6 shadow/1/data/test/partition_428/19700102_2_2_0/partition.dat -3726312af62aec86b64a7708d5751787 shadow/1/data/test/partition_428/19700201_1_1_0/partition.dat -37855b06a39b79a67ea4e86e4a3299aa shadow/1/data/test/partition_428/19700102_2_2_0/checksums.txt -38e62ff37e1e5064e9a3f605dfe09d13 shadow/1/data/test/partition_428/19700102_2_2_0/v1.bin -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/v1.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/v1.mrk -55a54008ad1ba589aa210d2629c1df41 shadow/1/data/test/partition_428/19700201_1_1_0/primary.idx -5f087cb3e7071bf9407e095821e2af8f shadow/1/data/test/partition_428/19700201_1_1_0/checksums.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700102_2_2_0/columns.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700201_1_1_0/columns.txt -88cdc31ded355e7572d68d8cde525d3a shadow/1/data/test/partition_428/19700201_1_1_0/p.bin -9e688c58a5487b8eaf69c9e1005ad0bf shadow/1/data/test/partition_428/19700102_2_2_0/primary.idx -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700102_2_2_0/count.txt -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700201_1_1_0/count.txt -cfcb770c3ecd0990dcceb1bde129e6c6 shadow/1/data/test/partition_428/19700102_2_2_0/p.bin -e2af3bef1fd129aea73a890ede1e7a30 shadow/1/data/test/partition_428/19700201_1_1_0/k.bin -f2312862cc01adf34a93151377be2ddf shadow/1/data/test/partition_428/19700201_1_1_0/minmax_p.idx -5 -5 -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700102_2_2_0/k.bin -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700201_1_1_0/v1.bin -13cae8e658e0ca4f75c56b1fc424e150 shadow/1/data/test/partition_428/19700102_2_2_0/minmax_p.idx -25daad3d9e60b45043a70c4ab7d3b1c6 shadow/1/data/test/partition_428/19700102_2_2_0/partition.dat -3726312af62aec86b64a7708d5751787 shadow/1/data/test/partition_428/19700201_1_1_0/partition.dat -37855b06a39b79a67ea4e86e4a3299aa shadow/1/data/test/partition_428/19700102_2_2_0/checksums.txt -38e62ff37e1e5064e9a3f605dfe09d13 shadow/1/data/test/partition_428/19700102_2_2_0/v1.bin -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/v1.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/v1.mrk -55a54008ad1ba589aa210d2629c1df41 shadow/1/data/test/partition_428/19700201_1_1_0/primary.idx -5f087cb3e7071bf9407e095821e2af8f shadow/1/data/test/partition_428/19700201_1_1_0/checksums.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700102_2_2_0/columns.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700201_1_1_0/columns.txt -88cdc31ded355e7572d68d8cde525d3a shadow/1/data/test/partition_428/19700201_1_1_0/p.bin -9e688c58a5487b8eaf69c9e1005ad0bf shadow/1/data/test/partition_428/19700102_2_2_0/primary.idx -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700102_2_2_0/count.txt -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700201_1_1_0/count.txt -cfcb770c3ecd0990dcceb1bde129e6c6 shadow/1/data/test/partition_428/19700102_2_2_0/p.bin -e2af3bef1fd129aea73a890ede1e7a30 shadow/1/data/test/partition_428/19700201_1_1_0/k.bin -f2312862cc01adf34a93151377be2ddf shadow/1/data/test/partition_428/19700201_1_1_0/minmax_p.idx -31,1,2 -1,2,3 diff --git a/dbms/tests/queries/0_stateless/00428_partition.sh b/dbms/tests/queries/0_stateless/00428_partition.sh deleted file mode 100755 index 033d5e24c13..00000000000 --- a/dbms/tests/queries/0_stateless/00428_partition.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -# Not found column date in block. There are only columns: x. - -# Test 1. Complex test checking columns.txt - -chl="$CLICKHOUSE_CLIENT -q" -ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` - -$chl "DROP TABLE IF EXISTS test.partition_428" -$chl "CREATE TABLE test.partition_428 (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0" -$chl "INSERT INTO test.partition_428 (p, k) VALUES(toDate(31), 1)" -$chl "INSERT INTO test.partition_428 (p, k) VALUES(toDate(1), 2)" - -for part in `$chl "SELECT name FROM system.parts WHERE database='test' AND table='partition_428'"`; do - # 2 header lines + 3 columns - (sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt 2>/dev/null || \ - cat $ch_dir/data/test/partition_428/$part/columns.txt) | wc -l -done - -$chl "ALTER TABLE test.partition_428 FREEZE" - -# Do `cd` for consistent output for reference -# Do not check increment.txt - it can be changed by other tests with FREEZE -cd $ch_dir && find shadow -type f -exec md5sum {} \; | grep "partition_428" | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g' | sort | uniq - -$chl "ALTER TABLE test.partition_428 DETACH PARTITION 197001" -$chl "ALTER TABLE test.partition_428 ATTACH PARTITION 197001" - -for part in `$chl "SELECT name FROM system.parts WHERE database='test' AND table='partition_428'"`; do - # 2 header lines + 3 columns - (sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt 2>/dev/null || \ - cat $ch_dir/data/test/partition_428/$part/columns.txt) | wc -l -done - -$chl "ALTER TABLE test.partition_428 MODIFY COLUMN v1 Int8" - -# Check the backup hasn't changed -cd $ch_dir && find shadow -type f -exec md5sum {} \; | grep "partition_428" | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g' | sort | uniq - -$chl "OPTIMIZE TABLE test.partition_428" - -$chl "SELECT toUInt16(p), k, v1 FROM test.partition_428 ORDER BY k FORMAT CSV" -$chl "DROP TABLE test.partition_428" - -# Test 2. Simple test - -$chl "drop table if exists test.partition_428" -$chl "create table test.partition_428 (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) SETTINGS index_granularity=8192, index_granularity_bytes=0" -$chl "insert into test.partition_428 ( x ) VALUES ( now() )" -$chl "insert into test.partition_428 ( x ) VALUES ( now()+1 )" -$chl "alter table test.partition_428 detach partition 197001" -$chl "alter table test.partition_428 attach partition 197001" -$chl "optimize table test.partition_428" -$chl "drop table test.partition_428" diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference deleted file mode 100644 index f30fc160dfb..00000000000 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference +++ /dev/null @@ -1,26 +0,0 @@ -=== cannot attach active === -OK1 -0_1_1_0 -1_2_2_0 -2_3_3_0 -3_4_4_0 -16 120 -=== check all parts before attaching === -OK2 -1_2_2_0 -1_4_4_0 -=== detached === -0_1_1_0 -0_3_3_0 -0_5_5_0 -attaching_0_6_6_0 -deleting_0_7_7_0 -=== attach === -0_5_5_0 -0_6_6_0 -1_2_2_0 -1_4_4_0 -16 120 -=== detached === -attaching_0_6_6_0 -deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh deleted file mode 100755 index db45cfe7f21..00000000000 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` -cur_db=`${CLICKHOUSE_CLIENT} --query "SELECT currentDatabase()"` - -echo '=== cannot attach active ==='; -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_active"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; -$CLICKHOUSE_CLIENT --query="INSERT INTO attach_active SELECT number FROM system.numbers LIMIT 16"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_active ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_active' AND database='${cur_db}' ORDER BY name FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_active FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="DROP TABLE attach_active"; - - - -$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_partitions"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_partitions (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; -$CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8"; -$CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; - -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part -sudo -n cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ - cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ -sudo -n cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ - cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ - -echo '=== check all parts before attaching ==='; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0" 2>&1 | grep "No columns in part 0_5_5_0" > /dev/null && echo 'OK2'; -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; -echo '=== detached ==='; -$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; - -echo '=== attach ==='; -sudo -n rm -r $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ - rm -r $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_partitions FORMAT TSV"; - -echo '=== detached ==='; -$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; - -$CLICKHOUSE_CLIENT --query="DROP TABLE attach_partitions"; -$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.reference b/dbms/tests/queries/0_stateless/00975_drop_detached.reference deleted file mode 100644 index 414ac4b1927..00000000000 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.reference +++ /dev/null @@ -1,15 +0,0 @@ -=== validate part name === -OK1 -OK2 -OK3 -=== drop detached part === -0_3_3_0 -1_2_2_0 -1_4_4_0 -attaching_0_6_6_0 -deleting_0_7_7_0 -prefix_1_2_2_0_0 -=== drop detached partition === -0_3_3_0 -attaching_0_6_6_0 -deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh deleted file mode 100755 index 8da831b019a..00000000000 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` -cur_db=`${CLICKHOUSE_CLIENT} --query "SELECT currentDatabase()"` - -$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS drop_detached"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; -$CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8"; -$CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; - -$CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 1"; -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0_0/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0_0/ -#sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0/ 2>/dev/null || \ -# mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0/ - -echo '=== validate part name ===' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK2' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK3' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" - -echo '=== drop detached part ===' -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; - -echo '=== drop detached partition ===' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PARTITION 1" -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; - -$CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; -$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From b5eee531a9cbd9b3df2c5c373d99680e50d6b8cb Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 8 Aug 2019 19:08:43 +0300 Subject: [PATCH 076/357] fix setting name --- dbms/src/Core/Settings.h | 2 +- dbms/src/Interpreters/InterpreterAlterQuery.cpp | 2 +- dbms/tests/integration/test_partition/test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index fd11d645bd5..ffc11cef4a6 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -341,7 +341,7 @@ struct Settings : public SettingsCollection /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ - M(SettingBool, allow_drop_detached_part, false, "Allow ALTER TABLE ... DROP DETACHED PART ... queries") + M(SettingBool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries") DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 074fbb7d4c2..bc419f1ff84 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -56,7 +56,7 @@ BlockIO InterpreterAlterQuery::execute() else if (auto partition_command = PartitionCommand::parse(command_ast)) { if (partition_command->type == PartitionCommand::DROP_DETACHED_PARTITION - && !context.getSettingsRef().allow_drop_detached_part) + && !context.getSettingsRef().allow_drop_detached) throw DB::Exception("Cannot execute query: DROP DETACHED PART is disabled " "(see allow_drop_detached setting)", ErrorCodes::SUPPORT_IS_DISABLED); partition_commands.emplace_back(std::move(*partition_command)); diff --git a/dbms/tests/integration/test_partition/test.py b/dbms/tests/integration/test_partition/test.py index 59c48e5d9e9..3365343b6fb 100644 --- a/dbms/tests/integration/test_partition/test.py +++ b/dbms/tests/integration/test_partition/test.py @@ -212,7 +212,7 @@ def drop_detached_parts_table(started_cluster): def test_drop_detached_parts(drop_detached_parts_table): - s = {"allow_drop_detached_part": 1} + s = {"allow_drop_detached": 1} q("ALTER TABLE test.drop_detached DETACH PARTITION 0") q("ALTER TABLE test.drop_detached DETACH PARTITION 1") From b780072a9d4fe86c884c087b88b552b641a03ac4 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Thu, 8 Aug 2019 21:49:29 +0300 Subject: [PATCH 077/357] raw data relative_analize --- dbms/programs/benchmark/Benchmark.cpp | 192 +++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 5 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 923745eb204..e97a499101b 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -67,6 +67,7 @@ public: connections.reserve(connections_cnt); comparison_info_total.reserve(connections_cnt); comparison_info_per_interval.reserve(connections_cnt); + comparison_relative.data.resize(connections_cnt); for (size_t i = 0; i < connections_cnt; ++i) { @@ -183,6 +184,178 @@ private: } }; + struct RelativeAnalysis + { + struct RelativeStats + { + size_t cnt = 0; + double sum = 0; + double squares_sum = 0; + + void add(double seconds) + { + ++cnt; + sum += seconds; + squares_sum += seconds * seconds; + } + + double avg() + { + return sum / cnt; + } + + double var() + { + return (squares_sum - (sum * sum / cnt)) / static_cast(cnt - 1); + } + }; + + std::vector confidence_level = { 80, 90, 95, 98, 99, 99.5 }; + + std::vector> students_table = { + /* inf */ { 1.282, 1.645, 1.960, 2.326, 2.576, 3.090 }, + /* 1. */ { 3.078, 6.314, 12.706, 31.821, 63.657, 318.313 }, + /* 2. */ { 1.886, 2.920, 4.303, 6.965, 9.925, 22.327 }, + /* 3. */ { 1.638, 2.353, 3.182, 4.541, 5.841, 10.215 }, + /* 4. */ { 1.533, 2.132, 2.776, 3.747, 4.604, 7.173 }, + /* 5. */ { 1.476, 2.015, 2.571, 3.365, 4.032, 5.893 }, + /* 6. */ { 1.440, 1.943, 2.447, 3.143, 3.707, 5.208 }, + /* 7. */ { 1.415, 1.895, 2.365, 2.998, 3.499, 4.782 }, + /* 8. */ { 1.397, 1.860, 2.306, 2.896, 3.355, 4.499 }, + /* 9. */ { 1.383, 1.833, 2.262, 2.821, 3.250, 4.296 }, + /* 10. */ { 1.372, 1.812, 2.228, 2.764, 3.169, 4.143 }, + /* 11. */ { 1.363, 1.796, 2.201, 2.718, 3.106, 4.024 }, + /* 12. */ { 1.356, 1.782, 2.179, 2.681, 3.055, 3.929 }, + /* 13. */ { 1.350, 1.771, 2.160, 2.650, 3.012, 3.852 }, + /* 14. */ { 1.345, 1.761, 2.145, 2.624, 2.977, 3.787 }, + /* 15. */ { 1.341, 1.753, 2.131, 2.602, 2.947, 3.733 }, + /* 16. */ { 1.337, 1.746, 2.120, 2.583, 2.921, 3.686 }, + /* 17. */ { 1.333, 1.740, 2.110, 2.567, 2.898, 3.646 }, + /* 18. */ { 1.330, 1.734, 2.101, 2.552, 2.878, 3.610 }, + /* 19. */ { 1.328, 1.729, 2.093, 2.539, 2.861, 3.579 }, + /* 20. */ { 1.325, 1.725, 2.086, 2.528, 2.845, 3.552 }, + /* 21. */ { 1.323, 1.721, 2.080, 2.518, 2.831, 3.527 }, + /* 22. */ { 1.321, 1.717, 2.074, 2.508, 2.819, 3.505 }, + /* 23. */ { 1.319, 1.714, 2.069, 2.500, 2.807, 3.485 }, + /* 24. */ { 1.318, 1.711, 2.064, 2.492, 2.797, 3.467 }, + /* 25. */ { 1.316, 1.708, 2.060, 2.485, 2.787, 3.450 }, + /* 26. */ { 1.315, 1.706, 2.056, 2.479, 2.779, 3.435 }, + /* 27. */ { 1.314, 1.703, 2.052, 2.473, 2.771, 3.421 }, + /* 28. */ { 1.313, 1.701, 2.048, 2.467, 2.763, 3.408 }, + /* 29. */ { 1.311, 1.699, 2.045, 2.462, 2.756, 3.396 }, + /* 30. */ { 1.310, 1.697, 2.042, 2.457, 2.750, 3.385 }, + /* 31. */ { 1.309, 1.696, 2.040, 2.453, 2.744, 3.375 }, + /* 32. */ { 1.309, 1.694, 2.037, 2.449, 2.738, 3.365 }, + /* 33. */ { 1.308, 1.692, 2.035, 2.445, 2.733, 3.356 }, + /* 34. */ { 1.307, 1.691, 2.032, 2.441, 2.728, 3.348 }, + /* 35. */ { 1.306, 1.690, 2.030, 2.438, 2.724, 3.340 }, + /* 36. */ { 1.306, 1.688, 2.028, 2.434, 2.719, 3.333 }, + /* 37. */ { 1.305, 1.687, 2.026, 2.431, 2.715, 3.326 }, + /* 38. */ { 1.304, 1.686, 2.024, 2.429, 2.712, 3.319 }, + /* 39. */ { 1.304, 1.685, 2.023, 2.426, 2.708, 3.313 }, + /* 40. */ { 1.303, 1.684, 2.021, 2.423, 2.704, 3.307 }, + /* 41. */ { 1.303, 1.683, 2.020, 2.421, 2.701, 3.301 }, + /* 42. */ { 1.302, 1.682, 2.018, 2.418, 2.698, 3.296 }, + /* 43. */ { 1.302, 1.681, 2.017, 2.416, 2.695, 3.291 }, + /* 44. */ { 1.301, 1.680, 2.015, 2.414, 2.692, 3.286 }, + /* 45. */ { 1.301, 1.679, 2.014, 2.412, 2.690, 3.281 }, + /* 46. */ { 1.300, 1.679, 2.013, 2.410, 2.687, 3.277 }, + /* 47. */ { 1.300, 1.678, 2.012, 2.408, 2.685, 3.273 }, + /* 48. */ { 1.299, 1.677, 2.011, 2.407, 2.682, 3.269 }, + /* 49. */ { 1.299, 1.677, 2.010, 2.405, 2.680, 3.265 }, + /* 50. */ { 1.299, 1.676, 2.009, 2.403, 2.678, 3.261 }, + /* 51. */ { 1.298, 1.675, 2.008, 2.402, 2.676, 3.258 }, + /* 52. */ { 1.298, 1.675, 2.007, 2.400, 2.674, 3.255 }, + /* 53. */ { 1.298, 1.674, 2.006, 2.399, 2.672, 3.251 }, + /* 54. */ { 1.297, 1.674, 2.005, 2.397, 2.670, 3.248 }, + /* 55. */ { 1.297, 1.673, 2.004, 2.396, 2.668, 3.245 }, + /* 56. */ { 1.297, 1.673, 2.003, 2.395, 2.667, 3.242 }, + /* 57. */ { 1.297, 1.672, 2.002, 2.394, 2.665, 3.239 }, + /* 58. */ { 1.296, 1.672, 2.002, 2.392, 2.663, 3.237 }, + /* 59. */ { 1.296, 1.671, 2.001, 2.391, 2.662, 3.234 }, + /* 60. */ { 1.296, 1.671, 2.000, 2.390, 2.660, 3.232 }, + /* 61. */ { 1.296, 1.670, 2.000, 2.389, 2.659, 3.229 }, + /* 62. */ { 1.295, 1.670, 1.999, 2.388, 2.657, 3.227 }, + /* 63. */ { 1.295, 1.669, 1.998, 2.387, 2.656, 3.225 }, + /* 64. */ { 1.295, 1.669, 1.998, 2.386, 2.655, 3.223 }, + /* 65. */ { 1.295, 1.669, 1.997, 2.385, 2.654, 3.220 }, + /* 66. */ { 1.295, 1.668, 1.997, 2.384, 2.652, 3.218 }, + /* 67. */ { 1.294, 1.668, 1.996, 2.383, 2.651, 3.216 }, + /* 68. */ { 1.294, 1.668, 1.995, 2.382, 2.650, 3.214 }, + /* 69. */ { 1.294, 1.667, 1.995, 2.382, 2.649, 3.213 }, + /* 70. */ { 1.294, 1.667, 1.994, 2.381, 2.648, 3.211 }, + /* 71. */ { 1.294, 1.667, 1.994, 2.380, 2.647, 3.209 }, + /* 72. */ { 1.293, 1.666, 1.993, 2.379, 2.646, 3.207 }, + /* 73. */ { 1.293, 1.666, 1.993, 2.379, 2.645, 3.206 }, + /* 74. */ { 1.293, 1.666, 1.993, 2.378, 2.644, 3.204 }, + /* 75. */ { 1.293, 1.665, 1.992, 2.377, 2.643, 3.202 }, + /* 76. */ { 1.293, 1.665, 1.992, 2.376, 2.642, 3.201 }, + /* 77. */ { 1.293, 1.665, 1.991, 2.376, 2.641, 3.199 }, + /* 78. */ { 1.292, 1.665, 1.991, 2.375, 2.640, 3.198 }, + /* 79. */ { 1.292, 1.664, 1.990, 2.374, 2.640, 3.197 }, + /* 80. */ { 1.292, 1.664, 1.990, 2.374, 2.639, 3.195 }, + /* 81. */ { 1.292, 1.664, 1.990, 2.373, 2.638, 3.194 }, + /* 82. */ { 1.292, 1.664, 1.989, 2.373, 2.637, 3.193 }, + /* 83. */ { 1.292, 1.663, 1.989, 2.372, 2.636, 3.191 }, + /* 84. */ { 1.292, 1.663, 1.989, 2.372, 2.636, 3.190 }, + /* 85. */ { 1.292, 1.663, 1.988, 2.371, 2.635, 3.189 }, + /* 86. */ { 1.291, 1.663, 1.988, 2.370, 2.634, 3.188 }, + /* 87. */ { 1.291, 1.663, 1.988, 2.370, 2.634, 3.187 }, + /* 88. */ { 1.291, 1.662, 1.987, 2.369, 2.633, 3.185 }, + /* 89. */ { 1.291, 1.662, 1.987, 2.369, 2.632, 3.184 }, + /* 90. */ { 1.291, 1.662, 1.987, 2.368, 2.632, 3.183 }, + /* 91. */ { 1.291, 1.662, 1.986, 2.368, 2.631, 3.182 }, + /* 92. */ { 1.291, 1.662, 1.986, 2.368, 2.630, 3.181 }, + /* 93. */ { 1.291, 1.661, 1.986, 2.367, 2.630, 3.180 }, + /* 94. */ { 1.291, 1.661, 1.986, 2.367, 2.629, 3.179 }, + /* 95. */ { 1.291, 1.661, 1.985, 2.366, 2.629, 3.178 }, + /* 96. */ { 1.290, 1.661, 1.985, 2.366, 2.628, 3.177 }, + /* 97. */ { 1.290, 1.661, 1.985, 2.365, 2.627, 3.176 }, + /* 98. */ { 1.290, 1.661, 1.984, 2.365, 2.627, 3.175 }, + /* 99. */ { 1.290, 1.660, 1.984, 2.365, 2.626, 3.175 }, + /* 100. */ { 1.290, 1.660, 1.984, 2.364, 2.626, 3.174 } + }; + + std::vector data; + + bool report(UInt8 confidence_index) + { + if (data.size() != 2) /// Works for two connections only + return true; + + size_t i = (data[0].cnt - 1) + (data[1].cnt - 1); + + double t = students_table[i > 100 ? 0 : i][confidence_index]; + + double spool = (data[0].cnt - 1) * data[0].var() + (data[1].cnt - 1) * data[1].var(); + spool = sqrt(spool / i); + + double s = spool * sqrt(1.0 / data[0].cnt + 1.0 / data[1].cnt); + + double d = data[0].avg() - data[1].avg(); + + double e = t * s; + + std::cerr << '\n'; + if (fabs(d) > e) + { + std::cerr << std::setprecision(1) << "Difference at " << confidence_level[confidence_index] << "% confidence\n" << std::setprecision(6); + std::cerr << "\t" << d << " +/- " << e << "\n"; + std::cerr << "\t" << d * 100 / data[1].avg() << " +/- " << e * 100 / data[1].avg() << "\n"; + std::cerr << "\t(Student's t, pooled s = " << spool << ")\n" << std::setprecision(3); + return false; + } + else + { + std::cerr << std::setprecision(1) << "No difference proven at " << confidence_level[confidence_index] << "% confidence\n" << std::setprecision(3); + return true; + } + } + + }; + + RelativeAnalysis comparison_relative; + using MultiStats = std::vector>; MultiStats comparison_info_per_interval; MultiStats comparison_info_total; @@ -254,6 +427,9 @@ private: { printNumberOfQueriesExecuted(queries_executed); cumulative ? report(comparison_info_total) : report(comparison_info_per_interval); + + comparison_relative.report(5); + delay_watch.restart(); } } @@ -300,6 +476,7 @@ private: printNumberOfQueriesExecuted(queries_executed); report(comparison_info_total); + comparison_relative.report(5); } @@ -366,6 +543,7 @@ private: comparison_info_per_interval[connection_index]->add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); comparison_info_total[connection_index]->add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes); + comparison_relative.data[connection_index].add(seconds); } void report(MultiStats & infos) @@ -380,14 +558,16 @@ private: if (0 == info->queries) return; + double seconds = info->work_time / concurrency; + std::cerr << "connection " << info_counter++ << ", " << "queries " << info->queries << ", " - << "QPS: " << (info->queries / info->work_time) << ", " - << "RPS: " << (info->read_rows / info->work_time) << ", " - << "MiB/s: " << (info->read_bytes / info->work_time / 1048576) << ", " - << "result RPS: " << (info->result_rows / info->work_time) << ", " - << "result MiB/s: " << (info->result_bytes / info->work_time / 1048576) << "." + << "QPS: " << (info->queries / seconds) << ", " + << "RPS: " << (info->read_rows / seconds) << ", " + << "MiB/s: " << (info->read_bytes / seconds / 1048576) << ", " + << "result RPS: " << (info->result_rows / seconds) << ", " + << "result MiB/s: " << (info->result_bytes / seconds / 1048576) << "." << "\n"; } std::cerr << "\n\t\t"; @@ -415,8 +595,10 @@ private: print_percentile(99.99); if (!cumulative) + { for (auto & info : infos) info->clear(); + } } void reportJSON(MultiStats & infos, const std::string & filename) From d1ebfaacd6df35b2a6b55f25f6f9319e00c1f5ac Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 8 Aug 2019 22:28:25 +0300 Subject: [PATCH 078/357] update docs --- docs/en/operations/system_tables.md | 3 ++- docs/en/query_language/alter.md | 12 +++++++++++- docs/ru/operations/system_tables.md | 6 ++++++ docs/ru/query_language/alter.md | 11 ++++++++++- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index e63a9115270..e5eac2f1f58 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -87,13 +87,14 @@ This table contains a single String column called 'name' – the name of a datab Each database that the server knows about has a corresponding entry in the table. This system table is used for implementing the `SHOW DATABASES` query. -## system.detached_parts +## system.detached_parts {#system_tables-detached_parts} Contains information about detached parts of [MergeTree](table_engines/mergetree.md) tables. The `reason` column specifies why the part was detached. For user-detached parts, the reason is empty. Such parts can be attached with [ALTER TABLE ATTACH PARTITION|PART](../query_language/query_language/alter/#alter_attach-partition) command. For the description of other columns, see [system.parts](#system_tables-parts). +If part name is invalid, values of some columns may be `NULL`. Such parts can be deleted with [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). ## system.dictionaries diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md index 6e8e712ff30..2d42c4cc354 100644 --- a/docs/en/query_language/alter.md +++ b/docs/en/query_language/alter.md @@ -210,6 +210,16 @@ Read about setting the partition expression in a section [How to specify the par The query is replicated – it deletes data on all replicas. +#### DROP DETACHED PARTITION|PART {#alter_drop-detached} + +```sql +ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr +``` + +Removes the specified part or all parts of the specified partition from `detached`. +Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr). + + #### ATTACH PARTITION|PART {#alter_attach-partition} ``` sql @@ -327,7 +337,7 @@ You can specify the partition expression in `ALTER ... PARTITION` queries in dif - As a value from the `partition` column of the `system.parts` table. For example, `ALTER TABLE visits DETACH PARTITION 201901`. - As the expression from the table column. Constants and constant expressions are supported. For example, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. - Using the partition ID. Partition ID is a string identifier of the partition (human-readable, if possible) that is used as the names of partitions in the file system and in ZooKeeper. The partition ID must be specified in the `PARTITION ID` clause, in a single quotes. For example, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- In the [ALTER ATTACH PART](#alter_attach-partition) query, to specify the name of a part, use a value from the `name` column of the `system.parts` table. For example, `ALTER TABLE visits ATTACH PART 201901_1_1_0`. +- In the [ALTER ATTACH PART](#alter_attach-partition) and [DROP DETACHED PART](#alter_drop-detached) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached_parts](../operations/system_tables.md#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. Usage of quotes when specifying the partition depends on the type of partition expression. For example, for the `String` type, you have to specify its name in quotes (`'`). For the `Date` and `Int*` types no quotes are needed. diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index eb452c8de4e..4345f83718b 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -47,6 +47,12 @@ default_expression String - выражение для значения по ум Для каждой базы данных, о которой знает сервер, будет присутствовать соответствующая запись в таблице. Эта системная таблица используется для реализации запроса `SHOW DATABASES`. +## system.detached_parts {#system_tables-detached_parts} + +Сожелржит информацию об отсоединённых кусках таблиц семейства [MergeTree](table_engines/mergetree.md). Столбец `reason` содержит причину, по которой кусок был отсоединён. Для кусов, отсоединённых пользователем, `reason` содержит пустую строку. +Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION|PART](../query_language/query_language/alter/#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). +Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). + ## system.dictionaries Содержит информацию о внешних словарях. diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md index 2367386172a..3e0030e948e 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/query_language/alter.md @@ -209,6 +209,15 @@ ALTER TABLE table_name DROP PARTITION partition_expr Запрос реплицируется — данные будут удалены на всех репликах. +#### DROP DETACHED PARTITION|PART {#alter_drop-detached} + +```sql +ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr +``` + +Удаляет из `detached` кусок или все куски, принадлежащие партиции. +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + #### ATTACH PARTITION|PART {#alter_attach-partition} ```sql @@ -328,7 +337,7 @@ ALTER TABLE users ATTACH PARTITION 201902; - Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../operations/system_tables.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. - Произвольное выражение из столбцов исходной таблицы. Также поддерживаются константы и константные выражения. Например, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. - Строковый идентификатор партиции. Идентификатор партиции используется для именования кусков партиции на файловой системе и в ZooKeeper. В запросах `ALTER` идентификатор партиции нужно указывать в секции `PARTITION ID`, в одинарных кавычках. Например, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- Для запросов [ATTACH PART](#alter_attach-partition): чтобы задать имя куска партиции, используйте значение из столбца `name` системной таблицы `system.parts`. Например, `ALTER TABLE visits ATTACH PART 201901_1_1_0`. +- Для запросов [ATTACH PART](#alter_attach-partition) и [DROP DETACHED PART](#alter_drop-detached): чтобы задать имя куска партиции, используйте строковой литерал со значением из столбца `name` системной таблицы [system.detached_parts](../operations/system_tables.md#system_tables-detached_parts). Например, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. Использование кавычек в имени партиций зависит от типа данных столбца, по которому задано партиционирование. Например, для столбца с типом `String` имя партиции необходимо указывать в кавычках (одинарных). Для типов `Date` и `Int*` кавычки указывать не нужно. From a03fcd9f121ac5ad655a4e714175266213d6666f Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 8 Aug 2019 22:29:56 +0300 Subject: [PATCH 079/357] Make settings values atomic to avoid race conditions --- dbms/programs/copier/ClusterCopier.cpp | 2 +- dbms/src/Core/SettingsCommon.cpp | 26 +++++--- dbms/src/Core/SettingsCommon.h | 63 +++++++++++++++---- .../LogicalExpressionsOptimizer.cpp | 4 +- .../LogicalExpressionsOptimizer.h | 2 +- dbms/src/Interpreters/SyntaxAnalyzer.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 +- .../Storages/MergeTree/MergeTreeSettings.cpp | 2 +- 8 files changed, 75 insertions(+), 30 deletions(-) diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 43158dedd71..0c383ace576 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -708,7 +708,7 @@ void DB::TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & c auto set_default_value = [] (auto && setting, auto && default_value) { - setting = setting.changed ? setting.value : default_value; + setting = setting.changed ? setting.getValue() : default_value; }; /// Override important settings diff --git a/dbms/src/Core/SettingsCommon.cpp b/dbms/src/Core/SettingsCommon.cpp index 988a4b2d736..639e1dea3ee 100644 --- a/dbms/src/Core/SettingsCommon.cpp +++ b/dbms/src/Core/SettingsCommon.cpp @@ -34,19 +34,19 @@ namespace ErrorCodes template String SettingNumber::toString() const { - return DB::toString(value); + return DB::toString(value.load(std::memory_order_relaxed)); } template Field SettingNumber::toField() const { - return value; + return value.load(std::memory_order_relaxed); } template void SettingNumber::set(Type x) { - value = x; + value.store(x, std::memory_order_relaxed); changed = true; } @@ -136,17 +136,17 @@ template struct SettingNumber; String SettingMaxThreads::toString() const { /// Instead of the `auto` value, we output the actual value to make it easier to see. - return is_auto ? ("auto(" + DB::toString(value) + ")") : DB::toString(value); + return is_auto ? ("auto(" + DB::toString(getValue()) + ")") : DB::toString(getValue()); } Field SettingMaxThreads::toField() const { - return is_auto ? 0 : value; + return is_auto ? 0 : getValue(); } void SettingMaxThreads::set(UInt64 x) { - value = x ? x : getAutoValue(); + value.store(x ? x : getAutoValue(), std::memory_order_relaxed); is_auto = x == 0; changed = true; } @@ -169,7 +169,7 @@ void SettingMaxThreads::set(const String & x) void SettingMaxThreads::serialize(WriteBuffer & buf) const { - writeVarUInt(is_auto ? 0 : value, buf); + writeVarUInt(is_auto ? 0 : getValue(), buf); } void SettingMaxThreads::deserialize(ReadBuffer & buf) @@ -195,18 +195,21 @@ UInt64 SettingMaxThreads::getAutoValue() const template String SettingTimespan::toString() const { + std::lock_guard lock(m); return DB::toString(value.totalMicroseconds() / microseconds_per_io_unit); } template Field SettingTimespan::toField() const { + std::lock_guard lock(m); return value.totalMicroseconds() / microseconds_per_io_unit; } template void SettingTimespan::set(const Poco::Timespan & x) { + std::lock_guard lock(m); value = x; changed = true; } @@ -235,6 +238,7 @@ void SettingTimespan::set(const String & x) template void SettingTimespan::serialize(WriteBuffer & buf) const { + std::lock_guard lock(m); writeVarUInt(value.totalMicroseconds() / microseconds_per_io_unit, buf); } @@ -252,16 +256,19 @@ template struct SettingTimespan; String SettingString::toString() const { + std::lock_guard lock(m); return value; } Field SettingString::toField() const { + std::lock_guard lock(m); return value; } void SettingString::set(const String & x) { + std::lock_guard lock(m); value = x; changed = true; } @@ -273,6 +280,7 @@ void SettingString::set(const Field & x) void SettingString::serialize(WriteBuffer & buf) const { + std::lock_guard lock(m); writeBinary(value, buf); } @@ -296,7 +304,7 @@ Field SettingChar::toField() const void SettingChar::set(char x) { - value = x; + value.store(x, std::memory_order_relaxed); changed = true; } @@ -351,7 +359,7 @@ void SettingEnum::deserialize(ReadBuffer & buf) { \ using EnumType = ENUM_NAME; \ using UnderlyingType = std::underlying_type::type; \ - switch (static_cast(value)) \ + switch (static_cast(getValue())) \ { \ LIST_OF_NAMES_MACRO(IMPLEMENT_SETTING_ENUM_TO_STRING_HELPER_) \ } \ diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index d4607e70904..46bf5bca5ce 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -32,13 +33,23 @@ namespace ErrorCodes template struct SettingNumber { - Type value; +public: + /// The value is atomic, because we want to avoid race conditions on value precisely. + /// It doesn't gurantee atomicy on whole structure. It just helps to avoid the most common + /// case: when we change setting from one thread and use in another (for example in MergeTreeSettings). + /// + std::atomic value; + bool changed = false; SettingNumber(Type x = 0) : value(x) {} + SettingNumber(const SettingNumber & o) : value(o.getValue()), changed(o.changed) { } + + operator Type() const { return getValue(); } + Type getValue() const { return value.load(std::memory_order_relaxed); } - operator Type() const { return value; } SettingNumber & operator= (Type x) { set(x); return *this; } + SettingNumber & operator= (SettingNumber o) { set(o.getValue()); return *this; } /// Serialize to a test string. String toString() const; @@ -73,14 +84,23 @@ using SettingBool = SettingNumber; */ struct SettingMaxThreads { - UInt64 value; + std::atomic value; bool is_auto; bool changed = false; SettingMaxThreads(UInt64 x = 0) : value(x ? x : getAutoValue()), is_auto(x == 0) {} + SettingMaxThreads(const SettingMaxThreads & o) + : is_auto(o.is_auto) + , changed(o.changed) + { + value.store(o.value, std::memory_order_relaxed); + } + + operator UInt64() const { return getValue(); } + UInt64 getValue() const { return value.load(std::memory_order_relaxed); } - operator UInt64() const { return value; } SettingMaxThreads & operator= (UInt64 x) { set(x); return *this; } + SettingMaxThreads & operator= (const SettingMaxThreads & o) { set(o.getValue()); return *this; } String toString() const; Field toField() const; @@ -102,16 +122,20 @@ enum class SettingTimespanIO { MILLISECOND, SECOND }; template struct SettingTimespan { + mutable std::mutex m; Poco::Timespan value; bool changed = false; SettingTimespan(UInt64 x = 0) : value(x * microseconds_per_io_unit) {} + SettingTimespan(const SettingTimespan & o) : value(o.value), changed(o.changed) {} - operator Poco::Timespan() const { return value; } + operator Poco::Timespan() const { return getValue(); } + Poco::Timespan getValue() const { std::lock_guard guard(m); return value; } SettingTimespan & operator= (const Poco::Timespan & x) { set(x); return *this; } + SettingTimespan & operator= (const SettingTimespan & o) { set(o.value); return *this; } - Poco::Timespan::TimeDiff totalSeconds() const { return value.totalSeconds(); } - Poco::Timespan::TimeDiff totalMilliseconds() const { return value.totalMilliseconds(); } + Poco::Timespan::TimeDiff totalSeconds() const { return getValue().totalSeconds(); } + Poco::Timespan::TimeDiff totalMilliseconds() const { return getValue().totalMilliseconds(); } String toString() const; Field toField() const; @@ -134,13 +158,18 @@ using SettingMilliseconds = SettingTimespan; struct SettingString { + mutable std::mutex m; String value; bool changed = false; SettingString(const String & x = String{}) : value(x) {} + SettingString(const SettingString & o) : value(o.value), changed(o.changed) {} + + operator String() const { return getValue(); } + String getValue() const { std::lock_guard guard(m); return value; } - operator String() const { return value; } SettingString & operator= (const String & x) { set(x); return *this; } + SettingString & operator= (const SettingString & o) { set(o.value); return *this; } String toString() const; Field toField() const; @@ -156,13 +185,15 @@ struct SettingString struct SettingChar { public: - char value; + std::atomic value; bool changed = false; SettingChar(char x = '\0') : value(x) {} + SettingChar(const SettingChar & o) : value(o.getValue()), changed(o.changed) { } - operator char() const { return value; } + operator char() const { return getValue(); } SettingChar & operator= (char x) { set(x); return *this; } + SettingChar & operator= (const SettingChar & o) { set(o.getValue()); return *this; } String toString() const; Field toField() const; @@ -173,6 +204,8 @@ public: void serialize(WriteBuffer & buf) const; void deserialize(ReadBuffer & buf); + + char getValue() const { return value.load(std::memory_order_relaxed); } }; @@ -180,23 +213,27 @@ public: template struct SettingEnum { - EnumType value; + std::atomic value; bool changed = false; SettingEnum(EnumType x) : value(x) {} + SettingEnum(const SettingEnum & o) : value(o.getValue()), changed(o.changed) { } - operator EnumType() const { return value; } + operator EnumType() const { return getValue(); } SettingEnum & operator= (EnumType x) { set(x); return *this; } + SettingEnum & operator= (SettingEnum o) { set(o.getValue()); return *this; } String toString() const; Field toField() const { return toString(); } - void set(EnumType x) { value = x; changed = true; } + void set(EnumType x) { value.store(x, std::memory_order_relaxed); changed = true; } void set(const Field & x) { set(safeGet(x)); } void set(const String & x); void serialize(WriteBuffer & buf) const; void deserialize(ReadBuffer & buf); + + EnumType getValue() const { return value.load(std::memory_order_relaxed); } }; diff --git a/dbms/src/Interpreters/LogicalExpressionsOptimizer.cpp b/dbms/src/Interpreters/LogicalExpressionsOptimizer.cpp index 1e00dd499ba..ca8fbb1c8f4 100644 --- a/dbms/src/Interpreters/LogicalExpressionsOptimizer.cpp +++ b/dbms/src/Interpreters/LogicalExpressionsOptimizer.cpp @@ -30,8 +30,8 @@ bool LogicalExpressionsOptimizer::OrWithExpression::operator<(const OrWithExpres return std::tie(this->or_function, this->expression) < std::tie(rhs.or_function, rhs.expression); } -LogicalExpressionsOptimizer::LogicalExpressionsOptimizer(ASTSelectQuery * select_query_, ExtractedSettings && settings_) - : select_query(select_query_), settings(settings_) +LogicalExpressionsOptimizer::LogicalExpressionsOptimizer(ASTSelectQuery * select_query_, UInt64 optimize_min_equality_disjunction_chain_length) + : select_query(select_query_), settings(optimize_min_equality_disjunction_chain_length) { } diff --git a/dbms/src/Interpreters/LogicalExpressionsOptimizer.h b/dbms/src/Interpreters/LogicalExpressionsOptimizer.h index 09c3931ce1d..fa5289b3f5f 100644 --- a/dbms/src/Interpreters/LogicalExpressionsOptimizer.h +++ b/dbms/src/Interpreters/LogicalExpressionsOptimizer.h @@ -36,7 +36,7 @@ class LogicalExpressionsOptimizer final public: /// Constructor. Accepts the root of the query DAG. - LogicalExpressionsOptimizer(ASTSelectQuery * select_query_, ExtractedSettings && settings_); + LogicalExpressionsOptimizer(ASTSelectQuery * select_query_, UInt64 optimize_min_equality_disjunction_chain_length); /** Replace all rather long homogeneous OR-chains expr = x1 OR ... OR expr = xN * on the expressions `expr` IN (x1, ..., xN). diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index 48f40d17cb3..464ef149e35 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -596,7 +596,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( InJoinSubqueriesPreprocessor(context).visit(query); /// Optimizes logical expressions. - LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.value).perform(); + LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length).perform(); } /// Creates a dictionary `aliases`: alias -> ASTPtr diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ccbcd04857d..513a713651b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -400,8 +400,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( if (relative_sample_size == RelativeSize(0)) relative_sample_size = 1; - relative_sample_size /= settings.parallel_replicas_count.value; - relative_sample_offset += relative_sample_size * RelativeSize(settings.parallel_replica_offset.value); + relative_sample_size /= settings.parallel_replicas_count.getValue(); + relative_sample_offset += relative_sample_size * RelativeSize(settings.parallel_replica_offset.getValue()); } if (relative_sample_offset >= RelativeSize(1)) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index 9eee33554ab..425a0851d67 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -69,7 +69,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) if (std::find_if(changes.begin(), changes.end(), \ [](const SettingChange & c) { return c.name == #NAME; }) \ == changes.end()) \ - changes.push_back(SettingChange{#NAME, NAME.value}); + changes.push_back(SettingChange{#NAME, NAME.getValue()}); APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(ADD_IF_ABSENT) #undef ADD_IF_ABSENT From 50cabe4ab0b5cfc492a88bed1ddfd8ebef1dd723 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 8 Aug 2019 22:30:32 +0300 Subject: [PATCH 080/357] Comment --- dbms/src/Core/SettingsCommon.h | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index 46bf5bca5ce..0939ccac7ca 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -37,7 +37,6 @@ public: /// The value is atomic, because we want to avoid race conditions on value precisely. /// It doesn't gurantee atomicy on whole structure. It just helps to avoid the most common /// case: when we change setting from one thread and use in another (for example in MergeTreeSettings). - /// std::atomic value; bool changed = false; From b2cbc1a747b520ce1157f03ffaad6b60c2079aa3 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Fri, 9 Aug 2019 13:46:57 +0300 Subject: [PATCH 081/357] Fixed some const's and added --confidence option. --- dbms/programs/benchmark/Benchmark.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index e97a499101b..1e39dc6889d 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -55,11 +55,12 @@ public: bool cumulative_, bool secure_, const String & default_database_, const String & user_, const String & password_, const String & stage, bool randomize_, size_t max_iterations_, double max_time_, - const String & json_path_, const Settings & settings_) + const String & json_path_, size_t confidence_, const Settings & settings_) : concurrency(concurrency_), delay(delay_), queue(concurrency), randomize(randomize_), cumulative(cumulative_), max_iterations(max_iterations_), max_time(max_time_), - json_path(json_path_), settings(settings_), global_context(Context::createGlobal()), pool(concurrency) + confidence(confidence_), json_path(json_path_), settings(settings_), + global_context(Context::createGlobal()), pool(concurrency) { const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; size_t connections_cnt = std::max(ports_.size(), hosts_.size()); @@ -139,6 +140,7 @@ private: bool cumulative; size_t max_iterations; double max_time; + size_t confidence; String json_path; Settings settings; Context global_context; @@ -199,20 +201,20 @@ private: squares_sum += seconds * seconds; } - double avg() + double avg() const { return sum / cnt; } - double var() + double var() const { return (squares_sum - (sum * sum / cnt)) / static_cast(cnt - 1); } }; - std::vector confidence_level = { 80, 90, 95, 98, 99, 99.5 }; + const std::vector confidence_level = { 80, 90, 95, 98, 99, 99.5 }; - std::vector> students_table = { + const std::vector> students_table = { /* inf */ { 1.282, 1.645, 1.960, 2.326, 2.576, 3.090 }, /* 1. */ { 3.078, 6.314, 12.706, 31.821, 63.657, 318.313 }, /* 2. */ { 1.886, 2.920, 4.303, 6.965, 9.925, 22.327 }, @@ -318,7 +320,7 @@ private: std::vector data; - bool report(UInt8 confidence_index) + bool report(size_t confidence_index) { if (data.size() != 2) /// Works for two connections only return true; @@ -427,8 +429,7 @@ private: { printNumberOfQueriesExecuted(queries_executed); cumulative ? report(comparison_info_total) : report(comparison_info_per_interval); - - comparison_relative.report(5); + comparison_relative.report(confidence); delay_watch.restart(); } @@ -476,7 +477,7 @@ private: printNumberOfQueriesExecuted(queries_executed); report(comparison_info_total); - comparison_relative.report(5); + comparison_relative.report(confidence); } @@ -693,6 +694,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ("password", value()->default_value(""), "") ("database", value()->default_value("default"), "") ("stacktrace", "print stack traces of exceptions") + ("confidence", value()->default_value(5), "set the level of confidence for T-test [0=80%, 1=90%, 2=95%, 3=98%, 4=99%, 5=99.5%(default)") ; Settings settings; @@ -728,6 +730,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) options["iterations"].as(), options["timelimit"].as(), options["json"].as(), + options["confidence"].as(), settings); return benchmark.run(); } From 4c85e9f4814e117e155e234ce64978b804fdff31 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 9 Aug 2019 14:17:33 +0300 Subject: [PATCH 082/357] atomic drop table --- dbms/src/Databases/DatabaseOrdinary.cpp | 27 ++++++++++++- .../src/Interpreters/InterpreterDropQuery.cpp | 21 ++++++++-- .../test_atomic_drop_table/__init__.py | 0 .../config.d/zookeeper_session_timeout.xml | 6 +++ .../configs/remote_servers.xml | 14 +++++++ .../test_atomic_drop_table/test.py | 39 +++++++++++++++++++ 6 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 dbms/tests/integration/test_atomic_drop_table/__init__.py create mode 100644 dbms/tests/integration/test_atomic_drop_table/configs/config.d/zookeeper_session_timeout.xml create mode 100644 dbms/tests/integration/test_atomic_drop_table/configs/remote_servers.xml create mode 100644 dbms/tests/integration/test_atomic_drop_table/test.py diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 25b3eb652b5..0bf703887ef 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -135,7 +135,25 @@ void DatabaseOrdinary::loadTables( if (endsWith(dir_it.name(), ".sql.bak")) continue; - /// There are files .sql.tmp - delete. + // There are files that we tried to delete previously + const std::string tmp_drop_ext = ".sql.tmp_drop"; + if (endsWith(dir_it.name(), ".sql.tmp_drop")) + { + const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - tmp_drop_ext.size()); + if (Poco::File(data_path + '/' + table_name).exists()) + { + Poco::File(dir_it->path()).renameTo(table_name + ".sql"); + LOG_WARNING(log, "Table was not dropped previously"); + } + else + { + LOG_INFO(log, "Removing file " << dir_it->path()); + Poco::File(dir_it->path()).remove(); + } + continue; + } + + /// There are files .sql.tmp and .sql.tmp_drop - delete if (endsWith(dir_it.name(), ".sql.tmp")) { LOG_INFO(log, "Removing file " << dir_it->path()); @@ -302,6 +320,12 @@ void DatabaseOrdinary::removeTable( } catch (...) { + try + { + Poco::File(table_metadata_path + ".tmp_drop").remove(); + return; + } + catch (...) {} attachTable(table_name, res); throw; } @@ -363,7 +387,6 @@ void DatabaseOrdinary::renameTable( throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED); StoragePtr table = tryGetTable(context, table_name); - if (!table) throw Exception("Table " + name + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 91213b6100e..13911d930e1 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -90,11 +90,26 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t /// If table was already dropped by anyone, an exception will be thrown auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId()); - /// Delete table metadata and table itself from memory + const auto prev_metadata_name = database_and_table.first->getMetadataPath() + database_and_table.second->getTableName() + ".sql"; + const auto drop_metadata_name = database_and_table.first->getMetadataPath() + database_and_table.second->getTableName() + ".sql.tmp_drop"; + + try + { + Poco::File(prev_metadata_name).renameTo(drop_metadata_name); + std::cout << "RENAMED" << std::endl; + /// Delete table data + database_and_table.second->drop(); + } + catch (...) + { + Poco::File(drop_metadata_name).renameTo(prev_metadata_name); + std::cout << "RENAMED BACK" << std::endl; + throw; + } + + /// Delete table metadata and table itself from memory database_and_table.first->removeTable(context, database_and_table.second->getTableName()); - /// Delete table data - database_and_table.second->drop(); database_and_table.second->is_dropped = true; String database_data_path = database_and_table.first->getDataPath(); diff --git a/dbms/tests/integration/test_atomic_drop_table/__init__.py b/dbms/tests/integration/test_atomic_drop_table/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_atomic_drop_table/configs/config.d/zookeeper_session_timeout.xml b/dbms/tests/integration/test_atomic_drop_table/configs/config.d/zookeeper_session_timeout.xml new file mode 100644 index 00000000000..071725b5391 --- /dev/null +++ b/dbms/tests/integration/test_atomic_drop_table/configs/config.d/zookeeper_session_timeout.xml @@ -0,0 +1,6 @@ + + + + 3000 + + diff --git a/dbms/tests/integration/test_atomic_drop_table/configs/remote_servers.xml b/dbms/tests/integration/test_atomic_drop_table/configs/remote_servers.xml new file mode 100644 index 00000000000..538aa72d386 --- /dev/null +++ b/dbms/tests/integration/test_atomic_drop_table/configs/remote_servers.xml @@ -0,0 +1,14 @@ + + + + + true + + shard_0 + node1 + 9000 + + + + + diff --git a/dbms/tests/integration/test_atomic_drop_table/test.py b/dbms/tests/integration/test_atomic_drop_table/test.py new file mode 100644 index 00000000000..4098cce446b --- /dev/null +++ b/dbms/tests/integration/test_atomic_drop_table/test.py @@ -0,0 +1,39 @@ +import time +import pytest + +from helpers.network import PartitionManager +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', config_dir="configs", with_zookeeper=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + node1.query("CREATE DATABASE zktest;") + node1.query( + ''' + CREATE TABLE zktest.atomic_drop_table (n UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/zktest/tables/atomic_drop_table', 'node1') + PARTITION BY n ORDER BY n + ''' + ) + yield cluster + finally: + cluster.shutdown() + +def test_atomic_delete_with_stopped_zookeeper(start_cluster): + + node1.query("select * from zktest.atomic_drop_table") + node1.query("insert into zktest.atomic_drop_table values (8192)") + + with PartitionManager() as pm: + pm.drop_instance_zk_connections(node1) + error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") + assert error != "" + + time.sleep(5) + assert '8192' in node1.query("select * from zktest.atomic_drop_table") From 596a3fe1a88b98c711deb898c28642403c7454af Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 9 Aug 2019 15:54:47 +0300 Subject: [PATCH 083/357] better --- dbms/src/Interpreters/InterpreterDropQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 13911d930e1..965aa2cc997 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -96,7 +96,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t try { - Poco::File(prev_metadata_name).renameTo(drop_metadata_name); + Poco::File(prev_metadata_name).renameTo(drop_metadata_name);git std::cout << "RENAMED" << std::endl; /// Delete table data database_and_table.second->drop(); From 636793fd890a59a445dc54d4678063abbfe9f499 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 9 Aug 2019 15:56:19 +0300 Subject: [PATCH 084/357] better --- dbms/src/Interpreters/InterpreterDropQuery.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 965aa2cc997..7a26bc3c740 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -96,15 +96,13 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t try { - Poco::File(prev_metadata_name).renameTo(drop_metadata_name);git - std::cout << "RENAMED" << std::endl; + Poco::File(prev_metadata_name).renameTo(drop_metadata_name); /// Delete table data database_and_table.second->drop(); } catch (...) { Poco::File(drop_metadata_name).renameTo(prev_metadata_name); - std::cout << "RENAMED BACK" << std::endl; throw; } From e3579f215f7ef75b7ab8fd53a4b2546967608129 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 9 Aug 2019 15:58:46 +0300 Subject: [PATCH 085/357] comment to test --- dbms/tests/integration/test_atomic_drop_table/test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dbms/tests/integration/test_atomic_drop_table/test.py b/dbms/tests/integration/test_atomic_drop_table/test.py index 4098cce446b..7d845baeec6 100644 --- a/dbms/tests/integration/test_atomic_drop_table/test.py +++ b/dbms/tests/integration/test_atomic_drop_table/test.py @@ -26,13 +26,11 @@ def start_cluster(): cluster.shutdown() def test_atomic_delete_with_stopped_zookeeper(start_cluster): - - node1.query("select * from zktest.atomic_drop_table") node1.query("insert into zktest.atomic_drop_table values (8192)") with PartitionManager() as pm: pm.drop_instance_zk_connections(node1) - error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") + error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") #Table won't drop assert error != "" time.sleep(5) From 9d2c9c8c2a9076880dcaa76748207a41b58433aa Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Fri, 9 Aug 2019 16:00:47 +0300 Subject: [PATCH 086/357] Minor style and logic fixes --- dbms/programs/benchmark/Benchmark.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 1e39dc6889d..e172ebc710a 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -216,9 +216,9 @@ private: const std::vector> students_table = { /* inf */ { 1.282, 1.645, 1.960, 2.326, 2.576, 3.090 }, - /* 1. */ { 3.078, 6.314, 12.706, 31.821, 63.657, 318.313 }, - /* 2. */ { 1.886, 2.920, 4.303, 6.965, 9.925, 22.327 }, - /* 3. */ { 1.638, 2.353, 3.182, 4.541, 5.841, 10.215 }, + /* 1. */ { 3.078, 6.314, 12.706, 31.821, 63.657, 318.313}, + /* 2. */ { 1.886, 2.920, 4.303, 6.965, 9.925, 22.327 }, + /* 3. */ { 1.638, 2.353, 3.182, 4.541, 5.841, 10.215 }, /* 4. */ { 1.533, 2.132, 2.776, 3.747, 4.604, 7.173 }, /* 5. */ { 1.476, 2.015, 2.571, 3.365, 4.032, 5.893 }, /* 6. */ { 1.440, 1.943, 2.447, 3.143, 3.707, 5.208 }, @@ -353,7 +353,6 @@ private: return true; } } - }; RelativeAnalysis comparison_relative; @@ -552,9 +551,10 @@ private: std::lock_guard lock(mutex); std::cerr << "\n"; - size_t info_counter = 1; - for (auto & info : infos) + for (size_t i = 1; i <= infos.size(); ++i) { + auto & info = infos[i - 1]; + /// Avoid zeros, nans or exceptions if (0 == info->queries) return; @@ -562,7 +562,7 @@ private: double seconds = info->work_time / concurrency; std::cerr - << "connection " << info_counter++ << ", " + << "connection " << i << ", " << "queries " << info->queries << ", " << "QPS: " << (info->queries / seconds) << ", " << "RPS: " << (info->read_rows / seconds) << ", " @@ -571,6 +571,7 @@ private: << "result MiB/s: " << (info->result_bytes / seconds / 1048576) << "." << "\n"; } + std::cerr << "\n\t\t"; for (size_t i = 1; i <= infos.size(); ++i) @@ -622,7 +623,7 @@ private: for (size_t i = 1; i <= infos.size(); ++i) { - auto info = infos[i - 1]; + auto & info = infos[i - 1]; json_out << double_quote << "connection_" + toString(i) << ": {\n"; json_out << double_quote << "statistics" << ": {\n"; From 2803fcc2bab351348672b20c95c380fa5703cd9b Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 9 Aug 2019 16:02:01 +0300 Subject: [PATCH 087/357] Make atomic settings --- CMakeLists.txt | 2 +- dbms/programs/client/Client.cpp | 2 +- dbms/programs/copier/ClusterCopier.cpp | 2 +- dbms/programs/server/TCPHandler.cpp | 6 +- dbms/src/Core/SettingsCommon.cpp | 135 ++++++++++++---- dbms/src/Core/SettingsCommon.h | 151 +++++++++++------- .../ClusterProxy/executeQuery.cpp | 6 +- dbms/src/Interpreters/SyntaxAnalyzer.cpp | 2 +- dbms/src/Interpreters/ThreadStatusExt.cpp | 2 +- dbms/src/Storages/Kafka/StorageKafka.cpp | 18 +-- .../ReplicatedMergeTreeCleanupThread.cpp | 8 +- dbms/src/Storages/StorageDistributed.cpp | 2 +- dbms/src/Storages/StorageJoin.cpp | 4 +- 13 files changed, 230 insertions(+), 110 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e03af27cfa..c0fbdd51f0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -350,7 +350,7 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L endif () # Add Libc. GLIBC is actually a collection of interdependent libraries. - set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc") + set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc -latomic") # Note: we'd rather use Musl libc library, but it's little bit more difficult to use. diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 091a1ac063f..509e5393c34 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -323,7 +323,7 @@ private: insert_format = "Values"; /// Setting value from cmd arg overrides one from config - if (context.getSettingsRef().max_insert_block_size.changed) + if (context.getSettingsRef().max_insert_block_size.isChanged()) insert_format_max_block_size = context.getSettingsRef().max_insert_block_size; else insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 0c383ace576..a63cd54a003 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -708,7 +708,7 @@ void DB::TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & c auto set_default_value = [] (auto && setting, auto && default_value) { - setting = setting.changed ? setting.getValue() : default_value; + setting = setting.isChanged() ? setting.getValue() : default_value; }; /// Override important settings diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 5091258acaf..c20a466e5b5 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -182,11 +182,11 @@ void TCPHandler::runImpl() /// Should we send internal logs to client? const auto client_logs_level = query_context->getSettingsRef().send_logs_level; if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS - && client_logs_level.value != LogsLevel::none) + && client_logs_level != LogsLevel::none) { state.logs_queue = std::make_shared(); state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); - CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level.value); + CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level); } query_context->setExternalTablesInitializer([&connection_settings, this] (Context & context) @@ -329,7 +329,7 @@ void TCPHandler::readData(const Settings & connection_settings) const auto receive_timeout = query_context->getSettingsRef().receive_timeout.value; /// Poll interval should not be greater than receive_timeout - const size_t default_poll_interval = connection_settings.poll_interval.value * 1000000; + const size_t default_poll_interval = connection_settings.poll_interval * 1000000; size_t current_poll_interval = static_cast(receive_timeout.totalMicroseconds()); constexpr size_t min_poll_interval = 5000; // 5 ms size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval)); diff --git a/dbms/src/Core/SettingsCommon.cpp b/dbms/src/Core/SettingsCommon.cpp index 639e1dea3ee..2d3ed4f6e14 100644 --- a/dbms/src/Core/SettingsCommon.cpp +++ b/dbms/src/Core/SettingsCommon.cpp @@ -34,20 +34,19 @@ namespace ErrorCodes template String SettingNumber::toString() const { - return DB::toString(value.load(std::memory_order_relaxed)); + return DB::toString(getValue()); } template Field SettingNumber::toField() const { - return value.load(std::memory_order_relaxed); + return getValue(); } template void SettingNumber::set(Type x) { - value.store(x, std::memory_order_relaxed); - changed = true; + data.store(Data{x, true}, std::memory_order_relaxed); } template @@ -59,6 +58,14 @@ void SettingNumber::set(const Field & x) set(applyVisitor(FieldVisitorConvertToNumber(), x)); } + +template +SettingNumber & SettingNumber::operator= (const SettingNumber & o) +{ + data.store(o.data.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; +} + template void SettingNumber::set(const String & x) { @@ -93,9 +100,9 @@ template void SettingNumber::serialize(WriteBuffer & buf) const { if constexpr (std::is_integral_v && std::is_unsigned_v) - writeVarUInt(static_cast(value), buf); + writeVarUInt(static_cast(getValue()), buf); else if constexpr (std::is_integral_v && std::is_signed_v) - writeVarInt(static_cast(value), buf); + writeVarInt(static_cast(getValue()), buf); else { static_assert(std::is_floating_point_v); @@ -133,22 +140,28 @@ template struct SettingNumber; template struct SettingNumber; +SettingMaxThreads & SettingMaxThreads::operator= (const SettingMaxThreads & o) +{ + data.store(o.data.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; +} + String SettingMaxThreads::toString() const { + auto d = data.load(std::memory_order_relaxed); /// Instead of the `auto` value, we output the actual value to make it easier to see. - return is_auto ? ("auto(" + DB::toString(getValue()) + ")") : DB::toString(getValue()); + return d.is_auto ? ("auto(" + DB::toString(d.value) + ")") : DB::toString(d.value); } Field SettingMaxThreads::toField() const { - return is_auto ? 0 : getValue(); + auto d = data.load(std::memory_order_relaxed); + return d.is_auto ? 0 : d.value; } void SettingMaxThreads::set(UInt64 x) { - value.store(x ? x : getAutoValue(), std::memory_order_relaxed); - is_auto = x == 0; - changed = true; + data.store({x ? x : getAutoValue(), x == 0, true}); } void SettingMaxThreads::set(const Field & x) @@ -169,7 +182,8 @@ void SettingMaxThreads::set(const String & x) void SettingMaxThreads::serialize(WriteBuffer & buf) const { - writeVarUInt(is_auto ? 0 : getValue(), buf); + auto d = data.load(std::memory_order_relaxed); + writeVarUInt(d.is_auto ? 0 : d.value, buf); } void SettingMaxThreads::deserialize(ReadBuffer & buf) @@ -181,8 +195,7 @@ void SettingMaxThreads::deserialize(ReadBuffer & buf) void SettingMaxThreads::setAuto() { - value = getAutoValue(); - is_auto = true; + data.store({getAutoValue(), true, isChanged()}); } UInt64 SettingMaxThreads::getAutoValue() const @@ -191,25 +204,54 @@ UInt64 SettingMaxThreads::getAutoValue() const return res; } +void SettingMaxThreads::setChanged(bool changed) +{ + auto d = data.load(std::memory_order_relaxed); + data.store({d.value, d.is_auto, changed}); +} + + +template +SettingTimespan & SettingTimespan::operator= (const SettingTimespan & o) +{ + std::shared_lock lock_o(o.mutex); + value = o.value; + changed = o.changed; + return *this; +} + +template +SettingTimespan::SettingTimespan(const SettingTimespan & o) +{ + std::shared_lock lock_o(o.mutex); + value = o.value; + changed = o.changed; +} + + +template +void SettingTimespan::setChanged(bool c) +{ + std::unique_lock lock(mutex); + changed = c; +} template String SettingTimespan::toString() const { - std::lock_guard lock(m); - return DB::toString(value.totalMicroseconds() / microseconds_per_io_unit); + return DB::toString(getValue().totalMicroseconds() / microseconds_per_io_unit); } template Field SettingTimespan::toField() const { - std::lock_guard lock(m); - return value.totalMicroseconds() / microseconds_per_io_unit; + return getValue().totalMicroseconds() / microseconds_per_io_unit; } template void SettingTimespan::set(const Poco::Timespan & x) { - std::lock_guard lock(m); + std::unique_lock lock(mutex); value = x; changed = true; } @@ -238,8 +280,7 @@ void SettingTimespan::set(const String & x) template void SettingTimespan::serialize(WriteBuffer & buf) const { - std::lock_guard lock(m); - writeVarUInt(value.totalMicroseconds() / microseconds_per_io_unit, buf); + writeVarUInt(getValue().totalMicroseconds() / microseconds_per_io_unit, buf); } template @@ -253,26 +294,47 @@ void SettingTimespan::deserialize(ReadBuffer & buf) template struct SettingTimespan; template struct SettingTimespan; +SettingString & SettingString::operator= (const SettingString & o) +{ + std::shared_lock lock_o(o.mutex); + value = o.value; + changed = o.changed; + return *this; +} + +SettingString::SettingString(const SettingString & o) +{ + std::shared_lock lock(o.mutex); + value = o.value; + changed = o.changed; +} + String SettingString::toString() const { - std::lock_guard lock(m); + std::shared_lock lock(mutex); return value; } Field SettingString::toField() const { - std::lock_guard lock(m); + std::shared_lock lock(mutex); return value; } void SettingString::set(const String & x) { - std::lock_guard lock(m); + std::unique_lock lock(mutex); value = x; changed = true; } +void SettingString::setChanged(bool c) +{ + std::unique_lock lock(mutex); + changed = c; +} + void SettingString::set(const Field & x) { set(safeGet(x)); @@ -280,7 +342,6 @@ void SettingString::set(const Field & x) void SettingString::serialize(WriteBuffer & buf) const { - std::lock_guard lock(m); writeBinary(value, buf); } @@ -291,10 +352,15 @@ void SettingString::deserialize(ReadBuffer & buf) set(s); } +SettingChar & SettingChar::operator= (const SettingChar & o) +{ + data.store(o.data.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; +} String SettingChar::toString() const { - return String(1, value); + return String(1, getValue()); } Field SettingChar::toField() const @@ -304,8 +370,7 @@ Field SettingChar::toField() const void SettingChar::set(char x) { - value.store(x, std::memory_order_relaxed); - changed = true; + data.store({x, true}); } void SettingChar::set(const String & x) @@ -335,6 +400,19 @@ void SettingChar::deserialize(ReadBuffer & buf) } +template +SettingEnum & SettingEnum::operator= (const SettingEnum & o) +{ + data.store(o.data.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; +} + +template +void SettingEnum::set(EnumType x) +{ + data.store({x, true}, std::memory_order_relaxed); +} + template void SettingEnum::serialize(WriteBuffer & buf) const { @@ -350,6 +428,7 @@ void SettingEnum::deserialize(ReadBuffer & buf) } + #define IMPLEMENT_SETTING_ENUM(ENUM_NAME, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME) \ IMPLEMENT_SETTING_ENUM_WITH_TAG(ENUM_NAME, void, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME) diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index 0939ccac7ca..83d90c28f53 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB @@ -33,22 +34,25 @@ namespace ErrorCodes template struct SettingNumber { -public: - /// The value is atomic, because we want to avoid race conditions on value precisely. - /// It doesn't gurantee atomicy on whole structure. It just helps to avoid the most common - /// case: when we change setting from one thread and use in another (for example in MergeTreeSettings). - std::atomic value; + struct Data + { + Type value; + bool changed; + }; - bool changed = false; + std::atomic data; - SettingNumber(Type x = 0) : value(x) {} - SettingNumber(const SettingNumber & o) : value(o.getValue()), changed(o.changed) { } + SettingNumber(Type x = 0) : data{{x, false}} {} + SettingNumber(const SettingNumber & o) : data{o.data.load(std::memory_order_relaxed)} {} + + bool isChanged() const { return data.load(std::memory_order_relaxed).changed; } + void setChanged(bool changed) { data.store({getValue(), changed}, std::memory_order_relaxed); } operator Type() const { return getValue(); } - Type getValue() const { return value.load(std::memory_order_relaxed); } + Type getValue() const { return data.load(std::memory_order_relaxed).value; } SettingNumber & operator= (Type x) { set(x); return *this; } - SettingNumber & operator= (SettingNumber o) { set(o.getValue()); return *this; } + SettingNumber & operator= (const SettingNumber & o); /// Serialize to a test string. String toString() const; @@ -83,23 +87,26 @@ using SettingBool = SettingNumber; */ struct SettingMaxThreads { - std::atomic value; - bool is_auto; - bool changed = false; - - SettingMaxThreads(UInt64 x = 0) : value(x ? x : getAutoValue()), is_auto(x == 0) {} - SettingMaxThreads(const SettingMaxThreads & o) - : is_auto(o.is_auto) - , changed(o.changed) + struct Data { - value.store(o.value, std::memory_order_relaxed); - } + UInt64 value; + bool is_auto; + bool changed; + }; + + std::atomic data; + + SettingMaxThreads(UInt64 x = 0) : data{{x ? x : getAutoValue(), x == 0, false}} {} + SettingMaxThreads(const SettingMaxThreads & o) : data{o.data.load(std::memory_order_relaxed)} {} + + bool isChanged() const { return data.load(std::memory_order_relaxed).changed; } + void setChanged(bool changed); operator UInt64() const { return getValue(); } - UInt64 getValue() const { return value.load(std::memory_order_relaxed); } + UInt64 getValue() const { return data.load(std::memory_order_relaxed).value; } SettingMaxThreads & operator= (UInt64 x) { set(x); return *this; } - SettingMaxThreads & operator= (const SettingMaxThreads & o) { set(o.getValue()); return *this; } + SettingMaxThreads & operator= (const SettingMaxThreads & o); String toString() const; Field toField() const; @@ -111,6 +118,7 @@ struct SettingMaxThreads void serialize(WriteBuffer & buf) const; void deserialize(ReadBuffer & buf); + bool isAuto() const { return data.load(std::memory_order_relaxed).is_auto; } void setAuto(); UInt64 getAutoValue() const; }; @@ -121,20 +129,37 @@ enum class SettingTimespanIO { MILLISECOND, SECOND }; template struct SettingTimespan { - mutable std::mutex m; + mutable std::shared_mutex mutex; Poco::Timespan value; bool changed = false; SettingTimespan(UInt64 x = 0) : value(x * microseconds_per_io_unit) {} - SettingTimespan(const SettingTimespan & o) : value(o.value), changed(o.changed) {} + SettingTimespan(const SettingTimespan & o); operator Poco::Timespan() const { return getValue(); } - Poco::Timespan getValue() const { std::lock_guard guard(m); return value; } + Poco::Timespan getValue() const { std::shared_lock lock(mutex); return value; } SettingTimespan & operator= (const Poco::Timespan & x) { set(x); return *this; } - SettingTimespan & operator= (const SettingTimespan & o) { set(o.value); return *this; } + SettingTimespan & operator= (const SettingTimespan & o); - Poco::Timespan::TimeDiff totalSeconds() const { return getValue().totalSeconds(); } - Poco::Timespan::TimeDiff totalMilliseconds() const { return getValue().totalMilliseconds(); } + Poco::Timespan::TimeDiff totalSeconds() const + { + std::shared_lock lock(mutex); + return value.totalSeconds(); + } + + Poco::Timespan::TimeDiff totalMilliseconds() const + { + std::shared_lock lock(mutex); + return value.totalMilliseconds(); + } + + bool isChanged() const + { + std::shared_lock lock(mutex); + return changed; + } + + void setChanged(bool changed); String toString() const; Field toField() const; @@ -157,18 +182,19 @@ using SettingMilliseconds = SettingTimespan; struct SettingString { - mutable std::mutex m; + mutable std::shared_mutex mutex; String value; bool changed = false; SettingString(const String & x = String{}) : value(x) {} - SettingString(const SettingString & o) : value(o.value), changed(o.changed) {} - - operator String() const { return getValue(); } - String getValue() const { std::lock_guard guard(m); return value; } + SettingString(const SettingString & o); + operator String() const { return getValue(); } + String getValue() const { std::shared_lock lock(mutex); return value; } SettingString & operator= (const String & x) { set(x); return *this; } - SettingString & operator= (const SettingString & o) { set(o.value); return *this; } + SettingString & operator= (const SettingString & o); + bool isChanged() const { std::shared_lock lock(mutex); return changed; } + void setChanged(bool changed); String toString() const; Field toField() const; @@ -184,15 +210,25 @@ struct SettingString struct SettingChar { public: - std::atomic value; - bool changed = false; + struct Data + { + char value; + bool changed; + }; - SettingChar(char x = '\0') : value(x) {} - SettingChar(const SettingChar & o) : value(o.getValue()), changed(o.changed) { } + std::atomic data; + + SettingChar(char x = '\0') : data({x, false}) {} + SettingChar(const SettingChar & o) : data{o.data.load(std::memory_order_relaxed)} {} operator char() const { return getValue(); } + char getValue() const { return data.load(std::memory_order_relaxed).value; } + SettingChar & operator= (char x) { set(x); return *this; } - SettingChar & operator= (const SettingChar & o) { set(o.getValue()); return *this; } + SettingChar & operator= (const SettingChar & o); + + bool isChanged() const { return data.load(std::memory_order_relaxed).changed; } + void setChanged(bool changed) { data.store({getValue(), changed}, std::memory_order_relaxed);} String toString() const; Field toField() const; @@ -203,8 +239,6 @@ public: void serialize(WriteBuffer & buf) const; void deserialize(ReadBuffer & buf); - - char getValue() const { return value.load(std::memory_order_relaxed); } }; @@ -212,27 +246,35 @@ public: template struct SettingEnum { - std::atomic value; - bool changed = false; + struct Data + { + EnumType value; + bool changed; + }; - SettingEnum(EnumType x) : value(x) {} - SettingEnum(const SettingEnum & o) : value(o.getValue()), changed(o.changed) { } + std::atomic data; + + SettingEnum(EnumType x) : data({x, false}) {} + SettingEnum(const SettingEnum & o) : data{o.data.load(std::memory_order_relaxed)} {} operator EnumType() const { return getValue(); } + EnumType getValue() const { return data.load(std::memory_order_relaxed).value; } + SettingEnum & operator= (EnumType x) { set(x); return *this; } - SettingEnum & operator= (SettingEnum o) { set(o.getValue()); return *this; } + SettingEnum & operator= (const SettingEnum & o); + + bool isChanged() const { return data.load(std::memory_order_relaxed).changed; } + void setChanged(bool changed) { data.store({getValue(), changed}, std::memory_order_relaxed);} String toString() const; Field toField() const { return toString(); } - void set(EnumType x) { value.store(x, std::memory_order_relaxed); changed = true; } + void set(EnumType x); void set(const Field & x) { set(safeGet(x)); } void set(const String & x); void serialize(WriteBuffer & buf) const; void deserialize(ReadBuffer & buf); - - EnumType getValue() const { return value.load(std::memory_order_relaxed); } }; @@ -344,6 +386,7 @@ private: Derived & castToDerived() { return *static_cast(this); } const Derived & castToDerived() const { return *static_cast(this); } + using IsChangedFunction = bool (*)(const Derived &); using GetStringFunction = String (*)(const Derived &); using GetFieldFunction = Field (*)(const Derived &); using SetStringFunction = void (*)(Derived &, const String &); @@ -354,7 +397,7 @@ private: struct MemberInfo { - size_t offset_of_changed; + IsChangedFunction is_changed; StringRef name; StringRef description; /// Can be updated after first load for config/definition. @@ -369,7 +412,7 @@ private: DeserializeFunction deserialize; CastValueWithoutApplyingFunction cast_value_without_applying; - bool isChanged(const Derived & collection) const { return *reinterpret_cast(reinterpret_cast(&collection) + offset_of_changed); } + bool isChanged(const Derived & collection) const { return is_changed(collection); } }; class MemberInfos @@ -729,8 +772,7 @@ public: #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ - static_assert(std::is_same_v().NAME.changed), bool>); \ - add({offsetof(Derived, NAME.changed), \ + add({[](const Derived & d) { return d.NAME.isChanged(); }, \ StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ @@ -738,8 +780,7 @@ public: &Functions::NAME##_castValueWithoutApplying }); #define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ - static_assert(std::is_same_v().NAME.changed), bool>); \ - add({offsetof(Derived, NAME.changed), \ + add({[](const Derived & d) { return d.NAME.isChanged(); }, \ StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index 1a202e064f1..e4e52eab2b7 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -26,9 +26,9 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin new_settings.max_memory_usage_for_all_queries = 0; /// Set as unchanged to avoid sending to remote server. - new_settings.max_concurrent_queries_for_user.changed = false; - new_settings.max_memory_usage_for_user.changed = false; - new_settings.max_memory_usage_for_all_queries.changed = false; + new_settings.max_concurrent_queries_for_user.setChanged(false); + new_settings.max_memory_usage_for_user.setChanged(false); + new_settings.max_memory_usage_for_all_queries.setChanged(false); Context new_context(context); new_context.setSettings(new_settings); diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index 464ef149e35..465e0fb786f 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -596,7 +596,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( InJoinSubqueriesPreprocessor(context).visit(query); /// Optimizes logical expressions. - LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length).perform(); + LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.getValue()).perform(); } /// Creates a dictionary `aliases`: alias -> ASTPtr diff --git a/dbms/src/Interpreters/ThreadStatusExt.cpp b/dbms/src/Interpreters/ThreadStatusExt.cpp index 8c46a3ba08f..28740417b71 100644 --- a/dbms/src/Interpreters/ThreadStatusExt.cpp +++ b/dbms/src/Interpreters/ThreadStatusExt.cpp @@ -251,7 +251,7 @@ void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log) { elem.client_info = query_context->getClientInfo(); - if (query_context->getSettingsRef().log_profile_events.value != 0) + if (query_context->getSettingsRef().log_profile_events != 0) { /// NOTE: Here we are in the same thread, so we can make memcpy() elem.profile_counters = std::make_shared(performance_counters.getPartiallyAtomicSnapshot()); diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 47f4fc8c610..2d431ed1368 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -199,7 +199,7 @@ BufferPtr StorageKafka::createBuffer() const Settings & settings = global_context.getSettingsRef(); size_t batch_size = max_block_size; if (!batch_size) - batch_size = settings.max_block_size.value; + batch_size = settings.max_block_size; size_t poll_timeout = settings.stream_poll_timeout_ms.totalMilliseconds(); return std::make_shared( @@ -350,7 +350,7 @@ bool StorageKafka::streamToViews() const Settings & settings = global_context.getSettingsRef(); size_t block_size = max_block_size; if (block_size == 0) - block_size = settings.max_block_size.value; + block_size = settings.max_block_size; // Create a stream for each consumer and join them in a union stream InterpreterInsertQuery interpreter{insert, global_context}; @@ -436,7 +436,7 @@ void registerStorageKafka(StorageFactory & factory) #define CHECK_KAFKA_STORAGE_ARGUMENT(ARG_NUM, PAR_NAME) \ /* One of the four required arguments is not specified */ \ if (args_count < ARG_NUM && ARG_NUM <= 4 && \ - !kafka_settings.PAR_NAME.changed) \ + !kafka_settings.PAR_NAME.isChanged()) \ { \ throw Exception( \ "Required parameter '" #PAR_NAME "' " \ @@ -445,7 +445,7 @@ void registerStorageKafka(StorageFactory & factory) } \ /* The same argument is given in two places */ \ if (has_settings && \ - kafka_settings.PAR_NAME.changed && \ + kafka_settings.PAR_NAME.isChanged() && \ args_count >= ARG_NUM) \ { \ throw Exception( \ @@ -469,7 +469,7 @@ void registerStorageKafka(StorageFactory & factory) #undef CHECK_KAFKA_STORAGE_ARGUMENT // Get and check broker list - String brokers = kafka_settings.kafka_broker_list.value; + String brokers = kafka_settings.kafka_broker_list; if (args_count >= 1) { const auto * ast = engine_args[0]->as(); @@ -524,7 +524,7 @@ void registerStorageKafka(StorageFactory & factory) } // Parse row delimiter (optional) - char row_delimiter = kafka_settings.kafka_row_delimiter.value; + char row_delimiter = kafka_settings.kafka_row_delimiter; if (args_count >= 5) { engine_args[4] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[4], args.local_context); @@ -571,7 +571,7 @@ void registerStorageKafka(StorageFactory & factory) } // Parse number of consumers (optional) - UInt64 num_consumers = kafka_settings.kafka_num_consumers.value; + UInt64 num_consumers = kafka_settings.kafka_num_consumers; if (args_count >= 7) { const auto * ast = engine_args[6]->as(); @@ -586,7 +586,7 @@ void registerStorageKafka(StorageFactory & factory) } // Parse max block size (optional) - UInt64 max_block_size = static_cast(kafka_settings.kafka_max_block_size.value); + UInt64 max_block_size = static_cast(kafka_settings.kafka_max_block_size); if (args_count >= 8) { const auto * ast = engine_args[7]->as(); @@ -601,7 +601,7 @@ void registerStorageKafka(StorageFactory & factory) } } - size_t skip_broken = static_cast(kafka_settings.kafka_skip_broken_messages.value); + size_t skip_broken = static_cast(kafka_settings.kafka_skip_broken_messages); if (args_count >= 9) { const auto * ast = engine_args[8]->as(); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 54da38df541..6108704b45a 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -100,8 +100,8 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() std::sort(entries.begin(), entries.end()); String min_saved_record_log_str = entries[ - entries.size() > storage.settings.max_replicated_logs_to_keep.value - ? entries.size() - storage.settings.max_replicated_logs_to_keep.value + entries.size() > storage.settings.max_replicated_logs_to_keep + ? entries.size() - storage.settings.max_replicated_logs_to_keep : 0]; /// Replicas that were marked is_lost but are active. @@ -203,7 +203,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() min_saved_log_pointer = std::min(min_saved_log_pointer, min_log_pointer_lost_candidate); /// We will not touch the last `min_replicated_logs_to_keep` records. - entries.erase(entries.end() - std::min(entries.size(), storage.settings.min_replicated_logs_to_keep.value), entries.end()); + entries.erase(entries.end() - std::min(entries.size(), storage.settings.min_replicated_logs_to_keep), entries.end()); /// We will not touch records that are no less than `min_saved_log_pointer`. entries.erase(std::lower_bound(entries.begin(), entries.end(), "log-" + padIndex(min_saved_log_pointer)), entries.end()); @@ -299,7 +299,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() /// Virtual node, all nodes that are "greater" than this one will be deleted NodeWithStat block_threshold{{}, time_threshold}; - size_t current_deduplication_window = std::min(timed_blocks.size(), storage.settings.replicated_deduplication_window.value); + size_t current_deduplication_window = std::min(timed_blocks.size(), storage.settings.replicated_deduplication_window); auto first_outdated_block_fixed_threshold = timed_blocks.begin() + current_deduplication_window; auto first_outdated_block_time_threshold = std::upper_bound(timed_blocks.begin(), timed_blocks.end(), block_threshold, NodeWithStat::greaterByTime); auto first_outdated_block = std::min(first_outdated_block_fixed_threshold, first_outdated_block_time_threshold); diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 88a7c07377c..62638e1982a 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -329,7 +329,7 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Context & c const auto & settings = context.getSettingsRef(); /// Ban an attempt to make async insert into the table belonging to DatabaseMemory - if (path.empty() && !owned_cluster && !settings.insert_distributed_sync.value) + if (path.empty() && !owned_cluster && !settings.insert_distributed_sync) { throw Exception("Storage " + getName() + " must has own data directory to enable asynchronous inserts", ErrorCodes::BAD_ARGUMENTS); diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index 3c90917b0f6..8d957a1c9cf 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -164,8 +164,8 @@ void registerStorageJoin(StorageFactory & factory) args.database_name, args.table_name, key_names, - join_use_nulls.value, - SizeLimits{max_rows_in_join.value, max_bytes_in_join.value, join_overflow_mode.value}, + join_use_nulls, + SizeLimits{max_rows_in_join, max_bytes_in_join, join_overflow_mode}, kind, strictness, args.columns, From aec386f60e8f3d281ec5e17dd3bc5c85d1f47c6a Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 9 Aug 2019 16:15:23 +0300 Subject: [PATCH 088/357] better comment --- dbms/src/Databases/DatabaseOrdinary.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 0bf703887ef..1e7f331c7aa 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -153,7 +153,7 @@ void DatabaseOrdinary::loadTables( continue; } - /// There are files .sql.tmp and .sql.tmp_drop - delete + /// There are files .sql.tmp - delete if (endsWith(dir_it.name(), ".sql.tmp")) { LOG_INFO(log, "Removing file " << dir_it->path()); From 2ce2bba496d39d239fb8498b0e9fc67e2ba05402 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Fri, 9 Aug 2019 18:08:10 +0300 Subject: [PATCH 089/357] Minor fixes --- dbms/programs/benchmark/Benchmark.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index e172ebc710a..72ae8f2e22b 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -80,6 +80,12 @@ public: comparison_info_total.emplace_back(std::make_shared()); } + if (confidence > 5) + { + std::cerr << "Confidence can't be set to " + toString(confidence) + ". It was set to 5 instead." << '\n'; + confidence = 5; + } + global_context.makeGlobalContext(); std::cerr << std::fixed << std::setprecision(3); @@ -334,7 +340,7 @@ private: double s = spool * sqrt(1.0 / data[0].cnt + 1.0 / data[1].cnt); - double d = data[0].avg() - data[1].avg(); + double d = data[1].avg() - data[0].avg(); double e = t * s; @@ -343,7 +349,7 @@ private: { std::cerr << std::setprecision(1) << "Difference at " << confidence_level[confidence_index] << "% confidence\n" << std::setprecision(6); std::cerr << "\t" << d << " +/- " << e << "\n"; - std::cerr << "\t" << d * 100 / data[1].avg() << " +/- " << e * 100 / data[1].avg() << "\n"; + std::cerr << "\t" << d * 100 / data[0].avg() << "% +/- " << e * 100 / data[0].avg() << "%\n"; std::cerr << "\t(Student's t, pooled s = " << spool << ")\n" << std::setprecision(3); return false; } From 9caa69ce8837eaaf75857c84fe855fb610723cd4 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Fri, 9 Aug 2019 18:18:21 +0300 Subject: [PATCH 090/357] Minor fixes --- dbms/programs/benchmark/Benchmark.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 72ae8f2e22b..43e109810ec 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -559,7 +559,7 @@ private: std::cerr << "\n"; for (size_t i = 1; i <= infos.size(); ++i) { - auto & info = infos[i - 1]; + const auto & info = infos[i - 1]; /// Avoid zeros, nans or exceptions if (0 == info->queries) @@ -587,7 +587,7 @@ private: auto print_percentile = [&](double percent) { std::cerr << percent << "%\t\t"; - for (auto & info : infos) + for (const auto & info : infos) { std::cerr << info->sampler.quantileInterpolated(percent / 100.0) << " sec." << "\t"; } @@ -629,7 +629,7 @@ private: for (size_t i = 1; i <= infos.size(); ++i) { - auto & info = infos[i - 1]; + const auto & info = infos[i - 1]; json_out << double_quote << "connection_" + toString(i) << ": {\n"; json_out << double_quote << "statistics" << ": {\n"; From 67331881356ee5dd1fb158ce728e8ab42016c9f2 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 5 Aug 2019 07:09:09 +0000 Subject: [PATCH 091/357] Added gcc-9 to docker/builder container --- docker/builder/Dockerfile | 6 ++++-- docker/builder/build.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index 03b4e242d6d..41a558f9eb8 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -1,6 +1,8 @@ FROM ubuntu:18.04 RUN apt-get update -y \ + && apt-get install -y software-properties-common \ + && add-apt-repository ppa:ubuntu-toolchain-r/test \ && env DEBIAN_FRONTEND=noninteractive \ apt-get install --yes --no-install-recommends \ bash \ @@ -8,8 +10,8 @@ RUN apt-get update -y \ cmake \ curl \ expect \ - g++ \ - gcc \ + g++-9 \ + gcc-9 \ libclang-6.0-dev \ libicu-dev \ liblld-6.0-dev \ diff --git a/docker/builder/build.sh b/docker/builder/build.sh index 6a5f1359bda..57999a4b483 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -3,7 +3,7 @@ #ccache -s mkdir -p /server/build_docker cd /server/build_docker -cmake -G Ninja /server -DENABLE_TESTS=1 +cmake -G Ninja /server -DENABLE_TESTS=1 -DCMAKE_C_COMPILER=`which gcc-9` -DCMAKE_CXX_COMPILER=`which g++-9` # Set the number of build jobs to the half of number of virtual CPU cores (rounded up). # By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time. From 0233f32f9b30e78d8f54c28ccdea16736a6293b1 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 11 Aug 2019 12:28:15 +0300 Subject: [PATCH 092/357] Fixed AddresSanitizer error --- .../DataStreams/CheckConstraintsBlockOutputStream.cpp | 11 +++++------ .../DataStreams/CheckConstraintsBlockOutputStream.h | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 5adf344cf0b..4b4865f004f 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -11,8 +11,9 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { for (size_t i = 0; i < expressions.size(); ++i) { + Block res = block; auto constraint_expr = expressions[i]; - auto res_column_uint8 = executeOnBlock(block, constraint_expr); + auto res_column_uint8 = executeOnBlock(res, constraint_expr); if (!memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1)) { auto indices_wrong = findAllWrong(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize()); @@ -48,13 +49,11 @@ void CheckConstraintsBlockOutputStream::writeSuffix() } const ColumnUInt8 *CheckConstraintsBlockOutputStream::executeOnBlock( - const Block & block, + Block & block, const ExpressionActionsPtr & constraint) { - Block res = block; - - constraint->execute(res); - ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); + constraint->execute(block); + ColumnWithTypeAndName res_column = block.safeGetByPosition(block.columns() - 1); return checkAndGetColumn(res_column.column.get()); } diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index ac2e7e974a1..7ab6832fd28 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -39,7 +39,7 @@ public: void writeSuffix() override; private: - const ColumnUInt8* executeOnBlock(const Block & block, const ExpressionActionsPtr & constraint); + const ColumnUInt8* executeOnBlock(Block & block, const ExpressionActionsPtr & constraint); std::vector findAllWrong(const void *data, size_t size); String table; From 93a635d18a2119cc018cdbb5d73cfd12da6fbdc6 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 11 Aug 2019 12:30:01 +0300 Subject: [PATCH 093/357] Added clang-8 to docker builder --- docker/builder/Dockerfile | 4 ++++ docker/builder/Makefile | 2 +- docker/builder/build.sh | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index 41a558f9eb8..5978dcd08d0 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -28,6 +28,10 @@ RUN apt-get update -y \ tzdata \ gperf +RUN apt install -y wget +RUN printf "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main\ndeb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list \ + && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt-get install -y clang-8 lldb-8 lld-8 + COPY build.sh / CMD ["/bin/bash", "/build.sh"] diff --git a/docker/builder/Makefile b/docker/builder/Makefile index 779e944b723..a9a7cddf3f2 100644 --- a/docker/builder/Makefile +++ b/docker/builder/Makefile @@ -1,6 +1,6 @@ build: image mkdir -p $(HOME)/.ccache - docker run --network=host --rm --workdir /server --volume $(realpath ../..):/server --mount=type=bind,source=$(HOME)/.ccache,destination=/ccache -e CCACHE_DIR=/ccache -it yandex/clickhouse-builder + docker run --network=host --rm --workdir /server --volume $(realpath ../..):/server --cap-add=SYS_PTRACE --mount=type=bind,source=$(HOME)/.ccache,destination=/ccache -e CCACHE_DIR=/ccache -it yandex/clickhouse-builder pull: docker pull yandex/clickhouse-builder diff --git a/docker/builder/build.sh b/docker/builder/build.sh index 57999a4b483..96468d8d820 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -3,7 +3,8 @@ #ccache -s mkdir -p /server/build_docker cd /server/build_docker -cmake -G Ninja /server -DENABLE_TESTS=1 -DCMAKE_C_COMPILER=`which gcc-9` -DCMAKE_CXX_COMPILER=`which g++-9` + +cmake -G Ninja /server -DCMAKE_C_COMPILER=`which clang-8` -DCMAKE_CXX_COMPILER=`which clang++-8` -DCMAKE_BUILD_TYPE=Debug # Set the number of build jobs to the half of number of virtual CPU cores (rounded up). # By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time. From 3b9e1f9bf727764e1a4d9787b6e58313f3d381bf Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 11 Aug 2019 13:39:17 +0300 Subject: [PATCH 094/357] Fixed getIdentifierName call in AlterCommand::parse --- dbms/src/Storages/AlterCommands.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 48690df071a..13141683be8 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -190,7 +190,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) { command.type = AlterCommand::DROP_COLUMN; - command.column_name = *getIdentifierName(command_ast->column); + command.column_name = getIdentifierName(command_ast->column); } return command; From fd4638aa64bb4874c3eea28bec165d60b13a9265 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 12 Aug 2019 13:13:03 +0300 Subject: [PATCH 095/357] drop materialized view fixed --- dbms/src/Interpreters/InterpreterDropQuery.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 7a26bc3c740..c8cfb1605d4 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -91,8 +91,8 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId()); - const auto prev_metadata_name = database_and_table.first->getMetadataPath() + database_and_table.second->getTableName() + ".sql"; - const auto drop_metadata_name = database_and_table.first->getMetadataPath() + database_and_table.second->getTableName() + ".sql.tmp_drop"; + const auto prev_metadata_name = database_and_table.first->getMetadataPath() + escapeForFileName(database_and_table.second->getTableName()) + ".sql"; + const auto drop_metadata_name = database_and_table.first->getMetadataPath() + escapeForFileName(database_and_table.second->getTableName()) + ".sql.tmp_drop"; try { @@ -102,7 +102,10 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t } catch (...) { - Poco::File(drop_metadata_name).renameTo(prev_metadata_name); + try + { + Poco::File(drop_metadata_name).renameTo(prev_metadata_name); + } catch (...) {} throw; } From fafbd2b1883947ceec2850f0ded5b864419ade19 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 12 Aug 2019 15:56:10 +0300 Subject: [PATCH 096/357] fix drop with memory engine --- dbms/src/Interpreters/InterpreterDropQuery.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index c8cfb1605d4..facf377b885 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -94,18 +94,19 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t const auto prev_metadata_name = database_and_table.first->getMetadataPath() + escapeForFileName(database_and_table.second->getTableName()) + ".sql"; const auto drop_metadata_name = database_and_table.first->getMetadataPath() + escapeForFileName(database_and_table.second->getTableName()) + ".sql.tmp_drop"; + //Try to rename metadata file and delete the data try { - Poco::File(prev_metadata_name).renameTo(drop_metadata_name); + //Memory database has no metadata on disk + if (database_and_table.first->getEngineName() != "Memory") + Poco::File(prev_metadata_name).renameTo(drop_metadata_name); /// Delete table data database_and_table.second->drop(); } catch (...) { - try - { + if (database_and_table.first->getEngineName() != "Memory") Poco::File(drop_metadata_name).renameTo(prev_metadata_name); - } catch (...) {} throw; } From 1b54a52488441b753c99fe6e17d19d458e90313c Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Mon, 12 Aug 2019 17:10:29 +0300 Subject: [PATCH 097/357] Temp --- dbms/src/Functions/neighbour.cpp | 217 +++++++++++++++++++++++++++++ dbms/src/Functions/nextInBlock.cpp | 159 --------------------- 2 files changed, 217 insertions(+), 159 deletions(-) create mode 100644 dbms/src/Functions/neighbour.cpp delete mode 100644 dbms/src/Functions/nextInBlock.cpp diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp new file mode 100644 index 00000000000..cf96282725a --- /dev/null +++ b/dbms/src/Functions/neighbour.cpp @@ -0,0 +1,217 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +// Implements function, giving value for column within range of given +// Example: +// | c1 | +// | 10 | +// | 20 | +// SELECT c1, neighbour(c1, 1) as c2: +// | c1 | c2 | +// | 10 | 20 | +// | 20 | 0 | +class FunctionNeighbour : public IFunction +{ +public: + static constexpr auto name = "neighbour"; + static FunctionPtr create(const Context & context) { return std::make_shared(context); } + + FunctionNeighbour(const Context & context_) : context(context_) {} + + /// Get the name of the function. + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isVariadic() const override { return true; } + + bool isDeterministic() const override { return false; } + + bool isDeterministicInScopeOfQuery() const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + size_t number_of_arguments = arguments.size(); + + if (number_of_arguments < 2 || number_of_arguments > 3) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(number_of_arguments) + + ", should be from 2 to 3", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + // second argument must be a positive integer + if (!isInteger(arguments[1])) + throw Exception( + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + + " - should be positive integer", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + // check that default value column has supertype with first argument + if (number_of_arguments == 3) + { + DataTypes types = {arguments[0], arguments[2]}; + try + { + return getLeastSupertype(types); + } + catch (const Exception &) + { + throw Exception( + "Illegal types of arguments (" + types[0]->getName() + ", " + types[1]->getName() + + ")" + " of function " + + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + + return arguments[0]; + } + + static void insertDefaults(const MutableColumnPtr & target, size_t row_count, ColumnPtr & default_values_column, size_t offset) + { + if (row_count == 0) { + return; + } + if (default_values_column) + { + if (isColumnConst(*default_values_column)) + { + Field constant_value = (*default_values_column)[0]; + for(size_t row = 0; row < row_count;row++) + { + target->insert(constant_value); + } + } else { + target->insertRangeFrom(*default_values_column, offset, row_count); + } + } else { + for(size_t row = 0; row <= row_count;row++) { + target->insertDefault(); + } + } + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto offset_structure = block.getByPosition(arguments[1]); + ColumnPtr & offset_column = offset_structure.column; + + auto is_constant_offset = isColumnConst(*offset_structure.column); + ColumnPtr default_values_column = nullptr; + if (arguments.size() == 3) + { + default_values_column = block.getByPosition(arguments[2]).column; + } + +// Field offset_field = (*block.getByPosition(arguments[1]).column)[0]; +// auto raw_value = safeGet(offset_field); + + ColumnWithTypeAndName &source_column_name_and_type = block.getByPosition(arguments[0]); + DataTypes types = {source_column_name_and_type.type}; + if (default_values_column) + { + types.push_back(block.getByPosition(arguments[2]).type); + } + const DataTypePtr & result_type = getLeastSupertype(types); + auto source_column = source_column_name_and_type.column; + + // adjust source and default values columns to resulting datatype + if (!source_column_name_and_type.type->equals(*result_type)) { + source_column = castColumn(source_column_name_and_type, result_type, context); + } + + if (default_values_column && !block.getByPosition(arguments[2]).type->equals(*result_type)) { + default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); + } + + auto column = result_type->createColumn(); + column->reserve(input_rows_count); + + const DataTypePtr desired_type = std::make_shared(); + if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) { + offset_column = castColumn(offset_structure, desired_type, context); + } + + // with constant offset - insertRangeFrom + if (is_constant_offset) + { + Int64 offset_value = offset_column->getInt(0); + + if (offset_value > 0) + { + // insert shifted value + column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); + // insert defaults into the end + insertDefaults(column, input_rows_count - offset_value, default_values_column, offset_value); + } else if(offset_value < 0) { + // insert defaults up to offset_value + insertDefaults(column, input_rows_count - std::abs(offset_value), default_values_column, std::abs(offset_value)); + // insert range, where possible + column->insertRangeFrom(*source_column, 0, input_rows_count - std::abs(offset_value)); + } else { + // populate column with source values + column->insertRangeFrom(*source_column, 0, input_rows_count); + } + } else { + // with dynamic offset - handle row by row + for (size_t row = 0; row < input_rows_count; row++) + { + Int64 offset_value = offset_column->getInt(row); + if (offset_value == 0) { + column->insertFrom(*source_column, row); + } else if (offset_value > 0) { + size_t real_offset = row + offset_value; + if (real_offset > input_rows_count) { + if (default_values_column) { + column->insertFrom(*default_values_column, row); + } else { + column->insertDefault(); + } + } else { + column->insertFrom(*column, real_offset); + } + } else { + // out of range + if ((size_t)std::abs(offset_value) > row) + { + if (default_values_column) { + column->insertFrom(*default_values_column, row); + } else { + column->insertDefault(); + } + } else { + column->insertFrom(*column, row - std::abs(offset_value)); + } + } + } + } + + + block.getByPosition(result).column = std::move(column); + } +private: + const Context & context; +}; + +void registerFunctionNextInBlock(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/nextInBlock.cpp b/dbms/src/Functions/nextInBlock.cpp deleted file mode 100644 index eeb33e28146..00000000000 --- a/dbms/src/Functions/nextInBlock.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ARGUMENT_OUT_OF_BOUND; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - -// Implements function, giving value for column in next row -// Example: -// | c1 | -// | 10 | -// | 20 | -// SELECT c1, nextInBlock(c1, 1) as c2: -// | c1 | c2 | -// | 10 | 20 | -// | 20 | 0 | -class FunctionNextInBlock : public IFunction -{ -public: - static constexpr auto name = "nextInBlock"; - static FunctionPtr create(const Context &) { return std::make_shared(); } - - /// Get the name of the function. - String getName() const override { return name; } - - size_t getNumberOfArguments() const override { return 0; } - - bool isVariadic() const override { return true; } - - bool isDeterministic() const override { return false; } - - bool isDeterministicInScopeOfQuery() const override { return false; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - size_t number_of_arguments = arguments.size(); - - if (number_of_arguments < 1 || number_of_arguments > 3) - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(number_of_arguments) - + ", should be from 1 to 3", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - // second argument must be a positive, constant column - if (number_of_arguments == 2 && !isUnsignedInteger(arguments[1])) - throw Exception( - "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + " - should be positive integer", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - // check that default value has supertype with first argument - if (number_of_arguments == 3) - { - DataTypes types = {arguments[0], arguments[2]}; - try - { - return getLeastSupertype(types); - } - catch (const Exception &) - { - throw Exception( - "Illegal types of arguments (" + types[0]->getName() + ", " + types[1]->getName() - + ")" - " of function " - + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - } - - return arguments[0]; - } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override - { - size_t offset_value = 1; - - if (arguments.size() > 1) - { - auto offset_column = block.getByPosition(arguments[1]); - if (!isColumnConst(*offset_column.column)) - throw Exception("Second argument of function " + getName() + " should be constant", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - Field offset_field = (*block.getByPosition(arguments[1]).column)[0]; - auto raw_value = safeGet(offset_field); - - if (raw_value == 0) - throw Exception( - "Second argument of function " + getName() + " should be positive integer, " + toString(raw_value) + " given", - ErrorCodes::ARGUMENT_OUT_OF_BOUND); - - offset_value = raw_value; - } - - auto has_column_for_missing = arguments.size() == 3; - - DataTypes types = {block.getByPosition(arguments[0]).type}; - if (has_column_for_missing) - { - types.push_back(block.getByPosition(arguments[2]).type); - } - const DataTypePtr & result_type = getLeastSupertype(types); - - auto column = result_type->createColumn(); - column->reserve(input_rows_count); - - auto source_column = block.getByPosition(arguments[0]).column; - - for (size_t i = offset_value; i < input_rows_count; i++) - { - column->insertFrom(*source_column, i); - } - - if (has_column_for_missing) - { - auto default_values_column = block.getByPosition(arguments[2]).column; - size_t starting_pos = offset_value > input_rows_count ? 0 : input_rows_count - offset_value; - if (isColumnConst(*default_values_column)) - { - Field constant_value = (*default_values_column)[0]; - for (size_t i = starting_pos; i < input_rows_count; i++) - { - column->insert(constant_value); - } - } - else - { - for (size_t i = starting_pos; i < input_rows_count; i++) - { - column->insertFrom(*default_values_column, i); - } - } - } - else - { - for (size_t i = 0; i < std::min(offset_value, input_rows_count); i++) - { - column->insertDefault(); - } - } - - block.getByPosition(result).column = std::move(column); - } -}; - -void registerFunctionNextInBlock(FunctionFactory & factory) -{ - factory.registerFunction(); -} - -} From 31fdc99efc3b668b18e8ea830dfc669f743d275d Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Mon, 12 Aug 2019 18:44:28 +0300 Subject: [PATCH 098/357] In progress --- dbms/src/Functions/neighbour.cpp | 10 ++++---- .../0_stateless/00957_next_in_block.sql | 24 ++++++++----------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index cf96282725a..3eff660c996 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -101,7 +101,7 @@ public: target->insertRangeFrom(*default_values_column, offset, row_count); } } else { - for(size_t row = 0; row <= row_count;row++) { + for(size_t row = 0; row < row_count;row++) { target->insertDefault(); } } @@ -156,9 +156,11 @@ public: if (offset_value > 0) { // insert shifted value - column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); - // insert defaults into the end - insertDefaults(column, input_rows_count - offset_value, default_values_column, offset_value); + if ((size_t)std::abs(offset_value) <= input_rows_count) { + column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); + // insert defaults into the end + insertDefaults(column, input_rows_count - offset_value, default_values_column, offset_value); + } } else if(offset_value < 0) { // insert defaults up to offset_value insertDefaults(column, input_rows_count - std::abs(offset_value), default_values_column, std::abs(offset_value)); diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.sql b/dbms/tests/queries/0_stateless/00957_next_in_block.sql index 7cbd932cf1a..1efda43339e 100644 --- a/dbms/tests/queries/0_stateless/00957_next_in_block.sql +++ b/dbms/tests/queries/0_stateless/00957_next_in_block.sql @@ -1,22 +1,18 @@ -- no arguments -select nextInBlock(); -- { serverError 42 } +select neighbour(); -- { serverError 42 } +-- single argument +select neighbour(1); -- { serverError 42 } -- greater than 3 arguments -select nextInBlock(1,2,3,4); -- { serverError 42 } --- zero offset value -select nextInBlock(dummy, 0); -- { serverError 69 } --- negative offset value -select nextInBlock(dummy, -1); -- { serverError 43 } --- non-constant offset value -select nextInBlock(dummy, dummy); -- { serverError 43 } +select neighbour(1,2,3,4); -- { serverError 42 } -- bad default value -select nextInBlock(dummy, 1, 'hello'); -- { serverError 43 } +select neighbour(dummy, 1, 'hello'); -- { serverError 43 } -- single argument test -select number, nextInBlock(number) from numbers(2); +select number, neighbour(number,1) from numbers(2); -- filling by column's default value -select number, nextInBlock(number, 2) from numbers(3); +select number, neighbour(number, 2) from numbers(3); -- offset is greater that block - should fill everything with defaults -select number, nextInBlock(number, 5) from numbers(2); +select number, neighbour(number, 5) from numbers(2); -- substitution by constant for missing values -select number, nextInBlock(number, 2, 1000) from numbers(5); +select number, neighbour(number, 2, 1000) from numbers(5); -- substitution by expression --- select number, nextInBlock(number, 2, number % 2) from numbers(5); \ No newline at end of file +-- select number, neighbour(number, 2, number % 2) from numbers(5); \ No newline at end of file From 8328a06eec77a7ec32cf5eb9d3f103ad3510051c Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 12 Aug 2019 19:20:31 +0300 Subject: [PATCH 099/357] update renaming metadata files --- dbms/src/Interpreters/InterpreterDropQuery.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index facf377b885..c93dedada82 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -97,16 +97,20 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t //Try to rename metadata file and delete the data try { - //Memory database has no metadata on disk - if (database_and_table.first->getEngineName() != "Memory") + //There some kind of tables that have no metadata - ignore renaming + try + { Poco::File(prev_metadata_name).renameTo(drop_metadata_name); + } catch (...) {} /// Delete table data database_and_table.second->drop(); } catch (...) { - if (database_and_table.first->getEngineName() != "Memory") + try + { Poco::File(drop_metadata_name).renameTo(prev_metadata_name); + } catch (...) {} throw; } From 4be51007c57f292a92756b4adc736685726baaef Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 13 Aug 2019 00:45:59 +0300 Subject: [PATCH 100/357] empty commit --- dbms/src/Databases/DatabaseOrdinary.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 1e7f331c7aa..10bf2ad871d 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -387,6 +387,7 @@ void DatabaseOrdinary::renameTable( throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED); StoragePtr table = tryGetTable(context, table_name); + if (!table) throw Exception("Table " + name + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); From 313f72e41a65a43fd5c380a3f63be3ae1cba1952 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 13 Aug 2019 11:03:31 +0300 Subject: [PATCH 101/357] remove try catch + submodule update --- dbms/src/Interpreters/InterpreterDropQuery.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index c93dedada82..a35b9599050 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -98,19 +98,15 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t try { //There some kind of tables that have no metadata - ignore renaming - try - { + if (Poco::File(prev_metadata_name).exists()) Poco::File(prev_metadata_name).renameTo(drop_metadata_name); - } catch (...) {} /// Delete table data database_and_table.second->drop(); } catch (...) { - try - { + if (Poco::File(drop_metadata_name).exists()) Poco::File(drop_metadata_name).renameTo(prev_metadata_name); - } catch (...) {} throw; } From 8e72d4c2ec1181dd84aea79227bf7732f6594339 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 11:35:49 +0300 Subject: [PATCH 102/357] Tryin COW settings --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 56 +++++++++++-------- dbms/src/Storages/MergeTree/MergeTreeData.h | 18 ++++-- .../Storages/MergeTree/MergeTreeSettings.cpp | 5 ++ .../Storages/MergeTree/MergeTreeSettings.h | 11 +++- .../MergeTree/registerStorageMergeTree.cpp | 6 +- 5 files changed, 64 insertions(+), 32 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b0453b04c8a..e6a8f2f7ba1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -104,7 +104,7 @@ MergeTreeData::MergeTreeData( const ASTPtr & sample_by_ast_, const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - const MergeTreeSettings & settings_, + MergeTreeSettingsPtr settings_, bool require_part_metadata_, bool attach, BrokenPartCallback broken_part_callback_) @@ -132,7 +132,7 @@ MergeTreeData::MergeTreeData( sampling_expr_column_name = sample_by_ast->getColumnName(); if (!primary_key_sample.has(sampling_expr_column_name) - && !attach && !settings.compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility. + && !attach && !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility. throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); auto syntax = SyntaxAnalyzer(global_context).analyze(sample_by_ast, getColumns().getAllPhysical()); @@ -727,6 +727,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { LOG_DEBUG(log, "Loading data parts"); + auto settings_ptr = getImmutableSettings(); Strings part_file_names; Poco::DirectoryIterator end; for (Poco::DirectoryIterator it(full_path); it != end; ++it) @@ -843,12 +844,12 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) throw Exception("Part " + part->name + " already exists", ErrorCodes::DUPLICATE_DATA_PART); } - if (has_non_adaptive_parts && has_adaptive_parts && !settings.enable_mixed_granularity_parts) + if (has_non_adaptive_parts && has_adaptive_parts && !settings_ptr->enable_mixed_granularity_parts) throw Exception("Table contains parts with adaptive and non adaptive marks, but `setting enable_mixed_granularity_parts` is disabled", ErrorCodes::LOGICAL_ERROR); has_non_adaptive_index_granularity_parts = has_non_adaptive_parts; - if (suspicious_broken_parts > settings.max_suspicious_broken_parts && !skip_sanity_checks) + if (suspicious_broken_parts > settings_ptr->max_suspicious_broken_parts && !skip_sanity_checks) throw Exception("Suspiciously many (" + toString(suspicious_broken_parts) + ") broken parts to remove.", ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS); @@ -938,7 +939,7 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life time_t current_time = time(nullptr); ssize_t deadline = (custom_directories_lifetime_seconds >= 0) ? current_time - custom_directories_lifetime_seconds - : current_time - settings.temporary_directories_lifetime.totalSeconds(); + : current_time - settings_ptr->temporary_directories_lifetime.totalSeconds(); /// Delete temporary directories older than a day. Poco::DirectoryIterator end; @@ -989,7 +990,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts() if (part.unique() && /// Grab only parts that are not used by anyone (SELECTs for example). part_remove_time < now && - now - part_remove_time > settings.old_parts_lifetime.totalSeconds()) + now - part_remove_time > getSettings()->old_parts_lifetime.totalSeconds()) { parts_to_delete.emplace_back(it); } @@ -1290,7 +1291,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name if (part) part_mrk_file_extension = part->index_granularity_info.marks_file_extension; else - part_mrk_file_extension = settings.index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(); + part_mrk_file_extension = mutable_settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(); using NameToType = std::map; NameToType new_types; @@ -1448,6 +1449,7 @@ void MergeTreeData::alterDataPart( bool skip_sanity_checks, AlterDataPartTransactionPtr & transaction) { + auto settings = getImmutableSettings(); ExpressionActionsPtr expression; const auto & part = transaction->getDataPart(); bool force_update_metadata; @@ -1463,12 +1465,12 @@ void MergeTreeData::alterDataPart( ++num_files_to_remove; if (!skip_sanity_checks - && (num_files_to_modify > settings.max_files_to_modify_in_alter_columns - || num_files_to_remove > settings.max_files_to_remove_in_alter_columns)) + && (num_files_to_modify > settings->max_files_to_modify_in_alter_columns + || num_files_to_remove > settings->max_files_to_remove_in_alter_columns)) { transaction->clear(); - const bool forbidden_because_of_modify = num_files_to_modify > settings.max_files_to_modify_in_alter_columns; + const bool forbidden_because_of_modify = num_files_to_modify > settings->max_files_to_modify_in_alter_columns; std::stringstream exception_message; exception_message @@ -1500,7 +1502,7 @@ void MergeTreeData::alterDataPart( << " If it is not an error, you could increase merge_tree/" << (forbidden_because_of_modify ? "max_files_to_modify_in_alter_columns" : "max_files_to_remove_in_alter_columns") << " parameter in configuration file (current value: " - << (forbidden_because_of_modify ? settings.max_files_to_modify_in_alter_columns : settings.max_files_to_remove_in_alter_columns) + << (forbidden_because_of_modify ? settings->max_files_to_modify_in_alter_columns : settings->max_files_to_remove_in_alter_columns) << ")"; throw Exception(exception_message.str(), ErrorCodes::TABLE_DIFFERS_TOO_MUCH); @@ -1590,13 +1592,17 @@ void MergeTreeData::alterSettings( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - settings.updateFromChanges(new_changes); - IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); + { + MutableMergeTreeSettingsPtr settings = std::move(*mutable_settings).mutate(); + settings->updateFromChanges(new_changes); + IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); + mutable_settings = std::move(settings); + } } bool MergeTreeData::hasSetting(const String & setting_name) const { - return settings.findIndex(setting_name) != MergeTreeSettings::npos; + return MergeTreeSettings::findIndex(setting_name) != MergeTreeSettings::npos; } void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part) @@ -2231,26 +2237,27 @@ std::optional MergeTreeData::getMinPartDataVersion() const void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const { + auto settings_ptr = getImmutableSettings(); const size_t parts_count_in_total = getPartsCount(); - if (parts_count_in_total >= settings.max_parts_in_total) + if (parts_count_in_total >= settings_ptr->max_parts_in_total) { ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception("Too many parts (" + toString(parts_count_in_total) + ") in all partitions in total. This indicates wrong choice of partition key. The threshold can be modified with 'max_parts_in_total' setting in element in config.xml or with per-table setting.", ErrorCodes::TOO_MANY_PARTS); } const size_t parts_count_in_partition = getMaxPartsCountForPartition(); - if (parts_count_in_partition < settings.parts_to_delay_insert) + if (parts_count_in_partition < settings_ptr->parts_to_delay_insert) return; - if (parts_count_in_partition >= settings.parts_to_throw_insert) + if (parts_count_in_partition >= settings_ptr->parts_to_throw_insert) { ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception("Too many parts (" + toString(parts_count_in_partition) + "). Merges are processing significantly slower than inserts.", ErrorCodes::TOO_MANY_PARTS); } - const size_t max_k = settings.parts_to_throw_insert - settings.parts_to_delay_insert; /// always > 0 - const size_t k = 1 + parts_count_in_partition - settings.parts_to_delay_insert; /// from 1 to max_k - const double delay_milliseconds = ::pow(settings.max_delay_to_insert * 1000, static_cast(k) / max_k); + const size_t max_k = settings_ptr->parts_to_throw_insert - settings_ptr->parts_to_delay_insert; /// always > 0 + const size_t k = 1 + parts_count_in_partition - settings_ptr->parts_to_delay_insert; /// from 1 to max_k + const double delay_milliseconds = ::pow(settings_ptr->max_delay_to_insert * 1000, static_cast(k) / max_k); ProfileEvents::increment(ProfileEvents::DelayedInserts); ProfileEvents::increment(ProfileEvents::DelayedInsertsMilliseconds, delay_milliseconds); @@ -2268,8 +2275,9 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const void MergeTreeData::throwInsertIfNeeded() const { + auto settings_ptr = getImmutableSettings(); const size_t parts_count_in_total = getPartsCount(); - if (parts_count_in_total >= settings.max_parts_in_total) + if (parts_count_in_total >= settings_ptr->max_parts_in_total) { ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception("Too many parts (" + toString(parts_count_in_total) + ") in all partitions in total. This indicates wrong choice of partition key. The threshold can be modified with 'max_parts_in_total' setting in element in config.xml or with per-table setting.", ErrorCodes::TOO_MANY_PARTS); @@ -2277,7 +2285,7 @@ void MergeTreeData::throwInsertIfNeeded() const const size_t parts_count_in_partition = getMaxPartsCountForPartition(); - if (parts_count_in_partition >= settings.parts_to_throw_insert) + if (parts_count_in_partition >= settings_ptr->parts_to_throw_insert) { ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception("Too many parts (" + toString(parts_count_in_partition) + "). Merges are processing significantly slower than inserts.", ErrorCodes::TOO_MANY_PARTS); @@ -2860,7 +2868,9 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const { - if (!settings.enable_mixed_granularity_parts || settings.index_granularity_bytes == 0) + auto settings_ptr = getImmutableSettings(); + + if (!settings_ptr->enable_mixed_granularity_parts || settings_ptr->index_granularity_bytes == 0) { if (!canUseAdaptiveGranularity() && src_part->index_granularity_info.is_adaptive) return false; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 756c188c724..38e58f3a6da 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -313,7 +313,7 @@ public: const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported. const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - const MergeTreeSettings & settings_, + MergeTreeSettingsPtr settings_, bool require_part_metadata_, bool attach, BrokenPartCallback broken_part_callback_ = [](const String &){}); @@ -591,8 +591,9 @@ public: bool canUseAdaptiveGranularity() const { - return settings.index_granularity_bytes != 0 && - (settings.enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts); + auto settings_ptr = getImmutableSettings(); + return settings_ptr->index_granularity_bytes != 0 && + (settings_ptr->enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts); } @@ -645,8 +646,6 @@ public: String sampling_expr_column_name; Names columns_required_for_sampling; - MergeTreeSettings settings; - /// Limiting parallel sends per one table, used in DataPartsExchange std::atomic_uint current_table_sends {0}; @@ -655,7 +654,13 @@ public: bool has_non_adaptive_index_granularity_parts = false; + MergeTreeSettingsPtr getImmutableSettings() const + { + return mutable_settings; + } + protected: + friend struct MergeTreeDataPart; friend class MergeTreeDataMergerMutator; friend class ReplicatedMergeTreeAlterThread; @@ -683,6 +688,9 @@ protected: String log_name; Logger * log; + /// Settings COW pointer. Data maybe changed at any point of time. + /// If you need consistent settings, just copy pointer to your scope. + MergeTreeSettingsPtr mutable_settings; /// Work with data parts diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index 425a0851d67..3b90bf293ba 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -75,4 +75,9 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) #undef ADD_IF_ABSENT } + +MergeTreeSettings * MergeTreeSettings::clone() const +{ + return new MergeTreeSettings(*this); +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index dc94ad5b11f..54416f934a6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -2,6 +2,7 @@ #include #include +#include namespace Poco @@ -21,9 +22,11 @@ class ASTStorage; /** Settings for the MergeTree family of engines. * Could be loaded from config or from a CREATE TABLE query (SETTINGS clause). */ -struct MergeTreeSettings : public SettingsCollection +struct MergeTreeSettings : public SettingsCollection, public COW { + friend class COW; + /// M (mutable) for normal settings, IM (immutable) for not updateable settings. #define LIST_OF_MERGE_TREE_SETTINGS(M, IM) \ IM(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \ @@ -93,6 +96,12 @@ struct MergeTreeSettings : public SettingsCollection /// NOTE: will rewrite the AST to add immutable settings. void loadFromQuery(ASTStorage & storage_def); + + MergeTreeSettings * clone() const; + ~MergeTreeSettings() {} }; +using MergeTreeSettingsPtr = MergeTreeSettings::Ptr; +using MutableMergeTreeSettingsPtr = MergeTreeSettings::MutablePtr; + } diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 138e7c14f9d..a8e2f137e1a 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -574,7 +574,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) ASTPtr sample_by_ast; ASTPtr ttl_table_ast; IndicesDescription indices_description; - MergeTreeSettings storage_settings = args.context.getMergeTreeSettings(); + MutableMergeTreeSettingsPtr storage_settings = MergeTreeSettings::create(args.context.getMergeTreeSettings()); if (is_extended_storage_def) { @@ -603,7 +603,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) std::dynamic_pointer_cast(index->clone())); - storage_settings.loadFromQuery(*args.storage_def); + storage_settings->loadFromQuery(*args.storage_def); } else { @@ -625,7 +625,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) const auto * ast = engine_args.back()->as(); if (ast && ast->value.getType() == Field::Types::UInt64) - storage_settings.index_granularity = safeGet(ast->value); + storage_settings->index_granularity = safeGet(ast->value); else throw Exception( "Index granularity must be a positive integer" + getMergeTreeVerboseHelp(is_extended_storage_def), From ad81c743c186cff9d2e0e5ff1019f9be44088392 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 13:29:31 +0300 Subject: [PATCH 103/357] Buildable code --- .../Storages/MergeTree/DataPartsExchange.cpp | 8 ++- .../MergeTree/IMergedBlockOutputStream.cpp | 7 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 54 ++++++++------- dbms/src/Storages/MergeTree/MergeTreeData.h | 36 +++++++--- .../MergeTree/MergeTreeDataMergerMutator.cpp | 32 +++++---- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 21 +++--- .../MergeTreeIndexGranularityInfo.cpp | 5 +- .../Storages/MergeTree/MergeTreeSettings.cpp | 4 +- .../Storages/MergeTree/MergeTreeSettings.h | 2 +- .../MergeTreeThreadSelectBlockInputStream.cpp | 2 +- .../ReplicatedMergeTreeCleanupThread.cpp | 30 +++++---- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 3 +- .../ReplicatedMergeTreeRestartingThread.cpp | 17 +++-- .../ReplicatedMergeTreeTableMetadata.cpp | 5 +- .../MergeTree/registerStorageMergeTree.cpp | 4 +- dbms/src/Storages/StorageMergeTree.cpp | 11 +-- dbms/src/Storages/StorageMergeTree.h | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 67 +++++++++++-------- .../src/Storages/StorageReplicatedMergeTree.h | 2 +- 19 files changed, 186 insertions(+), 126 deletions(-) diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index f01b384d441..795cc68f1ea 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -54,14 +54,15 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); String part_name = params.get("part"); + const auto data_settings = data.getCOWSettings(); /// Validation of the input that may come from malicious replica. MergeTreePartInfo::fromPartName(part_name, data.format_version); static std::atomic_uint total_sends {0}; - if ((data.settings.replicated_max_parallel_sends && total_sends >= data.settings.replicated_max_parallel_sends) - || (data.settings.replicated_max_parallel_sends_for_table && data.current_table_sends >= data.settings.replicated_max_parallel_sends_for_table)) + if ((data_settings->replicated_max_parallel_sends && total_sends >= data_settings->replicated_max_parallel_sends) + || (data_settings->replicated_max_parallel_sends_for_table && data.current_table_sends >= data_settings->replicated_max_parallel_sends_for_table)) { response.setStatus(std::to_string(HTTP_TOO_MANY_REQUESTS)); response.setReason("Too many concurrent fetches, try again later"); @@ -174,6 +175,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( { /// Validation of the input that may come from malicious replica. MergeTreePartInfo::fromPartName(part_name, data.format_version); + const auto data_settings = data.getCOWSettings(); Poco::URI uri; uri.setScheme(interserver_scheme); @@ -200,7 +202,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( timeouts, creds, DBMS_DEFAULT_BUFFER_SIZE, - data.settings.replicated_max_parallel_fetches_for_host + data_settings->replicated_max_parallel_fetches_for_host }; static const String TMP_PREFIX = "tmp_fetch_"; diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 2bd0ebb61ea..a20b03cc7b0 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -32,7 +32,7 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( , index_granularity(index_granularity_) , compute_granularity(index_granularity.empty()) , codec(std::move(codec_)) - , with_final_mark(storage.settings.write_final_mark && storage.canUseAdaptiveGranularity()) + , with_final_mark(storage.getCOWSettings()->write_final_mark && storage.canUseAdaptiveGranularity()) { if (blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); @@ -133,10 +133,11 @@ void fillIndexGranularityImpl( void IMergedBlockOutputStream::fillIndexGranularity(const Block & block) { + const auto storage_settings = storage.getCOWSettings(); fillIndexGranularityImpl( block, - storage.settings.index_granularity_bytes, - storage.settings.index_granularity, + storage_settings->index_granularity_bytes, + storage_settings->index_granularity, blocks_are_granules_size, index_offset, index_granularity, diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index f8ff2ab87b1..4d47f60a2fe 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -110,7 +110,6 @@ MergeTreeData::MergeTreeData( BrokenPartCallback broken_part_callback_) : global_context(context_), merging_params(merging_params_), - settings(settings_), partition_by_ast(partition_by_ast_), sample_by_ast(sample_by_ast_), ttl_table_ast(ttl_table_ast_), @@ -119,9 +118,11 @@ MergeTreeData::MergeTreeData( full_path(full_path_), broken_part_callback(broken_part_callback_), log_name(database_name + "." + table_name), log(&Logger::get(log_name)), + guarded_settings(std::move(settings_)), data_parts_by_info(data_parts_indexes.get()), data_parts_by_state_and_info(data_parts_indexes.get()) { + const auto settings = getCOWSettings(); setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_); /// NOTE: using the same columns list as is read when performing actual merges. @@ -725,7 +726,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { LOG_DEBUG(log, "Loading data parts"); - auto settings_ptr = getImmutableSettings(); + const auto settings = getCOWSettings(); Strings part_file_names; Poco::DirectoryIterator end; for (Poco::DirectoryIterator it(full_path); it != end; ++it) @@ -747,7 +748,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) } /// Parallel loading of data parts. - size_t num_threads = std::min(size_t(settings.max_part_loading_threads), part_file_names.size()); + size_t num_threads = std::min(size_t(settings->max_part_loading_threads), part_file_names.size()); std::mutex mutex; @@ -866,12 +867,12 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) pool.wait(); - if (has_non_adaptive_parts && has_adaptive_parts && !settings_ptr->enable_mixed_granularity_parts) + if (has_non_adaptive_parts && has_adaptive_parts && !settings->enable_mixed_granularity_parts) throw Exception("Table contains parts with adaptive and non adaptive marks, but `setting enable_mixed_granularity_parts` is disabled", ErrorCodes::LOGICAL_ERROR); has_non_adaptive_index_granularity_parts = has_non_adaptive_parts; - if (suspicious_broken_parts > settings_ptr->max_suspicious_broken_parts && !skip_sanity_checks) + if (suspicious_broken_parts > settings->max_suspicious_broken_parts && !skip_sanity_checks) throw Exception("Suspiciously many (" + toString(suspicious_broken_parts) + ") broken parts to remove.", ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS); @@ -958,10 +959,11 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life if (!lock.try_lock()) return; + const auto settings = getCOWSettings(); time_t current_time = time(nullptr); ssize_t deadline = (custom_directories_lifetime_seconds >= 0) ? current_time - custom_directories_lifetime_seconds - : current_time - settings_ptr->temporary_directories_lifetime.totalSeconds(); + : current_time - settings->temporary_directories_lifetime.totalSeconds(); /// Delete temporary directories older than a day. Poco::DirectoryIterator end; @@ -1012,7 +1014,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts() if (part.unique() && /// Grab only parts that are not used by anyone (SELECTs for example). part_remove_time < now && - now - part_remove_time > getSettings()->old_parts_lifetime.totalSeconds()) + now - part_remove_time > getCOWSettings()->old_parts_lifetime.totalSeconds()) { parts_to_delete.emplace_back(it); } @@ -1096,11 +1098,12 @@ void MergeTreeData::clearOldPartsFromFilesystem() void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_remove) { - if (parts_to_remove.size() > 1 && settings.max_part_removal_threads > 1 && parts_to_remove.size() > settings.concurrent_part_removal_threshold) + const auto settings = getCOWSettings(); + if (parts_to_remove.size() > 1 && settings->max_part_removal_threads > 1 && parts_to_remove.size() > settings->concurrent_part_removal_threshold) { /// Parallel parts removal. - size_t num_threads = std::min(size_t(settings.max_part_removal_threads), parts_to_remove.size()); + size_t num_threads = std::min(size_t(settings->max_part_removal_threads), parts_to_remove.size()); ThreadPool pool(num_threads); /// NOTE: Under heavy system load you may get "Cannot schedule a task" from ThreadPool. @@ -1332,6 +1335,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name const IndicesASTs & old_indices, const IndicesASTs & new_indices, ExpressionActionsPtr & out_expression, NameToNameMap & out_rename_map, bool & out_force_update_metadata) const { + const auto settings = getCOWSettings(); out_expression = nullptr; out_rename_map = {}; out_force_update_metadata = false; @@ -1339,7 +1343,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name if (part) part_mrk_file_extension = part->index_granularity_info.marks_file_extension; else - part_mrk_file_extension = mutable_settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(); + part_mrk_file_extension = settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(); using NameToType = std::map; NameToType new_types; @@ -1497,7 +1501,7 @@ void MergeTreeData::alterDataPart( bool skip_sanity_checks, AlterDataPartTransactionPtr & transaction) { - auto settings = getImmutableSettings(); + const auto settings = getCOWSettings(); ExpressionActionsPtr expression; const auto & part = transaction->getDataPart(); bool force_update_metadata; @@ -1641,10 +1645,10 @@ void MergeTreeData::alterSettings( TableStructureWriteLockHolder & table_lock_holder) { { - MutableMergeTreeSettingsPtr settings = std::move(*mutable_settings).mutate(); + MutableMergeTreeSettingsPtr settings = std::move(*guarded_settings.getPtr()).mutate(); settings->updateFromChanges(new_changes); IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); - mutable_settings = std::move(settings); + guarded_settings.setPtr(std::move(settings)); } } @@ -2285,27 +2289,27 @@ std::optional MergeTreeData::getMinPartDataVersion() const void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const { - auto settings_ptr = getImmutableSettings(); + const auto settings = getCOWSettings(); const size_t parts_count_in_total = getPartsCount(); - if (parts_count_in_total >= settings_ptr->max_parts_in_total) + if (parts_count_in_total >= settings->max_parts_in_total) { ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception("Too many parts (" + toString(parts_count_in_total) + ") in all partitions in total. This indicates wrong choice of partition key. The threshold can be modified with 'max_parts_in_total' setting in element in config.xml or with per-table setting.", ErrorCodes::TOO_MANY_PARTS); } const size_t parts_count_in_partition = getMaxPartsCountForPartition(); - if (parts_count_in_partition < settings_ptr->parts_to_delay_insert) + if (parts_count_in_partition < settings->parts_to_delay_insert) return; - if (parts_count_in_partition >= settings_ptr->parts_to_throw_insert) + if (parts_count_in_partition >= settings->parts_to_throw_insert) { ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception("Too many parts (" + toString(parts_count_in_partition) + "). Merges are processing significantly slower than inserts.", ErrorCodes::TOO_MANY_PARTS); } - const size_t max_k = settings_ptr->parts_to_throw_insert - settings_ptr->parts_to_delay_insert; /// always > 0 - const size_t k = 1 + parts_count_in_partition - settings_ptr->parts_to_delay_insert; /// from 1 to max_k - const double delay_milliseconds = ::pow(settings_ptr->max_delay_to_insert * 1000, static_cast(k) / max_k); + const size_t max_k = settings->parts_to_throw_insert - settings->parts_to_delay_insert; /// always > 0 + const size_t k = 1 + parts_count_in_partition - settings->parts_to_delay_insert; /// from 1 to max_k + const double delay_milliseconds = ::pow(settings->max_delay_to_insert * 1000, static_cast(k) / max_k); ProfileEvents::increment(ProfileEvents::DelayedInserts); ProfileEvents::increment(ProfileEvents::DelayedInsertsMilliseconds, delay_milliseconds); @@ -2323,9 +2327,9 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const void MergeTreeData::throwInsertIfNeeded() const { - auto settings_ptr = getImmutableSettings(); + const auto settings = getCOWSettings(); const size_t parts_count_in_total = getPartsCount(); - if (parts_count_in_total >= settings_ptr->max_parts_in_total) + if (parts_count_in_total >= settings->max_parts_in_total) { ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception("Too many parts (" + toString(parts_count_in_total) + ") in all partitions in total. This indicates wrong choice of partition key. The threshold can be modified with 'max_parts_in_total' setting in element in config.xml or with per-table setting.", ErrorCodes::TOO_MANY_PARTS); @@ -2333,7 +2337,7 @@ void MergeTreeData::throwInsertIfNeeded() const const size_t parts_count_in_partition = getMaxPartsCountForPartition(); - if (parts_count_in_partition >= settings_ptr->parts_to_throw_insert) + if (parts_count_in_partition >= settings->parts_to_throw_insert) { ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception("Too many parts (" + toString(parts_count_in_partition) + "). Merges are processing significantly slower than inserts.", ErrorCodes::TOO_MANY_PARTS); @@ -2916,9 +2920,9 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const { - auto settings_ptr = getImmutableSettings(); + const auto settings = getCOWSettings(); - if (!settings_ptr->enable_mixed_granularity_parts || settings_ptr->index_granularity_bytes == 0) + if (!settings->enable_mixed_granularity_parts || settings->index_granularity_bytes == 0) { if (!canUseAdaptiveGranularity() && src_part->index_granularity_info.is_adaptive) return false; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index c07e2fc34f4..b94555477f9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -594,9 +594,9 @@ public: /// Has additional constraint in replicated version virtual bool canUseAdaptiveGranularity() const { - auto settings_ptr = getImmutableSettings(); - return settings_ptr->index_granularity_bytes != 0 && - (settings_ptr->enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts); + const auto settings = getCOWSettings(); + return settings->index_granularity_bytes != 0 && + (settings->enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts); } @@ -657,9 +657,12 @@ public: bool has_non_adaptive_index_granularity_parts = false; - MergeTreeSettingsPtr getImmutableSettings() const + /// Get copy-on-write pointer to storage settings. + /// Copy this pointer into your scope and you will + /// get consistent settings. + const MergeTreeSettingsPtr getCOWSettings() const { - return mutable_settings; + return guarded_settings.copyPtr(); } protected: @@ -691,9 +694,26 @@ protected: String log_name; Logger * log; - /// Settings COW pointer. Data maybe changed at any point of time. - /// If you need consistent settings, just copy pointer to your scope. - MergeTreeSettingsPtr mutable_settings; + /// Just hides settings pointer from direct usage + class MergeTreeSettingsGuard + { + private: + /// Settings COW pointer. Data maybe changed at any point of time. + /// If you need consistent settings, just copy pointer to your scope. + MergeTreeSettingsPtr settings_ptr; + public: + MergeTreeSettingsGuard(MergeTreeSettingsPtr settings_ptr_) + : settings_ptr(settings_ptr_) + {} + + const MergeTreeSettingsPtr copyPtr() const { return settings_ptr; } + MergeTreeSettingsPtr & getPtr() { return settings_ptr; } + void setPtr(MergeTreeSettingsPtr ptr) { settings_ptr = ptr; } + }; + + /// Storage settings. Don't use this field directly, if you + /// want readonly settings. Prefer getCOWSettings() method. + MergeTreeSettingsGuard guarded_settings; /// Work with data parts diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 3a4d4038317..fad63f35aec 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -141,15 +141,16 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_siz throw Exception("Logical error: invalid arguments passed to getMaxSourcePartsSize: pool_used > pool_size", ErrorCodes::LOGICAL_ERROR); size_t free_entries = pool_size - pool_used; + const auto data_settings = data.getCOWSettings(); UInt64 max_size = 0; - if (free_entries >= data.settings.number_of_free_entries_in_pool_to_lower_max_size_of_merge) - max_size = data.settings.max_bytes_to_merge_at_max_space_in_pool; + if (free_entries >= data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge) + max_size = data_settings->max_bytes_to_merge_at_max_space_in_pool; else max_size = interpolateExponential( - data.settings.max_bytes_to_merge_at_min_space_in_pool, - data.settings.max_bytes_to_merge_at_max_space_in_pool, - static_cast(free_entries) / data.settings.number_of_free_entries_in_pool_to_lower_max_size_of_merge); + data_settings->max_bytes_to_merge_at_min_space_in_pool, + data_settings->max_bytes_to_merge_at_max_space_in_pool, + static_cast(free_entries) / data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge); return std::min(max_size, static_cast(DiskSpaceMonitor::getUnreservedFreeSpace(data.full_path) / DISK_USAGE_COEFFICIENT_TO_SELECT)); } @@ -169,6 +170,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( String * out_disable_reason) { MergeTreeData::DataPartsVector data_parts = data.getDataPartsVector(); + const auto data_settings = data.getCOWSettings(); if (data_parts.empty()) { @@ -223,7 +225,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( merge_settings.base = 1; bool can_merge_with_ttl = - (current_time - last_merge_with_ttl > data.settings.merge_with_ttl_timeout); + (current_time - last_merge_with_ttl > data_settings->merge_with_ttl_timeout); /// NOTE Could allow selection of different merge strategy. if (can_merge_with_ttl && has_part_with_expired_ttl && !ttl_merges_blocker.isCancelled()) @@ -545,6 +547,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor Names all_column_names = data.getColumns().getNamesOfPhysical(); NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); + const auto data_settings = data.getCOWSettings(); NamesAndTypesList gathering_columns, merging_columns; Names gathering_column_names, merging_column_names; @@ -617,13 +620,13 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// We count total amount of bytes in parts /// and use direct_io + aio if there is more than min_merge_bytes_to_use_direct_io bool read_with_direct_io = false; - if (data.settings.min_merge_bytes_to_use_direct_io != 0) + if (data_settings->min_merge_bytes_to_use_direct_io != 0) { size_t total_size = 0; for (const auto & part : parts) { total_size += part->bytes_on_disk; - if (total_size >= data.settings.min_merge_bytes_to_use_direct_io) + if (total_size >= data_settings->min_merge_bytes_to_use_direct_io) { LOG_DEBUG(log, "Will merge parts reading files in O_DIRECT"); read_with_direct_io = true; @@ -720,7 +723,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor merging_columns, compression_codec, merged_column_to_size, - data.settings.min_merge_bytes_to_use_direct_io, + data_settings->min_merge_bytes_to_use_direct_io, blocks_are_granules_size}; merged_stream->readPrefix(); @@ -938,6 +941,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor auto in = mutations_interpreter.execute(); NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); + const auto data_settings = data.getCOWSettings(); Block in_header = in->getHeader(); @@ -995,7 +999,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor } NameSet files_to_skip = {"checksums.txt", "columns.txt"}; - auto mrk_extension = data.settings.index_granularity_bytes ? getAdaptiveMrkExtension() : getNonAdaptiveMrkExtension(); + auto mrk_extension = data_settings->index_granularity_bytes ? getAdaptiveMrkExtension() : getNonAdaptiveMrkExtension(); for (const auto & entry : in_header) { IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path) @@ -1092,9 +1096,11 @@ MergeTreeDataMergerMutator::MergeAlgorithm MergeTreeDataMergerMutator::chooseMer const MergeTreeData::DataPartsVector & parts, size_t sum_rows_upper_bound, const NamesAndTypesList & gathering_columns, bool deduplicate, bool need_remove_expired_values) const { + const auto data_settings = data.getCOWSettings(); + if (deduplicate) return MergeAlgorithm::Horizontal; - if (data.settings.enable_vertical_merge_algorithm == 0) + if (data_settings->enable_vertical_merge_algorithm == 0) return MergeAlgorithm::Horizontal; if (need_remove_expired_values) return MergeAlgorithm::Horizontal; @@ -1105,9 +1111,9 @@ MergeTreeDataMergerMutator::MergeAlgorithm MergeTreeDataMergerMutator::chooseMer data.merging_params.mode == MergeTreeData::MergingParams::Replacing || data.merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; - bool enough_ordinary_cols = gathering_columns.size() >= data.settings.vertical_merge_algorithm_min_columns_to_activate; + bool enough_ordinary_cols = gathering_columns.size() >= data_settings->vertical_merge_algorithm_min_columns_to_activate; - bool enough_total_rows = sum_rows_upper_bound >= data.settings.vertical_merge_algorithm_min_rows_to_activate; + bool enough_total_rows = sum_rows_upper_bound >= data_settings->vertical_merge_algorithm_min_rows_to_activate; bool no_parts_overflow = parts.size() <= RowSourcePart::MAX_PARTS; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 513a713651b..314967b485d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -672,6 +672,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( size_t sum_marks = 0; size_t total_rows = 0; + const auto data_settings = data.getCOWSettings(); size_t adaptive_parts = 0; for (size_t i = 0; i < parts.size(); ++i) { @@ -688,18 +689,18 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( size_t index_granularity_bytes = 0; if (adaptive_parts > parts.size() / 2) - index_granularity_bytes = data.settings.index_granularity_bytes; + index_granularity_bytes = data_settings->index_granularity_bytes; const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks( settings.merge_tree_max_rows_to_use_cache, settings.merge_tree_max_bytes_to_use_cache, - data.settings.index_granularity, + data_settings->index_granularity, index_granularity_bytes); const size_t min_marks_for_concurrent_read = roundRowsOrBytesToMarks( settings.merge_tree_min_rows_for_concurrent_read, settings.merge_tree_min_bytes_for_concurrent_read, - data.settings.index_granularity, + data_settings->index_granularity, index_granularity_bytes); if (sum_marks > max_marks_to_use_cache) @@ -830,6 +831,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO SortingInfoPtr sorting_info = query_info.sorting_info; size_t adaptive_parts = 0; std::vector sum_marks_in_parts(parts.size()); + const auto data_settings = data.getCOWSettings(); /// In case of reverse order let's split ranges to avoid reading much data. auto split_ranges = [max_block_size](const auto & ranges, size_t rows_granularity, size_t num_marks_in_part) @@ -862,7 +864,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO sum_marks += sum_marks_in_parts[i]; if (sorting_info->direction == -1) - parts[i].ranges = split_ranges(parts[i].ranges, data.settings.index_granularity, sum_marks_in_parts[i]); + parts[i].ranges = split_ranges(parts[i].ranges, data_settings->index_granularity, sum_marks_in_parts[i]); /// Let the ranges be listed from right to left so that the leftmost range can be dropped using `pop_back()`. std::reverse(parts[i].ranges.begin(), parts[i].ranges.end()); @@ -873,18 +875,18 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO size_t index_granularity_bytes = 0; if (adaptive_parts > parts.size() / 2) - index_granularity_bytes = data.settings.index_granularity_bytes; + index_granularity_bytes = data_settings->index_granularity_bytes; const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks( settings.merge_tree_max_rows_to_use_cache, settings.merge_tree_max_bytes_to_use_cache, - data.settings.index_granularity, + data_settings->index_granularity, index_granularity_bytes); const size_t min_marks_for_concurrent_read = roundRowsOrBytesToMarks( settings.merge_tree_min_rows_for_concurrent_read, settings.merge_tree_min_bytes_for_concurrent_read, - data.settings.index_granularity, + data_settings->index_granularity, index_granularity_bytes); if (sum_marks > max_marks_to_use_cache) @@ -1012,6 +1014,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal const Names & virt_columns, const Settings & settings) const { + const auto data_settings = data.getCOWSettings(); size_t sum_marks = 0; size_t adaptive_parts = 0; for (size_t i = 0; i < parts.size(); ++i) @@ -1025,12 +1028,12 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal size_t index_granularity_bytes = 0; if (adaptive_parts >= parts.size() / 2) - index_granularity_bytes = data.settings.index_granularity_bytes; + index_granularity_bytes = data_settings->index_granularity_bytes; const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks( settings.merge_tree_max_rows_to_use_cache, settings.merge_tree_max_bytes_to_use_cache, - data.settings.index_granularity, + data_settings->index_granularity, index_granularity_bytes); if (sum_marks > max_marks_to_use_cache) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 63b19da9e64..143af37c10d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -25,12 +25,13 @@ std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS( MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo( const MergeTreeData & storage) { - fixed_index_granularity = storage.settings.index_granularity; + const auto storage_settings = storage.getCOWSettings(); + fixed_index_granularity = storage_settings->index_granularity; /// Granularity is fixed if (!storage.canUseAdaptiveGranularity()) setNonAdaptive(); else - setAdaptive(storage.settings.index_granularity_bytes); + setAdaptive(storage_settings->index_granularity_bytes); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index 3b90bf293ba..7c3c7ef02cc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -76,8 +76,8 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) } -MergeTreeSettings * MergeTreeSettings::clone() const +MergeTreeSettings::MutablePtr MergeTreeSettings::clone() const { - return new MergeTreeSettings(*this); + return COW::create(*this); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 480209c740a..dff7cc2e523 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -100,7 +100,7 @@ struct MergeTreeSettings : public SettingsCollection, public /// NOTE: will rewrite the AST to add immutable settings. void loadFromQuery(ASTStorage & storage_def); - MergeTreeSettings * clone() const; + MutablePtr clone() const; ~MergeTreeSettings() {} }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp index 9c34782dec8..7a09bde0998 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp @@ -31,7 +31,7 @@ MergeTreeThreadSelectBlockInputStream::MergeTreeThreadSelectBlockInputStream( /// Maybe it will make sence to add settings `max_block_size_bytes` if (max_block_size_rows && !storage.canUseAdaptiveGranularity()) { - size_t fixed_index_granularity = storage.settings.index_granularity; + size_t fixed_index_granularity = storage.getCOWSettings()->index_granularity; min_marks_to_read = (min_marks_to_read_ * fixed_index_granularity + max_block_size_rows - 1) / max_block_size_rows * max_block_size_rows / fixed_index_granularity; } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 6108704b45a..2b03ed86895 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -27,8 +27,9 @@ ReplicatedMergeTreeCleanupThread::ReplicatedMergeTreeCleanupThread(StorageReplic void ReplicatedMergeTreeCleanupThread::run() { - const auto CLEANUP_SLEEP_MS = storage.settings.cleanup_delay_period * 1000 - + std::uniform_int_distribution(0, storage.settings.cleanup_delay_period_random_add * 1000)(rng); + auto storage_settings = storage.getCOWSettings(); + const auto CLEANUP_SLEEP_MS = storage_settings->cleanup_delay_period * 1000 + + std::uniform_int_distribution(0, storage_settings->cleanup_delay_period_random_add * 1000)(rng); try { @@ -74,6 +75,7 @@ void ReplicatedMergeTreeCleanupThread::iterate() void ReplicatedMergeTreeCleanupThread::clearOldLogs() { auto zookeeper = storage.getZooKeeper(); + auto storage_settings = storage.getCOWSettings(); Coordination::Stat stat; if (!zookeeper->exists(storage.zookeeper_path + "/log", &stat)) @@ -82,7 +84,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() int children_count = stat.numChildren; /// We will wait for 1.1 times more records to accumulate than necessary. - if (static_cast(children_count) < storage.settings.min_replicated_logs_to_keep * 1.1) + if (static_cast(children_count) < storage_settings->min_replicated_logs_to_keep * 1.1) return; Strings replicas = zookeeper->getChildren(storage.zookeeper_path + "/replicas", &stat); @@ -100,8 +102,8 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() std::sort(entries.begin(), entries.end()); String min_saved_record_log_str = entries[ - entries.size() > storage.settings.max_replicated_logs_to_keep - ? entries.size() - storage.settings.max_replicated_logs_to_keep + entries.size() > storage_settings->max_replicated_logs_to_keep + ? entries.size() - storage_settings->max_replicated_logs_to_keep : 0]; /// Replicas that were marked is_lost but are active. @@ -203,7 +205,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() min_saved_log_pointer = std::min(min_saved_log_pointer, min_log_pointer_lost_candidate); /// We will not touch the last `min_replicated_logs_to_keep` records. - entries.erase(entries.end() - std::min(entries.size(), storage.settings.min_replicated_logs_to_keep), entries.end()); + entries.erase(entries.end() - std::min(entries.size(), storage_settings->min_replicated_logs_to_keep), entries.end()); /// We will not touch records that are no less than `min_saved_log_pointer`. entries.erase(std::lower_bound(entries.begin(), entries.end(), "log-" + padIndex(min_saved_log_pointer)), entries.end()); @@ -285,6 +287,7 @@ struct ReplicatedMergeTreeCleanupThread::NodeWithStat void ReplicatedMergeTreeCleanupThread::clearOldBlocks() { auto zookeeper = storage.getZooKeeper(); + auto storage_settings = storage.getCOWSettings(); std::vector timed_blocks; getBlocksSortedByTime(*zookeeper, timed_blocks); @@ -294,12 +297,12 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() /// Use ZooKeeper's first node (last according to time) timestamp as "current" time. Int64 current_time = timed_blocks.front().ctime; - Int64 time_threshold = std::max(static_cast(0), current_time - static_cast(1000 * storage.settings.replicated_deduplication_window_seconds)); + Int64 time_threshold = std::max(static_cast(0), current_time - static_cast(1000 * storage_settings->replicated_deduplication_window_seconds)); /// Virtual node, all nodes that are "greater" than this one will be deleted NodeWithStat block_threshold{{}, time_threshold}; - size_t current_deduplication_window = std::min(timed_blocks.size(), storage.settings.replicated_deduplication_window); + size_t current_deduplication_window = std::min(timed_blocks.size(), storage_settings->replicated_deduplication_window); auto first_outdated_block_fixed_threshold = timed_blocks.begin() + current_deduplication_window; auto first_outdated_block_time_threshold = std::upper_bound(timed_blocks.begin(), timed_blocks.end(), block_threshold, NodeWithStat::greaterByTime); auto first_outdated_block = std::min(first_outdated_block_fixed_threshold, first_outdated_block_time_threshold); @@ -401,10 +404,11 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & void ReplicatedMergeTreeCleanupThread::clearOldMutations() { - if (!storage.settings.finished_mutations_to_keep) + auto storage_settings = storage.getCOWSettings(); + if (!storage_settings->finished_mutations_to_keep) return; - if (storage.queue.countFinishedMutations() <= storage.settings.finished_mutations_to_keep) + if (storage.queue.countFinishedMutations() <= storage_settings->finished_mutations_to_keep) { /// Not strictly necessary, but helps to avoid unnecessary ZooKeeper requests. /// If even this replica hasn't finished enough mutations yet, then we don't need to clean anything. @@ -431,10 +435,10 @@ void ReplicatedMergeTreeCleanupThread::clearOldMutations() /// Do not remove entries that are greater than `min_pointer` (they are not done yet). entries.erase(std::upper_bound(entries.begin(), entries.end(), padIndex(min_pointer)), entries.end()); - /// Do not remove last `storage.settings.finished_mutations_to_keep` entries. - if (entries.size() <= storage.settings.finished_mutations_to_keep) + /// Do not remove last `storage_settings->finished_mutations_to_keep` entries. + if (entries.size() <= storage_settings->finished_mutations_to_keep) return; - entries.erase(entries.end() - storage.settings.finished_mutations_to_keep, entries.end()); + entries.erase(entries.end() - storage_settings->finished_mutations_to_keep, entries.end()); if (entries.empty()) return; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 665e8c9bd5c..5d044fbf839 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -961,7 +961,8 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( * (because it may be ordered by OPTIMIZE or early with differrent settings). */ UInt64 max_source_parts_size = merger_mutator.getMaxSourcePartsSizeForMerge(); - if (max_source_parts_size != data.settings.max_bytes_to_merge_at_max_space_in_pool + const auto data_settings = data.getCOWSettings(); + if (max_source_parts_size != data_settings->max_bytes_to_merge_at_max_space_in_pool && sum_parts_size_in_bytes > max_source_parts_size) { String reason = "Not executing log entry for part " + entry.new_part_name diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index a98625336c5..6145713492f 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -44,11 +44,12 @@ ReplicatedMergeTreeRestartingThread::ReplicatedMergeTreeRestartingThread(Storage , log(&Logger::get(log_name)) , active_node_identifier(generateActiveNodeIdentifier()) { - check_period_ms = storage.settings.zookeeper_session_expiration_check_period.totalSeconds() * 1000; + const auto storage_settings = storage.getCOWSettings(); + check_period_ms = storage_settings->zookeeper_session_expiration_check_period.totalSeconds() * 1000; /// Periodicity of checking lag of replica. - if (check_period_ms > static_cast(storage.settings.check_delay_period) * 1000) - check_period_ms = storage.settings.check_delay_period * 1000; + if (check_period_ms > static_cast(storage_settings->check_delay_period) * 1000) + check_period_ms = storage_settings->check_delay_period * 1000; task = storage.global_context.getSchedulePool().createTask(log_name, [this]{ run(); }); } @@ -121,7 +122,8 @@ void ReplicatedMergeTreeRestartingThread::run() } time_t current_time = time(nullptr); - if (current_time >= prev_time_of_check_delay + static_cast(storage.settings.check_delay_period)) + const auto storage_settings = storage.getCOWSettings(); + if (current_time >= prev_time_of_check_delay + static_cast(storage_settings->check_delay_period)) { /// Find out lag of replicas. time_t absolute_delay = 0; @@ -136,10 +138,10 @@ void ReplicatedMergeTreeRestartingThread::run() /// We give up leadership if the relative lag is greater than threshold. if (storage.is_leader - && relative_delay > static_cast(storage.settings.min_relative_delay_to_yield_leadership)) + && relative_delay > static_cast(storage_settings->min_relative_delay_to_yield_leadership)) { LOG_INFO(log, "Relative replica delay (" << relative_delay << " seconds) is bigger than threshold (" - << storage.settings.min_relative_delay_to_yield_leadership << "). Will yield leadership."); + << storage_settings->min_relative_delay_to_yield_leadership << "). Will yield leadership."); ProfileEvents::increment(ProfileEvents::ReplicaYieldLeadership); @@ -169,6 +171,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() activateReplica(); const auto & zookeeper = storage.getZooKeeper(); + const auto storage_settings = storage.getCOWSettings(); storage.cloneReplicaIfNeeded(zookeeper); @@ -181,7 +184,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() updateQuorumIfWeHavePart(); - if (storage.settings.replicated_can_become_leader) + if (storage_settings->replicated_can_become_leader) storage.enterLeaderElection(); else LOG_INFO(log, "Will not enter leader election because replicated_can_become_leader=0"); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 10f77358b7d..381d1f47412 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -27,8 +27,9 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr if (data.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) date_column = data.minmax_idx_columns[data.minmax_idx_date_column_pos]; + const auto data_settings = data.getCOWSettings(); sampling_expression = formattedAST(data.sample_by_ast); - index_granularity = data.settings.index_granularity; + index_granularity = data_settings->index_granularity; merging_params_mode = static_cast(data.merging_params.mode); sign_column = data.merging_params.sign_column; @@ -48,7 +49,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr ttl_table = formattedAST(data.ttl_table_ast); skip_indices = data.getIndices().toString(); if (data.canUseAdaptiveGranularity()) - index_granularity_bytes = data.settings.index_granularity_bytes; + index_granularity_bytes = data_settings->index_granularity_bytes; else index_granularity_bytes = 0; } diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 81ead0fd20d..1e9f1a252f4 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -641,13 +641,13 @@ static StoragePtr create(const StorageFactory::Arguments & args) zookeeper_path, replica_name, args.attach, args.data_path, args.database_name, args.table_name, args.columns, indices_description, args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast, - sample_by_ast, ttl_table_ast, merging_params, storage_settings, + sample_by_ast, ttl_table_ast, merging_params, std::move(storage_settings), args.has_force_restore_data_flag); else return StorageMergeTree::create( args.data_path, args.database_name, args.table_name, args.columns, indices_description, args.attach, args.context, date_column_name, partition_by_ast, order_by_ast, - primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, storage_settings, + primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, std::move(storage_settings), args.has_force_restore_data_flag); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 9ed413157a2..f6975e34dfa 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -62,7 +62,7 @@ StorageMergeTree::StorageMergeTree( const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported. const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - const MergeTreeSettings & settings_, + MergeTreeSettingsPtr settings_, bool has_force_restore_data_flag) : MergeTreeData(database_name_, table_name_, path_ + escapeForFileName(table_name_) + '/', @@ -801,14 +801,15 @@ Int64 StorageMergeTree::getCurrentMutationVersion( void StorageMergeTree::clearOldMutations() { - if (!settings.finished_mutations_to_keep) + const auto settings = getCOWSettings(); + if (!settings->finished_mutations_to_keep) return; std::vector mutations_to_delete; { std::lock_guard lock(currently_merging_mutex); - if (current_mutations_by_version.size() <= settings.finished_mutations_to_keep) + if (current_mutations_by_version.size() <= settings->finished_mutations_to_keep) return; auto begin_it = current_mutations_by_version.begin(); @@ -819,10 +820,10 @@ void StorageMergeTree::clearOldMutations() end_it = current_mutations_by_version.upper_bound(*min_version); size_t done_count = std::distance(begin_it, end_it); - if (done_count <= settings.finished_mutations_to_keep) + if (done_count <= settings->finished_mutations_to_keep) return; - size_t to_delete_count = done_count - settings.finished_mutations_to_keep; + size_t to_delete_count = done_count - settings->finished_mutations_to_keep; auto it = begin_it; for (size_t i = 0; i < to_delete_count; ++i) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 0df90604f67..556cf7999b8 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -151,7 +151,7 @@ protected: const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported. const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - const MergeTreeSettings & settings_, + MergeTreeSettingsPtr settings_, bool has_force_restore_data_flag); }; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 0b66e98d0dc..c788c940efc 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -202,7 +202,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const ASTPtr & sample_by_ast_, const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - const MergeTreeSettings & settings_, + MergeTreeSettingsPtr settings_, bool has_force_restore_data_flag) : MergeTreeData(database_name_, table_name_, path_ + escapeForFileName(table_name_) + '/', @@ -376,7 +376,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists() } -/** Verify that list of columns and table settings match those specified in ZK (/ metadata). +/** Verify that list of columns and table storage_settings match those specified in ZK (/ metadata). * If not, throw an exception. */ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bool allow_alter) @@ -633,7 +633,8 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) for (const auto & part : parts) total_rows_on_filesystem += part->rows_count; - bool insane = unexpected_parts_rows > total_rows_on_filesystem * settings.replicated_max_ratio_of_wrong_parts; + const auto storage_settings = getCOWSettings(); + bool insane = unexpected_parts_rows > total_rows_on_filesystem * storage_settings->replicated_max_ratio_of_wrong_parts; if (insane && !skip_sanity_checks) { @@ -776,12 +777,13 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (!has_been_already_added) { + const auto storage_settings = getCOWSettings(); String part_path = replica_path + "/parts/" + part_name; ops.emplace_back(zkutil::makeCheckRequest( zookeeper_path + "/columns", expected_columns_version)); - if (settings.use_minimalistic_part_header_in_zookeeper) + if (storage_settings->use_minimalistic_part_header_in_zookeeper) { ops.emplace_back(zkutil::makeCreateRequest( part_path, local_part_header.toString(), zkutil::CreateMode::Persistent)); @@ -858,7 +860,7 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd String StorageReplicatedMergeTree::getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums) const { return MinimalisticDataPartChecksums::getSerializedString(checksums, - static_cast(settings.use_minimalistic_checksums_in_zookeeper)); + getCOWSettings()->use_minimalistic_checksums_in_zookeeper); } @@ -1029,13 +1031,14 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) parts.push_back(part); } + const auto storage_settings = getCOWSettings(); if (!have_all_parts) { /// If you do not have all the necessary parts, try to take some already merged part from someone. LOG_DEBUG(log, "Don't have all parts for merge " << entry.new_part_name << "; will try to fetch it instead"); return false; } - else if (entry.create_time + settings.prefer_fetch_merged_part_time_threshold.totalSeconds() <= time(nullptr)) + else if (entry.create_time + storage_settings->prefer_fetch_merged_part_time_threshold.totalSeconds() <= time(nullptr)) { /// If entry is old enough, and have enough size, and part are exists in any replica, /// then prefer fetching of merged part from replica. @@ -1044,7 +1047,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) for (const auto & part : parts) sum_parts_bytes_on_disk += part->bytes_on_disk; - if (sum_parts_bytes_on_disk >= settings.prefer_fetch_merged_part_size_threshold) + if (sum_parts_bytes_on_disk >= storage_settings->prefer_fetch_merged_part_size_threshold) { String replica = findReplicaHavingPart(entry.new_part_name, true); /// NOTE excessive ZK requests for same data later, may remove. if (!replica.empty()) @@ -1154,6 +1157,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedMergeTree::LogEntry & entry) { const String & source_part_name = entry.source_parts.at(0); + const auto storage_settings = getCOWSettings(); LOG_TRACE(log, "Executing log entry to mutate part " << source_part_name << " to " << entry.new_part_name); DataPartPtr source_part = getActiveContainingPart(source_part_name); @@ -1173,8 +1177,8 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM /// TODO - some better heuristic? size_t estimated_space_for_result = MergeTreeDataMergerMutator::estimateNeededDiskSpace({source_part}); - if (entry.create_time + settings.prefer_fetch_merged_part_time_threshold.totalSeconds() <= time(nullptr) - && estimated_space_for_result >= settings.prefer_fetch_merged_part_size_threshold) + if (entry.create_time + storage_settings->prefer_fetch_merged_part_time_threshold.totalSeconds() <= time(nullptr) + && estimated_space_for_result >= storage_settings->prefer_fetch_merged_part_size_threshold) { /// If entry is old enough, and have enough size, and some replica has the desired part, /// then prefer fetching from replica. @@ -1268,20 +1272,21 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) { String replica = findReplicaHavingCoveringPart(entry, true); + const auto storage_settings = getCOWSettings(); static std::atomic_uint total_fetches {0}; - if (settings.replicated_max_parallel_fetches && total_fetches >= settings.replicated_max_parallel_fetches) + if (storage_settings->replicated_max_parallel_fetches && total_fetches >= storage_settings->replicated_max_parallel_fetches) { - throw Exception("Too many total fetches from replicas, maximum: " + settings.replicated_max_parallel_fetches.toString(), + throw Exception("Too many total fetches from replicas, maximum: " + storage_settings->replicated_max_parallel_fetches.toString(), ErrorCodes::TOO_MANY_FETCHES); } ++total_fetches; SCOPE_EXIT({--total_fetches;}); - if (settings.replicated_max_parallel_fetches_for_table && current_table_fetches >= settings.replicated_max_parallel_fetches_for_table) + if (storage_settings->replicated_max_parallel_fetches_for_table && current_table_fetches >= storage_settings->replicated_max_parallel_fetches_for_table) { - throw Exception("Too many fetches from replicas for table, maximum: " + settings.replicated_max_parallel_fetches_for_table.toString(), + throw Exception("Too many fetches from replicas for table, maximum: " + storage_settings->replicated_max_parallel_fetches_for_table.toString(), ErrorCodes::TOO_MANY_FETCHES); } @@ -2162,6 +2167,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() if (!is_leader) return; + const auto storage_settings = getCOWSettings(); const bool deduplicate = false; /// TODO: read deduplicate option from table config const bool force_ttl = false; @@ -2181,16 +2187,16 @@ void StorageReplicatedMergeTree::mergeSelectingTask() /// Otherwise merge queue could be filled with only large merges, /// and in the same time, many small parts could be created and won't be merged. size_t merges_and_mutations_queued = queue.countMergesAndPartMutations(); - if (merges_and_mutations_queued >= settings.max_replicated_merges_in_queue) + if (merges_and_mutations_queued >= storage_settings->max_replicated_merges_in_queue) { LOG_TRACE(log, "Number of queued merges and part mutations (" << merges_and_mutations_queued << ") is greater than max_replicated_merges_in_queue (" - << settings.max_replicated_merges_in_queue << "), so won't select new parts to merge or mutate."); + << storage_settings->max_replicated_merges_in_queue << "), so won't select new parts to merge or mutate."); } else { UInt64 max_source_parts_size_for_merge = merger_mutator.getMaxSourcePartsSizeForMerge( - settings.max_replicated_merges_in_queue, merges_and_mutations_queued); + storage_settings->max_replicated_merges_in_queue, merges_and_mutations_queued); UInt64 max_source_part_size_for_mutation = merger_mutator.getMaxSourcePartSizeForMutation(); FutureMergedMutatedPart future_merged_part; @@ -2973,10 +2979,11 @@ void StorageReplicatedMergeTree::assertNotReadonly() const BlockOutputStreamPtr StorageReplicatedMergeTree::write(const ASTPtr & /*query*/, const Context & context) { + const auto storage_settings = getCOWSettings(); assertNotReadonly(); const Settings & query_settings = context.getSettingsRef(); - bool deduplicate = settings.replicated_deduplication_window != 0 && query_settings.insert_deduplicate; + bool deduplicate = storage_settings->replicated_deduplication_window != 0 && query_settings.insert_deduplicate; return std::make_shared(*this, query_settings.insert_quorum, query_settings.insert_quorum_timeout.totalMilliseconds(), query_settings.max_partitions_per_insert_block, deduplicate); @@ -3010,6 +3017,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p }; bool force_ttl = (final && (hasTableTTL() || hasAnyColumnTTL())); + const auto storage_settings = getCOWSettings(); if (!partition && final) { @@ -3042,7 +3050,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p if (!partition) { selected = merger_mutator.selectPartsToMerge( - future_merged_part, true, settings.max_bytes_to_merge_at_max_space_in_pool, can_merge, &disable_reason); + future_merged_part, true, storage_settings->max_bytes_to_merge_at_max_space_in_pool, can_merge, &disable_reason); } else { @@ -3092,9 +3100,9 @@ void StorageReplicatedMergeTree::alter( if (params.isSettingsAlter()) { - /// We don't replicate settings ALTER. It's local operation. + /// We don't replicate storage_settings ALTER. It's local operation. /// Also we don't upgrade alter lock to table structure lock. - LOG_DEBUG(log, "ALTER settings only"); + LOG_DEBUG(log, "ALTER storage_settings only"); SettingsChanges new_changes; params.applyForSettingsOnly(new_changes); alterSettings(new_changes, current_database_name, current_table_name, query_context, table_lock_holder); @@ -3920,9 +3928,10 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) { auto zookeeper = tryGetZooKeeper(); + const auto storage_settings = getCOWSettings(); res.is_leader = is_leader; - res.can_become_leader = settings.replicated_can_become_leader; + res.can_become_leader = storage_settings->replicated_can_become_leader; res.is_readonly = is_readonly; res.is_session_expired = !zookeeper || zookeeper->expired(); @@ -4112,13 +4121,14 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t out_absolute_delay = getAbsoluteDelay(); out_relative_delay = 0; + const auto storage_settings = getCOWSettings(); /** Relative delay is the maximum difference of absolute delay from any other replica, * (if this replica lags behind any other live replica, or zero, otherwise). * Calculated only if the absolute delay is large enough. */ - if (out_absolute_delay < static_cast(settings.min_relative_delay_to_yield_leadership)) + if (out_absolute_delay < static_cast(storage_settings->min_relative_delay_to_yield_leadership)) return; auto zookeeper = getZooKeeper(); @@ -4376,7 +4386,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const /// instead. /// /// Mutations of individual parts are in fact pretty similar to merges, e.g. their assignment and execution - /// is governed by the same settings. TODO: support a single "merge-mutation" operation when the data + /// is governed by the same storage_settings. TODO: support a single "merge-mutation" operation when the data /// read from the the source parts is first mutated on the fly to some uniform mutation version and then /// merged to a resulting part. /// @@ -4939,6 +4949,7 @@ void StorageReplicatedMergeTree::getCommitPartOps( const String & block_id_path) const { const String & part_name = part->name; + const auto storage_settings = getCOWSettings(); if (!block_id_path.empty()) { @@ -4956,7 +4967,7 @@ void StorageReplicatedMergeTree::getCommitPartOps( zookeeper_path + "/columns", columns_version)); - if (settings.use_minimalistic_part_header_in_zookeeper) + if (storage_settings->use_minimalistic_part_header_in_zookeeper) { ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, @@ -4985,11 +4996,12 @@ void StorageReplicatedMergeTree::updatePartHeaderInZooKeeperAndCommit( AlterDataPartTransaction & transaction) { String part_path = replica_path + "/parts/" + transaction.getPartName(); + const auto storage_settings = getCOWSettings(); bool need_delete_columns_and_checksums_nodes = false; try { - if (settings.use_minimalistic_part_header_in_zookeeper) + if (storage_settings->use_minimalistic_part_header_in_zookeeper) { auto part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( transaction.getNewColumns(), transaction.getNewChecksums()); @@ -5169,8 +5181,9 @@ CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, const C bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const { - return settings.index_granularity_bytes != 0 && - (settings.enable_mixed_granularity_parts || + const auto storage_settings = getCOWSettings(); + return storage_settings->index_granularity_bytes != 0 && + (storage_settings->enable_mixed_granularity_parts || (!has_non_adaptive_index_granularity_parts && !other_replicas_fixed_granularity)); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 7f632ab4cb4..a32a9f19473 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -534,7 +534,7 @@ protected: const ASTPtr & sample_by_ast_, const ASTPtr & table_ttl_ast_, const MergingParams & merging_params_, - const MergeTreeSettings & settings_, + MergeTreeSettingsPtr settings_, bool has_force_restore_data_flag); }; From 9e78781378fba7cae91239ffd02cdfcbd5314e57 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 13:39:06 +0300 Subject: [PATCH 104/357] Fix tests --- dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql | 4 ++-- .../0_stateless/00980_zookeeper_merge_tree_alter_settings.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql index 866cf960710..973d9b95d20 100644 --- a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql @@ -5,7 +5,7 @@ CREATE TABLE log_for_alter ( Data String ) ENGINE = Log(); -ALTER TABLE log_for_alter MODIFY SETTING aaa=123; -- { serverError 468 } +ALTER TABLE log_for_alter MODIFY SETTING aaa=123; -- { serverError 469 } DROP TABLE IF EXISTS log_for_alter; @@ -16,7 +16,7 @@ CREATE TABLE table_for_alter ( Data String ) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096; -ALTER TABLE table_for_alter MODIFY SETTING index_granularity=555; -- { serverError 469 } +ALTER TABLE table_for_alter MODIFY SETTING index_granularity=555; -- { serverError 470 } SHOW CREATE TABLE table_for_alter; diff --git a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql index 9c7a7c33329..24c1135c247 100644 --- a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql +++ b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql @@ -13,7 +13,7 @@ CREATE TABLE replicated_table_for_alter2 ( SHOW CREATE TABLE replicated_table_for_alter1; -ALTER TABLE replicated_table_for_alter1 MODIFY SETTING index_granularity = 4096; -- { serverError 469 } +ALTER TABLE replicated_table_for_alter1 MODIFY SETTING index_granularity = 4096; -- { serverError 470 } SHOW CREATE TABLE replicated_table_for_alter1; From 1b68d79c59e7037f52263a3d1ae1331037cb16f0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 13:56:58 +0300 Subject: [PATCH 105/357] Revert immutable settings --- dbms/src/Core/SettingsCommon.cpp | 129 +++++------------------------- dbms/src/Core/SettingsCommon.h | 133 +++++++------------------------ 2 files changed, 49 insertions(+), 213 deletions(-) diff --git a/dbms/src/Core/SettingsCommon.cpp b/dbms/src/Core/SettingsCommon.cpp index 2d3ed4f6e14..988a4b2d736 100644 --- a/dbms/src/Core/SettingsCommon.cpp +++ b/dbms/src/Core/SettingsCommon.cpp @@ -34,19 +34,20 @@ namespace ErrorCodes template String SettingNumber::toString() const { - return DB::toString(getValue()); + return DB::toString(value); } template Field SettingNumber::toField() const { - return getValue(); + return value; } template void SettingNumber::set(Type x) { - data.store(Data{x, true}, std::memory_order_relaxed); + value = x; + changed = true; } template @@ -58,14 +59,6 @@ void SettingNumber::set(const Field & x) set(applyVisitor(FieldVisitorConvertToNumber(), x)); } - -template -SettingNumber & SettingNumber::operator= (const SettingNumber & o) -{ - data.store(o.data.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; -} - template void SettingNumber::set(const String & x) { @@ -100,9 +93,9 @@ template void SettingNumber::serialize(WriteBuffer & buf) const { if constexpr (std::is_integral_v && std::is_unsigned_v) - writeVarUInt(static_cast(getValue()), buf); + writeVarUInt(static_cast(value), buf); else if constexpr (std::is_integral_v && std::is_signed_v) - writeVarInt(static_cast(getValue()), buf); + writeVarInt(static_cast(value), buf); else { static_assert(std::is_floating_point_v); @@ -140,28 +133,22 @@ template struct SettingNumber; template struct SettingNumber; -SettingMaxThreads & SettingMaxThreads::operator= (const SettingMaxThreads & o) -{ - data.store(o.data.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; -} - String SettingMaxThreads::toString() const { - auto d = data.load(std::memory_order_relaxed); /// Instead of the `auto` value, we output the actual value to make it easier to see. - return d.is_auto ? ("auto(" + DB::toString(d.value) + ")") : DB::toString(d.value); + return is_auto ? ("auto(" + DB::toString(value) + ")") : DB::toString(value); } Field SettingMaxThreads::toField() const { - auto d = data.load(std::memory_order_relaxed); - return d.is_auto ? 0 : d.value; + return is_auto ? 0 : value; } void SettingMaxThreads::set(UInt64 x) { - data.store({x ? x : getAutoValue(), x == 0, true}); + value = x ? x : getAutoValue(); + is_auto = x == 0; + changed = true; } void SettingMaxThreads::set(const Field & x) @@ -182,8 +169,7 @@ void SettingMaxThreads::set(const String & x) void SettingMaxThreads::serialize(WriteBuffer & buf) const { - auto d = data.load(std::memory_order_relaxed); - writeVarUInt(d.is_auto ? 0 : d.value, buf); + writeVarUInt(is_auto ? 0 : value, buf); } void SettingMaxThreads::deserialize(ReadBuffer & buf) @@ -195,7 +181,8 @@ void SettingMaxThreads::deserialize(ReadBuffer & buf) void SettingMaxThreads::setAuto() { - data.store({getAutoValue(), true, isChanged()}); + value = getAutoValue(); + is_auto = true; } UInt64 SettingMaxThreads::getAutoValue() const @@ -204,54 +191,22 @@ UInt64 SettingMaxThreads::getAutoValue() const return res; } -void SettingMaxThreads::setChanged(bool changed) -{ - auto d = data.load(std::memory_order_relaxed); - data.store({d.value, d.is_auto, changed}); -} - - -template -SettingTimespan & SettingTimespan::operator= (const SettingTimespan & o) -{ - std::shared_lock lock_o(o.mutex); - value = o.value; - changed = o.changed; - return *this; -} - -template -SettingTimespan::SettingTimespan(const SettingTimespan & o) -{ - std::shared_lock lock_o(o.mutex); - value = o.value; - changed = o.changed; -} - - -template -void SettingTimespan::setChanged(bool c) -{ - std::unique_lock lock(mutex); - changed = c; -} template String SettingTimespan::toString() const { - return DB::toString(getValue().totalMicroseconds() / microseconds_per_io_unit); + return DB::toString(value.totalMicroseconds() / microseconds_per_io_unit); } template Field SettingTimespan::toField() const { - return getValue().totalMicroseconds() / microseconds_per_io_unit; + return value.totalMicroseconds() / microseconds_per_io_unit; } template void SettingTimespan::set(const Poco::Timespan & x) { - std::unique_lock lock(mutex); value = x; changed = true; } @@ -280,7 +235,7 @@ void SettingTimespan::set(const String & x) template void SettingTimespan::serialize(WriteBuffer & buf) const { - writeVarUInt(getValue().totalMicroseconds() / microseconds_per_io_unit, buf); + writeVarUInt(value.totalMicroseconds() / microseconds_per_io_unit, buf); } template @@ -294,47 +249,23 @@ void SettingTimespan::deserialize(ReadBuffer & buf) template struct SettingTimespan; template struct SettingTimespan; -SettingString & SettingString::operator= (const SettingString & o) -{ - std::shared_lock lock_o(o.mutex); - value = o.value; - changed = o.changed; - return *this; -} - -SettingString::SettingString(const SettingString & o) -{ - std::shared_lock lock(o.mutex); - value = o.value; - changed = o.changed; -} - String SettingString::toString() const { - std::shared_lock lock(mutex); return value; } Field SettingString::toField() const { - std::shared_lock lock(mutex); return value; } void SettingString::set(const String & x) { - std::unique_lock lock(mutex); value = x; changed = true; } -void SettingString::setChanged(bool c) -{ - std::unique_lock lock(mutex); - changed = c; -} - void SettingString::set(const Field & x) { set(safeGet(x)); @@ -352,15 +283,10 @@ void SettingString::deserialize(ReadBuffer & buf) set(s); } -SettingChar & SettingChar::operator= (const SettingChar & o) -{ - data.store(o.data.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; -} String SettingChar::toString() const { - return String(1, getValue()); + return String(1, value); } Field SettingChar::toField() const @@ -370,7 +296,8 @@ Field SettingChar::toField() const void SettingChar::set(char x) { - data.store({x, true}); + value = x; + changed = true; } void SettingChar::set(const String & x) @@ -400,19 +327,6 @@ void SettingChar::deserialize(ReadBuffer & buf) } -template -SettingEnum & SettingEnum::operator= (const SettingEnum & o) -{ - data.store(o.data.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; -} - -template -void SettingEnum::set(EnumType x) -{ - data.store({x, true}, std::memory_order_relaxed); -} - template void SettingEnum::serialize(WriteBuffer & buf) const { @@ -428,7 +342,6 @@ void SettingEnum::deserialize(ReadBuffer & buf) } - #define IMPLEMENT_SETTING_ENUM(ENUM_NAME, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME) \ IMPLEMENT_SETTING_ENUM_WITH_TAG(ENUM_NAME, void, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME) @@ -438,7 +351,7 @@ void SettingEnum::deserialize(ReadBuffer & buf) { \ using EnumType = ENUM_NAME; \ using UnderlyingType = std::underlying_type::type; \ - switch (static_cast(getValue())) \ + switch (static_cast(value)) \ { \ LIST_OF_NAMES_MACRO(IMPLEMENT_SETTING_ENUM_TO_STRING_HELPER_) \ } \ diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index 83d90c28f53..d4607e70904 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -8,8 +8,6 @@ #include #include #include -#include -#include namespace DB @@ -34,25 +32,13 @@ namespace ErrorCodes template struct SettingNumber { - struct Data - { - Type value; - bool changed; - }; + Type value; + bool changed = false; - std::atomic data; - - SettingNumber(Type x = 0) : data{{x, false}} {} - SettingNumber(const SettingNumber & o) : data{o.data.load(std::memory_order_relaxed)} {} - - bool isChanged() const { return data.load(std::memory_order_relaxed).changed; } - void setChanged(bool changed) { data.store({getValue(), changed}, std::memory_order_relaxed); } - - operator Type() const { return getValue(); } - Type getValue() const { return data.load(std::memory_order_relaxed).value; } + SettingNumber(Type x = 0) : value(x) {} + operator Type() const { return value; } SettingNumber & operator= (Type x) { set(x); return *this; } - SettingNumber & operator= (const SettingNumber & o); /// Serialize to a test string. String toString() const; @@ -87,26 +73,14 @@ using SettingBool = SettingNumber; */ struct SettingMaxThreads { - struct Data - { - UInt64 value; - bool is_auto; - bool changed; - }; + UInt64 value; + bool is_auto; + bool changed = false; - std::atomic data; - - SettingMaxThreads(UInt64 x = 0) : data{{x ? x : getAutoValue(), x == 0, false}} {} - SettingMaxThreads(const SettingMaxThreads & o) : data{o.data.load(std::memory_order_relaxed)} {} - - bool isChanged() const { return data.load(std::memory_order_relaxed).changed; } - void setChanged(bool changed); - - operator UInt64() const { return getValue(); } - UInt64 getValue() const { return data.load(std::memory_order_relaxed).value; } + SettingMaxThreads(UInt64 x = 0) : value(x ? x : getAutoValue()), is_auto(x == 0) {} + operator UInt64() const { return value; } SettingMaxThreads & operator= (UInt64 x) { set(x); return *this; } - SettingMaxThreads & operator= (const SettingMaxThreads & o); String toString() const; Field toField() const; @@ -118,7 +92,6 @@ struct SettingMaxThreads void serialize(WriteBuffer & buf) const; void deserialize(ReadBuffer & buf); - bool isAuto() const { return data.load(std::memory_order_relaxed).is_auto; } void setAuto(); UInt64 getAutoValue() const; }; @@ -129,37 +102,16 @@ enum class SettingTimespanIO { MILLISECOND, SECOND }; template struct SettingTimespan { - mutable std::shared_mutex mutex; Poco::Timespan value; bool changed = false; SettingTimespan(UInt64 x = 0) : value(x * microseconds_per_io_unit) {} - SettingTimespan(const SettingTimespan & o); - operator Poco::Timespan() const { return getValue(); } - Poco::Timespan getValue() const { std::shared_lock lock(mutex); return value; } + operator Poco::Timespan() const { return value; } SettingTimespan & operator= (const Poco::Timespan & x) { set(x); return *this; } - SettingTimespan & operator= (const SettingTimespan & o); - Poco::Timespan::TimeDiff totalSeconds() const - { - std::shared_lock lock(mutex); - return value.totalSeconds(); - } - - Poco::Timespan::TimeDiff totalMilliseconds() const - { - std::shared_lock lock(mutex); - return value.totalMilliseconds(); - } - - bool isChanged() const - { - std::shared_lock lock(mutex); - return changed; - } - - void setChanged(bool changed); + Poco::Timespan::TimeDiff totalSeconds() const { return value.totalSeconds(); } + Poco::Timespan::TimeDiff totalMilliseconds() const { return value.totalMilliseconds(); } String toString() const; Field toField() const; @@ -182,19 +134,13 @@ using SettingMilliseconds = SettingTimespan; struct SettingString { - mutable std::shared_mutex mutex; String value; bool changed = false; SettingString(const String & x = String{}) : value(x) {} - SettingString(const SettingString & o); - operator String() const { return getValue(); } - String getValue() const { std::shared_lock lock(mutex); return value; } + operator String() const { return value; } SettingString & operator= (const String & x) { set(x); return *this; } - SettingString & operator= (const SettingString & o); - bool isChanged() const { std::shared_lock lock(mutex); return changed; } - void setChanged(bool changed); String toString() const; Field toField() const; @@ -210,25 +156,13 @@ struct SettingString struct SettingChar { public: - struct Data - { - char value; - bool changed; - }; + char value; + bool changed = false; - std::atomic data; - - SettingChar(char x = '\0') : data({x, false}) {} - SettingChar(const SettingChar & o) : data{o.data.load(std::memory_order_relaxed)} {} - - operator char() const { return getValue(); } - char getValue() const { return data.load(std::memory_order_relaxed).value; } + SettingChar(char x = '\0') : value(x) {} + operator char() const { return value; } SettingChar & operator= (char x) { set(x); return *this; } - SettingChar & operator= (const SettingChar & o); - - bool isChanged() const { return data.load(std::memory_order_relaxed).changed; } - void setChanged(bool changed) { data.store({getValue(), changed}, std::memory_order_relaxed);} String toString() const; Field toField() const; @@ -246,30 +180,18 @@ public: template struct SettingEnum { - struct Data - { - EnumType value; - bool changed; - }; + EnumType value; + bool changed = false; - std::atomic data; - - SettingEnum(EnumType x) : data({x, false}) {} - SettingEnum(const SettingEnum & o) : data{o.data.load(std::memory_order_relaxed)} {} - - operator EnumType() const { return getValue(); } - EnumType getValue() const { return data.load(std::memory_order_relaxed).value; } + SettingEnum(EnumType x) : value(x) {} + operator EnumType() const { return value; } SettingEnum & operator= (EnumType x) { set(x); return *this; } - SettingEnum & operator= (const SettingEnum & o); - - bool isChanged() const { return data.load(std::memory_order_relaxed).changed; } - void setChanged(bool changed) { data.store({getValue(), changed}, std::memory_order_relaxed);} String toString() const; Field toField() const { return toString(); } - void set(EnumType x); + void set(EnumType x) { value = x; changed = true; } void set(const Field & x) { set(safeGet(x)); } void set(const String & x); @@ -386,7 +308,6 @@ private: Derived & castToDerived() { return *static_cast(this); } const Derived & castToDerived() const { return *static_cast(this); } - using IsChangedFunction = bool (*)(const Derived &); using GetStringFunction = String (*)(const Derived &); using GetFieldFunction = Field (*)(const Derived &); using SetStringFunction = void (*)(Derived &, const String &); @@ -397,7 +318,7 @@ private: struct MemberInfo { - IsChangedFunction is_changed; + size_t offset_of_changed; StringRef name; StringRef description; /// Can be updated after first load for config/definition. @@ -412,7 +333,7 @@ private: DeserializeFunction deserialize; CastValueWithoutApplyingFunction cast_value_without_applying; - bool isChanged(const Derived & collection) const { return is_changed(collection); } + bool isChanged(const Derived & collection) const { return *reinterpret_cast(reinterpret_cast(&collection) + offset_of_changed); } }; class MemberInfos @@ -772,7 +693,8 @@ public: #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ - add({[](const Derived & d) { return d.NAME.isChanged(); }, \ + static_assert(std::is_same_v().NAME.changed), bool>); \ + add({offsetof(Derived, NAME.changed), \ StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ @@ -780,7 +702,8 @@ public: &Functions::NAME##_castValueWithoutApplying }); #define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ - add({[](const Derived & d) { return d.NAME.isChanged(); }, \ + static_assert(std::is_same_v().NAME.changed), bool>); \ + add({offsetof(Derived, NAME.changed), \ StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ From 7b1039568ce987f02329e347251e779070d67380 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 14:24:18 +0300 Subject: [PATCH 106/357] Revert old changed --- dbms/programs/client/Client.cpp | 4 ++-- dbms/programs/copier/ClusterCopier.cpp | 2 +- dbms/src/Core/SettingsCommon.h | 13 ++++++------- dbms/src/Interpreters/ClusterProxy/executeQuery.cpp | 6 +++--- dbms/src/Interpreters/SyntaxAnalyzer.cpp | 2 +- dbms/src/Storages/Kafka/StorageKafka.cpp | 4 ++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- dbms/src/Storages/MergeTree/MergeTreeSettings.cpp | 2 +- 8 files changed, 18 insertions(+), 19 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index f509ebf8c1f..feffd7e6cfe 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -225,7 +225,7 @@ private: for (auto && setting : context.getSettingsRef()) { const String & name = setting.getName().toString(); - if (config().has(name) && !setting.isChanged()) + if (config().has(name) && setting.isChanged()) setting.setValue(config().getString(name)); } @@ -323,7 +323,7 @@ private: insert_format = "Values"; /// Setting value from cmd arg overrides one from config - if (context.getSettingsRef().max_insert_block_size.isChanged()) + if (context.getSettingsRef().max_insert_block_size.changed) insert_format_max_block_size = context.getSettingsRef().max_insert_block_size; else insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 5ced5677569..435d06da854 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -708,7 +708,7 @@ void DB::TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & c auto set_default_value = [] (auto && setting, auto && default_value) { - setting = setting.isChanged() ? setting.getValue() : default_value; + setting = setting.changed ? setting.value : default_value; }; /// Override important settings diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index d4607e70904..1410dc2611f 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -308,6 +308,7 @@ private: Derived & castToDerived() { return *static_cast(this); } const Derived & castToDerived() const { return *static_cast(this); } + using IsChangedFunction = bool (*)(const Derived &); using GetStringFunction = String (*)(const Derived &); using GetFieldFunction = Field (*)(const Derived &); using SetStringFunction = void (*)(Derived &, const String &); @@ -318,11 +319,11 @@ private: struct MemberInfo { - size_t offset_of_changed; + IsChangedFunction is_changed; StringRef name; StringRef description; /// Can be updated after first load for config/definition. - /// Non updatable settings can be isChanged, + /// Non updatable settings can be `changed`, /// if they were overwritten in config/definition. bool updateable; GetStringFunction get_string; @@ -333,7 +334,7 @@ private: DeserializeFunction deserialize; CastValueWithoutApplyingFunction cast_value_without_applying; - bool isChanged(const Derived & collection) const { return *reinterpret_cast(reinterpret_cast(&collection) + offset_of_changed); } + bool isChanged(const Derived & collection) const { return is_changed(collection); } }; class MemberInfos @@ -693,8 +694,7 @@ public: #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ - static_assert(std::is_same_v().NAME.changed), bool>); \ - add({offsetof(Derived, NAME.changed), \ + add({[](const Derived & d) { return d.NAME.changed; }, \ StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ @@ -702,8 +702,7 @@ public: &Functions::NAME##_castValueWithoutApplying }); #define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ - static_assert(std::is_same_v().NAME.changed), bool>); \ - add({offsetof(Derived, NAME.changed), \ + add({[](const Derived & d) { return d.NAME.changed; }, \ StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index e4e52eab2b7..25d5a6eb0d4 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -26,9 +26,9 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin new_settings.max_memory_usage_for_all_queries = 0; /// Set as unchanged to avoid sending to remote server. - new_settings.max_concurrent_queries_for_user.setChanged(false); - new_settings.max_memory_usage_for_user.setChanged(false); - new_settings.max_memory_usage_for_all_queries.setChanged(false); + new_settings.max_concurrent_queries_for_user.changed = false; + new_settings.max_memory_usage_for_user.changed = false; + new_settings.max_memory_usage_for_all_queries = false; Context new_context(context); new_context.setSettings(new_settings); diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index dfd0ec63643..282b19991b1 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -788,7 +788,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( InJoinSubqueriesPreprocessor(context).visit(query); /// Optimizes logical expressions. - LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.getValue()).perform(); + LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.value).perform(); } /// Creates a dictionary `aliases`: alias -> ASTPtr diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index afd26fde113..f5a9c2ac4b6 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -440,7 +440,7 @@ void registerStorageKafka(StorageFactory & factory) #define CHECK_KAFKA_STORAGE_ARGUMENT(ARG_NUM, PAR_NAME) \ /* One of the four required arguments is not specified */ \ if (args_count < ARG_NUM && ARG_NUM <= 4 && \ - !kafka_settings.PAR_NAME.isChanged()) \ + !kafka_settings.PAR_NAME.changed) \ { \ throw Exception( \ "Required parameter '" #PAR_NAME "' " \ @@ -449,7 +449,7 @@ void registerStorageKafka(StorageFactory & factory) } \ /* The same argument is given in two places */ \ if (has_settings && \ - kafka_settings.PAR_NAME.isChanged() && \ + kafka_settings.PAR_NAME.changed && \ args_count >= ARG_NUM) \ { \ throw Exception( \ diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 314967b485d..15a19bbc924 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -400,8 +400,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( if (relative_sample_size == RelativeSize(0)) relative_sample_size = 1; - relative_sample_size /= settings.parallel_replicas_count.getValue(); - relative_sample_offset += relative_sample_size * RelativeSize(settings.parallel_replica_offset.getValue()); + relative_sample_size /= settings.parallel_replicas_count.value; + relative_sample_offset += relative_sample_size * RelativeSize(settings.parallel_replica_offset.value); } if (relative_sample_offset >= RelativeSize(1)) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index 7c3c7ef02cc..5bc44eee842 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -69,7 +69,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) if (std::find_if(changes.begin(), changes.end(), \ [](const SettingChange & c) { return c.name == #NAME; }) \ == changes.end()) \ - changes.push_back(SettingChange{#NAME, NAME.getValue()}); + changes.push_back(SettingChange{#NAME, NAME.value}); APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(ADD_IF_ABSENT) #undef ADD_IF_ABSENT From a4d6e8d5b2f9f53487599ad510f471d4e5589492 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 14:28:28 +0300 Subject: [PATCH 107/357] Revert latomic --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac0224ac56b..d369dca7e78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -350,7 +350,7 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L endif () # Add Libc. GLIBC is actually a collection of interdependent libraries. - set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc -latomic") + set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc") # Note: we'd rather use Musl libc library, but it's little bit more difficult to use. From bf5ec73582d413d9bb3dca1b32877bcad86eae86 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 14:40:45 +0300 Subject: [PATCH 108/357] Fix clang build --- dbms/src/Interpreters/Context.cpp | 7 ++++--- dbms/src/Storages/MergeTree/MergeTreeSettings.h | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 0b0cf657987..17b031dbd64 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -148,7 +148,7 @@ struct ContextShared std::unique_ptr ddl_worker; /// Process ddl commands from zk. /// Rules for selecting the compression settings, depending on the size of the part. mutable std::unique_ptr compression_codec_selector; - std::optional merge_tree_settings; /// Settings of MergeTree* engines. + MergeTreeSettingsPtr merge_tree_settings; /// Settings of MergeTree* engines. size_t max_table_size_to_drop = 50000000000lu; /// Protects MergeTree tables from accidental DROP (50GB by default) size_t max_partition_size_to_drop = 50000000000lu; /// Protects MergeTree partitions from accidental DROP (50GB by default) String format_schema_path; /// Path to a directory that contains schema files used by input formats. @@ -1759,8 +1759,9 @@ const MergeTreeSettings & Context::getMergeTreeSettings() const if (!shared->merge_tree_settings) { auto & config = getConfigRef(); - shared->merge_tree_settings.emplace(); - shared->merge_tree_settings->loadFromConfig("merge_tree", config); + MutableMergeTreeSettingsPtr settings_ptr = MergeTreeSettings::create(); + settings_ptr->loadFromConfig("merge_tree", config); + shared->merge_tree_settings = std::move(settings_ptr); } return *shared->merge_tree_settings; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index dff7cc2e523..c098aac2526 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -101,7 +101,9 @@ struct MergeTreeSettings : public SettingsCollection, public void loadFromQuery(ASTStorage & storage_def); MutablePtr clone() const; - ~MergeTreeSettings() {} +private: + MergeTreeSettings() = default; + MergeTreeSettings(const MergeTreeSettings & o) = default; }; using MergeTreeSettingsPtr = MergeTreeSettings::Ptr; From 5b85373c566293aa52cd6720525f35021d57b619 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 14:58:59 +0300 Subject: [PATCH 109/357] Fix stupid bug --- dbms/programs/client/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index feffd7e6cfe..cf72d7a87c3 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -225,7 +225,7 @@ private: for (auto && setting : context.getSettingsRef()) { const String & name = setting.getName().toString(); - if (config().has(name) && setting.isChanged()) + if (config().has(name) && !setting.isChanged()) setting.setValue(config().getString(name)); } From b00903330fe26276049929cb30079129e5d8bf85 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 15:03:42 +0300 Subject: [PATCH 110/357] Remove empty block --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 4d47f60a2fe..97d74d7a037 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1644,12 +1644,11 @@ void MergeTreeData::alterSettings( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - { - MutableMergeTreeSettingsPtr settings = std::move(*guarded_settings.getPtr()).mutate(); - settings->updateFromChanges(new_changes); - IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); - guarded_settings.setPtr(std::move(settings)); - } + /// No additional locking required, because we took table_lock_holder + MutableMergeTreeSettingsPtr settings = std::move(*guarded_settings.getPtr()).mutate(); + settings->updateFromChanges(new_changes); + IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); + guarded_settings.setPtr(std::move(settings)); } bool MergeTreeData::hasSetting(const String & setting_name) const From f988090111cf28fb8a8a300cabda0aa0d5d8ae21 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 15:10:58 +0300 Subject: [PATCH 111/357] Remove redundant move --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 97d74d7a037..aab384552cc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -118,7 +118,7 @@ MergeTreeData::MergeTreeData( full_path(full_path_), broken_part_callback(broken_part_callback_), log_name(database_name + "." + table_name), log(&Logger::get(log_name)), - guarded_settings(std::move(settings_)), + guarded_settings(settings_), data_parts_by_info(data_parts_indexes.get()), data_parts_by_state_and_info(data_parts_indexes.get()) { From ea9cf3a62f42f0ed48efe7baac2be46bfaa7ae5e Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Tue, 13 Aug 2019 16:11:24 +0300 Subject: [PATCH 112/357] Done --- dbms/src/Functions/neighbour.cpp | 105 +++++++++--------- .../0_stateless/00957_neighbour.reference | 42 +++++++ .../queries/0_stateless/00957_neighbour.sql | 30 +++++ .../0_stateless/00957_next_in_block.reference | 12 -- .../0_stateless/00957_next_in_block.sql | 18 --- .../functions/other_functions.md | 43 +++++++ .../functions/other_functions.md | 42 +++++++ 7 files changed, 209 insertions(+), 83 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00957_neighbour.reference create mode 100644 dbms/tests/queries/0_stateless/00957_neighbour.sql delete mode 100644 dbms/tests/queries/0_stateless/00957_next_in_block.reference delete mode 100644 dbms/tests/queries/0_stateless/00957_next_in_block.sql diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index 3eff660c996..79a1feec002 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -119,9 +119,6 @@ public: default_values_column = block.getByPosition(arguments[2]).column; } -// Field offset_field = (*block.getByPosition(arguments[1]).column)[0]; -// auto raw_value = safeGet(offset_field); - ColumnWithTypeAndName &source_column_name_and_type = block.getByPosition(arguments[0]); DataTypes types = {source_column_name_and_type.type}; if (default_values_column) @@ -140,72 +137,74 @@ public: default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); } - auto column = result_type->createColumn(); - column->reserve(input_rows_count); - const DataTypePtr desired_type = std::make_shared(); if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) { offset_column = castColumn(offset_structure, desired_type, context); } - - // with constant offset - insertRangeFrom - if (is_constant_offset) - { - Int64 offset_value = offset_column->getInt(0); - if (offset_value > 0) - { - // insert shifted value - if ((size_t)std::abs(offset_value) <= input_rows_count) { - column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); - // insert defaults into the end - insertDefaults(column, input_rows_count - offset_value, default_values_column, offset_value); - } - } else if(offset_value < 0) { - // insert defaults up to offset_value - insertDefaults(column, input_rows_count - std::abs(offset_value), default_values_column, std::abs(offset_value)); - // insert range, where possible - column->insertRangeFrom(*source_column, 0, input_rows_count - std::abs(offset_value)); - } else { - // populate column with source values - column->insertRangeFrom(*source_column, 0, input_rows_count); - } + if (isColumnConst(*source_column)) { + auto column = result_type->createColumnConst(input_rows_count, (*source_column)[0]); + block.getByPosition(result).column = std::move(column); } else { - // with dynamic offset - handle row by row - for (size_t row = 0; row < input_rows_count; row++) + auto column = result_type->createColumn(); + column->reserve(input_rows_count); + // with constant offset - insertRangeFrom + if (is_constant_offset) { - Int64 offset_value = offset_column->getInt(row); - if (offset_value == 0) { - column->insertFrom(*source_column, row); - } else if (offset_value > 0) { - size_t real_offset = row + offset_value; - if (real_offset > input_rows_count) { - if (default_values_column) { - column->insertFrom(*default_values_column, row); - } else { - column->insertDefault(); - } - } else { - column->insertFrom(*column, real_offset); + Int64 offset_value = offset_column->getInt(0); + + if (offset_value > 0) + { + // insert shifted value + if ((size_t)offset_value <= input_rows_count) { + column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); } + size_t row_count = (size_t)offset_value > input_rows_count ? input_rows_count : offset_value; + insertDefaults(column, row_count, default_values_column, input_rows_count - row_count); + } else if (offset_value < 0) { + size_t row_count = (size_t)std::abs(offset_value) > input_rows_count ? input_rows_count : std::abs(offset_value); + // insert defaults up to offset_value + insertDefaults(column, row_count, default_values_column, 0); + column->insertRangeFrom(*source_column, 0, input_rows_count - row_count); } else { - // out of range - if ((size_t)std::abs(offset_value) > row) - { - if (default_values_column) { - column->insertFrom(*default_values_column, row); + // populate column with source values + column->insertRangeFrom(*source_column, 0, input_rows_count); + } + } else { + // with dynamic offset - handle row by row + for (size_t row = 0; row < input_rows_count; row++) + { + Int64 offset_value = offset_column->getInt(row); + if (offset_value == 0) { + column->insertFrom(*source_column, row); + } else if (offset_value > 0) { + size_t real_offset = row + offset_value; + if (real_offset > input_rows_count) { + if (default_values_column) { + column->insertFrom(*default_values_column, row); + } else { + column->insertDefault(); + } } else { - column->insertDefault(); + column->insertFrom(*column, real_offset); } } else { - column->insertFrom(*column, row - std::abs(offset_value)); + // out of range + if ((size_t)std::abs(offset_value) > row) + { + if (default_values_column) { + column->insertFrom(*default_values_column, row); + } else { + column->insertDefault(); + } + } else { + column->insertFrom(*column, row - std::abs(offset_value)); + } } } } + block.getByPosition(result).column = std::move(column); } - - - block.getByPosition(result).column = std::move(column); } private: const Context & context; diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.reference b/dbms/tests/queries/0_stateless/00957_neighbour.reference new file mode 100644 index 00000000000..cd8c6310f22 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_neighbour.reference @@ -0,0 +1,42 @@ +Result with different type +0 1 +1 2 +2 -10 +Offset > block +0 0 +1 0 +2 0 +Abs(Offset) > block +0 0 +1 0 +2 0 +Positive offset +0 1 +1 2 +2 0 +Negative offset +0 1 +1 2 +2 0 +Positive offset with defaults +0 2 +1 3 +2 12 +3 13 +Negative offset with defaults +0 10 +1 11 +2 0 +3 1 +Positive offset with const defaults +0 1 +1 2 +2 1000 +Negative offset with const defaults +0 1000 +1 0 +2 1 +Constant column +0 1000 +1 1000 +2 1000 diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.sql b/dbms/tests/queries/0_stateless/00957_neighbour.sql new file mode 100644 index 00000000000..156672155a8 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_neighbour.sql @@ -0,0 +1,30 @@ +-- no arguments +select neighbour(); -- { serverError 42 } +-- single argument +select neighbour(1); -- { serverError 42 } +-- greater than 3 arguments +select neighbour(1,2,3,4); -- { serverError 42 } +-- bad default value +select neighbour(dummy, 1, 'hello'); -- { serverError 43 } +-- types without common supertype (UInt64 and Int8) +select number, neighbour(number, 1, -10) from numbers(3); -- { serverError 43 } +select 'Result with different type'; +select toInt32(number) as n, neighbour(n, 1, -10) from numbers(3); +select 'Offset > block'; +select number, neighbour(number, 10) from numbers(3); +select 'Abs(Offset) > block'; +select number, neighbour(number, -10) from numbers(3); +select 'Positive offset'; +select number, neighbour(number, 1) from numbers(3); +select 'Negative offset'; +select number, neighbour(number, 1) from numbers(3); +select 'Positive offset with defaults'; +select number, neighbour(number, 2, number + 10) from numbers(4); +select 'Negative offset with defaults'; +select number, neighbour(number, -2, number + 10) from numbers(4); +select 'Positive offset with const defaults'; +select number, neighbour(number, 1, 1000) from numbers(3); +select 'Negative offset with const defaults'; +select number, neighbour(number, -1, 1000) from numbers(3); +select 'Constant column'; +select number, neighbour(1000, 10) from numbers(3); \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.reference b/dbms/tests/queries/0_stateless/00957_next_in_block.reference deleted file mode 100644 index 860ce6dc1ba..00000000000 --- a/dbms/tests/queries/0_stateless/00957_next_in_block.reference +++ /dev/null @@ -1,12 +0,0 @@ -0 1 -1 0 -0 2 -1 0 -2 0 -0 0 -1 0 -0 2 -1 3 -2 4 -3 1000 -4 1000 diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.sql b/dbms/tests/queries/0_stateless/00957_next_in_block.sql deleted file mode 100644 index 1efda43339e..00000000000 --- a/dbms/tests/queries/0_stateless/00957_next_in_block.sql +++ /dev/null @@ -1,18 +0,0 @@ --- no arguments -select neighbour(); -- { serverError 42 } --- single argument -select neighbour(1); -- { serverError 42 } --- greater than 3 arguments -select neighbour(1,2,3,4); -- { serverError 42 } --- bad default value -select neighbour(dummy, 1, 'hello'); -- { serverError 43 } --- single argument test -select number, neighbour(number,1) from numbers(2); --- filling by column's default value -select number, neighbour(number, 2) from numbers(3); --- offset is greater that block - should fill everything with defaults -select number, neighbour(number, 5) from numbers(2); --- substitution by constant for missing values -select number, neighbour(number, 2, 1000) from numbers(5); --- substitution by expression --- select number, neighbour(number, 2, number % 2) from numbers(5); \ No newline at end of file diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md index 57fa8acfee3..05efe0fceb4 100644 --- a/docs/en/query_language/functions/other_functions.md +++ b/docs/en/query_language/functions/other_functions.md @@ -311,6 +311,49 @@ Returns the ordinal number of the row in the data block. Different data blocks a Returns the ordinal number of the row in the data block. This function only considers the affected data blocks. +## neighbour(column, offset\[, default_value\]) + +Returns value for `column`, in `offset` distance from current row. +This function is a partial implementation of [window functions](https://en.wikipedia.org/wiki/SQL_window_function) LEAD() and LAG(). + +The result of the function depends on the affected data blocks and the order of data in the block. +If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +If `offset` value is outside block bounds, a default value for `column` returned. If `default_value` is given, then it will be used. +This function can be used to compute year-over-year metric value: + +``` sql +WITH toDate('2018-01-01') AS start_date +SELECT + toStartOfMonth(start_date + (number * 32)) AS month, + toInt32(month) % 100 AS money, + neighbour(money, -12) AS prev_year, + round(prev_year / money, 2) AS year_over_year +FROM numbers(16) +``` + +``` +┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ +│ 2018-01-01 │ 32 │ 0 │ 0 │ +│ 2018-02-01 │ 63 │ 0 │ 0 │ +│ 2018-03-01 │ 91 │ 0 │ 0 │ +│ 2018-04-01 │ 22 │ 0 │ 0 │ +│ 2018-05-01 │ 52 │ 0 │ 0 │ +│ 2018-06-01 │ 83 │ 0 │ 0 │ +│ 2018-07-01 │ 13 │ 0 │ 0 │ +│ 2018-08-01 │ 44 │ 0 │ 0 │ +│ 2018-09-01 │ 75 │ 0 │ 0 │ +│ 2018-10-01 │ 5 │ 0 │ 0 │ +│ 2018-11-01 │ 36 │ 0 │ 0 │ +│ 2018-12-01 │ 66 │ 0 │ 0 │ +│ 2019-01-01 │ 97 │ 32 │ 0.33 │ +│ 2019-02-01 │ 28 │ 63 │ 2.25 │ +│ 2019-03-01 │ 56 │ 91 │ 1.62 │ +│ 2019-04-01 │ 87 │ 22 │ 0.25 │ +└────────────┴───────┴───────────┴────────────────┘ +``` + + ## runningDifference(x) {#other_functions-runningdifference} Calculates the difference between successive row values ​​in the data block. diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md index 1637c7bda93..6f5b6f1ff0d 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/query_language/functions/other_functions.md @@ -288,6 +288,48 @@ SELECT ## rowNumberInAllBlocks() Возвращает порядковый номер строки в блоке данных. Функция учитывает только задействованные блоки данных. +## neighbour(column, offset\[, default_value\]) + +Функция позволяет получить доступ к значению в колонке `column`, находящемуся на смещении `offset` относительно текущей строки. +Является частичной реализацией [оконных функций](https://en.wikipedia.org/wiki/SQL_window_function) LEAD() и LAG(). + +Результат функции зависит от затронутых блоков данных и порядка данных в блоке. +Если сделать подзапрос с ORDER BY и вызывать функцию извне подзапроса, можно будет получить ожидаемый результат. + +Если значение `offset` выходит за пределы блока данных, то берётся значение по-умолчанию для колонки `column`. Если передан параметр `default_value`, то значение берётся из него. +Например, эта функция может использоваться чтобы оценить year-over-year значение показателя: + +``` sql +WITH toDate('2018-01-01') AS start_date +SELECT + toStartOfMonth(start_date + (number * 32)) AS month, + toInt32(month) % 100 AS money, + neighbour(money, -12) AS prev_year, + round(prev_year / money, 2) AS year_over_year +FROM numbers(16) +``` + +``` +┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ +│ 2018-01-01 │ 32 │ 0 │ 0 │ +│ 2018-02-01 │ 63 │ 0 │ 0 │ +│ 2018-03-01 │ 91 │ 0 │ 0 │ +│ 2018-04-01 │ 22 │ 0 │ 0 │ +│ 2018-05-01 │ 52 │ 0 │ 0 │ +│ 2018-06-01 │ 83 │ 0 │ 0 │ +│ 2018-07-01 │ 13 │ 0 │ 0 │ +│ 2018-08-01 │ 44 │ 0 │ 0 │ +│ 2018-09-01 │ 75 │ 0 │ 0 │ +│ 2018-10-01 │ 5 │ 0 │ 0 │ +│ 2018-11-01 │ 36 │ 0 │ 0 │ +│ 2018-12-01 │ 66 │ 0 │ 0 │ +│ 2019-01-01 │ 97 │ 32 │ 0.33 │ +│ 2019-02-01 │ 28 │ 63 │ 2.25 │ +│ 2019-03-01 │ 56 │ 91 │ 1.62 │ +│ 2019-04-01 │ 87 │ 22 │ 0.25 │ +└────────────┴───────┴───────────┴────────────────┘ +``` + ## runningDifference(x) Считает разницу между последовательными значениями строк в блоке данных. Возвращает 0 для первой строки и разницу с предыдущей строкой для каждой последующей строки. From 6bf3902ce5d814758381bf15689c4585346478f0 Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Tue, 13 Aug 2019 16:20:32 +0300 Subject: [PATCH 113/357] Format file --- dbms/src/Functions/neighbour.cpp | 88 ++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index 79a1feec002..2efca01a66f 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -1,10 +1,10 @@ #include +#include #include #include #include #include #include -#include namespace DB { @@ -85,7 +85,8 @@ public: static void insertDefaults(const MutableColumnPtr & target, size_t row_count, ColumnPtr & default_values_column, size_t offset) { - if (row_count == 0) { + if (row_count == 0) + { return; } if (default_values_column) @@ -93,15 +94,20 @@ public: if (isColumnConst(*default_values_column)) { Field constant_value = (*default_values_column)[0]; - for(size_t row = 0; row < row_count;row++) + for (size_t row = 0; row < row_count; row++) { target->insert(constant_value); } - } else { + } + else + { target->insertRangeFrom(*default_values_column, offset, row_count); } - } else { - for(size_t row = 0; row < row_count;row++) { + } + else + { + for (size_t row = 0; row < row_count; row++) + { target->insertDefault(); } } @@ -119,7 +125,7 @@ public: default_values_column = block.getByPosition(arguments[2]).column; } - ColumnWithTypeAndName &source_column_name_and_type = block.getByPosition(arguments[0]); + ColumnWithTypeAndName & source_column_name_and_type = block.getByPosition(arguments[0]); DataTypes types = {source_column_name_and_type.type}; if (default_values_column) { @@ -129,23 +135,29 @@ public: auto source_column = source_column_name_and_type.column; // adjust source and default values columns to resulting datatype - if (!source_column_name_and_type.type->equals(*result_type)) { + if (!source_column_name_and_type.type->equals(*result_type)) + { source_column = castColumn(source_column_name_and_type, result_type, context); } - if (default_values_column && !block.getByPosition(arguments[2]).type->equals(*result_type)) { + if (default_values_column && !block.getByPosition(arguments[2]).type->equals(*result_type)) + { default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); } const DataTypePtr desired_type = std::make_shared(); - if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) { + if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) + { offset_column = castColumn(offset_structure, desired_type, context); } - if (isColumnConst(*source_column)) { + if (isColumnConst(*source_column)) + { auto column = result_type->createColumnConst(input_rows_count, (*source_column)[0]); block.getByPosition(result).column = std::move(column); - } else { + } + else + { auto column = result_type->createColumn(); column->reserve(input_rows_count); // with constant offset - insertRangeFrom @@ -156,48 +168,71 @@ public: if (offset_value > 0) { // insert shifted value - if ((size_t)offset_value <= input_rows_count) { + if ((size_t)offset_value <= input_rows_count) + { column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); } size_t row_count = (size_t)offset_value > input_rows_count ? input_rows_count : offset_value; insertDefaults(column, row_count, default_values_column, input_rows_count - row_count); - } else if (offset_value < 0) { + } + else if (offset_value < 0) + { size_t row_count = (size_t)std::abs(offset_value) > input_rows_count ? input_rows_count : std::abs(offset_value); // insert defaults up to offset_value insertDefaults(column, row_count, default_values_column, 0); column->insertRangeFrom(*source_column, 0, input_rows_count - row_count); - } else { + } + else + { // populate column with source values column->insertRangeFrom(*source_column, 0, input_rows_count); } - } else { + } + else + { // with dynamic offset - handle row by row for (size_t row = 0; row < input_rows_count; row++) { Int64 offset_value = offset_column->getInt(row); - if (offset_value == 0) { + if (offset_value == 0) + { column->insertFrom(*source_column, row); - } else if (offset_value > 0) { + } + else if (offset_value > 0) + { size_t real_offset = row + offset_value; - if (real_offset > input_rows_count) { - if (default_values_column) { + if (real_offset > input_rows_count) + { + if (default_values_column) + { column->insertFrom(*default_values_column, row); - } else { + } + else + { column->insertDefault(); } - } else { + } + else + { column->insertFrom(*column, real_offset); } - } else { + } + else + { // out of range if ((size_t)std::abs(offset_value) > row) { - if (default_values_column) { + if (default_values_column) + { column->insertFrom(*default_values_column, row); - } else { + } + else + { column->insertDefault(); } - } else { + } + else + { column->insertFrom(*column, row - std::abs(offset_value)); } } @@ -206,6 +241,7 @@ public: block.getByPosition(result).column = std::move(column); } } + private: const Context & context; }; From 031bfc7bf331ecc24345f9f9b5b8040b2994090e Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 17:46:18 +0300 Subject: [PATCH 114/357] Remove reference --- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index b94555477f9..8149ef9ddfe 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -707,7 +707,7 @@ protected: {} const MergeTreeSettingsPtr copyPtr() const { return settings_ptr; } - MergeTreeSettingsPtr & getPtr() { return settings_ptr; } + MergeTreeSettingsPtr getPtr() { return settings_ptr; } void setPtr(MergeTreeSettingsPtr ptr) { settings_ptr = ptr; } }; From 96f62fefcc162ceb1700e2fd91efffbc0fe964e4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 13 Aug 2019 19:34:52 +0300 Subject: [PATCH 115/357] Add mutex for settings --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.h | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index aab384552cc..3c14b7b1084 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1644,7 +1644,7 @@ void MergeTreeData::alterSettings( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - /// No additional locking required, because we took table_lock_holder + std::unique_lock lock(settings_mutex); MutableMergeTreeSettingsPtr settings = std::move(*guarded_settings.getPtr()).mutate(); settings->updateFromChanges(new_changes); IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 8149ef9ddfe..e1cfe7b0032 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -662,6 +662,7 @@ public: /// get consistent settings. const MergeTreeSettingsPtr getCOWSettings() const { + std::shared_lock lock(settings_mutex); return guarded_settings.copyPtr(); } @@ -801,6 +802,8 @@ protected: std::mutex grab_old_parts_mutex; /// The same for clearOldTemporaryDirectories. std::mutex clear_old_temporary_directories_mutex; + /// Mutex for settings usage + mutable std::shared_mutex settings_mutex; void setPrimaryKeyIndicesAndColumns(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, const ColumnsDescription & new_columns, From 986d56ba0c1d80b0fc879f22750e42a1f8b213f1 Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Wed, 14 Aug 2019 14:32:03 +0300 Subject: [PATCH 116/357] Fix casting style, work with Nullable --- dbms/src/Functions/neighbour.cpp | 30 ++++++++++++------- .../registerFunctionsMiscellaneous.cpp | 4 +-- .../queries/0_stateless/00957_neighbour.sql | 4 +++ 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index 2efca01a66f..153307abd4b 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -54,13 +54,14 @@ public: + ", should be from 2 to 3", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - // second argument must be a positive integer + // second argument must be an integer if (!isInteger(arguments[1])) throw Exception( "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + " - should be positive integer", + + " - should be an integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + // check that default value column has supertype with first argument if (number_of_arguments == 3) { @@ -116,7 +117,13 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { auto offset_structure = block.getByPosition(arguments[1]); + ColumnPtr & offset_column = offset_structure.column; + if (isColumnNullable(*offset_column)) + throw Exception( + "Illegal type " + offset_structure.type->getName() + " of second argument of function " + getName() + + " - can not be Nullable", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto is_constant_offset = isColumnConst(*offset_structure.column); ColumnPtr default_values_column = nullptr; @@ -145,6 +152,7 @@ public: default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); } + // since we are working with both signed and unsigned - we'll try to use Int64 for handling all of them const DataTypePtr desired_type = std::make_shared(); if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) { @@ -165,26 +173,26 @@ public: { Int64 offset_value = offset_column->getInt(0); + auto offset_value_casted = static_cast(std::abs(offset_value)); + size_t default_value_count = std::min(offset_value_casted, input_rows_count); if (offset_value > 0) { // insert shifted value - if ((size_t)offset_value <= input_rows_count) + if (offset_value_casted <= input_rows_count) { - column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); + column->insertRangeFrom(*source_column, offset_value_casted, input_rows_count - offset_value_casted); } - size_t row_count = (size_t)offset_value > input_rows_count ? input_rows_count : offset_value; - insertDefaults(column, row_count, default_values_column, input_rows_count - row_count); + insertDefaults(column, default_value_count, default_values_column, input_rows_count - default_value_count); } else if (offset_value < 0) { - size_t row_count = (size_t)std::abs(offset_value) > input_rows_count ? input_rows_count : std::abs(offset_value); // insert defaults up to offset_value - insertDefaults(column, row_count, default_values_column, 0); - column->insertRangeFrom(*source_column, 0, input_rows_count - row_count); + insertDefaults(column, default_value_count, default_values_column, 0); + column->insertRangeFrom(*source_column, 0, input_rows_count - default_value_count); } else { - // populate column with source values + // populate column with source values, when offset is equal to zero column->insertRangeFrom(*source_column, 0, input_rows_count); } } @@ -246,7 +254,7 @@ private: const Context & context; }; -void registerFunctionNextInBlock(FunctionFactory & factory) +void registerFunctionNeighbour(FunctionFactory & factory) { factory.registerFunction(); } diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp index 57ccfcd11c9..c96f5f05c7b 100644 --- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp @@ -17,7 +17,7 @@ void registerFunctionBlockSize(FunctionFactory &); void registerFunctionBlockNumber(FunctionFactory &); void registerFunctionRowNumberInBlock(FunctionFactory &); void registerFunctionRowNumberInAllBlocks(FunctionFactory &); -void registerFunctionNextInBlock(FunctionFactory &); +void registerFunctionNeighbour(FunctionFactory &); void registerFunctionSleep(FunctionFactory &); void registerFunctionSleepEachRow(FunctionFactory &); void registerFunctionMaterialize(FunctionFactory &); @@ -68,7 +68,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionBlockNumber(factory); registerFunctionRowNumberInBlock(factory); registerFunctionRowNumberInAllBlocks(factory); - registerFunctionNextInBlock(factory); + registerFunctionNeighbour(factory); registerFunctionSleep(factory); registerFunctionSleepEachRow(factory); registerFunctionMaterialize(factory); diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.sql b/dbms/tests/queries/0_stateless/00957_neighbour.sql index 156672155a8..665936fd70f 100644 --- a/dbms/tests/queries/0_stateless/00957_neighbour.sql +++ b/dbms/tests/queries/0_stateless/00957_neighbour.sql @@ -8,6 +8,10 @@ select neighbour(1,2,3,4); -- { serverError 42 } select neighbour(dummy, 1, 'hello'); -- { serverError 43 } -- types without common supertype (UInt64 and Int8) select number, neighbour(number, 1, -10) from numbers(3); -- { serverError 43 } +-- nullable offset is not allowed +select number, if(number > 1, number, null) as offset, neighbour(number, offset) from numbers(3); -- { serverError 43 } +select 'Zero offset'; +select number, neighbour(number, 0) from numbers(3); select 'Result with different type'; select toInt32(number) as n, neighbour(n, 1, -10) from numbers(3); select 'Offset > block'; From 2126196c8946126cd32322f259571c241b7f857d Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Wed, 14 Aug 2019 15:09:51 +0300 Subject: [PATCH 117/357] Nullable correct handling --- dbms/src/Functions/neighbour.cpp | 21 ++++++++++--------- .../0_stateless/00957_neighbour.reference | 8 +++++++ .../queries/0_stateless/00957_neighbour.sql | 2 ++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index 153307abd4b..06307884f22 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -44,6 +44,8 @@ public: bool isDeterministicInScopeOfQuery() const override { return false; } + bool useDefaultImplementationForNulls() const override { return false; } + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { size_t number_of_arguments = arguments.size(); @@ -57,8 +59,11 @@ public: // second argument must be an integer if (!isInteger(arguments[1])) throw Exception( - "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + " - should be an integer", + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + " - should be an integer", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else if (arguments[1]->isNullable()) + throw Exception( + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + " - can not be Nullable", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -119,11 +124,6 @@ public: auto offset_structure = block.getByPosition(arguments[1]); ColumnPtr & offset_column = offset_structure.column; - if (isColumnNullable(*offset_column)) - throw Exception( - "Illegal type " + offset_structure.type->getName() + " of second argument of function " + getName() - + " - can not be Nullable", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto is_constant_offset = isColumnConst(*offset_structure.column); ColumnPtr default_values_column = nullptr; @@ -222,13 +222,14 @@ public: } else { - column->insertFrom(*column, real_offset); + column->insertFrom(*source_column, real_offset); } } else { // out of range - if ((size_t)std::abs(offset_value) > row) + auto offset_value_casted = static_cast(std::abs(offset_value)); + if (offset_value_casted > row) { if (default_values_column) { @@ -241,7 +242,7 @@ public: } else { - column->insertFrom(*column, row - std::abs(offset_value)); + column->insertFrom(*column, row - offset_value_casted); } } } diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.reference b/dbms/tests/queries/0_stateless/00957_neighbour.reference index cd8c6310f22..1983488cfc2 100644 --- a/dbms/tests/queries/0_stateless/00957_neighbour.reference +++ b/dbms/tests/queries/0_stateless/00957_neighbour.reference @@ -1,3 +1,11 @@ +Zero offset +0 0 +1 1 +2 2 +Nullable values +\N 0 \N +\N 1 2 +2 2 \N Result with different type 0 1 1 2 diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.sql b/dbms/tests/queries/0_stateless/00957_neighbour.sql index 665936fd70f..753ae8493a3 100644 --- a/dbms/tests/queries/0_stateless/00957_neighbour.sql +++ b/dbms/tests/queries/0_stateless/00957_neighbour.sql @@ -12,6 +12,8 @@ select number, neighbour(number, 1, -10) from numbers(3); -- { serverError 43 } select number, if(number > 1, number, null) as offset, neighbour(number, offset) from numbers(3); -- { serverError 43 } select 'Zero offset'; select number, neighbour(number, 0) from numbers(3); +select 'Nullable values'; +select if(number > 1, number, null) as value, number as offset, neighbour(value, offset) as neighbour from numbers(3); select 'Result with different type'; select toInt32(number) as n, neighbour(n, 1, -10) from numbers(3); select 'Offset > block'; From ab1c4139deaad29f00104580df09f2334bb4efaa Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Wed, 14 Aug 2019 22:51:03 +0300 Subject: [PATCH 118/357] Added ReplicatedMergeTree support and test for constraints, also added VIOLATED_CONSTRAINT error --- dbms/programs/server/config.xml | 5 ++- dbms/src/Common/ErrorCodes.cpp | 1 + .../CheckConstraintsBlockOutputStream.cpp | 2 +- .../CheckConstraintsBlockOutputStream.h | 2 +- .../ReplicatedMergeTreeTableMetadata.cpp | 6 +++ .../Storages/StorageReplicatedMergeTree.cpp | 4 ++ ...onstraints_replication_zookeeper.reference | 0 ...0988_constraints_replication_zookeeper.sql | 43 +++++++++++++++++++ 8 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.reference create mode 100644 dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index c09913cbd87..188a98779e9 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -217,7 +217,10 @@ See https://clickhouse.yandex/docs/en/table_engines/replication/ --> - + + + testkeeper + - - testkeeper - + + From e68775e3d0f3a8201d8f4d1cc319dcd99295099c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 21 Aug 2019 17:51:32 +0300 Subject: [PATCH 142/357] Fix TotalsHavingTransform. --- dbms/src/Processors/Transforms/TotalsHavingTransform.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/Processors/Transforms/TotalsHavingTransform.cpp b/dbms/src/Processors/Transforms/TotalsHavingTransform.cpp index cce091b1999..b6931f2c8e1 100644 --- a/dbms/src/Processors/Transforms/TotalsHavingTransform.cpp +++ b/dbms/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -161,6 +161,8 @@ void TotalsHavingTransform::transform(Chunk & chunk) if (const_filter_description.always_true) { addToTotals(chunk, nullptr); + auto num_rows = columns.front()->size(); + chunk.setColumns(std::move(columns), num_rows); return; } From 51f6d9751156a506807aba97962b7ca52bbf425e Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 21 Aug 2019 18:20:42 +0300 Subject: [PATCH 143/357] Update formats.md --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index c5dbbce674d..2f409706c61 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -146,7 +146,7 @@ SELECT * FROM t_null FORMAT TSKV x=1 y=\N ``` -When there is a large number of small columns, this format is ineffective, and there is generally no reason to use it. It is used in some departments of Yandex. +When there is a large number of small columns, this format is ineffective, and there is generally no reason to use it. Nevertheless, it is no worse than JSONEachRow in terms of efficiency. Both data output and parsing are supported in this format. For parsing, any order is supported for the values of different columns. It is acceptable for some values to be omitted – they are treated as equal to their default values. In this case, zeros and blank rows are used as default values. Complex values that could be specified in the table are not supported as defaults. From a1560448d632ea5dbef1e9bbe2b66c03285598f5 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 21 Aug 2019 18:21:36 +0300 Subject: [PATCH 144/357] Update formats.md --- docs/ru/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 20c919665d5..130a32d63fa 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -145,7 +145,7 @@ SELECT * FROM t_null FORMAT TSKV x=1 y=\N ``` -При большом количестве маленьких столбцов, этот формат существенно неэффективен, и обычно нет причин его использовать. Он реализован, так как используется в некоторых отделах Яндекса. +При большом количестве маленьких столбцов, этот формат существенно неэффективен, и обычно нет причин его использовать. Впрочем, он не хуже формата JSONEachRow по производительности. Поддерживается как вывод, так и парсинг данных в этом формате. При парсинге, поддерживается расположение значений разных столбцов в произвольном порядке. Допустимо отсутствие некоторых значений - тогда они воспринимаются как равные значениям по умолчанию. В этом случае в качестве значений по умолчанию используются нули и пустые строки. Сложные значения, которые могут быть заданы в таблице не поддерживаются как значения по умолчанию. From 2e03ac09ecdc103ee0fb029aa6445c0003df595b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 21 Aug 2019 19:38:27 +0300 Subject: [PATCH 145/357] Fix MergingSortedTransform. --- dbms/src/Processors/Transforms/MergingSortedTransform.cpp | 3 ++- dbms/src/Processors/Transforms/MergingSortedTransform.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Processors/Transforms/MergingSortedTransform.cpp b/dbms/src/Processors/Transforms/MergingSortedTransform.cpp index 8857ec876d7..e37eae82de1 100644 --- a/dbms/src/Processors/Transforms/MergingSortedTransform.cpp +++ b/dbms/src/Processors/Transforms/MergingSortedTransform.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB { @@ -13,7 +14,7 @@ MergingSortedTransform::MergingSortedTransform( UInt64 limit_, bool quiet_, bool have_all_inputs_) - : IProcessor(InputPorts(num_inputs, header), {header}) + : IProcessor(InputPorts(num_inputs, header), {materializeBlock(header)}) , description(description_), max_block_size(max_block_size_), limit(limit_), quiet(quiet_) , have_all_inputs(have_all_inputs_) , merged_data(header), source_chunks(num_inputs), cursors(num_inputs) diff --git a/dbms/src/Processors/Transforms/MergingSortedTransform.h b/dbms/src/Processors/Transforms/MergingSortedTransform.h index 5a1f417fdb6..223d5253e62 100644 --- a/dbms/src/Processors/Transforms/MergingSortedTransform.h +++ b/dbms/src/Processors/Transforms/MergingSortedTransform.h @@ -93,7 +93,7 @@ protected: columns = chunk.mutateColumns(); if (limit_rows && num_rows > limit_rows) for (auto & column : columns) - column = (*column->cut(0, limit_rows)).mutate(); + column = (*column->cut(0, limit_rows)->convertToFullColumnIfConst()).mutate(); total_merged_rows += num_rows; merged_rows = num_rows; From 321233ddaa5934985686c99e290b66621369575d Mon Sep 17 00:00:00 2001 From: akonyaev Date: Wed, 21 Aug 2019 19:39:20 +0300 Subject: [PATCH 146/357] ADQM-34 fix hardcode for,at name in ArrowColumnToCHColumn class --- .../src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp | 8 ++++---- dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h | 2 +- dbms/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp | 2 +- .../Processors/Formats/Impl/ParquetBlockInputFormat.cpp | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 24a144b10b5..fc1277c4a88 100644 --- a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -268,7 +268,7 @@ namespace DB void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk &res, std::shared_ptr &table, arrow::Status &read_status, const Block &header, - int &row_group_current, const Context &context) + int &row_group_current, const Context &context, std::string format_name) { Columns columns_list; UInt64 num_rows = 0; @@ -277,7 +277,7 @@ namespace DB using NameToColumnPtr = std::unordered_map>; if (!read_status.ok()) - throw Exception{"Error while reading ORC data: " + read_status.ToString(), + throw Exception{"Error while reading " + format_name + " data: " + read_status.ToString(), ErrorCodes::CANNOT_READ_ALL_DATA}; if (0 == table->num_rows()) @@ -333,7 +333,7 @@ namespace DB throw Exception { "The type \"" + arrow_column->type()->name() + "\" of an input column \"" + arrow_column->name() - + "\" is not supported for conversion from a ORC data format", + + "\" is not supported for conversion from a " + format_name + " data format", ErrorCodes::CANNOT_CONVERT_TYPE}; } @@ -392,7 +392,7 @@ namespace DB default: throw Exception { - "Unsupported ORC type \"" + arrow_column->type()->name() + "\" of an input column \"" + "Unsupported " + format_name + " type \"" + arrow_column->type()->name() + "\" of an input column \"" + arrow_column->name() + "\"", ErrorCodes::UNKNOWN_TYPE }; diff --git a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index cf4f021f8c0..b5f4732d107 100644 --- a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -39,7 +39,7 @@ namespace DB static void arrowTableToCHChunk(Chunk &res, std::shared_ptr &table, arrow::Status &read_status, const Block &header, - int &row_group_current, const Context &context); + int &row_group_current, const Context &context, std::string format_name); }; } #endif diff --git a/dbms/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/dbms/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 4da6b10f3ab..2069c3f3cbf 100644 --- a/dbms/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -57,7 +57,7 @@ namespace DB arrow::Status read_status = file_reader->Read(&table); - ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, context); + ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, context, "ORC"); return res; } diff --git a/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 32a55c70e55..54bab6d7467 100644 --- a/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -59,7 +59,7 @@ namespace DB std::shared_ptr table; arrow::Status read_status = file_reader->ReadRowGroup(row_group_current, &table); - ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, context); + ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, context, "Parquet"); return res; } From 6d81087bdc4c777cbdb317a3c0e5d1885fb2264c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 21 Aug 2019 19:53:40 +0300 Subject: [PATCH 147/357] Remove _dummy column if it is not needed. --- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 27be35ad57f..2ccfc762087 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -1550,7 +1550,11 @@ void InterpreterSelectQuery::executeFetchColumns( { /// Unify streams in case they have different headers. auto first_header = streams.at(0)->getHeader(); - for (size_t i = 1; i < streams.size(); ++i) + + if (first_header.columns() > 1 && first_header.has("_dummy")) + first_header.erase("_dummy"); + + for (size_t i = 0; i < streams.size(); ++i) { auto & stream = streams[i]; auto header = stream->getHeader(); From d573c4ec3e1263a590d19b5de751fb7e6e8fd1b1 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 21 Aug 2019 20:42:44 +0300 Subject: [PATCH 148/357] fix wrong check for disabling "read in order" optimization --- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 2 +- .../00995_optimize_read_in_order_with_aggregation.reference | 1 + .../00995_optimize_read_in_order_with_aggregation.sql | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.reference create mode 100644 dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.sql diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 27be35ad57f..dbc0cf85824 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -884,7 +884,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS } SortingInfoPtr sorting_info; - if (settings.optimize_read_in_order && storage && query.orderBy() && !query.groupBy() && !query.final() && !query.join()) + if (settings.optimize_read_in_order && storage && query.orderBy() && !query_analyzer->hasAggregation() && !query.final() && !query.join()) { if (const MergeTreeData * merge_tree_data = dynamic_cast(storage.get())) sorting_info = optimizeReadInOrder(*merge_tree_data, query, context, syntax_analyzer_result); diff --git a/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.reference b/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.reference new file mode 100644 index 00000000000..d567f8a0b01 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.reference @@ -0,0 +1 @@ +4950 diff --git a/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.sql b/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.sql new file mode 100644 index 00000000000..93c907811a5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00995_optimize_read_in_order_with_aggregation.sql @@ -0,0 +1,6 @@ +SET optimize_read_in_order = 1; +DROP TABLE IF EXISTS order_with_aggr; +CREATE TABLE order_with_aggr(a Int) ENGINE = MergeTree ORDER BY a; + +INSERT INTO order_with_aggr SELECT * FROM numbers(100); +SELECT sum(a) as s FROM order_with_aggr ORDER BY s; From 93d380ddaad180df67ec150e03f2bded4c5a52d7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 21 Aug 2019 23:32:58 +0300 Subject: [PATCH 149/357] Fixed test --- dbms/src/Functions/array/arrayIntersect.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Functions/array/arrayIntersect.cpp b/dbms/src/Functions/array/arrayIntersect.cpp index e6cc24a49ba..6aec4f94dc8 100644 --- a/dbms/src/Functions/array/arrayIntersect.cpp +++ b/dbms/src/Functions/array/arrayIntersect.cpp @@ -398,7 +398,7 @@ ColumnPtr FunctionArrayIntersect::execute(const UnpackedArrays & arrays, Mutable all_nullable = false; } - auto & result_data = assert_cast(*result_data_ptr); + auto & result_data = static_cast(*result_data_ptr); auto result_offsets_ptr = ColumnArray::ColumnOffsets::create(rows); auto & result_offsets = assert_cast(*result_offsets_ptr); auto null_map_column = ColumnUInt8::create(); From e573b06373c7215b25b5e2859575c1a3ccfeebb0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 21 Aug 2019 23:33:38 +0300 Subject: [PATCH 150/357] Updated comment --- dbms/src/Functions/nullIf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Functions/nullIf.cpp b/dbms/src/Functions/nullIf.cpp index 91fb311f444..507abc8d3e1 100644 --- a/dbms/src/Functions/nullIf.cpp +++ b/dbms/src/Functions/nullIf.cpp @@ -43,7 +43,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - /// nullIf(col1, col2) == if(col1 = col2, NULL, 1) + /// nullIf(col1, col2) == if(col1 = col2, NULL, col1) Block temp_block = block; From a7fa71aaf1a19ddf2715acd2048a861afffa9e06 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 22 Aug 2019 00:28:05 +0300 Subject: [PATCH 151/357] Fixed flappy test --- .../queries/0_stateless/00704_drop_truncate_memory_table.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh b/dbms/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh index 170259f0e24..1bbb69f4dd7 100755 --- a/dbms/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh +++ b/dbms/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -e +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh From 6bbf3bd6341910844f185a376e7db994dafa2fa0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 22 Aug 2019 04:30:49 +0300 Subject: [PATCH 152/357] Added a metric for the number of files to send in Distributed tables --- dbms/src/Common/CurrentMetrics.cpp | 1 + dbms/src/Storages/Distributed/DirectoryMonitor.cpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/dbms/src/Common/CurrentMetrics.cpp b/dbms/src/Common/CurrentMetrics.cpp index b8e30f3cccd..6bd99fb8f01 100644 --- a/dbms/src/Common/CurrentMetrics.cpp +++ b/dbms/src/Common/CurrentMetrics.cpp @@ -49,6 +49,7 @@ M(GlobalThreadActive, "Number of threads in global thread pool running a task.") \ M(LocalThread, "Number of threads in local thread pools. Should be similar to GlobalThreadActive.") \ M(LocalThreadActive, "Number of threads in local thread pools running a task.") \ + M(DistributedFilesToInsert, "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.") \ namespace CurrentMetrics diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 7eefc68f3a8..13c9cf3050a 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -23,6 +23,7 @@ namespace CurrentMetrics { extern const Metric DistributedSend; + extern const Metric DistributedFilesToInsert; } namespace DB @@ -209,6 +210,8 @@ bool StorageDistributedDirectoryMonitor::processFiles() if (files.empty()) return false; + CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedFilesToInsert, CurrentMetrics::Value(files.size())}; + if (should_batch_inserts) { processFilesWithBatching(files); From ae7ae6d660361d25d12b0dc5f555ef924e1ffd9a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 22 Aug 2019 05:05:46 +0300 Subject: [PATCH 153/357] Update CHANGELOG.md --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 607f650deeb..eceeb5db0ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,16 @@ * Fix build with external `libcxx` [#6010](https://github.com/yandex/ClickHouse/pull/6010) ([Ivan](https://github.com/abyss7)) * Fix shared build with `rdkafka` library [#6101](https://github.com/yandex/ClickHouse/pull/6101) ([Ivan](https://github.com/abyss7)) +## ClickHouse release 19.11.8.46, 2019-08-22 + +### Bug Fix +* Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [#6543](https://github.com/yandex/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +* Fix NPE when using IN clause with a subquery with a tuple. [#6125](https://github.com/yandex/ClickHouse/issues/6125) [#6550](https://github.com/yandex/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +* Fixed an issue that if a stale replica become alive, it may still have data parts that were removed by DROP PARTITION. [#6522](https://github.com/yandex/ClickHouse/issues/6522) [#6523](https://github.com/yandex/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +* Fixed issue with parsing CSV [#6426](https://github.com/yandex/ClickHouse/issues/6426) [#6559](https://github.com/yandex/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +* Fixed data race in system.parts table and ALTER query. This fixes [#6245](https://github.com/yandex/ClickHouse/issues/6245). [#6513](https://github.com/yandex/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [#6514](https://github.com/yandex/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) + ## ClickHouse release 19.11.7.40, 2019-08-14 ### Bug fix From 19cb429b06d3ca454621f45caa8ac86e9331bcb5 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 22 Aug 2019 05:08:12 +0300 Subject: [PATCH 154/357] Update CHANGELOG.md --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eceeb5db0ac..32eb446ac01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## ClickHouse release 19.13.3, 2019-08-22 + +### Bug Fix +* Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [#6543](https://github.com/yandex/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) +* Fix NPE when using IN clause with a subquery with a tuple. [#6125](https://github.com/yandex/ClickHouse/issues/6125) [#6550](https://github.com/yandex/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) +* Fixed an issue that if a stale replica become alive, it may still have data parts that were removed by DROP PARTITION. [#6522](https://github.com/yandex/ClickHouse/issues/6522) [#6523](https://github.com/yandex/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +* Fixed issue with parsing CSV [#6426](https://github.com/yandex/ClickHouse/issues/6426) [#6559](https://github.com/yandex/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) +* Fixed data race in system.parts table and ALTER query. This fixes [#6245](https://github.com/yandex/ClickHouse/issues/6245). [#6513](https://github.com/yandex/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [#6514](https://github.com/yandex/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) + ## ClickHouse release 19.13.2.19, 2019-08-14 ### New Feature From bbf2911d61089d643524ef08672abf67a488677a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 22 Aug 2019 05:14:01 +0300 Subject: [PATCH 155/357] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32eb446ac01..6263a197f45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## ClickHouse release 19.13.3, 2019-08-22 +## ClickHouse release 19.13.3.26, 2019-08-22 ### Bug Fix * Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [#6543](https://github.com/yandex/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) From 833d6d60a67764ba688f4b8139a58475fb151e32 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 22 Aug 2019 05:14:46 +0300 Subject: [PATCH 156/357] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6263a197f45..74012668d1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### Bug Fix * Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [#6543](https://github.com/yandex/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) * Fix NPE when using IN clause with a subquery with a tuple. [#6125](https://github.com/yandex/ClickHouse/issues/6125) [#6550](https://github.com/yandex/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -* Fixed an issue that if a stale replica become alive, it may still have data parts that were removed by DROP PARTITION. [#6522](https://github.com/yandex/ClickHouse/issues/6522) [#6523](https://github.com/yandex/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +* Fixed an issue that if a stale replica becomes alive, it may still have data parts that were removed by DROP PARTITION. [#6522](https://github.com/yandex/ClickHouse/issues/6522) [#6523](https://github.com/yandex/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) * Fixed issue with parsing CSV [#6426](https://github.com/yandex/ClickHouse/issues/6426) [#6559](https://github.com/yandex/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) * Fixed data race in system.parts table and ALTER query. This fixes [#6245](https://github.com/yandex/ClickHouse/issues/6245). [#6513](https://github.com/yandex/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [#6514](https://github.com/yandex/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) From ee89ee0218c86613db1c6856fda8fb3d1140b33b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 22 Aug 2019 05:15:05 +0300 Subject: [PATCH 157/357] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74012668d1b..7d6714b6474 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,7 @@ ### Bug Fix * Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [#6543](https://github.com/yandex/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) * Fix NPE when using IN clause with a subquery with a tuple. [#6125](https://github.com/yandex/ClickHouse/issues/6125) [#6550](https://github.com/yandex/ClickHouse/pull/6550) ([tavplubix](https://github.com/tavplubix)) -* Fixed an issue that if a stale replica become alive, it may still have data parts that were removed by DROP PARTITION. [#6522](https://github.com/yandex/ClickHouse/issues/6522) [#6523](https://github.com/yandex/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) +* Fixed an issue that if a stale replica becomes alive, it may still have data parts that were removed by DROP PARTITION. [#6522](https://github.com/yandex/ClickHouse/issues/6522) [#6523](https://github.com/yandex/ClickHouse/pull/6523) ([tavplubix](https://github.com/tavplubix)) * Fixed issue with parsing CSV [#6426](https://github.com/yandex/ClickHouse/issues/6426) [#6559](https://github.com/yandex/ClickHouse/pull/6559) ([tavplubix](https://github.com/tavplubix)) * Fixed data race in system.parts table and ALTER query. This fixes [#6245](https://github.com/yandex/ClickHouse/issues/6245). [#6513](https://github.com/yandex/ClickHouse/pull/6513) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed wrong code in mutations that may lead to memory corruption. Fixed segfault with read of address `0x14c0` that may happed due to concurrent `DROP TABLE` and `SELECT` from `system.parts` or `system.parts_columns`. Fixed race condition in preparation of mutation queries. Fixed deadlock caused by `OPTIMIZE` of Replicated tables and concurrent modification operations like ALTERs. [#6514](https://github.com/yandex/ClickHouse/pull/6514) ([alexey-milovidov](https://github.com/alexey-milovidov)) From 85d3ba099ff93e8edc696feaa93c579a6967ea86 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 22 Aug 2019 05:31:49 +0300 Subject: [PATCH 158/357] Added a comment --- dbms/src/Common/SymbolIndex.cpp | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/dbms/src/Common/SymbolIndex.cpp b/dbms/src/Common/SymbolIndex.cpp index 05268c4b6f1..7f494f0dd9b 100644 --- a/dbms/src/Common/SymbolIndex.cpp +++ b/dbms/src/Common/SymbolIndex.cpp @@ -10,6 +10,49 @@ //#include #include +/** + +ELF object can contain three different places with symbol names and addresses: + +1. Symbol table in section headers. It is used for static linking and usually left in executable. +It is not loaded in memory and they are not necessary for program to run. +It does not relate to debug info and present regardless to -g flag. +You can use strip to get rid of this symbol table. +If you have this symbol table in your binary, you can manually read it and get symbol names, even for symbols from anonymous namespaces. + +2. Hashes in program headers such as DT_HASH and DT_GNU_HASH. +It is necessary for dynamic object (.so libraries and any dynamically linked executable that depend on .so libraries) +because it is used for dynamic linking that happens in runtime and performed by dynamic loader. +Only exported symbols will be presented in that hash tables. Symbols from anonymous namespaces are not. +This part of executable binary is loaded in memory and accessible via 'dl_iterate_phdr', 'dladdr' and 'backtrace_symbols' functions from libc. +ClickHouse versions prior to 19.13 has used just these symbol names to symbolize stack traces +and stack traces may be incomplete due to lack of symbols with internal linkage. +But because ClickHouse is linked with most of the symbols exported (-rdynamic flag) it can still provide good enough stack traces. + +3. DWARF debug info. It contains the most detailed information about symbols and everything else. +It allows to get source file names and line numbers from addresses. Only available if you use -g option for compiler. +It is also used by default for ClickHouse builds, but because of its weight (about two gigabytes) +it is splitted to separate binary and provided in clickhouse-common-static-dbg package. +This separate binary is placed in /usr/lib/debug/usr/bin/clickhouse and is loaded automatically by tools like gdb, addr2line. +When you build ClickHouse by yourself, debug info is not splitted and present in a single huge binary. + +What ClickHouse is using to provide good stack traces? + +In versions prior to 19.13, only "program headers" (2) was used. + +In version 19.13, ClickHouse will read program headers (2) and cache them, +also it will read itself as ELF binary and extract symbol tables from section headers (1) +to also symbolize functions that are not exported for dynamic linking. +And finally, it will read DWARF info (3) if available to display file names and line numbers. + +What detail can you obtain depending on your binary? + +If you have debug info (you build ClickHouse by yourself or install clickhouse-common-static-dbg package), you will get source file names and line numbers. +Otherwise you will get only symbol names. If your binary contains symbol table in section headers (the default, unless stripped), you will get all symbol names. +Otherwise you will get only exported symbols from program headers. + +*/ + namespace DB { From 78af6d793d472f6f8903d5a6cff2a2eeb8c004a0 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 22 Aug 2019 12:34:06 +0800 Subject: [PATCH 159/357] Fix segfault when decoding symbol table. --- dbms/src/Common/SymbolIndex.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dbms/src/Common/SymbolIndex.cpp b/dbms/src/Common/SymbolIndex.cpp index 7f494f0dd9b..5cb424ef48b 100644 --- a/dbms/src/Common/SymbolIndex.cpp +++ b/dbms/src/Common/SymbolIndex.cpp @@ -107,13 +107,14 @@ void collectSymbolsFromProgramHeaders(dl_phdr_info * info, size_t sym_cnt = 0; for (auto it = dyn_begin; it->d_tag != DT_NULL; ++it) { - if (it->d_tag == DT_HASH) - { - const ElfW(Word) * hash = reinterpret_cast(correct_address(info->dlpi_addr, it->d_un.d_ptr)); - sym_cnt = hash[1]; - break; - } - else if (it->d_tag == DT_GNU_HASH) + // TODO: this branch leads to invalid address of the hash table. Need further investigation. + // if (it->d_tag == DT_HASH) + // { + // const ElfW(Word) * hash = reinterpret_cast(correct_address(info->dlpi_addr, it->d_un.d_ptr)); + // sym_cnt = hash[1]; + // break; + // } + if (it->d_tag == DT_GNU_HASH) { /// This code based on Musl-libc. From 8ac4ee3dca4d51380fd3e2235bb842ce18017509 Mon Sep 17 00:00:00 2001 From: sev7e0 Date: Thu, 22 Aug 2019 14:29:21 +0800 Subject: [PATCH 160/357] Fix data type (enum) table name error in docs --- docs/en/data_types/enum.md | 2 +- docs/ru/data_types/enum.md | 2 +- docs/zh/data_types/enum.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md index 247ec070190..3fbe5b3708b 100644 --- a/docs/en/data_types/enum.md +++ b/docs/en/data_types/enum.md @@ -94,7 +94,7 @@ ENGINE = TinyLog it can store not only `'hello'` and `'world'`, but `NULL`, as well. ``` -INSERT INTO t_enum_null Values('hello'),('world'),(NULL) +INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) ``` In RAM, an `Enum` column is stored in the same way as `Int8` or `Int16` of the corresponding numerical values. diff --git a/docs/ru/data_types/enum.md b/docs/ru/data_types/enum.md index 7ed0150e65f..9191dc5d2b0 100644 --- a/docs/ru/data_types/enum.md +++ b/docs/ru/data_types/enum.md @@ -90,7 +90,7 @@ ENGINE = TinyLog , то в ней можно будет хранить не только `'hello'` и `'world'`, но и `NULL`. ``` -INSERT INTO t_enum_null Values('hello'),('world'),(NULL) +INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) ``` В оперативке столбец типа `Enum` представлен так же, как `Int8` или `Int16` соответствующими числовыми значениями. diff --git a/docs/zh/data_types/enum.md b/docs/zh/data_types/enum.md index ca8488b4345..41133b56d45 100644 --- a/docs/zh/data_types/enum.md +++ b/docs/zh/data_types/enum.md @@ -91,7 +91,7 @@ ENGINE = TinyLog 不仅可以存储 `'hello'` 和 `'world'` ，还可以存储 `NULL`。 ``` -INSERT INTO t_enum_null Values('hello'),('world'),(NULL) +INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) ``` 在内存中，`Enum` 列的存储方式与相应数值的 `Int8` 或 `Int16` 相同。 From 1683547b2d7ac65f22cda65dd8a0c769e8ec91e9 Mon Sep 17 00:00:00 2001 From: akonyaev Date: Thu, 22 Aug 2019 10:24:04 +0300 Subject: [PATCH 161/357] ADQM-34 fix style --- dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 15d0ad861ba..8fb6ab5a359 100644 --- a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -27,7 +27,6 @@ namespace DB extern const int CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN; extern const int THERE_IS_NO_COLUMN; } - const std::unordered_map> arrow_type_to_internal_type = { //{arrow::Type::DECIMAL, std::make_shared()}, {arrow::Type::UINT8, std::make_shared()}, @@ -63,7 +62,7 @@ namespace DB // 2. JSON -> String // Full list of types: contrib/arrow/cpp/src/arrow/type.h }; - + /// Inserts numeric data right into internal column data to reduce an overhead template > static void fillColumnWithNumericData(std::shared_ptr & arrow_column, MutableColumnPtr & internal_column) From c3aaf583a0df53bdbc62286c170df733993fe404 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 11:26:50 +0300 Subject: [PATCH 162/357] Fix RollupTransform. --- .../Processors/Transforms/RollupTransform.cpp | 33 ++++++++++++------- .../Processors/Transforms/RollupTransform.h | 10 +++--- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/dbms/src/Processors/Transforms/RollupTransform.cpp b/dbms/src/Processors/Transforms/RollupTransform.cpp index 9c8270ce091..1f5c140aafe 100644 --- a/dbms/src/Processors/Transforms/RollupTransform.cpp +++ b/dbms/src/Processors/Transforms/RollupTransform.cpp @@ -5,7 +5,7 @@ namespace DB { RollupTransform::RollupTransform(Block header, AggregatingTransformParamsPtr params_) - : IInflatingTransform(std::move(header), params_->getHeader()) + : IAccumulatingTransform(std::move(header), params_->getHeader()) , params(std::move(params_)) , keys(params->params.keys) { @@ -13,18 +13,33 @@ RollupTransform::RollupTransform(Block header, AggregatingTransformParamsPtr par void RollupTransform::consume(Chunk chunk) { - consumed_chunk = std::move(chunk); - last_removed_key = keys.size(); + consumed_chunks.emplace_back(std::move(chunk)); } -bool RollupTransform::canGenerate() +Chunk RollupTransform::merge(Chunks && chunks, bool final) { - return consumed_chunk; + BlocksList rollup_blocks; + for (auto & chunk : chunks) + rollup_blocks.emplace_back(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())); + + auto rollup_block = params->aggregator.mergeBlocks(rollup_blocks, final); + auto num_rows = rollup_block.rows(); + return Chunk(rollup_block.getColumns(), num_rows); } Chunk RollupTransform::generate() { - auto gen_chunk = std::move(consumed_chunk); + if (!consumed_chunks.empty()) + { + if (consumed_chunks.size() > 1) + rollup_chunk = merge(std::move(consumed_chunks), false); + else + rollup_chunk = std::move(consumed_chunks.front()); + + consumed_chunks.clear(); + } + + auto gen_chunk = std::move(rollup_chunk); if (last_removed_key) { @@ -35,11 +50,7 @@ Chunk RollupTransform::generate() auto columns = gen_chunk.getColumns(); columns[key] = columns[key]->cloneEmpty()->cloneResized(num_rows); - BlocksList rollup_blocks = { getInputPort().getHeader().cloneWithColumns(columns) }; - auto rollup_block = params->aggregator.mergeBlocks(rollup_blocks, false); - - num_rows = rollup_block.rows(); - consumed_chunk = Chunk(rollup_block.getColumns(), num_rows); + rollup_chunk = merge({Chunk(std::move(columns), num_rows)}, false); } finalizeChunk(gen_chunk); diff --git a/dbms/src/Processors/Transforms/RollupTransform.h b/dbms/src/Processors/Transforms/RollupTransform.h index 754e0237357..fd435740a63 100644 --- a/dbms/src/Processors/Transforms/RollupTransform.h +++ b/dbms/src/Processors/Transforms/RollupTransform.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include namespace DB @@ -7,7 +7,7 @@ namespace DB /// Takes blocks after grouping, with non-finalized aggregate functions. /// Calculates subtotals and grand totals values for a set of columns. -class RollupTransform : public IInflatingTransform +class RollupTransform : public IAccumulatingTransform { public: RollupTransform(Block header, AggregatingTransformParamsPtr params); @@ -15,14 +15,16 @@ public: protected: void consume(Chunk chunk) override; - bool canGenerate() override; Chunk generate() override; private: AggregatingTransformParamsPtr params; ColumnNumbers keys; - Chunk consumed_chunk; + Chunks consumed_chunks; + Chunk rollup_chunk; size_t last_removed_key = 0; + + Chunk merge(Chunks && chunks, bool final); }; } From 705e2c7437a7ee937bd47e8966c18fa04071ce7c Mon Sep 17 00:00:00 2001 From: akonyaev Date: Thu, 22 Aug 2019 11:28:32 +0300 Subject: [PATCH 163/357] ADQM-34 fix parquet tests --- .../queries/0_stateless/00900_parquet_load.reference | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00900_parquet_load.reference b/dbms/tests/queries/0_stateless/00900_parquet_load.reference index 83d0e8efde9..4e3977e0e96 100644 --- a/dbms/tests/queries/0_stateless/00900_parquet_load.reference +++ b/dbms/tests/queries/0_stateless/00900_parquet_load.reference @@ -39,7 +39,7 @@ 23.0 24.0 === Try load data from datapage_v2.snappy.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: IOError: Arrow error: IOError: Corrupt snappy compressed data. +Code: 33. DB::Ex---tion: Error while reading Parquet data: IOError: Arrow error: IOError: Corrupt snappy compressed data. === Try load data from fixed_length_decimal_1.parquet 1.0 @@ -171,19 +171,19 @@ Code: 33. DB::Ex---tion: Error while reading parquet data: IOError: Arrow error: Code: 8. DB::Ex---tion: Column "element" is not presented in input data === Try load data from nested_maps.snappy.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: NotImplemented: Currently only nesting with Lists is supported. +Code: 33. DB::Ex---tion: Error while reading Parquet data: NotImplemented: Currently only nesting with Lists is supported. === Try load data from nonnullable.impala.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: NotImplemented: Currently only nesting with Lists is supported. +Code: 33. DB::Ex---tion: Error while reading Parquet data: NotImplemented: Currently only nesting with Lists is supported. === Try load data from nullable.impala.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: NotImplemented: Currently only nesting with Lists is supported. +Code: 33. DB::Ex---tion: Error while reading Parquet data: NotImplemented: Currently only nesting with Lists is supported. === Try load data from nulls.snappy.parquet Code: 8. DB::Ex---tion: Column "b_c_int" is not presented in input data === Try load data from repeated_no_annotation.parquet -Code: 33. DB::Ex---tion: Error while reading parquet data: NotImplemented: Currently only nesting with Lists is supported. +Code: 33. DB::Ex---tion: Error while reading Parquet data: NotImplemented: Currently only nesting with Lists is supported. === Try load data from userdata1.parquet 1454486129 1 Amanda Jordan ajordan0@com.com Female 1.197.201.2 6759521864920116 Indonesia 3/8/1971 49756.53 Internal Auditor 1E+02 From 8228871821b60f0ab4af7f9aba6118ce26c129cb Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 11:29:29 +0300 Subject: [PATCH 164/357] Fix RollupTransform. --- dbms/src/Processors/Transforms/RollupTransform.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/src/Processors/Transforms/RollupTransform.cpp b/dbms/src/Processors/Transforms/RollupTransform.cpp index 1f5c140aafe..d60007b102e 100644 --- a/dbms/src/Processors/Transforms/RollupTransform.cpp +++ b/dbms/src/Processors/Transforms/RollupTransform.cpp @@ -50,7 +50,9 @@ Chunk RollupTransform::generate() auto columns = gen_chunk.getColumns(); columns[key] = columns[key]->cloneEmpty()->cloneResized(num_rows); - rollup_chunk = merge({Chunk(std::move(columns), num_rows)}, false); + Chunks chunks; + chunks.emplace_back(std::move(columns), num_rows); + rollup_chunk = merge(std::move(chunks), false); } finalizeChunk(gen_chunk); From 4f38b08481f572207e0263b701d15cbeb9f8f5bc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 11:53:34 +0300 Subject: [PATCH 165/357] Fix RollupTransform. --- dbms/src/Processors/Transforms/RollupTransform.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Processors/Transforms/RollupTransform.cpp b/dbms/src/Processors/Transforms/RollupTransform.cpp index d60007b102e..fb51b5f6b45 100644 --- a/dbms/src/Processors/Transforms/RollupTransform.cpp +++ b/dbms/src/Processors/Transforms/RollupTransform.cpp @@ -37,6 +37,7 @@ Chunk RollupTransform::generate() rollup_chunk = std::move(consumed_chunks.front()); consumed_chunks.clear(); + last_removed_key = keys.size(); } auto gen_chunk = std::move(rollup_chunk); From 11f8ec76df2086c112522ab4668d8e61fb5285ab Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 12:28:23 +0300 Subject: [PATCH 166/357] Fix CubeTransform. --- .../Processors/Transforms/CubeTransform.cpp | 53 ++++++++++++------- .../src/Processors/Transforms/CubeTransform.h | 10 ++-- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/dbms/src/Processors/Transforms/CubeTransform.cpp b/dbms/src/Processors/Transforms/CubeTransform.cpp index 5809a480d09..106bd0fef51 100644 --- a/dbms/src/Processors/Transforms/CubeTransform.cpp +++ b/dbms/src/Processors/Transforms/CubeTransform.cpp @@ -5,7 +5,7 @@ namespace DB { CubeTransform::CubeTransform(Block header, AggregatingTransformParamsPtr params_) - : IInflatingTransform(std::move(header), params_->getHeader()) + : IAccumulatingTransform(std::move(header), params_->getHeader()) , params(std::move(params_)) , keys(params->params.keys) { @@ -13,28 +13,45 @@ CubeTransform::CubeTransform(Block header, AggregatingTransformParamsPtr params_ throw Exception("Too many keys are used for CubeTransform.", ErrorCodes::LOGICAL_ERROR); } -void CubeTransform::consume(Chunk chunk) +Chunk CubeTransform::merge(Chunks && chunks, bool final) { - consumed_chunk = std::move(chunk); - auto num_rows = consumed_chunk.getNumRows(); - mask = (UInt64(1) << keys.size()) - 1; + BlocksList rollup_blocks; + for (auto & chunk : chunks) + rollup_blocks.emplace_back(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())); - current_columns = consumed_chunk.getColumns(); - current_zero_columns.clear(); - current_zero_columns.reserve(keys.size()); - - for (auto key : keys) - current_zero_columns.emplace_back(current_columns[key]->cloneEmpty()->cloneResized(num_rows)); + auto rollup_block = params->aggregator.mergeBlocks(rollup_blocks, final); + auto num_rows = rollup_block.rows(); + return Chunk(rollup_block.getColumns(), num_rows); } -bool CubeTransform::canGenerate() +void CubeTransform::consume(Chunk chunk) { - return consumed_chunk; + consumed_chunks.emplace_back(std::move(chunk)); } Chunk CubeTransform::generate() { - auto gen_chunk = std::move(consumed_chunk); + if (!consumed_chunks.empty()) + { + if (consumed_chunks.size() > 1) + cube_chunk = merge(std::move(consumed_chunks), false); + else + cube_chunk = std::move(consumed_chunks.front()); + + consumed_chunks.clear(); + + auto num_rows = cube_chunk.getNumRows(); + mask = (UInt64(1) << keys.size()) - 1; + + current_columns = cube_chunk.getColumns(); + current_zero_columns.clear(); + current_zero_columns.reserve(keys.size()); + + for (auto key : keys) + current_zero_columns.emplace_back(current_columns[key]->cloneEmpty()->cloneResized(num_rows)); + } + + auto gen_chunk = std::move(cube_chunk); if (mask) { @@ -47,11 +64,9 @@ Chunk CubeTransform::generate() if ((mask & (UInt64(1) << (size - i - 1))) == 0) columns[keys[i]] = current_zero_columns[i]; - BlocksList cube_blocks = { getInputPort().getHeader().cloneWithColumns(columns) }; - auto cube_block = params->aggregator.mergeBlocks(cube_blocks, false); - - auto num_rows = cube_block.rows(); - consumed_chunk = Chunk(cube_block.getColumns(), num_rows); + Chunks chunks; + chunks.emplace_back(std::move(columns), current_columns.front()->size()); + cube_chunk = merge(std::move(chunks), false); } finalizeChunk(gen_chunk); diff --git a/dbms/src/Processors/Transforms/CubeTransform.h b/dbms/src/Processors/Transforms/CubeTransform.h index 60259832e40..6d0e2338174 100644 --- a/dbms/src/Processors/Transforms/CubeTransform.h +++ b/dbms/src/Processors/Transforms/CubeTransform.h @@ -8,7 +8,7 @@ namespace DB /// Takes blocks after grouping, with non-finalized aggregate functions. /// Calculates all subsets of columns and aggregates over them. -class CubeTransform : public IInflatingTransform +class CubeTransform : public IAccumulatingTransform { public: CubeTransform(Block header, AggregatingTransformParamsPtr params); @@ -16,20 +16,20 @@ public: protected: void consume(Chunk chunk) override; - - bool canGenerate() override; - Chunk generate() override; private: AggregatingTransformParamsPtr params; ColumnNumbers keys; - Chunk consumed_chunk; + Chunks consumed_chunks; + Chunk cube_chunk; Columns current_columns; Columns current_zero_columns; UInt64 mask = 0; + + Chunk merge(Chunks && chunks, bool final); }; } From b93ffdd0387ea998be71bac361d6eda85a9c8c66 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 13:16:56 +0300 Subject: [PATCH 167/357] Fix MergingSortedTransform. --- dbms/src/Processors/Transforms/MergingSortedTransform.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dbms/src/Processors/Transforms/MergingSortedTransform.h b/dbms/src/Processors/Transforms/MergingSortedTransform.h index 223d5253e62..f1175c8d347 100644 --- a/dbms/src/Processors/Transforms/MergingSortedTransform.h +++ b/dbms/src/Processors/Transforms/MergingSortedTransform.h @@ -165,6 +165,13 @@ private: void updateCursor(Chunk chunk, size_t source_num) { + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + for (auto & column : columns) + column = column->convertToFullColumnIfConst(); + + chunk.setColumns(std::move(columns), num_rows); + auto & shared_chunk_ptr = source_chunks[source_num]; if (!shared_chunk_ptr) From 1f9087dbb22cf648a580aa7c286c45cc6a16e5c6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 13:46:23 +0300 Subject: [PATCH 168/357] Increase memory limit in 00284_external_aggregation. --- dbms/tests/queries/0_stateless/00284_external_aggregation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00284_external_aggregation.sql b/dbms/tests/queries/0_stateless/00284_external_aggregation.sql index 75d2c0b9bc2..03403b90b6c 100644 --- a/dbms/tests/queries/0_stateless/00284_external_aggregation.sql +++ b/dbms/tests/queries/0_stateless/00284_external_aggregation.sql @@ -1,5 +1,5 @@ SET max_bytes_before_external_group_by = 100000000; -SET max_memory_usage = 201000000; +SET max_memory_usage = 301000000; SELECT sum(k), sum(c) FROM (SELECT number AS k, count() AS c FROM (SELECT * FROM system.numbers LIMIT 10000000) GROUP BY k); SELECT sum(k), sum(c), max(u) FROM (SELECT number AS k, count() AS c, uniqArray(range(number % 16)) AS u FROM (SELECT * FROM system.numbers LIMIT 1000000) GROUP BY k); From fb534bd1ce0b290ba9380d47cbb04f75ccfdb1f0 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Thu, 22 Aug 2019 14:03:01 +0300 Subject: [PATCH 169/357] Added symlink to clickhouse-client package --- debian/clickhouse-client.install | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/clickhouse-client.install b/debian/clickhouse-client.install index 7331f3eb5ed..5e730db669f 100644 --- a/debian/clickhouse-client.install +++ b/debian/clickhouse-client.install @@ -3,5 +3,6 @@ usr/bin/clickhouse-local usr/bin/clickhouse-compressor usr/bin/clickhouse-benchmark usr/bin/clickhouse-format +usr/bin/clickhouse-obfuscator etc/clickhouse-client/config.xml usr/bin/clickhouse-extract-from-config From 63411b4d4a568bdebec9a463c607e74404a8ae9d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 15:51:33 +0300 Subject: [PATCH 170/357] Try to fix 00093_union_race_conditions_4. --- dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh b/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh index fcdbe4cbcdd..2d255a0c2f6 100755 --- a/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh +++ b/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh @@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -o errexit set -o pipefail -for i in {1..10}; do seq 1 10 | sed 's/.*/SELECT 1 % ((number + 500) % 1000) FROM system.numbers_mt LIMIT 1000;/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 --max_block_size=1 >/dev/null 2>&1 && echo 'Fail!' && break; echo -n '.'; done; echo +for i in {1..10}; do seq 1 10 | sed 's/.*/SELECT 1 % ((number + 500) % 1000) FROM numbers_mt(1000);/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 --max_block_size=1 >/dev/null 2>&1 && echo 'Fail!' && break; echo -n '.'; done; echo From 095124dcba3df720a1bc79adc2277c1d53884418 Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Thu, 22 Aug 2019 17:03:37 +0300 Subject: [PATCH 171/357] setOptionsDescription() function added --- dbms/programs/client/Client.cpp | 14 +++----- .../performance-test/PerformanceTestSuite.cpp | 18 ++-------- dbms/src/Common/SetOptionsDescription.h | 35 +++++++++++++++++++ 3 files changed, 41 insertions(+), 26 deletions(-) create mode 100644 dbms/src/Common/SetOptionsDescription.h diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index cf72d7a87c3..fa9431cb9e9 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -67,6 +67,7 @@ #include #include #include +#include #if USE_READLINE #include "Suggest.h" @@ -1641,23 +1642,16 @@ public: } stdin_is_not_tty = !isatty(STDIN_FILENO); - - namespace po = boost::program_options; - - unsigned line_length = po::options_description::m_default_line_length; - unsigned min_description_length = line_length / 2; if (!stdin_is_not_tty) { if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", ErrorCodes::SYSTEM_ERROR); - line_length = std::max( - static_cast(strlen("--http_native_compression_disable_checksumming_on_decompress ")), - static_cast(terminal_size.ws_col)); - min_description_length = std::min(min_description_length, line_length - 2); } + namespace po = boost::program_options; + /// Main commandline options related to client functionality and all parameters from Settings. - po::options_description main_description("Main options", line_length, min_description_length); + po::options_description main_description = setOptionsDescription("Main options"); main_description.add_options() ("help", "produce help message") ("config-file,C", po::value(), "config-file path") diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp index 943ee4fe6c2..d7b2d10e90f 100644 --- a/dbms/programs/performance-test/PerformanceTestSuite.cpp +++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "TestStopConditions.h" #include "TestStats.h" @@ -325,22 +326,7 @@ try using namespace DB; using po::value; - unsigned line_length = po::options_description::m_default_line_length; - unsigned min_description_length = line_length / 2; - winsize terminal_size {}; - - if (isatty(STDIN_FILENO)) - { - if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) - throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", ErrorCodes::SYSTEM_ERROR); - - line_length = std::max( - static_cast(strlen("--http_native_compression_disable_checksumming_on_decompress ")), - static_cast(terminal_size.ws_col)); - min_description_length = std::min(min_description_length, line_length - 2); - } - - po::options_description desc("Allowed options", line_length, min_description_length); + po::options_description desc = setOptionsDescription("Allowed options"); desc.add_options() ("help", "produce help message") ("lite", "use lite version of output") diff --git a/dbms/src/Common/SetOptionsDescription.h b/dbms/src/Common/SetOptionsDescription.h new file mode 100644 index 00000000000..a9c4031b064 --- /dev/null +++ b/dbms/src/Common/SetOptionsDescription.h @@ -0,0 +1,35 @@ +#include +#include +#include +#include + + +namespace po = boost::program_options; + + +namespace DB::ErrorCodes +{ + extern const int SYSTEM_ERROR; +} + +static po::options_description setOptionsDescription(const std::string & caption) +{ + unsigned line_length = po::options_description::m_default_line_length; + unsigned min_description_length = line_length / 2; + + if (isatty(STDIN_FILENO)) + { + winsize terminal_size {}; + + if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) + DB::throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", DB::ErrorCodes::SYSTEM_ERROR); + + std::string longest_option_desc = "--http_native_compression_disable_checksumming_on_decompress"; + + line_length = std::max(static_cast(longest_option_desc.size()), terminal_size.ws_col); + + min_description_length = std::min(min_description_length, line_length - 2); + } + + return po::options_description(caption, line_length, min_description_length); +} From e9d813bff79f3af4784bb905ca61054f6fec79e1 Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Thu, 22 Aug 2019 17:30:18 +0300 Subject: [PATCH 172/357] Minor fixes --- dbms/programs/client/Client.cpp | 2 +- dbms/programs/performance-test/PerformanceTestSuite.cpp | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index fa9431cb9e9..726a607812c 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1666,7 +1666,7 @@ public: * the "\n" is used to distinguish this case because there is hardly a chance an user would use "\n" * as the password. */ - ("password", po::value()->implicit_value("\n"), "password") + ("password", po::value()->implicit_value("\n","\\n"), "password") ("ask-password", "ask-password") ("query_id", po::value(), "query_id") ("query,q", po::value(), "query") diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp index d7b2d10e90f..99408c72f37 100644 --- a/dbms/programs/performance-test/PerformanceTestSuite.cpp +++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp @@ -46,7 +46,6 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int FILE_DOESNT_EXIST; - extern const int SYSTEM_ERROR; } /** Tests launcher for ClickHouse. @@ -323,8 +322,8 @@ std::unordered_map> getTestQueryIndexes(co int mainEntryClickHousePerformanceTest(int argc, char ** argv) try { - using namespace DB; using po::value; + using Strings = DB::Strings; po::options_description desc = setOptionsDescription("Allowed options"); desc.add_options() From b3f91a717cabca06b03aec29851d0776f45f06bc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 17:38:44 +0300 Subject: [PATCH 173/357] Use ThreadFromGlobalPool in PipelineExecutor instead of ThreadPool. --- .../Processors/Executors/PipelineExecutor.cpp | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index 31cca4e1a48..6aad6f96b5c 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -590,32 +590,45 @@ void PipelineExecutor::executeImpl(size_t num_threads) for (size_t i = 0; i < num_threads; ++i) executor_contexts.emplace_back(std::make_unique()); + auto thread_group = CurrentThread::getGroup(); + + using ThreadsData = std::vector; + ThreadsData threads; + threads.reserve(num_threads); + + bool finished_flag = false; + + SCOPE_EXIT( + if (!finished_flag) + { + finish(); + + for (auto & thread : threads) + thread.join(); + } + ); + addChildlessProcessorsToStack(stack); - while (!stack.empty()) { - UInt64 proc = stack.top(); - stack.pop(); + std::lock_guard lock(task_queue_mutex); - if (prepareProcessor(proc, stack, stack, 0, false)) + while (!stack.empty()) { - auto cur_state = graph[proc].execution_state.get(); - task_queue.push(cur_state); + UInt64 proc = stack.top(); + stack.pop(); + + if (prepareProcessor(proc, stack, stack, 0, false)) + { + auto cur_state = graph[proc].execution_state.get(); + task_queue.push(cur_state); + } } } - ThreadPool pool(num_threads); - - SCOPE_EXIT( - finish(); - pool.wait() - ); - - auto thread_group = CurrentThread::getGroup(); - for (size_t i = 0; i < num_threads; ++i) { - pool.schedule([this, thread_group, thread_num = i, num_threads] + threads.emplace_back([this, thread_group, thread_num = i, num_threads] { /// ThreadStatus thread_status; @@ -631,7 +644,10 @@ void PipelineExecutor::executeImpl(size_t num_threads) }); } - pool.wait(); + for (auto & thread : threads) + thread.join(); + + finished_flag = true; } String PipelineExecutor::dumpPipeline() const From 6d78e3be94b56f5e844238af7a688f0125d7e274 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 21 Aug 2019 11:12:39 +0300 Subject: [PATCH 174/357] hasToken function implementation * Function to check if given token is present in a string; * Special case for hasToken to 'tokenbf_v1' index; * Test cases for hasToken() * Test case for hasToken() + 'tokenbf_v1' integration --- dbms/src/Common/StringSearcher.h | 79 +++++++++- dbms/src/Common/Volnitsky.h | 14 +- dbms/src/Functions/FunctionsStringSearch.cpp | 76 ++++++++++ .../MergeTree/MergeTreeIndexFullText.cpp | 13 ++ .../queries/0_stateless/00990_hasToken.python | 124 ++++++++++++++++ .../0_stateless/00990_hasToken.reference | 139 ++++++++++++++++++ .../queries/0_stateless/00990_hasToken.sh | 8 + .../00990_hasToken_and_tokenbf.reference | 3 + .../00990_hasToken_and_tokenbf.sql | 33 +++++ 9 files changed, 481 insertions(+), 8 deletions(-) create mode 100755 dbms/tests/queries/0_stateless/00990_hasToken.python create mode 100644 dbms/tests/queries/0_stateless/00990_hasToken.reference create mode 100755 dbms/tests/queries/0_stateless/00990_hasToken.sh create mode 100644 dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.reference create mode 100644 dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql diff --git a/dbms/src/Common/StringSearcher.h b/dbms/src/Common/StringSearcher.h index 5e78ff23df1..fecf1a7ca81 100644 --- a/dbms/src/Common/StringSearcher.h +++ b/dbms/src/Common/StringSearcher.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include #include @@ -23,6 +25,7 @@ namespace DB namespace ErrorCodes { extern const int UNSUPPORTED_PARAMETER; + extern const int BAD_ARGUMENTS; } @@ -157,7 +160,7 @@ public: #endif } - ALWAYS_INLINE bool compare(const UInt8 * pos) const + ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const { static const Poco::UTF8Encoding utf8; @@ -374,7 +377,7 @@ public: #endif } - ALWAYS_INLINE bool compare(const UInt8 * pos) const + ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const { #ifdef __SSE4_1__ if (pageSafe(pos)) @@ -567,7 +570,7 @@ public: #endif } - ALWAYS_INLINE bool compare(const UInt8 * pos) const + ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const { #ifdef __SSE4_1__ if (pageSafe(pos)) @@ -697,11 +700,81 @@ public: } }; +// Searches for needle surrounded by token-separators. +// Separators are anything inside ASCII (0-128) and not alphanum. +// Any value outside of basic ASCII (>=128) is considered a non-separator symbol, hence UTF-8 strings +// should work just fine. But any Unicode whitespace is not considered a token separtor. +template +class TokenSearcher +{ + StringSearcher searcher; + size_t needle_size; + +public: + TokenSearcher(const char * const needle_, const size_t needle_size_) + : searcher{needle_, needle_size_}, + needle_size(needle_size_) + { + if (std::any_of(reinterpret_cast(needle_), reinterpret_cast(needle_) + needle_size_, isTokenSeparator)) + { + throw Exception{"needle must not contain whitespace characters", ErrorCodes::BAD_ARGUMENTS}; + } + + } + + ALWAYS_INLINE bool compare(const UInt8 * haystack, const UInt8 * haystack_end, const UInt8 * pos) const + { + // use searcher only if pos is in the beginning of token and pos + searcher.needle_size is end of token. + if (isToken(haystack, haystack_end, pos)) + return searcher.compare(haystack, haystack_end, pos); + + return false; + } + + const UInt8 * search(const UInt8 * haystack, const UInt8 * const haystack_end) const + { + // use searcher.search(), then verify that returned value is a token + // if it is not, skip it and re-run + + const UInt8 * pos = haystack; + while (pos < haystack_end) + { + pos = searcher.search(pos, haystack_end); + if (pos == haystack_end || isToken(haystack, haystack_end, pos)) + return pos; + + // assuming that heendle does not contain any token separators. + pos += needle_size; + } + return haystack_end; + } + + const UInt8 * search(const UInt8 * haystack, const size_t haystack_size) const + { + return search(haystack, haystack + haystack_size); + } + + ALWAYS_INLINE bool isToken(const UInt8 * haystack, const UInt8 * const haystack_end, const UInt8* p) const + { + return (p == haystack || isTokenSeparator(*(p - 1))) + && (p + needle_size >= haystack_end || isTokenSeparator(*(p + needle_size))); + } + + ALWAYS_INLINE static bool isTokenSeparator(const UInt8 c) + { + if (isAlphaNumericASCII(c) || !isASCII(c)) + return false; + + return true; + } +}; + using ASCIICaseSensitiveStringSearcher = StringSearcher; using ASCIICaseInsensitiveStringSearcher = StringSearcher; using UTF8CaseSensitiveStringSearcher = StringSearcher; using UTF8CaseInsensitiveStringSearcher = StringSearcher; +using ASCIICaseSensitiveTokenSearcher = TokenSearcher; /** Uses functions from libc. diff --git a/dbms/src/Common/Volnitsky.h b/dbms/src/Common/Volnitsky.h index 748cbe09138..c87bdd79dab 100644 --- a/dbms/src/Common/Volnitsky.h +++ b/dbms/src/Common/Volnitsky.h @@ -327,6 +327,8 @@ protected: FallbackSearcher fallback_searcher; public: + using Searcher = FallbackSearcher; + /** haystack_size_hint - the expected total size of the haystack for `search` calls. Optional (zero means unspecified). * If you specify it small enough, the fallback algorithm will be used, * since it is considered that it's useless to waste time initializing the hash table. @@ -373,7 +375,7 @@ public: const auto res = pos - (hash[cell_num] - 1); /// pointer in the code is always padded array so we can use pagesafe semantics - if (fallback_searcher.compare(res)) + if (fallback_searcher.compare(haystack, haystack_end, res)) return res; } } @@ -520,7 +522,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(res)) + if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) return true; } } @@ -552,7 +554,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(res)) + if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) ans = std::min(ans, ind); } } @@ -590,7 +592,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(res)) + if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) ans = std::min(ans, res - haystack); } } @@ -625,7 +627,7 @@ public: { const auto * res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (ans[ind] == 0 && res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(res)) + if (ans[ind] == 0 && res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) ans[ind] = count_chars(haystack, res); } } @@ -650,6 +652,8 @@ using VolnitskyUTF8 = VolnitskyBase; /// ignores non-ASCII bytes using VolnitskyCaseInsensitiveUTF8 = VolnitskyBase; +using VolnitskyToken = VolnitskyBase; + using MultiVolnitsky = MultiVolnitskyBase; using MultiVolnitskyUTF8 = MultiVolnitskyBase; using MultiVolnitskyCaseInsensitive = MultiVolnitskyBase; diff --git a/dbms/src/Functions/FunctionsStringSearch.cpp b/dbms/src/Functions/FunctionsStringSearch.cpp index 726eb8738af..5d688232bd4 100644 --- a/dbms/src/Functions/FunctionsStringSearch.cpp +++ b/dbms/src/Functions/FunctionsStringSearch.cpp @@ -434,6 +434,74 @@ struct MultiSearchFirstIndexImpl } }; +/** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation. + */ +template +struct HasTokenImpl +{ + using ResultType = UInt8; + + static void vector_constant( + const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & pattern, PaddedPODArray & res) + { + if (offsets.empty()) + return; + + const UInt8 * begin = data.data(); + const UInt8 * pos = begin; + const UInt8 * end = pos + data.size(); + + /// The current index in the array of strings. + size_t i = 0; + + VolnitskyToken searcher(pattern.data(), pattern.size(), end - pos); + + /// We will search for the next occurrence in all rows at once. + while (pos < end && end != (pos = searcher.search(pos, end - pos))) + { + /// Let's determine which index it refers to. + while (begin + offsets[i] <= pos) + { + res[i] = negate_result; + ++i; + } + + /// We check that the entry does not pass through the boundaries of strings. + if (pos + pattern.size() < begin + offsets[i]) + res[i] = !negate_result; + else + res[i] = negate_result; + + pos = begin + offsets[i]; + ++i; + } + + /// Tail, in which there can be no substring. + if (i < res.size()) + memset(&res[i], negate_result, (res.size() - i) * sizeof(res[0])); + } + + static void constant_constant(const std::string & data, const std::string & pattern, UInt8 & res) + { + VolnitskyToken searcher(pattern.data(), pattern.size(), data.size()); + const auto found = searcher.search(data.c_str(), data.size()) != data.end().base(); + res = negate_result ^ found; + } + + template + static void vector_vector(Args &&...) + { + throw Exception("Function 'hasToken' does not support non-constant needle argument", ErrorCodes::ILLEGAL_COLUMN); + } + + /// Search different needles in single haystack. + template + static void constant_vector(Args &&...) + { + throw Exception("Function 'hasToken' does not support non-constant needle argument", ErrorCodes::ILLEGAL_COLUMN); + } +}; + struct NamePosition { @@ -516,6 +584,11 @@ struct NameMultiSearchFirstPositionCaseInsensitiveUTF8 static constexpr auto name = "multiSearchFirstPositionCaseInsensitiveUTF8"; }; +struct NameHasToken +{ + static constexpr auto name = "hasToken"; +}; + using FunctionPosition = FunctionsStringSearch, NamePosition>; using FunctionPositionUTF8 = FunctionsStringSearch, NamePositionUTF8>; @@ -542,6 +615,7 @@ using FunctionMultiSearchFirstPositionUTF8 = FunctionsMultiStringSearch, NameMultiSearchFirstPositionCaseInsensitive>; using FunctionMultiSearchFirstPositionCaseInsensitiveUTF8 = FunctionsMultiStringSearch, NameMultiSearchFirstPositionCaseInsensitiveUTF8>; +using FunctionHasToken = FunctionsStringSearch, NameHasToken>; void registerFunctionsStringSearch(FunctionFactory & factory) { @@ -570,6 +644,8 @@ void registerFunctionsStringSearch(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerAlias("locate", NamePosition::name, FunctionFactory::CaseInsensitive); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index 3625c6f1aa5..246ad6784b2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -168,6 +168,19 @@ const MergeTreeConditionFullText::AtomMap MergeTreeConditionFullText::atom_map return true; } }, + { + "hasToken", + [] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) + { + out.function = RPNElement::FUNCTION_EQUALS; + out.bloom_filter = std::make_unique( + idx.bloom_filter_size, idx.bloom_filter_hashes, idx.seed); + + const auto & str = value.get(); + stringToBloomFilter(str.c_str(), str.size(), idx.token_extractor_func, *out.bloom_filter); + return true; + } + }, { "startsWith", [] (RPNElement & out, const Field & value, const MergeTreeIndexFullText & idx) diff --git a/dbms/tests/queries/0_stateless/00990_hasToken.python b/dbms/tests/queries/0_stateless/00990_hasToken.python new file mode 100755 index 00000000000..217d96dfe52 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken.python @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# encoding: utf-8 + +import re + +HAYSTACKS = [ + "hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay needle", + "needle hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay", + "hay hay hay hay hay hay hay hay hay needle hay hay hay hay hay hay hay hay hay", +] + +NEEDLE = "needle" + +HAY_RE = re.compile(r'\bhay\b', re.IGNORECASE) +NEEDLE_RE = re.compile(r'\bneedle\b', re.IGNORECASE) + +def replace_follow_case(replacement): + def func(match): + g = match.group() + if g.islower(): return replacement.lower() + if g.istitle(): return replacement.title() + if g.isupper(): return replacement.upper() + return replacement + return func + +def replace_separators(query, new_sep): + SEP_RE = re.compile('\\s+') + result = SEP_RE.sub(new_sep, query) + return result + +def enlarge_haystack(query, times, separator=''): + return HAY_RE.sub(replace_follow_case(('hay' + separator) * times), query) + +def small_needle(query): + return NEEDLE_RE.sub(replace_follow_case('n'), query) + +def remove_needle(query): + return NEEDLE_RE.sub('', query) + +def replace_needle(query, new_needle): + return NEEDLE_RE.sub(new_needle, query) + +# with str.lower, str.uppert, str.title and such +def transform_needle(query, string_transformation_func): + def replace_with_transformation(match): + g = match.group() + return string_transformation_func(g) + + return NEEDLE_RE.sub(replace_with_transformation, query) + + +def create_cases(table_row_template, table_query_template, const_query_template): + const_queries = [] + table_rows = [] + table_queries = set() + + def add_case(haystack, needle, match): + match = int(match) + const_queries.append(const_query_template.format(haystack=haystack, needle=needle, match=match)) + table_queries.add(table_query_template.format(haystack=haystack, needle=needle, match=match)) + table_rows.append(table_row_template.format(haystack=haystack, needle=needle, match=match)) + + # Negative cases + add_case(remove_needle(HAYSTACKS[0]), NEEDLE, False) + for haystack in HAYSTACKS: + add_case(transform_needle(haystack, str.title), NEEDLE, False) + sep = '' + h = replace_separators(haystack, sep) + add_case(h, NEEDLE, False) + add_case(small_needle(h), small_needle(NEEDLE), False) + add_case(enlarge_haystack(h, 10, sep), NEEDLE, False) + + # positive cases + for haystack in HAYSTACKS: + add_case(transform_needle(haystack, str.title), transform_needle(NEEDLE, str.title), True) + add_case(transform_needle(haystack, str.upper), transform_needle(NEEDLE, str.upper), True) + + # Not checking all separators since some (like ' and \n) cause issues when coupled with + # re-based replacement and quoting in query + # other are rare in practice and checking all separators makes this test too lengthy. + + # r'\\\\' turns into a single '\' in query + #separators = list(''' \t`~!@#$%^&*()-=+|]}[{";:/?.>,<''') + [r'\\\\'] + separators = list(''' \t;:?.,''') + [r'\\\\'] + for sep in separators: + h = replace_separators(haystack, sep) + add_case(h, NEEDLE, True) + add_case(small_needle(h), small_needle(NEEDLE), True) + add_case(enlarge_haystack(h, 200, sep), NEEDLE, True) + add_case(replace_needle(h, 'иголка'), replace_needle(NEEDLE, 'иголка'), True) + add_case(replace_needle(h, '指针'), replace_needle(NEEDLE, '指针'), True) + + return table_rows, table_queries, const_queries + +def main(): + + def query(x): + print x + + CONST_QUERY = """SELECT hasToken('{haystack}', '{needle}'), ' expecting ', {match};""" + #SELECT hasToken(haystack, '{needle}') FROM ht WHERE needle = '{needle}' AND match = {match};""" + TABLE_QUERY = """WITH '{needle}' as n SELECT haystack, needle, hasToken(haystack, n) as result FROM ht WHERE needle = n AND result != match;""" + TABLE_ROW = """('{haystack}', '{needle}', {match})""" + + rows, table_queries, const_queries = create_cases(TABLE_ROW, TABLE_QUERY, CONST_QUERY) + for q in const_queries: + query(q) + + query("""DROP TABLE IF EXISTS ht; + CREATE TABLE IF NOT EXISTS + ht +( + haystack String, + needle String, + match UInt8 +) +ENGINE MergeTree() +ORDER BY haystack; +INSERT INTO ht VALUES {values};""".format(values=", ".join(rows))) + for q in sorted(table_queries): + query(q) + +if __name__ == '__main__': + main() diff --git a/dbms/tests/queries/0_stateless/00990_hasToken.reference b/dbms/tests/queries/0_stateless/00990_hasToken.reference new file mode 100644 index 00000000000..867c0c1c691 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken.reference @@ -0,0 +1,139 @@ +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 diff --git a/dbms/tests/queries/0_stateless/00990_hasToken.sh b/dbms/tests/queries/0_stateless/00990_hasToken.sh new file mode 100755 index 00000000000..4ccb77b8ecc --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +# We should have correct env vars from shell_config.sh to run this test + +python $CURDIR/00990_hasToken.python | ${CLICKHOUSE_CLIENT} -nm diff --git a/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.reference b/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.reference new file mode 100644 index 00000000000..10e8f0d2c59 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.reference @@ -0,0 +1,3 @@ +2007 +2007 +2007 diff --git a/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql b/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql new file mode 100644 index 00000000000..60e4d959417 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql @@ -0,0 +1,33 @@ +SET allow_experimental_data_skipping_indices = 1; + +DROP TABLE IF EXISTS bloom_filter; + +CREATE TABLE bloom_filter +( + id UInt64, + s String, + INDEX tok_bf (s, lower(s)) TYPE tokenbf_v1(512, 3, 0) GRANULARITY 1 +) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 8; + +insert into bloom_filter select number, 'yyy,uuu' from numbers(1024); +insert into bloom_filter select number+2000, 'abc,def,zzz' from numbers(8); +insert into bloom_filter select number+3000, 'yyy,uuu' from numbers(1024); +insert into bloom_filter select number+3000, 'abcdefzzz' from numbers(1024); + +set max_rows_to_read = 16; + +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'abc'); +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'def'); +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'zzz'); + +-- invert result +-- this does not work as expected, reading more rows that it should +-- SELECT max(id) FROM bloom_filter WHERE NOT hasToken(s, 'yyy'); + +-- accessing to many rows +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'yyy'); -- { serverError 158 } + +-- this syntax is not supported by tokenbf +SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'zzz') == 1; -- { serverError 158 } + +DROP TABLE bloom_filter; \ No newline at end of file From a50d6e713299f8b54c7d1b81cc9742bf4a6a211e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 18:07:07 +0300 Subject: [PATCH 175/357] Update 00416_pocopatch_progress_in_http_headers. --- ...copatch_progress_in_http_headers.reference | 23 ++++++++++--------- ...0416_pocopatch_progress_in_http_headers.sh | 5 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference index e838f583cdf..a2c79e66928 100644 --- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference +++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.reference @@ -3,17 +3,18 @@ < X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} < X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} 9 -< X-ClickHouse-Progress: {"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"2","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"4","read_bytes":"32","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"5","read_bytes":"40","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"6","read_bytes":"48","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"7","read_bytes":"56","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"8","read_bytes":"64","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"9","read_bytes":"72","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} -< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"} +< X-ClickHouse-Progress: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"2","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"4","read_bytes":"32","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"5","read_bytes":"40","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"6","read_bytes":"48","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"7","read_bytes":"56","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"8","read_bytes":"64","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"9","read_bytes":"72","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} +< X-ClickHouse-Progress: {"read_rows":"10","read_bytes":"80","written_rows":"0","written_bytes":"0","total_rows_to_read":"10"} 0 1 2 diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh index d95798bc95c..c86154a8402 100755 --- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh +++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh @@ -6,9 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=5&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT max(number) FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' # This test will fail with external poco (progress not supported) -# "grep -v 11" in order to skip extra progress header for 11-th row (for processors pipeline) -${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&experimental_use_processors=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' | grep -v 11 -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1&experimental_use_processors=0" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d # 'send_progress_in_http_headers' is false by default ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -q 'X-ClickHouse-Progress' && echo 'Fail' || true From b6c8a492445e99d1d4ca318c6cd580c6362fb634 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Aug 2019 20:08:10 +0300 Subject: [PATCH 176/357] Update 00284_external_aggregation --- dbms/tests/queries/0_stateless/00284_external_aggregation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00284_external_aggregation.sql b/dbms/tests/queries/0_stateless/00284_external_aggregation.sql index 03403b90b6c..cd9abec59a8 100644 --- a/dbms/tests/queries/0_stateless/00284_external_aggregation.sql +++ b/dbms/tests/queries/0_stateless/00284_external_aggregation.sql @@ -1,5 +1,5 @@ SET max_bytes_before_external_group_by = 100000000; -SET max_memory_usage = 301000000; +SET max_memory_usage = 351000000; SELECT sum(k), sum(c) FROM (SELECT number AS k, count() AS c FROM (SELECT * FROM system.numbers LIMIT 10000000) GROUP BY k); SELECT sum(k), sum(c), max(u) FROM (SELECT number AS k, count() AS c, uniqArray(range(number % 16)) AS u FROM (SELECT * FROM system.numbers LIMIT 1000000) GROUP BY k); From acce56095fc78c7d4fbe6e80746c4241fe698d89 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 22 Aug 2019 22:35:46 +0300 Subject: [PATCH 177/357] limit number of background threads for mutations --- .../MergeTree/MergeTreeDataMergerMutator.cpp | 9 ++++++++- .../src/Storages/MergeTree/MergeTreeSettings.h | 1 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 18 +++++++++++------- dbms/src/Storages/StorageMergeTree.cpp | 7 ++----- .../Storages/StorageReplicatedMergeTree.cpp | 3 +-- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 74193fa7156..5a9affaacd4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -157,7 +157,14 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_siz UInt64 MergeTreeDataMergerMutator::getMaxSourcePartSizeForMutation() { - return static_cast(DiskSpaceMonitor::getUnreservedFreeSpace(data.full_path) / DISK_USAGE_COEFFICIENT_TO_RESERVE); + size_t total_threads_in_pool = pool.getNumberOfThreads(); + size_t busy_threads_in_pool = CurrentMetrics::values[CurrentMetrics::BackgroundPoolTask].load(std::memory_order_relaxed); + + /// Allow mutations only if there are enough threads, leave free threads for merges else + if (total_threads_in_pool - busy_threads_in_pool >= data.settings.number_of_free_entries_in_pool_to_execute_mutation) + return static_cast(DiskSpaceMonitor::getUnreservedFreeSpace(data.full_path) / DISK_USAGE_COEFFICIENT_TO_RESERVE); + + return 0; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 36e82b96961..afd0772a937 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -33,6 +33,7 @@ struct MergeTreeSettings : public SettingsCollection M(SettingUInt64, max_replicated_merges_in_queue, 16, "How many tasks of merging and mutating parts are allowed simultaneously in ReplicatedMergeTree queue.") \ M(SettingUInt64, max_replicated_mutations_in_queue, 8, "How many tasks of mutating parts are allowed simultaneously in ReplicatedMergeTree queue.") \ M(SettingUInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge, 8, "When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue). This is to allow small merges to process - not filling the pool with long running merges.") \ + M(SettingUInt64, number_of_free_entries_in_pool_to_execute_mutation, 10, "When there is less than specified number of free entries in pool, do not execute part mutations. This is to leave free threads for regular merges and avoid \"Too many parts\"") \ M(SettingSeconds, old_parts_lifetime, 8 * 60, "How many seconds to keep obsolete parts.") \ M(SettingSeconds, temporary_directories_lifetime, 86400, "How many seconds to keep tmp_-directories.") \ \ diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index e6251502576..fd65f14fedb 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -956,15 +956,19 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( return false; } - /** Execute merge only if there are enough free threads in background pool to do merges of that size. - * But if all threads are free (maximal size of merge is allowed) then execute any merge, - * (because it may be ordered by OPTIMIZE or early with differrent settings). + UInt64 max_source_parts_size = entry.type == LogEntry::MERGE_PARTS ? merger_mutator.getMaxSourcePartsSizeForMerge() + : merger_mutator.getMaxSourcePartSizeForMutation(); + /** If there are enough free threads in background pool to do large merges (maximal size of merge is allowed), + * then ignore value returned by getMaxSourcePartsSizeForMerge() and execute merge of any size, + * because it may be ordered by OPTIMIZE or early with different settings. + * Setting max_bytes_to_merge_at_max_space_in_pool still working for regular merges, + * because the leader replica does not assign merges of greater size (except OPTIMIZE PARTITION and OPTIMIZE FINAL). */ - UInt64 max_source_parts_size = merger_mutator.getMaxSourcePartsSizeForMerge(); - if (max_source_parts_size != data.settings.max_bytes_to_merge_at_max_space_in_pool - && sum_parts_size_in_bytes > max_source_parts_size) + bool ignore_max_size = (entry.type == LogEntry::MERGE_PARTS) && (max_source_parts_size == data.settings.max_bytes_to_merge_at_max_space_in_pool); + + if (!ignore_max_size && sum_parts_size_in_bytes > max_source_parts_size) { - String reason = "Not executing log entry for part " + entry.new_part_name + String reason = "Not executing log entry " + entry.typeToString() + " for part " + entry.new_part_name + " because source parts size (" + formatReadableSizeWithBinarySuffix(sum_parts_size_in_bytes) + ") is greater than the current maximum (" + formatReadableSizeWithBinarySuffix(max_source_parts_size) + ")."; LOG_DEBUG(log, reason); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index d062bb197ca..779efe95a8c 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -624,8 +624,6 @@ bool StorageMergeTree::tryMutatePart() /// You must call destructor with unlocked `currently_merging_mutex`. std::optional tagger; { - auto disk_space = DiskSpaceMonitor::getUnreservedFreeSpace(full_path); - std::lock_guard lock(currently_merging_mutex); if (current_mutations_by_version.empty()) @@ -641,8 +639,7 @@ bool StorageMergeTree::tryMutatePart() if (mutations_begin_it == mutations_end_it) continue; - auto estimated_needed_space = MergeTreeDataMergerMutator::estimateNeededDiskSpace({part}); - if (estimated_needed_space > disk_space) + if (merger_mutator.getMaxSourcePartSizeForMutation() < part->bytes_on_disk) continue; for (auto it = mutations_begin_it; it != mutations_end_it; ++it) @@ -655,7 +652,7 @@ bool StorageMergeTree::tryMutatePart() future_part.part_info = new_part_info; future_part.name = part->getNewName(new_part_info); - tagger.emplace(future_part, estimated_needed_space, *this); + tagger.emplace(future_part, MergeTreeDataMergerMutator::estimateNeededDiskSpace({part}), *this); break; } } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 028235d9eef..7a946400658 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2208,14 +2208,13 @@ void StorageReplicatedMergeTree::mergeSelectingTask() UInt64 max_source_part_size_for_mutation = merger_mutator.getMaxSourcePartSizeForMutation(); FutureMergedMutatedPart future_merged_part; - - /// If there are many mutations in queue it may happen, that we cannot enqueue enough merges to merge all new parts if (max_source_parts_size_for_merge > 0 && merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, merge_pred)) { success = createLogEntryToMergeParts(zookeeper, future_merged_part.parts, future_merged_part.name, deduplicate, force_ttl); } + /// If there are many mutations in queue it may happen, that we cannot enqueue enough merges to merge all new parts else if (max_source_part_size_for_mutation > 0 && queue.countMutations() > 0 && merges_and_mutations_queued.second < settings.max_replicated_mutations_in_queue) { From 9c054419323fb8db91b586f6c19208507c7452c5 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 22 Aug 2019 22:54:42 +0300 Subject: [PATCH 178/357] better test --- .../configs/merge_tree_max_parts.xml | 6 +++ .../configs/merge_tree_queue.xml | 7 --- .../test_replicated_mutations/test.py | 50 ++++++++++++------- 3 files changed, 39 insertions(+), 24 deletions(-) create mode 100644 dbms/tests/integration/test_replicated_mutations/configs/merge_tree_max_parts.xml delete mode 100644 dbms/tests/integration/test_replicated_mutations/configs/merge_tree_queue.xml diff --git a/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_max_parts.xml b/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_max_parts.xml new file mode 100644 index 00000000000..60047dcab2c --- /dev/null +++ b/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_max_parts.xml @@ -0,0 +1,6 @@ + + + 50 + 50 + + \ No newline at end of file diff --git a/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_queue.xml b/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_queue.xml deleted file mode 100644 index ccc63f2eaec..00000000000 --- a/dbms/tests/integration/test_replicated_mutations/configs/merge_tree_queue.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - 50 - 8 - 4 - - \ No newline at end of file diff --git a/dbms/tests/integration/test_replicated_mutations/test.py b/dbms/tests/integration/test_replicated_mutations/test.py index dd42a70e280..0347ba4782c 100644 --- a/dbms/tests/integration/test_replicated_mutations/test.py +++ b/dbms/tests/integration/test_replicated_mutations/test.py @@ -14,10 +14,12 @@ node1 = cluster.add_instance('node1', macros={'cluster': 'test1'}, with_zookeepe # Check, that limits on max part size for merges doesn`t affect mutations node2 = cluster.add_instance('node2', macros={'cluster': 'test1'}, main_configs=["configs/merge_tree.xml"], with_zookeeper=True) -node3 = cluster.add_instance('node3', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_queue.xml"], with_zookeeper=True) -node4 = cluster.add_instance('node4', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_queue.xml"], with_zookeeper=True) +node3 = cluster.add_instance('node3', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], with_zookeeper=True) +node4 = cluster.add_instance('node4', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], with_zookeeper=True) -all_nodes = [node1, node2, node3, node4] +node5 = cluster.add_instance('node5', macros={'cluster': 'test3'}, main_configs=["configs/merge_tree_max_parts.xml"]) + +all_nodes = [node1, node2, node3, node4, node5] @pytest.fixture(scope="module") def started_cluster(): @@ -27,9 +29,11 @@ def started_cluster(): for node in all_nodes: node.query("DROP TABLE IF EXISTS test_mutations") - for node in all_nodes: + for node in [node1, node2, node3, node4]: node.query("CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/test/test_mutations', '{instance}') ORDER BY x PARTITION BY toYYYYMM(d)") + node5.query("CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE MergeTree() ORDER BY x PARTITION BY toYYYYMM(d)") + yield cluster finally: @@ -56,7 +60,7 @@ class Runner: self.exceptions = [] - def do_insert(self, thread_num): + def do_insert(self, thread_num, partitions_num): self.stop_ev.wait(random.random()) # Each thread inserts a small random number of rows with random year, month 01 and day determined @@ -74,7 +78,7 @@ class Runner: for x in xs: self.currently_inserting_xs[x] += 1 - year = 2000 + random.randint(0, 10) + year = 2000 + random.randint(0, partitions_num) date_str = '{year}-{month}-{day}'.format(year=year, month=month, day=day) payload = '' for x in xs: @@ -158,7 +162,7 @@ def test_mutations(started_cluster): threads = [] for thread_num in range(5): - threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, ))) + threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, 10))) for thread_num in (11, 12, 13): threads.append(threading.Thread(target=runner.do_delete, args=(thread_num,))) @@ -178,7 +182,9 @@ def test_mutations(started_cluster): all_done = wait_for_mutations(nodes, runner.total_mutations) - print node1.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") + print "Total mutations: ", runner.total_mutations + for node in nodes: + print node.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") assert all_done expected_sum = runner.total_inserted_xs - runner.total_deleted_xs @@ -188,24 +194,30 @@ def test_mutations(started_cluster): assert actual_sums[i] == expected_sum -def test_mutations_dont_prevent_merges(started_cluster): - nodes = [node3, node4] - for year in range(2000, 2008): +@pytest.mark.parametrize( + ('nodes', ), + [ + ([node5, ], ), # MergeTree + ([node3, node4], ), # ReplicatedMergeTree + ] +) +def test_mutations_dont_prevent_merges(started_cluster, nodes): + for year in range(2000, 2016): rows = '' date_str = '{}-01-{}'.format(year, random.randint(1, 10)) for i in range(10): rows += '{} {} {}\n'.format(date_str, random.randint(1, 10), i) - node3.query("INSERT INTO test_mutations FORMAT TSV", rows) + nodes[0].query("INSERT INTO test_mutations FORMAT TSV", rows) - # will run mutations of 8 parts in parallel, mutations will sleep for about 20 seconds - node3.query("ALTER TABLE test_mutations UPDATE i = sleepEachRow(2) WHERE 1") + # will run mutations of 16 parts in parallel, mutations will sleep for about 20 seconds + nodes[0].query("ALTER TABLE test_mutations UPDATE i = sleepEachRow(2) WHERE 1") runner = Runner(nodes) threads = [] - for thread_num in range(10): - threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, ))) + for thread_num in range(2): + threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, 0))) - # will insert approx 4-5 new parts per 1 second into each partition + # will insert approx 8-10 new parts per 1 second into one partition for t in threads: t.start() @@ -215,5 +227,9 @@ def test_mutations_dont_prevent_merges(started_cluster): for t in threads: t.join() + for node in nodes: + print node.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") + print node.query("SELECT partition, count(name), sum(active), sum(active*rows) FROM system.parts WHERE table ='test_mutations' GROUP BY partition FORMAT TSVWithNames") + assert all_done assert all([str(e).find("Too many parts") < 0 for e in runner.exceptions]) From e8bc2189840613f6cbe1caaca987c293adff1b16 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 00:35:16 +0300 Subject: [PATCH 179/357] Rewrite flappy test --- .../00600_replace_running_query.reference | 8 ++++--- .../00600_replace_running_query.sh | 22 ++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00600_replace_running_query.reference b/dbms/tests/queries/0_stateless/00600_replace_running_query.reference index 237dd6b5309..804267a1c11 100644 --- a/dbms/tests/queries/0_stateless/00600_replace_running_query.reference +++ b/dbms/tests/queries/0_stateless/00600_replace_running_query.reference @@ -1,5 +1,7 @@ 0 -1 0 -3 0 -2 0 +1 +1 +1 +finished 42 readonly SELECT 2, count() FROM system.numbers +1 44 diff --git a/dbms/tests/queries/0_stateless/00600_replace_running_query.sh b/dbms/tests/queries/0_stateless/00600_replace_running_query.sh index 8e4677bb1d5..513f6d8440e 100755 --- a/dbms/tests/queries/0_stateless/00600_replace_running_query.sh +++ b/dbms/tests/queries/0_stateless/00600_replace_running_query.sh @@ -1,17 +1,18 @@ #!/usr/bin/env bash +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -set -e -o pipefail function wait_for_query_to_start() { while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done } -$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=hello&replace_running_query=1" -d 'SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(100000000) GROUP BY k)' 2>&1 > /dev/null & +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=hello&replace_running_query=1" -d 'SELECT 1, count() FROM system.numbers' 2>&1 > /dev/null & wait_for_query_to_start 'hello' # Replace it @@ -20,15 +21,20 @@ $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=hello&replace_running_query=1" -d # Wait for it to be replaced wait -${CLICKHOUSE_CLIENT} --user=readonly --query_id=42 --query='SELECT 1, sleep(1)' & +${CLICKHOUSE_CLIENT} --user=readonly --query_id=42 --query='SELECT 2, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & wait_for_query_to_start '42' -( ${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 43' ||: ) 2>&1 | grep -F 'is already running by user' > /dev/null + +# Trying to run another query with the same query_id +${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 43' 2>&1 | grep -cF 'is already running by user' + +# Trying to replace query of a different user +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=42&replace_running_query=1" -d 'SELECT 1' | grep -cF 'is already running by user' + +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = '42' SYNC" wait -${CLICKHOUSE_CLIENT} --query='SELECT 3, sleep(1)' & -sleep 0.1 -${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 2, sleep(1)' & +${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 3, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & wait_for_query_to_start '42' -( ${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --queue_max_wait_ms=500 --query='SELECT 43' ||: ) 2>&1 | grep -F "can't be stopped" > /dev/null +${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --queue_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null ${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --query='SELECT 44' wait From dce975321599838a03ed22ab3a52a493a1f01f5d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 00:44:40 +0300 Subject: [PATCH 180/357] Fix splitted build. --- contrib/arrow-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 7b94acc9031..bc229deeced 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -219,7 +219,7 @@ endif() add_library(${ARROW_LIBRARY} ${ARROW_SRCS}) add_dependencies(${ARROW_LIBRARY} protoc) target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS}) -target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} Threads::Threads) +target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${PROTOBUF_LIBRARIES} Threads::Threads) if (ARROW_WITH_LZ4) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY}) endif() From 859736d935275ef9d66e1f84c853f7f69f780394 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 01:41:30 +0300 Subject: [PATCH 181/357] Basic code quality of Live View --- .../PushingToViewsBlockOutputStream.cpp | 3 +- dbms/src/Storages/StorageLiveView.cpp | 217 ++++++++++++++++-- dbms/src/Storages/StorageLiveView.h | 184 +-------------- 3 files changed, 207 insertions(+), 197 deletions(-) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 840d3479ab9..d16d68bf72b 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -106,8 +106,7 @@ void PushingToViewsBlockOutputStream::write(const Block & block) if (auto * live_view = dynamic_cast(storage.get())) { - BlockOutputStreamPtr output_ = std::make_shared(*live_view); - StorageLiveView::writeIntoLiveView(*live_view, block, context, output_); + StorageLiveView::writeIntoLiveView(*live_view, block, context); } else { diff --git a/dbms/src/Storages/StorageLiveView.cpp b/dbms/src/Storages/StorageLiveView.cpp index 6e42b9780e1..fdfa6a3a80d 100644 --- a/dbms/src/Storages/StorageLiveView.cpp +++ b/dbms/src/Storages/StorageLiveView.cpp @@ -9,6 +9,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + #include #include #include @@ -17,10 +18,17 @@ limitations under the License. */ #include #include #include +#include #include #include #include #include +#include +#include +#include +#include +#include +#include #include #include @@ -98,6 +106,174 @@ static void checkAllowedQueries(const ASTSelectQuery & query) } } + +class LiveViewBlockOutputStream : public IBlockOutputStream +{ +public: + explicit LiveViewBlockOutputStream(StorageLiveView & storage_) : storage(storage_) {} + + void writePrefix() override + { + new_blocks = std::make_shared(); + new_blocks_metadata = std::make_shared(); + new_hash = std::make_shared(); + } + + void writeSuffix() override + { + UInt128 key; + String key_str; + + new_hash->get128(key.low, key.high); + key_str = key.toHexString(); + + Poco::FastMutex::ScopedLock lock(storage.mutex); + + if (storage.getBlocksHashKey() != key_str) + { + new_blocks_metadata->hash = key_str; + new_blocks_metadata->version = storage.getBlocksVersion() + 1; + + for (auto & block : *new_blocks) + { + block.insert({DataTypeUInt64().createColumnConst( + block.rows(), new_blocks_metadata->version)->convertToFullColumnIfConst(), + std::make_shared(), + "_version"}); + } + + (*storage.blocks_ptr) = new_blocks; + (*storage.blocks_metadata_ptr) = new_blocks_metadata; + + storage.condition.broadcast(); + } + + new_blocks.reset(); + new_blocks_metadata.reset(); + new_hash.reset(); + } + + void write(const Block & block) override + { + new_blocks->push_back(block); + block.updateHash(*new_hash); + } + + Block getHeader() const override { return storage.getHeader(); } + +private: + using SipHashPtr = std::shared_ptr; + + BlocksPtr new_blocks; + BlocksMetadataPtr new_blocks_metadata; + SipHashPtr new_hash; + StorageLiveView & storage; +}; + + +void StorageLiveView::writeIntoLiveView( + StorageLiveView & live_view, + const Block & block, + const Context & context) +{ + BlockOutputStreamPtr output = std::make_shared(live_view); + + /// Check if live view has any readers if not + /// just reset blocks to empty and do nothing else + /// When first reader comes the blocks will be read. + { + Poco::FastMutex::ScopedLock lock(live_view.mutex); + if (!live_view.hasActiveUsers()) + { + live_view.reset(); + return; + } + } + + bool is_block_processed = false; + BlockInputStreams from; + BlocksPtrs mergeable_blocks; + BlocksPtr new_mergeable_blocks = std::make_shared(); + + { + Poco::FastMutex::ScopedLock lock(live_view.mutex); + + mergeable_blocks = live_view.getMergeableBlocks(); + if (!mergeable_blocks || mergeable_blocks->size() >= context.getGlobalContext().getSettingsRef().max_live_view_insert_blocks_before_refresh) + { + mergeable_blocks = std::make_shared>(); + BlocksPtr base_mergeable_blocks = std::make_shared(); + InterpreterSelectQuery interpreter(live_view.getInnerQuery(), context, SelectQueryOptions(QueryProcessingStage::WithMergeableState), Names()); + auto view_mergeable_stream = std::make_shared( + interpreter.execute().in); + while (Block this_block = view_mergeable_stream->read()) + base_mergeable_blocks->push_back(this_block); + mergeable_blocks->push_back(base_mergeable_blocks); + live_view.setMergeableBlocks(mergeable_blocks); + + /// Create from streams + for (auto & blocks_ : *mergeable_blocks) + { + if (blocks_->empty()) + continue; + auto sample_block = blocks_->front().cloneEmpty(); + BlockInputStreamPtr stream = std::make_shared(std::make_shared(blocks_), sample_block); + from.push_back(std::move(stream)); + } + + is_block_processed = true; + } + } + + if (!is_block_processed) + { + auto parent_storage = context.getTable(live_view.getSelectDatabaseName(), live_view.getSelectTableName()); + BlockInputStreams streams = {std::make_shared(block)}; + auto proxy_storage = std::make_shared(parent_storage, std::move(streams), QueryProcessingStage::FetchColumns); + InterpreterSelectQuery select_block(live_view.getInnerQuery(), + context, proxy_storage, + QueryProcessingStage::WithMergeableState); + auto data_mergeable_stream = std::make_shared( + select_block.execute().in); + while (Block this_block = data_mergeable_stream->read()) + new_mergeable_blocks->push_back(this_block); + + if (new_mergeable_blocks->empty()) + return; + + { + Poco::FastMutex::ScopedLock lock(live_view.mutex); + + mergeable_blocks = live_view.getMergeableBlocks(); + mergeable_blocks->push_back(new_mergeable_blocks); + + /// Create from streams + for (auto & blocks_ : *mergeable_blocks) + { + if (blocks_->empty()) + continue; + auto sample_block = blocks_->front().cloneEmpty(); + BlockInputStreamPtr stream = std::make_shared(std::make_shared(blocks_), sample_block); + from.push_back(std::move(stream)); + } + } + } + + auto parent_storage = context.getTable(live_view.getSelectDatabaseName(), live_view.getSelectTableName()); + auto proxy_storage = std::make_shared(parent_storage, std::move(from), QueryProcessingStage::WithMergeableState); + InterpreterSelectQuery select(live_view.getInnerQuery(), context, proxy_storage, QueryProcessingStage::Complete); + BlockInputStreamPtr data = std::make_shared(select.execute().in); + + /// Squashing is needed here because the view query can generate a lot of blocks + /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY + /// and two-level aggregation is triggered). + data = std::make_shared( + data, context.getGlobalContext().getSettingsRef().min_insert_block_size_rows, context.getGlobalContext().getSettingsRef().min_insert_block_size_bytes); + + copyData(*data, *output); +} + + StorageLiveView::StorageLiveView( const String & table_name_, const String & database_name_, @@ -259,11 +435,10 @@ void StorageLiveView::noUsersThread(const UInt64 & timeout) { while (1) { - Poco::FastMutex::ScopedLock lock(noUsersThreadMutex); - if (!noUsersThreadWakeUp && !noUsersThreadCondition.tryWait(noUsersThreadMutex, - timeout * 1000)) + Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + if (!no_users_thread_wakeup && !no_users_thread_condition.tryWait(no_users_thread_mutex, timeout * 1000)) { - noUsersThreadWakeUp = false; + no_users_thread_wakeup = false; if (shutdown_called) return; if (hasUsers()) @@ -301,7 +476,7 @@ void StorageLiveView::noUsersThread(const UInt64 & timeout) void StorageLiveView::startNoUsersThread(const UInt64 & timeout) { bool expected = false; - if (!startnousersthread_called.compare_exchange_strong(expected, true)) + if (!start_no_users_thread_called.compare_exchange_strong(expected, true)) return; if (is_dropped) @@ -312,20 +487,20 @@ void StorageLiveView::startNoUsersThread(const UInt64 & timeout) if (no_users_thread.joinable()) { { - Poco::FastMutex::ScopedLock lock(noUsersThreadMutex); - noUsersThreadWakeUp = true; - noUsersThreadCondition.signal(); + Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + no_users_thread_wakeup = true; + no_users_thread_condition.signal(); } no_users_thread.join(); } { - Poco::FastMutex::ScopedLock lock(noUsersThreadMutex); - noUsersThreadWakeUp = false; + Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + no_users_thread_wakeup = false; } if (!is_dropped) no_users_thread = std::thread(&StorageLiveView::noUsersThread, this, timeout); } - startnousersthread_called = false; + start_no_users_thread_called = false; } void StorageLiveView::startup() @@ -341,13 +516,13 @@ void StorageLiveView::shutdown() if (no_users_thread.joinable()) { - Poco::FastMutex::ScopedLock lock(noUsersThreadMutex); - noUsersThreadWakeUp = true; - noUsersThreadCondition.signal(); + Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + no_users_thread_wakeup = true; + no_users_thread_condition.signal(); /// Must detach the no users thread /// as we can't join it as it will result /// in a deadlock - no_users_thread.detach(); + no_users_thread.detach(); /// TODO Not viable at all. } } @@ -423,9 +598,9 @@ BlockInputStreams StorageLiveView::watch( if (no_users_thread.joinable()) { - Poco::FastMutex::ScopedLock lock(noUsersThreadMutex); - noUsersThreadWakeUp = true; - noUsersThreadCondition.signal(); + Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + no_users_thread_wakeup = true; + no_users_thread_condition.signal(); } { @@ -448,9 +623,9 @@ BlockInputStreams StorageLiveView::watch( if (no_users_thread.joinable()) { - Poco::FastMutex::ScopedLock lock(noUsersThreadMutex); - noUsersThreadWakeUp = true; - noUsersThreadCondition.signal(); + Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + no_users_thread_wakeup = true; + no_users_thread_condition.signal(); } { diff --git a/dbms/src/Storages/StorageLiveView.h b/dbms/src/Storages/StorageLiveView.h index 6bff9c2dc85..3b2398a421f 100644 --- a/dbms/src/Storages/StorageLiveView.h +++ b/dbms/src/Storages/StorageLiveView.h @@ -12,33 +12,25 @@ limitations under the License. */ #pragma once #include -#include -#include -#include -#include -#include -#include -#include #include #include #include #include #include + namespace DB { -class IAST; - struct BlocksMetadata { String hash; UInt64 version; }; +class IAST; using ASTPtr = std::shared_ptr; using BlocksMetadataPtr = std::shared_ptr; -using SipHashPtr = std::shared_ptr; class StorageLiveView : public ext::shared_ptr_helper, public IStorage { @@ -87,9 +79,9 @@ public: /// Background thread for temporary tables /// which drops this table if there are no users void startNoUsersThread(const UInt64 & timeout); - Poco::FastMutex noUsersThreadMutex; - bool noUsersThreadWakeUp{false}; - Poco::Condition noUsersThreadCondition; + Poco::FastMutex no_users_thread_mutex; + bool no_users_thread_wakeup{false}; + Poco::Condition no_users_thread_condition; /// Get blocks hash /// must be called with mutex locked String getBlocksHashKey() @@ -150,105 +142,10 @@ public: Block getHeader() const; - static void writeIntoLiveView(StorageLiveView & live_view, - const Block & block, - const Context & context, - BlockOutputStreamPtr & output) - { - /// Check if live view has any readers if not - /// just reset blocks to empty and do nothing else - /// When first reader comes the blocks will be read. - { - Poco::FastMutex::ScopedLock lock(live_view.mutex); - if (!live_view.hasActiveUsers()) - { - live_view.reset(); - return; - } - } - - bool is_block_processed = false; - BlockInputStreams from; - BlocksPtrs mergeable_blocks; - BlocksPtr new_mergeable_blocks = std::make_shared(); - - { - Poco::FastMutex::ScopedLock lock(live_view.mutex); - - mergeable_blocks = live_view.getMergeableBlocks(); - if (!mergeable_blocks || mergeable_blocks->size() >= context.getGlobalContext().getSettingsRef().max_live_view_insert_blocks_before_refresh) - { - mergeable_blocks = std::make_shared>(); - BlocksPtr base_mergeable_blocks = std::make_shared(); - InterpreterSelectQuery interpreter(live_view.getInnerQuery(), context, SelectQueryOptions(QueryProcessingStage::WithMergeableState), Names()); - auto view_mergeable_stream = std::make_shared( - interpreter.execute().in); - while (Block this_block = view_mergeable_stream->read()) - base_mergeable_blocks->push_back(this_block); - mergeable_blocks->push_back(base_mergeable_blocks); - live_view.setMergeableBlocks(mergeable_blocks); - - /// Create from streams - for (auto & blocks_ : *mergeable_blocks) - { - if (blocks_->empty()) - continue; - auto sample_block = blocks_->front().cloneEmpty(); - BlockInputStreamPtr stream = std::make_shared(std::make_shared(blocks_), sample_block); - from.push_back(std::move(stream)); - } - - is_block_processed = true; - } - } - - if (!is_block_processed) - { - auto parent_storage = context.getTable(live_view.getSelectDatabaseName(), live_view.getSelectTableName()); - BlockInputStreams streams = {std::make_shared(block)}; - auto proxy_storage = std::make_shared(parent_storage, std::move(streams), QueryProcessingStage::FetchColumns); - InterpreterSelectQuery select_block(live_view.getInnerQuery(), - context, proxy_storage, - QueryProcessingStage::WithMergeableState); - auto data_mergeable_stream = std::make_shared( - select_block.execute().in); - while (Block this_block = data_mergeable_stream->read()) - new_mergeable_blocks->push_back(this_block); - - if (new_mergeable_blocks->empty()) - return; - - { - Poco::FastMutex::ScopedLock lock(live_view.mutex); - - mergeable_blocks = live_view.getMergeableBlocks(); - mergeable_blocks->push_back(new_mergeable_blocks); - - /// Create from streams - for (auto & blocks_ : *mergeable_blocks) - { - if (blocks_->empty()) - continue; - auto sample_block = blocks_->front().cloneEmpty(); - BlockInputStreamPtr stream = std::make_shared(std::make_shared(blocks_), sample_block); - from.push_back(std::move(stream)); - } - } - } - - auto parent_storage = context.getTable(live_view.getSelectDatabaseName(), live_view.getSelectTableName()); - auto proxy_storage = std::make_shared(parent_storage, std::move(from), QueryProcessingStage::WithMergeableState); - InterpreterSelectQuery select(live_view.getInnerQuery(), context, proxy_storage, QueryProcessingStage::Complete); - BlockInputStreamPtr data = std::make_shared(select.execute().in); - - /// Squashing is needed here because the view query can generate a lot of blocks - /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY - /// and two-level aggregation is triggered). - data = std::make_shared( - data, context.getGlobalContext().getSettingsRef().min_insert_block_size_rows, context.getGlobalContext().getSettingsRef().min_insert_block_size_bytes); - - copyData(*data, *output); - } + static void writeIntoLiveView( + StorageLiveView & live_view, + const Block & block, + const Context & context); private: String select_database_name; @@ -271,7 +168,7 @@ private: void noUsersThread(const UInt64 & timeout); std::thread no_users_thread; std::atomic shutdown_called{false}; - std::atomic startnousersthread_called{false}; + std::atomic start_no_users_thread_called{false}; UInt64 temporary_live_view_timeout; StorageLiveView( @@ -283,65 +180,4 @@ private: ); }; -class LiveViewBlockOutputStream : public IBlockOutputStream -{ -public: - explicit LiveViewBlockOutputStream(StorageLiveView & storage_) : storage(storage_) {} - - void writePrefix() override - { - new_blocks = std::make_shared(); - new_blocks_metadata = std::make_shared(); - new_hash = std::make_shared(); - } - - void writeSuffix() override - { - UInt128 key; - String key_str; - - new_hash->get128(key.low, key.high); - key_str = key.toHexString(); - - Poco::FastMutex::ScopedLock lock(storage.mutex); - - if (storage.getBlocksHashKey() != key_str) - { - new_blocks_metadata->hash = key_str; - new_blocks_metadata->version = storage.getBlocksVersion() + 1; - - for (auto & block : *new_blocks) - { - block.insert({DataTypeUInt64().createColumnConst( - block.rows(), new_blocks_metadata->version)->convertToFullColumnIfConst(), - std::make_shared(), - "_version"}); - } - - (*storage.blocks_ptr) = new_blocks; - (*storage.blocks_metadata_ptr) = new_blocks_metadata; - - storage.condition.broadcast(); - } - - new_blocks.reset(); - new_blocks_metadata.reset(); - new_hash.reset(); - } - - void write(const Block & block) override - { - new_blocks->push_back(block); - block.updateHash(*new_hash); - } - - Block getHeader() const override { return storage.getHeader(); } - -private: - BlocksPtr new_blocks; - BlocksMetadataPtr new_blocks_metadata; - SipHashPtr new_hash; - StorageLiveView & storage; -}; - } From c4712f1e6e2376232747e839850127238a008061 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 02:22:57 +0300 Subject: [PATCH 182/357] Make the code less bad --- dbms/CMakeLists.txt | 1 + .../PushingToViewsBlockOutputStream.cpp | 2 +- .../PushingToViewsBlockOutputStream.h | 1 - .../Interpreters/InterpreterAlterQuery.cpp | 4 +- .../LiveView}/LiveViewBlockInputStream.h | 38 ++---- .../LiveView/LiveViewBlockOutputStream.h | 74 +++++++++++ .../{ => LiveView}/LiveViewCommands.h | 4 +- .../LiveViewEventsBlockInputStream.h | 27 ++-- .../Storages/{ => LiveView}/ProxyStorage.h | 0 .../{ => LiveView}/StorageLiveView.cpp | 125 +++++------------- .../Storages/{ => LiveView}/StorageLiveView.h | 27 ++-- dbms/src/Storages/StorageFactory.cpp | 14 -- dbms/src/Storages/registerStorages.cpp | 2 - 13 files changed, 152 insertions(+), 167 deletions(-) rename dbms/src/{DataStreams => Storages/LiveView}/LiveViewBlockInputStream.h (81%) create mode 100644 dbms/src/Storages/LiveView/LiveViewBlockOutputStream.h rename dbms/src/Storages/{ => LiveView}/LiveViewCommands.h (97%) rename dbms/src/{DataStreams => Storages/LiveView}/LiveViewEventsBlockInputStream.h (91%) rename dbms/src/Storages/{ => LiveView}/ProxyStorage.h (100%) rename dbms/src/Storages/{ => LiveView}/StorageLiveView.cpp (85%) rename dbms/src/Storages/{ => LiveView}/StorageLiveView.h (94%) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index b589c398238..f011cc21103 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -114,6 +114,7 @@ add_headers_and_sources(dbms src/Columns) add_headers_and_sources(dbms src/Storages) add_headers_and_sources(dbms src/Storages/Distributed) add_headers_and_sources(dbms src/Storages/MergeTree) +add_headers_and_sources(dbms src/Storages/LiveView) add_headers_and_sources(dbms src/Client) add_headers_and_sources(dbms src/Formats) add_headers_and_sources(dbms src/Processors) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index d16d68bf72b..807a9129a75 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include namespace DB { diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h index e3f96241b1d..c9b0538e470 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -5,7 +5,6 @@ #include #include #include -#include namespace DB { diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index a3e6824c3a5..61d5e011d37 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -8,9 +8,9 @@ #include #include #include -#include +#include +#include #include -#include #include diff --git a/dbms/src/DataStreams/LiveViewBlockInputStream.h b/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h similarity index 81% rename from dbms/src/DataStreams/LiveViewBlockInputStream.h rename to dbms/src/Storages/LiveView/LiveViewBlockInputStream.h index b3756c9ff6d..345fceaf095 100644 --- a/dbms/src/DataStreams/LiveViewBlockInputStream.h +++ b/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h @@ -1,23 +1,6 @@ -/* Copyright (c) 2018 BlackBerry Limited - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ #pragma once -#include - -#include -#include -#include #include -#include namespace DB @@ -61,8 +44,8 @@ public: if (isCancelled() || storage->is_dropped) return; IBlockInputStream::cancel(kill); - Poco::FastMutex::ScopedLock lock(storage->mutex); - storage->condition.broadcast(); + std::lock_guard lock(storage->mutex); + storage->condition.notify_all(); } Block getHeader() const override { return storage->getHeader(); } @@ -92,14 +75,14 @@ public: NonBlockingResult tryRead() { - return tryRead_(false); + return tryReadImpl(false); } protected: Block readImpl() override { /// try reading - return tryRead_(true).first; + return tryReadImpl(true).first; } /** tryRead method attempts to read a block in either blocking @@ -107,7 +90,7 @@ protected: * then method return empty block with flag set to false * to indicate that method would block to get the next block. */ - NonBlockingResult tryRead_(bool blocking) + NonBlockingResult tryReadImpl(bool blocking) { Block res; @@ -118,7 +101,7 @@ protected: /// If blocks were never assigned get blocks if (!blocks) { - Poco::FastMutex::ScopedLock lock(storage->mutex); + std::lock_guard lock(storage->mutex); if (!active) return { Block(), false }; blocks = (*blocks_ptr); @@ -135,7 +118,7 @@ protected: if (it == end) { { - Poco::FastMutex::ScopedLock lock(storage->mutex); + std::unique_lock lock(storage->mutex); if (!active) return { Block(), false }; /// If we are done iterating over our blocks @@ -162,7 +145,10 @@ protected: while (true) { UInt64 timestamp_usec = static_cast(timestamp.epochMicroseconds()); - bool signaled = storage->condition.tryWait(storage->mutex, std::max(static_cast(0), heartbeat_interval_usec - (timestamp_usec - last_event_timestamp_usec)) / 1000); + + /// Or spurious wakeup. + bool signaled = std::cv_status::no_timeout == storage->condition.wait_for(lock, + std::chrono::microseconds(std::max(UInt64(0), heartbeat_interval_usec - (timestamp_usec - last_event_timestamp_usec)))); if (isCancelled() || storage->is_dropped) { @@ -181,7 +167,7 @@ protected: } } } - return tryRead_(blocking); + return tryReadImpl(blocking); } res = *it; diff --git a/dbms/src/Storages/LiveView/LiveViewBlockOutputStream.h b/dbms/src/Storages/LiveView/LiveViewBlockOutputStream.h new file mode 100644 index 00000000000..548bcf1b86a --- /dev/null +++ b/dbms/src/Storages/LiveView/LiveViewBlockOutputStream.h @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class LiveViewBlockOutputStream : public IBlockOutputStream +{ +public: + explicit LiveViewBlockOutputStream(StorageLiveView & storage_) : storage(storage_) {} + + void writePrefix() override + { + new_blocks = std::make_shared(); + new_blocks_metadata = std::make_shared(); + new_hash = std::make_shared(); + } + + void writeSuffix() override + { + UInt128 key; + String key_str; + + new_hash->get128(key.low, key.high); + key_str = key.toHexString(); + + std::lock_guard lock(storage.mutex); + + if (storage.getBlocksHashKey() != key_str) + { + new_blocks_metadata->hash = key_str; + new_blocks_metadata->version = storage.getBlocksVersion() + 1; + + for (auto & block : *new_blocks) + { + block.insert({DataTypeUInt64().createColumnConst( + block.rows(), new_blocks_metadata->version)->convertToFullColumnIfConst(), + std::make_shared(), + "_version"}); + } + + (*storage.blocks_ptr) = new_blocks; + (*storage.blocks_metadata_ptr) = new_blocks_metadata; + + storage.condition.notify_all(); + } + + new_blocks.reset(); + new_blocks_metadata.reset(); + new_hash.reset(); + } + + void write(const Block & block) override + { + new_blocks->push_back(block); + block.updateHash(*new_hash); + } + + Block getHeader() const override { return storage.getHeader(); } + +private: + using SipHashPtr = std::shared_ptr; + + BlocksPtr new_blocks; + BlocksMetadataPtr new_blocks_metadata; + SipHashPtr new_hash; + StorageLiveView & storage; +}; + +} diff --git a/dbms/src/Storages/LiveViewCommands.h b/dbms/src/Storages/LiveView/LiveViewCommands.h similarity index 97% rename from dbms/src/Storages/LiveViewCommands.h rename to dbms/src/Storages/LiveView/LiveViewCommands.h index 35015a7e5aa..54048c28a5f 100644 --- a/dbms/src/Storages/LiveViewCommands.h +++ b/dbms/src/Storages/LiveView/LiveViewCommands.h @@ -12,9 +12,9 @@ limitations under the License. */ #pragma once -#include #include -#include +#include +#include namespace DB { diff --git a/dbms/src/DataStreams/LiveViewEventsBlockInputStream.h b/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h similarity index 91% rename from dbms/src/DataStreams/LiveViewEventsBlockInputStream.h rename to dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h index 93fb6a76372..120d0098536 100644 --- a/dbms/src/DataStreams/LiveViewEventsBlockInputStream.h +++ b/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h @@ -9,11 +9,9 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + #pragma once -#include - -#include #include #include #include @@ -21,7 +19,7 @@ limitations under the License. */ #include #include #include -#include +#include namespace DB @@ -66,8 +64,8 @@ public: if (isCancelled() || storage->is_dropped) return; IBlockInputStream::cancel(kill); - Poco::FastMutex::ScopedLock lock(storage->mutex); - storage->condition.broadcast(); + std::lock_guard lock(storage->mutex); + storage->condition.notify_all(); } Block getHeader() const override @@ -103,7 +101,7 @@ public: NonBlockingResult tryRead() { - return tryRead_(false); + return tryReadImpl(false); } Block getEventBlock() @@ -120,7 +118,7 @@ protected: Block readImpl() override { /// try reading - return tryRead_(true).first; + return tryReadImpl(true).first; } /** tryRead method attempts to read a block in either blocking @@ -128,7 +126,7 @@ protected: * then method return empty block with flag set to false * to indicate that method would block to get the next block. */ - NonBlockingResult tryRead_(bool blocking) + NonBlockingResult tryReadImpl(bool blocking) { if (has_limit && num_updates == static_cast(limit)) { @@ -137,7 +135,7 @@ protected: /// If blocks were never assigned get blocks if (!blocks) { - Poco::FastMutex::ScopedLock lock(storage->mutex); + std::lock_guard lock(storage->mutex); if (!active) return { Block(), false }; blocks = (*blocks_ptr); @@ -155,7 +153,7 @@ protected: if (it == end) { { - Poco::FastMutex::ScopedLock lock(storage->mutex); + std::unique_lock lock(storage->mutex); if (!active) return { Block(), false }; /// If we are done iterating over our blocks @@ -183,7 +181,10 @@ protected: while (true) { UInt64 timestamp_usec = static_cast(timestamp.epochMicroseconds()); - bool signaled = storage->condition.tryWait(storage->mutex, std::max(static_cast(0), heartbeat_interval_usec - (timestamp_usec - last_event_timestamp_usec)) / 1000); + + /// Or spurious wakeup. + bool signaled = std::cv_status::no_timeout == storage->condition.wait_for(lock, + std::chrono::microseconds(std::max(UInt64(0), heartbeat_interval_usec - (timestamp_usec - last_event_timestamp_usec)))); if (isCancelled() || storage->is_dropped) { @@ -202,7 +203,7 @@ protected: } } } - return tryRead_(blocking); + return tryReadImpl(blocking); } // move right to the end diff --git a/dbms/src/Storages/ProxyStorage.h b/dbms/src/Storages/LiveView/ProxyStorage.h similarity index 100% rename from dbms/src/Storages/ProxyStorage.h rename to dbms/src/Storages/LiveView/ProxyStorage.h diff --git a/dbms/src/Storages/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp similarity index 85% rename from dbms/src/Storages/StorageLiveView.cpp rename to dbms/src/Storages/LiveView/StorageLiveView.cpp index fdfa6a3a80d..98d48392e09 100644 --- a/dbms/src/Storages/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -20,9 +20,6 @@ limitations under the License. */ #include #include #include -#include -#include -#include #include #include #include @@ -30,10 +27,15 @@ limitations under the License. */ #include #include #include +#include + +#include +#include +#include +#include +#include -#include #include -#include #include #include #include @@ -107,70 +109,6 @@ static void checkAllowedQueries(const ASTSelectQuery & query) } -class LiveViewBlockOutputStream : public IBlockOutputStream -{ -public: - explicit LiveViewBlockOutputStream(StorageLiveView & storage_) : storage(storage_) {} - - void writePrefix() override - { - new_blocks = std::make_shared(); - new_blocks_metadata = std::make_shared(); - new_hash = std::make_shared(); - } - - void writeSuffix() override - { - UInt128 key; - String key_str; - - new_hash->get128(key.low, key.high); - key_str = key.toHexString(); - - Poco::FastMutex::ScopedLock lock(storage.mutex); - - if (storage.getBlocksHashKey() != key_str) - { - new_blocks_metadata->hash = key_str; - new_blocks_metadata->version = storage.getBlocksVersion() + 1; - - for (auto & block : *new_blocks) - { - block.insert({DataTypeUInt64().createColumnConst( - block.rows(), new_blocks_metadata->version)->convertToFullColumnIfConst(), - std::make_shared(), - "_version"}); - } - - (*storage.blocks_ptr) = new_blocks; - (*storage.blocks_metadata_ptr) = new_blocks_metadata; - - storage.condition.broadcast(); - } - - new_blocks.reset(); - new_blocks_metadata.reset(); - new_hash.reset(); - } - - void write(const Block & block) override - { - new_blocks->push_back(block); - block.updateHash(*new_hash); - } - - Block getHeader() const override { return storage.getHeader(); } - -private: - using SipHashPtr = std::shared_ptr; - - BlocksPtr new_blocks; - BlocksMetadataPtr new_blocks_metadata; - SipHashPtr new_hash; - StorageLiveView & storage; -}; - - void StorageLiveView::writeIntoLiveView( StorageLiveView & live_view, const Block & block, @@ -182,7 +120,7 @@ void StorageLiveView::writeIntoLiveView( /// just reset blocks to empty and do nothing else /// When first reader comes the blocks will be read. { - Poco::FastMutex::ScopedLock lock(live_view.mutex); + std::lock_guard lock(live_view.mutex); if (!live_view.hasActiveUsers()) { live_view.reset(); @@ -196,7 +134,7 @@ void StorageLiveView::writeIntoLiveView( BlocksPtr new_mergeable_blocks = std::make_shared(); { - Poco::FastMutex::ScopedLock lock(live_view.mutex); + std::lock_guard lock(live_view.mutex); mergeable_blocks = live_view.getMergeableBlocks(); if (!mergeable_blocks || mergeable_blocks->size() >= context.getGlobalContext().getSettingsRef().max_live_view_insert_blocks_before_refresh) @@ -242,7 +180,7 @@ void StorageLiveView::writeIntoLiveView( return; { - Poco::FastMutex::ScopedLock lock(live_view.mutex); + std::lock_guard lock(live_view.mutex); mergeable_blocks = live_view.getMergeableBlocks(); mergeable_blocks->push_back(new_mergeable_blocks); @@ -435,8 +373,8 @@ void StorageLiveView::noUsersThread(const UInt64 & timeout) { while (1) { - Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); - if (!no_users_thread_wakeup && !no_users_thread_condition.tryWait(no_users_thread_mutex, timeout * 1000)) + std::unique_lock lock(no_users_thread_mutex); + if (!no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return no_users_thread_wakeup; })) { no_users_thread_wakeup = false; if (shutdown_called) @@ -487,14 +425,14 @@ void StorageLiveView::startNoUsersThread(const UInt64 & timeout) if (no_users_thread.joinable()) { { - Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + std::lock_guard lock(no_users_thread_mutex); no_users_thread_wakeup = true; - no_users_thread_condition.signal(); + no_users_thread_condition.notify_one(); } no_users_thread.join(); } { - Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + std::lock_guard lock(no_users_thread_mutex); no_users_thread_wakeup = false; } if (!is_dropped) @@ -516,9 +454,9 @@ void StorageLiveView::shutdown() if (no_users_thread.joinable()) { - Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + std::lock_guard lock(no_users_thread_mutex); no_users_thread_wakeup = true; - no_users_thread_condition.signal(); + no_users_thread_condition.notify_one(); /// Must detach the no users thread /// as we can't join it as it will result /// in a deadlock @@ -536,18 +474,19 @@ void StorageLiveView::drop() global_context.removeDependency( DatabaseAndTableName(select_database_name, select_table_name), DatabaseAndTableName(database_name, table_name)); - Poco::FastMutex::ScopedLock lock(mutex); + + std::lock_guard lock(mutex); is_dropped = true; - condition.broadcast(); + condition.notify_all(); } void StorageLiveView::refresh(const Context & context) { auto alter_lock = lockAlterIntention(context.getCurrentQueryId()); { - Poco::FastMutex::ScopedLock lock(mutex); + std::lock_guard lock(mutex); if (getNewBlocks()) - condition.broadcast(); + condition.notify_all(); } } @@ -562,11 +501,11 @@ BlockInputStreams StorageLiveView::read( /// add user to the blocks_ptr std::shared_ptr stream_blocks_ptr = blocks_ptr; { - Poco::FastMutex::ScopedLock lock(mutex); + std::lock_guard lock(mutex); if (!(*blocks_ptr)) { if (getNewBlocks()) - condition.broadcast(); + condition.notify_all(); } } return { std::make_shared(stream_blocks_ptr, getHeader()) }; @@ -598,17 +537,17 @@ BlockInputStreams StorageLiveView::watch( if (no_users_thread.joinable()) { - Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + std::lock_guard lock(no_users_thread_mutex); no_users_thread_wakeup = true; - no_users_thread_condition.signal(); + no_users_thread_condition.notify_one(); } { - Poco::FastMutex::ScopedLock lock(mutex); + std::lock_guard lock(mutex); if (!(*blocks_ptr)) { if (getNewBlocks()) - condition.broadcast(); + condition.notify_all(); } } @@ -623,17 +562,17 @@ BlockInputStreams StorageLiveView::watch( if (no_users_thread.joinable()) { - Poco::FastMutex::ScopedLock lock(no_users_thread_mutex); + std::lock_guard lock(no_users_thread_mutex); no_users_thread_wakeup = true; - no_users_thread_condition.signal(); + no_users_thread_condition.notify_one(); } { - Poco::FastMutex::ScopedLock lock(mutex); + std::lock_guard lock(mutex); if (!(*blocks_ptr)) { if (getNewBlocks()) - condition.broadcast(); + condition.notify_all(); } } diff --git a/dbms/src/Storages/StorageLiveView.h b/dbms/src/Storages/LiveView/StorageLiveView.h similarity index 94% rename from dbms/src/Storages/StorageLiveView.h rename to dbms/src/Storages/LiveView/StorageLiveView.h index 3b2398a421f..9930d8d6154 100644 --- a/dbms/src/Storages/StorageLiveView.h +++ b/dbms/src/Storages/LiveView/StorageLiveView.h @@ -11,12 +11,11 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include -#include #include -#include #include -#include + +#include +#include namespace DB @@ -35,6 +34,8 @@ using BlocksMetadataPtr = std::shared_ptr; class StorageLiveView : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; +friend class LiveViewBlockInputStream; +friend class LiveViewEventsBlockInputStream; friend class LiveViewBlockOutputStream; public: @@ -55,12 +56,6 @@ public: bool supportsSampling() const override { return true; } bool supportsFinal() const override { return true; } - /// Mutex for the blocks and ready condition - Poco::FastMutex mutex; - /// New blocks ready condition to broadcast to readers - /// that new blocks are available - Poco::Condition condition; - bool isTemporary() { return is_temporary; } /// Check if we have any readers @@ -79,16 +74,16 @@ public: /// Background thread for temporary tables /// which drops this table if there are no users void startNoUsersThread(const UInt64 & timeout); - Poco::FastMutex no_users_thread_mutex; + std::mutex no_users_thread_mutex; bool no_users_thread_wakeup{false}; - Poco::Condition no_users_thread_condition; + std::condition_variable no_users_thread_condition; /// Get blocks hash /// must be called with mutex locked String getBlocksHashKey() { if (*blocks_metadata_ptr) return (*blocks_metadata_ptr)->hash; - return ""; + return {}; } /// Get blocks version /// must be called with mutex locked @@ -157,6 +152,12 @@ private: bool is_temporary {false}; mutable Block sample_block; + /// Mutex for the blocks and ready condition + std::mutex mutex; + /// New blocks ready condition to broadcast to readers + /// that new blocks are available + std::condition_variable condition; + /// Active users std::shared_ptr active_ptr; /// Current data blocks that store query result diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 7d92ce0ea2c..862f76bc3ce 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -68,14 +68,6 @@ StoragePtr StorageFactory::get( name = "LiveView"; } - else if (query.is_live_channel) - { - - if (query.storage) - throw Exception("Specifying ENGINE is not allowed for a LiveChannel", ErrorCodes::INCORRECT_QUERY); - - name = "LiveChannel"; - } else { /// Check for some special types, that are not allowed to be stored in tables. Example: NULL data type. @@ -137,12 +129,6 @@ StoragePtr StorageFactory::get( "Direct creation of tables with ENGINE LiveView is not supported, use CREATE LIVE VIEW statement", ErrorCodes::INCORRECT_QUERY); } - else if (name == "LiveChannel") - { - throw Exception( - "Direct creation of tables with ENGINE LiveChannel is not supported, use CREATE LIVE CHANNEL statement", - ErrorCodes::INCORRECT_QUERY); - } } } diff --git a/dbms/src/Storages/registerStorages.cpp b/dbms/src/Storages/registerStorages.cpp index 811a031c7a3..c21156ea44d 100644 --- a/dbms/src/Storages/registerStorages.cpp +++ b/dbms/src/Storages/registerStorages.cpp @@ -25,7 +25,6 @@ void registerStorageJoin(StorageFactory & factory); void registerStorageView(StorageFactory & factory); void registerStorageMaterializedView(StorageFactory & factory); void registerStorageLiveView(StorageFactory & factory); -//void registerStorageLiveChannel(StorageFactory & factory); #if USE_HDFS void registerStorageHDFS(StorageFactory & factory); @@ -67,7 +66,6 @@ void registerStorages() registerStorageView(factory); registerStorageMaterializedView(factory); registerStorageLiveView(factory); - //registerStorageLiveChannel(factory); #if USE_HDFS registerStorageHDFS(factory); From 78c3be175ca2ca52b04e18c5e2c690242a6eeb44 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 02:46:40 +0300 Subject: [PATCH 183/357] Added experimental setting for LIVE VIEWs --- dbms/src/Core/Settings.h | 2 ++ dbms/src/Interpreters/InterpreterWatchQuery.cpp | 4 ++++ dbms/src/Storages/LiveView/StorageLiveView.cpp | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 22b0b5c8d03..6a22869c8dc 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -342,6 +342,8 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.") \ M(SettingBool, check_query_single_value_result, true, "Return check query result as single 1/0 value") \ \ + M(SettingBool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.") \ + \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ diff --git a/dbms/src/Interpreters/InterpreterWatchQuery.cpp b/dbms/src/Interpreters/InterpreterWatchQuery.cpp index 3ba8e2eadaa..da09022e252 100644 --- a/dbms/src/Interpreters/InterpreterWatchQuery.cpp +++ b/dbms/src/Interpreters/InterpreterWatchQuery.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes extern const int UNKNOWN_STORAGE; extern const int UNKNOWN_TABLE; extern const int TOO_MANY_COLUMNS; + extern const int SUPPORT_IS_DISABLED; } BlockInputStreamPtr InterpreterWatchQuery::executeImpl() @@ -34,6 +35,9 @@ BlockInputStreamPtr InterpreterWatchQuery::executeImpl() BlockIO InterpreterWatchQuery::execute() { + if (!context.getSettingsRef().allow_experimental_live_view) + throw Exception("Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')", ErrorCodes::SUPPORT_IS_DISABLED); + BlockIO res; const ASTWatchQuery & query = typeid_cast(*query_ptr); String database; diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 98d48392e09..3c0d205fa3f 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -19,7 +19,6 @@ limitations under the License. */ #include #include #include -#include #include #include #include @@ -50,6 +49,7 @@ namespace ErrorCodes extern const int INCORRECT_QUERY; extern const int TABLE_WAS_NOT_DROPPED; extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW; + extern const int SUPPORT_IS_DISABLED; } static void extractDependentTable(ASTSelectQuery & query, String & select_database_name, String & select_table_name) @@ -586,6 +586,9 @@ void registerStorageLiveView(StorageFactory & factory) { factory.registerStorage("LiveView", [](const StorageFactory::Arguments & args) { + if (!args.local_context.getSettingsRef().allow_experimental_live_view) + throw Exception("Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')", ErrorCodes::SUPPORT_IS_DISABLED); + return StorageLiveView::create(args.table_name, args.database_name, args.local_context, args.query, args.columns); }); } From 96869d405fc132a7443cfbc13d1e583079503db8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 02:48:52 +0300 Subject: [PATCH 184/357] Temporarily disable all LIVE VIEW tests because this feature has subtle bugs that manifestate under TSan --- .../00960_live_view_watch_events_live.py | 42 ---- ...0960_live_view_watch_events_live.reference | 0 .../00961_temporary_live_view_watch.reference | 3 - .../00961_temporary_live_view_watch.sql | 18 -- .../00962_temporary_live_view_watch_live.py | 42 ---- ...2_temporary_live_view_watch_live.reference | 0 ...y_live_view_watch_live_timeout.py.disabled | 49 ----- ...ary_live_view_watch_live_timeout.reference | 0 .../00964_live_view_watch_events_heartbeat.py | 44 ---- ...live_view_watch_events_heartbeat.reference | 0 .../00965_live_view_watch_heartbeat.py | 45 ---- .../00965_live_view_watch_heartbeat.reference | 0 .../00966_live_view_watch_events_http.py | 37 ---- ...0966_live_view_watch_events_http.reference | 0 .../0_stateless/00967_live_view_watch_http.py | 37 ---- .../00967_live_view_watch_http.reference | 0 ...t_format_jsoneachrowwithprogress.reference | 4 - ..._select_format_jsoneachrowwithprogress.sql | 12 - ...h_format_jsoneachrowwithprogress.reference | 6 - ...w_watch_format_jsoneachrowwithprogress.sql | 18 -- ...0_live_view_watch_events_http_heartbeat.py | 43 ---- ...view_watch_events_http_heartbeat.reference | 0 .../00971_live_view_watch_http_heartbeat.py | 43 ---- ...1_live_view_watch_http_heartbeat.reference | 0 .../00972_live_view_select_1.reference | 1 - .../0_stateless/00972_live_view_select_1.sql | 7 - .../00973_live_view_select.reference | 4 - .../0_stateless/00973_live_view_select.sql | 18 -- ...ive_view_select_with_aggregation.reference | 2 - ...0974_live_view_select_with_aggregation.sql | 16 -- .../00975_live_view_create.reference | 0 .../0_stateless/00975_live_view_create.sql | 7 - .../00976_live_view_select_version.reference | 3 - .../00976_live_view_select_version.sql | 12 - .../00977_live_view_watch_events.reference | 3 - .../00977_live_view_watch_events.sql | 18 -- .../00978_live_view_watch.reference | 3 - .../0_stateless/00978_live_view_watch.sql | 18 -- .../0_stateless/00979_live_view_watch_live.py | 48 ---- .../00979_live_view_watch_live.reference | 0 ...00980_create_temporary_live_view.reference | 3 - .../00980_create_temporary_live_view.sql | 15 -- .../00991_live_view_watch_event_live.python | 81 ------- ...00991_live_view_watch_event_live.reference | 7 - ...991_live_view_watch_event_live.sh.disabled | 6 - .../00991_live_view_watch_http.python | 63 ------ .../00991_live_view_watch_http.reference | 4 - .../00991_live_view_watch_http.sh.disabled | 6 - ...ry_live_view_watch_events_heartbeat.python | 83 ------- ...live_view_watch_events_heartbeat.reference | 0 ...ve_view_watch_events_heartbeat.sh.disabled | 6 - ...0991_temporary_live_view_watch_live.python | 81 ------- ...1_temporary_live_view_watch_live.reference | 7 - ...temporary_live_view_watch_live.sh.disabled | 6 - .../queries/0_stateless/helpers/client.py | 36 --- .../queries/0_stateless/helpers/httpclient.py | 14 -- .../queries/0_stateless/helpers/httpexpect.py | 73 ------- .../queries/0_stateless/helpers/uexpect.py | 206 ------------------ 58 files changed, 1300 deletions(-) delete mode 100755 dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py delete mode 100644 dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference delete mode 100644 dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference delete mode 100644 dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql delete mode 100755 dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py delete mode 100644 dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference delete mode 100755 dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled delete mode 100644 dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference delete mode 100755 dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py delete mode 100644 dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference delete mode 100755 dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py delete mode 100644 dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference delete mode 100755 dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py delete mode 100644 dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference delete mode 100755 dbms/tests/queries/0_stateless/00967_live_view_watch_http.py delete mode 100644 dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference delete mode 100644 dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference delete mode 100644 dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql delete mode 100644 dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference delete mode 100644 dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql delete mode 100755 dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py delete mode 100644 dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference delete mode 100755 dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py delete mode 100644 dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference delete mode 100644 dbms/tests/queries/0_stateless/00972_live_view_select_1.reference delete mode 100644 dbms/tests/queries/0_stateless/00972_live_view_select_1.sql delete mode 100644 dbms/tests/queries/0_stateless/00973_live_view_select.reference delete mode 100644 dbms/tests/queries/0_stateless/00973_live_view_select.sql delete mode 100644 dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference delete mode 100644 dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql delete mode 100644 dbms/tests/queries/0_stateless/00975_live_view_create.reference delete mode 100644 dbms/tests/queries/0_stateless/00975_live_view_create.sql delete mode 100644 dbms/tests/queries/0_stateless/00976_live_view_select_version.reference delete mode 100644 dbms/tests/queries/0_stateless/00976_live_view_select_version.sql delete mode 100644 dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference delete mode 100644 dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql delete mode 100644 dbms/tests/queries/0_stateless/00978_live_view_watch.reference delete mode 100644 dbms/tests/queries/0_stateless/00978_live_view_watch.sql delete mode 100755 dbms/tests/queries/0_stateless/00979_live_view_watch_live.py delete mode 100644 dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference delete mode 100644 dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference delete mode 100644 dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql delete mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python delete mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference delete mode 100755 dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.sh.disabled delete mode 100755 dbms/tests/queries/0_stateless/00991_live_view_watch_http.python delete mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference delete mode 100755 dbms/tests/queries/0_stateless/00991_live_view_watch_http.sh.disabled delete mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python delete mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference delete mode 100755 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.sh.disabled delete mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python delete mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference delete mode 100755 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.sh.disabled delete mode 100644 dbms/tests/queries/0_stateless/helpers/client.py delete mode 100644 dbms/tests/queries/0_stateless/helpers/httpclient.py delete mode 100644 dbms/tests/queries/0_stateless/helpers/httpexpect.py delete mode 100644 dbms/tests/queries/0_stateless/helpers/uexpect.py diff --git a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py deleted file mode 100755 index b7fc3f4e3a6..00000000000 --- a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv EVENTS') - client1.expect('1.*' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect('2.*' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect('3.*' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference deleted file mode 100644 index 6fbbedf1b21..00000000000 --- a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference +++ /dev/null @@ -1,3 +0,0 @@ -0 1 -6 2 -21 3 diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql deleted file mode 100644 index c3e2ab8d102..00000000000 --- a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py deleted file mode 100755 index f27b1213c70..00000000000 --- a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled deleted file mode 100755 index df627c84e49..00000000000 --- a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET temporary_live_view_timeout=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client2.expect(prompt) - client1.expect(r'6.*2' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client2.expect(prompt) - client1.expect(r'21.*3' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('SELECT sleep(1)') - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect('Table test.lv doesn\'t exist') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py deleted file mode 100755 index 5664c0e6c6d..00000000000 --- a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET live_view_heartbeat_interval=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv EVENTS') - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect('2.*' + end_of_block) - client1.expect('Progress: 2.00 rows.*\)') - # wait for heartbeat - client1.expect('Progress: 2.00 rows.*\)') - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py deleted file mode 100755 index 03e22175dff..00000000000 --- a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET live_view_heartbeat_interval=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client1.expect('Progress: 2.00 rows.*\)') - # wait for heartbeat - client1.expect('Progress: 2.00 rows.*\)') - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py deleted file mode 100755 index bb9d6152200..00000000000 --- a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - - with http_client({'method':'GET', 'url': '/?query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: - client2.expect('.*1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - client2.expect('.*2\n') - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py deleted file mode 100755 index d3439431eb3..00000000000 --- a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - - with http_client({'method':'GET', 'url':'/?query=WATCH%20test.lv'}, name='client2>', log=log) as client2: - client2.expect('.*0\t1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - client2.expect('.*6\t2\n') - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference deleted file mode 100644 index 5ae423d90d1..00000000000 --- a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference +++ /dev/null @@ -1,4 +0,0 @@ -{"row":{"a":1}} -{"row":{"a":2}} -{"row":{"a":3}} -{"progress":{"read_rows":"3","read_bytes":"36","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql deleted file mode 100644 index 8c6f4197d54..00000000000 --- a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql +++ /dev/null @@ -1,12 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT * FROM test.lv FORMAT JSONEachRowWithProgress; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference deleted file mode 100644 index 287a1ced92d..00000000000 --- a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference +++ /dev/null @@ -1,6 +0,0 @@ -{"row":{"sum(a)":"0","_version":"1"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} -{"row":{"sum(a)":"6","_version":"2"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} -{"row":{"sum(a)":"21","_version":"3"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql deleted file mode 100644 index 725a4ad00ed..00000000000 --- a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py deleted file mode 100755 index 63628c4a76f..00000000000 --- a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - - with http_client({'method':'GET', 'url': '/?live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}\n', escape=True) - client2.expect('{"row":{"version":"1"}', escape=True) - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) - # heartbeat is provided by progress message - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) - - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - - client2.expect('{"row":{"version":"2"}}\n', escape=True) - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py deleted file mode 100755 index 7bdb47b7caa..00000000000 --- a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - with http_client({'method':'GET', 'url':'/?live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: - client2.expect('"progress".*',) - client2.expect('{"row":{"sum(a)":"0","_version":"1"}}\n', escape=True) - client2.expect('"progress".*\n') - # heartbeat is provided by progress message - client2.expect('"progress".*\n') - - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - - client2.expect('"progress".*"read_rows":"2".*\n') - client2.expect('{"row":{"sum(a)":"6","_version":"2"}}\n', escape=True) - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference b/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql b/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql deleted file mode 100644 index 661080b577b..00000000000 --- a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP TABLE IF EXISTS test.lv; - -CREATE LIVE VIEW test.lv AS SELECT 1; - -SELECT * FROM test.lv; - -DROP TABLE test.lv; diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.reference b/dbms/tests/queries/0_stateless/00973_live_view_select.reference deleted file mode 100644 index 75236c0daf7..00000000000 --- a/dbms/tests/queries/0_stateless/00973_live_view_select.reference +++ /dev/null @@ -1,4 +0,0 @@ -6 1 -6 1 -12 2 -12 2 diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.sql b/dbms/tests/queries/0_stateless/00973_live_view_select.sql deleted file mode 100644 index ff4a45ffcc1..00000000000 --- a/dbms/tests/queries/0_stateless/00973_live_view_select.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; -SELECT *,_version FROM test.lv; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; -SELECT *,_version FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference deleted file mode 100644 index 6d50f0e9c3a..00000000000 --- a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference +++ /dev/null @@ -1,2 +0,0 @@ -6 -21 diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql deleted file mode 100644 index 3c11f855c9d..00000000000 --- a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql +++ /dev/null @@ -1,16 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT sum(a) FROM test.lv; - -INSERT INTO test.mt VALUES (4),(5),(6); - -SELECT sum(a) FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.reference b/dbms/tests/queries/0_stateless/00975_live_view_create.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.sql b/dbms/tests/queries/0_stateless/00975_live_view_create.sql deleted file mode 100644 index 1c929b15b00..00000000000 --- a/dbms/tests/queries/0_stateless/00975_live_view_create.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference b/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference deleted file mode 100644 index 453bd800469..00000000000 --- a/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference +++ /dev/null @@ -1,3 +0,0 @@ -1 1 -2 1 -3 1 diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql b/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql deleted file mode 100644 index 5f3ab1f7546..00000000000 --- a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql +++ /dev/null @@ -1,12 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference deleted file mode 100644 index 01e79c32a8c..00000000000 --- a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference +++ /dev/null @@ -1,3 +0,0 @@ -1 -2 -3 diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql deleted file mode 100644 index a3b84e8d4c1..00000000000 --- a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv EVENTS LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv EVENTS LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv EVENTS LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.reference b/dbms/tests/queries/0_stateless/00978_live_view_watch.reference deleted file mode 100644 index 6fbbedf1b21..00000000000 --- a/dbms/tests/queries/0_stateless/00978_live_view_watch.reference +++ /dev/null @@ -1,3 +0,0 @@ -0 1 -6 2 -21 3 diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql b/dbms/tests/queries/0_stateless/00978_live_view_watch.sql deleted file mode 100644 index abe4a6c32ae..00000000000 --- a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py deleted file mode 100755 index 948e4c93662..00000000000 --- a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client2.expect(prompt) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) - client2.expect(prompt) - for i in range(1,129): - client2.send('INSERT INTO test.mt VALUES (1)') - client1.expect(r'%d.*%d' % (21+i, 3+i) + end_of_block) - client2.expect(prompt) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference deleted file mode 100644 index 7f9fcbb2e9c..00000000000 --- a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference +++ /dev/null @@ -1,3 +0,0 @@ -temporary_live_view_timeout 5 -live_view_heartbeat_interval 15 -0 diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql deleted file mode 100644 index 8cd6ee06ace..00000000000 --- a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql +++ /dev/null @@ -1,15 +0,0 @@ -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -SELECT name, value from system.settings WHERE name = 'temporary_live_view_timeout'; -SELECT name, value from system.settings WHERE name = 'live_view_heartbeat_interval'; - -SET temporary_live_view_timeout=1; -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -SHOW TABLES LIKE 'lv'; -SELECT sleep(2); -SHOW TABLES LIKE 'lv'; - -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python deleted file mode 100644 index 782671cdfaf..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - line = q.get() - print(line) - assert (line == '21\t3') - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - print(line) - assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference deleted file mode 100644 index 1e94cdade41..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference +++ /dev/null @@ -1,7 +0,0 @@ -0 1 -0 1 -6 2 -6 2 -21 3 -21 3 -None diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.sh.disabled b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.sh.disabled deleted file mode 100755 index 10e4e98b2e3..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.sh.disabled +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -python $CURDIR/00991_live_view_watch_event_live.python diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python deleted file mode 100755 index 938547ca0cb..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_http_query(query): - cmd = list(CLICKHOUSE_CURL.split()) # list(['curl', '-sSN', '--max-time', '10']) - cmd += ['-sSN', CLICKHOUSE_URL, '-d', query] - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def read_lines_and_push_to_queue(pipe, queue): - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - pipe = send_http_query('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(pipe, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference deleted file mode 100644 index 489457d751b..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference +++ /dev/null @@ -1,4 +0,0 @@ -0 1 -0 1 -6 2 -6 2 diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.sh.disabled b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.sh.disabled deleted file mode 100755 index 88cce77f595..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.sh.disabled +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -python $CURDIR/00991_live_view_watch_http.python diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python deleted file mode 100644 index 70063adc6e3..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query, '--live_view_heartbeat_interval=1', '--progress'] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - # print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - # print(line) - assert (line.endswith('0\t1')) - assert ('Progress: 0.00 rows' in line) - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - assert (line.endswith('6\t2')) - assert ('Progress: 1.00 rows' in line) - - # send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - # line = q.get() - # print(line) - # assert (line.endswith('6\t2')) - # assert ('Progress: 1.00 rows' in line) - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - # print(line) - # assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.sh.disabled b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.sh.disabled deleted file mode 100755 index f7aa13d52b3..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.sh.disabled +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -python $CURDIR/00991_temporary_live_view_watch_events_heartbeat.python diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python deleted file mode 100644 index d290018a02c..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - line = q.get() - print(line) - assert (line == '21\t3') - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - print(line) - assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference deleted file mode 100644 index 1e94cdade41..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference +++ /dev/null @@ -1,7 +0,0 @@ -0 1 -0 1 -6 2 -6 2 -21 3 -21 3 -None diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.sh.disabled b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.sh.disabled deleted file mode 100755 index 4d01d1c3a8e..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.sh.disabled +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -python $CURDIR/00991_temporary_live_view_watch_live.python diff --git a/dbms/tests/queries/0_stateless/helpers/client.py b/dbms/tests/queries/0_stateless/helpers/client.py deleted file mode 100644 index f3938d3bf63..00000000000 --- a/dbms/tests/queries/0_stateless/helpers/client.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import sys -import time - -CURDIR = os.path.dirname(os.path.realpath(__file__)) - -sys.path.insert(0, os.path.join(CURDIR)) - -import uexpect - -prompt = ':\) ' -end_of_block = r'.*\r\n.*\r\n' - -class client(object): - def __init__(self, command=None, name='', log=None): - self.client = uexpect.spawn(['/bin/bash','--noediting']) - if command is None: - command = os.environ.get('CLICKHOUSE_BINARY', 'clickhouse') + '-client' - self.client.command = command - self.client.eol('\r') - self.client.logger(log, prefix=name) - self.client.timeout(20) - self.client.expect('[#\$] ', timeout=2) - self.client.send(command) - - def __enter__(self): - return self.client.__enter__() - - def __exit__(self, type, value, traceback): - self.client.reader['kill_event'].set() - # send Ctrl-C - self.client.send('\x03', eol='') - time.sleep(0.3) - self.client.send('quit', eol='\r') - self.client.send('\x03', eol='') - return self.client.__exit__(type, value, traceback) diff --git a/dbms/tests/queries/0_stateless/helpers/httpclient.py b/dbms/tests/queries/0_stateless/helpers/httpclient.py deleted file mode 100644 index a42fad2cbc3..00000000000 --- a/dbms/tests/queries/0_stateless/helpers/httpclient.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) - -sys.path.insert(0, os.path.join(CURDIR)) - -import httpexpect - -def client(request, name='', log=None): - client = httpexpect.spawn({'host':'localhost','port':8123}, request) - client.logger(log, prefix=name) - client.timeout(20) - return client diff --git a/dbms/tests/queries/0_stateless/helpers/httpexpect.py b/dbms/tests/queries/0_stateless/helpers/httpexpect.py deleted file mode 100644 index e440dafce4e..00000000000 --- a/dbms/tests/queries/0_stateless/helpers/httpexpect.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (c) 2019 Vitaliy Zakaznikov -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys -import httplib - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, CURDIR) - -import uexpect - -from threading import Thread, Event -from Queue import Queue, Empty - -class IO(uexpect.IO): - def __init__(self, connection, response, queue, reader): - self.connection = connection - self.response = response - super(IO, self).__init__(None, None, queue, reader) - - def write(self, data): - raise NotImplementedError - - def close(self, force=True): - self.reader['kill_event'].set() - self.connection.close() - if self._logger: - self._logger.write('\n') - self._logger.flush() - - -def reader(response, queue, kill_event): - while True: - try: - if kill_event.is_set(): - break - data = response.read(1) - queue.put(data) - except Exception, e: - if kill_event.is_set(): - break - raise - -def spawn(connection, request): - connection = httplib.HTTPConnection(**connection) - connection.request(**request) - response = connection.getresponse() - - queue = Queue() - reader_kill_event = Event() - thread = Thread(target=reader, args=(response, queue, reader_kill_event)) - thread.daemon = True - thread.start() - - return IO(connection, response, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) - -if __name__ == '__main__': - with http({'host':'localhost','port':8123},{'method':'GET', 'url':'?query=SELECT%201'}) as client: - client.logger(sys.stdout) - client.timeout(2) - print client.response.status, client.response.reason - client.expect('1\n') diff --git a/dbms/tests/queries/0_stateless/helpers/uexpect.py b/dbms/tests/queries/0_stateless/helpers/uexpect.py deleted file mode 100644 index f71b32a53e1..00000000000 --- a/dbms/tests/queries/0_stateless/helpers/uexpect.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) 2019 Vitaliy Zakaznikov -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import pty -import time -import sys -import re - -from threading import Thread, Event -from subprocess import Popen -from Queue import Queue, Empty - -class TimeoutError(Exception): - def __init__(self, timeout): - self.timeout = timeout - - def __str__(self): - return 'Timeout %.3fs' % float(self.timeout) - -class ExpectTimeoutError(Exception): - def __init__(self, pattern, timeout, buffer): - self.pattern = pattern - self.timeout = timeout - self.buffer = buffer - - def __str__(self): - s = 'Timeout %.3fs ' % float(self.timeout) - if self.pattern: - s += 'for %s ' % repr(self.pattern.pattern) - if self.buffer: - s += 'buffer %s ' % repr(self.buffer[:]) - s += 'or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]]) - return s - -class IO(object): - class EOF(object): - pass - - class Timeout(object): - pass - - EOF = EOF - TIMEOUT = Timeout - - class Logger(object): - def __init__(self, logger, prefix=''): - self._logger = logger - self._prefix = prefix - - def write(self, data): - self._logger.write(('\n' + data).replace('\n','\n' + self._prefix)) - - def flush(self): - self._logger.flush() - - def __init__(self, process, master, queue, reader): - self.process = process - self.master = master - self.queue = queue - self.buffer = None - self.before = None - self.after = None - self.match = None - self.pattern = None - self.reader = reader - self._timeout = None - self._logger = None - self._eol = '' - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def logger(self, logger=None, prefix=''): - if logger: - self._logger = self.Logger(logger, prefix=prefix) - return self._logger - - def timeout(self, timeout=None): - if timeout: - self._timeout = timeout - return self._timeout - - def eol(self, eol=None): - if eol: - self._eol = eol - return self._eol - - def close(self, force=True): - self.reader['kill_event'].set() - os.system('pkill -TERM -P %d' % self.process.pid) - if force: - self.process.kill() - else: - self.process.terminate() - os.close(self.master) - if self._logger: - self._logger.write('\n') - self._logger.flush() - - def send(self, data, eol=None): - if eol is None: - eol = self._eol - return self.write(data + eol) - - def write(self, data): - return os.write(self.master, data) - - def expect(self, pattern, timeout=None, escape=False): - self.match = None - self.before = None - self.after = None - if escape: - pattern = re.escape(pattern) - pattern = re.compile(pattern) - if timeout is None: - timeout = self._timeout - timeleft = timeout - while True: - start_time = time.time() - if self.buffer is not None: - self.match = pattern.search(self.buffer, 0) - if self.match is not None: - self.after = self.buffer[self.match.start():self.match.end()] - self.before = self.buffer[:self.match.start()] - self.buffer = self.buffer[self.match.end():] - break - if timeleft < 0: - break - try: - data = self.read(timeout=timeleft, raise_exception=True) - except TimeoutError: - if self._logger: - self._logger.write((self.buffer or '') + '\n') - self._logger.flush() - exception = ExpectTimeoutError(pattern, timeout, self.buffer) - self.buffer = None - raise exception - timeleft -= (time.time() - start_time) - if data: - self.buffer = (self.buffer + data) if self.buffer else data - if self._logger: - self._logger.write((self.before or '') + (self.after or '')) - self._logger.flush() - if self.match is None: - exception = ExpectTimeoutError(pattern, timeout, self.buffer) - self.buffer = None - raise exception - return self.match - - def read(self, timeout=0, raise_exception=False): - data = '' - timeleft = timeout - try: - while timeleft >= 0 : - start_time = time.time() - data += self.queue.get(timeout=timeleft) - if data: - break - timeleft -= (time.time() - start_time) - except Empty: - if data: - return data - if raise_exception: - raise TimeoutError(timeout) - pass - if not data and raise_exception: - raise TimeoutError(timeout) - - return data - -def spawn(command): - master, slave = pty.openpty() - process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1) - os.close(slave) - - queue = Queue() - reader_kill_event = Event() - thread = Thread(target=reader, args=(process, master, queue, reader_kill_event)) - thread.daemon = True - thread.start() - - return IO(process, master, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) - -def reader(process, out, queue, kill_event): - while True: - try: - data = os.read(out, 65536) - queue.put(data) - except: - if kill_event.is_set(): - break - raise From cff8ec43f992ca2daff2141ea15cd964c7401168 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 04:31:04 +0300 Subject: [PATCH 185/357] Rename neighbour -> neighbor --- dbms/src/Functions/{neighbour.cpp => neighbor.cpp} | 14 +++++++------- .../Functions/registerFunctionsMiscellaneous.cpp | 4 ++-- .../en/query_language/functions/other_functions.md | 4 ++-- .../ru/query_language/functions/other_functions.md | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) rename dbms/src/Functions/{neighbour.cpp => neighbor.cpp} (96%) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbor.cpp similarity index 96% rename from dbms/src/Functions/neighbour.cpp rename to dbms/src/Functions/neighbor.cpp index 02cd4df4996..6ac2c966016 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbor.cpp @@ -21,17 +21,17 @@ namespace ErrorCodes // | c1 | // | 10 | // | 20 | -// SELECT c1, neighbour(c1, 1) as c2: +// SELECT c1, neighbor(c1, 1) as c2: // | c1 | c2 | // | 10 | 20 | // | 20 | 0 | -class FunctionNeighbour : public IFunction +class FunctionNeighbor : public IFunction { public: - static constexpr auto name = "neighbour"; - static FunctionPtr create(const Context & context) { return std::make_shared(context); } + static constexpr auto name = "neighbor"; + static FunctionPtr create(const Context & context) { return std::make_shared(context); } - FunctionNeighbour(const Context & context_) : context(context_) {} + FunctionNeighbor(const Context & context_) : context(context_) {} /// Get the name of the function. String getName() const override { return name; } @@ -255,9 +255,9 @@ private: const Context & context; }; -void registerFunctionNeighbour(FunctionFactory & factory) +void registerFunctionNeighbor(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); } } diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp index e3d4714638e..1d95844ce9b 100644 --- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp @@ -18,7 +18,7 @@ void registerFunctionBlockSize(FunctionFactory &); void registerFunctionBlockNumber(FunctionFactory &); void registerFunctionRowNumberInBlock(FunctionFactory &); void registerFunctionRowNumberInAllBlocks(FunctionFactory &); -void registerFunctionNeighbour(FunctionFactory &); +void registerFunctionNeighbor(FunctionFactory &); void registerFunctionSleep(FunctionFactory &); void registerFunctionSleepEachRow(FunctionFactory &); void registerFunctionMaterialize(FunctionFactory &); @@ -70,7 +70,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionBlockNumber(factory); registerFunctionRowNumberInBlock(factory); registerFunctionRowNumberInAllBlocks(factory); - registerFunctionNeighbour(factory); + registerFunctionNeighbor(factory); registerFunctionSleep(factory); registerFunctionSleepEachRow(factory); registerFunctionMaterialize(factory); diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md index 36c40fc4b02..349397059af 100644 --- a/docs/en/query_language/functions/other_functions.md +++ b/docs/en/query_language/functions/other_functions.md @@ -314,7 +314,7 @@ Returns the ordinal number of the row in the data block. Different data blocks a Returns the ordinal number of the row in the data block. This function only considers the affected data blocks. -## neighbour(column, offset\[, default_value\]) +## neighbor(column, offset\[, default_value\]) Returns value for `column`, in `offset` distance from current row. This function is a partial implementation of [window functions](https://en.wikipedia.org/wiki/SQL_window_function) LEAD() and LAG(). @@ -330,7 +330,7 @@ WITH toDate('2018-01-01') AS start_date SELECT toStartOfMonth(start_date + (number * 32)) AS month, toInt32(month) % 100 AS money, - neighbour(money, -12) AS prev_year, + neighbor(money, -12) AS prev_year, round(prev_year / money, 2) AS year_over_year FROM numbers(16) ``` diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md index 41cd7c8c63b..fdc46a0d4ee 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/query_language/functions/other_functions.md @@ -291,7 +291,7 @@ SELECT ## rowNumberInAllBlocks() Возвращает порядковый номер строки в блоке данных. Функция учитывает только задействованные блоки данных. -## neighbour(column, offset\[, default_value\]) +## neighbor(column, offset\[, default_value\]) Функция позволяет получить доступ к значению в колонке `column`, находящемуся на смещении `offset` относительно текущей строки. Является частичной реализацией [оконных функций](https://en.wikipedia.org/wiki/SQL_window_function) LEAD() и LAG(). @@ -307,7 +307,7 @@ WITH toDate('2018-01-01') AS start_date SELECT toStartOfMonth(start_date + (number * 32)) AS month, toInt32(month) % 100 AS money, - neighbour(money, -12) AS prev_year, + neighbor(money, -12) AS prev_year, round(prev_year / money, 2) AS year_over_year FROM numbers(16) ``` From e3bd572fc75f20d83f421e00981524e6bcd1b8ba Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 05:00:35 +0300 Subject: [PATCH 186/357] Removed unused settings --- dbms/src/Core/Settings.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 1c2cab8b860..3cc5d291708 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -350,8 +350,6 @@ struct Settings : public SettingsCollection \ M(SettingSeconds, live_view_heartbeat_interval, DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC, "The heartbeat interval in seconds to indicate live query is alive.") \ M(SettingSeconds, temporary_live_view_timeout, DEFAULT_TEMPORARY_LIVE_VIEW_TIMEOUT_SEC, "Timeout after which temporary live view is deleted.") \ - M(SettingSeconds, temporary_live_channel_timeout, DEFAULT_TEMPORARY_LIVE_CHANNEL_TIMEOUT_SEC, "Timeout after which temporary live channel is deleted.") \ - M(SettingMilliseconds, alter_channel_wait_ms, DEFAULT_ALTER_LIVE_CHANNEL_WAIT_MS, "The wait time for alter channel request.") \ M(SettingUInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.") \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) From 99f4c9c8130a742d532294ce5edf2ac6cfd86a8c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 05:01:24 +0300 Subject: [PATCH 187/357] Moved settings that were in a wrong place --- dbms/src/Core/Settings.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 3cc5d291708..ff3c8fb5890 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -343,14 +343,13 @@ struct Settings : public SettingsCollection M(SettingBool, check_query_single_value_result, true, "Return check query result as single 1/0 value") \ \ M(SettingBool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.") \ + M(SettingSeconds, live_view_heartbeat_interval, DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC, "The heartbeat interval in seconds to indicate live query is alive.") \ + M(SettingSeconds, temporary_live_view_timeout, DEFAULT_TEMPORARY_LIVE_VIEW_TIMEOUT_SEC, "Timeout after which temporary live view is deleted.") \ + M(SettingUInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.") \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ - \ - M(SettingSeconds, live_view_heartbeat_interval, DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC, "The heartbeat interval in seconds to indicate live query is alive.") \ - M(SettingSeconds, temporary_live_view_timeout, DEFAULT_TEMPORARY_LIVE_VIEW_TIMEOUT_SEC, "Timeout after which temporary live view is deleted.") \ - M(SettingUInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.") \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) From 1222973cb3fa6e267c6bb7d29c44fef2beafe173 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 05:02:14 +0300 Subject: [PATCH 188/357] Function "neighbor": merging #5925 --- dbms/src/Functions/neighbor.cpp | 50 +++++++++++++++------------------ 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/dbms/src/Functions/neighbor.cpp b/dbms/src/Functions/neighbor.cpp index 6ac2c966016..02d28028d3b 100644 --- a/dbms/src/Functions/neighbor.cpp +++ b/dbms/src/Functions/neighbor.cpp @@ -66,7 +66,6 @@ public: "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + " - can not be Nullable", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - // check that default value column has supertype with first argument if (number_of_arguments == 3) { @@ -99,10 +98,10 @@ public: { if (isColumnConst(*default_values_column)) { - Field constant_value = (*default_values_column)[0]; - for (size_t row = 0; row < row_count; row++) + const IColumn & constant_content = assert_cast(*default_values_column).getDataColumn(); + for (size_t row = 0; row < row_count; ++row) { - target->insert(constant_value); + target->insertFrom(constant_content, 0); } } else @@ -112,7 +111,7 @@ public: } else { - for (size_t row = 0; row < row_count; row++) + for (size_t row = 0; row < row_count; ++row) { target->insertDefault(); } @@ -121,37 +120,33 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - auto offset_structure = block.getByPosition(arguments[1]); + const ColumnWithTypeAndName & source_column_name_and_type = block.getByPosition(arguments[0]); + const DataTypePtr & result_type = block.getByPosition(result).type; - ColumnPtr & offset_column = offset_structure.column; + ColumnPtr source_column = source_column_name_and_type.column; - auto is_constant_offset = isColumnConst(*offset_structure.column); - ColumnPtr default_values_column = nullptr; - if (arguments.size() == 3) - { - default_values_column = block.getByPosition(arguments[2]).column; - } - - ColumnWithTypeAndName & source_column_name_and_type = block.getByPosition(arguments[0]); - DataTypes types = {source_column_name_and_type.type}; - if (default_values_column) - { - types.push_back(block.getByPosition(arguments[2]).type); - } - const DataTypePtr & result_type = getLeastSupertype(types); - auto source_column = source_column_name_and_type.column; - - // adjust source and default values columns to resulting datatype + // adjust source and default values columns to resulting data type if (!source_column_name_and_type.type->equals(*result_type)) { source_column = castColumn(source_column_name_and_type, result_type, context); } - if (default_values_column && !block.getByPosition(arguments[2]).type->equals(*result_type)) + ColumnPtr default_values_column; + + /// Has argument with default value: neighbor(source, offset, default) + if (arguments.size() == 3) { - default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); + default_values_column = block.getByPosition(arguments[2]).column; + + if (!block.getByPosition(arguments[2]).type->equals(*result_type)) + default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); } + const auto & offset_structure = block.getByPosition(arguments[1]); + ColumnPtr offset_column = offset_structure.column; + + auto is_constant_offset = isColumnConst(*offset_structure.column); + // since we are working with both signed and unsigned - we'll try to use Int64 for handling all of them const DataTypePtr desired_type = std::make_shared(); if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) @@ -161,6 +156,7 @@ public: if (isColumnConst(*source_column)) { + /// NOTE Inconsistency when default_values are specified. auto column = result_type->createColumnConst(input_rows_count, (*source_column)[0]); block.getByPosition(result).column = std::move(column); } @@ -199,7 +195,7 @@ public: else { // with dynamic offset - handle row by row - for (size_t row = 0; row < input_rows_count; row++) + for (size_t row = 0; row < input_rows_count; ++row) { Int64 offset_value = offset_column->getInt(row); if (offset_value == 0) From ff9e92eab9ace56801f032e1bf3990217ee456d5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 05:53:09 +0300 Subject: [PATCH 189/357] Renamed function in test --- .../0_stateless/00957_neighbor.reference | 69 +++++++++++++++++++ .../queries/0_stateless/00957_neighbor.sql | 42 +++++++++++ 2 files changed, 111 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00957_neighbor.reference create mode 100644 dbms/tests/queries/0_stateless/00957_neighbor.sql diff --git a/dbms/tests/queries/0_stateless/00957_neighbor.reference b/dbms/tests/queries/0_stateless/00957_neighbor.reference new file mode 100644 index 00000000000..5a523d2c575 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_neighbor.reference @@ -0,0 +1,69 @@ +Zero offset +0 0 +1 1 +2 2 +Nullable values +\N 0 \N +\N 1 2 +2 2 \N +Result with different type +0 1 +1 2 +2 -10 +Offset > block +0 0 +1 0 +2 0 +Abs(Offset) > block +0 0 +1 0 +2 0 +Positive offset +0 1 +1 2 +2 0 +Negative offset +0 1 +1 2 +2 0 +Positive offset with defaults +0 2 +1 3 +2 12 +3 13 +Negative offset with defaults +0 10 +1 11 +2 0 +3 1 +Positive offset with const defaults +0 1 +1 2 +2 1000 +Negative offset with const defaults +0 1000 +1 0 +2 1 +Dynamic column and offset, out of bounds +0 0 0 +1 2 3 +2 4 20 +3 6 30 +Dynamic column and offset, negative +0 0 0 +1 -2 10 +2 -4 20 +3 -6 30 +4 -8 40 +5 -10 50 +Dynamic column and offset, without defaults +0 4 4 +1 2 3 +2 0 2 +3 -2 1 +4 -4 0 +5 -6 0 +Constant column +0 1000 +1 1000 +2 1000 diff --git a/dbms/tests/queries/0_stateless/00957_neighbor.sql b/dbms/tests/queries/0_stateless/00957_neighbor.sql new file mode 100644 index 00000000000..2f701d4d5c7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_neighbor.sql @@ -0,0 +1,42 @@ +-- no arguments +select neighbor(); -- { serverError 42 } +-- single argument +select neighbor(1); -- { serverError 42 } +-- greater than 3 arguments +select neighbor(1,2,3,4); -- { serverError 42 } +-- bad default value +select neighbor(dummy, 1, 'hello'); -- { serverError 43 } +-- types without common supertype (UInt64 and Int8) +select number, neighbor(number, 1, -10) from numbers(3); -- { serverError 43 } +-- nullable offset is not allowed +select number, if(number > 1, number, null) as offset, neighbor(number, offset) from numbers(3); -- { serverError 43 } +select 'Zero offset'; +select number, neighbor(number, 0) from numbers(3); +select 'Nullable values'; +select if(number > 1, number, null) as value, number as offset, neighbor(value, offset) as neighbor from numbers(3); +select 'Result with different type'; +select toInt32(number) as n, neighbor(n, 1, -10) from numbers(3); +select 'Offset > block'; +select number, neighbor(number, 10) from numbers(3); +select 'Abs(Offset) > block'; +select number, neighbor(number, -10) from numbers(3); +select 'Positive offset'; +select number, neighbor(number, 1) from numbers(3); +select 'Negative offset'; +select number, neighbor(number, 1) from numbers(3); +select 'Positive offset with defaults'; +select number, neighbor(number, 2, number + 10) from numbers(4); +select 'Negative offset with defaults'; +select number, neighbor(number, -2, number + 10) from numbers(4); +select 'Positive offset with const defaults'; +select number, neighbor(number, 1, 1000) from numbers(3); +select 'Negative offset with const defaults'; +select number, neighbor(number, -1, 1000) from numbers(3); +select 'Dynamic column and offset, out of bounds'; +select number, number * 2 as offset, neighbor(number, offset, number * 10) from numbers(4); +select 'Dynamic column and offset, negative'; +select number, -number * 2 as offset, neighbor(number, offset, number * 10) from numbers(6); +select 'Dynamic column and offset, without defaults'; +select number, -(number - 2) * 2 as offset, neighbor(number, offset) from numbers(6); +select 'Constant column'; +select number, neighbor(1000, 10) from numbers(3); \ No newline at end of file From 341e2e4587a18065c2da1ca888c73389f48ce36c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 06:00:06 +0300 Subject: [PATCH 190/357] Step 1: make it correct. --- dbms/src/Functions/neighbor.cpp | 197 ++++-------------- .../0_stateless/00957_neighbor.reference | 6 +- .../queries/0_stateless/00957_neighbor.sql | 4 +- .../0_stateless/00957_neighbour.reference | 69 ------ .../queries/0_stateless/00957_neighbour.sql | 42 ---- 5 files changed, 42 insertions(+), 276 deletions(-) delete mode 100644 dbms/tests/queries/0_stateless/00957_neighbour.reference delete mode 100644 dbms/tests/queries/0_stateless/00957_neighbour.sql diff --git a/dbms/src/Functions/neighbor.cpp b/dbms/src/Functions/neighbor.cpp index 02d28028d3b..1c640ffb76c 100644 --- a/dbms/src/Functions/neighbor.cpp +++ b/dbms/src/Functions/neighbor.cpp @@ -46,6 +46,8 @@ public: bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return false; } + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { size_t number_of_arguments = arguments.size(); @@ -68,183 +70,58 @@ public: // check that default value column has supertype with first argument if (number_of_arguments == 3) - { - DataTypes types = {arguments[0], arguments[2]}; - try - { - return getLeastSupertype(types); - } - catch (const Exception &) - { - throw Exception( - "Illegal types of arguments (" + types[0]->getName() + ", " + types[1]->getName() - + ")" - " of function " - + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - } + return getLeastSupertype({arguments[0], arguments[2]}); return arguments[0]; } - static void insertDefaults(const MutableColumnPtr & target, size_t row_count, ColumnPtr & default_values_column, size_t offset) - { - if (row_count == 0) - { - return; - } - if (default_values_column) - { - if (isColumnConst(*default_values_column)) - { - const IColumn & constant_content = assert_cast(*default_values_column).getDataColumn(); - for (size_t row = 0; row < row_count; ++row) - { - target->insertFrom(constant_content, 0); - } - } - else - { - target->insertRangeFrom(*default_values_column, offset, row_count); - } - } - else - { - for (size_t row = 0; row < row_count; ++row) - { - target->insertDefault(); - } - } - } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const ColumnWithTypeAndName & source_column_name_and_type = block.getByPosition(arguments[0]); const DataTypePtr & result_type = block.getByPosition(result).type; - ColumnPtr source_column = source_column_name_and_type.column; + const ColumnWithTypeAndName & source_elem = block.getByPosition(arguments[0]); + const ColumnWithTypeAndName & offset_elem = block.getByPosition(arguments[1]); + bool has_defaults = arguments.size() == 3; - // adjust source and default values columns to resulting data type - if (!source_column_name_and_type.type->equals(*result_type)) + ColumnPtr source_column_casted = castColumn(source_elem, result_type, context); + ColumnPtr offset_column = offset_elem.column; + + ColumnPtr default_column_casted; + if (has_defaults) { - source_column = castColumn(source_column_name_and_type, result_type, context); + const ColumnWithTypeAndName & default_elem = block.getByPosition(arguments[2]); + default_column_casted = castColumn(default_elem, result_type, context); } - ColumnPtr default_values_column; + bool source_is_constant = isColumnConst(*source_column_casted); + bool offset_is_constant = isColumnConst(*offset_column); - /// Has argument with default value: neighbor(source, offset, default) - if (arguments.size() == 3) + bool default_is_constant = false; + if (has_defaults) + default_is_constant = isColumnConst(*default_column_casted); + + if (source_is_constant) + source_column_casted = assert_cast(*source_column_casted).getDataColumnPtr(); + if (offset_is_constant) + offset_column = assert_cast(*offset_column).getDataColumnPtr(); + if (default_is_constant) + default_column_casted = assert_cast(*default_column_casted).getDataColumnPtr(); + + auto column = result_type->createColumn(); + + for (size_t row = 0; row < input_rows_count; ++row) { - default_values_column = block.getByPosition(arguments[2]).column; + Int64 src_idx = row + offset_column->getInt(offset_is_constant ? 0 : row); - if (!block.getByPosition(arguments[2]).type->equals(*result_type)) - default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); - } - - const auto & offset_structure = block.getByPosition(arguments[1]); - ColumnPtr offset_column = offset_structure.column; - - auto is_constant_offset = isColumnConst(*offset_structure.column); - - // since we are working with both signed and unsigned - we'll try to use Int64 for handling all of them - const DataTypePtr desired_type = std::make_shared(); - if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) - { - offset_column = castColumn(offset_structure, desired_type, context); - } - - if (isColumnConst(*source_column)) - { - /// NOTE Inconsistency when default_values are specified. - auto column = result_type->createColumnConst(input_rows_count, (*source_column)[0]); - block.getByPosition(result).column = std::move(column); - } - else - { - auto column = result_type->createColumn(); - column->reserve(input_rows_count); - // with constant offset - insertRangeFrom - if (is_constant_offset) - { - Int64 offset_value = offset_column->getInt(0); - - auto offset_value_casted = static_cast(std::abs(offset_value)); - size_t default_value_count = std::min(offset_value_casted, input_rows_count); - if (offset_value > 0) - { - // insert shifted value - if (offset_value_casted <= input_rows_count) - { - column->insertRangeFrom(*source_column, offset_value_casted, input_rows_count - offset_value_casted); - } - insertDefaults(column, default_value_count, default_values_column, input_rows_count - default_value_count); - } - else if (offset_value < 0) - { - // insert defaults up to offset_value - insertDefaults(column, default_value_count, default_values_column, 0); - column->insertRangeFrom(*source_column, 0, input_rows_count - default_value_count); - } - else - { - // populate column with source values, when offset is equal to zero - column->insertRangeFrom(*source_column, 0, input_rows_count); - } - } + if (src_idx >= 0 && src_idx < Int64(input_rows_count)) + column->insertFrom(*source_column_casted, source_is_constant ? 0 : src_idx); + else if (has_defaults) + column->insertFrom(*default_column_casted, default_is_constant ? 0 : row); else - { - // with dynamic offset - handle row by row - for (size_t row = 0; row < input_rows_count; ++row) - { - Int64 offset_value = offset_column->getInt(row); - if (offset_value == 0) - { - column->insertFrom(*source_column, row); - } - else if (offset_value > 0) - { - size_t real_offset = row + offset_value; - if (real_offset > input_rows_count) - { - if (default_values_column) - { - column->insertFrom(*default_values_column, row); - } - else - { - column->insertDefault(); - } - } - else - { - column->insertFrom(*source_column, real_offset); - } - } - else - { - // out of range - auto offset_value_casted = static_cast(std::abs(offset_value)); - if (offset_value_casted > row) - { - if (default_values_column) - { - column->insertFrom(*default_values_column, row); - } - else - { - column->insertDefault(); - } - } - else - { - column->insertFrom(*source_column, row - offset_value_casted); - } - } - } - } - block.getByPosition(result).column = std::move(column); + column->insertDefault(); } + + block.getByPosition(result).column = std::move(column); } private: diff --git a/dbms/tests/queries/0_stateless/00957_neighbor.reference b/dbms/tests/queries/0_stateless/00957_neighbor.reference index 5a523d2c575..d25d727da5d 100644 --- a/dbms/tests/queries/0_stateless/00957_neighbor.reference +++ b/dbms/tests/queries/0_stateless/00957_neighbor.reference @@ -64,6 +64,6 @@ Dynamic column and offset, without defaults 4 -4 0 5 -6 0 Constant column -0 1000 -1 1000 -2 1000 +0 0 +1 0 +2 0 diff --git a/dbms/tests/queries/0_stateless/00957_neighbor.sql b/dbms/tests/queries/0_stateless/00957_neighbor.sql index 2f701d4d5c7..c4b801c80cb 100644 --- a/dbms/tests/queries/0_stateless/00957_neighbor.sql +++ b/dbms/tests/queries/0_stateless/00957_neighbor.sql @@ -5,9 +5,9 @@ select neighbor(1); -- { serverError 42 } -- greater than 3 arguments select neighbor(1,2,3,4); -- { serverError 42 } -- bad default value -select neighbor(dummy, 1, 'hello'); -- { serverError 43 } +select neighbor(dummy, 1, 'hello'); -- { serverError 386 } -- types without common supertype (UInt64 and Int8) -select number, neighbor(number, 1, -10) from numbers(3); -- { serverError 43 } +select number, neighbor(number, 1, -10) from numbers(3); -- { serverError 386 } -- nullable offset is not allowed select number, if(number > 1, number, null) as offset, neighbor(number, offset) from numbers(3); -- { serverError 43 } select 'Zero offset'; diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.reference b/dbms/tests/queries/0_stateless/00957_neighbour.reference deleted file mode 100644 index 5a523d2c575..00000000000 --- a/dbms/tests/queries/0_stateless/00957_neighbour.reference +++ /dev/null @@ -1,69 +0,0 @@ -Zero offset -0 0 -1 1 -2 2 -Nullable values -\N 0 \N -\N 1 2 -2 2 \N -Result with different type -0 1 -1 2 -2 -10 -Offset > block -0 0 -1 0 -2 0 -Abs(Offset) > block -0 0 -1 0 -2 0 -Positive offset -0 1 -1 2 -2 0 -Negative offset -0 1 -1 2 -2 0 -Positive offset with defaults -0 2 -1 3 -2 12 -3 13 -Negative offset with defaults -0 10 -1 11 -2 0 -3 1 -Positive offset with const defaults -0 1 -1 2 -2 1000 -Negative offset with const defaults -0 1000 -1 0 -2 1 -Dynamic column and offset, out of bounds -0 0 0 -1 2 3 -2 4 20 -3 6 30 -Dynamic column and offset, negative -0 0 0 -1 -2 10 -2 -4 20 -3 -6 30 -4 -8 40 -5 -10 50 -Dynamic column and offset, without defaults -0 4 4 -1 2 3 -2 0 2 -3 -2 1 -4 -4 0 -5 -6 0 -Constant column -0 1000 -1 1000 -2 1000 diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.sql b/dbms/tests/queries/0_stateless/00957_neighbour.sql deleted file mode 100644 index b60cc3123b6..00000000000 --- a/dbms/tests/queries/0_stateless/00957_neighbour.sql +++ /dev/null @@ -1,42 +0,0 @@ --- no arguments -select neighbour(); -- { serverError 42 } --- single argument -select neighbour(1); -- { serverError 42 } --- greater than 3 arguments -select neighbour(1,2,3,4); -- { serverError 42 } --- bad default value -select neighbour(dummy, 1, 'hello'); -- { serverError 43 } --- types without common supertype (UInt64 and Int8) -select number, neighbour(number, 1, -10) from numbers(3); -- { serverError 43 } --- nullable offset is not allowed -select number, if(number > 1, number, null) as offset, neighbour(number, offset) from numbers(3); -- { serverError 43 } -select 'Zero offset'; -select number, neighbour(number, 0) from numbers(3); -select 'Nullable values'; -select if(number > 1, number, null) as value, number as offset, neighbour(value, offset) as neighbour from numbers(3); -select 'Result with different type'; -select toInt32(number) as n, neighbour(n, 1, -10) from numbers(3); -select 'Offset > block'; -select number, neighbour(number, 10) from numbers(3); -select 'Abs(Offset) > block'; -select number, neighbour(number, -10) from numbers(3); -select 'Positive offset'; -select number, neighbour(number, 1) from numbers(3); -select 'Negative offset'; -select number, neighbour(number, 1) from numbers(3); -select 'Positive offset with defaults'; -select number, neighbour(number, 2, number + 10) from numbers(4); -select 'Negative offset with defaults'; -select number, neighbour(number, -2, number + 10) from numbers(4); -select 'Positive offset with const defaults'; -select number, neighbour(number, 1, 1000) from numbers(3); -select 'Negative offset with const defaults'; -select number, neighbour(number, -1, 1000) from numbers(3); -select 'Dynamic column and offset, out of bounds'; -select number, number * 2 as offset, neighbour(number, offset, number * 10) from numbers(4); -select 'Dynamic column and offset, negative'; -select number, -number * 2 as offset, neighbour(number, offset, number * 10) from numbers(6); -select 'Dynamic column and offset, without defaults'; -select number, -(number - 2) * 2 as offset, neighbour(number, offset) from numbers(6); -select 'Constant column'; -select number, neighbour(1000, 10) from numbers(3); \ No newline at end of file From 6685365ab8c5b74f9650492c88a012596eb1b0c6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 07:25:32 +0300 Subject: [PATCH 191/357] Added optimized case --- dbms/src/Functions/neighbor.cpp | 78 ++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/dbms/src/Functions/neighbor.cpp b/dbms/src/Functions/neighbor.cpp index 1c640ffb76c..994e6311678 100644 --- a/dbms/src/Functions/neighbor.cpp +++ b/dbms/src/Functions/neighbor.cpp @@ -107,21 +107,79 @@ public: if (default_is_constant) default_column_casted = assert_cast(*default_column_casted).getDataColumnPtr(); - auto column = result_type->createColumn(); - - for (size_t row = 0; row < input_rows_count; ++row) + if (offset_is_constant) { - Int64 src_idx = row + offset_column->getInt(offset_is_constant ? 0 : row); + /// Optimization for the case when we can copy many values at once. - if (src_idx >= 0 && src_idx < Int64(input_rows_count)) - column->insertFrom(*source_column_casted, source_is_constant ? 0 : src_idx); - else if (has_defaults) - column->insertFrom(*default_column_casted, default_is_constant ? 0 : row); + Int64 offset = offset_column->getInt(0); + + auto result_column = result_type->createColumn(); + + auto insert_range_from = [&](bool is_const, const ColumnPtr & src, Int64 begin, Int64 size) + { + /// Saturation of bounds. + if (begin < 0) + { + size += begin; + begin = 0; + } + if (size <= 0) + return; + if (size > Int64(input_rows_count)) + size = input_rows_count; + + if (!src) + { + for (Int64 i = 0; i < size; ++i) + result_column->insertDefault(); + } + else if (is_const) + { + for (Int64 i = 0; i < size; ++i) + result_column->insertFrom(*src, 0); + } + else + { + result_column->insertRangeFrom(*src, begin, size); + } + }; + + if (offset == 0) + { + /// Degenerate case, just copy source column as is. + block.getByPosition(result).column = source_column_casted; /// TODO + } + else if (offset > 0) + { + insert_range_from(source_is_constant, source_column_casted, offset, Int64(input_rows_count) - offset); + insert_range_from(default_is_constant, default_column_casted, Int64(input_rows_count) - offset, offset); + block.getByPosition(result).column = std::move(result_column); + } else - column->insertDefault(); + { + insert_range_from(default_is_constant, default_column_casted, 0, -offset); + insert_range_from(source_is_constant, source_column_casted, 0, Int64(input_rows_count) + offset); + block.getByPosition(result).column = std::move(result_column); + } } + else + { + auto result_column = result_type->createColumn(); - block.getByPosition(result).column = std::move(column); + for (size_t row = 0; row < input_rows_count; ++row) + { + Int64 src_idx = row + offset_column->getInt(offset_is_constant ? 0 : row); + + if (src_idx >= 0 && src_idx < Int64(input_rows_count)) + result_column->insertFrom(*source_column_casted, source_is_constant ? 0 : src_idx); + else if (has_defaults) + result_column->insertFrom(*default_column_casted, default_is_constant ? 0 : row); + else + result_column->insertDefault(); + } + + block.getByPosition(result).column = std::move(result_column); + } } private: From 20b9af29f555f6e125c811e95cc19c39ca7857d1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 07:34:59 +0300 Subject: [PATCH 192/357] More tests --- dbms/src/Functions/neighbor.cpp | 2 +- .../0_stateless/00996_neigbor.reference | 270 ++++++++++++++++++ .../queries/0_stateless/00996_neigbor.sql | 42 +++ 3 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/00996_neigbor.reference create mode 100644 dbms/tests/queries/0_stateless/00996_neigbor.sql diff --git a/dbms/src/Functions/neighbor.cpp b/dbms/src/Functions/neighbor.cpp index 994e6311678..078f704c771 100644 --- a/dbms/src/Functions/neighbor.cpp +++ b/dbms/src/Functions/neighbor.cpp @@ -147,7 +147,7 @@ public: if (offset == 0) { /// Degenerate case, just copy source column as is. - block.getByPosition(result).column = source_column_casted; /// TODO + block.getByPosition(result).column = source_is_constant ? ColumnConst::create(source_column_casted, input_rows_count) : source_column_casted; } else if (offset > 0) { diff --git a/dbms/tests/queries/0_stateless/00996_neigbor.reference b/dbms/tests/queries/0_stateless/00996_neigbor.reference new file mode 100644 index 00000000000..ebdb8c9e684 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00996_neigbor.reference @@ -0,0 +1,270 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +0 5 +1 6 +2 7 +3 8 +4 9 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 0 +6 1 +7 2 +8 3 +9 4 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 5 +1 6 +2 7 +3 8 +4 9 +5 Hello +6 Hello +7 Hello +8 Hello +9 Hello +0 World +1 World +2 World +3 World +4 World +5 0 +6 1 +7 2 +8 3 +9 4 +0 5 +1 6 +2 7 +3 8 +4 9 +5 Hello 5 +6 Hello 6 +7 Hello 7 +8 Hello 8 +9 Hello 9 +0 World 0 +1 World 1 +2 World 2 +3 World 3 +4 World 4 +5 0 +6 1 +7 2 +8 3 +9 4 +0 ClickHouse +1 ClickHouse +2 ClickHouse +3 ClickHouse +4 ClickHouse +5 ClickHouse +6 ClickHouse +7 ClickHouse +8 ClickHouse +9 ClickHouse +0 ClickHouse +1 ClickHouse +2 ClickHouse +3 ClickHouse +4 ClickHouse +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 ClickHouse +6 ClickHouse +7 ClickHouse +8 ClickHouse +9 ClickHouse +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 ClickHouse +1 ClickHouse +2 ClickHouse +3 ClickHouse +4 ClickHouse +5 Hello +6 Hello +7 Hello +8 Hello +9 Hello +0 World +1 World +2 World +3 World +4 World +5 ClickHouse +6 ClickHouse +7 ClickHouse +8 ClickHouse +9 ClickHouse +0 ClickHouse +1 ClickHouse +2 ClickHouse +3 ClickHouse +4 ClickHouse +5 Hello 5 +6 Hello 6 +7 Hello 7 +8 Hello 8 +9 Hello 9 +0 World 0 +1 World 1 +2 World 2 +3 World 3 +4 World 4 +5 ClickHouse +6 ClickHouse +7 ClickHouse +8 ClickHouse +9 ClickHouse +0 0 +1 2 +2 4 +3 6 +4 8 +5 +6 +7 +8 +9 +0 0 +1 1 +2 3 +3 4 +4 6 +5 7 +6 9 +7 +8 +9 +0 Hello +1 Hello +2 Hello +3 Hello +4 Hello +5 +6 +7 +8 +9 +0 +1 +2 +3 Hello +4 Hello +5 Hello +6 Hello +7 Hello +8 Hello +9 Hello +0 World +1 World +2 World +3 Hello +4 Hello +5 Hello +6 Hello +7 Hello +8 Hello +9 Hello diff --git a/dbms/tests/queries/0_stateless/00996_neigbor.sql b/dbms/tests/queries/0_stateless/00996_neigbor.sql new file mode 100644 index 00000000000..25c20b1b896 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00996_neigbor.sql @@ -0,0 +1,42 @@ +SELECT number, neighbor(toString(number), 0) FROM numbers(10); + +SELECT number, neighbor(toString(number), 5) FROM numbers(10); +SELECT number, neighbor(toString(number), -5) FROM numbers(10); + +SELECT number, neighbor(toString(number), 10) FROM numbers(10); +SELECT number, neighbor(toString(number), -10) FROM numbers(10); + +SELECT number, neighbor(toString(number), 15) FROM numbers(10); +SELECT number, neighbor(toString(number), -15) FROM numbers(10); + +SELECT number, neighbor(toString(number), 5, 'Hello') FROM numbers(10); +SELECT number, neighbor(toString(number), -5, 'World') FROM numbers(10); + +SELECT number, neighbor(toString(number), 5, concat('Hello ', toString(number))) FROM numbers(10); +SELECT number, neighbor(toString(number), -5, concat('World ', toString(number))) FROM numbers(10); + + +SELECT number, neighbor('ClickHouse', 0) FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 5) FROM numbers(10); +SELECT number, neighbor('ClickHouse', -5) FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 10) FROM numbers(10); +SELECT number, neighbor('ClickHouse', -10) FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 15) FROM numbers(10); +SELECT number, neighbor('ClickHouse', -15) FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 5, 'Hello') FROM numbers(10); +SELECT number, neighbor('ClickHouse', -5, 'World') FROM numbers(10); + +SELECT number, neighbor('ClickHouse', 5, concat('Hello ', toString(number))) FROM numbers(10); +SELECT number, neighbor('ClickHouse', -5, concat('World ', toString(number))) FROM numbers(10); + + +SELECT number, neighbor(toString(number), number) FROM numbers(10); +SELECT number, neighbor(toString(number), intDiv(number, 2)) FROM numbers(10); + +SELECT number, neighbor('Hello', number) FROM numbers(10); +SELECT number, neighbor('Hello', -3) FROM numbers(10); +SELECT number, neighbor('Hello', -3, 'World') FROM numbers(10); From 3d8613f8dfe1750e7f7c5be0b71921db2b2cea2a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 07:36:01 +0300 Subject: [PATCH 193/357] More tests --- .../{00996_neigbor.reference => 00996_neighbor.reference} | 0 .../queries/0_stateless/{00996_neigbor.sql => 00996_neighbor.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename dbms/tests/queries/0_stateless/{00996_neigbor.reference => 00996_neighbor.reference} (100%) rename dbms/tests/queries/0_stateless/{00996_neigbor.sql => 00996_neighbor.sql} (100%) diff --git a/dbms/tests/queries/0_stateless/00996_neigbor.reference b/dbms/tests/queries/0_stateless/00996_neighbor.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00996_neigbor.reference rename to dbms/tests/queries/0_stateless/00996_neighbor.reference diff --git a/dbms/tests/queries/0_stateless/00996_neigbor.sql b/dbms/tests/queries/0_stateless/00996_neighbor.sql similarity index 100% rename from dbms/tests/queries/0_stateless/00996_neigbor.sql rename to dbms/tests/queries/0_stateless/00996_neighbor.sql From 84b0f709aa0c1b26d15e4de0abea45c865fb1934 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 08:08:06 +0300 Subject: [PATCH 194/357] Removed useless code --- dbms/src/Core/Defines.h | 2 - dbms/src/Parsers/ASTAlterQuery.cpp | 43 ------------------ dbms/src/Parsers/ASTAlterQuery.h | 19 -------- dbms/src/Parsers/ASTCreateQuery.cpp | 3 -- dbms/src/Parsers/ASTCreateQuery.h | 1 - dbms/src/Parsers/ParserAlterQuery.cpp | 63 ++------------------------ dbms/src/Parsers/ParserAlterQuery.h | 13 +----- dbms/src/Parsers/ParserCreateQuery.cpp | 47 ++++++------------- dbms/src/Parsers/ParserCreateQuery.h | 2 +- 9 files changed, 20 insertions(+), 173 deletions(-) diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 0c72c926006..a172cf6e243 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -33,8 +33,6 @@ #define DEFAULT_MERGE_BLOCK_SIZE 8192 #define DEFAULT_TEMPORARY_LIVE_VIEW_TIMEOUT_SEC 5 -#define DEFAULT_TEMPORARY_LIVE_CHANNEL_TIMEOUT_SEC 15 -#define DEFAULT_ALTER_LIVE_CHANNEL_WAIT_MS 10000 #define SHOW_CHARS_ON_SYNTAX_ERROR ptrdiff_t(160) #define DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC 15 #define DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE 1024 diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index 54ba460d75a..6cfba1d6a79 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -45,11 +45,6 @@ ASTPtr ASTAlterCommand::clone() const res->ttl = ttl->clone(); res->children.push_back(res->ttl); } - if (values) - { - res->values = values->clone(); - res->children.push_back(res->values); - } return res; } @@ -226,42 +221,6 @@ void ASTAlterCommand::formatImpl( { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "REFRESH " << (settings.hilite ? hilite_none : ""); } - else if (type == ASTAlterCommand::LIVE_CHANNEL_ADD) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ADD " << (settings.hilite ? hilite_none : ""); - - values->formatImpl(settings, state, frame); - } - else if (type == ASTAlterCommand::LIVE_CHANNEL_DROP) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DROP " << (settings.hilite ? hilite_none : ""); - - values->formatImpl(settings, state, frame); - } - else if (type == ASTAlterCommand::LIVE_CHANNEL_MODIFY) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY " << (settings.hilite ? hilite_none : ""); - - values->formatImpl(settings, state, frame); - } - else if (type == ASTAlterCommand::LIVE_CHANNEL_SUSPEND) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "SUSPEND " << (settings.hilite ? hilite_none : ""); - - values->formatImpl(settings, state, frame); - } - else if (type == ASTAlterCommand::LIVE_CHANNEL_RESUME) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "RESUME " << (settings.hilite ? hilite_none : ""); - - values->formatImpl(settings, state, frame); - } - else if (type == ASTAlterCommand::LIVE_CHANNEL_REFRESH) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "REFRESH " << (settings.hilite ? hilite_none : ""); - - values->formatImpl(settings, state, frame); - } else throw Exception("Unexpected type of ALTER", ErrorCodes::UNEXPECTED_AST_STRUCTURE); } @@ -316,8 +275,6 @@ void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState if (is_live_view) settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ALTER LIVE VIEW " << (settings.hilite ? hilite_none : ""); - else if (is_live_channel) - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ALTER LIVE CHANNEL " << (settings.hilite ? hilite_none : ""); else settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ALTER TABLE " << (settings.hilite ? hilite_none : ""); diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 5b04fcdffb9..2563abfac6e 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -17,13 +17,6 @@ namespace DB * COMMENT_COLUMN col_name 'comment', * ALTER LIVE VIEW [db.]name_type * REFRESH - * ALTER CHANNEL [db.]name_type - * ADD live_view,... - * DROP live_view,... - * SUSPEND live_view,... - * RESUME live_view,... - * REFRESH live_view,... - * MODIFY live_view,... */ class ASTAlterCommand : public IAST @@ -59,13 +52,6 @@ public: NO_TYPE, LIVE_VIEW_REFRESH, - - LIVE_CHANNEL_ADD, - LIVE_CHANNEL_DROP, - LIVE_CHANNEL_SUSPEND, - LIVE_CHANNEL_RESUME, - LIVE_CHANNEL_REFRESH, - LIVE_CHANNEL_MODIFY }; Type type = NO_TYPE; @@ -121,10 +107,6 @@ public: /// For MODIFY TTL query ASTPtr ttl; - /** In ALTER CHANNEL, ADD, DROP, SUSPEND, RESUME, REFRESH, MODIFY queries, the list of live views is stored here - */ - ASTPtr values; - bool detach = false; /// true for DETACH PARTITION bool part = false; /// true for ATTACH PART and DROP DETACHED PART @@ -182,7 +164,6 @@ class ASTAlterQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCl { public: bool is_live_view{false}; /// true for ALTER LIVE VIEW - bool is_live_channel{false}; /// true for ALTER LIVE CHANNEL ASTAlterCommandList * command_list = nullptr; diff --git a/dbms/src/Parsers/ASTCreateQuery.cpp b/dbms/src/Parsers/ASTCreateQuery.cpp index 87e9301329d..f22460d8716 100644 --- a/dbms/src/Parsers/ASTCreateQuery.cpp +++ b/dbms/src/Parsers/ASTCreateQuery.cpp @@ -220,9 +220,6 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat what = "MATERIALIZED VIEW"; if (is_live_view) what = "LIVE VIEW"; - if (is_live_channel) - what = "LIVE CHANNEL"; - settings.ostr << (settings.hilite ? hilite_keyword : "") diff --git a/dbms/src/Parsers/ASTCreateQuery.h b/dbms/src/Parsers/ASTCreateQuery.h index 3893fa2c82a..5ca0c067a3c 100644 --- a/dbms/src/Parsers/ASTCreateQuery.h +++ b/dbms/src/Parsers/ASTCreateQuery.h @@ -57,7 +57,6 @@ public: bool is_view{false}; bool is_materialized_view{false}; bool is_live_view{false}; - bool is_live_channel{false}; bool is_populate{false}; bool replace_view{false}; /// CREATE OR REPLACE VIEW ASTColumns * columns_list = nullptr; diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 41b046e2877..21a4fd586c6 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -89,53 +89,6 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected else return false; } - else if (is_live_channel) - { - if (s_add.ignore(pos, expected)) - { - if (!values_p.parse(pos, command->values, expected)) - return false; - - command->type = ASTAlterCommand::LIVE_CHANNEL_ADD; - } - else if (s_drop.ignore(pos, expected)) - { - if (!values_p.parse(pos, command->values, expected)) - return false; - - command->type = ASTAlterCommand::LIVE_CHANNEL_DROP; - } - else if (s_suspend.ignore(pos, expected)) - { - if (!values_p.parse(pos, command->values, expected)) - return false; - - command->type = ASTAlterCommand::LIVE_CHANNEL_SUSPEND; - } - else if (s_resume.ignore(pos, expected)) - { - if (!values_p.parse(pos, command->values, expected)) - return false; - - command->type = ASTAlterCommand::LIVE_CHANNEL_RESUME; - } - else if (s_refresh.ignore(pos, expected)) - { - if (!values_p.parse(pos, command->values, expected)) - return false; - - command->type = ASTAlterCommand::LIVE_CHANNEL_REFRESH; - } - else if (s_modify.ignore(pos, expected)) - { - if (!values_p.parse(pos, command->values, expected)) - return false; - - command->type = ASTAlterCommand::LIVE_CHANNEL_MODIFY; - } - else - return false; - } else { if (s_add_column.ignore(pos, expected)) @@ -466,7 +419,7 @@ bool ParserAlterCommandList::parseImpl(Pos & pos, ASTPtr & node, Expected & expe node = command_list; ParserToken s_comma(TokenType::Comma); - ParserAlterCommand p_command(is_live_view, is_live_channel); + ParserAlterCommand p_command(is_live_view); do { @@ -516,20 +469,13 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_alter_table("ALTER TABLE"); ParserKeyword s_alter_live_view("ALTER LIVE VIEW"); - ParserKeyword s_alter_live_channel("ALTER LIVE CHANNEL"); bool is_live_view = false; - bool is_live_channel = false; if (!s_alter_table.ignore(pos, expected)) { if (!s_alter_live_view.ignore(pos, expected)) - { - if (!s_alter_live_channel.ignore(pos, expected)) - return false; - else - is_live_channel = true; - } + return false; else is_live_view = true; } @@ -537,9 +483,6 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (is_live_view) query->is_live_view = true; - if (is_live_channel) - query->is_live_channel = true; - if (!parseDatabaseAndTableName(pos, expected, query->database, query->table)) return false; @@ -551,7 +494,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } query->cluster = cluster_str; - ParserAlterCommandList p_command_list(is_live_view, is_live_channel); + ParserAlterCommandList p_command_list(is_live_view); ASTPtr command_list; if (!p_command_list.parse(pos, command_list, expected)) return false; diff --git a/dbms/src/Parsers/ParserAlterQuery.h b/dbms/src/Parsers/ParserAlterQuery.h index 13f5681a9da..450e64ee9f4 100644 --- a/dbms/src/Parsers/ParserAlterQuery.h +++ b/dbms/src/Parsers/ParserAlterQuery.h @@ -21,13 +21,6 @@ namespace DB * [UPDATE col_name = expr, ... WHERE ...] * ALTER LIVE VIEW [db.name] * [REFRESH] - * ALTER LIVE CHANNEL [db.name] [ON CLUSTER cluster] - * [ADD live_view, ...] - * [DROP live_view, ...] - * [SUSPEND live_view, ...] - * [RESUME live_view, ...] - * [REFRESH live_view, ...] - * [MODIFY live_view, ...] */ class ParserAlterQuery : public IParserBase @@ -46,9 +39,8 @@ protected: public: bool is_live_view; - bool is_live_channel; - ParserAlterCommandList(bool is_live_view_ = false, bool is_live_channel_ = false) : is_live_view(is_live_view_), is_live_channel(is_live_channel_) {} + ParserAlterCommandList(bool is_live_view_ = false) : is_live_view(is_live_view_) {} }; @@ -60,9 +52,8 @@ protected: public: bool is_live_view; - bool is_live_channel; - ParserAlterCommand(bool is_live_view_ = false, bool is_live_channel_ = false) : is_live_view(is_live_view_), is_live_channel(is_live_channel_) {} + ParserAlterCommand(bool is_live_view_ = false) : is_live_view(is_live_view_) {} }; diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index d2cbd920d0f..acbf2fb89ba 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -365,7 +365,6 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_with("WITH"); ParserKeyword s_materialized("MATERIALIZED"); ParserKeyword s_live("LIVE"); - ParserKeyword s_channel("CHANNEL"); ParserKeyword s_populate("POPULATE"); ParserKeyword s_or_replace("OR REPLACE"); ParserToken s_dot(TokenType::Dot); @@ -396,7 +395,6 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool is_view = false; bool is_materialized_view = false; bool is_live_view = false; - bool is_live_channel = false; bool is_populate = false; bool is_temporary = false; bool replace_view = false; @@ -494,9 +492,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } else if (s_live.ignore(pos, expected)) { - if (s_channel.ignore(pos, expected)) - is_live_channel = true; - else if (s_view.ignore(pos, expected)) + if (s_view.ignore(pos, expected)) is_live_view = true; else return false; @@ -520,50 +516,36 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; } - if (!is_live_channel) + // TO [db.]table + if (ParserKeyword{"TO"}.ignore(pos, expected)) { - // TO [db.]table - if (ParserKeyword{"TO"}.ignore(pos, expected)) + if (!name_p.parse(pos, to_table, expected)) + return false; + + if (s_dot.ignore(pos, expected)) { + to_database = to_table; if (!name_p.parse(pos, to_table, expected)) return false; - - if (s_dot.ignore(pos, expected)) - { - to_database = to_table; - if (!name_p.parse(pos, to_table, expected)) - return false; - } } } /// Optional - a list of columns can be specified. It must fully comply with SELECT. if (s_lparen.ignore(pos, expected)) { - if (!columns_or_indices_p.parse(pos, columns_list, expected)) + if (!table_properties_p.parse(pos, columns_list, expected)) return false; if (!s_rparen.ignore(pos, expected)) return false; } - if (is_live_channel) - { - if (s_with.ignore(pos, expected)) - { - if (!names_p.parse(pos, tables, expected)) - return false; - } - } - else - { - /// AS SELECT ... - if (!s_as.ignore(pos, expected)) - return false; + /// AS SELECT ... + if (!s_as.ignore(pos, expected)) + return false; - if (!select_p.parse(pos, select, expected)) - return false; - } + if (!select_p.parse(pos, select, expected)) + return false; } else if (is_temporary) return false; @@ -673,7 +655,6 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->is_view = is_view; query->is_materialized_view = is_materialized_view; query->is_live_view = is_live_view; - query->is_live_channel = is_live_channel; query->is_populate = is_populate; query->temporary = is_temporary; query->replace_view = replace_view; diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index 1199c712585..a4f4da8907e 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -321,7 +321,7 @@ protected: * CREATE|ATTACH DATABASE db [ENGINE = engine] * * Or: - * CREATE[OR REPLACE]|ATTACH [[MATERIALIZED] VIEW] | [[TEMPORARY] LIVE [CHANNEL] | [VIEW]] [IF NOT EXISTS] [db.]name [TO [db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... + * CREATE[OR REPLACE]|ATTACH [[MATERIALIZED] VIEW] | [VIEW]] [IF NOT EXISTS] [db.]name [TO [db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... */ class ParserCreateQuery : public IParserBase { From dae2aa61387a619cf5db94d8a046d5c3353fde37 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 08:11:11 +0300 Subject: [PATCH 195/357] Removed useless code --- dbms/src/Parsers/ASTAlterQuery.cpp | 5 +++++ dbms/src/Parsers/ASTAlterQuery.h | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index 6cfba1d6a79..e8fa3630442 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -45,6 +45,11 @@ ASTPtr ASTAlterCommand::clone() const res->ttl = ttl->clone(); res->children.push_back(res->ttl); } + if (values) + { + res->values = values->clone(); + res->children.push_back(res->values); + } return res; } diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 2563abfac6e..a4962087a9b 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -107,6 +107,10 @@ public: /// For MODIFY TTL query ASTPtr ttl; + /** In ALTER CHANNEL, ADD, DROP, SUSPEND, RESUME, REFRESH, MODIFY queries, the list of live views is stored here + */ + ASTPtr values; + bool detach = false; /// true for DETACH PARTITION bool part = false; /// true for ATTACH PART and DROP DETACHED PART From 1cc8bf7fe040afe05978bec43fab525bd85e68d7 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Fri, 23 Aug 2019 12:11:43 +0300 Subject: [PATCH 196/357] DOCAPI-7783: RU Translation (#6623) --- docs/ru/query_language/misc.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/ru/query_language/misc.md b/docs/ru/query_language/misc.md index 93f548bf73c..e366abf2f3d 100644 --- a/docs/ru/query_language/misc.md +++ b/docs/ru/query_language/misc.md @@ -193,18 +193,21 @@ RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... Все таблицы переименовываются под глобальной блокировкой. Переименовывание таблицы является лёгкой операцией. Если вы указали после TO другую базу данных, то таблица будет перенесена в эту базу данных. При этом, директории с базами данных должны быть расположены в одной файловой системе (иначе возвращается ошибка). -## SET +## SET {#query-set} ```sql SET param = value ``` -Позволяет установить настройку `param` в значение `value`. Также можно одним запросом установить все настройки из заданного профиля настроек. Для этого укажите 'profile' в качестве имени настройки. Подробнее смотрите в разделе "Настройки". -Настройка устанавливается на сессию, или на сервер (глобально), если указано `GLOBAL`. -При установке глобальных настроек, эти настройки не применяются к уже запущенной сессии, включая текущую сессию. Она будет использована только для новых сессий. +Устанавливает значение `value` для настройки `param` в текущей сессии. [Конфигурационные параметры сервера](../operations/server_settings/index.md) нельзя изменить подобным образом. -При перезапуске сервера теряются настройки, установленные с помощью `SET`. -Установить настройки, которые переживут перезапуск сервера, можно только с помощью конфигурационного файла сервера. +Можно одним запросом установить все настройки из заданного профиля настроек. + +```sql +SET profile = 'profile-name-from-the-settings-file' +``` + +Подробности смотрите в разделе [Настройки](../operations/settings/settings.md). ## SHOW CREATE TABLE From baf121c864550cdcafdcd70b9b2ab36afc7e9c53 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Fri, 23 Aug 2019 18:14:19 +0800 Subject: [PATCH 197/357] Translate database engine documentation(zh) (#6625) --- docs/zh/database_engines/index.md | 12 +- docs/zh/database_engines/mysql.md | 125 +++++++++++++++++- docs/zh/operations/table_engines/mergetree.md | 2 +- 3 files changed, 136 insertions(+), 3 deletions(-) diff --git a/docs/zh/database_engines/index.md b/docs/zh/database_engines/index.md index bbdb762a4ad..f8ae05e2520 120000 --- a/docs/zh/database_engines/index.md +++ b/docs/zh/database_engines/index.md @@ -1 +1,11 @@ -../../en/database_engines/index.md \ No newline at end of file +# 数据库引擎 + +您使用的所有表都是由数据库引擎所提供的 + +默认情况下，ClickHouse使用自己的数据库引擎，该引擎提供可配置的[表引擎](../operations/table_engines/index.md)和[所有支持的SQL语法](../query_language/syntax.md). + +除此之外，您还可以选择使用以下的数据库引擎： + +- [MySQL](mysql.md) + +[来源文章](https://clickhouse.yandex/docs/en/database_engines/) diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md index 51ac4126e2d..38dfcb5ef64 120000 --- a/docs/zh/database_engines/mysql.md +++ b/docs/zh/database_engines/mysql.md @@ -1 +1,124 @@ -../../en/database_engines/mysql.md \ No newline at end of file +# MySQL + +MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并允许您对表进行`INSERT`和`SELECT`查询，以方便您在ClickHouse与MySQL之间进行数据交换。 + +`MySQL`数据库引擎会将对其的查询转换为MySQL语法并发送到MySQL服务器中，因此您可以执行诸如`SHOW TABLES`或`SHOW CREATE TABLE`之类的操作。 + +但您无法对其执行以下操作： + +- `ATTACH`/`DETACH` +- `DROP` +- `RENAME` +- `CREATE TABLE` +- `ALTER` + + +## CREATE DATABASE + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +ENGINE = MySQL('host:port', 'database', 'user', 'password') +``` + +**MySQL数据库引擎参数** + +- `host:port` — 链接的MySQL地址。 +- `database` — 链接的MySQL数据库。 +- `user` — 链接的MySQL用户。 +- `password` — 链接的MySQL用户密码。 + + +## 支持的类型对应 + +MySQL | ClickHouse +------|------------ +UNSIGNED TINYINT | [UInt8](../data_types/int_uint.md) +TINYINT | [Int8](../data_types/int_uint.md) +UNSIGNED SMALLINT | [UInt16](../data_types/int_uint.md) +SMALLINT | [Int16](../data_types/int_uint.md) +UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../data_types/int_uint.md) +INT, MEDIUMINT | [Int32](../data_types/int_uint.md) +UNSIGNED BIGINT | [UInt64](../data_types/int_uint.md) +BIGINT | [Int64](../data_types/int_uint.md) +FLOAT | [Float32](../data_types/float.md) +DOUBLE | [Float64](../data_types/float.md) +DATE | [Date](../data_types/date.md) +DATETIME, TIMESTAMP | [DateTime](../data_types/datetime.md) +BINARY | [FixedString](../data_types/fixedstring.md) + +其他的MySQL数据类型将全部都转换为[String](../data_types/string.md)。 + +同时以上的所有类型都支持[Nullable](../data_types/nullable.md)。 + + +## 使用示例 + +在MySQL中创建表: + +``` +mysql> USE test; +Database changed + +mysql> CREATE TABLE `mysql_table` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `float` FLOAT NOT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from mysql_table; ++--------+-------+ +| int_id | value | ++--------+-------+ +| 1 | 2 | ++--------+-------+ +1 row in set (0,00 sec) +``` + +在ClickHouse中创建MySQL类型的数据库，同时与MySQL服务器交换数据： + +```sql +CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') +``` +```sql +SHOW DATABASES +``` +```text +┌─name─────┐ +│ default │ +│ mysql_db │ +│ system │ +└──────────┘ +``` +```sql +SHOW TABLES FROM mysql_db +``` +```text +┌─name─────────┐ +│ mysql_table │ +└──────────────┘ +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +└────────┴───────┘ +``` +```sql +INSERT INTO mysql_db.mysql_table VALUES (3,4) +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +│ 3 │ 4 │ +└────────┴───────┘ +``` + +[来源文章](https://clickhouse.yandex/docs/en/database_engines/mysql/) diff --git a/docs/zh/operations/table_engines/mergetree.md b/docs/zh/operations/table_engines/mergetree.md index 5ddf837708a..5e330164c5a 100644 --- a/docs/zh/operations/table_engines/mergetree.md +++ b/docs/zh/operations/table_engines/mergetree.md @@ -48,7 +48,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **子句** -- `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`. `MergeTree` 引擎没有参数。 +- `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`。 `MergeTree` 引擎不需要其他参数。 - `PARTITION BY` — [分区键](custom_partitioning_key.md) 。 From a367f94923c13445d5ed46d4bfe58d7b94950b69 Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Fri, 23 Aug 2019 16:19:12 +0300 Subject: [PATCH 198/357] Fix after review --- dbms/programs/benchmark/Benchmark.cpp | 3 ++- dbms/programs/client/Client.cpp | 16 +++++------ dbms/programs/compressor/Compressor.cpp | 5 ++-- dbms/programs/format/Format.cpp | 4 +-- dbms/programs/local/LocalServer.cpp | 13 ++------- dbms/programs/obfuscator/Obfuscator.cpp | 3 ++- .../performance-test/PerformanceTestSuite.cpp | 4 +-- ...nsDescription.h => TerminalDisplaying.cpp} | 27 ++++++++++--------- dbms/src/Common/TerminalDisplaying.h | 17 ++++++++++++ 9 files changed, 52 insertions(+), 40 deletions(-) rename dbms/src/Common/{SetOptionsDescription.h => TerminalDisplaying.cpp} (55%) create mode 100644 dbms/src/Common/TerminalDisplaying.h diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index c69e9a54feb..7fc33559b54 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -32,6 +32,7 @@ #include #include #include +#include /** A tool for evaluating ClickHouse performance. @@ -439,7 +440,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) { using boost::program_options::value; - boost::program_options::options_description desc("Allowed options"); + boost::program_options::options_description desc = setOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help", "produce help message") ("concurrency,c", value()->default_value(1), "number of parallel queries") diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 726a607812c..f77ddbea827 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -67,7 +67,7 @@ #include #include #include -#include +#include #if USE_READLINE #include "Suggest.h" @@ -131,7 +131,7 @@ private: bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode. bool stdin_is_not_tty = false; /// stdin is not a terminal. - winsize terminal_size {}; /// Terminal size is needed to render progress bar. + unsigned short int terminal_width; /// Terminal width is needed to render progress bar. std::unique_ptr connection; /// Connection to DB. String query_id; /// Current query_id. @@ -1466,7 +1466,7 @@ private: if (show_progress_bar) { - ssize_t width_of_progress_bar = static_cast(terminal_size.ws_col) - written_progress_chars - strlen(" 99%"); + ssize_t width_of_progress_bar = static_cast(terminal_width) - written_progress_chars - strlen(" 99%"); if (width_of_progress_bar > 0) { std::string bar = UnicodeBar::render(UnicodeBar::getWidth(progress.read_rows, 0, total_rows_corrected, width_of_progress_bar)); @@ -1642,16 +1642,14 @@ public: } stdin_is_not_tty = !isatty(STDIN_FILENO); + if (!stdin_is_not_tty) - { - if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) - throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", ErrorCodes::SYSTEM_ERROR); - } + terminal_width = getTerminalWidth(); namespace po = boost::program_options; /// Main commandline options related to client functionality and all parameters from Settings. - po::options_description main_description = setOptionsDescription("Main options"); + po::options_description main_description = setOptionsDescription("Main options", terminal_width); main_description.add_options() ("help", "produce help message") ("config-file,C", po::value(), "config-file path") @@ -1697,7 +1695,7 @@ public: context.getSettingsRef().addProgramOptions(main_description); /// Commandline options related to external tables. - po::options_description external_description("External tables options"); + po::options_description external_description = setOptionsDescription("External tables options", terminal_width); external_description.add_options() ("file", po::value(), "data file or - for stdin") ("name", po::value()->default_value("_data"), "name of the table") diff --git a/dbms/programs/compressor/Compressor.cpp b/dbms/programs/compressor/Compressor.cpp index 427d58cbdc6..f990b3ac690 100644 --- a/dbms/programs/compressor/Compressor.cpp +++ b/dbms/programs/compressor/Compressor.cpp @@ -12,8 +12,9 @@ #include #include #include - #include +#include + namespace DB { @@ -59,7 +60,7 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out) int mainEntryClickHouseCompressor(int argc, char ** argv) { - boost::program_options::options_description desc("Allowed options"); + boost::program_options::options_description desc = setOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help,h", "produce help message") ("decompress,d", "decompress") diff --git a/dbms/programs/format/Format.cpp b/dbms/programs/format/Format.cpp index b7e2629df16..fbc17e44e63 100644 --- a/dbms/programs/format/Format.cpp +++ b/dbms/programs/format/Format.cpp @@ -6,13 +6,13 @@ #include #include #include - +#include int mainEntryClickHouseFormat(int argc, char ** argv) { using namespace DB; - boost::program_options::options_description desc("Allowed options"); + boost::program_options::options_description desc = setOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help,h", "produce help message") ("hilite", "add syntax highlight with ANSI terminal escape sequences") diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index bed55a0fc5f..c894e13ed47 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -35,6 +35,7 @@ #include #include #include +#include namespace DB @@ -409,17 +410,7 @@ void LocalServer::init(int argc, char ** argv) /// Don't parse options with Poco library, we prefer neat boost::program_options stopOptionsProcessing(); - unsigned line_length = po::options_description::m_default_line_length; - unsigned min_description_length = line_length / 2; - if (isatty(STDIN_FILENO)) - { - winsize terminal_size{}; - ioctl(0, TIOCGWINSZ, &terminal_size); - line_length = std::max(3U, static_cast(terminal_size.ws_col)); - min_description_length = std::min(min_description_length, line_length - 2); - } - - po::options_description description("Main options", line_length, min_description_length); + po::options_description description = setOptionsDescription("Main options", getTerminalWidth()); description.add_options() ("help", "produce help message") ("config-file,c", po::value(), "config-file path") diff --git a/dbms/programs/obfuscator/Obfuscator.cpp b/dbms/programs/obfuscator/Obfuscator.cpp index a96c10072dc..a2cae4ed1a5 100644 --- a/dbms/programs/obfuscator/Obfuscator.cpp +++ b/dbms/programs/obfuscator/Obfuscator.cpp @@ -36,6 +36,7 @@ #include #include #include +#include static const char * documantation = R"( @@ -948,7 +949,7 @@ try using namespace DB; namespace po = boost::program_options; - po::options_description description("Options"); + po::options_description description = setOptionsDescription("Options", getTerminalWidth()); description.add_options() ("help", "produce help message") ("structure,S", po::value(), "structure of the initial table (list of column and type names)") diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp index 99408c72f37..37a9e61d169 100644 --- a/dbms/programs/performance-test/PerformanceTestSuite.cpp +++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include "TestStopConditions.h" #include "TestStats.h" @@ -325,7 +325,7 @@ try using po::value; using Strings = DB::Strings; - po::options_description desc = setOptionsDescription("Allowed options"); + po::options_description desc = setOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help", "produce help message") ("lite", "use lite version of output") diff --git a/dbms/src/Common/SetOptionsDescription.h b/dbms/src/Common/TerminalDisplaying.cpp similarity index 55% rename from dbms/src/Common/SetOptionsDescription.h rename to dbms/src/Common/TerminalDisplaying.cpp index a9c4031b064..d9a8f5ebfa9 100644 --- a/dbms/src/Common/SetOptionsDescription.h +++ b/dbms/src/Common/TerminalDisplaying.cpp @@ -1,8 +1,7 @@ #include #include #include -#include - +#include namespace po = boost::program_options; @@ -12,11 +11,8 @@ namespace DB::ErrorCodes extern const int SYSTEM_ERROR; } -static po::options_description setOptionsDescription(const std::string & caption) +unsigned short int getTerminalWidth() { - unsigned line_length = po::options_description::m_default_line_length; - unsigned min_description_length = line_length / 2; - if (isatty(STDIN_FILENO)) { winsize terminal_size {}; @@ -24,12 +20,19 @@ static po::options_description setOptionsDescription(const std::string & caption if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) DB::throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", DB::ErrorCodes::SYSTEM_ERROR); - std::string longest_option_desc = "--http_native_compression_disable_checksumming_on_decompress"; - - line_length = std::max(static_cast(longest_option_desc.size()), terminal_size.ws_col); - - min_description_length = std::min(min_description_length, line_length - 2); + return terminal_size.ws_col; } + return 0; +} + +po::options_description setOptionsDescription(const std::string & caption, unsigned short terminal_width) +{ + unsigned line_length = po::options_description::m_default_line_length; + unsigned min_description_length = line_length / 2; + std::string longest_option_desc = "--http_native_compression_disable_checksumming_on_decompress"; + + line_length = std::max(static_cast(longest_option_desc.size()), terminal_width); + min_description_length = std::min(min_description_length, line_length - 2); return po::options_description(caption, line_length, min_description_length); -} +} \ No newline at end of file diff --git a/dbms/src/Common/TerminalDisplaying.h b/dbms/src/Common/TerminalDisplaying.h new file mode 100644 index 00000000000..598969b62c7 --- /dev/null +++ b/dbms/src/Common/TerminalDisplaying.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + + +namespace po = boost::program_options; + + +unsigned short int getTerminalWidth(); + +/** Sets a name and an appropriate size for option displaying + * when program is called with option --help + * */ +po::options_description setOptionsDescription(const std::string & caption, unsigned short terminal_width); + From 6e098eb953c29dfb51fd81b35f06a4ff625ef52b Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Fri, 23 Aug 2019 16:25:46 +0300 Subject: [PATCH 199/357] New line added :) --- dbms/src/Common/TerminalDisplaying.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/TerminalDisplaying.cpp b/dbms/src/Common/TerminalDisplaying.cpp index d9a8f5ebfa9..243ba7f8288 100644 --- a/dbms/src/Common/TerminalDisplaying.cpp +++ b/dbms/src/Common/TerminalDisplaying.cpp @@ -35,4 +35,4 @@ po::options_description setOptionsDescription(const std::string & caption, unsig min_description_length = std::min(min_description_length, line_length - 2); return po::options_description(caption, line_length, min_description_length); -} \ No newline at end of file +} From d001c7e554ccedfba05b271a91687e918ab4e6d3 Mon Sep 17 00:00:00 2001 From: Alex Ryndin Date: Fri, 23 Aug 2019 17:27:07 +0300 Subject: [PATCH 200/357] Fix typo (#6631) --- docs/en/operations/settings/query_complexity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md index 77699c868b4..c00f2132ebd 100644 --- a/docs/en/operations/settings/query_complexity.md +++ b/docs/en/operations/settings/query_complexity.md @@ -79,7 +79,7 @@ Enables or disables execution of `GROUP BY` clauses in external memory. See [GRO Possible values: -- Maximum volume or RAM (in bytes) that can be used by the single [GROUP BY](../../query_language/select.md#select-group-by-clause) operation. +- Maximum volume of RAM (in bytes) that can be used by the single [GROUP BY](../../query_language/select.md#select-group-by-clause) operation. - 0 — `GROUP BY` in external memory disabled. Default value: 0. From 9e6c629134b14952a7c3b143809ffdf75bee8b0b Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Fri, 23 Aug 2019 17:54:10 +0300 Subject: [PATCH 201/357] Style fixed --- dbms/src/Common/T_test.h | 206 ++++++++++++++++++++------------------- 1 file changed, 104 insertions(+), 102 deletions(-) diff --git a/dbms/src/Common/T_test.h b/dbms/src/Common/T_test.h index 456ed464e8f..1160fd0805e 100644 --- a/dbms/src/Common/T_test.h +++ b/dbms/src/Common/T_test.h @@ -50,110 +50,112 @@ struct T_test std::vector data; - const std::vector> students_table = + /// First row corresponds to infinity size of distributions case + const double students_table[101][6] = { - /* inf */ { 1.282, 1.645, 1.960, 2.326, 2.576, 3.090 }, - /* 1. */ { 3.078, 6.314, 12.706, 31.821, 63.657, 318.313}, - /* 2. */ { 1.886, 2.920, 4.303, 6.965, 9.925, 22.327 }, - /* 3. */ { 1.638, 2.353, 3.182, 4.541, 5.841, 10.215 }, - /* 4. */ { 1.533, 2.132, 2.776, 3.747, 4.604, 7.173 }, - /* 5. */ { 1.476, 2.015, 2.571, 3.365, 4.032, 5.893 }, - /* 6. */ { 1.440, 1.943, 2.447, 3.143, 3.707, 5.208 }, - /* 7. */ { 1.415, 1.895, 2.365, 2.998, 3.499, 4.782 }, - /* 8. */ { 1.397, 1.860, 2.306, 2.896, 3.355, 4.499 }, - /* 9. */ { 1.383, 1.833, 2.262, 2.821, 3.250, 4.296 }, - /* 10. */ { 1.372, 1.812, 2.228, 2.764, 3.169, 4.143 }, - /* 11. */ { 1.363, 1.796, 2.201, 2.718, 3.106, 4.024 }, - /* 12. */ { 1.356, 1.782, 2.179, 2.681, 3.055, 3.929 }, - /* 13. */ { 1.350, 1.771, 2.160, 2.650, 3.012, 3.852 }, - /* 14. */ { 1.345, 1.761, 2.145, 2.624, 2.977, 3.787 }, - /* 15. */ { 1.341, 1.753, 2.131, 2.602, 2.947, 3.733 }, - /* 16. */ { 1.337, 1.746, 2.120, 2.583, 2.921, 3.686 }, - /* 17. */ { 1.333, 1.740, 2.110, 2.567, 2.898, 3.646 }, - /* 18. */ { 1.330, 1.734, 2.101, 2.552, 2.878, 3.610 }, - /* 19. */ { 1.328, 1.729, 2.093, 2.539, 2.861, 3.579 }, - /* 20. */ { 1.325, 1.725, 2.086, 2.528, 2.845, 3.552 }, - /* 21. */ { 1.323, 1.721, 2.080, 2.518, 2.831, 3.527 }, - /* 22. */ { 1.321, 1.717, 2.074, 2.508, 2.819, 3.505 }, - /* 23. */ { 1.319, 1.714, 2.069, 2.500, 2.807, 3.485 }, - /* 24. */ { 1.318, 1.711, 2.064, 2.492, 2.797, 3.467 }, - /* 25. */ { 1.316, 1.708, 2.060, 2.485, 2.787, 3.450 }, - /* 26. */ { 1.315, 1.706, 2.056, 2.479, 2.779, 3.435 }, - /* 27. */ { 1.314, 1.703, 2.052, 2.473, 2.771, 3.421 }, - /* 28. */ { 1.313, 1.701, 2.048, 2.467, 2.763, 3.408 }, - /* 29. */ { 1.311, 1.699, 2.045, 2.462, 2.756, 3.396 }, - /* 30. */ { 1.310, 1.697, 2.042, 2.457, 2.750, 3.385 }, - /* 31. */ { 1.309, 1.696, 2.040, 2.453, 2.744, 3.375 }, - /* 32. */ { 1.309, 1.694, 2.037, 2.449, 2.738, 3.365 }, - /* 33. */ { 1.308, 1.692, 2.035, 2.445, 2.733, 3.356 }, - /* 34. */ { 1.307, 1.691, 2.032, 2.441, 2.728, 3.348 }, - /* 35. */ { 1.306, 1.690, 2.030, 2.438, 2.724, 3.340 }, - /* 36. */ { 1.306, 1.688, 2.028, 2.434, 2.719, 3.333 }, - /* 37. */ { 1.305, 1.687, 2.026, 2.431, 2.715, 3.326 }, - /* 38. */ { 1.304, 1.686, 2.024, 2.429, 2.712, 3.319 }, - /* 39. */ { 1.304, 1.685, 2.023, 2.426, 2.708, 3.313 }, - /* 40. */ { 1.303, 1.684, 2.021, 2.423, 2.704, 3.307 }, - /* 41. */ { 1.303, 1.683, 2.020, 2.421, 2.701, 3.301 }, - /* 42. */ { 1.302, 1.682, 2.018, 2.418, 2.698, 3.296 }, - /* 43. */ { 1.302, 1.681, 2.017, 2.416, 2.695, 3.291 }, - /* 44. */ { 1.301, 1.680, 2.015, 2.414, 2.692, 3.286 }, - /* 45. */ { 1.301, 1.679, 2.014, 2.412, 2.690, 3.281 }, - /* 46. */ { 1.300, 1.679, 2.013, 2.410, 2.687, 3.277 }, - /* 47. */ { 1.300, 1.678, 2.012, 2.408, 2.685, 3.273 }, - /* 48. */ { 1.299, 1.677, 2.011, 2.407, 2.682, 3.269 }, - /* 49. */ { 1.299, 1.677, 2.010, 2.405, 2.680, 3.265 }, - /* 50. */ { 1.299, 1.676, 2.009, 2.403, 2.678, 3.261 }, - /* 51. */ { 1.298, 1.675, 2.008, 2.402, 2.676, 3.258 }, - /* 52. */ { 1.298, 1.675, 2.007, 2.400, 2.674, 3.255 }, - /* 53. */ { 1.298, 1.674, 2.006, 2.399, 2.672, 3.251 }, - /* 54. */ { 1.297, 1.674, 2.005, 2.397, 2.670, 3.248 }, - /* 55. */ { 1.297, 1.673, 2.004, 2.396, 2.668, 3.245 }, - /* 56. */ { 1.297, 1.673, 2.003, 2.395, 2.667, 3.242 }, - /* 57. */ { 1.297, 1.672, 2.002, 2.394, 2.665, 3.239 }, - /* 58. */ { 1.296, 1.672, 2.002, 2.392, 2.663, 3.237 }, - /* 59. */ { 1.296, 1.671, 2.001, 2.391, 2.662, 3.234 }, - /* 60. */ { 1.296, 1.671, 2.000, 2.390, 2.660, 3.232 }, - /* 61. */ { 1.296, 1.670, 2.000, 2.389, 2.659, 3.229 }, - /* 62. */ { 1.295, 1.670, 1.999, 2.388, 2.657, 3.227 }, - /* 63. */ { 1.295, 1.669, 1.998, 2.387, 2.656, 3.225 }, - /* 64. */ { 1.295, 1.669, 1.998, 2.386, 2.655, 3.223 }, - /* 65. */ { 1.295, 1.669, 1.997, 2.385, 2.654, 3.220 }, - /* 66. */ { 1.295, 1.668, 1.997, 2.384, 2.652, 3.218 }, - /* 67. */ { 1.294, 1.668, 1.996, 2.383, 2.651, 3.216 }, - /* 68. */ { 1.294, 1.668, 1.995, 2.382, 2.650, 3.214 }, - /* 69. */ { 1.294, 1.667, 1.995, 2.382, 2.649, 3.213 }, - /* 70. */ { 1.294, 1.667, 1.994, 2.381, 2.648, 3.211 }, - /* 71. */ { 1.294, 1.667, 1.994, 2.380, 2.647, 3.209 }, - /* 72. */ { 1.293, 1.666, 1.993, 2.379, 2.646, 3.207 }, - /* 73. */ { 1.293, 1.666, 1.993, 2.379, 2.645, 3.206 }, - /* 74. */ { 1.293, 1.666, 1.993, 2.378, 2.644, 3.204 }, - /* 75. */ { 1.293, 1.665, 1.992, 2.377, 2.643, 3.202 }, - /* 76. */ { 1.293, 1.665, 1.992, 2.376, 2.642, 3.201 }, - /* 77. */ { 1.293, 1.665, 1.991, 2.376, 2.641, 3.199 }, - /* 78. */ { 1.292, 1.665, 1.991, 2.375, 2.640, 3.198 }, - /* 79. */ { 1.292, 1.664, 1.990, 2.374, 2.640, 3.197 }, - /* 80. */ { 1.292, 1.664, 1.990, 2.374, 2.639, 3.195 }, - /* 81. */ { 1.292, 1.664, 1.990, 2.373, 2.638, 3.194 }, - /* 82. */ { 1.292, 1.664, 1.989, 2.373, 2.637, 3.193 }, - /* 83. */ { 1.292, 1.663, 1.989, 2.372, 2.636, 3.191 }, - /* 84. */ { 1.292, 1.663, 1.989, 2.372, 2.636, 3.190 }, - /* 85. */ { 1.292, 1.663, 1.988, 2.371, 2.635, 3.189 }, - /* 86. */ { 1.291, 1.663, 1.988, 2.370, 2.634, 3.188 }, - /* 87. */ { 1.291, 1.663, 1.988, 2.370, 2.634, 3.187 }, - /* 88. */ { 1.291, 1.662, 1.987, 2.369, 2.633, 3.185 }, - /* 89. */ { 1.291, 1.662, 1.987, 2.369, 2.632, 3.184 }, - /* 90. */ { 1.291, 1.662, 1.987, 2.368, 2.632, 3.183 }, - /* 91. */ { 1.291, 1.662, 1.986, 2.368, 2.631, 3.182 }, - /* 92. */ { 1.291, 1.662, 1.986, 2.368, 2.630, 3.181 }, - /* 93. */ { 1.291, 1.661, 1.986, 2.367, 2.630, 3.180 }, - /* 94. */ { 1.291, 1.661, 1.986, 2.367, 2.629, 3.179 }, - /* 95. */ { 1.291, 1.661, 1.985, 2.366, 2.629, 3.178 }, - /* 96. */ { 1.290, 1.661, 1.985, 2.366, 2.628, 3.177 }, - /* 97. */ { 1.290, 1.661, 1.985, 2.365, 2.627, 3.176 }, - /* 98. */ { 1.290, 1.661, 1.984, 2.365, 2.627, 3.175 }, - /* 99. */ { 1.290, 1.660, 1.984, 2.365, 2.626, 3.175 }, - /* 100. */ { 1.290, 1.660, 1.984, 2.364, 2.626, 3.174 } + { 1.282, 1.645, 1.960, 2.326, 2.576, 3.090 }, + { 3.078, 6.314, 12.706, 31.821, 63.657, 318.313 }, + { 1.886, 2.920, 4.303, 6.965, 9.925, 22.327 }, + { 1.638, 2.353, 3.182, 4.541, 5.841, 10.215 }, + { 1.533, 2.132, 2.776, 3.747, 4.604, 7.173 }, + { 1.476, 2.015, 2.571, 3.365, 4.032, 5.893 }, + { 1.440, 1.943, 2.447, 3.143, 3.707, 5.208 }, + { 1.415, 1.895, 2.365, 2.998, 3.499, 4.782 }, + { 1.397, 1.860, 2.306, 2.896, 3.355, 4.499 }, + { 1.383, 1.833, 2.262, 2.821, 3.250, 4.296 }, + { 1.372, 1.812, 2.228, 2.764, 3.169, 4.143 }, + { 1.363, 1.796, 2.201, 2.718, 3.106, 4.024 }, + { 1.356, 1.782, 2.179, 2.681, 3.055, 3.929 }, + { 1.350, 1.771, 2.160, 2.650, 3.012, 3.852 }, + { 1.345, 1.761, 2.145, 2.624, 2.977, 3.787 }, + { 1.341, 1.753, 2.131, 2.602, 2.947, 3.733 }, + { 1.337, 1.746, 2.120, 2.583, 2.921, 3.686 }, + { 1.333, 1.740, 2.110, 2.567, 2.898, 3.646 }, + { 1.330, 1.734, 2.101, 2.552, 2.878, 3.610 }, + { 1.328, 1.729, 2.093, 2.539, 2.861, 3.579 }, + { 1.325, 1.725, 2.086, 2.528, 2.845, 3.552 }, + { 1.323, 1.721, 2.080, 2.518, 2.831, 3.527 }, + { 1.321, 1.717, 2.074, 2.508, 2.819, 3.505 }, + { 1.319, 1.714, 2.069, 2.500, 2.807, 3.485 }, + { 1.318, 1.711, 2.064, 2.492, 2.797, 3.467 }, + { 1.316, 1.708, 2.060, 2.485, 2.787, 3.450 }, + { 1.315, 1.706, 2.056, 2.479, 2.779, 3.435 }, + { 1.314, 1.703, 2.052, 2.473, 2.771, 3.421 }, + { 1.313, 1.701, 2.048, 2.467, 2.763, 3.408 }, + { 1.311, 1.699, 2.045, 2.462, 2.756, 3.396 }, + { 1.310, 1.697, 2.042, 2.457, 2.750, 3.385 }, + { 1.309, 1.696, 2.040, 2.453, 2.744, 3.375 }, + { 1.309, 1.694, 2.037, 2.449, 2.738, 3.365 }, + { 1.308, 1.692, 2.035, 2.445, 2.733, 3.356 }, + { 1.307, 1.691, 2.032, 2.441, 2.728, 3.348 }, + { 1.306, 1.690, 2.030, 2.438, 2.724, 3.340 }, + { 1.306, 1.688, 2.028, 2.434, 2.719, 3.333 }, + { 1.305, 1.687, 2.026, 2.431, 2.715, 3.326 }, + { 1.304, 1.686, 2.024, 2.429, 2.712, 3.319 }, + { 1.304, 1.685, 2.023, 2.426, 2.708, 3.313 }, + { 1.303, 1.684, 2.021, 2.423, 2.704, 3.307 }, + { 1.303, 1.683, 2.020, 2.421, 2.701, 3.301 }, + { 1.302, 1.682, 2.018, 2.418, 2.698, 3.296 }, + { 1.302, 1.681, 2.017, 2.416, 2.695, 3.291 }, + { 1.301, 1.680, 2.015, 2.414, 2.692, 3.286 }, + { 1.301, 1.679, 2.014, 2.412, 2.690, 3.281 }, + { 1.300, 1.679, 2.013, 2.410, 2.687, 3.277 }, + { 1.300, 1.678, 2.012, 2.408, 2.685, 3.273 }, + { 1.299, 1.677, 2.011, 2.407, 2.682, 3.269 }, + { 1.299, 1.677, 2.010, 2.405, 2.680, 3.265 }, + { 1.299, 1.676, 2.009, 2.403, 2.678, 3.261 }, + { 1.298, 1.675, 2.008, 2.402, 2.676, 3.258 }, + { 1.298, 1.675, 2.007, 2.400, 2.674, 3.255 }, + { 1.298, 1.674, 2.006, 2.399, 2.672, 3.251 }, + { 1.297, 1.674, 2.005, 2.397, 2.670, 3.248 }, + { 1.297, 1.673, 2.004, 2.396, 2.668, 3.245 }, + { 1.297, 1.673, 2.003, 2.395, 2.667, 3.242 }, + { 1.297, 1.672, 2.002, 2.394, 2.665, 3.239 }, + { 1.296, 1.672, 2.002, 2.392, 2.663, 3.237 }, + { 1.296, 1.671, 2.001, 2.391, 2.662, 3.234 }, + { 1.296, 1.671, 2.000, 2.390, 2.660, 3.232 }, + { 1.296, 1.670, 2.000, 2.389, 2.659, 3.229 }, + { 1.295, 1.670, 1.999, 2.388, 2.657, 3.227 }, + { 1.295, 1.669, 1.998, 2.387, 2.656, 3.225 }, + { 1.295, 1.669, 1.998, 2.386, 2.655, 3.223 }, + { 1.295, 1.669, 1.997, 2.385, 2.654, 3.220 }, + { 1.295, 1.668, 1.997, 2.384, 2.652, 3.218 }, + { 1.294, 1.668, 1.996, 2.383, 2.651, 3.216 }, + { 1.294, 1.668, 1.995, 2.382, 2.650, 3.214 }, + { 1.294, 1.667, 1.995, 2.382, 2.649, 3.213 }, + { 1.294, 1.667, 1.994, 2.381, 2.648, 3.211 }, + { 1.294, 1.667, 1.994, 2.380, 2.647, 3.209 }, + { 1.293, 1.666, 1.993, 2.379, 2.646, 3.207 }, + { 1.293, 1.666, 1.993, 2.379, 2.645, 3.206 }, + { 1.293, 1.666, 1.993, 2.378, 2.644, 3.204 }, + { 1.293, 1.665, 1.992, 2.377, 2.643, 3.202 }, + { 1.293, 1.665, 1.992, 2.376, 2.642, 3.201 }, + { 1.293, 1.665, 1.991, 2.376, 2.641, 3.199 }, + { 1.292, 1.665, 1.991, 2.375, 2.640, 3.198 }, + { 1.292, 1.664, 1.990, 2.374, 2.640, 3.197 }, + { 1.292, 1.664, 1.990, 2.374, 2.639, 3.195 }, + { 1.292, 1.664, 1.990, 2.373, 2.638, 3.194 }, + { 1.292, 1.664, 1.989, 2.373, 2.637, 3.193 }, + { 1.292, 1.663, 1.989, 2.372, 2.636, 3.191 }, + { 1.292, 1.663, 1.989, 2.372, 2.636, 3.190 }, + { 1.292, 1.663, 1.988, 2.371, 2.635, 3.189 }, + { 1.291, 1.663, 1.988, 2.370, 2.634, 3.188 }, + { 1.291, 1.663, 1.988, 2.370, 2.634, 3.187 }, + { 1.291, 1.662, 1.987, 2.369, 2.633, 3.185 }, + { 1.291, 1.662, 1.987, 2.369, 2.632, 3.184 }, + { 1.291, 1.662, 1.987, 2.368, 2.632, 3.183 }, + { 1.291, 1.662, 1.986, 2.368, 2.631, 3.182 }, + { 1.291, 1.662, 1.986, 2.368, 2.630, 3.181 }, + { 1.291, 1.661, 1.986, 2.367, 2.630, 3.180 }, + { 1.291, 1.661, 1.986, 2.367, 2.629, 3.179 }, + { 1.291, 1.661, 1.985, 2.366, 2.629, 3.178 }, + { 1.290, 1.661, 1.985, 2.366, 2.628, 3.177 }, + { 1.290, 1.661, 1.985, 2.365, 2.627, 3.176 }, + { 1.290, 1.661, 1.984, 2.365, 2.627, 3.175 }, + { 1.290, 1.660, 1.984, 2.365, 2.626, 3.175 }, + { 1.290, 1.660, 1.984, 2.364, 2.626, 3.174 }, }; + const std::vector confidence_level = { 80, 90, 95, 98, 99, 99.5 }; T_test() From f757de462584ec8cad5d6cba4c73f3fe2b480324 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 23 Aug 2019 17:56:38 +0300 Subject: [PATCH 202/357] Revert "Translate database engine documentation(zh) (#6625)" This reverts commit baf121c864550cdcafdcd70b9b2ab36afc7e9c53. --- docs/zh/database_engines/index.md | 12 +- docs/zh/database_engines/mysql.md | 125 +----------------- docs/zh/operations/table_engines/mergetree.md | 2 +- 3 files changed, 3 insertions(+), 136 deletions(-) diff --git a/docs/zh/database_engines/index.md b/docs/zh/database_engines/index.md index f8ae05e2520..bbdb762a4ad 120000 --- a/docs/zh/database_engines/index.md +++ b/docs/zh/database_engines/index.md @@ -1,11 +1 @@ -# 数据库引擎 - -您使用的所有表都是由数据库引擎所提供的 - -默认情况下，ClickHouse使用自己的数据库引擎，该引擎提供可配置的[表引擎](../operations/table_engines/index.md)和[所有支持的SQL语法](../query_language/syntax.md). - -除此之外，您还可以选择使用以下的数据库引擎： - -- [MySQL](mysql.md) - -[来源文章](https://clickhouse.yandex/docs/en/database_engines/) +../../en/database_engines/index.md \ No newline at end of file diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md index 38dfcb5ef64..51ac4126e2d 120000 --- a/docs/zh/database_engines/mysql.md +++ b/docs/zh/database_engines/mysql.md @@ -1,124 +1 @@ -# MySQL - -MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并允许您对表进行`INSERT`和`SELECT`查询，以方便您在ClickHouse与MySQL之间进行数据交换。 - -`MySQL`数据库引擎会将对其的查询转换为MySQL语法并发送到MySQL服务器中，因此您可以执行诸如`SHOW TABLES`或`SHOW CREATE TABLE`之类的操作。 - -但您无法对其执行以下操作： - -- `ATTACH`/`DETACH` -- `DROP` -- `RENAME` -- `CREATE TABLE` -- `ALTER` - - -## CREATE DATABASE - -``` sql -CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MySQL('host:port', 'database', 'user', 'password') -``` - -**MySQL数据库引擎参数** - -- `host:port` — 链接的MySQL地址。 -- `database` — 链接的MySQL数据库。 -- `user` — 链接的MySQL用户。 -- `password` — 链接的MySQL用户密码。 - - -## 支持的类型对应 - -MySQL | ClickHouse -------|------------ -UNSIGNED TINYINT | [UInt8](../data_types/int_uint.md) -TINYINT | [Int8](../data_types/int_uint.md) -UNSIGNED SMALLINT | [UInt16](../data_types/int_uint.md) -SMALLINT | [Int16](../data_types/int_uint.md) -UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../data_types/int_uint.md) -INT, MEDIUMINT | [Int32](../data_types/int_uint.md) -UNSIGNED BIGINT | [UInt64](../data_types/int_uint.md) -BIGINT | [Int64](../data_types/int_uint.md) -FLOAT | [Float32](../data_types/float.md) -DOUBLE | [Float64](../data_types/float.md) -DATE | [Date](../data_types/date.md) -DATETIME, TIMESTAMP | [DateTime](../data_types/datetime.md) -BINARY | [FixedString](../data_types/fixedstring.md) - -其他的MySQL数据类型将全部都转换为[String](../data_types/string.md)。 - -同时以上的所有类型都支持[Nullable](../data_types/nullable.md)。 - - -## 使用示例 - -在MySQL中创建表: - -``` -mysql> USE test; -Database changed - -mysql> CREATE TABLE `mysql_table` ( - -> `int_id` INT NOT NULL AUTO_INCREMENT, - -> `float` FLOAT NOT NULL, - -> PRIMARY KEY (`int_id`)); -Query OK, 0 rows affected (0,09 sec) - -mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); -Query OK, 1 row affected (0,00 sec) - -mysql> select * from mysql_table; -+--------+-------+ -| int_id | value | -+--------+-------+ -| 1 | 2 | -+--------+-------+ -1 row in set (0,00 sec) -``` - -在ClickHouse中创建MySQL类型的数据库，同时与MySQL服务器交换数据： - -```sql -CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') -``` -```sql -SHOW DATABASES -``` -```text -┌─name─────┐ -│ default │ -│ mysql_db │ -│ system │ -└──────────┘ -``` -```sql -SHOW TABLES FROM mysql_db -``` -```text -┌─name─────────┐ -│ mysql_table │ -└──────────────┘ -``` -```sql -SELECT * FROM mysql_db.mysql_table -``` -```text -┌─int_id─┬─value─┐ -│ 1 │ 2 │ -└────────┴───────┘ -``` -```sql -INSERT INTO mysql_db.mysql_table VALUES (3,4) -``` -```sql -SELECT * FROM mysql_db.mysql_table -``` -```text -┌─int_id─┬─value─┐ -│ 1 │ 2 │ -│ 3 │ 4 │ -└────────┴───────┘ -``` - -[来源文章](https://clickhouse.yandex/docs/en/database_engines/mysql/) +../../en/database_engines/mysql.md \ No newline at end of file diff --git a/docs/zh/operations/table_engines/mergetree.md b/docs/zh/operations/table_engines/mergetree.md index 5e330164c5a..5ddf837708a 100644 --- a/docs/zh/operations/table_engines/mergetree.md +++ b/docs/zh/operations/table_engines/mergetree.md @@ -48,7 +48,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **子句** -- `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`。 `MergeTree` 引擎不需要其他参数。 +- `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`. `MergeTree` 引擎没有参数。 - `PARTITION BY` — [分区键](custom_partitioning_key.md) 。 From abdd70fcc4b02297618c8ed29751e6bbff917fa1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 18:01:36 +0300 Subject: [PATCH 203/357] Fixed "splitted" build --- contrib/arrow-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index bc229deeced..843ff9cd8af 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -219,7 +219,7 @@ endif() add_library(${ARROW_LIBRARY} ${ARROW_SRCS}) add_dependencies(${ARROW_LIBRARY} protoc) target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS}) -target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${PROTOBUF_LIBRARIES} Threads::Threads) +target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY} Threads::Threads) if (ARROW_WITH_LZ4) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY}) endif() From 75e124f3909d1a6820938986f9d7f82b05acb309 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 18:05:27 +0300 Subject: [PATCH 204/357] Removed misleading flag from CMake --- CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ac4d67f6ae..f84a181a39c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,11 +264,10 @@ if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}") endif () -# TODO: only make this extra-checks in CI builds, since a lot of contrib libs won't link - -# CI works around this problem by explicitly adding GLIBC_COMPATIBILITY flag. -if (NOT SANITIZE AND YANDEX_OFFICIAL_BUILD) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") +# Make this extra-checks for correct library dependencies. +if (NOT SANITIZE) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") endif () include (cmake/find_unwind.cmake) From 5fd649e663e126813b074f2c819baca8a6eedc7c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 18:10:33 +0300 Subject: [PATCH 205/357] Check for broken symlinks #6625 --- utils/check-style/check-style | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index fed4b6b8670..ef3bf6cfad4 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -25,3 +25,6 @@ find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' | find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' | grep -vP 'Compiler|build' | xargs grep $@ -P '}\s*//+\s*namespace\s*' + +# Broken symlinks +find -L $ROOT_PATH -type l | grep -v contrib && echo "^ Broken symlinks found" From 016b1ee2f77646806d95aa28b562ab7c8e0fd41e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 18:42:45 +0300 Subject: [PATCH 206/357] Increase timeout for "stack overflow" test because it may take a long time in debug build --- dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh | 2 ++ dbms/tests/queries/shell_config.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh b/dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh index 64fae3fb0f9..14f2a8e31fb 100755 --- a/dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh +++ b/dbms/tests/queries/0_stateless/00984_parser_stack_overflow.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +CLICKHOUSE_CURL_TIMEOUT=30 + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh diff --git a/dbms/tests/queries/shell_config.sh b/dbms/tests/queries/shell_config.sh index d4ab11be927..b3058a6cdbe 100644 --- a/dbms/tests/queries/shell_config.sh +++ b/dbms/tests/queries/shell_config.sh @@ -46,7 +46,8 @@ export CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:="9009"} export CLICKHOUSE_URL_INTERSERVER=${CLICKHOUSE_URL_INTERSERVER:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_INTERSERVER}/"} export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"} -export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} --max-time 10"} +export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="10"} +export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."} mkdir -p ${CLICKHOUSE_TMP} From 859d97e8fdf4ef3d51d95de4047ee94621f19e0f Mon Sep 17 00:00:00 2001 From: dimarub2000 Date: Fri, 23 Aug 2019 18:47:27 +0300 Subject: [PATCH 207/357] Renaming --- dbms/programs/benchmark/Benchmark.cpp | 4 ++-- dbms/programs/client/Client.cpp | 6 +++--- dbms/programs/compressor/Compressor.cpp | 4 ++-- dbms/programs/format/Format.cpp | 4 ++-- dbms/programs/local/LocalServer.cpp | 4 ++-- dbms/programs/obfuscator/Obfuscator.cpp | 4 ++-- dbms/programs/performance-test/PerformanceTestSuite.cpp | 4 ++-- .../src/Common/{TerminalDisplaying.cpp => TerminalSize.cpp} | 4 ++-- dbms/src/Common/{TerminalDisplaying.h => TerminalSize.h} | 4 ++-- 9 files changed, 19 insertions(+), 19 deletions(-) rename dbms/src/Common/{TerminalDisplaying.cpp => TerminalSize.cpp} (87%) rename dbms/src/Common/{TerminalDisplaying.h => TerminalSize.h} (52%) diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index 7fc33559b54..f56e3e6da2c 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -32,7 +32,7 @@ #include #include #include -#include +#include /** A tool for evaluating ClickHouse performance. @@ -440,7 +440,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) { using boost::program_options::value; - boost::program_options::options_description desc = setOptionsDescription("Allowed options", getTerminalWidth()); + boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help", "produce help message") ("concurrency,c", value()->default_value(1), "number of parallel queries") diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index f77ddbea827..3032c9a5a57 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -67,7 +67,7 @@ #include #include #include -#include +#include #if USE_READLINE #include "Suggest.h" @@ -1649,7 +1649,7 @@ public: namespace po = boost::program_options; /// Main commandline options related to client functionality and all parameters from Settings. - po::options_description main_description = setOptionsDescription("Main options", terminal_width); + po::options_description main_description = createOptionsDescription("Main options", terminal_width); main_description.add_options() ("help", "produce help message") ("config-file,C", po::value(), "config-file path") @@ -1695,7 +1695,7 @@ public: context.getSettingsRef().addProgramOptions(main_description); /// Commandline options related to external tables. - po::options_description external_description = setOptionsDescription("External tables options", terminal_width); + po::options_description external_description = createOptionsDescription("External tables options", terminal_width); external_description.add_options() ("file", po::value(), "data file or - for stdin") ("name", po::value()->default_value("_data"), "name of the table") diff --git a/dbms/programs/compressor/Compressor.cpp b/dbms/programs/compressor/Compressor.cpp index f990b3ac690..a073a79b416 100644 --- a/dbms/programs/compressor/Compressor.cpp +++ b/dbms/programs/compressor/Compressor.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include namespace DB @@ -60,7 +60,7 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out) int mainEntryClickHouseCompressor(int argc, char ** argv) { - boost::program_options::options_description desc = setOptionsDescription("Allowed options", getTerminalWidth()); + boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help,h", "produce help message") ("decompress,d", "decompress") diff --git a/dbms/programs/format/Format.cpp b/dbms/programs/format/Format.cpp index fbc17e44e63..ff415d88e1b 100644 --- a/dbms/programs/format/Format.cpp +++ b/dbms/programs/format/Format.cpp @@ -6,13 +6,13 @@ #include #include #include -#include +#include int mainEntryClickHouseFormat(int argc, char ** argv) { using namespace DB; - boost::program_options::options_description desc = setOptionsDescription("Allowed options", getTerminalWidth()); + boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help,h", "produce help message") ("hilite", "add syntax highlight with ANSI terminal escape sequences") diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index c894e13ed47..54383050b6c 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -35,7 +35,7 @@ #include #include #include -#include +#include namespace DB @@ -410,7 +410,7 @@ void LocalServer::init(int argc, char ** argv) /// Don't parse options with Poco library, we prefer neat boost::program_options stopOptionsProcessing(); - po::options_description description = setOptionsDescription("Main options", getTerminalWidth()); + po::options_description description = createOptionsDescription("Main options", getTerminalWidth()); description.add_options() ("help", "produce help message") ("config-file,c", po::value(), "config-file path") diff --git a/dbms/programs/obfuscator/Obfuscator.cpp b/dbms/programs/obfuscator/Obfuscator.cpp index a2cae4ed1a5..0fb6b495982 100644 --- a/dbms/programs/obfuscator/Obfuscator.cpp +++ b/dbms/programs/obfuscator/Obfuscator.cpp @@ -36,7 +36,7 @@ #include #include #include -#include +#include static const char * documantation = R"( @@ -949,7 +949,7 @@ try using namespace DB; namespace po = boost::program_options; - po::options_description description = setOptionsDescription("Options", getTerminalWidth()); + po::options_description description = createOptionsDescription("Options", getTerminalWidth()); description.add_options() ("help", "produce help message") ("structure,S", po::value(), "structure of the initial table (list of column and type names)") diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp index 37a9e61d169..eaa4e24cde9 100644 --- a/dbms/programs/performance-test/PerformanceTestSuite.cpp +++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include "TestStopConditions.h" #include "TestStats.h" @@ -325,7 +325,7 @@ try using po::value; using Strings = DB::Strings; - po::options_description desc = setOptionsDescription("Allowed options", getTerminalWidth()); + po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() ("help", "produce help message") ("lite", "use lite version of output") diff --git a/dbms/src/Common/TerminalDisplaying.cpp b/dbms/src/Common/TerminalSize.cpp similarity index 87% rename from dbms/src/Common/TerminalDisplaying.cpp rename to dbms/src/Common/TerminalSize.cpp index 243ba7f8288..ffc301cf77c 100644 --- a/dbms/src/Common/TerminalDisplaying.cpp +++ b/dbms/src/Common/TerminalSize.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include namespace po = boost::program_options; @@ -25,7 +25,7 @@ unsigned short int getTerminalWidth() return 0; } -po::options_description setOptionsDescription(const std::string & caption, unsigned short terminal_width) +po::options_description createOptionsDescription(const std::string &caption, unsigned short terminal_width) { unsigned line_length = po::options_description::m_default_line_length; unsigned min_description_length = line_length / 2; diff --git a/dbms/src/Common/TerminalDisplaying.h b/dbms/src/Common/TerminalSize.h similarity index 52% rename from dbms/src/Common/TerminalDisplaying.h rename to dbms/src/Common/TerminalSize.h index 598969b62c7..4dd58b5c8ca 100644 --- a/dbms/src/Common/TerminalDisplaying.h +++ b/dbms/src/Common/TerminalSize.h @@ -10,8 +10,8 @@ namespace po = boost::program_options; unsigned short int getTerminalWidth(); -/** Sets a name and an appropriate size for option displaying +/** Creates po::options_description with name and an appropriate size for option displaying * when program is called with option --help * */ -po::options_description setOptionsDescription(const std::string & caption, unsigned short terminal_width); +po::options_description createOptionsDescription(const std::string &caption, unsigned short terminal_width); From 50b927a9d74ed6de672ec9cc58e230c545c6ce43 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 23 Aug 2019 19:08:27 +0300 Subject: [PATCH 208/357] Update StringSearcher.h --- dbms/src/Common/StringSearcher.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/StringSearcher.h b/dbms/src/Common/StringSearcher.h index fecf1a7ca81..3cb6e56ab78 100644 --- a/dbms/src/Common/StringSearcher.h +++ b/dbms/src/Common/StringSearcher.h @@ -717,7 +717,7 @@ public: { if (std::any_of(reinterpret_cast(needle_), reinterpret_cast(needle_) + needle_size_, isTokenSeparator)) { - throw Exception{"needle must not contain whitespace characters", ErrorCodes::BAD_ARGUMENTS}; + throw Exception{"Needle must not contain whitespace or separator characters", ErrorCodes::BAD_ARGUMENTS}; } } From c781e1c6a7640ddf29bb0af52608fa97a1793736 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 23 Aug 2019 19:09:24 +0300 Subject: [PATCH 209/357] Update StringSearcher.h --- dbms/src/Common/StringSearcher.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Common/StringSearcher.h b/dbms/src/Common/StringSearcher.h index 3cb6e56ab78..25287db11f5 100644 --- a/dbms/src/Common/StringSearcher.h +++ b/dbms/src/Common/StringSearcher.h @@ -160,7 +160,7 @@ public: #endif } - ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const + ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const { static const Poco::UTF8Encoding utf8; @@ -377,7 +377,7 @@ public: #endif } - ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const + ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const { #ifdef __SSE4_1__ if (pageSafe(pos)) @@ -570,7 +570,7 @@ public: #endif } - ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const + ALWAYS_INLINE bool compare(const UInt8 * /*haystack*/, const UInt8 * /*haystack_end*/, const UInt8 * pos) const { #ifdef __SSE4_1__ if (pageSafe(pos)) @@ -722,7 +722,7 @@ public: } - ALWAYS_INLINE bool compare(const UInt8 * haystack, const UInt8 * haystack_end, const UInt8 * pos) const + ALWAYS_INLINE bool compare(const UInt8 * haystack, const UInt8 * haystack_end, const UInt8 * pos) const { // use searcher only if pos is in the beginning of token and pos + searcher.needle_size is end of token. if (isToken(haystack, haystack_end, pos)) From b42f85e16bd683b1f03fe4a0833370c742e1445b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 21:30:04 +0300 Subject: [PATCH 210/357] Added a check for double whitespaces --- dbms/src/AggregateFunctions/QuantileExact.h | 2 +- dbms/src/DataTypes/DataTypeEnum.cpp | 2 +- dbms/src/Functions/GeoUtils.cpp | 2 +- .../Formats/Impl/ProtobufRowInputFormat.cpp | 2 +- .../MergeTree/MergeTreeRangeReader.cpp | 2 +- .../Storages/MergeTree/MergeTreeRangeReader.h | 2 +- .../Storages/System/StorageSystemNumbers.cpp | 2 +- utils/check-style/check-style | 3 ++ utils/check-style/double-whitespaces.pl | 33 +++++++++++++++++++ 9 files changed, 43 insertions(+), 7 deletions(-) create mode 100755 utils/check-style/double-whitespaces.pl diff --git a/dbms/src/AggregateFunctions/QuantileExact.h b/dbms/src/AggregateFunctions/QuantileExact.h index 5a1343b1399..4a2aa574ae9 100644 --- a/dbms/src/AggregateFunctions/QuantileExact.h +++ b/dbms/src/AggregateFunctions/QuantileExact.h @@ -176,7 +176,7 @@ struct QuantileExactExclusive : public QuantileExact } }; -/// QuantileExactInclusive is equivalent to Excel PERCENTILE and PERCENTILE.INC, R-7, SciPy-(1,1) +/// QuantileExactInclusive is equivalent to Excel PERCENTILE and PERCENTILE.INC, R-7, SciPy-(1,1) template struct QuantileExactInclusive : public QuantileExact { diff --git a/dbms/src/DataTypes/DataTypeEnum.cpp b/dbms/src/DataTypes/DataTypeEnum.cpp index cffc29feaf8..add7052195a 100644 --- a/dbms/src/DataTypes/DataTypeEnum.cpp +++ b/dbms/src/DataTypes/DataTypeEnum.cpp @@ -234,7 +234,7 @@ void DataTypeEnum::deserializeBinaryBulk( } template -void DataTypeEnum::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const +void DataTypeEnum::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const { if (value_index) return; diff --git a/dbms/src/Functions/GeoUtils.cpp b/dbms/src/Functions/GeoUtils.cpp index 5134343dae0..847d934c6b4 100644 --- a/dbms/src/Functions/GeoUtils.cpp +++ b/dbms/src/Functions/GeoUtils.cpp @@ -18,7 +18,7 @@ const UInt8 geohash_base32_decode_lookup_table[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 10, 11, 12, 13, 14, 15, 16, 0xFF, 17, 18, 0xFF, 19, 20, 0xFF, diff --git a/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp index 25fecc5c642..1cd9d329c9d 100644 --- a/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp @@ -12,7 +12,7 @@ namespace DB ProtobufRowInputFormat::ProtobufRowInputFormat(ReadBuffer & in_, const Block & header_, Params params_, const FormatSchemaInfo & info_) : IRowInputFormat(header_, in_, params_) - , data_types(header_.getDataTypes()) + , data_types(header_.getDataTypes()) , reader(in, ProtobufSchemas::instance().getMessageTypeForFormatSchema(info_), header_.getNames()) { } diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 2aae847217e..932721eb028 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -108,7 +108,7 @@ size_t MergeTreeRangeReader::DelayedStream::finalize(Block & block) MergeTreeRangeReader::Stream::Stream( - size_t from_mark, size_t to_mark, MergeTreeReader * merge_tree_reader_) + size_t from_mark, size_t to_mark, MergeTreeReader * merge_tree_reader_) : current_mark(from_mark), offset_after_current_mark(0) , last_mark(to_mark) , merge_tree_reader(merge_tree_reader_) diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h index 9552373901c..0eae69ee17e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -41,7 +41,7 @@ public: { public: DelayedStream() = default; - DelayedStream(size_t from_mark, MergeTreeReader * merge_tree_reader); + DelayedStream(size_t from_mark, MergeTreeReader * merge_tree_reader); /// Read @num_rows rows from @from_mark starting from @offset row /// Returns the number of rows added to block. diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index 2afe2a7c018..2f155e22a11 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -146,7 +146,7 @@ BlockInputStreams StorageSystemNumbers::read( res[i] = std::make_shared(max_block_size, offset + i * max_block_size, num_streams * max_block_size); if (limit) /// This formula is how to split 'limit' elements to 'num_streams' chunks almost uniformly. - res[i] = std::make_shared(res[i], *limit * (i + 1) / num_streams - *limit * i / num_streams, 0, false, true); + res[i] = std::make_shared(res[i], *limit * (i + 1) / num_streams - *limit * i / num_streams, 0, false, true); } return res; diff --git a/utils/check-style/check-style b/utils/check-style/check-style index ef3bf6cfad4..deed481f043 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -28,3 +28,6 @@ find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' | # Broken symlinks find -L $ROOT_PATH -type l | grep -v contrib && echo "^ Broken symlinks found" + +# Double whitespaces +find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' | while read i; do $ROOT_PATH/utils/check-style/double-whitespaces.pl < $i || echo -e "^ File $i contains double whitespaces\n"; done diff --git a/utils/check-style/double-whitespaces.pl b/utils/check-style/double-whitespaces.pl new file mode 100755 index 00000000000..47b03cb74ab --- /dev/null +++ b/utils/check-style/double-whitespaces.pl @@ -0,0 +1,33 @@ +#!/usr/bin/perl + +use strict; + +# Find double whitespace such as "a, b, c" that looks very ugly and annoying. +# But skip double whitespaces if they are used as an alignment - by comparing to surrounding lines. + +my @array; + +while (<>) +{ + push @array, $_; +} + +my $ret = 0; + +for (my $i = 1; $i < $#array; ++$i) +{ + if ($array[$i] =~ ',( {2,3})[^ /]') + { + # https://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl + + if ((substr($array[$i - 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) # whitespaces are not part of alignment + && (substr($array[$i + 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) + && $array[$i] !~ /(-?\d+\w*,\s+){3,}/) # this is not a number table like { 10, -1, 2 } + { + print(($i + 1) . ":" . $array[$i]); + $ret = 1; + } + } +} + +exit $ret; From a21b43913fd97d0a97232764ad2f338da93a9561 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 23 Aug 2019 21:40:42 +0300 Subject: [PATCH 211/357] fix crash in OptimizedRegularExpression --- dbms/src/Common/OptimizedRegularExpression.cpp | 5 +++-- .../0_stateless/00997_extract_all_crash_6627.reference | 1 + .../queries/0_stateless/00997_extract_all_crash_6627.sql | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.reference create mode 100644 dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.sql diff --git a/dbms/src/Common/OptimizedRegularExpression.cpp b/dbms/src/Common/OptimizedRegularExpression.cpp index c87d87fc2df..3a224709447 100644 --- a/dbms/src/Common/OptimizedRegularExpression.cpp +++ b/dbms/src/Common/OptimizedRegularExpression.cpp @@ -1,4 +1,5 @@ #include +#include #include #define MIN_LENGTH_FOR_STRSTR 3 @@ -413,9 +414,9 @@ unsigned OptimizedRegularExpressionImpl::match(const char * subject return 0; } - StringPieceType pieces[MAX_SUBPATTERNS]; + DB::PODArrayWithStackMemory pieces(limit); - if (!re2->Match(StringPieceType(subject, subject_size), 0, subject_size, RegexType::UNANCHORED, pieces, limit)) + if (!re2->Match(StringPieceType(subject, subject_size), 0, subject_size, RegexType::UNANCHORED, pieces.data(), pieces.size())) return 0; else { diff --git a/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.reference b/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.reference new file mode 100644 index 00000000000..acb53e80e6d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.reference @@ -0,0 +1 @@ +['9'] diff --git a/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.sql b/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.sql new file mode 100644 index 00000000000..06de4ec8afb --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_extract_all_crash_6627.sql @@ -0,0 +1 @@ +SELECT extractAll('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.143 YaBrowser/19.7.2.455 Yowser/2.5 Safari/537.36', '[Y][a-zA-Z]{8}/[1-9]([1-9]+)?(((.?)([0-9]+)?){0,4})?'); From 7dffa0fe9fbca08d6f25bc83b53fdee1ffeadbf5 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Fri, 23 Aug 2019 22:19:36 +0300 Subject: [PATCH 212/357] added wait for mutation to indices tests --- dbms/tests/queries/0_stateless/00942_mutate_index.sh | 4 ++-- dbms/tests/queries/0_stateless/00943_materialize_index.sh | 6 ++++++ .../queries/0_stateless/00944_clear_index_in_partition.sh | 3 ++- .../00975_indices_mutation_replicated_zookeeper.sh | 7 +++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00942_mutate_index.sh b/dbms/tests/queries/0_stateless/00942_mutate_index.sh index c6dd1dfb836..467eb9ab671 100755 --- a/dbms/tests/queries/0_stateless/00942_mutate_index.sh +++ b/dbms/tests/queries/0_stateless/00942_mutate_index.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;" @@ -35,8 +36,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 1;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 5;" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx UPDATE i64 = 5 WHERE i64 = 1;" - -sleep 0.1 +wait_for_mutation "minmax_idx" "mutation_2.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 1;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 5;" diff --git a/dbms/tests/queries/0_stateless/00943_materialize_index.sh b/dbms/tests/queries/0_stateless/00943_materialize_index.sh index f51b66993aa..ab2fd1e5355 100755 --- a/dbms/tests/queries/0_stateless/00943_materialize_index.sh +++ b/dbms/tests/queries/0_stateless/00943_materialize_index.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;" @@ -38,22 +39,27 @@ SET allow_experimental_data_skipping_indices=1; ALTER TABLE test.minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1;" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" +wait_for_mutation "minmax_idx" "mutation_1.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 2;" +wait_for_mutation "minmax_idx" "mutation_2.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 1;" +wait_for_mutation "minmax_idx" "mutation_3.txt" "test" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 2;" +wait_for_mutation "minmax_idx" "mutation_4.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx;" +wait_for_mutation "minmax_idx" "mutation_5.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh index 9047bbb3a72..7d68bac8c83 100755 --- a/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh +++ b/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;" @@ -42,7 +43,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" -sleep 0.5 +wait_for_mutation "minmax_idx" "mutation_1.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index 613226a3fb7..1bcb4f17edd 100755 --- a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -2,6 +2,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.indices_mutaions1;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.indices_mutaions2;" @@ -47,13 +48,15 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" -sleep 1 +wait_for_mutation "indices_mutaions1" "mutation_1.txt" "test" +wait_for_mutation "indices_mutaions2" "mutation_1.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" -sleep 1 +wait_for_mutation "indices_mutaions1" "mutation_2.txt" "test" +wait_for_mutation "indices_mutaions2" "mutation_2.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" From ba2d17c12a68ede2bb47e34fdfbdd6ea0ec33a8e Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Fri, 23 Aug 2019 22:36:17 +0300 Subject: [PATCH 213/357] fix --- .../0_stateless/00943_materialize_index.sh | 17 ++++++++--------- .../00944_clear_index_in_partition.sh | 2 +- ...975_indices_mutation_replicated_zookeeper.sh | 3 +-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00943_materialize_index.sh b/dbms/tests/queries/0_stateless/00943_materialize_index.sh index ab2fd1e5355..feab59b368e 100755 --- a/dbms/tests/queries/0_stateless/00943_materialize_index.sh +++ b/dbms/tests/queries/0_stateless/00943_materialize_index.sh @@ -39,27 +39,26 @@ SET allow_experimental_data_skipping_indices=1; ALTER TABLE test.minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1;" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" -wait_for_mutation "minmax_idx" "mutation_1.txt" "test" - -$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" - -$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 2;" wait_for_mutation "minmax_idx" "mutation_2.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 1;" +$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 2;" wait_for_mutation "minmax_idx" "mutation_3.txt" "test" + +$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 1;" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 2;" -wait_for_mutation "minmax_idx" "mutation_4.txt" "test" +sleep 0.5 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx;" -wait_for_mutation "minmax_idx" "mutation_5.txt" "test" +wait_for_mutation "minmax_idx" "mutation_4.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh index 7d68bac8c83..5a7bdd8e3ae 100755 --- a/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh +++ b/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh @@ -43,7 +43,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" -wait_for_mutation "minmax_idx" "mutation_1.txt" "test" +wait_for_mutation "minmax_idx" "mutation_2.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index 1bcb4f17edd..5e6159475f8 100755 --- a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -48,8 +48,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" -wait_for_mutation "indices_mutaions1" "mutation_1.txt" "test" -wait_for_mutation "indices_mutaions2" "mutation_1.txt" "test" +sleep 0.5 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" From 13f59d3f6c470b41ada777a88625130cbf9fa910 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 23 Aug 2019 23:13:08 +0300 Subject: [PATCH 214/357] Update Client.cpp --- dbms/programs/client/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 3032c9a5a57..76f67bd6de3 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -131,7 +131,7 @@ private: bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode. bool stdin_is_not_tty = false; /// stdin is not a terminal. - unsigned short int terminal_width; /// Terminal width is needed to render progress bar. + uint16_t terminal_width{}; /// Terminal width is needed to render progress bar. std::unique_ptr connection; /// Connection to DB. String query_id; /// Current query_id. From d38e9ee229cede07348b2531c3949a9f11b9abe3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Aug 2019 23:32:31 +0300 Subject: [PATCH 215/357] Fixed "trim" functions (in progress) --- dbms/src/Functions/trim.cpp | 64 ++---- libs/libcommon/include/common/find_symbols.h | 198 +++++++++++++------ 2 files changed, 157 insertions(+), 105 deletions(-) diff --git a/dbms/src/Functions/trim.cpp b/dbms/src/Functions/trim.cpp index f2e2543cc90..81916604d63 100644 --- a/dbms/src/Functions/trim.cpp +++ b/dbms/src/Functions/trim.cpp @@ -1,10 +1,8 @@ #include #include #include +#include -#ifdef __SSE4_2__ -#include -#endif namespace DB { @@ -60,7 +58,7 @@ public: execute(reinterpret_cast(&data[prev_offset]), offsets[i] - prev_offset - 1, start, length); res_data.resize(res_data.size() + length + 1); - memcpy(&res_data[res_offset], start, length); + memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], start, length); res_offset += length + 1; res_data[res_offset - 1] = '\0'; @@ -77,59 +75,27 @@ public: private: static void execute(const UInt8 * data, size_t size, const UInt8 *& res_data, size_t & res_size) { - size_t chars_to_trim_left = 0; - size_t chars_to_trim_right = 0; - char whitespace = ' '; -#ifdef __SSE4_2__ - const auto bytes_sse = sizeof(__m128i); - const auto size_sse = size - (size % bytes_sse); - const auto whitespace_mask = _mm_set1_epi8(whitespace); - constexpr auto base_sse_mode = _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_EACH | _SIDD_NEGATIVE_POLARITY; - auto mask = bytes_sse; -#endif + const char * char_data = reinterpret_cast(data); + const char * char_end = char_data + size; if constexpr (mode::trim_left) { -#ifdef __SSE4_2__ - /// skip whitespace from left in blocks of up to 16 characters - - /// Avoid gcc bug: _mm_cmpistri: error: the third argument must be an 8-bit immediate - enum { left_sse_mode = base_sse_mode | _SIDD_LEAST_SIGNIFICANT }; - while (mask == bytes_sse && chars_to_trim_left < size_sse) - { - const auto chars = _mm_loadu_si128(reinterpret_cast(data + chars_to_trim_left)); - mask = _mm_cmpistri(whitespace_mask, chars, left_sse_mode); - chars_to_trim_left += mask; - } -#endif - /// skip remaining whitespace from left, character by character - while (chars_to_trim_left < size && data[chars_to_trim_left] == whitespace) - ++chars_to_trim_left; + const char * found = find_first_not_symbols<' '>(char_data, char_end); + size_t num_chars = found - char_data; + char_data += num_chars; } - if constexpr (mode::trim_right) + if constexpr (mode::trim_left) { - const auto trim_right_size = size - chars_to_trim_left; -#ifdef __SSE4_2__ - /// try to skip whitespace from right in blocks of up to 16 characters - - /// Avoid gcc bug: _mm_cmpistri: error: the third argument must be an 8-bit immediate - enum { right_sse_mode = base_sse_mode | _SIDD_MOST_SIGNIFICANT }; - const auto trim_right_size_sse = trim_right_size - (trim_right_size % bytes_sse); - while (mask == bytes_sse && chars_to_trim_right < trim_right_size_sse) - { - const auto chars = _mm_loadu_si128(reinterpret_cast(data + size - chars_to_trim_right - bytes_sse)); - mask = _mm_cmpistri(whitespace_mask, chars, right_sse_mode); - chars_to_trim_right += mask; - } -#endif - /// skip remaining whitespace from right, character by character - while (chars_to_trim_right < trim_right_size && data[size - chars_to_trim_right - 1] == whitespace) - ++chars_to_trim_right; + const char * found = find_last_not_symbols_or_null<' '>(char_data, char_end); + if (found) + char_end = found + 1; + else + char_end = char_data; } - res_data = data + chars_to_trim_left; - res_size = size - chars_to_trim_left - chars_to_trim_right; + res_data = reinterpret_cast(char_data); + res_size = char_end - char_data; } }; diff --git a/libs/libcommon/include/common/find_symbols.h b/libs/libcommon/include/common/find_symbols.h index 68b49397683..920a7df04c5 100644 --- a/libs/libcommon/include/common/find_symbols.h +++ b/libs/libcommon/include/common/find_symbols.h @@ -65,115 +65,153 @@ inline __m128i mm_is_in(__m128i bytes) } #endif - -template -inline const char * find_first_symbols_sse2(const char * begin, const char * end) +template +bool maybe_negate(bool x) { + if constexpr (positive) + return x; + else + return !x; +} + +template +uint16_t maybe_negate(uint16_t x) +{ + if constexpr (positive) + return x; + else + return ~x; +} + +enum class ReturnMode +{ + End, + Nullptr, +}; + + +template +inline const char * find_first_symbols_sse2(const char * const begin, const char * const end) +{ + const char * pos = begin; + #if defined(__SSE2__) - for (; begin + 15 < end; begin += 16) + for (; pos + 15 < end; pos += 16) { - __m128i bytes = _mm_loadu_si128(reinterpret_cast(begin)); + __m128i bytes = _mm_loadu_si128(reinterpret_cast(pos)); __m128i eq = mm_is_in(bytes); - uint16_t bit_mask = _mm_movemask_epi8(eq); + uint16_t bit_mask = maybe_negate(uint16_t(_mm_movemask_epi8(eq))); if (bit_mask) - return begin + __builtin_ctz(bit_mask); + return pos + __builtin_ctz(bit_mask); } #endif - for (; begin < end; ++begin) - if (is_in(*begin)) - return begin; - return end; + for (; pos < end; ++pos) + if (maybe_negate(is_in(*pos))) + return pos; + + return return_mode == ReturnMode::End ? end : nullptr; } -template -inline const char * find_last_symbols_or_null_sse2(const char * begin, const char * end) +template +inline const char * find_last_symbols_sse2(const char * const begin, const char * const end) { + const char * pos = end; + #if defined(__SSE2__) - for (; end - 16 >= begin; end -= 16) /// Assuming the pointer cannot overflow. Assuming we can compare these pointers. + for (; pos - 16 >= begin; pos -= 16) /// Assuming the pointer cannot overflow. Assuming we can compare these pointers. { - __m128i bytes = _mm_loadu_si128(reinterpret_cast(end - 16)); + __m128i bytes = _mm_loadu_si128(reinterpret_cast(pos - 16)); __m128i eq = mm_is_in(bytes); - uint16_t bit_mask = _mm_movemask_epi8(eq); + uint16_t bit_mask = maybe_negate(uint16_t(_mm_movemask_epi8(eq))); if (bit_mask) - return end - 1 - (__builtin_clz(bit_mask) - 16); /// because __builtin_clz works with mask as uint32. + return pos - 1 - (__builtin_clz(bit_mask) - 16); /// because __builtin_clz works with mask as uint32. } #endif - --end; - for (; end >= begin; --end) - if (is_in(*end)) - return end; + --pos; + for (; pos >= begin; --pos) + if (maybe_negate(is_in(*pos))) + return pos; - return nullptr; + return return_mode == ReturnMode::End ? end : nullptr; } -template -inline const char * find_first_symbols_sse42_impl(const char * begin, const char * end) +inline const char * find_first_symbols_sse42_impl(const char * const begin, const char * const end) { + const char * pos = begin; + #if defined(__SSE4_2__) #define MODE (_SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT) __m128i set = _mm_setr_epi8(c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16); - for (; begin + 15 < end; begin += 16) + for (; pos + 15 < end; pos += 16) { - __m128i bytes = _mm_loadu_si128(reinterpret_cast(begin)); + __m128i bytes = _mm_loadu_si128(reinterpret_cast(pos)); - if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE)) - return begin + _mm_cmpestri(set, num_chars, bytes, 16, MODE); + if constexpr (positive) + { + if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE)) + return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE); + } + else + { + if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY)) + return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY); + } } #undef MODE #endif - for (; begin < end; ++begin) - if ( (num_chars >= 1 && *begin == c01) - || (num_chars >= 2 && *begin == c02) - || (num_chars >= 3 && *begin == c03) - || (num_chars >= 4 && *begin == c04) - || (num_chars >= 5 && *begin == c05) - || (num_chars >= 6 && *begin == c06) - || (num_chars >= 7 && *begin == c07) - || (num_chars >= 8 && *begin == c08) - || (num_chars >= 9 && *begin == c09) - || (num_chars >= 10 && *begin == c10) - || (num_chars >= 11 && *begin == c11) - || (num_chars >= 12 && *begin == c12) - || (num_chars >= 13 && *begin == c13) - || (num_chars >= 14 && *begin == c14) - || (num_chars >= 15 && *begin == c15) - || (num_chars >= 16 && *begin == c16)) - return begin; - return end; + for (; pos < end; ++pos) + if ( (num_chars >= 1 && maybe_negate(*pos == c01)) + || (num_chars >= 2 && maybe_negate(*pos == c02)) + || (num_chars >= 3 && maybe_negate(*pos == c03)) + || (num_chars >= 4 && maybe_negate(*pos == c04)) + || (num_chars >= 5 && maybe_negate(*pos == c05)) + || (num_chars >= 6 && maybe_negate(*pos == c06)) + || (num_chars >= 7 && maybe_negate(*pos == c07)) + || (num_chars >= 8 && maybe_negate(*pos == c08)) + || (num_chars >= 9 && maybe_negate(*pos == c09)) + || (num_chars >= 10 && maybe_negate(*pos == c10)) + || (num_chars >= 11 && maybe_negate(*pos == c11)) + || (num_chars >= 12 && maybe_negate(*pos == c12)) + || (num_chars >= 13 && maybe_negate(*pos == c13)) + || (num_chars >= 15 && maybe_negate(*pos == c15)) + || (num_chars >= 16 && maybe_negate(*pos == c16))) + return pos; + return return_mode == ReturnMode::End ? end : nullptr; } -template +template inline const char * find_first_symbols_sse42(const char * begin, const char * end) { - return find_first_symbols_sse42_impl(begin, end); + return find_first_symbols_sse42_impl(begin, end); } /// NOTE No SSE 4.2 implementation for find_last_symbols_or_null. Not worth to do. -template +template inline const char * find_first_symbols_dispatch(const char * begin, const char * end) { #if defined(__SSE4_2__) if (sizeof...(symbols) >= 5) - return find_first_symbols_sse42(begin, end); + return find_first_symbols_sse42(begin, end); else #endif - return find_first_symbols_sse2(begin, end); + return find_first_symbols_sse2(begin, end); } } @@ -182,7 +220,7 @@ inline const char * find_first_symbols_dispatch(const char * begin, const char * template inline const char * find_first_symbols(const char * begin, const char * end) { - return detail::find_first_symbols_dispatch(begin, end); + return detail::find_first_symbols_dispatch(begin, end); } /// Returning non const result for non const arguments. @@ -190,18 +228,66 @@ inline const char * find_first_symbols(const char * begin, const char * end) template inline char * find_first_symbols(char * begin, char * end) { - return const_cast(detail::find_first_symbols_dispatch(begin, end)); + return const_cast(detail::find_first_symbols_dispatch(begin, end)); +} + +template +inline const char * find_first_not_symbols(const char * begin, const char * end) +{ + return detail::find_first_symbols_dispatch(begin, end); +} + +template +inline char * find_first_not_symbols(char * begin, char * end) +{ + return const_cast(detail::find_first_symbols_dispatch(begin, end)); +} + +template +inline const char * find_first_symbols_or_null(const char * begin, const char * end) +{ + return detail::find_first_symbols_dispatch(begin, end); +} + +template +inline char * find_first_symbols_or_null(char * begin, char * end) +{ + return const_cast(detail::find_first_symbols_dispatch(begin, end)); +} + +template +inline const char * find_first_not_symbols_or_null(const char * begin, const char * end) +{ + return detail::find_first_symbols_dispatch(begin, end); +} + +template +inline char * find_first_not_symbols_or_null(char * begin, char * end) +{ + return const_cast(detail::find_first_symbols_dispatch(begin, end)); } template inline const char * find_last_symbols_or_null(const char * begin, const char * end) { - return detail::find_last_symbols_or_null_sse2(begin, end); + return detail::find_last_symbols_sse2(begin, end); } template inline char * find_last_symbols_or_null(char * begin, char * end) { - return const_cast(detail::find_last_symbols_or_null_sse2(begin, end)); + return const_cast(detail::find_last_symbols_sse2(begin, end)); +} + +template +inline const char * find_last_not_symbols_or_null(const char * begin, const char * end) +{ + return detail::find_last_symbols_sse2(begin, end); +} + +template +inline char * find_last_not_symbols_or_null(char * begin, char * end) +{ + return const_cast(detail::find_last_symbols_sse2(begin, end)); } From 44d3e1e837fd5e804c2d0452e8a8f1b4af6f93f8 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 23 Aug 2019 23:35:36 +0300 Subject: [PATCH 216/357] Update Client.cpp --- dbms/programs/client/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 76f67bd6de3..e1f1bd92840 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1664,7 +1664,7 @@ public: * the "\n" is used to distinguish this case because there is hardly a chance an user would use "\n" * as the password. */ - ("password", po::value()->implicit_value("\n","\\n"), "password") + ("password", po::value()->implicit_value("\n", ""), "password") ("ask-password", "ask-password") ("query_id", po::value(), "query_id") ("query,q", po::value(), "query") From cdd6dca51771520b70df52550b3ae427b4fc82f9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Aug 2019 00:10:26 +0300 Subject: [PATCH 217/357] Remove Compiler --- dbms/programs/CMakeLists.txt | 28 +- dbms/programs/clang/CMakeLists.txt | 38 -- .../clang/Compiler-5.0.0/CMakeLists.txt | 53 -- .../programs/clang/Compiler-5.0.0/LICENSE.TXT | 63 -- .../clang/Compiler-5.0.0/cc1_main.cpp | 242 -------- .../clang/Compiler-5.0.0/cc1as_main.cpp | 540 ----------------- dbms/programs/clang/Compiler-5.0.0/driver.cpp | 519 ---------------- dbms/programs/clang/Compiler-5.0.0/lld.cpp | 23 - dbms/programs/clang/Compiler-5.0.1 | 1 - dbms/programs/clang/Compiler-5.0.2 | 1 - .../clang/Compiler-6.0.0/CMakeLists.txt | 54 -- .../programs/clang/Compiler-6.0.0/LICENSE.TXT | 63 -- .../clang/Compiler-6.0.0/cc1_main.cpp | 242 -------- .../clang/Compiler-6.0.0/cc1as_main.cpp | 540 ----------------- dbms/programs/clang/Compiler-6.0.0/driver.cpp | 520 ---------------- dbms/programs/clang/Compiler-6.0.0/lld.cpp | 23 - dbms/programs/clang/Compiler-6.0.0svn | 1 - dbms/programs/clang/Compiler-6.0.1 | 1 - .../clang/Compiler-7.0.0/CMakeLists.txt | 49 -- .../clang/Compiler-7.0.0/cc1_main.cpp | 239 -------- .../clang/Compiler-7.0.0/cc1as_main.cpp | 572 ------------------ .../Compiler-7.0.0/cc1gen_reproducer_main.cpp | 196 ------ dbms/programs/clang/Compiler-7.0.0/driver.cpp | 514 ---------------- dbms/programs/clang/Compiler-7.0.0/lld.cpp | 150 ----- .../Compiler-7.0.0bundled/CMakeLists.txt | 49 -- .../clang/Compiler-7.0.0bundled/cc1_main.cpp | 243 -------- .../Compiler-7.0.0bundled/cc1as_main.cpp | 555 ----------------- .../clang/Compiler-7.0.0bundled/driver.cpp | 512 ---------------- .../clang/Compiler-7.0.0bundled/lld.cpp | 10 - dbms/programs/clang/Compiler-7.0.0svn | 1 - dbms/programs/clang/Compiler-7.0.1 | 1 - dbms/programs/clang/clickhouse-clang.cpp | 2 - dbms/programs/clang/clickhouse-lld.cpp | 2 - dbms/programs/clang/copy_headers.sh | 100 --- dbms/programs/main.cpp | 16 - dbms/src/Core/Settings.h | 2 +- dbms/src/Interpreters/Aggregator.cpp | 254 +------- dbms/src/Interpreters/Aggregator.h | 58 +- dbms/src/Interpreters/CMakeLists.txt | 67 -- dbms/src/Interpreters/Compiler.cpp | 326 ---------- dbms/src/Interpreters/Compiler.h | 88 --- dbms/src/Interpreters/Context.cpp | 13 - .../Interpreters/InterpreterSelectQuery.cpp | 4 - dbms/src/Interpreters/SpecializedAggregator.h | 215 ------- dbms/src/Interpreters/config_compile.h.in | 26 - dbms/src/Interpreters/tests/CMakeLists.txt | 3 - dbms/src/Interpreters/tests/compiler_test.cpp | 57 -- .../00281_compile_sizeof_packed.reference | 2 - .../00281_compile_sizeof_packed.sql | 2 - .../00568_compile_catch_throw.reference | 2 - .../0_stateless/00568_compile_catch_throw.sh | 14 - 51 files changed, 8 insertions(+), 7288 deletions(-) delete mode 100644 dbms/programs/clang/CMakeLists.txt delete mode 100644 dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt delete mode 100644 dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT delete mode 100644 dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp delete mode 100644 dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp delete mode 100644 dbms/programs/clang/Compiler-5.0.0/driver.cpp delete mode 100644 dbms/programs/clang/Compiler-5.0.0/lld.cpp delete mode 120000 dbms/programs/clang/Compiler-5.0.1 delete mode 120000 dbms/programs/clang/Compiler-5.0.2 delete mode 100644 dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt delete mode 100644 dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT delete mode 100644 dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp delete mode 100644 dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp delete mode 100644 dbms/programs/clang/Compiler-6.0.0/driver.cpp delete mode 100644 dbms/programs/clang/Compiler-6.0.0/lld.cpp delete mode 120000 dbms/programs/clang/Compiler-6.0.0svn delete mode 120000 dbms/programs/clang/Compiler-6.0.1 delete mode 100644 dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt delete mode 100644 dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp delete mode 100644 dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp delete mode 100644 dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp delete mode 100644 dbms/programs/clang/Compiler-7.0.0/driver.cpp delete mode 100644 dbms/programs/clang/Compiler-7.0.0/lld.cpp delete mode 100644 dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt delete mode 100644 dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp delete mode 100644 dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp delete mode 100644 dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp delete mode 100644 dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp delete mode 120000 dbms/programs/clang/Compiler-7.0.0svn delete mode 120000 dbms/programs/clang/Compiler-7.0.1 delete mode 100644 dbms/programs/clang/clickhouse-clang.cpp delete mode 100644 dbms/programs/clang/clickhouse-lld.cpp delete mode 100755 dbms/programs/clang/copy_headers.sh delete mode 100644 dbms/src/Interpreters/Compiler.cpp delete mode 100644 dbms/src/Interpreters/Compiler.h delete mode 100644 dbms/src/Interpreters/SpecializedAggregator.h delete mode 100644 dbms/src/Interpreters/config_compile.h.in delete mode 100644 dbms/src/Interpreters/tests/compiler_test.cpp delete mode 100644 dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.reference delete mode 100644 dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.sql delete mode 100644 dbms/tests/queries/0_stateless/00568_compile_catch_throw.reference delete mode 100755 dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index 03eba470949..0dcd4d7ab91 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -81,7 +81,6 @@ add_subdirectory (extract-from-config) add_subdirectory (compressor) add_subdirectory (copier) add_subdirectory (format) -add_subdirectory (clang) add_subdirectory (obfuscator) if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) @@ -89,9 +88,9 @@ if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) endif () if (CLICKHOUSE_ONE_SHARED) - add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_COMPILER_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES}) - target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_COMPILER_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK}) - target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_COMPILER_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}) + add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES}) + target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK}) + target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}) set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "") install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse) endif() @@ -104,10 +103,6 @@ if (CLICKHOUSE_SPLIT_BINARY) list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge) endif () - if (USE_EMBEDDED_COMPILER) - list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-clang clickhouse-lld) - endif () - set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS}) @@ -115,10 +110,6 @@ if (CLICKHOUSE_SPLIT_BINARY) install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse) else () - if (USE_EMBEDDED_COMPILER) - # before add_executable ! - link_directories (${LLVM_LIBRARY_DIRS}) - endif () add_executable (clickhouse main.cpp) target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils) target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) @@ -154,9 +145,6 @@ else () if (ENABLE_CLICKHOUSE_OBFUSCATOR) clickhouse_target_link_split_lib(clickhouse obfuscator) endif () - if (USE_EMBEDDED_COMPILER) - target_link_libraries(clickhouse PRIVATE clickhouse-compiler-lib) - endif () set (CLICKHOUSE_BUNDLE) if (ENABLE_CLICKHOUSE_SERVER) @@ -213,18 +201,8 @@ else () list(APPEND CLICKHOUSE_BUNDLE clickhouse-odbc-bridge) endif() - # install always because depian package want this files: - add_custom_target (clickhouse-clang ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-clang DEPENDS clickhouse) - add_custom_target (clickhouse-lld ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-lld DEPENDS clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-clang clickhouse-lld) - install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - install (FILES - ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-clang - ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-lld - DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE}) endif () diff --git a/dbms/programs/clang/CMakeLists.txt b/dbms/programs/clang/CMakeLists.txt deleted file mode 100644 index 82f520614f4..00000000000 --- a/dbms/programs/clang/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -if (USE_EMBEDDED_COMPILER) - add_subdirectory ("Compiler-${LLVM_VERSION}") -endif () - -if (CLICKHOUSE_SPLIT_BINARY) - if (USE_EMBEDDED_COMPILER) - link_directories (${LLVM_LIBRARY_DIRS}) - add_executable (clickhouse-clang clickhouse-clang.cpp) - target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib) - add_executable (clickhouse-lld clickhouse-lld.cpp) - target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib) - install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - endif () -endif () - -set (TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/${INTERNAL_COMPILER_HEADERS_RELATIVE}") -# Make and install empty dir for debian package if compiler disabled -add_custom_target (make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR}) -install (DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse/${INTERNAL_COMPILER_HEADERS_DIR} COMPONENT clickhouse) -# TODO: fix on macos copy_headers.sh: sed --posix - -if (USE_EMBEDDED_COMPILER) - set (COPY_HEADERS_COMPILER "${CMAKE_CURRENT_BINARY_DIR}/../${INTERNAL_COMPILER_EXECUTABLE}") - set (COPY_HEADERS_DEPENDS clickhouse-clang) -elseif (EXISTS ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}) - set (COPY_HEADERS_COMPILER "${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}") -endif () - -if (COPY_HEADERS_COMPILER) - add_custom_target (copy-headers [ -f ${TMP_HEADERS_DIR}/dbms/src/Interpreters/SpecializedAggregator.h ] || env CLANG=${COPY_HEADERS_COMPILER} BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} CMAKE_CXX_COMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_CURRENT_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS ${COPY_HEADERS_DEPENDS} WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES copy_headers.sh) - - if (USE_INTERNAL_LLVM_LIBRARY) - set (CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers") - set (CLANG_HEADERS_DEST "${TMP_HEADERS_DIR}/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include - add_custom_target (copy-headers-clang ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} ) - add_dependencies (copy-headers copy-headers-clang) - endif () -endif () diff --git a/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt deleted file mode 100644 index 83e38cea257..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) - -link_directories(${LLVM_LIBRARY_DIRS}) - -add_library(clickhouse-compiler-lib - driver.cpp - cc1_main.cpp - cc1as_main.cpp - lld.cpp) - -target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti - -llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - -message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") - -target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) - -# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. - -target_link_libraries(clickhouse-compiler-lib PRIVATE - -clangBasic clangCodeGen clangDriver clangFrontend clangFrontendTool -clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend -clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers -clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic - -lldCOFF -lldDriver -lldELF -#lldMinGW -lldMachO -lldReaderWriter -lldYAML -#lldCommon -lldCore -lldConfig - -${REQUIRED_LLVM_LIBRARIES} - -LLVMSupport - -#Polly -#PollyISL -#PollyPPCG - -PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads -${MALLOC_LIBRARIES} -${GLIBC_COMPATIBILITY_LIBRARIES} -${MEMCPY_LIBRARIES} -) diff --git a/dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT b/dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT deleted file mode 100644 index b452ca2efd8..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT +++ /dev/null @@ -1,63 +0,0 @@ -============================================================================== -LLVM Release License -============================================================================== -University of Illinois/NCSA -Open Source License - -Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign. -All rights reserved. - -Developed by: - - LLVM Team - - University of Illinois at Urbana-Champaign - - http://llvm.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== -The LLVM software contains code written by third parties. Such software will -have its own individual LICENSE.TXT file in the directory in which it appears. -This file will describe the copyrights, license, and restrictions which apply -to that code. - -The disclaimer of warranty in the University of Illinois Open Source License -applies to all code in the LLVM Distribution, and nothing in any of the -other licenses gives permission to use the names of the LLVM Team or the -University of Illinois to endorse or promote products derived from this -Software. - -The following pieces of software have additional or alternate copyrights, -licenses, and/or restrictions: - -Program Directory -------- --------- - - diff --git a/dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp deleted file mode 100644 index f6eabaf3387..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp +++ /dev/null @@ -1,242 +0,0 @@ -//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1 functionality, which implements the -// core compiler functionality along with a number of additional tools for -// demonstration and testing purposes. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Option/Arg.h" -#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" -#include "clang/Config/config.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticBuffer.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "clang/FrontendTool/Utils.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include - -#ifdef CLANG_HAVE_RLIMITS -#include -#endif - -// have no .a version in packages -#undef LINK_POLLY_INTO_TOOLS - -using namespace clang; -using namespace llvm::opt; - -//===----------------------------------------------------------------------===// -// Main driver -//===----------------------------------------------------------------------===// - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // Run the interrupt handlers to make sure any special cleanups get done, in - // particular that we remove files registered with RemoveFileOnSignal. - llvm::sys::RunInterruptHandlers(); - - // We cannot recover from llvm errors. When reporting a fatal error, exit - // with status 70 to generate crash diagnostics. For BSD systems this is - // defined as an internal software error. Otherwise, exit with status 1. - exit(GenCrashDiag ? 70 : 1); -} - -#ifdef LINK_POLLY_INTO_TOOLS -namespace polly { -void initializePollyPasses(llvm::PassRegistry &Registry); -} -#endif - -#ifdef CLANG_HAVE_RLIMITS -// The amount of stack we think is "sufficient". If less than this much is -// available, we may be unable to reach our template instantiation depth -// limit and other similar limits. -// FIXME: Unify this with the stack we request when spawning a thread to build -// a module. -static const int kSufficientStack = 8 << 20; - -#if defined(__linux__) && defined(__PIE__) -static size_t getCurrentStackAllocation() { - // If we can't compute the current stack usage, allow for 512K of command - // line arguments and environment. - size_t Usage = 512 * 1024; - if (FILE *StatFile = fopen("/proc/self/stat", "r")) { - // We assume that the stack extends from its current address to the end of - // the environment space. In reality, there is another string literal (the - // program name) after the environment, but this is close enough (we only - // need to be within 100K or so). - unsigned long StackPtr, EnvEnd; - // Disable silly GCC -Wformat warning that complains about length - // modifiers on ignored format specifiers. We want to retain these - // for documentation purposes even though they have no effect. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat" -#endif - if (fscanf(StatFile, - "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " - "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " - "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " - "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", - &StackPtr, &EnvEnd) == 2) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; - } - fclose(StatFile); - } - return Usage; -} - -#include - -LLVM_ATTRIBUTE_NOINLINE -static void ensureStackAddressSpace(int ExtraChunks = 0) { - // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary - // relatively close to the stack (they are only guaranteed to be 128MiB - // apart). This results in crashes if we happen to heap-allocate more than - // 128MiB before we reach our stack high-water mark. - // - // To avoid these crashes, ensure that we have sufficient virtual memory - // pages allocated before we start running. - size_t Curr = getCurrentStackAllocation(); - const int kTargetStack = kSufficientStack - 256 * 1024; - if (Curr < kTargetStack) { - volatile char *volatile Alloc = - static_cast(alloca(kTargetStack - Curr)); - Alloc[0] = 0; - Alloc[kTargetStack - Curr - 1] = 0; - } -} -#else -static void ensureStackAddressSpace() {} -#endif - -/// Attempt to ensure that we have at least 8MiB of usable stack space. -static void ensureSufficientStack() { - struct rlimit rlim; - if (getrlimit(RLIMIT_STACK, &rlim) != 0) - return; - - // Increase the soft stack limit to our desired level, if necessary and - // possible. - if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) { - // Try to allocate sufficient stack. - if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack) - rlim.rlim_cur = kSufficientStack; - else if (rlim.rlim_cur == rlim.rlim_max) - return; - else - rlim.rlim_cur = rlim.rlim_max; - - if (setrlimit(RLIMIT_STACK, &rlim) != 0 || - rlim.rlim_cur != kSufficientStack) - return; - } - - // We should now have a stack of size at least kSufficientStack. Ensure - // that we can actually use that much, if necessary. - ensureStackAddressSpace(); -} -#else -static void ensureSufficientStack() {} -#endif - -int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - ensureSufficientStack(); - - std::unique_ptr Clang(new CompilerInstance()); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - // Register the support for object-file-wrapped Clang modules. - auto PCHOps = Clang->getPCHContainerOperations(); - PCHOps->registerWriter(llvm::make_unique()); - PCHOps->registerReader(llvm::make_unique()); - - // Initialize targets first, so that --version shows registered targets. - llvm::InitializeAllTargets(); - llvm::InitializeAllTargetMCs(); - llvm::InitializeAllAsmPrinters(); - llvm::InitializeAllAsmParsers(); - -#ifdef LINK_POLLY_INTO_TOOLS - llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); - polly::initializePollyPasses(Registry); -#endif - - // Buffer diagnostics from argument parsing so that we can output them using a - // well formed diagnostic object. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - bool Success = CompilerInvocation::CreateFromArgs( - Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); - - // Infer the builtin include path if unspecified. - if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && - Clang->getHeaderSearchOpts().ResourceDir.empty()) - Clang->getHeaderSearchOpts().ResourceDir = - CompilerInvocation::GetResourcesPath(Argv0, MainAddr); - - // Create the actual diagnostics engine. - Clang->createDiagnostics(); - if (!Clang->hasDiagnostics()) - return 1; - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - llvm::install_fatal_error_handler(LLVMErrorHandler, - static_cast(&Clang->getDiagnostics())); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); - if (!Success) - return 1; - - // Execute the frontend actions. - Success = ExecuteCompilerInvocation(Clang.get()); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - - // Our error handler depends on the Diagnostics object, which we're - // potentially about to delete. Uninstall the handler now so that any - // later errors use the default handling behavior instead. - llvm::remove_fatal_error_handler(); - - // When running with -disable-free, don't do any destruction or shutdown. - if (Clang->getFrontendOpts().DisableFree) { - BuryPointer(std::move(Clang)); - return !Success; - } - - return !Success; -} diff --git a/dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp deleted file mode 100644 index 2fc2b508ef2..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp +++ /dev/null @@ -1,540 +0,0 @@ -//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1as functionality, which implements -// the direct interface to the LLVM MC based assembler. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/MC/MCAsmBackend.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCCodeEmitter.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCParser/MCAsmParser.h" -#include "llvm/MC/MCParser/MCTargetAsmParser.h" -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" -#include "llvm/Option/Arg.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace clang::driver::options; -using namespace llvm; -using namespace llvm::opt; - -namespace { - -/// \brief Helper class for representing a single invocation of the assembler. -struct AssemblerInvocation { - /// @name Target Options - /// @{ - - /// The name of the target triple to assemble for. - std::string Triple; - - /// If given, the name of the target CPU to determine which instructions - /// are legal. - std::string CPU; - - /// The list of target specific features to enable or disable -- this should - /// be a list of strings starting with '+' or '-'. - std::vector Features; - - /// The list of symbol definitions. - std::vector SymbolDefs; - - /// @} - /// @name Language Options - /// @{ - - std::vector IncludePaths; - unsigned NoInitialTextSection : 1; - unsigned SaveTemporaryLabels : 1; - unsigned GenDwarfForAssembly : 1; - unsigned RelaxELFRelocations : 1; - unsigned DwarfVersion; - std::string DwarfDebugFlags; - std::string DwarfDebugProducer; - std::string DebugCompilationDir; - llvm::DebugCompressionType CompressDebugSections = - llvm::DebugCompressionType::None; - std::string MainFileName; - - /// @} - /// @name Frontend Options - /// @{ - - std::string InputFile; - std::vector LLVMArgs; - std::string OutputPath; - enum FileType { - FT_Asm, ///< Assembly (.s) output, transliterate mode. - FT_Null, ///< No output, for timing purposes. - FT_Obj ///< Object file output. - }; - FileType OutputType; - unsigned ShowHelp : 1; - unsigned ShowVersion : 1; - - /// @} - /// @name Transliterate Options - /// @{ - - unsigned OutputAsmVariant; - unsigned ShowEncoding : 1; - unsigned ShowInst : 1; - - /// @} - /// @name Assembler Options - /// @{ - - unsigned RelaxAll : 1; - unsigned NoExecStack : 1; - unsigned FatalWarnings : 1; - unsigned IncrementalLinkerCompatible : 1; - - /// The name of the relocation model to use. - std::string RelocationModel; - - /// @} - -public: - AssemblerInvocation() { - Triple = ""; - NoInitialTextSection = 0; - InputFile = "-"; - OutputPath = "-"; - OutputType = FT_Asm; - OutputAsmVariant = 0; - ShowInst = 0; - ShowEncoding = 0; - RelaxAll = 0; - NoExecStack = 0; - FatalWarnings = 0; - IncrementalLinkerCompatible = 0; - DwarfVersion = 0; - } - - static bool CreateFromArgs(AssemblerInvocation &Res, - ArrayRef Argv, - DiagnosticsEngine &Diags); -}; - -} - -bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, - ArrayRef Argv, - DiagnosticsEngine &Diags) { - bool Success = true; - - // Parse the arguments. - std::unique_ptr OptTbl(createDriverOptTable()); - - const unsigned IncludedFlagsBitmask = options::CC1AsOption; - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, - IncludedFlagsBitmask); - - // Check for missing argument error. - if (MissingArgCount) { - Diags.Report(diag::err_drv_missing_argument) - << Args.getArgString(MissingArgIndex) << MissingArgCount; - Success = false; - } - - // Issue errors on unknown arguments. - for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - - // Construct the invocation. - - // Target Options - Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); - Opts.CPU = Args.getLastArgValue(OPT_target_cpu); - Opts.Features = Args.getAllArgValues(OPT_target_feature); - - // Use the default target triple if unspecified. - if (Opts.Triple.empty()) - Opts.Triple = llvm::sys::getDefaultTargetTriple(); - - // Language Options - Opts.IncludePaths = Args.getAllArgValues(OPT_I); - Opts.NoInitialTextSection = Args.hasArg(OPT_n); - Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); - // Any DebugInfoKind implies GenDwarfForAssembly. - Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); - - if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, - OPT_compress_debug_sections_EQ)) { - if (A->getOption().getID() == OPT_compress_debug_sections) { - // TODO: be more clever about the compression type auto-detection - Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; - } else { - Opts.CompressDebugSections = - llvm::StringSwitch(A->getValue()) - .Case("none", llvm::DebugCompressionType::None) - .Case("zlib", llvm::DebugCompressionType::Z) - .Case("zlib-gnu", llvm::DebugCompressionType::GNU) - .Default(llvm::DebugCompressionType::None); - } - } - - Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); - Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); - Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); - Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); - Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); - Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); - - // Frontend Options - if (Args.hasArg(OPT_INPUT)) { - bool First = true; - for (const Arg *A : Args.filtered(OPT_INPUT)) { - if (First) { - Opts.InputFile = A->getValue(); - First = false; - } else { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - } - } - Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); - Opts.OutputPath = Args.getLastArgValue(OPT_o); - if (Arg *A = Args.getLastArg(OPT_filetype)) { - StringRef Name = A->getValue(); - unsigned OutputType = StringSwitch(Name) - .Case("asm", FT_Asm) - .Case("null", FT_Null) - .Case("obj", FT_Obj) - .Default(~0U); - if (OutputType == ~0U) { - Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; - Success = false; - } else - Opts.OutputType = FileType(OutputType); - } - Opts.ShowHelp = Args.hasArg(OPT_help); - Opts.ShowVersion = Args.hasArg(OPT_version); - - // Transliterate Options - Opts.OutputAsmVariant = - getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); - Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); - Opts.ShowInst = Args.hasArg(OPT_show_inst); - - // Assemble Options - Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); - Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); - Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); - Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); - Opts.IncrementalLinkerCompatible = - Args.hasArg(OPT_mincremental_linker_compatible); - Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); - - return Success; -} - -static std::unique_ptr -getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags, - bool Binary) { - if (Opts.OutputPath.empty()) - Opts.OutputPath = "-"; - - // Make sure that the Out file gets unlinked from the disk if we get a - // SIGINT. - if (Opts.OutputPath != "-") - sys::RemoveFileOnSignal(Opts.OutputPath); - - std::error_code EC; - auto Out = llvm::make_unique( - Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); - if (EC) { - Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath - << EC.message(); - return nullptr; - } - - return Out; -} - -static bool ExecuteAssembler(AssemblerInvocation &Opts, - DiagnosticsEngine &Diags) { - // Get the target specific parser. - std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); - if (!TheTarget) - return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - ErrorOr> Buffer = - MemoryBuffer::getFileOrSTDIN(Opts.InputFile); - - if (std::error_code EC = Buffer.getError()) { - Error = EC.message(); - return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; - } - - SourceMgr SrcMgr; - - // Tell SrcMgr about this buffer, which is what the parser will pick up. - SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); - - // Record the location of the include directories so that the lexer can find - // it later. - SrcMgr.setIncludeDirs(Opts.IncludePaths); - - std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); - assert(MRI && "Unable to create target register info!"); - - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); - assert(MAI && "Unable to create target asm info!"); - - // Ensure MCAsmInfo initialization occurs before any use, otherwise sections - // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); - - bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; - std::unique_ptr FDOS = getOutputStream(Opts, Diags, IsBinary); - if (!FDOS) - return true; - - // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and - // MCObjectFileInfo needs a MCContext reference in order to initialize itself. - std::unique_ptr MOFI(new MCObjectFileInfo()); - - MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); - - bool PIC = false; - if (Opts.RelocationModel == "static") { - PIC = false; - } else if (Opts.RelocationModel == "pic") { - PIC = true; - } else { - assert(Opts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - PIC = false; - } - - MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, CodeModel::Default, Ctx); - if (Opts.SaveTemporaryLabels) - Ctx.setAllowTemporaryLabels(false); - if (Opts.GenDwarfForAssembly) - Ctx.setGenDwarfForAssembly(true); - if (!Opts.DwarfDebugFlags.empty()) - Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); - if (!Opts.DwarfDebugProducer.empty()) - Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); - if (!Opts.DebugCompilationDir.empty()) - Ctx.setCompilationDir(Opts.DebugCompilationDir); - if (!Opts.MainFileName.empty()) - Ctx.setMainFileName(StringRef(Opts.MainFileName)); - Ctx.setDwarfVersion(Opts.DwarfVersion); - - // Build up the feature string from the target feature list. - std::string FS; - if (!Opts.Features.empty()) { - FS = Opts.Features[0]; - for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) - FS += "," + Opts.Features[i]; - } - - std::unique_ptr Str; - - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr STI( - TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); - - raw_pwrite_stream *Out = FDOS.get(); - std::unique_ptr BOS; - - // FIXME: There is a bit of code duplication with addPassesToEmitFile. - if (Opts.OutputType == AssemblerInvocation::FT_Asm) { - MCInstPrinter *IP = TheTarget->createMCInstPrinter( - llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); - MCCodeEmitter *CE = nullptr; - MCAsmBackend *MAB = nullptr; - if (Opts.ShowEncoding) { - CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); - MCTargetOptions Options; - MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, Opts.CPU, Options); - } - auto FOut = llvm::make_unique(*Out); - Str.reset(TheTarget->createAsmStreamer( - Ctx, std::move(FOut), /*asmverbose*/ true, - /*useDwarfDirectory*/ true, IP, CE, MAB, Opts.ShowInst)); - } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { - Str.reset(createNullStreamer(Ctx)); - } else { - assert(Opts.OutputType == AssemblerInvocation::FT_Obj && - "Invalid file type!"); - if (!FDOS->supportsSeeking()) { - BOS = make_unique(*FDOS); - Out = BOS.get(); - } - - MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); - MCTargetOptions Options; - MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*MRI, Opts.Triple, - Opts.CPU, Options); - Triple T(Opts.Triple); - Str.reset(TheTarget->createMCObjectStreamer( - T, Ctx, *MAB, *Out, CE, *STI, Opts.RelaxAll, - Opts.IncrementalLinkerCompatible, - /*DWARFMustBeAtTheEnd*/ true)); - Str.get()->InitSections(Opts.NoExecStack); - } - - bool Failed = false; - - std::unique_ptr Parser( - createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); - - // FIXME: init MCTargetOptions from sanitizer flags here. - MCTargetOptions Options; - std::unique_ptr TAP( - TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); - if (!TAP) - Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - // Set values for symbols, if any. - for (auto &S : Opts.SymbolDefs) { - auto Pair = StringRef(S).split('='); - auto Sym = Pair.first; - auto Val = Pair.second; - int64_t Value; - // We have already error checked this in the driver. - Val.getAsInteger(0, Value); - Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); - } - - if (!Failed) { - Parser->setTargetParser(*TAP.get()); - Failed = Parser->Run(Opts.NoInitialTextSection); - } - - // Close Streamer first. - // It might have a reference to the output stream. - Str.reset(); - // Close the output stream early. - BOS.reset(); - FDOS.reset(); - - // Delete output file if there were errors. - if (Failed && Opts.OutputPath != "-") - sys::fs::remove(Opts.OutputPath); - - return Failed; -} - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // We cannot recover from llvm errors. - exit(1); -} - -int cc1as_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - // Initialize targets and assembly printers/parsers. - InitializeAllTargetInfos(); - InitializeAllTargetMCs(); - InitializeAllAsmParsers(); - - // Construct our diagnostic client. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(errs(), &*DiagOpts); - DiagClient->setPrefix("clang -cc1as"); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - ScopedFatalErrorHandler FatalErrorHandler - (LLVMErrorHandler, static_cast(&Diags)); - - // Parse the arguments. - AssemblerInvocation Asm; - if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) - return 1; - - if (Asm.ShowHelp) { - std::unique_ptr Opts(driver::createDriverOptTable()); - Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", - /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0); - return 0; - } - - // Honor -version. - // - // FIXME: Use a better -version message? - if (Asm.ShowVersion) { - llvm::cl::PrintVersionMessage(); - return 0; - } - - // Honor -mllvm. - // - // FIXME: Remove this, one day. - if (!Asm.LLVMArgs.empty()) { - unsigned NumArgs = Asm.LLVMArgs.size(); - auto Args = llvm::make_unique(NumArgs + 2); - Args[0] = "clang (LLVM option parsing)"; - for (unsigned i = 0; i != NumArgs; ++i) - Args[i + 1] = Asm.LLVMArgs[i].c_str(); - Args[NumArgs + 1] = nullptr; - llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); - } - - // Execute the invocation, unless there were parsing errors. - bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); - - // If any timers were active but haven't been destroyed yet, print their - // results now. - TimerGroup::printAll(errs()); - - return !!Failed; -} diff --git a/dbms/programs/clang/Compiler-5.0.0/driver.cpp b/dbms/programs/clang/Compiler-5.0.0/driver.cpp deleted file mode 100644 index 5aec2759f9e..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/driver.cpp +++ /dev/null @@ -1,519 +0,0 @@ -//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang driver; it is a thin wrapper -// for functionality in the Driver clang library. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/Driver.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Driver/ToolChain.h" -#include "clang/Frontend/ChainedDiagnosticConsumer.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/SerializedDiagnosticPrinter.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Process.h" -#include "llvm/Support/Program.h" -#include "llvm/Support/Regex.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/StringSaver.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace llvm::opt; - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { - if (!CanonicalPrefixes) { - SmallString<128> ExecutablePath(Argv0); - // Do a PATH lookup if Argv0 isn't a valid path. - if (!llvm::sys::fs::exists(ExecutablePath)) - if (llvm::ErrorOr P = - llvm::sys::findProgramByName(ExecutablePath)) - ExecutablePath = *P; - return ExecutablePath.str(); - } - - // This just needs to be some symbol in the binary; C++ doesn't - // allow taking the address of ::main however. - void *P = (void*) (intptr_t) GetExecutablePath; - return llvm::sys::fs::getMainExecutable(Argv0, P); -} - -static const char *GetStableCStr(std::set &SavedStrings, - StringRef S) { - return SavedStrings.insert(S).first->c_str(); -} - -/// ApplyQAOverride - Apply a list of edits to the input argument lists. -/// -/// The input string is a space separate list of edits to perform, -/// they are applied in order to the input argument lists. Edits -/// should be one of the following forms: -/// -/// '#': Silence information about the changes to the command line arguments. -/// -/// '^': Add FOO as a new argument at the beginning of the command line. -/// -/// '+': Add FOO as a new argument at the end of the command line. -/// -/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command -/// line. -/// -/// 'xOPTION': Removes all instances of the literal argument OPTION. -/// -/// 'XOPTION': Removes all instances of the literal argument OPTION, -/// and the following argument. -/// -/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' -/// at the end of the command line. -/// -/// \param OS - The stream to write edit information to. -/// \param Args - The vector of command line arguments. -/// \param Edit - The override command to perform. -/// \param SavedStrings - Set to use for storing string representations. -static void ApplyOneQAOverride(raw_ostream &OS, - SmallVectorImpl &Args, - StringRef Edit, - std::set &SavedStrings) { - // This does not need to be efficient. - - if (Edit[0] == '^') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at beginning\n"; - Args.insert(Args.begin() + 1, Str); - } else if (Edit[0] == '+') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at end\n"; - Args.push_back(Str); - } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { - StringRef MatchPattern = Edit.substr(2).split('/').first; - StringRef ReplPattern = Edit.substr(2).split('/').second; - ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); - - for (unsigned i = 1, e = Args.size(); i != e; ++i) { - // Ignore end-of-line response file markers - if (Args[i] == nullptr) - continue; - std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); - - if (Repl != Args[i]) { - OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; - Args[i] = GetStableCStr(SavedStrings, Repl); - } - } - } else if (Edit[0] == 'x' || Edit[0] == 'X') { - auto Option = Edit.substr(1); - for (unsigned i = 1; i < Args.size();) { - if (Option == Args[i]) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - if (Edit[0] == 'X') { - if (i < Args.size()) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - OS << "### Invalid X edit, end of command line!\n"; - } - } else - ++i; - } - } else if (Edit[0] == 'O') { - for (unsigned i = 1; i < Args.size();) { - const char *A = Args[i]; - // Ignore end-of-line response file markers - if (A == nullptr) - continue; - if (A[0] == '-' && A[1] == 'O' && - (A[2] == '\0' || - (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || - ('0' <= A[2] && A[2] <= '9'))))) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - ++i; - } - OS << "### Adding argument " << Edit << " at end\n"; - Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); - } else { - OS << "### Unrecognized edit: " << Edit << "\n"; - } -} - -/// ApplyQAOverride - Apply a comma separate list of edits to the -/// input argument lists. See ApplyOneQAOverride. -static void ApplyQAOverride(SmallVectorImpl &Args, - const char *OverrideStr, - std::set &SavedStrings) { - raw_ostream *OS = &llvm::errs(); - - if (OverrideStr[0] == '#') { - ++OverrideStr; - OS = &llvm::nulls(); - } - - *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; - - // This does not need to be efficient. - - const char *S = OverrideStr; - while (*S) { - const char *End = ::strchr(S, ' '); - if (!End) - End = S + strlen(S); - if (End != S) - ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); - S = End; - if (*S != '\0') - ++S; - } -} - -extern int cc1_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1as_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); - -static void insertTargetAndModeArgs(StringRef Target, StringRef Mode, - SmallVectorImpl &ArgVector, - std::set &SavedStrings) { - if (!Mode.empty()) { - // Add the mode flag to the arguments. - auto it = ArgVector.begin(); - if (it != ArgVector.end()) - ++it; - ArgVector.insert(it, GetStableCStr(SavedStrings, Mode)); - } - - if (!Target.empty()) { - auto it = ArgVector.begin(); - if (it != ArgVector.end()) - ++it; - const char *arr[] = {"-target", GetStableCStr(SavedStrings, Target)}; - ArgVector.insert(it, std::begin(arr), std::end(arr)); - } -} - -static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, - SmallVectorImpl &Opts) { - llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); - // The first instance of '#' should be replaced with '=' in each option. - for (const char *Opt : Opts) - if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) - *NumberSignPtr = '='; -} - -static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { - // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. - TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); - if (TheDriver.CCPrintOptions) - TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); - - // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. - TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); - if (TheDriver.CCPrintHeaders) - TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); - - // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. - TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); - if (TheDriver.CCLogDiagnostics) - TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); -} - -static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, - const std::string &Path) { - // If the clang binary happens to be named cl.exe for compatibility reasons, - // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. - StringRef ExeBasename(llvm::sys::path::filename(Path)); - if (ExeBasename.equals_lower("cl.exe")) - ExeBasename = "clang-cl.exe"; - DiagClient->setPrefix(ExeBasename); -} - -// This lets us create the DiagnosticsEngine with a properly-filled-out -// DiagnosticOptions instance. -static DiagnosticOptions * -CreateAndPopulateDiagOpts(ArrayRef argv) { - auto *DiagOpts = new DiagnosticOptions; - std::unique_ptr Opts(createDriverOptTable()); - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = - Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); - // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. - // Any errors that would be diagnosed here will also be diagnosed later, - // when the DiagnosticsEngine actually exists. - (void)ParseDiagnosticArgs(*DiagOpts, Args); - return DiagOpts; -} - -static void SetInstallDir(SmallVectorImpl &argv, - Driver &TheDriver, bool CanonicalPrefixes) { - // Attempt to find the original path used to invoke the driver, to determine - // the installed path. We do this manually, because we want to support that - // path being a symlink. - SmallString<128> InstalledPath(argv[0]); - - // Do a PATH lookup, if there are no directory components. - if (llvm::sys::path::filename(InstalledPath) == InstalledPath) - if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( - llvm::sys::path::filename(InstalledPath.str()))) - InstalledPath = *Tmp; - - // FIXME: We don't actually canonicalize this, we just make it absolute. - if (CanonicalPrefixes) - llvm::sys::fs::make_absolute(InstalledPath); - - StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); - if (llvm::sys::fs::exists(InstalledPathParent)) - TheDriver.setInstalledDir(InstalledPathParent); -} - -static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { - void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") - return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") - return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - - // Reject unknown tools. - llvm::errs() << "error: unknown integrated tool '" << Tool << "'\n"; - return 1; -} - -int mainEntryClickHouseClang(int argc_, char **argv_) { - llvm::sys::PrintStackTraceOnErrorSignal(argv_[0]); - llvm::PrettyStackTraceProgram X(argc_, argv_); - llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. - - if (llvm::sys::Process::FixupStandardFileDescriptors()) - return 1; - - SmallVector argv; - llvm::SpecificBumpPtrAllocator ArgAllocator; - std::error_code EC = llvm::sys::Process::GetArgumentVector( - argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator); - if (EC) { - llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n'; - return 1; - } - - llvm::InitializeAllTargets(); - std::string ProgName = argv[0]; - std::pair TargetAndMode = - ToolChain::getTargetAndModeFromProgramName(ProgName); - - llvm::BumpPtrAllocator A; - llvm::StringSaver Saver(A); - - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: argv[0] is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. - bool ClangCLMode = false; - if (TargetAndMode.second == "--driver-mode=cl" || - std::find_if(argv.begin(), argv.end(), [](const char *F) { - return F && strcmp(F, "--driver-mode=cl") == 0; - }) != argv.end()) { - ClangCLMode = true; - } - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : argv) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - - // Determines whether we want nullptr markers in argv to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); - - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), - [](const char *A) { return A != nullptr; }); - if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); - argv.resize(newEnd - argv.begin()); - } - return ExecuteCC1Tool(argv, argv[1] + 4); - } - - bool CanonicalPrefixes = true; - for (int i = 1, size = argv.size(); i < size; ++i) { - // Skip end-of-line response file markers - if (argv[i] == nullptr) - continue; - if (StringRef(argv[i]) == "-no-canonical-prefixes") { - CanonicalPrefixes = false; - break; - } - } - - // Handle CL and _CL_ which permits additional command line options to be - // prepended or appended. - if (ClangCLMode) { - // Arguments in "CL" are prepended. - llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); - if (OptCL.hasValue()) { - SmallVector PrependedOpts; - getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); - - // Insert right after the program name to prepend to the argument list. - argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); - } - // Arguments in "_CL_" are appended. - llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); - if (Opt_CL_.hasValue()) { - SmallVector AppendedOpts; - getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); - - // Insert at the end of the argument list to append. - argv.append(AppendedOpts.begin(), AppendedOpts.end()); - } - } - - std::set SavedStrings; - // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the - // scenes. - if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { - // FIXME: Driver shouldn't take extra initial argument. - ApplyQAOverride(argv, OverrideStr, SavedStrings); - } - - std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); - - IntrusiveRefCntPtr DiagOpts = - CreateAndPopulateDiagOpts(argv); - - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - FixupDiagPrefixExeName(DiagClient, Path); - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - if (!DiagOpts->DiagnosticSerializationFile.empty()) { - auto SerializedConsumer = - clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, - &*DiagOpts, /*MergeChildRecords=*/true); - Diags.setClient(new ChainedDiagnosticConsumer( - Diags.takeClient(), std::move(SerializedConsumer))); - } - - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - - Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); - SetInstallDir(argv, TheDriver, CanonicalPrefixes); - - insertTargetAndModeArgs(TargetAndMode.first, TargetAndMode.second, argv, - SavedStrings); - - SetBackdoorDriverOutputsFromEnvVars(TheDriver); - - std::unique_ptr C(TheDriver.BuildCompilation(argv)); - int Res = 1; - if (C && !C->containsError()) { - SmallVector, 4> FailingCommands; - Res = TheDriver.ExecuteCompilation(*C, FailingCommands); - - // Force a crash to test the diagnostics. - if (TheDriver.GenReproducer) { - Diags.Report(diag::err_drv_force_crash) - << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); - - // Pretend that every command failed. - FailingCommands.clear(); - for (const auto &J : C->getJobs()) - if (const Command *C = dyn_cast(&J)) - FailingCommands.push_back(std::make_pair(-1, C)); - } - - for (const auto &P : FailingCommands) { - int CommandRes = P.first; - const Command *FailingCommand = P.second; - if (!Res) - Res = CommandRes; - - // If result status is < 0, then the driver command signalled an error. - // If result status is 70, then the driver command reported a fatal error. - // On Windows, abort will return an exit code of 3. In these cases, - // generate additional diagnostic information if possible. - bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; -#ifdef LLVM_ON_WIN32 - DiagnoseCrash |= CommandRes == 3; -#endif - if (DiagnoseCrash) { - TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); - break; - } - } - } - - Diags.getClient()->finish(); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - -#ifdef LLVM_ON_WIN32 - // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be - // propagated. - if (Res < 0) - Res = 1; -#endif - - // If we have multiple failing commands, we return the result of the first - // failing command. - return Res; -} diff --git a/dbms/programs/clang/Compiler-5.0.0/lld.cpp b/dbms/programs/clang/Compiler-5.0.0/lld.cpp deleted file mode 100644 index 5af29868864..00000000000 --- a/dbms/programs/clang/Compiler-5.0.0/lld.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "lld/Driver/Driver.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Signals.h" - -using namespace lld; -using namespace llvm; -using namespace llvm::sys; - -int mainEntryClickHouseLLD(int Argc, char **Argv) -{ - // Standard set up, so program fails gracefully. - sys::PrintStackTraceOnErrorSignal(Argv[0]); - PrettyStackTraceProgram StackPrinter(Argc, Argv); - llvm_shutdown_obj Shutdown; - - std::vector Args(Argv, Argv + Argc); - return !elf::link(Args, true); -} diff --git a/dbms/programs/clang/Compiler-5.0.1 b/dbms/programs/clang/Compiler-5.0.1 deleted file mode 120000 index 7c8af57399f..00000000000 --- a/dbms/programs/clang/Compiler-5.0.1 +++ /dev/null @@ -1 +0,0 @@ -Compiler-5.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-5.0.2 b/dbms/programs/clang/Compiler-5.0.2 deleted file mode 120000 index 7c8af57399f..00000000000 --- a/dbms/programs/clang/Compiler-5.0.2 +++ /dev/null @@ -1 +0,0 @@ -Compiler-5.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt deleted file mode 100644 index 4a046674afc..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ - -add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) - -link_directories(${LLVM_LIBRARY_DIRS}) - -add_library(clickhouse-compiler-lib - driver.cpp - cc1_main.cpp - cc1as_main.cpp - lld.cpp) - -target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti - -llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - -message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") - -target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) - -# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. - -target_link_libraries(clickhouse-compiler-lib PRIVATE - -clangBasic clangCodeGen clangDriver -clangFrontend -clangFrontendTool -clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend -clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers -clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic - -lldCOFF -lldDriver -lldELF -lldMinGW -lldMachO -lldReaderWriter -lldYAML -lldCommon -lldCore -#lldWasm - -${REQUIRED_LLVM_LIBRARIES} - -#Polly -#PollyISL -#PollyPPCG - -PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads -${MALLOC_LIBRARIES} -${GLIBC_COMPATIBILITY_LIBRARIES} -${MEMCPY_LIBRARIES} -) diff --git a/dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT b/dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT deleted file mode 100644 index b452ca2efd8..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT +++ /dev/null @@ -1,63 +0,0 @@ -============================================================================== -LLVM Release License -============================================================================== -University of Illinois/NCSA -Open Source License - -Copyright (c) 2007-2016 University of Illinois at Urbana-Champaign. -All rights reserved. - -Developed by: - - LLVM Team - - University of Illinois at Urbana-Champaign - - http://llvm.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of the LLVM Team, University of Illinois at - Urbana-Champaign, nor the names of its contributors may be used to - endorse or promote products derived from this Software without specific - prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== -The LLVM software contains code written by third parties. Such software will -have its own individual LICENSE.TXT file in the directory in which it appears. -This file will describe the copyrights, license, and restrictions which apply -to that code. - -The disclaimer of warranty in the University of Illinois Open Source License -applies to all code in the LLVM Distribution, and nothing in any of the -other licenses gives permission to use the names of the LLVM Team or the -University of Illinois to endorse or promote products derived from this -Software. - -The following pieces of software have additional or alternate copyrights, -licenses, and/or restrictions: - -Program Directory -------- --------- - - diff --git a/dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp deleted file mode 100644 index f6eabaf3387..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp +++ /dev/null @@ -1,242 +0,0 @@ -//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1 functionality, which implements the -// core compiler functionality along with a number of additional tools for -// demonstration and testing purposes. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Option/Arg.h" -#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" -#include "clang/Config/config.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticBuffer.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "clang/FrontendTool/Utils.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include - -#ifdef CLANG_HAVE_RLIMITS -#include -#endif - -// have no .a version in packages -#undef LINK_POLLY_INTO_TOOLS - -using namespace clang; -using namespace llvm::opt; - -//===----------------------------------------------------------------------===// -// Main driver -//===----------------------------------------------------------------------===// - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // Run the interrupt handlers to make sure any special cleanups get done, in - // particular that we remove files registered with RemoveFileOnSignal. - llvm::sys::RunInterruptHandlers(); - - // We cannot recover from llvm errors. When reporting a fatal error, exit - // with status 70 to generate crash diagnostics. For BSD systems this is - // defined as an internal software error. Otherwise, exit with status 1. - exit(GenCrashDiag ? 70 : 1); -} - -#ifdef LINK_POLLY_INTO_TOOLS -namespace polly { -void initializePollyPasses(llvm::PassRegistry &Registry); -} -#endif - -#ifdef CLANG_HAVE_RLIMITS -// The amount of stack we think is "sufficient". If less than this much is -// available, we may be unable to reach our template instantiation depth -// limit and other similar limits. -// FIXME: Unify this with the stack we request when spawning a thread to build -// a module. -static const int kSufficientStack = 8 << 20; - -#if defined(__linux__) && defined(__PIE__) -static size_t getCurrentStackAllocation() { - // If we can't compute the current stack usage, allow for 512K of command - // line arguments and environment. - size_t Usage = 512 * 1024; - if (FILE *StatFile = fopen("/proc/self/stat", "r")) { - // We assume that the stack extends from its current address to the end of - // the environment space. In reality, there is another string literal (the - // program name) after the environment, but this is close enough (we only - // need to be within 100K or so). - unsigned long StackPtr, EnvEnd; - // Disable silly GCC -Wformat warning that complains about length - // modifiers on ignored format specifiers. We want to retain these - // for documentation purposes even though they have no effect. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat" -#endif - if (fscanf(StatFile, - "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " - "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " - "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " - "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", - &StackPtr, &EnvEnd) == 2) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; - } - fclose(StatFile); - } - return Usage; -} - -#include - -LLVM_ATTRIBUTE_NOINLINE -static void ensureStackAddressSpace(int ExtraChunks = 0) { - // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary - // relatively close to the stack (they are only guaranteed to be 128MiB - // apart). This results in crashes if we happen to heap-allocate more than - // 128MiB before we reach our stack high-water mark. - // - // To avoid these crashes, ensure that we have sufficient virtual memory - // pages allocated before we start running. - size_t Curr = getCurrentStackAllocation(); - const int kTargetStack = kSufficientStack - 256 * 1024; - if (Curr < kTargetStack) { - volatile char *volatile Alloc = - static_cast(alloca(kTargetStack - Curr)); - Alloc[0] = 0; - Alloc[kTargetStack - Curr - 1] = 0; - } -} -#else -static void ensureStackAddressSpace() {} -#endif - -/// Attempt to ensure that we have at least 8MiB of usable stack space. -static void ensureSufficientStack() { - struct rlimit rlim; - if (getrlimit(RLIMIT_STACK, &rlim) != 0) - return; - - // Increase the soft stack limit to our desired level, if necessary and - // possible. - if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) { - // Try to allocate sufficient stack. - if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack) - rlim.rlim_cur = kSufficientStack; - else if (rlim.rlim_cur == rlim.rlim_max) - return; - else - rlim.rlim_cur = rlim.rlim_max; - - if (setrlimit(RLIMIT_STACK, &rlim) != 0 || - rlim.rlim_cur != kSufficientStack) - return; - } - - // We should now have a stack of size at least kSufficientStack. Ensure - // that we can actually use that much, if necessary. - ensureStackAddressSpace(); -} -#else -static void ensureSufficientStack() {} -#endif - -int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - ensureSufficientStack(); - - std::unique_ptr Clang(new CompilerInstance()); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - // Register the support for object-file-wrapped Clang modules. - auto PCHOps = Clang->getPCHContainerOperations(); - PCHOps->registerWriter(llvm::make_unique()); - PCHOps->registerReader(llvm::make_unique()); - - // Initialize targets first, so that --version shows registered targets. - llvm::InitializeAllTargets(); - llvm::InitializeAllTargetMCs(); - llvm::InitializeAllAsmPrinters(); - llvm::InitializeAllAsmParsers(); - -#ifdef LINK_POLLY_INTO_TOOLS - llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); - polly::initializePollyPasses(Registry); -#endif - - // Buffer diagnostics from argument parsing so that we can output them using a - // well formed diagnostic object. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - bool Success = CompilerInvocation::CreateFromArgs( - Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); - - // Infer the builtin include path if unspecified. - if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && - Clang->getHeaderSearchOpts().ResourceDir.empty()) - Clang->getHeaderSearchOpts().ResourceDir = - CompilerInvocation::GetResourcesPath(Argv0, MainAddr); - - // Create the actual diagnostics engine. - Clang->createDiagnostics(); - if (!Clang->hasDiagnostics()) - return 1; - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - llvm::install_fatal_error_handler(LLVMErrorHandler, - static_cast(&Clang->getDiagnostics())); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); - if (!Success) - return 1; - - // Execute the frontend actions. - Success = ExecuteCompilerInvocation(Clang.get()); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - - // Our error handler depends on the Diagnostics object, which we're - // potentially about to delete. Uninstall the handler now so that any - // later errors use the default handling behavior instead. - llvm::remove_fatal_error_handler(); - - // When running with -disable-free, don't do any destruction or shutdown. - if (Clang->getFrontendOpts().DisableFree) { - BuryPointer(std::move(Clang)); - return !Success; - } - - return !Success; -} diff --git a/dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp deleted file mode 100644 index caf8409054a..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp +++ /dev/null @@ -1,540 +0,0 @@ -//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1as functionality, which implements -// the direct interface to the LLVM MC based assembler. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/MC/MCAsmBackend.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCCodeEmitter.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCParser/MCAsmParser.h" -#include "llvm/MC/MCParser/MCTargetAsmParser.h" -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" -#include "llvm/Option/Arg.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace clang::driver::options; -using namespace llvm; -using namespace llvm::opt; - -namespace { - -/// \brief Helper class for representing a single invocation of the assembler. -struct AssemblerInvocation { - /// @name Target Options - /// @{ - - /// The name of the target triple to assemble for. - std::string Triple; - - /// If given, the name of the target CPU to determine which instructions - /// are legal. - std::string CPU; - - /// The list of target specific features to enable or disable -- this should - /// be a list of strings starting with '+' or '-'. - std::vector Features; - - /// The list of symbol definitions. - std::vector SymbolDefs; - - /// @} - /// @name Language Options - /// @{ - - std::vector IncludePaths; - unsigned NoInitialTextSection : 1; - unsigned SaveTemporaryLabels : 1; - unsigned GenDwarfForAssembly : 1; - unsigned RelaxELFRelocations : 1; - unsigned DwarfVersion; - std::string DwarfDebugFlags; - std::string DwarfDebugProducer; - std::string DebugCompilationDir; - llvm::DebugCompressionType CompressDebugSections = - llvm::DebugCompressionType::None; - std::string MainFileName; - - /// @} - /// @name Frontend Options - /// @{ - - std::string InputFile; - std::vector LLVMArgs; - std::string OutputPath; - enum FileType { - FT_Asm, ///< Assembly (.s) output, transliterate mode. - FT_Null, ///< No output, for timing purposes. - FT_Obj ///< Object file output. - }; - FileType OutputType; - unsigned ShowHelp : 1; - unsigned ShowVersion : 1; - - /// @} - /// @name Transliterate Options - /// @{ - - unsigned OutputAsmVariant; - unsigned ShowEncoding : 1; - unsigned ShowInst : 1; - - /// @} - /// @name Assembler Options - /// @{ - - unsigned RelaxAll : 1; - unsigned NoExecStack : 1; - unsigned FatalWarnings : 1; - unsigned IncrementalLinkerCompatible : 1; - - /// The name of the relocation model to use. - std::string RelocationModel; - - /// @} - -public: - AssemblerInvocation() { - Triple = ""; - NoInitialTextSection = 0; - InputFile = "-"; - OutputPath = "-"; - OutputType = FT_Asm; - OutputAsmVariant = 0; - ShowInst = 0; - ShowEncoding = 0; - RelaxAll = 0; - NoExecStack = 0; - FatalWarnings = 0; - IncrementalLinkerCompatible = 0; - DwarfVersion = 0; - } - - static bool CreateFromArgs(AssemblerInvocation &Res, - ArrayRef Argv, - DiagnosticsEngine &Diags); -}; - -} - -bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, - ArrayRef Argv, - DiagnosticsEngine &Diags) { - bool Success = true; - - // Parse the arguments. - std::unique_ptr OptTbl(createDriverOptTable()); - - const unsigned IncludedFlagsBitmask = options::CC1AsOption; - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, - IncludedFlagsBitmask); - - // Check for missing argument error. - if (MissingArgCount) { - Diags.Report(diag::err_drv_missing_argument) - << Args.getArgString(MissingArgIndex) << MissingArgCount; - Success = false; - } - - // Issue errors on unknown arguments. - for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - - // Construct the invocation. - - // Target Options - Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); - Opts.CPU = Args.getLastArgValue(OPT_target_cpu); - Opts.Features = Args.getAllArgValues(OPT_target_feature); - - // Use the default target triple if unspecified. - if (Opts.Triple.empty()) - Opts.Triple = llvm::sys::getDefaultTargetTriple(); - - // Language Options - Opts.IncludePaths = Args.getAllArgValues(OPT_I); - Opts.NoInitialTextSection = Args.hasArg(OPT_n); - Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); - // Any DebugInfoKind implies GenDwarfForAssembly. - Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); - - if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, - OPT_compress_debug_sections_EQ)) { - if (A->getOption().getID() == OPT_compress_debug_sections) { - // TODO: be more clever about the compression type auto-detection - Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; - } else { - Opts.CompressDebugSections = - llvm::StringSwitch(A->getValue()) - .Case("none", llvm::DebugCompressionType::None) - .Case("zlib", llvm::DebugCompressionType::Z) - .Case("zlib-gnu", llvm::DebugCompressionType::GNU) - .Default(llvm::DebugCompressionType::None); - } - } - - Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); - Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); - Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); - Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); - Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); - Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); - - // Frontend Options - if (Args.hasArg(OPT_INPUT)) { - bool First = true; - for (const Arg *A : Args.filtered(OPT_INPUT)) { - if (First) { - Opts.InputFile = A->getValue(); - First = false; - } else { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - } - } - Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); - Opts.OutputPath = Args.getLastArgValue(OPT_o); - if (Arg *A = Args.getLastArg(OPT_filetype)) { - StringRef Name = A->getValue(); - unsigned OutputType = StringSwitch(Name) - .Case("asm", FT_Asm) - .Case("null", FT_Null) - .Case("obj", FT_Obj) - .Default(~0U); - if (OutputType == ~0U) { - Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; - Success = false; - } else - Opts.OutputType = FileType(OutputType); - } - Opts.ShowHelp = Args.hasArg(OPT_help); - Opts.ShowVersion = Args.hasArg(OPT_version); - - // Transliterate Options - Opts.OutputAsmVariant = - getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); - Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); - Opts.ShowInst = Args.hasArg(OPT_show_inst); - - // Assemble Options - Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); - Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); - Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); - Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); - Opts.IncrementalLinkerCompatible = - Args.hasArg(OPT_mincremental_linker_compatible); - Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); - - return Success; -} - -static std::unique_ptr -getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags, - bool Binary) { - if (Opts.OutputPath.empty()) - Opts.OutputPath = "-"; - - // Make sure that the Out file gets unlinked from the disk if we get a - // SIGINT. - if (Opts.OutputPath != "-") - sys::RemoveFileOnSignal(Opts.OutputPath); - - std::error_code EC; - auto Out = llvm::make_unique( - Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); - if (EC) { - Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath - << EC.message(); - return nullptr; - } - - return Out; -} - -static bool ExecuteAssembler(AssemblerInvocation &Opts, - DiagnosticsEngine &Diags) { - // Get the target specific parser. - std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); - if (!TheTarget) - return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - ErrorOr> Buffer = - MemoryBuffer::getFileOrSTDIN(Opts.InputFile); - - if (std::error_code EC = Buffer.getError()) { - Error = EC.message(); - return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; - } - - SourceMgr SrcMgr; - - // Tell SrcMgr about this buffer, which is what the parser will pick up. - SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); - - // Record the location of the include directories so that the lexer can find - // it later. - SrcMgr.setIncludeDirs(Opts.IncludePaths); - - std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); - assert(MRI && "Unable to create target register info!"); - - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); - assert(MAI && "Unable to create target asm info!"); - - // Ensure MCAsmInfo initialization occurs before any use, otherwise sections - // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); - - bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; - std::unique_ptr FDOS = getOutputStream(Opts, Diags, IsBinary); - if (!FDOS) - return true; - - // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and - // MCObjectFileInfo needs a MCContext reference in order to initialize itself. - std::unique_ptr MOFI(new MCObjectFileInfo()); - - MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); - - bool PIC = false; - if (Opts.RelocationModel == "static") { - PIC = false; - } else if (Opts.RelocationModel == "pic") { - PIC = true; - } else { - assert(Opts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - PIC = false; - } - - MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); - if (Opts.SaveTemporaryLabels) - Ctx.setAllowTemporaryLabels(false); - if (Opts.GenDwarfForAssembly) - Ctx.setGenDwarfForAssembly(true); - if (!Opts.DwarfDebugFlags.empty()) - Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); - if (!Opts.DwarfDebugProducer.empty()) - Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); - if (!Opts.DebugCompilationDir.empty()) - Ctx.setCompilationDir(Opts.DebugCompilationDir); - if (!Opts.MainFileName.empty()) - Ctx.setMainFileName(StringRef(Opts.MainFileName)); - Ctx.setDwarfVersion(Opts.DwarfVersion); - - // Build up the feature string from the target feature list. - std::string FS; - if (!Opts.Features.empty()) { - FS = Opts.Features[0]; - for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) - FS += "," + Opts.Features[i]; - } - - std::unique_ptr Str; - - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr STI( - TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); - - raw_pwrite_stream *Out = FDOS.get(); - std::unique_ptr BOS; - - // FIXME: There is a bit of code duplication with addPassesToEmitFile. - if (Opts.OutputType == AssemblerInvocation::FT_Asm) { - MCInstPrinter *IP = TheTarget->createMCInstPrinter( - llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); - MCCodeEmitter *CE = nullptr; - MCAsmBackend *MAB = nullptr; - if (Opts.ShowEncoding) { - CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); - MCTargetOptions Options; - MAB = TheTarget->createMCAsmBackend(*STI, *MRI, Options); - } - auto FOut = llvm::make_unique(*Out); - Str.reset(TheTarget->createAsmStreamer( - Ctx, std::move(FOut), /*asmverbose*/ true, - /*useDwarfDirectory*/ true, IP, CE, MAB, Opts.ShowInst)); - } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { - Str.reset(createNullStreamer(Ctx)); - } else { - assert(Opts.OutputType == AssemblerInvocation::FT_Obj && - "Invalid file type!"); - if (!FDOS->supportsSeeking()) { - BOS = make_unique(*FDOS); - Out = BOS.get(); - } - - MCCodeEmitter *CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); - MCTargetOptions Options; - MCAsmBackend *MAB = TheTarget->createMCAsmBackend(*STI, *MRI, Options); - Triple T(Opts.Triple); - Str.reset(TheTarget->createMCObjectStreamer( - T, Ctx, std::unique_ptr(MAB), *Out, std::unique_ptr(CE), *STI, - Opts.RelaxAll, Opts.IncrementalLinkerCompatible, - /*DWARFMustBeAtTheEnd*/ true)); - Str.get()->InitSections(Opts.NoExecStack); - } - - bool Failed = false; - - std::unique_ptr Parser( - createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); - - // FIXME: init MCTargetOptions from sanitizer flags here. - MCTargetOptions Options; - std::unique_ptr TAP( - TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); - if (!TAP) - Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - // Set values for symbols, if any. - for (auto &S : Opts.SymbolDefs) { - auto Pair = StringRef(S).split('='); - auto Sym = Pair.first; - auto Val = Pair.second; - int64_t Value = 0; - // We have already error checked this in the driver. - Val.getAsInteger(0, Value); - Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); - } - - if (!Failed) { - Parser->setTargetParser(*TAP.get()); - Failed = Parser->Run(Opts.NoInitialTextSection); - } - - // Close Streamer first. - // It might have a reference to the output stream. - Str.reset(); - // Close the output stream early. - BOS.reset(); - FDOS.reset(); - - // Delete output file if there were errors. - if (Failed && Opts.OutputPath != "-") - sys::fs::remove(Opts.OutputPath); - - return Failed; -} - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // We cannot recover from llvm errors. - exit(1); -} - -int cc1as_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - // Initialize targets and assembly printers/parsers. - InitializeAllTargetInfos(); - InitializeAllTargetMCs(); - InitializeAllAsmParsers(); - - // Construct our diagnostic client. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(errs(), &*DiagOpts); - DiagClient->setPrefix("clang -cc1as"); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - ScopedFatalErrorHandler FatalErrorHandler - (LLVMErrorHandler, static_cast(&Diags)); - - // Parse the arguments. - AssemblerInvocation Asm; - if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) - return 1; - - if (Asm.ShowHelp) { - std::unique_ptr Opts(driver::createDriverOptTable()); - Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", - /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, - /*ShowAllAliases=*/false); - return 0; - } - - // Honor -version. - // - // FIXME: Use a better -version message? - if (Asm.ShowVersion) { - llvm::cl::PrintVersionMessage(); - return 0; - } - - // Honor -mllvm. - // - // FIXME: Remove this, one day. - if (!Asm.LLVMArgs.empty()) { - unsigned NumArgs = Asm.LLVMArgs.size(); - auto Args = llvm::make_unique(NumArgs + 2); - Args[0] = "clang (LLVM option parsing)"; - for (unsigned i = 0; i != NumArgs; ++i) - Args[i + 1] = Asm.LLVMArgs[i].c_str(); - Args[NumArgs + 1] = nullptr; - llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); - } - - // Execute the invocation, unless there were parsing errors. - bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); - - // If any timers were active but haven't been destroyed yet, print their - // results now. - TimerGroup::printAll(errs()); - - return !!Failed; -} diff --git a/dbms/programs/clang/Compiler-6.0.0/driver.cpp b/dbms/programs/clang/Compiler-6.0.0/driver.cpp deleted file mode 100644 index 30511b8253a..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/driver.cpp +++ /dev/null @@ -1,520 +0,0 @@ -//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang driver; it is a thin wrapper -// for functionality in the Driver clang library. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/Driver.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Driver/ToolChain.h" -#include "clang/Frontend/ChainedDiagnosticConsumer.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/SerializedDiagnosticPrinter.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Process.h" -#include "llvm/Support/Program.h" -#include "llvm/Support/Regex.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/StringSaver.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace llvm::opt; - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { - if (!CanonicalPrefixes) { - SmallString<128> ExecutablePath(Argv0); - // Do a PATH lookup if Argv0 isn't a valid path. - if (!llvm::sys::fs::exists(ExecutablePath)) - if (llvm::ErrorOr P = - llvm::sys::findProgramByName(ExecutablePath)) - ExecutablePath = *P; - return ExecutablePath.str(); - } - - // This just needs to be some symbol in the binary; C++ doesn't - // allow taking the address of ::main however. - void *P = (void*) (intptr_t) GetExecutablePath; - return llvm::sys::fs::getMainExecutable(Argv0, P); -} - -static const char *GetStableCStr(std::set &SavedStrings, - StringRef S) { - return SavedStrings.insert(S).first->c_str(); -} - -/// ApplyQAOverride - Apply a list of edits to the input argument lists. -/// -/// The input string is a space separate list of edits to perform, -/// they are applied in order to the input argument lists. Edits -/// should be one of the following forms: -/// -/// '#': Silence information about the changes to the command line arguments. -/// -/// '^': Add FOO as a new argument at the beginning of the command line. -/// -/// '+': Add FOO as a new argument at the end of the command line. -/// -/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command -/// line. -/// -/// 'xOPTION': Removes all instances of the literal argument OPTION. -/// -/// 'XOPTION': Removes all instances of the literal argument OPTION, -/// and the following argument. -/// -/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' -/// at the end of the command line. -/// -/// \param OS - The stream to write edit information to. -/// \param Args - The vector of command line arguments. -/// \param Edit - The override command to perform. -/// \param SavedStrings - Set to use for storing string representations. -static void ApplyOneQAOverride(raw_ostream &OS, - SmallVectorImpl &Args, - StringRef Edit, - std::set &SavedStrings) { - // This does not need to be efficient. - - if (Edit[0] == '^') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at beginning\n"; - Args.insert(Args.begin() + 1, Str); - } else if (Edit[0] == '+') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at end\n"; - Args.push_back(Str); - } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { - StringRef MatchPattern = Edit.substr(2).split('/').first; - StringRef ReplPattern = Edit.substr(2).split('/').second; - ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); - - for (unsigned i = 1, e = Args.size(); i != e; ++i) { - // Ignore end-of-line response file markers - if (Args[i] == nullptr) - continue; - std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); - - if (Repl != Args[i]) { - OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; - Args[i] = GetStableCStr(SavedStrings, Repl); - } - } - } else if (Edit[0] == 'x' || Edit[0] == 'X') { - auto Option = Edit.substr(1); - for (unsigned i = 1; i < Args.size();) { - if (Option == Args[i]) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - if (Edit[0] == 'X') { - if (i < Args.size()) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - OS << "### Invalid X edit, end of command line!\n"; - } - } else - ++i; - } - } else if (Edit[0] == 'O') { - for (unsigned i = 1; i < Args.size();) { - const char *A = Args[i]; - // Ignore end-of-line response file markers - if (A == nullptr) - continue; - if (A[0] == '-' && A[1] == 'O' && - (A[2] == '\0' || - (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || - ('0' <= A[2] && A[2] <= '9'))))) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - ++i; - } - OS << "### Adding argument " << Edit << " at end\n"; - Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); - } else { - OS << "### Unrecognized edit: " << Edit << "\n"; - } -} - -/// ApplyQAOverride - Apply a comma separate list of edits to the -/// input argument lists. See ApplyOneQAOverride. -static void ApplyQAOverride(SmallVectorImpl &Args, - const char *OverrideStr, - std::set &SavedStrings) { - raw_ostream *OS = &llvm::errs(); - - if (OverrideStr[0] == '#') { - ++OverrideStr; - OS = &llvm::nulls(); - } - - *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; - - // This does not need to be efficient. - - const char *S = OverrideStr; - while (*S) { - const char *End = ::strchr(S, ' '); - if (!End) - End = S + strlen(S); - if (End != S) - ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); - S = End; - if (*S != '\0') - ++S; - } -} - -extern int cc1_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1as_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); - -static void insertTargetAndModeArgs(const ParsedClangName &NameParts, - SmallVectorImpl &ArgVector, - std::set &SavedStrings) { - // Put target and mode arguments at the start of argument list so that - // arguments specified in command line could override them. Avoid putting - // them at index 0, as an option like '-cc1' must remain the first. - auto InsertionPoint = ArgVector.begin(); - if (InsertionPoint != ArgVector.end()) - ++InsertionPoint; - - if (NameParts.DriverMode) { - // Add the mode flag to the arguments. - ArgVector.insert(InsertionPoint, - GetStableCStr(SavedStrings, NameParts.DriverMode)); - } - - if (NameParts.TargetIsValid) { - const char *arr[] = {"-target", GetStableCStr(SavedStrings, - NameParts.TargetPrefix)}; - ArgVector.insert(InsertionPoint, std::begin(arr), std::end(arr)); - } -} - -static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, - SmallVectorImpl &Opts) { - llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); - // The first instance of '#' should be replaced with '=' in each option. - for (const char *Opt : Opts) - if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) - *NumberSignPtr = '='; -} - -static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { - // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. - TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); - if (TheDriver.CCPrintOptions) - TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); - - // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. - TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); - if (TheDriver.CCPrintHeaders) - TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); - - // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. - TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); - if (TheDriver.CCLogDiagnostics) - TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); -} - -static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, - const std::string &Path) { - // If the clang binary happens to be named cl.exe for compatibility reasons, - // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. - StringRef ExeBasename(llvm::sys::path::filename(Path)); - if (ExeBasename.equals_lower("cl.exe")) - ExeBasename = "clang-cl.exe"; - DiagClient->setPrefix(ExeBasename); -} - -// This lets us create the DiagnosticsEngine with a properly-filled-out -// DiagnosticOptions instance. -static DiagnosticOptions * -CreateAndPopulateDiagOpts(ArrayRef argv) { - auto *DiagOpts = new DiagnosticOptions; - std::unique_ptr Opts(createDriverOptTable()); - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = - Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); - // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. - // Any errors that would be diagnosed here will also be diagnosed later, - // when the DiagnosticsEngine actually exists. - (void)ParseDiagnosticArgs(*DiagOpts, Args); - return DiagOpts; -} - -static void SetInstallDir(SmallVectorImpl &argv, - Driver &TheDriver, bool CanonicalPrefixes) { - // Attempt to find the original path used to invoke the driver, to determine - // the installed path. We do this manually, because we want to support that - // path being a symlink. - SmallString<128> InstalledPath(argv[0]); - - // Do a PATH lookup, if there are no directory components. - if (llvm::sys::path::filename(InstalledPath) == InstalledPath) - if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( - llvm::sys::path::filename(InstalledPath.str()))) - InstalledPath = *Tmp; - - // FIXME: We don't actually canonicalize this, we just make it absolute. - if (CanonicalPrefixes) - llvm::sys::fs::make_absolute(InstalledPath); - - StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); - if (llvm::sys::fs::exists(InstalledPathParent)) - TheDriver.setInstalledDir(InstalledPathParent); -} - -static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { - void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") - return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") - return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - - // Reject unknown tools. - llvm::errs() << "error: unknown integrated tool '" << Tool << "'\n"; - return 1; -} - -int mainEntryClickHouseClang(int argc_, char **argv_) { - llvm::sys::PrintStackTraceOnErrorSignal(argv_[0]); - llvm::PrettyStackTraceProgram X(argc_, argv_); - llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit. - - if (llvm::sys::Process::FixupStandardFileDescriptors()) - return 1; - - SmallVector argv; - llvm::SpecificBumpPtrAllocator ArgAllocator; - std::error_code EC = llvm::sys::Process::GetArgumentVector( - argv, llvm::makeArrayRef(argv_, argc_), ArgAllocator); - if (EC) { - llvm::errs() << "error: couldn't get arguments: " << EC.message() << '\n'; - return 1; - } - - llvm::InitializeAllTargets(); - auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); - - llvm::BumpPtrAllocator A; - llvm::StringSaver Saver(A); - - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: argv[0] is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. - bool ClangCLMode = false; - if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || - std::find_if(argv.begin(), argv.end(), [](const char *F) { - return F && strcmp(F, "--driver-mode=cl") == 0; - }) != argv.end()) { - ClangCLMode = true; - } - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : argv) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - - // Determines whether we want nullptr markers in argv to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); - - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), - [](const char *A) { return A != nullptr; }); - if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); - argv.resize(newEnd - argv.begin()); - } - return ExecuteCC1Tool(argv, argv[1] + 4); - } - - bool CanonicalPrefixes = true; - for (int i = 1, size = argv.size(); i < size; ++i) { - // Skip end-of-line response file markers - if (argv[i] == nullptr) - continue; - if (StringRef(argv[i]) == "-no-canonical-prefixes") { - CanonicalPrefixes = false; - break; - } - } - - // Handle CL and _CL_ which permits additional command line options to be - // prepended or appended. - if (ClangCLMode) { - // Arguments in "CL" are prepended. - llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); - if (OptCL.hasValue()) { - SmallVector PrependedOpts; - getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); - - // Insert right after the program name to prepend to the argument list. - argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); - } - // Arguments in "_CL_" are appended. - llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); - if (Opt_CL_.hasValue()) { - SmallVector AppendedOpts; - getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); - - // Insert at the end of the argument list to append. - argv.append(AppendedOpts.begin(), AppendedOpts.end()); - } - } - - std::set SavedStrings; - // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the - // scenes. - if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { - // FIXME: Driver shouldn't take extra initial argument. - ApplyQAOverride(argv, OverrideStr, SavedStrings); - } - - std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); - - IntrusiveRefCntPtr DiagOpts = - CreateAndPopulateDiagOpts(argv); - - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - FixupDiagPrefixExeName(DiagClient, Path); - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - if (!DiagOpts->DiagnosticSerializationFile.empty()) { - auto SerializedConsumer = - clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, - &*DiagOpts, /*MergeChildRecords=*/true); - Diags.setClient(new ChainedDiagnosticConsumer( - Diags.takeClient(), std::move(SerializedConsumer))); - } - - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - - Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); - SetInstallDir(argv, TheDriver, CanonicalPrefixes); - TheDriver.setTargetAndMode(TargetAndMode); - - insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); - - SetBackdoorDriverOutputsFromEnvVars(TheDriver); - - std::unique_ptr C(TheDriver.BuildCompilation(argv)); - int Res = 1; - if (C && !C->containsError()) { - SmallVector, 4> FailingCommands; - Res = TheDriver.ExecuteCompilation(*C, FailingCommands); - - // Force a crash to test the diagnostics. - if (TheDriver.GenReproducer) { - Diags.Report(diag::err_drv_force_crash) - << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); - - // Pretend that every command failed. - FailingCommands.clear(); - for (const auto &J : C->getJobs()) - if (const Command *C = dyn_cast(&J)) - FailingCommands.push_back(std::make_pair(-1, C)); - } - - for (const auto &P : FailingCommands) { - int CommandRes = P.first; - const Command *FailingCommand = P.second; - if (!Res) - Res = CommandRes; - - // If result status is < 0, then the driver command signalled an error. - // If result status is 70, then the driver command reported a fatal error. - // On Windows, abort will return an exit code of 3. In these cases, - // generate additional diagnostic information if possible. - bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; -#ifdef LLVM_ON_WIN32 - DiagnoseCrash |= CommandRes == 3; -#endif - if (DiagnoseCrash) { - TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); - break; - } - } - } - - Diags.getClient()->finish(); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - -#ifdef LLVM_ON_WIN32 - // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be - // propagated. - if (Res < 0) - Res = 1; -#endif - - // If we have multiple failing commands, we return the result of the first - // failing command. - return Res; -} diff --git a/dbms/programs/clang/Compiler-6.0.0/lld.cpp b/dbms/programs/clang/Compiler-6.0.0/lld.cpp deleted file mode 100644 index 696ff84dfe6..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0/lld.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "lld/Common/Driver.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Signals.h" - -using namespace lld; -using namespace llvm; -using namespace llvm::sys; - -int mainEntryClickHouseLLD(int Argc, char **Argv) -{ - // Standard set up, so program fails gracefully. - sys::PrintStackTraceOnErrorSignal(Argv[0]); - PrettyStackTraceProgram StackPrinter(Argc, Argv); - llvm_shutdown_obj Shutdown; - - std::vector Args(Argv, Argv + Argc); - return !elf::link(Args, true); -} diff --git a/dbms/programs/clang/Compiler-6.0.0svn b/dbms/programs/clang/Compiler-6.0.0svn deleted file mode 120000 index 7eba9cc37d0..00000000000 --- a/dbms/programs/clang/Compiler-6.0.0svn +++ /dev/null @@ -1 +0,0 @@ -Compiler-6.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-6.0.1 b/dbms/programs/clang/Compiler-6.0.1 deleted file mode 120000 index 7eba9cc37d0..00000000000 --- a/dbms/programs/clang/Compiler-6.0.1 +++ /dev/null @@ -1 +0,0 @@ -Compiler-6.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt deleted file mode 100644 index a042c821ec4..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) - -link_directories(${LLVM_LIBRARY_DIRS}) - -add_library(clickhouse-compiler-lib - driver.cpp - cc1_main.cpp - cc1gen_reproducer_main.cpp - cc1as_main.cpp - lld.cpp) - -target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti - -llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - -message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") - -target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) - -# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. - -target_link_libraries(clickhouse-compiler-lib PRIVATE -clangBasic clangCodeGen clangDriver -clangFrontend -clangFrontendTool -clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend -clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers -clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic -clangCrossTU clangIndex - -lldCOFF -lldDriver -lldELF -lldMinGW -lldMachO -lldReaderWriter -lldYAML -lldCommon -lldCore - -${REQUIRED_LLVM_LIBRARIES} - -PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads -${MALLOC_LIBRARIES} -${GLIBC_COMPATIBILITY_LIBRARIES} -${MEMCPY_LIBRARIES} -) diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp deleted file mode 100644 index 214bfa72476..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp +++ /dev/null @@ -1,239 +0,0 @@ -//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1 functionality, which implements the -// core compiler functionality along with a number of additional tools for -// demonstration and testing purposes. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Option/Arg.h" -#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" -#include "clang/Config/config.h" -#include "clang/Basic/Stack.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticBuffer.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "clang/FrontendTool/Utils.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include - -#ifdef CLANG_HAVE_RLIMITS -#include -#endif - -// have no .a version in packages -#undef LINK_POLLY_INTO_TOOLS - -using namespace clang; -using namespace llvm::opt; - -//===----------------------------------------------------------------------===// -// Main driver -//===----------------------------------------------------------------------===// - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // Run the interrupt handlers to make sure any special cleanups get done, in - // particular that we remove files registered with RemoveFileOnSignal. - llvm::sys::RunInterruptHandlers(); - - // We cannot recover from llvm errors. When reporting a fatal error, exit - // with status 70 to generate crash diagnostics. For BSD systems this is - // defined as an internal software error. Otherwise, exit with status 1. - exit(GenCrashDiag ? 70 : 1); -} - -#ifdef LINK_POLLY_INTO_TOOLS -namespace polly { -void initializePollyPasses(llvm::PassRegistry &Registry); -} -#endif - -#ifdef CLANG_HAVE_RLIMITS -#if defined(__linux__) && defined(__PIE__) -static size_t getCurrentStackAllocation() { - // If we can't compute the current stack usage, allow for 512K of command - // line arguments and environment. - size_t Usage = 512 * 1024; - if (FILE *StatFile = fopen("/proc/self/stat", "r")) { - // We assume that the stack extends from its current address to the end of - // the environment space. In reality, there is another string literal (the - // program name) after the environment, but this is close enough (we only - // need to be within 100K or so). - unsigned long StackPtr, EnvEnd; - // Disable silly GCC -Wformat warning that complains about length - // modifiers on ignored format specifiers. We want to retain these - // for documentation purposes even though they have no effect. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat" -#endif - if (fscanf(StatFile, - "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " - "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " - "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " - "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", - &StackPtr, &EnvEnd) == 2) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; - } - fclose(StatFile); - } - return Usage; -} - -#include - -LLVM_ATTRIBUTE_NOINLINE -static void ensureStackAddressSpace() { - // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary - // relatively close to the stack (they are only guaranteed to be 128MiB - // apart). This results in crashes if we happen to heap-allocate more than - // 128MiB before we reach our stack high-water mark. - // - // To avoid these crashes, ensure that we have sufficient virtual memory - // pages allocated before we start running. - size_t Curr = getCurrentStackAllocation(); - const int kTargetStack = DesiredStackSize - 256 * 1024; - if (Curr < kTargetStack) { - volatile char *volatile Alloc = - static_cast(alloca(kTargetStack - Curr)); - Alloc[0] = 0; - Alloc[kTargetStack - Curr - 1] = 0; - } -} -#else -static void ensureStackAddressSpace() {} -#endif - -/// Attempt to ensure that we have at least 8MiB of usable stack space. -static void ensureSufficientStack() { - struct rlimit rlim; - if (getrlimit(RLIMIT_STACK, &rlim) != 0) - return; - - // Increase the soft stack limit to our desired level, if necessary and - // possible. - if (rlim.rlim_cur != RLIM_INFINITY && - rlim.rlim_cur < rlim_t(DesiredStackSize)) { - // Try to allocate sufficient stack. - if (rlim.rlim_max == RLIM_INFINITY || - rlim.rlim_max >= rlim_t(DesiredStackSize)) - rlim.rlim_cur = DesiredStackSize; - else if (rlim.rlim_cur == rlim.rlim_max) - return; - else - rlim.rlim_cur = rlim.rlim_max; - - if (setrlimit(RLIMIT_STACK, &rlim) != 0 || - rlim.rlim_cur != DesiredStackSize) - return; - } - - // We should now have a stack of size at least DesiredStackSize. Ensure - // that we can actually use that much, if necessary. - ensureStackAddressSpace(); -} -#else -static void ensureSufficientStack() {} -#endif - -int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - ensureSufficientStack(); - - std::unique_ptr Clang(new CompilerInstance()); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - // Register the support for object-file-wrapped Clang modules. - auto PCHOps = Clang->getPCHContainerOperations(); - PCHOps->registerWriter(llvm::make_unique()); - PCHOps->registerReader(llvm::make_unique()); - - // Initialize targets first, so that --version shows registered targets. - llvm::InitializeAllTargets(); - llvm::InitializeAllTargetMCs(); - llvm::InitializeAllAsmPrinters(); - llvm::InitializeAllAsmParsers(); - -#ifdef LINK_POLLY_INTO_TOOLS - llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); - polly::initializePollyPasses(Registry); -#endif - - // Buffer diagnostics from argument parsing so that we can output them using a - // well formed diagnostic object. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - bool Success = CompilerInvocation::CreateFromArgs( - Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); - - // Infer the builtin include path if unspecified. - if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && - Clang->getHeaderSearchOpts().ResourceDir.empty()) - Clang->getHeaderSearchOpts().ResourceDir = - CompilerInvocation::GetResourcesPath(Argv0, MainAddr); - - // Create the actual diagnostics engine. - Clang->createDiagnostics(); - if (!Clang->hasDiagnostics()) - return 1; - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - llvm::install_fatal_error_handler(LLVMErrorHandler, - static_cast(&Clang->getDiagnostics())); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); - if (!Success) - return 1; - - // Execute the frontend actions. - Success = ExecuteCompilerInvocation(Clang.get()); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - - // Our error handler depends on the Diagnostics object, which we're - // potentially about to delete. Uninstall the handler now so that any - // later errors use the default handling behavior instead. - llvm::remove_fatal_error_handler(); - - // When running with -disable-free, don't do any destruction or shutdown. - if (Clang->getFrontendOpts().DisableFree) { - BuryPointer(std::move(Clang)); - return !Success; - } - - return !Success; -} diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp deleted file mode 100644 index d93b1f5cb1d..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp +++ /dev/null @@ -1,572 +0,0 @@ -//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1as functionality, which implements -// the direct interface to the LLVM MC based assembler. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/MC/MCAsmBackend.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCCodeEmitter.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCObjectWriter.h" -#include "llvm/MC/MCParser/MCAsmParser.h" -#include "llvm/MC/MCParser/MCTargetAsmParser.h" -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" -#include "llvm/Option/Arg.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace clang::driver::options; -using namespace llvm; -using namespace llvm::opt; - -namespace { - -/// Helper class for representing a single invocation of the assembler. -struct AssemblerInvocation { - /// @name Target Options - /// @{ - - /// The name of the target triple to assemble for. - std::string Triple; - - /// If given, the name of the target CPU to determine which instructions - /// are legal. - std::string CPU; - - /// The list of target specific features to enable or disable -- this should - /// be a list of strings starting with '+' or '-'. - std::vector Features; - - /// The list of symbol definitions. - std::vector SymbolDefs; - - /// @} - /// @name Language Options - /// @{ - - std::vector IncludePaths; - unsigned NoInitialTextSection : 1; - unsigned SaveTemporaryLabels : 1; - unsigned GenDwarfForAssembly : 1; - unsigned RelaxELFRelocations : 1; - unsigned DwarfVersion; - std::string DwarfDebugFlags; - std::string DwarfDebugProducer; - std::string DebugCompilationDir; - std::map DebugPrefixMap; - llvm::DebugCompressionType CompressDebugSections = - llvm::DebugCompressionType::None; - std::string MainFileName; - std::string SplitDwarfFile; - - /// @} - /// @name Frontend Options - /// @{ - - std::string InputFile; - std::vector LLVMArgs; - std::string OutputPath; - enum FileType { - FT_Asm, ///< Assembly (.s) output, transliterate mode. - FT_Null, ///< No output, for timing purposes. - FT_Obj ///< Object file output. - }; - FileType OutputType; - unsigned ShowHelp : 1; - unsigned ShowVersion : 1; - - /// @} - /// @name Transliterate Options - /// @{ - - unsigned OutputAsmVariant; - unsigned ShowEncoding : 1; - unsigned ShowInst : 1; - - /// @} - /// @name Assembler Options - /// @{ - - unsigned RelaxAll : 1; - unsigned NoExecStack : 1; - unsigned FatalWarnings : 1; - unsigned IncrementalLinkerCompatible : 1; - - /// The name of the relocation model to use. - std::string RelocationModel; - - /// @} - -public: - AssemblerInvocation() { - Triple = ""; - NoInitialTextSection = 0; - InputFile = "-"; - OutputPath = "-"; - OutputType = FT_Asm; - OutputAsmVariant = 0; - ShowInst = 0; - ShowEncoding = 0; - RelaxAll = 0; - NoExecStack = 0; - FatalWarnings = 0; - IncrementalLinkerCompatible = 0; - DwarfVersion = 0; - } - - static bool CreateFromArgs(AssemblerInvocation &Res, - ArrayRef Argv, - DiagnosticsEngine &Diags); -}; - -} - -bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, - ArrayRef Argv, - DiagnosticsEngine &Diags) { - bool Success = true; - - // Parse the arguments. - std::unique_ptr OptTbl(createDriverOptTable()); - - const unsigned IncludedFlagsBitmask = options::CC1AsOption; - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, - IncludedFlagsBitmask); - - // Check for missing argument error. - if (MissingArgCount) { - Diags.Report(diag::err_drv_missing_argument) - << Args.getArgString(MissingArgIndex) << MissingArgCount; - Success = false; - } - - // Issue errors on unknown arguments. - for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { - auto ArgString = A->getAsString(Args); - std::string Nearest; - if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1) - Diags.Report(diag::err_drv_unknown_argument) << ArgString; - else - Diags.Report(diag::err_drv_unknown_argument_with_suggestion) - << ArgString << Nearest; - Success = false; - } - - // Construct the invocation. - - // Target Options - Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); - Opts.CPU = Args.getLastArgValue(OPT_target_cpu); - Opts.Features = Args.getAllArgValues(OPT_target_feature); - - // Use the default target triple if unspecified. - if (Opts.Triple.empty()) - Opts.Triple = llvm::sys::getDefaultTargetTriple(); - - // Language Options - Opts.IncludePaths = Args.getAllArgValues(OPT_I); - Opts.NoInitialTextSection = Args.hasArg(OPT_n); - Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); - // Any DebugInfoKind implies GenDwarfForAssembly. - Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); - - if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, - OPT_compress_debug_sections_EQ)) { - if (A->getOption().getID() == OPT_compress_debug_sections) { - // TODO: be more clever about the compression type auto-detection - Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; - } else { - Opts.CompressDebugSections = - llvm::StringSwitch(A->getValue()) - .Case("none", llvm::DebugCompressionType::None) - .Case("zlib", llvm::DebugCompressionType::Z) - .Case("zlib-gnu", llvm::DebugCompressionType::GNU) - .Default(llvm::DebugCompressionType::None); - } - } - - Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); - Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); - Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); - Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); - Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); - Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); - - for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ)) - Opts.DebugPrefixMap.insert(StringRef(Arg).split('=')); - - // Frontend Options - if (Args.hasArg(OPT_INPUT)) { - bool First = true; - for (const Arg *A : Args.filtered(OPT_INPUT)) { - if (First) { - Opts.InputFile = A->getValue(); - First = false; - } else { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - } - } - Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); - Opts.OutputPath = Args.getLastArgValue(OPT_o); - Opts.SplitDwarfFile = Args.getLastArgValue(OPT_split_dwarf_file); - if (Arg *A = Args.getLastArg(OPT_filetype)) { - StringRef Name = A->getValue(); - unsigned OutputType = StringSwitch(Name) - .Case("asm", FT_Asm) - .Case("null", FT_Null) - .Case("obj", FT_Obj) - .Default(~0U); - if (OutputType == ~0U) { - Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; - Success = false; - } else - Opts.OutputType = FileType(OutputType); - } - Opts.ShowHelp = Args.hasArg(OPT_help); - Opts.ShowVersion = Args.hasArg(OPT_version); - - // Transliterate Options - Opts.OutputAsmVariant = - getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); - Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); - Opts.ShowInst = Args.hasArg(OPT_show_inst); - - // Assemble Options - Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); - Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); - Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); - Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); - Opts.IncrementalLinkerCompatible = - Args.hasArg(OPT_mincremental_linker_compatible); - Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); - - return Success; -} - -static std::unique_ptr -getOutputStream(StringRef Path, DiagnosticsEngine &Diags, bool Binary) { - // Make sure that the Out file gets unlinked from the disk if we get a - // SIGINT. - if (Path != "-") - sys::RemoveFileOnSignal(Path); - - std::error_code EC; - auto Out = llvm::make_unique( - Path, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); - if (EC) { - Diags.Report(diag::err_fe_unable_to_open_output) << Path << EC.message(); - return nullptr; - } - - return Out; -} - -static bool ExecuteAssembler(AssemblerInvocation &Opts, - DiagnosticsEngine &Diags) { - // Get the target specific parser. - std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); - if (!TheTarget) - return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - ErrorOr> Buffer = - MemoryBuffer::getFileOrSTDIN(Opts.InputFile); - - if (std::error_code EC = Buffer.getError()) { - Error = EC.message(); - return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; - } - - SourceMgr SrcMgr; - - // Tell SrcMgr about this buffer, which is what the parser will pick up. - SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); - - // Record the location of the include directories so that the lexer can find - // it later. - SrcMgr.setIncludeDirs(Opts.IncludePaths); - - std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); - assert(MRI && "Unable to create target register info!"); - - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); - assert(MAI && "Unable to create target asm info!"); - - // Ensure MCAsmInfo initialization occurs before any use, otherwise sections - // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); - - bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; - if (Opts.OutputPath.empty()) - Opts.OutputPath = "-"; - std::unique_ptr FDOS = - getOutputStream(Opts.OutputPath, Diags, IsBinary); - if (!FDOS) - return true; - std::unique_ptr DwoOS; - if (!Opts.SplitDwarfFile.empty()) - DwoOS = getOutputStream(Opts.SplitDwarfFile, Diags, IsBinary); - - // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and - // MCObjectFileInfo needs a MCContext reference in order to initialize itself. - std::unique_ptr MOFI(new MCObjectFileInfo()); - - MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); - - bool PIC = false; - if (Opts.RelocationModel == "static") { - PIC = false; - } else if (Opts.RelocationModel == "pic") { - PIC = true; - } else { - assert(Opts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - PIC = false; - } - - MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); - if (Opts.SaveTemporaryLabels) - Ctx.setAllowTemporaryLabels(false); - if (Opts.GenDwarfForAssembly) - Ctx.setGenDwarfForAssembly(true); - if (!Opts.DwarfDebugFlags.empty()) - Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); - if (!Opts.DwarfDebugProducer.empty()) - Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); - if (!Opts.DebugCompilationDir.empty()) - Ctx.setCompilationDir(Opts.DebugCompilationDir); - if (!Opts.DebugPrefixMap.empty()) - for (const auto &KV : Opts.DebugPrefixMap) - Ctx.addDebugPrefixMapEntry(KV.first, KV.second); - if (!Opts.MainFileName.empty()) - Ctx.setMainFileName(StringRef(Opts.MainFileName)); - Ctx.setDwarfVersion(Opts.DwarfVersion); - - // Build up the feature string from the target feature list. - std::string FS; - if (!Opts.Features.empty()) { - FS = Opts.Features[0]; - for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) - FS += "," + Opts.Features[i]; - } - - std::unique_ptr Str; - - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr STI( - TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); - - raw_pwrite_stream *Out = FDOS.get(); - std::unique_ptr BOS; - - // FIXME: There is a bit of code duplication with addPassesToEmitFile. - if (Opts.OutputType == AssemblerInvocation::FT_Asm) { - MCInstPrinter *IP = TheTarget->createMCInstPrinter( - llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); - - std::unique_ptr CE; - if (Opts.ShowEncoding) - CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MCTargetOptions MCOptions; - std::unique_ptr MAB( - TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); - - auto FOut = llvm::make_unique(*Out); - Str.reset(TheTarget->createAsmStreamer( - Ctx, std::move(FOut), /*asmverbose*/ true, - /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB), - Opts.ShowInst)); - } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { - Str.reset(createNullStreamer(Ctx)); - } else { - assert(Opts.OutputType == AssemblerInvocation::FT_Obj && - "Invalid file type!"); - if (!FDOS->supportsSeeking()) { - BOS = make_unique(*FDOS); - Out = BOS.get(); - } - - std::unique_ptr CE( - TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MCTargetOptions MCOptions; - std::unique_ptr MAB( - TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); - std::unique_ptr OW = - DwoOS ? MAB->createDwoObjectWriter(*Out, *DwoOS) - : MAB->createObjectWriter(*Out); - - Triple T(Opts.Triple); - Str.reset(TheTarget->createMCObjectStreamer( - T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI, - Opts.RelaxAll, Opts.IncrementalLinkerCompatible, - /*DWARFMustBeAtTheEnd*/ true)); - Str.get()->InitSections(Opts.NoExecStack); - } - - // Assembly to object compilation should leverage assembly info. - Str->setUseAssemblerInfoForParsing(true); - - bool Failed = false; - - std::unique_ptr Parser( - createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); - - // FIXME: init MCTargetOptions from sanitizer flags here. - MCTargetOptions Options; - std::unique_ptr TAP( - TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); - if (!TAP) - Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - // Set values for symbols, if any. - for (auto &S : Opts.SymbolDefs) { - auto Pair = StringRef(S).split('='); - auto Sym = Pair.first; - auto Val = Pair.second; - int64_t Value = 1; - // We have already error checked this in the driver. - Val.getAsInteger(0, Value); - Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); - } - - if (!Failed) { - Parser->setTargetParser(*TAP.get()); - Failed = Parser->Run(Opts.NoInitialTextSection); - } - - // Close Streamer first. - // It might have a reference to the output stream. - Str.reset(); - // Close the output stream early. - BOS.reset(); - FDOS.reset(); - - // Delete output file if there were errors. - if (Failed) { - if (Opts.OutputPath != "-") - sys::fs::remove(Opts.OutputPath); - if (!Opts.SplitDwarfFile.empty() && Opts.SplitDwarfFile != "-") - sys::fs::remove(Opts.SplitDwarfFile); - } - - return Failed; -} - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // We cannot recover from llvm errors. - exit(1); -} - -int cc1as_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - // Initialize targets and assembly printers/parsers. - InitializeAllTargetInfos(); - InitializeAllTargetMCs(); - InitializeAllAsmParsers(); - - // Construct our diagnostic client. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(errs(), &*DiagOpts); - DiagClient->setPrefix("clang -cc1as"); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - ScopedFatalErrorHandler FatalErrorHandler - (LLVMErrorHandler, static_cast(&Diags)); - - // Parse the arguments. - AssemblerInvocation Asm; - if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) - return 1; - - if (Asm.ShowHelp) { - std::unique_ptr Opts(driver::createDriverOptTable()); - Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", - /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, - /*ShowAllAliases=*/false); - return 0; - } - - // Honor -version. - // - // FIXME: Use a better -version message? - if (Asm.ShowVersion) { - llvm::cl::PrintVersionMessage(); - return 0; - } - - // Honor -mllvm. - // - // FIXME: Remove this, one day. - if (!Asm.LLVMArgs.empty()) { - unsigned NumArgs = Asm.LLVMArgs.size(); - auto Args = llvm::make_unique(NumArgs + 2); - Args[0] = "clang (LLVM option parsing)"; - for (unsigned i = 0; i != NumArgs; ++i) - Args[i + 1] = Asm.LLVMArgs[i].c_str(); - Args[NumArgs + 1] = nullptr; - llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); - } - - // Execute the invocation, unless there were parsing errors. - bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); - - // If any timers were active but haven't been destroyed yet, print their - // results now. - TimerGroup::printAll(errs()); - - return !!Failed; -} diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp deleted file mode 100644 index a4c034d8d35..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp +++ /dev/null @@ -1,196 +0,0 @@ -//===-- cc1gen_reproducer_main.cpp - Clang reproducer generator ----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1gen-reproducer functionality, which -// generates reproducers for invocations for clang-based tools. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/LLVM.h" -#include "clang/Basic/VirtualFileSystem.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/Driver.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/YAMLTraits.h" -#include "llvm/Support/raw_ostream.h" - -using namespace clang; - -namespace { - -struct UnsavedFileHash { - std::string Name; - std::string MD5; -}; - -struct ClangInvocationInfo { - std::string Toolchain; - std::string LibclangOperation; - std::string LibclangOptions; - std::vector Arguments; - std::vector InvocationArguments; - std::vector UnsavedFileHashes; - bool Dump = false; -}; - -} // end anonymous namespace - -LLVM_YAML_IS_SEQUENCE_VECTOR(UnsavedFileHash) - -namespace llvm { -namespace yaml { - -template <> struct MappingTraits { - static void mapping(IO &IO, UnsavedFileHash &Info) { - IO.mapRequired("name", Info.Name); - IO.mapRequired("md5", Info.MD5); - } -}; - -template <> struct MappingTraits { - static void mapping(IO &IO, ClangInvocationInfo &Info) { - IO.mapRequired("toolchain", Info.Toolchain); - IO.mapOptional("libclang.operation", Info.LibclangOperation); - IO.mapOptional("libclang.opts", Info.LibclangOptions); - IO.mapRequired("args", Info.Arguments); - IO.mapOptional("invocation-args", Info.InvocationArguments); - IO.mapOptional("unsaved_file_hashes", Info.UnsavedFileHashes); - } -}; - -} // end namespace yaml -} // end namespace llvm - -static std::string generateReproducerMetaInfo(const ClangInvocationInfo &Info) { - std::string Result; - llvm::raw_string_ostream OS(Result); - OS << '{'; - bool NeedComma = false; - auto EmitKey = [&](StringRef Key) { - if (NeedComma) - OS << ", "; - NeedComma = true; - OS << '"' << Key << "\": "; - }; - auto EmitStringKey = [&](StringRef Key, StringRef Value) { - if (Value.empty()) - return; - EmitKey(Key); - OS << '"' << Value << '"'; - }; - EmitStringKey("libclang.operation", Info.LibclangOperation); - EmitStringKey("libclang.opts", Info.LibclangOptions); - if (!Info.InvocationArguments.empty()) { - EmitKey("invocation-args"); - OS << '['; - for (const auto &Arg : llvm::enumerate(Info.InvocationArguments)) { - if (Arg.index()) - OS << ','; - OS << '"' << Arg.value() << '"'; - } - OS << ']'; - } - OS << '}'; - // FIXME: Compare unsaved file hashes and report mismatch in the reproducer. - if (Info.Dump) - llvm::outs() << "REPRODUCER METAINFO: " << OS.str() << "\n"; - return std::move(OS.str()); -} - -/// Generates a reproducer for a set of arguments from a specific invocation. -static llvm::Optional -generateReproducerForInvocationArguments(ArrayRef Argv, - const ClangInvocationInfo &Info) { - using namespace driver; - auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(Argv[0]); - - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions; - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, new IgnoringDiagConsumer()); - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - Driver TheDriver(Argv[0], llvm::sys::getDefaultTargetTriple(), Diags); - TheDriver.setTargetAndMode(TargetAndMode); - - std::unique_ptr C(TheDriver.BuildCompilation(Argv)); - if (C && !C->containsError()) { - for (const auto &J : C->getJobs()) { - if (const Command *Cmd = dyn_cast(&J)) { - Driver::CompilationDiagnosticReport Report; - TheDriver.generateCompilationDiagnostics( - *C, *Cmd, generateReproducerMetaInfo(Info), &Report); - return Report; - } - } - } - - return None; -} - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes); - -static void printReproducerInformation( - llvm::raw_ostream &OS, const ClangInvocationInfo &Info, - const driver::Driver::CompilationDiagnosticReport &Report) { - OS << "REPRODUCER:\n"; - OS << "{\n"; - OS << R"("files":[)"; - for (const auto &File : llvm::enumerate(Report.TemporaryFiles)) { - if (File.index()) - OS << ','; - OS << '"' << File.value() << '"'; - } - OS << "]\n}\n"; -} - -int cc1gen_reproducer_main(ArrayRef Argv, const char *Argv0, - void *MainAddr) { - if (Argv.size() < 1) { - llvm::errs() << "error: missing invocation file\n"; - return 1; - } - // Parse the invocation descriptor. - StringRef Input = Argv[0]; - llvm::ErrorOr> Buffer = - llvm::MemoryBuffer::getFile(Input); - if (!Buffer) { - llvm::errs() << "error: failed to read " << Input << ": " - << Buffer.getError().message() << "\n"; - return 1; - } - llvm::yaml::Input YAML(Buffer.get()->getBuffer()); - ClangInvocationInfo InvocationInfo; - YAML >> InvocationInfo; - if (Argv.size() > 1 && Argv[1] == StringRef("-v")) - InvocationInfo.Dump = true; - - // Create an invocation that will produce the reproducer. - std::vector DriverArgs; - for (const auto &Arg : InvocationInfo.Arguments) - DriverArgs.push_back(Arg.c_str()); - std::string Path = GetExecutablePath(Argv0, /*CanonicalPrefixes=*/true); - DriverArgs[0] = Path.c_str(); - llvm::Optional Report = - generateReproducerForInvocationArguments(DriverArgs, InvocationInfo); - - // Emit the information about the reproduce files to stdout. - int Result = 1; - if (Report) { - printReproducerInformation(llvm::outs(), InvocationInfo, *Report); - Result = 0; - } - - // Remove the input file. - llvm::sys::fs::remove(Input); - return Result; -} diff --git a/dbms/programs/clang/Compiler-7.0.0/driver.cpp b/dbms/programs/clang/Compiler-7.0.0/driver.cpp deleted file mode 100644 index 79d71b08ba7..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/driver.cpp +++ /dev/null @@ -1,514 +0,0 @@ -//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang driver; it is a thin wrapper -// for functionality in the Driver clang library. -// -//===----------------------------------------------------------------------===// - -#include "clang/Driver/Driver.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Driver/ToolChain.h" -#include "clang/Frontend/ChainedDiagnosticConsumer.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/SerializedDiagnosticPrinter.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/InitLLVM.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Process.h" -#include "llvm/Support/Program.h" -#include "llvm/Support/Regex.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/StringSaver.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace llvm::opt; - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { - if (!CanonicalPrefixes) { - SmallString<128> ExecutablePath(Argv0); - // Do a PATH lookup if Argv0 isn't a valid path. - if (!llvm::sys::fs::exists(ExecutablePath)) - if (llvm::ErrorOr P = - llvm::sys::findProgramByName(ExecutablePath)) - ExecutablePath = *P; - return ExecutablePath.str(); - } - - // This just needs to be some symbol in the binary; C++ doesn't - // allow taking the address of ::main however. - void *P = (void*) (intptr_t) GetExecutablePath; - return llvm::sys::fs::getMainExecutable(Argv0, P); -} - -static const char *GetStableCStr(std::set &SavedStrings, - StringRef S) { - return SavedStrings.insert(S).first->c_str(); -} - -/// ApplyQAOverride - Apply a list of edits to the input argument lists. -/// -/// The input string is a space separate list of edits to perform, -/// they are applied in order to the input argument lists. Edits -/// should be one of the following forms: -/// -/// '#': Silence information about the changes to the command line arguments. -/// -/// '^': Add FOO as a new argument at the beginning of the command line. -/// -/// '+': Add FOO as a new argument at the end of the command line. -/// -/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command -/// line. -/// -/// 'xOPTION': Removes all instances of the literal argument OPTION. -/// -/// 'XOPTION': Removes all instances of the literal argument OPTION, -/// and the following argument. -/// -/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' -/// at the end of the command line. -/// -/// \param OS - The stream to write edit information to. -/// \param Args - The vector of command line arguments. -/// \param Edit - The override command to perform. -/// \param SavedStrings - Set to use for storing string representations. -static void ApplyOneQAOverride(raw_ostream &OS, - SmallVectorImpl &Args, - StringRef Edit, - std::set &SavedStrings) { - // This does not need to be efficient. - - if (Edit[0] == '^') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at beginning\n"; - Args.insert(Args.begin() + 1, Str); - } else if (Edit[0] == '+') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at end\n"; - Args.push_back(Str); - } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { - StringRef MatchPattern = Edit.substr(2).split('/').first; - StringRef ReplPattern = Edit.substr(2).split('/').second; - ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); - - for (unsigned i = 1, e = Args.size(); i != e; ++i) { - // Ignore end-of-line response file markers - if (Args[i] == nullptr) - continue; - std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); - - if (Repl != Args[i]) { - OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; - Args[i] = GetStableCStr(SavedStrings, Repl); - } - } - } else if (Edit[0] == 'x' || Edit[0] == 'X') { - auto Option = Edit.substr(1); - for (unsigned i = 1; i < Args.size();) { - if (Option == Args[i]) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - if (Edit[0] == 'X') { - if (i < Args.size()) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - OS << "### Invalid X edit, end of command line!\n"; - } - } else - ++i; - } - } else if (Edit[0] == 'O') { - for (unsigned i = 1; i < Args.size();) { - const char *A = Args[i]; - // Ignore end-of-line response file markers - if (A == nullptr) - continue; - if (A[0] == '-' && A[1] == 'O' && - (A[2] == '\0' || - (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || - ('0' <= A[2] && A[2] <= '9'))))) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - ++i; - } - OS << "### Adding argument " << Edit << " at end\n"; - Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); - } else { - OS << "### Unrecognized edit: " << Edit << "\n"; - } -} - -/// ApplyQAOverride - Apply a comma separate list of edits to the -/// input argument lists. See ApplyOneQAOverride. -static void ApplyQAOverride(SmallVectorImpl &Args, - const char *OverrideStr, - std::set &SavedStrings) { - raw_ostream *OS = &llvm::errs(); - - if (OverrideStr[0] == '#') { - ++OverrideStr; - OS = &llvm::nulls(); - } - - *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; - - // This does not need to be efficient. - - const char *S = OverrideStr; - while (*S) { - const char *End = ::strchr(S, ' '); - if (!End) - End = S + strlen(S); - if (End != S) - ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); - S = End; - if (*S != '\0') - ++S; - } -} - -extern int cc1_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1as_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1gen_reproducer_main(ArrayRef Argv, - const char *Argv0, void *MainAddr); - -static void insertTargetAndModeArgs(const ParsedClangName &NameParts, - SmallVectorImpl &ArgVector, - std::set &SavedStrings) { - // Put target and mode arguments at the start of argument list so that - // arguments specified in command line could override them. Avoid putting - // them at index 0, as an option like '-cc1' must remain the first. - int InsertionPoint = 0; - if (ArgVector.size() > 0) - ++InsertionPoint; - - if (NameParts.DriverMode) { - // Add the mode flag to the arguments. - ArgVector.insert(ArgVector.begin() + InsertionPoint, - GetStableCStr(SavedStrings, NameParts.DriverMode)); - } - - if (NameParts.TargetIsValid) { - const char *arr[] = {"-target", GetStableCStr(SavedStrings, - NameParts.TargetPrefix)}; - ArgVector.insert(ArgVector.begin() + InsertionPoint, - std::begin(arr), std::end(arr)); - } -} - -static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, - SmallVectorImpl &Opts) { - llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); - // The first instance of '#' should be replaced with '=' in each option. - for (const char *Opt : Opts) - if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) - *NumberSignPtr = '='; -} - -static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { - // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. - TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); - if (TheDriver.CCPrintOptions) - TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); - - // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. - TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); - if (TheDriver.CCPrintHeaders) - TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); - - // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. - TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); - if (TheDriver.CCLogDiagnostics) - TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); -} - -static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, - const std::string &Path) { - // If the clang binary happens to be named cl.exe for compatibility reasons, - // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. - StringRef ExeBasename(llvm::sys::path::filename(Path)); - if (ExeBasename.equals_lower("cl.exe")) - ExeBasename = "clang-cl.exe"; - DiagClient->setPrefix(ExeBasename); -} - -// This lets us create the DiagnosticsEngine with a properly-filled-out -// DiagnosticOptions instance. -static DiagnosticOptions * -CreateAndPopulateDiagOpts(ArrayRef argv) { - auto *DiagOpts = new DiagnosticOptions; - std::unique_ptr Opts(createDriverOptTable()); - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = - Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); - // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. - // Any errors that would be diagnosed here will also be diagnosed later, - // when the DiagnosticsEngine actually exists. - (void)ParseDiagnosticArgs(*DiagOpts, Args); - return DiagOpts; -} - -static void SetInstallDir(SmallVectorImpl &argv, - Driver &TheDriver, bool CanonicalPrefixes) { - // Attempt to find the original path used to invoke the driver, to determine - // the installed path. We do this manually, because we want to support that - // path being a symlink. - SmallString<128> InstalledPath(argv[0]); - - // Do a PATH lookup, if there are no directory components. - if (llvm::sys::path::filename(InstalledPath) == InstalledPath) - if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( - llvm::sys::path::filename(InstalledPath.str()))) - InstalledPath = *Tmp; - - // FIXME: We don't actually canonicalize this, we just make it absolute. - if (CanonicalPrefixes) - llvm::sys::fs::make_absolute(InstalledPath); - - StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); - if (llvm::sys::fs::exists(InstalledPathParent)) - TheDriver.setInstalledDir(InstalledPathParent); -} - -static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { - void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") - return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") - return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "gen-reproducer") - return cc1gen_reproducer_main(argv.slice(2), argv[0], GetExecutablePathVP); - - // Reject unknown tools. - llvm::errs() << "error: unknown integrated tool '" << Tool << "'. " - << "Valid tools include '-cc1' and '-cc1as'.\n"; - return 1; -} - -int mainEntryClickHouseClang(int argc_, /* const */ char **argv_) { - llvm::InitLLVM X(argc_, argv_); - SmallVector argv(argv_, argv_ + argc_); - - if (llvm::sys::Process::FixupStandardFileDescriptors()) - return 1; - - llvm::InitializeAllTargets(); - auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); - - llvm::BumpPtrAllocator A; - llvm::StringSaver Saver(A); - - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: argv[0] is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. - bool ClangCLMode = false; - if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || - std::find_if(argv.begin(), argv.end(), [](const char *F) { - return F && strcmp(F, "--driver-mode=cl") == 0; - }) != argv.end()) { - ClangCLMode = true; - } - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : argv) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - - // Determines whether we want nullptr markers in argv to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); - - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), - [](const char *A) { return A != nullptr; }); - if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); - argv.resize(newEnd - argv.begin()); - } - return ExecuteCC1Tool(argv, argv[1] + 4); - } - - bool CanonicalPrefixes = true; - for (int i = 1, size = argv.size(); i < size; ++i) { - // Skip end-of-line response file markers - if (argv[i] == nullptr) - continue; - if (StringRef(argv[i]) == "-no-canonical-prefixes") { - CanonicalPrefixes = false; - break; - } - } - - // Handle CL and _CL_ which permits additional command line options to be - // prepended or appended. - if (ClangCLMode) { - // Arguments in "CL" are prepended. - llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); - if (OptCL.hasValue()) { - SmallVector PrependedOpts; - getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); - - // Insert right after the program name to prepend to the argument list. - argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); - } - // Arguments in "_CL_" are appended. - llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); - if (Opt_CL_.hasValue()) { - SmallVector AppendedOpts; - getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); - - // Insert at the end of the argument list to append. - argv.append(AppendedOpts.begin(), AppendedOpts.end()); - } - } - - std::set SavedStrings; - // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the - // scenes. - if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { - // FIXME: Driver shouldn't take extra initial argument. - ApplyQAOverride(argv, OverrideStr, SavedStrings); - } - - std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); - - IntrusiveRefCntPtr DiagOpts = - CreateAndPopulateDiagOpts(argv); - - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - FixupDiagPrefixExeName(DiagClient, Path); - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - if (!DiagOpts->DiagnosticSerializationFile.empty()) { - auto SerializedConsumer = - clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, - &*DiagOpts, /*MergeChildRecords=*/true); - Diags.setClient(new ChainedDiagnosticConsumer( - Diags.takeClient(), std::move(SerializedConsumer))); - } - - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - - Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); - SetInstallDir(argv, TheDriver, CanonicalPrefixes); - TheDriver.setTargetAndMode(TargetAndMode); - - insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); - - SetBackdoorDriverOutputsFromEnvVars(TheDriver); - - std::unique_ptr C(TheDriver.BuildCompilation(argv)); - int Res = 1; - if (C && !C->containsError()) { - SmallVector, 4> FailingCommands; - Res = TheDriver.ExecuteCompilation(*C, FailingCommands); - - // Force a crash to test the diagnostics. - if (TheDriver.GenReproducer) { - Diags.Report(diag::err_drv_force_crash) - << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); - - // Pretend that every command failed. - FailingCommands.clear(); - for (const auto &J : C->getJobs()) - if (const Command *C = dyn_cast(&J)) - FailingCommands.push_back(std::make_pair(-1, C)); - } - - for (const auto &P : FailingCommands) { - int CommandRes = P.first; - const Command *FailingCommand = P.second; - if (!Res) - Res = CommandRes; - - // If result status is < 0, then the driver command signalled an error. - // If result status is 70, then the driver command reported a fatal error. - // On Windows, abort will return an exit code of 3. In these cases, - // generate additional diagnostic information if possible. - bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; -#ifdef _WIN32 - DiagnoseCrash |= CommandRes == 3; -#endif - if (DiagnoseCrash) { - TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); - break; - } - } - } - - Diags.getClient()->finish(); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - -#ifdef _WIN32 - // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be - // propagated. - if (Res < 0) - Res = 1; -#endif - - // If we have multiple failing commands, we return the result of the first - // failing command. - return Res; -} diff --git a/dbms/programs/clang/Compiler-7.0.0/lld.cpp b/dbms/programs/clang/Compiler-7.0.0/lld.cpp deleted file mode 100644 index 8e118b6e24b..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0/lld.cpp +++ /dev/null @@ -1,150 +0,0 @@ -//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===// -// -// The LLVM Linker -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains the main function of the lld executable. The main -// function is a thin wrapper which dispatches to the platform specific -// driver. -// -// lld is a single executable that contains four different linkers for ELF, -// COFF, WebAssembly and Mach-O. The main function dispatches according to -// argv[0] (i.e. command name). The most common name for each target is shown -// below: -// -// - ld.lld: ELF (Unix) -// - ld64: Mach-O (macOS) -// - lld-link: COFF (Windows) -// - ld-wasm: WebAssembly -// -// lld can be invoked as "lld" along with "-flavor" option. This is for -// backward compatibility and not recommended. -// -//===----------------------------------------------------------------------===// - -#include "lld/Common/Driver.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/InitLLVM.h" -#include "llvm/Support/Path.h" -#include -using namespace lld; -using namespace llvm; -using namespace llvm::sys; - -/* - -enum Flavor { - Invalid, - Gnu, // -flavor gnu - WinLink, // -flavor link - Darwin, // -flavor darwin - Wasm, // -flavor wasm -}; - -LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) { - errs() << S << "\n"; - exit(1); -} - -static Flavor getFlavor(StringRef S) { - return StringSwitch(S) - .CasesLower("ld", "ld.lld", "gnu", Gnu) - .CasesLower("wasm", "ld-wasm", Wasm) - .CaseLower("link", WinLink) - .CasesLower("ld64", "ld64.lld", "darwin", Darwin) - .Default(Invalid); -} - -static bool isPETarget(const std::vector &V) { - for (auto It = V.begin(); It + 1 != V.end(); ++It) { - if (StringRef(*It) != "-m") - continue; - StringRef S = *(It + 1); - return S == "i386pe" || S == "i386pep" || S == "thumb2pe" || S == "arm64pe"; - } - return false; -} - -static Flavor parseProgname(StringRef Progname) { -#if __APPLE__ - // Use Darwin driver for "ld" on Darwin. - if (Progname == "ld") - return Darwin; -#endif - -#if LLVM_ON_UNIX - // Use GNU driver for "ld" on other Unix-like system. - if (Progname == "ld") - return Gnu; -#endif - - // Progname may be something like "lld-gnu". Parse it. - SmallVector V; - Progname.split(V, "-"); - for (StringRef S : V) - if (Flavor F = getFlavor(S)) - return F; - return Invalid; -} - -static Flavor parseFlavor(std::vector &V) { - // Parse -flavor option. - if (V.size() > 1 && V[1] == StringRef("-flavor")) { - if (V.size() <= 2) - die("missing arg value for '-flavor'"); - Flavor F = getFlavor(V[2]); - if (F == Invalid) - die("Unknown flavor: " + StringRef(V[2])); - V.erase(V.begin() + 1, V.begin() + 3); - return F; - } - - // Deduct the flavor from argv[0]. - StringRef Arg0 = path::filename(V[0]); - if (Arg0.endswith_lower(".exe")) - Arg0 = Arg0.drop_back(4); - return parseProgname(Arg0); -} -*/ - -// If this function returns true, lld calls _exit() so that it quickly -// exits without invoking destructors of globally allocated objects. -// -// We don't want to do that if we are running tests though, because -// doing that breaks leak sanitizer. So, lit sets this environment variable, -// and we use it to detect whether we are running tests or not. -static bool canExitEarly() { return StringRef(getenv("LLD_IN_TEST")) != "1"; } - -/// Universal linker main(). This linker emulates the gnu, darwin, or -/// windows linker based on the argv[0] or -flavor option. -int mainEntryClickHouseLLD(int Argc, /* const */ char **Argv) { - InitLLVM X(Argc, Argv); - - std::vector Args(Argv, Argv + Argc); -/* - switch (parseFlavor(Args)) { - case Gnu: - if (isPETarget(Args)) - return !mingw::link(Args); -*/ - return !elf::link(Args, canExitEarly()); -/* - case WinLink: - return !coff::link(Args, canExitEarly()); - case Darwin: - return !mach_o::link(Args, canExitEarly()); - case Wasm: - return !wasm::link(Args, canExitEarly()); - default: - die("lld is a generic driver.\n" - "Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-lld" - " (WebAssembly) instead"); - } -*/ -} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt deleted file mode 100644 index a5f8314b862..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) - -link_directories(${LLVM_LIBRARY_DIRS}) - -add_library(clickhouse-compiler-lib - driver.cpp - cc1_main.cpp - cc1as_main.cpp - lld.cpp) - -target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti - -llvm_libs_all(REQUIRED_LLVM_LIBRARIES) - -message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") - -target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) - -# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. - -target_link_libraries(clickhouse-compiler-lib PRIVATE - -clangBasic clangCodeGen clangDriver -clangFrontend -clangFrontendTool -clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend -clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers -clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic -clangCrossTU clangIndex - -lldCOFF -lldDriver -lldELF -lldMinGW -lldMachO -lldReaderWriter -lldYAML -lldCommon -lldCore - -${REQUIRED_LLVM_LIBRARIES} - -PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARIES} Threads::Threads -${MALLOC_LIBRARIES} -${GLIBC_COMPATIBILITY_LIBRARIES} -${MEMCPY_LIBRARIES} -) diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp deleted file mode 100644 index 3686475dd42..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp +++ /dev/null @@ -1,243 +0,0 @@ -//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1 functionality, which implements the -// core compiler functionality along with a number of additional tools for -// demonstration and testing purposes. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Option/Arg.h" -#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" -#include "clang/Config/config.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticBuffer.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "clang/FrontendTool/Utils.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/LinkAllPasses.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include - -#ifdef CLANG_HAVE_RLIMITS -#include -#endif - -// have no .a version in packages -#undef LINK_POLLY_INTO_TOOLS - -using namespace clang; -using namespace llvm::opt; - -//===----------------------------------------------------------------------===// -// Main driver -//===----------------------------------------------------------------------===// - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool GenCrashDiag) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // Run the interrupt handlers to make sure any special cleanups get done, in - // particular that we remove files registered with RemoveFileOnSignal. - llvm::sys::RunInterruptHandlers(); - - // We cannot recover from llvm errors. When reporting a fatal error, exit - // with status 70 to generate crash diagnostics. For BSD systems this is - // defined as an internal software error. Otherwise, exit with status 1. - exit(GenCrashDiag ? 70 : 1); -} - -#ifdef LINK_POLLY_INTO_TOOLS -namespace polly { -void initializePollyPasses(llvm::PassRegistry &Registry); -} -#endif - -#ifdef CLANG_HAVE_RLIMITS -// The amount of stack we think is "sufficient". If less than this much is -// available, we may be unable to reach our template instantiation depth -// limit and other similar limits. -// FIXME: Unify this with the stack we request when spawning a thread to build -// a module. -static const int kSufficientStack = 8 << 20; - -#if defined(__linux__) && defined(__PIE__) -static size_t getCurrentStackAllocation() { - // If we can't compute the current stack usage, allow for 512K of command - // line arguments and environment. - size_t Usage = 512 * 1024; - if (FILE *StatFile = fopen("/proc/self/stat", "r")) { - // We assume that the stack extends from its current address to the end of - // the environment space. In reality, there is another string literal (the - // program name) after the environment, but this is close enough (we only - // need to be within 100K or so). - unsigned long StackPtr, EnvEnd; - // Disable silly GCC -Wformat warning that complains about length - // modifiers on ignored format specifiers. We want to retain these - // for documentation purposes even though they have no effect. -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat" -#endif - if (fscanf(StatFile, - "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " - "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " - "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " - "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", - &StackPtr, &EnvEnd) == 2) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; - } - fclose(StatFile); - } - return Usage; -} - -#include - -LLVM_ATTRIBUTE_NOINLINE -static void ensureStackAddressSpace(int ExtraChunks = 0) { - // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary - // relatively close to the stack (they are only guaranteed to be 128MiB - // apart). This results in crashes if we happen to heap-allocate more than - // 128MiB before we reach our stack high-water mark. - // - // To avoid these crashes, ensure that we have sufficient virtual memory - // pages allocated before we start running. - size_t Curr = getCurrentStackAllocation(); - const int kTargetStack = kSufficientStack - 256 * 1024; - if (Curr < kTargetStack) { - volatile char *volatile Alloc = - static_cast(alloca(kTargetStack - Curr)); - Alloc[0] = 0; - Alloc[kTargetStack - Curr - 1] = 0; - } -} -#else -static void ensureStackAddressSpace() {} -#endif - -/// Attempt to ensure that we have at least 8MiB of usable stack space. -static void ensureSufficientStack() { - struct rlimit rlim; - if (getrlimit(RLIMIT_STACK, &rlim) != 0) - return; - - // Increase the soft stack limit to our desired level, if necessary and - // possible. - if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) { - // Try to allocate sufficient stack. - if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack) - rlim.rlim_cur = kSufficientStack; - else if (rlim.rlim_cur == rlim.rlim_max) - return; - else - rlim.rlim_cur = rlim.rlim_max; - - if (setrlimit(RLIMIT_STACK, &rlim) != 0 || - rlim.rlim_cur != kSufficientStack) - return; - } - - // We should now have a stack of size at least kSufficientStack. Ensure - // that we can actually use that much, if necessary. - ensureStackAddressSpace(); -} -#else -static void ensureSufficientStack() {} -#endif - -int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { - ensureSufficientStack(); - - std::unique_ptr Clang(new CompilerInstance()); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - // Register the support for object-file-wrapped Clang modules. - auto PCHOps = Clang->getPCHContainerOperations(); - PCHOps->registerWriter(llvm::make_unique()); - PCHOps->registerReader(llvm::make_unique()); - - // Initialize targets first, so that --version shows registered targets. - llvm::InitializeAllTargets(); - llvm::InitializeAllTargetMCs(); - llvm::InitializeAllAsmPrinters(); - llvm::InitializeAllAsmParsers(); - -#ifdef LINK_POLLY_INTO_TOOLS - llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); - polly::initializePollyPasses(Registry); -#endif - - // Buffer diagnostics from argument parsing so that we can output them using a - // well formed diagnostic object. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - bool Success = CompilerInvocation::CreateFromArgs( - Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); - - // Infer the builtin include path if unspecified. - if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && - Clang->getHeaderSearchOpts().ResourceDir.empty()) - Clang->getHeaderSearchOpts().ResourceDir = - CompilerInvocation::GetResourcesPath(Argv0, MainAddr); - - // Create the actual diagnostics engine. - Clang->createDiagnostics(); - if (!Clang->hasDiagnostics()) - return 1; - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - llvm::install_fatal_error_handler(LLVMErrorHandler, - static_cast(&Clang->getDiagnostics())); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); - if (!Success) - return 1; - - // Execute the frontend actions. - Success = ExecuteCompilerInvocation(Clang.get()); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - - // Our error handler depends on the Diagnostics object, which we're - // potentially about to delete. Uninstall the handler now so that any - // later errors use the default handling behavior instead. - llvm::remove_fatal_error_handler(); - - // When running with -disable-free, don't do any destruction or shutdown. - if (Clang->getFrontendOpts().DisableFree) { - BuryPointer(std::move(Clang)); - return !Success; - } - - return !Success; -} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp deleted file mode 100644 index ce23422077f..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp +++ /dev/null @@ -1,555 +0,0 @@ -//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang -cc1as functionality, which implements -// the direct interface to the LLVM MC based assembler. -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Frontend/FrontendDiagnostic.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/MC/MCAsmBackend.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCCodeEmitter.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCInstrInfo.h" -#include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCParser/MCAsmParser.h" -#include "llvm/MC/MCParser/MCTargetAsmParser.h" -#include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/MC/MCTargetOptions.h" -#include "llvm/Option/Arg.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace clang::driver::options; -using namespace llvm; -using namespace llvm::opt; - - -namespace { - -/// \brief Helper class for representing a single invocation of the assembler. -struct AssemblerInvocation { - /// @name Target Options - /// @{ - - /// The name of the target triple to assemble for. - std::string Triple; - - /// If given, the name of the target CPU to determine which instructions - /// are legal. - std::string CPU; - - /// The list of target specific features to enable or disable -- this should - /// be a list of strings starting with '+' or '-'. - std::vector Features; - - /// The list of symbol definitions. - std::vector SymbolDefs; - - /// @} - /// @name Language Options - /// @{ - - std::vector IncludePaths; - unsigned NoInitialTextSection : 1; - unsigned SaveTemporaryLabels : 1; - unsigned GenDwarfForAssembly : 1; - unsigned RelaxELFRelocations : 1; - unsigned DwarfVersion; - std::string DwarfDebugFlags; - std::string DwarfDebugProducer; - std::string DebugCompilationDir; - llvm::DebugCompressionType CompressDebugSections = - llvm::DebugCompressionType::None; - std::string MainFileName; - - /// @} - /// @name Frontend Options - /// @{ - - std::string InputFile; - std::vector LLVMArgs; - std::string OutputPath; - enum FileType { - FT_Asm, ///< Assembly (.s) output, transliterate mode. - FT_Null, ///< No output, for timing purposes. - FT_Obj ///< Object file output. - }; - FileType OutputType; - unsigned ShowHelp : 1; - unsigned ShowVersion : 1; - - /// @} - /// @name Transliterate Options - /// @{ - - unsigned OutputAsmVariant; - unsigned ShowEncoding : 1; - unsigned ShowInst : 1; - - /// @} - /// @name Assembler Options - /// @{ - - unsigned RelaxAll : 1; - unsigned NoExecStack : 1; - unsigned FatalWarnings : 1; - unsigned IncrementalLinkerCompatible : 1; - - /// The name of the relocation model to use. - std::string RelocationModel; - - /// @} - -public: - AssemblerInvocation() { - Triple = ""; - NoInitialTextSection = 0; - InputFile = "-"; - OutputPath = "-"; - OutputType = FT_Asm; - OutputAsmVariant = 0; - ShowInst = 0; - ShowEncoding = 0; - RelaxAll = 0; - NoExecStack = 0; - FatalWarnings = 0; - IncrementalLinkerCompatible = 0; - DwarfVersion = 0; - } - - static bool CreateFromArgs(AssemblerInvocation &Res, - ArrayRef Argv, - DiagnosticsEngine &Diags); -}; - -} - -bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, - ArrayRef Argv, - DiagnosticsEngine &Diags) { - bool Success = true; - - // Parse the arguments. - std::unique_ptr OptTbl(createDriverOptTable()); - - const unsigned IncludedFlagsBitmask = options::CC1AsOption; - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, - IncludedFlagsBitmask); - - // Check for missing argument error. - if (MissingArgCount) { - Diags.Report(diag::err_drv_missing_argument) - << Args.getArgString(MissingArgIndex) << MissingArgCount; - Success = false; - } - - // Issue errors on unknown arguments. - for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { - auto ArgString = A->getAsString(Args); - std::string Nearest; - if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1) - Diags.Report(diag::err_drv_unknown_argument) << ArgString; - else - Diags.Report(diag::err_drv_unknown_argument_with_suggestion) - << ArgString << Nearest; - Success = false; - } - - // Construct the invocation. - - // Target Options - Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); - Opts.CPU = Args.getLastArgValue(OPT_target_cpu); - Opts.Features = Args.getAllArgValues(OPT_target_feature); - - // Use the default target triple if unspecified. - if (Opts.Triple.empty()) - Opts.Triple = llvm::sys::getDefaultTargetTriple(); - - // Language Options - Opts.IncludePaths = Args.getAllArgValues(OPT_I); - Opts.NoInitialTextSection = Args.hasArg(OPT_n); - Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); - // Any DebugInfoKind implies GenDwarfForAssembly. - Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); - - if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, - OPT_compress_debug_sections_EQ)) { - if (A->getOption().getID() == OPT_compress_debug_sections) { - // TODO: be more clever about the compression type auto-detection - Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; - } else { - Opts.CompressDebugSections = - llvm::StringSwitch(A->getValue()) - .Case("none", llvm::DebugCompressionType::None) - .Case("zlib", llvm::DebugCompressionType::Z) - .Case("zlib-gnu", llvm::DebugCompressionType::GNU) - .Default(llvm::DebugCompressionType::None); - } - } - - Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); - Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); - Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); - Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); - Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); - Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); - - // Frontend Options - if (Args.hasArg(OPT_INPUT)) { - bool First = true; - for (const Arg *A : Args.filtered(OPT_INPUT)) { - if (First) { - Opts.InputFile = A->getValue(); - First = false; - } else { - Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); - Success = false; - } - } - } - Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); - Opts.OutputPath = Args.getLastArgValue(OPT_o); - if (Arg *A = Args.getLastArg(OPT_filetype)) { - StringRef Name = A->getValue(); - unsigned OutputType = StringSwitch(Name) - .Case("asm", FT_Asm) - .Case("null", FT_Null) - .Case("obj", FT_Obj) - .Default(~0U); - if (OutputType == ~0U) { - Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; - Success = false; - } else - Opts.OutputType = FileType(OutputType); - } - Opts.ShowHelp = Args.hasArg(OPT_help); - Opts.ShowVersion = Args.hasArg(OPT_version); - - // Transliterate Options - Opts.OutputAsmVariant = - getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); - Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); - Opts.ShowInst = Args.hasArg(OPT_show_inst); - - // Assemble Options - Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); - Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); - Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); - Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); - Opts.IncrementalLinkerCompatible = - Args.hasArg(OPT_mincremental_linker_compatible); - Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); - - return Success; -} - -static std::unique_ptr -getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags, - bool Binary) { - if (Opts.OutputPath.empty()) - Opts.OutputPath = "-"; - - // Make sure that the Out file gets unlinked from the disk if we get a - // SIGINT. - if (Opts.OutputPath != "-") - sys::RemoveFileOnSignal(Opts.OutputPath); - - std::error_code EC; - auto Out = llvm::make_unique( - Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); - if (EC) { - Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath - << EC.message(); - return nullptr; - } - - return Out; -} - -static bool ExecuteAssembler(AssemblerInvocation &Opts, - DiagnosticsEngine &Diags) { - // Get the target specific parser. - std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); - if (!TheTarget) - return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - ErrorOr> Buffer = - MemoryBuffer::getFileOrSTDIN(Opts.InputFile); - - if (std::error_code EC = Buffer.getError()) { - Error = EC.message(); - return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; - } - - SourceMgr SrcMgr; - - // Tell SrcMgr about this buffer, which is what the parser will pick up. - SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); - - // Record the location of the include directories so that the lexer can find - // it later. - SrcMgr.setIncludeDirs(Opts.IncludePaths); - - std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); - assert(MRI && "Unable to create target register info!"); - - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); - assert(MAI && "Unable to create target asm info!"); - - // Ensure MCAsmInfo initialization occurs before any use, otherwise sections - // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); - - bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; - std::unique_ptr FDOS = getOutputStream(Opts, Diags, IsBinary); - if (!FDOS) - return true; - - // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and - // MCObjectFileInfo needs a MCContext reference in order to initialize itself. - std::unique_ptr MOFI(new MCObjectFileInfo()); - - MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); - - bool PIC = false; - if (Opts.RelocationModel == "static") { - PIC = false; - } else if (Opts.RelocationModel == "pic") { - PIC = true; - } else { - assert(Opts.RelocationModel == "dynamic-no-pic" && - "Invalid PIC model!"); - PIC = false; - } - - MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); - if (Opts.SaveTemporaryLabels) - Ctx.setAllowTemporaryLabels(false); - if (Opts.GenDwarfForAssembly) - Ctx.setGenDwarfForAssembly(true); - if (!Opts.DwarfDebugFlags.empty()) - Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); - if (!Opts.DwarfDebugProducer.empty()) - Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); - if (!Opts.DebugCompilationDir.empty()) - Ctx.setCompilationDir(Opts.DebugCompilationDir); - if (!Opts.MainFileName.empty()) - Ctx.setMainFileName(StringRef(Opts.MainFileName)); - Ctx.setDwarfVersion(Opts.DwarfVersion); - - // Build up the feature string from the target feature list. - std::string FS; - if (!Opts.Features.empty()) { - FS = Opts.Features[0]; - for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) - FS += "," + Opts.Features[i]; - } - - std::unique_ptr Str; - - std::unique_ptr MCII(TheTarget->createMCInstrInfo()); - std::unique_ptr STI( - TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); - - raw_pwrite_stream *Out = FDOS.get(); - std::unique_ptr BOS; - - // FIXME: There is a bit of code duplication with addPassesToEmitFile. - if (Opts.OutputType == AssemblerInvocation::FT_Asm) { - MCInstPrinter *IP = TheTarget->createMCInstPrinter( - llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); - - std::unique_ptr CE; - if (Opts.ShowEncoding) - CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MCTargetOptions MCOptions; - std::unique_ptr MAB( - TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); - - auto FOut = llvm::make_unique(*Out); - Str.reset(TheTarget->createAsmStreamer( - Ctx, std::move(FOut), /*asmverbose*/ true, - /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB), - Opts.ShowInst)); - } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { - Str.reset(createNullStreamer(Ctx)); - } else { - assert(Opts.OutputType == AssemblerInvocation::FT_Obj && - "Invalid file type!"); - if (!FDOS->supportsSeeking()) { - BOS = make_unique(*FDOS); - Out = BOS.get(); - } - - std::unique_ptr CE( - TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); - MCTargetOptions MCOptions; - std::unique_ptr MAB( - TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); - - Triple T(Opts.Triple); - Str.reset(TheTarget->createMCObjectStreamer( - T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll, - Opts.IncrementalLinkerCompatible, - /*DWARFMustBeAtTheEnd*/ true)); - Str.get()->InitSections(Opts.NoExecStack); - } - - // Assembly to object compilation should leverage assembly info. - Str->setUseAssemblerInfoForParsing(true); - - bool Failed = false; - - std::unique_ptr Parser( - createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); - - // FIXME: init MCTargetOptions from sanitizer flags here. - MCTargetOptions Options; - std::unique_ptr TAP( - TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); - if (!TAP) - Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; - - // Set values for symbols, if any. - for (auto &S : Opts.SymbolDefs) { - auto Pair = StringRef(S).split('='); - auto Sym = Pair.first; - auto Val = Pair.second; - int64_t Value; - // We have already error checked this in the driver. - Val.getAsInteger(0, Value); - Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); - } - - if (!Failed) { - Parser->setTargetParser(*TAP.get()); - Failed = Parser->Run(Opts.NoInitialTextSection); - } - - // Close Streamer first. - // It might have a reference to the output stream. - Str.reset(); - // Close the output stream early. - BOS.reset(); - FDOS.reset(); - - // Delete output file if there were errors. - if (Failed && Opts.OutputPath != "-") - sys::fs::remove(Opts.OutputPath); - - return Failed; -} - -static void LLVMErrorHandler(void *UserData, const std::string &Message, - bool /*GenCrashDiag*/) { - DiagnosticsEngine &Diags = *static_cast(UserData); - - Diags.Report(diag::err_fe_error_backend) << Message; - - // We cannot recover from llvm errors. - exit(1); -} - -int cc1as_main(ArrayRef Argv, const char */*Argv0*/, void */*MainAddr*/) { - // Initialize targets and assembly printers/parsers. - InitializeAllTargetInfos(); - InitializeAllTargetMCs(); - InitializeAllAsmParsers(); - - // Construct our diagnostic client. - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(errs(), &*DiagOpts); - DiagClient->setPrefix("clang -cc1as"); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - // Set an error handler, so that any LLVM backend diagnostics go through our - // error handler. - ScopedFatalErrorHandler FatalErrorHandler - (LLVMErrorHandler, static_cast(&Diags)); - - // Parse the arguments. - AssemblerInvocation Asm; - if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) - return 1; - - if (Asm.ShowHelp) { - std::unique_ptr Opts(driver::createDriverOptTable()); - Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", - /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, - /*ShowAllAliases=*/false); - return 0; - } - - // Honor -version. - // - // FIXME: Use a better -version message? - if (Asm.ShowVersion) { - llvm::cl::PrintVersionMessage(); - return 0; - } - - // Honor -mllvm. - // - // FIXME: Remove this, one day. - if (!Asm.LLVMArgs.empty()) { - unsigned NumArgs = Asm.LLVMArgs.size(); - auto Args = llvm::make_unique(NumArgs + 2); - Args[0] = "clang (LLVM option parsing)"; - for (unsigned i = 0; i != NumArgs; ++i) - Args[i + 1] = Asm.LLVMArgs[i].c_str(); - Args[NumArgs + 1] = nullptr; - llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); - } - - // Execute the invocation, unless there were parsing errors. - bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); - - // If any timers were active but haven't been destroyed yet, print their - // results now. - TimerGroup::printAll(errs()); - - return !!Failed; -} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp deleted file mode 100644 index 9a061b9d137..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp +++ /dev/null @@ -1,512 +0,0 @@ -//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This is the entry point to the clang driver; it is a thin wrapper -// for functionality in the Driver clang library. -// -//===----------------------------------------------------------------------===// - -#include "clang/Driver/Driver.h" -#include "clang/Basic/DiagnosticOptions.h" -#include "clang/Driver/Compilation.h" -#include "clang/Driver/DriverDiagnostic.h" -#include "clang/Driver/Options.h" -#include "clang/Driver/ToolChain.h" -#include "clang/Frontend/ChainedDiagnosticConsumer.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/SerializedDiagnosticPrinter.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Frontend/Utils.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Option/ArgList.h" -#include "llvm/Option/OptTable.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/InitLLVM.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Process.h" -#include "llvm/Support/Program.h" -#include "llvm/Support/Regex.h" -#include "llvm/Support/Signals.h" -#include "llvm/Support/StringSaver.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include -using namespace clang; -using namespace clang::driver; -using namespace llvm::opt; - -std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { - if (!CanonicalPrefixes) { - SmallString<128> ExecutablePath(Argv0); - // Do a PATH lookup if Argv0 isn't a valid path. - if (!llvm::sys::fs::exists(ExecutablePath)) - if (llvm::ErrorOr P = - llvm::sys::findProgramByName(ExecutablePath)) - ExecutablePath = *P; - return ExecutablePath.str(); - } - - // This just needs to be some symbol in the binary; C++ doesn't - // allow taking the address of ::main however. - void *P = (void*) (intptr_t) GetExecutablePath; - return llvm::sys::fs::getMainExecutable(Argv0, P); -} - -static const char *GetStableCStr(std::set &SavedStrings, - StringRef S) { - return SavedStrings.insert(S).first->c_str(); -} - -/// ApplyQAOverride - Apply a list of edits to the input argument lists. -/// -/// The input string is a space separate list of edits to perform, -/// they are applied in order to the input argument lists. Edits -/// should be one of the following forms: -/// -/// '#': Silence information about the changes to the command line arguments. -/// -/// '^': Add FOO as a new argument at the beginning of the command line. -/// -/// '+': Add FOO as a new argument at the end of the command line. -/// -/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command -/// line. -/// -/// 'xOPTION': Removes all instances of the literal argument OPTION. -/// -/// 'XOPTION': Removes all instances of the literal argument OPTION, -/// and the following argument. -/// -/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' -/// at the end of the command line. -/// -/// \param OS - The stream to write edit information to. -/// \param Args - The vector of command line arguments. -/// \param Edit - The override command to perform. -/// \param SavedStrings - Set to use for storing string representations. -static void ApplyOneQAOverride(raw_ostream &OS, - SmallVectorImpl &Args, - StringRef Edit, - std::set &SavedStrings) { - // This does not need to be efficient. - - if (Edit[0] == '^') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at beginning\n"; - Args.insert(Args.begin() + 1, Str); - } else if (Edit[0] == '+') { - const char *Str = - GetStableCStr(SavedStrings, Edit.substr(1)); - OS << "### Adding argument " << Str << " at end\n"; - Args.push_back(Str); - } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && - Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { - StringRef MatchPattern = Edit.substr(2).split('/').first; - StringRef ReplPattern = Edit.substr(2).split('/').second; - ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); - - for (unsigned i = 1, e = Args.size(); i != e; ++i) { - // Ignore end-of-line response file markers - if (Args[i] == nullptr) - continue; - std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); - - if (Repl != Args[i]) { - OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; - Args[i] = GetStableCStr(SavedStrings, Repl); - } - } - } else if (Edit[0] == 'x' || Edit[0] == 'X') { - auto Option = Edit.substr(1); - for (unsigned i = 1; i < Args.size();) { - if (Option == Args[i]) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - if (Edit[0] == 'X') { - if (i < Args.size()) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - OS << "### Invalid X edit, end of command line!\n"; - } - } else - ++i; - } - } else if (Edit[0] == 'O') { - for (unsigned i = 1; i < Args.size();) { - const char *A = Args[i]; - // Ignore end-of-line response file markers - if (A == nullptr) - continue; - if (A[0] == '-' && A[1] == 'O' && - (A[2] == '\0' || - (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || - ('0' <= A[2] && A[2] <= '9'))))) { - OS << "### Deleting argument " << Args[i] << '\n'; - Args.erase(Args.begin() + i); - } else - ++i; - } - OS << "### Adding argument " << Edit << " at end\n"; - Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); - } else { - OS << "### Unrecognized edit: " << Edit << "\n"; - } -} - -/// ApplyQAOverride - Apply a comma separate list of edits to the -/// input argument lists. See ApplyOneQAOverride. -static void ApplyQAOverride(SmallVectorImpl &Args, - const char *OverrideStr, - std::set &SavedStrings) { - raw_ostream *OS = &llvm::errs(); - - if (OverrideStr[0] == '#') { - ++OverrideStr; - OS = &llvm::nulls(); - } - - *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; - - // This does not need to be efficient. - - const char *S = OverrideStr; - while (*S) { - const char *End = ::strchr(S, ' '); - if (!End) - End = S + strlen(S); - if (End != S) - ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); - S = End; - if (*S != '\0') - ++S; - } -} - -extern int cc1_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1as_main(ArrayRef Argv, const char *Argv0, - void *MainAddr); -extern int cc1gen_reproducer_main(ArrayRef Argv, - const char *Argv0, void *MainAddr); - -static void insertTargetAndModeArgs(const ParsedClangName &NameParts, - SmallVectorImpl &ArgVector, - std::set &SavedStrings) { - // Put target and mode arguments at the start of argument list so that - // arguments specified in command line could override them. Avoid putting - // them at index 0, as an option like '-cc1' must remain the first. - int InsertionPoint = 0; - if (ArgVector.size() > 0) - ++InsertionPoint; - - if (NameParts.DriverMode) { - // Add the mode flag to the arguments. - ArgVector.insert(ArgVector.begin() + InsertionPoint, - GetStableCStr(SavedStrings, NameParts.DriverMode)); - } - - if (NameParts.TargetIsValid) { - const char *arr[] = {"-target", GetStableCStr(SavedStrings, - NameParts.TargetPrefix)}; - ArgVector.insert(ArgVector.begin() + InsertionPoint, - std::begin(arr), std::end(arr)); - } -} - -static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, - SmallVectorImpl &Opts) { - llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); - // The first instance of '#' should be replaced with '=' in each option. - for (const char *Opt : Opts) - if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) - *NumberSignPtr = '='; -} - -static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { - // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. - TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); - if (TheDriver.CCPrintOptions) - TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); - - // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. - TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); - if (TheDriver.CCPrintHeaders) - TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); - - // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. - TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); - if (TheDriver.CCLogDiagnostics) - TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); -} - -static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, - const std::string &Path) { - // If the clang binary happens to be named cl.exe for compatibility reasons, - // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. - StringRef ExeBasename(llvm::sys::path::filename(Path)); - if (ExeBasename.equals_lower("cl.exe")) - ExeBasename = "clang-cl.exe"; - DiagClient->setPrefix(ExeBasename); -} - -// This lets us create the DiagnosticsEngine with a properly-filled-out -// DiagnosticOptions instance. -static DiagnosticOptions * -CreateAndPopulateDiagOpts(ArrayRef argv) { - auto *DiagOpts = new DiagnosticOptions; - std::unique_ptr Opts(createDriverOptTable()); - unsigned MissingArgIndex, MissingArgCount; - InputArgList Args = - Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); - // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. - // Any errors that would be diagnosed here will also be diagnosed later, - // when the DiagnosticsEngine actually exists. - (void)ParseDiagnosticArgs(*DiagOpts, Args); - return DiagOpts; -} - -static void SetInstallDir(SmallVectorImpl &argv, - Driver &TheDriver, bool CanonicalPrefixes) { - // Attempt to find the original path used to invoke the driver, to determine - // the installed path. We do this manually, because we want to support that - // path being a symlink. - SmallString<128> InstalledPath(argv[0]); - - // Do a PATH lookup, if there are no directory components. - if (llvm::sys::path::filename(InstalledPath) == InstalledPath) - if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( - llvm::sys::path::filename(InstalledPath.str()))) - InstalledPath = *Tmp; - - // FIXME: We don't actually canonicalize this, we just make it absolute. - if (CanonicalPrefixes) - llvm::sys::fs::make_absolute(InstalledPath); - - StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); - if (llvm::sys::fs::exists(InstalledPathParent)) - TheDriver.setInstalledDir(InstalledPathParent); -} - -static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { - void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; - if (Tool == "") - return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); - if (Tool == "as") - return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); - - // Reject unknown tools. - llvm::errs() << "error: unknown integrated tool '" << Tool << "'. " - << "Valid tools include '-cc1' and '-cc1as'.\n"; - return 1; -} - -int mainEntryClickHouseClang(int argc_, char **argv_) { - llvm::InitLLVM X(argc_, argv_); - SmallVector argv(argv_, argv_ + argc_); - - if (llvm::sys::Process::FixupStandardFileDescriptors()) - return 1; - - llvm::InitializeAllTargets(); - auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); - - llvm::BumpPtrAllocator A; - llvm::StringSaver Saver(A); - - // Parse response files using the GNU syntax, unless we're in CL mode. There - // are two ways to put clang in CL compatibility mode: argv[0] is either - // clang-cl or cl, or --driver-mode=cl is on the command line. The normal - // command line parsing can't happen until after response file parsing, so we - // have to manually search for a --driver-mode=cl argument the hard way. - // Finally, our -cc1 tools don't care which tokenization mode we use because - // response files written by clang will tokenize the same way in either mode. - bool ClangCLMode = false; - if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || - std::find_if(argv.begin(), argv.end(), [](const char *F) { - return F && strcmp(F, "--driver-mode=cl") == 0; - }) != argv.end()) { - ClangCLMode = true; - } - enum { Default, POSIX, Windows } RSPQuoting = Default; - for (const char *F : argv) { - if (strcmp(F, "--rsp-quoting=posix") == 0) - RSPQuoting = POSIX; - else if (strcmp(F, "--rsp-quoting=windows") == 0) - RSPQuoting = Windows; - } - - // Determines whether we want nullptr markers in argv to indicate response - // files end-of-lines. We only use this for the /LINK driver argument with - // clang-cl.exe on Windows. - bool MarkEOLs = ClangCLMode; - - llvm::cl::TokenizerCallback Tokenizer; - if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) - Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; - else - Tokenizer = &llvm::cl::TokenizeGNUCommandLine; - - if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) - MarkEOLs = false; - llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); - - // Handle -cc1 integrated tools, even if -cc1 was expanded from a response - // file. - auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), - [](const char *A) { return A != nullptr; }); - if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { - // If -cc1 came from a response file, remove the EOL sentinels. - if (MarkEOLs) { - auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); - argv.resize(newEnd - argv.begin()); - } - return ExecuteCC1Tool(argv, argv[1] + 4); - } - - bool CanonicalPrefixes = true; - for (int i = 1, size = argv.size(); i < size; ++i) { - // Skip end-of-line response file markers - if (argv[i] == nullptr) - continue; - if (StringRef(argv[i]) == "-no-canonical-prefixes") { - CanonicalPrefixes = false; - break; - } - } - - // Handle CL and _CL_ which permits additional command line options to be - // prepended or appended. - if (ClangCLMode) { - // Arguments in "CL" are prepended. - llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); - if (OptCL.hasValue()) { - SmallVector PrependedOpts; - getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); - - // Insert right after the program name to prepend to the argument list. - argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); - } - // Arguments in "_CL_" are appended. - llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); - if (Opt_CL_.hasValue()) { - SmallVector AppendedOpts; - getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); - - // Insert at the end of the argument list to append. - argv.append(AppendedOpts.begin(), AppendedOpts.end()); - } - } - - std::set SavedStrings; - // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the - // scenes. - if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { - // FIXME: Driver shouldn't take extra initial argument. - ApplyQAOverride(argv, OverrideStr, SavedStrings); - } - - std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); - - IntrusiveRefCntPtr DiagOpts = - CreateAndPopulateDiagOpts(argv); - - TextDiagnosticPrinter *DiagClient - = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - FixupDiagPrefixExeName(DiagClient, Path); - - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); - - if (!DiagOpts->DiagnosticSerializationFile.empty()) { - auto SerializedConsumer = - clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, - &*DiagOpts, /*MergeChildRecords=*/true); - Diags.setClient(new ChainedDiagnosticConsumer( - Diags.takeClient(), std::move(SerializedConsumer))); - } - - ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); - - Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); - SetInstallDir(argv, TheDriver, CanonicalPrefixes); - TheDriver.setTargetAndMode(TargetAndMode); - - insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); - - SetBackdoorDriverOutputsFromEnvVars(TheDriver); - - std::unique_ptr C(TheDriver.BuildCompilation(argv)); - int Res = 1; - if (C && !C->containsError()) { - SmallVector, 4> FailingCommands; - Res = TheDriver.ExecuteCompilation(*C, FailingCommands); - - // Force a crash to test the diagnostics. - if (TheDriver.GenReproducer) { - Diags.Report(diag::err_drv_force_crash) - << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); - - // Pretend that every command failed. - FailingCommands.clear(); - for (const auto &J : C->getJobs()) - if (const Command *C = dyn_cast(&J)) - FailingCommands.push_back(std::make_pair(-1, C)); - } - - for (const auto &P : FailingCommands) { - int CommandRes = P.first; - const Command *FailingCommand = P.second; - if (!Res) - Res = CommandRes; - - // If result status is < 0, then the driver command signalled an error. - // If result status is 70, then the driver command reported a fatal error. - // On Windows, abort will return an exit code of 3. In these cases, - // generate additional diagnostic information if possible. - bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; -#ifdef _WIN32 - DiagnoseCrash |= CommandRes == 3; -#endif - if (DiagnoseCrash) { - TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); - break; - } - } - } - - Diags.getClient()->finish(); - - // If any timers were active but haven't been destroyed yet, print their - // results now. This happens in -disable-free mode. - llvm::TimerGroup::printAll(llvm::errs()); - -#ifdef _WIN32 - // Exit status should not be negative on Win32, unless abnormal termination. - // Once abnormal termiation was caught, negative status should not be - // propagated. - if (Res < 0) - Res = 1; -#endif - - // If we have multiple failing commands, we return the result of the first - // failing command. - return Res; -} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp deleted file mode 100644 index 203e50d42a9..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "lld/Common/Driver.h" -#include "llvm/Support/InitLLVM.h" -#include - -int mainEntryClickHouseLLD(int argc, char ** argv) -{ - llvm::InitLLVM X(argc, argv); - std::vector args(argv, argv + argc); - return !lld::elf::link(args, false); -} diff --git a/dbms/programs/clang/Compiler-7.0.0svn b/dbms/programs/clang/Compiler-7.0.0svn deleted file mode 120000 index eeeb5bbc2c0..00000000000 --- a/dbms/programs/clang/Compiler-7.0.0svn +++ /dev/null @@ -1 +0,0 @@ -Compiler-7.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/Compiler-7.0.1 b/dbms/programs/clang/Compiler-7.0.1 deleted file mode 120000 index eeeb5bbc2c0..00000000000 --- a/dbms/programs/clang/Compiler-7.0.1 +++ /dev/null @@ -1 +0,0 @@ -Compiler-7.0.0 \ No newline at end of file diff --git a/dbms/programs/clang/clickhouse-clang.cpp b/dbms/programs/clang/clickhouse-clang.cpp deleted file mode 100644 index 261ae18b6d3..00000000000 --- a/dbms/programs/clang/clickhouse-clang.cpp +++ /dev/null @@ -1,2 +0,0 @@ -int mainEntryClickHouseClang(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseClang(argc_, argv_); } diff --git a/dbms/programs/clang/clickhouse-lld.cpp b/dbms/programs/clang/clickhouse-lld.cpp deleted file mode 100644 index baa6182d66d..00000000000 --- a/dbms/programs/clang/clickhouse-lld.cpp +++ /dev/null @@ -1,2 +0,0 @@ -int mainEntryClickHouseLLD(int argc, char ** argv); -int main(int argc_, char ** argv_) { return mainEntryClickHouseLLD(argc_, argv_); } diff --git a/dbms/programs/clang/copy_headers.sh b/dbms/programs/clang/copy_headers.sh deleted file mode 100755 index 45a58855c91..00000000000 --- a/dbms/programs/clang/copy_headers.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -set -e -#set -x -#echo "Args: $*"; env | sort - -# Этот скрипт собирает все заголовочные файлы, нужные для компиляции некоторого translation unit-а -# и копирует их с сохранением путей в директорию DST. -# Это затем может быть использовано, чтобы скомпилировать translation unit на другом сервере, -# используя ровно такой же набор заголовочных файлов. -# -# Требуется clang, желательно наиболее свежий (trunk). -# -# Используется при сборке пакетов. -# Заголовочные файлы записываются в пакет clickhouse-common, в директорию /usr/share/clickhouse/headers. -# -# Если вы хотите установить их самостоятельно, без сборки пакета, -# чтобы clickhouse-server видел их там, где ожидается, выполните: -# -# sudo ./copy_headers.sh . /usr/share/clickhouse/headers/ - -SOURCE_PATH=${1:-../../..} -DST=${2:-$SOURCE_PATH/../headers} -BUILD_PATH=${BUILD_PATH=${3:-$SOURCE_PATH/build}} - -PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:$PATH" - -if [[ -z $CLANG ]]; then - CLANG="clang" -fi - -START_HEADERS=$(echo \ - $BUILD_PATH/dbms/src/Common/config_version.h \ - $SOURCE_PATH/dbms/src/Interpreters/SpecializedAggregator.h \ - $SOURCE_PATH/dbms/src/AggregateFunctions/AggregateFunction*.h) - -for header in $START_HEADERS; do - START_HEADERS_INCLUDE+="-include $header " -done - - -GCC_ROOT=`$CLANG -v 2>&1 | grep "Selected GCC installation"| sed -n -e 's/^.*: //p'` - -# TODO: Does not work on macos? -GCC_ROOT=${GCC_ROOT:=/usr/lib/clang/${CMAKE_CXX_COMPILER_VERSION}} - -# Опция -mcx16 для того, чтобы выбиралось больше заголовочных файлов (с запасом). -# The latter options are the same that are added while building packages. -for src_file in $(echo | $CLANG -M -xc++ -std=c++1z -Wall -Werror -msse2 -msse4 -mcx16 -mpopcnt -O3 -g -fPIC -fstack-protector -D_FORTIFY_SOURCE=2 \ - -I $GCC_ROOT/include \ - -I $GCC_ROOT/include-fixed \ - $(cat "$BUILD_PATH/include_directories.txt") \ - $START_HEADERS_INCLUDE \ - - | - tr -d '\\' | - sed -E -e 's/^-\.o://'); -do - dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") - dst_file=$(echo $dst_file | sed -E -e 's/build\///') # for simplicity reasons, will put generated headers near the rest. - mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$dst_file"; -done - - -# Копируем больше заголовочных файлов с интринсиками, так как на серверах, куда будут устанавливаться -# заголовочные файлы, будет использоваться опция -march=native. - -for src_file in $(ls -1 $($CLANG -v -xc++ - <<<'' 2>&1 | grep '^ /' | grep 'include' | grep -E '/lib/clang/|/include/clang/')/*.h | grep -vE 'arm|altivec|Intrin'); -do - dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$dst_file"; -done - -if [ -d "$SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail" ]; then - # Even more platform-specific headers - for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail/*); - do - dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$dst_file"; - done -fi - -if [ -d "$SOURCE_PATH/contrib/boost/boost/smart_ptr/detail" ]; then - for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/boost/smart_ptr/detail/*); - do - dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$dst_file"; - done -fi diff --git a/dbms/programs/main.cpp b/dbms/programs/main.cpp index 57821d854e9..3fbbcee0f15 100644 --- a/dbms/programs/main.cpp +++ b/dbms/programs/main.cpp @@ -56,11 +56,6 @@ int mainEntryClickHouseObfuscator(int argc, char ** argv); #endif -#if USE_EMBEDDED_COMPILER - int mainEntryClickHouseClang(int argc, char ** argv); - int mainEntryClickHouseLLD(int argc, char ** argv); -#endif - namespace { @@ -100,12 +95,6 @@ std::pair clickhouse_applications[] = #if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR) {"obfuscator", mainEntryClickHouseObfuscator}, #endif - -#if USE_EMBEDDED_COMPILER - {"clang", mainEntryClickHouseClang}, - {"clang++", mainEntryClickHouseClang}, - {"lld", mainEntryClickHouseLLD}, -#endif }; @@ -152,11 +141,6 @@ int main(int argc_, char ** argv_) /// will work only after additional call of this function. updatePHDRCache(); -#if USE_EMBEDDED_COMPILER - if (argc_ >= 2 && 0 == strcmp(argv_[1], "-cc1")) - return mainEntryClickHouseClang(argc_, argv_); -#endif - #if USE_TCMALLOC /** Without this option, tcmalloc returns memory to OS too frequently for medium-sized memory allocations * (like IO buffers, column vectors, hash tables, etc.), diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index b7db44700bb..3adae6d9e93 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -85,7 +85,6 @@ struct Settings : public SettingsCollection M(SettingTotalsMode, totals_mode, TotalsMode::AFTER_HAVING_EXCLUSIVE, "How to calculate TOTALS when HAVING is present, as well as when max_rows_to_group_by and group_by_overflow_mode = ‘any’ are present.") \ M(SettingFloat, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.") \ \ - M(SettingBool, compile, false, "Whether query compilation is enabled.") \ M(SettingBool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.") \ M(SettingBool, compile_expressions, false, "Compile some scalar functions and operators to native code.") \ M(SettingUInt64, min_count_to_compile, 3, "The number of structurally identical queries before they are compiled.") \ @@ -351,6 +350,7 @@ struct Settings : public SettingsCollection /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ + M(SettingBool, compile, false, "Whether query compilation is enabled. Will be removed after 2020-03-13") \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 33fbb903497..373b47f7315 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -25,10 +25,6 @@ #include #include -#if __has_include() -#include -#endif - namespace ProfileEvents { @@ -47,7 +43,6 @@ namespace DB namespace ErrorCodes { - extern const int CANNOT_COMPILE_CODE; extern const int TOO_MANY_ROWS; extern const int EMPTY_DATA_PASSED; extern const int CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS; @@ -195,200 +190,6 @@ Aggregator::Aggregator(const Params & params_) } -void Aggregator::compileIfPossible(AggregatedDataVariants::Type type) -{ - std::lock_guard lock(mutex); - - if (compiled_if_possible) - return; - - compiled_if_possible = true; - -#if !defined(INTERNAL_COMPILER_HEADERS) - throw Exception("Cannot compile code: Compiler disabled", ErrorCodes::CANNOT_COMPILE_CODE); -#else - std::string method_typename_single_level; - std::string method_typename_two_level; - - if (false) {} -#define M(NAME) \ - else if (type == AggregatedDataVariants::Type::NAME) \ - { \ - method_typename_single_level = "decltype(AggregatedDataVariants::" #NAME ")::element_type"; \ - method_typename_two_level = "decltype(AggregatedDataVariants::" #NAME "_two_level)::element_type"; \ - } - - APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) -#undef M - -#define M(NAME) \ - else if (type == AggregatedDataVariants::Type::NAME) \ - method_typename_single_level = "decltype(AggregatedDataVariants::" #NAME ")::element_type"; - - APPLY_FOR_VARIANTS_NOT_CONVERTIBLE_TO_TWO_LEVEL(M) -#undef M - else if (type == AggregatedDataVariants::Type::without_key) {} - else - throw Exception("Unknown aggregated data variant.", ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT); - - auto compiler_headers = Poco::Util::Application::instance().config().getString("compiler_headers", INTERNAL_COMPILER_HEADERS); - - /// List of types of aggregate functions. - std::stringstream aggregate_functions_typenames_str; - std::stringstream aggregate_functions_headers_args; - for (size_t i = 0; i < params.aggregates_size; ++i) - { - IAggregateFunction & func = *aggregate_functions[i]; - - int status = 0; - std::string type_name = demangle(typeid(func).name(), status); - - if (status) - throw Exception("Cannot compile code: cannot demangle name " + String(typeid(func).name()) - + ", status: " + toString(status), ErrorCodes::CANNOT_COMPILE_CODE); - - aggregate_functions_typenames_str << ((i != 0) ? ", " : "") << type_name; - - std::string header_path = func.getHeaderFilePath(); - auto pos = header_path.find("/AggregateFunctions/"); - - if (pos == std::string::npos) - throw Exception("Cannot compile code: unusual path of header file for aggregate function: " + header_path, - ErrorCodes::CANNOT_COMPILE_CODE); - - aggregate_functions_headers_args << "-include '" << compiler_headers << "/dbms/src"; - aggregate_functions_headers_args.write(&header_path[pos], header_path.size() - pos); - aggregate_functions_headers_args << "' "; - } - - aggregate_functions_headers_args << "-include '" << compiler_headers << "/dbms/src/Interpreters/SpecializedAggregator.h'"; - - std::string aggregate_functions_typenames = aggregate_functions_typenames_str.str(); - - std::stringstream key_str; - key_str << "Aggregate: "; - if (!method_typename_single_level.empty()) - key_str << method_typename_single_level + ", "; - key_str << aggregate_functions_typenames; - std::string key = key_str.str(); - - auto get_code = [method_typename_single_level, method_typename_two_level, aggregate_functions_typenames] - { - /// A short piece of code, which is an explicit instantiation of the template. - std::stringstream code; - code << /// No explicit inclusion of the header file. It is included using the -include compiler option. - "namespace DB\n" - "{\n" - "\n"; - - /// There can be up to two instantiations for the template - for normal and two_level options. - auto append_code_for_specialization = - [&code, &aggregate_functions_typenames] (const std::string & method_typename, const std::string & suffix) - { - code << - "template void Aggregator::executeSpecialized<\n" - " " << method_typename << ", TypeList<" << aggregate_functions_typenames << ">>(\n" - " " << method_typename << " &, Arena *, size_t, ColumnRawPtrs &,\n" - " AggregateColumns &, bool, AggregateDataPtr) const;\n" - "\n" - "static void wrapper" << suffix << "(\n" - " const Aggregator & aggregator,\n" - " " << method_typename << " & method,\n" - " Arena * arena,\n" - " size_t rows,\n" - " ColumnRawPtrs & key_columns,\n" - " Aggregator::AggregateColumns & aggregate_columns,\n" - " bool no_more_keys,\n" - " AggregateDataPtr overflow_row)\n" - "{\n" - " aggregator.executeSpecialized<\n" - " " << method_typename << ", TypeList<" << aggregate_functions_typenames << ">>(\n" - " method, arena, rows, key_columns, aggregate_columns, no_more_keys, overflow_row);\n" - "}\n" - "\n" - "void * getPtr" << suffix << "() __attribute__((__visibility__(\"default\")));\n" - "void * getPtr" << suffix << "()\n" /// Without this wrapper, it's not clear how to get the desired symbol from the compiled library. - "{\n" - " return reinterpret_cast(&wrapper" << suffix << ");\n" - "}\n"; - }; - - if (!method_typename_single_level.empty()) - append_code_for_specialization(method_typename_single_level, ""); - else - { - /// For `without_key` method. - code << - "template void Aggregator::executeSpecializedWithoutKey<\n" - " " << "TypeList<" << aggregate_functions_typenames << ">>(\n" - " AggregatedDataWithoutKey &, size_t, AggregateColumns &, Arena *) const;\n" - "\n" - "static void wrapper(\n" - " const Aggregator & aggregator,\n" - " AggregatedDataWithoutKey & method,\n" - " size_t rows,\n" - " Aggregator::AggregateColumns & aggregate_columns,\n" - " Arena * arena)\n" - "{\n" - " aggregator.executeSpecializedWithoutKey<\n" - " TypeList<" << aggregate_functions_typenames << ">>(\n" - " method, rows, aggregate_columns, arena);\n" - "}\n" - "\n" - "void * getPtr() __attribute__((__visibility__(\"default\")));\n" - "void * getPtr()\n" - "{\n" - " return reinterpret_cast(&wrapper);\n" - "}\n"; - } - - if (!method_typename_two_level.empty()) - append_code_for_specialization(method_typename_two_level, "TwoLevel"); - else - { - /// The stub. - code << - "void * getPtrTwoLevel() __attribute__((__visibility__(\"default\")));\n" - "void * getPtrTwoLevel()\n" - "{\n" - " return nullptr;\n" - "}\n"; - } - - code << - "}\n"; - - return code.str(); - }; - - auto compiled_data_owned_by_callback = compiled_data; - auto on_ready = [compiled_data_owned_by_callback] (SharedLibraryPtr & lib) - { - if (compiled_data_owned_by_callback.unique()) /// Aggregator is already destroyed. - return; - - compiled_data_owned_by_callback->compiled_aggregator = lib; - compiled_data_owned_by_callback->compiled_method_ptr = lib->get("_ZN2DB6getPtrEv")(); - compiled_data_owned_by_callback->compiled_two_level_method_ptr = lib->get("_ZN2DB14getPtrTwoLevelEv")(); - }; - - /** If the library has already been compiled, a non-zero SharedLibraryPtr is returned. - * If the library was not compiled, then the counter is incremented, and nullptr is returned. - * If the counter has reached the value min_count_to_compile, then the compilation starts asynchronously (in a separate thread) - * at the end of which `on_ready` callback is called. - */ - aggregate_functions_headers_args << " -Wno-unused-function"; - SharedLibraryPtr lib = params.compiler->getOrCount(key, params.min_count_to_compile, - aggregate_functions_headers_args.str(), - get_code, on_ready); - - /// If the result is already ready. - if (lib) - on_ready(lib); -#endif -} - - AggregatedDataVariants::Type Aggregator::chooseAggregationMethod() { /// If no keys. All aggregating to single row. @@ -720,9 +521,6 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re result.keys_size = params.keys_size; result.key_sizes = key_sizes; LOG_TRACE(log, "Aggregation method: " << result.getMethodName()); - - if (params.compiler) - compileIfPossible(result.type); } if (isCancelled()) @@ -794,67 +592,21 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re /// For the case when there are no keys (all aggregate into one row). if (result.type == AggregatedDataVariants::Type::without_key) { - /// If there is a dynamically compiled code. - if (compiled_data->compiled_method_ptr) - { - reinterpret_cast< - void (*)(const Aggregator &, AggregatedDataWithoutKey &, size_t, AggregateColumns &, Arena *)> - (compiled_data->compiled_method_ptr)(*this, result.without_key, rows, aggregate_columns, result.aggregates_pool); - } - else - executeWithoutKeyImpl(result.without_key, rows, aggregate_functions_instructions.data(), result.aggregates_pool); + executeWithoutKeyImpl(result.without_key, rows, aggregate_functions_instructions.data(), result.aggregates_pool); } else { /// This is where data is written that does not fit in `max_rows_to_group_by` with `group_by_overflow_mode = any`. AggregateDataPtr overflow_row_ptr = params.overflow_row ? result.without_key : nullptr; - bool is_two_level = result.isTwoLevel(); - - /// Compiled code, for the normal structure. - if (!is_two_level && compiled_data->compiled_method_ptr) - { - #define M(NAME, IS_TWO_LEVEL) \ - else if (result.type == AggregatedDataVariants::Type::NAME) \ - reinterpret_cast(compiled_data->compiled_method_ptr) \ - (*this, *result.NAME, result.aggregates_pool, rows, key_columns, aggregate_columns, \ - no_more_keys, overflow_row_ptr); - - if (false) {} - APPLY_FOR_AGGREGATED_VARIANTS(M) - #undef M - } - /// Compiled code, for a two-level structure. - else if (is_two_level && compiled_data->compiled_two_level_method_ptr) - { - #define M(NAME) \ - else if (result.type == AggregatedDataVariants::Type::NAME) \ - reinterpret_cast(compiled_data->compiled_two_level_method_ptr) \ - (*this, *result.NAME, result.aggregates_pool, rows, key_columns, aggregate_columns, \ - no_more_keys, overflow_row_ptr); - - if (false) {} - APPLY_FOR_VARIANTS_TWO_LEVEL(M) - #undef M - } - /// When there is no dynamically compiled code. - else - { #define M(NAME, IS_TWO_LEVEL) \ else if (result.type == AggregatedDataVariants::Type::NAME) \ executeImpl(*result.NAME, result.aggregates_pool, rows, key_columns, aggregate_functions_instructions.data(), \ no_more_keys, overflow_row_ptr); - if (false) {} - APPLY_FOR_AGGREGATED_VARIANTS(M) + if (false) {} + APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M - } } size_t result_size = result.sizeWithoutOverflowRow(); diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 41fd957345e..b48663ff689 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -24,7 +24,6 @@ #include #include -#include #include #include @@ -778,10 +777,6 @@ public: const size_t max_rows_to_group_by; const OverflowMode group_by_overflow_mode; - /// For dynamic compilation. - Compiler * compiler; - const UInt32 min_count_to_compile; - /// Two-level aggregation settings (used for a large number of keys). /** With how many keys or the size of the aggregation state in bytes, * two-level aggregation begins to be used. Enough to reach of at least one of the thresholds. @@ -805,7 +800,6 @@ public: const Block & src_header_, const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_rows_to_group_by_, OverflowMode group_by_overflow_mode_, - Compiler * compiler_, UInt32 min_count_to_compile_, size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, size_t max_bytes_before_external_group_by_, bool empty_result_for_aggregation_by_empty_set_, @@ -813,7 +807,6 @@ public: : src_header(src_header_), keys(keys_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()), overflow_row(overflow_row_), max_rows_to_group_by(max_rows_to_group_by_), group_by_overflow_mode(group_by_overflow_mode_), - compiler(compiler_), min_count_to_compile(min_count_to_compile_), group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_), max_bytes_before_external_group_by(max_bytes_before_external_group_by_), empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_), @@ -824,7 +817,7 @@ public: /// Only parameters that matter during merge. Params(const Block & intermediate_header_, const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_) - : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, nullptr, 0, 0, 0, 0, false, "", max_threads_) + : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, "", max_threads_) { intermediate_header = intermediate_header_; } @@ -956,26 +949,6 @@ protected: Logger * log = &Logger::get("Aggregator"); - /** Dynamically compiled library for aggregation, if any. - * The meaning of dynamic compilation is to specialize code - * for a specific list of aggregate functions. - * This allows you to expand the loop to create and update states of aggregate functions, - * and also use inline-code instead of virtual calls. - */ - struct CompiledData - { - SharedLibraryPtr compiled_aggregator; - - /// Obtained with dlsym. It is still necessary to make reinterpret_cast to the function pointer. - void * compiled_method_ptr = nullptr; - void * compiled_two_level_method_ptr = nullptr; - }; - /// shared_ptr - to pass into a callback, that can survive Aggregator. - std::shared_ptr compiled_data { new CompiledData }; - - bool compiled_if_possible = false; - void compileIfPossible(AggregatedDataVariants::Type type); - /// Returns true if you can abort the current task. CancellationHook isCancelled; @@ -1037,35 +1010,6 @@ protected: Method & method, IBlockOutputStream & out); -public: - /// Templates that are instantiated by dynamic code compilation - see SpecializedAggregator.h - - template - void executeSpecialized( - Method & method, - Arena * aggregates_pool, - size_t rows, - ColumnRawPtrs & key_columns, - AggregateColumns & aggregate_columns, - bool no_more_keys, - AggregateDataPtr overflow_row) const; - - template - void executeSpecializedCase( - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - size_t rows, - AggregateColumns & aggregate_columns, - AggregateDataPtr overflow_row) const; - - template - void executeSpecializedWithoutKey( - AggregatedDataWithoutKey & res, - size_t rows, - AggregateColumns & aggregate_columns, - Arena * arena) const; - protected: /// Merge NULL key data from hash table `src` into `dst`. template diff --git a/dbms/src/Interpreters/CMakeLists.txt b/dbms/src/Interpreters/CMakeLists.txt index 75771a07027..65172356645 100644 --- a/dbms/src/Interpreters/CMakeLists.txt +++ b/dbms/src/Interpreters/CMakeLists.txt @@ -1,70 +1,3 @@ - -if (OS_FREEBSD) - set (PATH_SHARE "/usr/local/share" CACHE STRING "") -else () - set (PATH_SHARE "/usr/share" CACHE STRING "") -endif () - -set (INTERNAL_COMPILER_BIN_ROOT "${CMAKE_INSTALL_FULL_BINDIR}/" CACHE STRING "") -set (INTERNAL_COMPILER_EXECUTABLE "clickhouse-clang" CACHE STRING "") -set (INTERNAL_LINKER_EXECUTABLE "clickhouse-lld" CACHE STRING "") - -# Disabling leak reporting for these tools -if (SANITIZE STREQUAL "address") - # Note that this doesn't work for setuid and setcap binaries - set(INTERNAL_COMPILER_ENV "env ASAN_OPTIONS=detect_leaks=0" CACHE STRING "") -else () - set(INTERNAL_COMPILER_ENV "" CACHE STRING "") -endif () - -set (INTERNAL_COMPILER_NO_WARNING OFF CACHE INTERNAL "") -set (INTERNAL_COMPILER_HEADERS_DIR "headers" CACHE STRING "") -set (INTERNAL_COMPILER_HEADERS_RELATIVE "${INTERNAL_COMPILER_HEADERS_DIR}/${VERSION_STRING}" CACHE STRING "") -set (INTERNAL_COMPILER_HEADERS "${PATH_SHARE}/clickhouse/${INTERNAL_COMPILER_HEADERS_RELATIVE}" CACHE STRING "") - -if(OS_FREEBSD) - set(INTERNAL_COMPILER_HEADERS_ROOT "" CACHE STRING "") -else() - set(INTERNAL_COMPILER_HEADERS_ROOT "${INTERNAL_COMPILER_HEADERS}" CACHE STRING "") - set(INTERNAL_COMPILER_CUSTOM_ROOT ON CACHE INTERNAL "") -endif() - -if(NOT INTERNAL_COMPILER_FLAGS) - set(INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -fPIC -fvisibility=hidden -fno-implement-inlines -Wno-unused-command-line-argument -Bprefix=${PATH_SHARE}/clickhouse" CACHE STRING "") - if(INTERNAL_COMPILER_CUSTOM_ROOT) - set(INTERNAL_COMPILER_FLAGS "${INTERNAL_COMPILER_FLAGS} -nostdinc -nostdinc++") - if(INTERNAL_COMPILER_HEADERS_ROOT) - set(INTERNAL_COMPILER_FLAGS "${INTERNAL_COMPILER_FLAGS} -isysroot=${INTERNAL_COMPILER_HEADERS_ROOT}") - endif() - endif() -endif() -# TODO: use libs from package: -nodefaultlibs -lm -lc -lgcc_s -lgcc -lc++ -lc++abi - -string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS}) -string(REPLACE "-no-pie" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS}) -if (INTERNAL_COMPILER_NO_WARNING) - string (REPLACE "-Wall" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS}) - string (REPLACE "-Wextra" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS}) - string (REPLACE "-Werror" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS}) -endif () - -list(GET Poco_INCLUDE_DIRS 0 Poco_Foundation_INCLUDE_DIR) -list(GET Poco_INCLUDE_DIRS 1 Poco_Util_INCLUDE_DIR) - -if (NOT DOUBLE_CONVERSION_INCLUDE_DIR) - get_target_property(DOUBLE_CONVERSION_INCLUDE_DIR ${DOUBLE_CONVERSION_LIBRARIES} INTERFACE_INCLUDE_DIRECTORIES) -endif () - -string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR ${DOUBLE_CONVERSION_INCLUDE_DIR}) -string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIRS}) -string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Poco_Foundation_INCLUDE_DIR ${Poco_Foundation_INCLUDE_DIR}) -string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Poco_Util_INCLUDE_DIR ${Poco_Util_INCLUDE_DIR}) - -message (STATUS "Using internal=${USE_INTERNAL_LLVM_LIBRARY} compiler=${USE_EMBEDDED_COMPILER}: headers=${INTERNAL_COMPILER_HEADERS} root=${INTERNAL_COMPILER_HEADERS_ROOT}: ${INTERNAL_COMPILER_ENV} ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE} ${INTERNAL_COMPILER_FLAGS}; ${INTERNAL_LINKER_EXECUTABLE}") - -set (CONFIG_COMPILE ${ClickHouse_BINARY_DIR}/dbms/src/Interpreters/config_compile.h) -configure_file (${ClickHouse_SOURCE_DIR}/dbms/src/Interpreters/config_compile.h.in ${CONFIG_COMPILE}) - if (ENABLE_TESTS) add_subdirectory (tests) endif () diff --git a/dbms/src/Interpreters/Compiler.cpp b/dbms/src/Interpreters/Compiler.cpp deleted file mode 100644 index 3b420b6acce..00000000000 --- a/dbms/src/Interpreters/Compiler.cpp +++ /dev/null @@ -1,326 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if __has_include() -#include -#endif - -namespace ProfileEvents -{ - extern const Event CompileAttempt; - extern const Event CompileSuccess; -} - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int CANNOT_DLOPEN; - extern const int CANNOT_COMPILE_CODE; -} - -Compiler::Compiler(const std::string & path_, size_t threads) - : path(path_), pool(threads) -{ - Poco::File(path).createDirectory(); - - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(path); dir_end != dir_it; ++dir_it) - { - const std::string & name = dir_it.name(); - if (endsWith(name, ".so")) - { - files.insert(name.substr(0, name.size() - 3)); - } - } - - LOG_INFO(log, "Having " << files.size() << " compiled files from previous start."); -} - -Compiler::~Compiler() -{ - LOG_DEBUG(log, "Waiting for threads to finish."); - pool.wait(); -} - - -static Compiler::HashedKey getHash(const std::string & key) -{ - SipHash hash; - - auto revision = ClickHouseRevision::get(); - hash.update(revision); - hash.update(key.data(), key.size()); - - Compiler::HashedKey res; - hash.get128(res.low, res.high); - return res; -} - - -/// Without .so extension. -static std::string hashedKeyToFileName(Compiler::HashedKey hashed_key) -{ - WriteBufferFromOwnString out; - out << hashed_key.low << '_' << hashed_key.high; - return out.str(); -} - - -SharedLibraryPtr Compiler::getOrCount( - const std::string & key, - UInt32 min_count_to_compile, - const std::string & additional_compiler_flags, - CodeGenerator get_code, - ReadyCallback on_ready) -{ - HashedKey hashed_key = getHash(key); - - std::lock_guard lock(mutex); - - UInt32 count = ++counts[hashed_key]; - - /// Is there a ready open library? Or, if the library is in the process of compiling, there will be nullptr. - Libraries::iterator libraries_it = libraries.find(hashed_key); - if (libraries.end() != libraries_it) - { - if (!libraries_it->second) - LOG_INFO(log, "Library " << hashedKeyToFileName(hashed_key) << " is already compiling or compilation was failed."); - - /// TODO In this case, after the compilation is finished, the callback will not be called. - - return libraries_it->second; - } - - /// Is there a file with the library left over from the previous launch? - std::string file_name = hashedKeyToFileName(hashed_key); - Files::iterator files_it = files.find(file_name); - if (files.end() != files_it) - { - std::string so_file_path = path + '/' + file_name + ".so"; - LOG_INFO(log, "Loading existing library " << so_file_path); - - SharedLibraryPtr lib; - - try - { - lib = std::make_shared(so_file_path); - } - catch (const Exception & e) - { - if (e.code() != ErrorCodes::CANNOT_DLOPEN) - throw; - - /// Found broken .so file (or file cannot be dlopened by whatever reason). - /// This could happen when filesystem is corrupted after server restart. - /// We remove the file - it will be recompiled on next attempt. - - tryLogCurrentException(log); - - files.erase(files_it); - Poco::File(so_file_path).remove(); - return nullptr; - } - - libraries[hashed_key] = lib; - return lib; - } - - /// Has min_count_to_compile been reached? - if (count >= min_count_to_compile) - { - /// The min_count_to_compile value of zero indicates the need for synchronous compilation. - - /// Indicates that the library is in the process of compiling. - libraries[hashed_key] = nullptr; - - LOG_INFO(log, "Compiling code " << file_name << ", key: " << key); - - if (min_count_to_compile == 0) - { - { - ext::unlock_guard unlock(mutex); - compile(hashed_key, file_name, additional_compiler_flags, get_code, on_ready); - } - - return libraries[hashed_key]; - } - else - { - bool res = pool.trySchedule([=] - { - try - { - compile(hashed_key, file_name, additional_compiler_flags, get_code, on_ready); - } - catch (...) - { - tryLogCurrentException("Compiler"); - } - }); - - if (!res) - LOG_INFO(log, "All threads are busy."); - } - } - - return nullptr; -} - - -/// This will guarantee that code will compile only when version of headers match version of running server. -static void addCodeToAssertHeadersMatch(WriteBuffer & out) -{ - out << - "#include \n" - "#if VERSION_REVISION != " << ClickHouseRevision::get() << "\n" - "#define STRING2(x) #x\n" - "#define STRING(x) STRING2(x)\n" - "#pragma message \"ClickHouse headers revision = \" STRING(VERSION_REVISION) \n" - "#error \"ClickHouse headers revision doesn't match runtime revision of the server (" << ClickHouseRevision::get() << ").\"\n" - "#endif\n\n"; -} - - -void Compiler::compile( - HashedKey hashed_key, - std::string file_name, - const std::string & additional_compiler_flags, - CodeGenerator get_code, - ReadyCallback on_ready) -{ - ProfileEvents::increment(ProfileEvents::CompileAttempt); - -#if !defined(INTERNAL_COMPILER_EXECUTABLE) - throw Exception("Cannot compile code: Compiler disabled", ErrorCodes::CANNOT_COMPILE_CODE); -#else - std::string prefix = path + "/" + file_name; - std::string cpp_file_path = prefix + ".cpp"; - std::string so_file_path = prefix + ".so"; - std::string so_tmp_file_path = prefix + ".so.tmp"; - - { - WriteBufferFromFile out(cpp_file_path); - - addCodeToAssertHeadersMatch(out); - out << get_code(); - } - - std::stringstream command; - - auto compiler_executable_root = Poco::Util::Application::instance().config().getString("compiler_executable_root", INTERNAL_COMPILER_BIN_ROOT); - auto compiler_headers = Poco::Util::Application::instance().config().getString("compiler_headers", INTERNAL_COMPILER_HEADERS); - auto compiler_headers_root = Poco::Util::Application::instance().config().getString("compiler_headers_root", INTERNAL_COMPILER_HEADERS_ROOT); - LOG_DEBUG(log, "Using internal compiler: compiler_executable_root=" << compiler_executable_root << "; compiler_headers_root=" << compiler_headers_root << "; compiler_headers=" << compiler_headers); - - /// Slightly unconvenient. - command << - "(" - INTERNAL_COMPILER_ENV - " " << compiler_executable_root << INTERNAL_COMPILER_EXECUTABLE - " " INTERNAL_COMPILER_FLAGS - /// It is hard to correctly call a ld program manually, because it is easy to skip critical flags, which might lead to - /// unhandled exceptions. Therefore pass path to llvm's lld directly to clang. - " -fuse-ld=" << compiler_executable_root << INTERNAL_LINKER_EXECUTABLE - " -fdiagnostics-color=never" - - /// Do not use libgcc and startup files. The library will work nevertheless and we avoid extra dependency. - " -nodefaultlibs -nostartfiles" - - #if INTERNAL_COMPILER_CUSTOM_ROOT - /// To get correct order merge this results carefully: - /// echo | clang -x c++ -E -Wp,-v - - /// echo | g++ -x c++ -E -Wp,-v - - - " -isystem " << compiler_headers_root << "/usr/include/c++/*" - #if defined(CMAKE_LIBRARY_ARCHITECTURE) - " -isystem " << compiler_headers_root << "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE "/c++/*" - #endif - " -isystem " << compiler_headers_root << "/usr/include/c++/*/backward" - " -isystem " << compiler_headers_root << "/usr/include/clang/*/include" /// if compiler is clang (from package) - " -isystem " << compiler_headers_root << "/usr/local/lib/clang/*/include" /// if clang installed manually - " -isystem " << compiler_headers_root << "/usr/lib/clang/*/include" /// if clang build from submodules - #if defined(CMAKE_LIBRARY_ARCHITECTURE) - " -isystem " << compiler_headers_root << "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include-fixed" - " -isystem " << compiler_headers_root << "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include" - #endif - " -isystem " << compiler_headers_root << "/usr/local/include" /// if something installed manually - #if defined(CMAKE_LIBRARY_ARCHITECTURE) - " -isystem " << compiler_headers_root << "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE - #endif - " -isystem " << compiler_headers_root << "/usr/include" - #endif - " -I " << compiler_headers << "/dbms/src/" - " -isystem " << compiler_headers << "/contrib/cityhash102/include/" - " -isystem " << compiler_headers << "/contrib/libpcg-random/include/" - #if USE_MIMALLOC - " -isystem " << compiler_headers << "/contrib/mimalloc/include/" - #endif - " -isystem " << compiler_headers << INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR - " -isystem " << compiler_headers << INTERNAL_Poco_Foundation_INCLUDE_DIR - " -isystem " << compiler_headers << INTERNAL_Boost_INCLUDE_DIRS - " -I " << compiler_headers << "/libs/libcommon/include/" - " " << additional_compiler_flags << - " -shared -o " << so_tmp_file_path << " " << cpp_file_path - << " 2>&1" - ") || echo Return code: $?"; - -#ifndef NDEBUG - LOG_TRACE(log, "Compile command: " << command.str()); -#endif - - std::string compile_result; - - { - auto process = ShellCommand::execute(command.str()); - readStringUntilEOF(compile_result, process->out); - process->wait(); - } - - if (!compile_result.empty()) - { - std::string error_message = "Cannot compile code:\n\n" + command.str() + "\n\n" + compile_result; - - Poco::File so_tmp_file(so_tmp_file_path); - if (so_tmp_file.exists() && so_tmp_file.canExecute()) - { - /// Compiler may emit information messages. This is suspicious, but we still can use compiled result. - LOG_WARNING(log, error_message); - } - else - throw Exception(error_message, ErrorCodes::CANNOT_COMPILE_CODE); - } - - /// If there was an error before, the file with the code remains for viewing. - Poco::File(cpp_file_path).remove(); - - Poco::File(so_tmp_file_path).renameTo(so_file_path); - SharedLibraryPtr lib(new SharedLibrary(so_file_path)); - - { - std::lock_guard lock(mutex); - libraries[hashed_key] = lib; - } - - LOG_INFO(log, "Compiled code " << file_name); - ProfileEvents::increment(ProfileEvents::CompileSuccess); - - on_ready(lib); - -#endif -} - - -} diff --git a/dbms/src/Interpreters/Compiler.h b/dbms/src/Interpreters/Compiler.h deleted file mode 100644 index b79cf26e0f0..00000000000 --- a/dbms/src/Interpreters/Compiler.h +++ /dev/null @@ -1,88 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace DB -{ - -/** Lets you compile a piece of code that uses the server's header files into the dynamic library. - * Conducts statistic of calls, and initiates compilation only on the N-th call for one key. - * Compilation is performed asynchronously, in separate threads, if there are free threads. - * NOTE: There is no cleaning of obsolete and unnecessary results. - */ -class Compiler -{ -public: - /** path - path to the directory with temporary files - the results of the compilation. - * The compilation results are saved when the server is restarted, - * but use the revision number as part of the key. That is, they become obsolete when the server is updated. - */ - Compiler(const std::string & path_, size_t threads); - ~Compiler(); - - using HashedKey = UInt128; - - using CodeGenerator = std::function; - using ReadyCallback = std::function; - - /** Increase the counter for the given key `key` by one. - * If the compilation result already exists (already open, or there is a file with the library), - * then return ready SharedLibrary. - * Otherwise, if min_count_to_compile == 0, then initiate the compilation in the same thread, wait for it, and return the result. - * Otherwise, if the counter has reached min_count_to_compile, - * initiate compilation in a separate thread, if there are free threads, and return nullptr. - * Otherwise, return nullptr. - */ - SharedLibraryPtr getOrCount( - const std::string & key, - UInt32 min_count_to_compile, - const std::string & additional_compiler_flags, - CodeGenerator get_code, - ReadyCallback on_ready); - -private: - using Counts = std::unordered_map; - using Libraries = std::unordered_map; - using Files = std::unordered_set; - - const std::string path; - ThreadPool pool; - - /// Number of calls to `getOrCount`. - Counts counts; - - /// Compiled and open libraries. Or nullptr for libraries in the compilation process. - Libraries libraries; - - /// Compiled files remaining from previous runs, but not yet open. - Files files; - - std::mutex mutex; - - Logger * log = &Logger::get("Compiler"); - - - void compile( - HashedKey hashed_key, - std::string file_name, - const std::string & additional_compiler_flags, - CodeGenerator get_code, - ReadyCallback on_ready); -}; - -} diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 53392f9ad9a..67a81a94b3a 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -140,7 +139,6 @@ struct ContextShared std::optional background_pool; /// The thread pool for the background work performed by the tables. std::optional schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables) MultiVersion macros; /// Substitutions extracted from config. - std::optional compiler; /// Used for dynamic compilation of queries' parts if it necessary. std::unique_ptr ddl_worker; /// Process ddl commands from zk. /// Rules for selecting the compression settings, depending on the size of the part. mutable std::unique_ptr compression_codec_selector; @@ -1634,17 +1632,6 @@ void Context::setCluster(const String & cluster_name, const std::shared_ptrcompiler) - shared->compiler.emplace(shared->path + "build/", 1); - - return *shared->compiler; -} - - void Context::initializeSystemLogs() { auto lock = getLock(); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index dcc3fa9b3ee..2d583c3c353 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -1654,7 +1654,6 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre Aggregator::Params params(header, keys, aggregates, overflow_row, settings.max_rows_to_group_by, settings.group_by_overflow_mode, - settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile, allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, @@ -1721,7 +1720,6 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const Aggregator::Params params(header_before_aggregation, keys, aggregates, overflow_row, settings.max_rows_to_group_by, settings.group_by_overflow_mode, - settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile, allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, @@ -1943,7 +1941,6 @@ void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificato Aggregator::Params params(header, keys, aggregates, false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, - settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile, SettingUInt64(0), SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, context.getTemporaryPath(), settings.max_threads); @@ -1973,7 +1970,6 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPipeline & pipeline, Modif Aggregator::Params params(header_before_transform, keys, aggregates, false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, - settings.compile ? &context.getCompiler() : nullptr, settings.min_count_to_compile, SettingUInt64(0), SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, context.getTemporaryPath(), settings.max_threads); diff --git a/dbms/src/Interpreters/SpecializedAggregator.h b/dbms/src/Interpreters/SpecializedAggregator.h deleted file mode 100644 index 9a238c77032..00000000000 --- a/dbms/src/Interpreters/SpecializedAggregator.h +++ /dev/null @@ -1,215 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ - - -/** An aggregation loop template that allows you to generate a custom variant for a specific combination of aggregate functions. - * It differs from the usual one in that calls to aggregate functions should be inlined, and the update loop of the aggregate functions should be unrolled. - * - * Since there are too many possible combinations, it is not possible to generate them all in advance. - * This template is intended to instantiate it in runtime, - * by running the compiler, compiling shared library, and using it with `dlopen`. - */ - - -struct AggregateFunctionsUpdater -{ - AggregateFunctionsUpdater( - const Aggregator::AggregateFunctionsPlainPtrs & aggregate_functions_, - const Sizes & offsets_of_aggregate_states_, - Aggregator::AggregateColumns & aggregate_columns_, - AggregateDataPtr & value_, - size_t row_num_, - Arena * arena_) - : aggregate_functions(aggregate_functions_), - offsets_of_aggregate_states(offsets_of_aggregate_states_), - aggregate_columns(aggregate_columns_), - value(value_), row_num(row_num_), arena(arena_) - { - } - - template - void operator()() ALWAYS_INLINE; - - const Aggregator::AggregateFunctionsPlainPtrs & aggregate_functions; - const Sizes & offsets_of_aggregate_states; - Aggregator::AggregateColumns & aggregate_columns; - AggregateDataPtr & value; - size_t row_num; - Arena * arena; -}; - -template -void AggregateFunctionsUpdater::operator()() -{ - static_cast(aggregate_functions[column_num])->add( - value + offsets_of_aggregate_states[column_num], - aggregate_columns[column_num].data(), - row_num, arena); -} - -struct AggregateFunctionsCreator -{ - AggregateFunctionsCreator( - const Aggregator::AggregateFunctionsPlainPtrs & aggregate_functions_, - const Sizes & offsets_of_aggregate_states_, - AggregateDataPtr & aggregate_data_) - : aggregate_functions(aggregate_functions_), - offsets_of_aggregate_states(offsets_of_aggregate_states_), - aggregate_data(aggregate_data_) - { - } - - template - void operator()() ALWAYS_INLINE; - - const Aggregator::AggregateFunctionsPlainPtrs & aggregate_functions; - const Sizes & offsets_of_aggregate_states; - AggregateDataPtr & aggregate_data; -}; - -template -void AggregateFunctionsCreator::operator()() -{ - AggregateFunction * func = static_cast(aggregate_functions[column_num]); - - try - { - /** An exception may occur if there is a shortage of memory. - * To ensure that everything is properly destroyed, we "roll back" some of the created states. - * The code is not very convenient. - */ - func->create(aggregate_data + offsets_of_aggregate_states[column_num]); - } - catch (...) - { - for (size_t rollback_j = 0; rollback_j < column_num; ++rollback_j) - func->destroy(aggregate_data + offsets_of_aggregate_states[rollback_j]); - - throw; - } -} - - -template -void NO_INLINE Aggregator::executeSpecialized( - Method & method, - Arena * aggregates_pool, - size_t rows, - ColumnRawPtrs & key_columns, - AggregateColumns & aggregate_columns, - bool no_more_keys, - AggregateDataPtr overflow_row) const -{ - typename Method::State state(key_columns, key_sizes, aggregation_state_cache); - - if (!no_more_keys) - executeSpecializedCase( - method, state, aggregates_pool, rows, aggregate_columns, overflow_row); - else - executeSpecializedCase( - method, state, aggregates_pool, rows, aggregate_columns, overflow_row); -} - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wuninitialized" - -template -void NO_INLINE Aggregator::executeSpecializedCase( - Method & method, - typename Method::State & state, - Arena * aggregates_pool, - size_t rows, - AggregateColumns & aggregate_columns, - AggregateDataPtr overflow_row) const -{ - /// For all rows. - for (size_t i = 0; i < rows; ++i) - { - AggregateDataPtr aggregate_data = nullptr; - - if (!no_more_keys) /// Insert. - { - auto emplace_result = state.emplaceKey(method.data, i, *aggregates_pool); - - /// If a new key is inserted, initialize the states of the aggregate functions, and possibly something related to the key. - if (emplace_result.isInserted()) - { - /// exception-safety - if you can not allocate memory or create states, then destructors will not be called. - emplace_result.setMapped(nullptr); - - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - AggregateFunctionsList::forEach(AggregateFunctionsCreator( - aggregate_functions, offsets_of_aggregate_states, aggregate_data)); - - emplace_result.setMapped(aggregate_data); - } - else - aggregate_data = emplace_result.getMapped(); - } - else - { - /// Add only if the key already exists. - auto find_result = state.findKey(method.data, i, *aggregates_pool); - if (find_result.isFound()) - aggregate_data = find_result.getMapped(); - } - - /// If the key does not fit, and the data does not need to be aggregated in a separate row, then there's nothing to do. - if (!aggregate_data && !overflow_row) - continue; - - auto value = aggregate_data ? aggregate_data : overflow_row; - - /// Add values into the aggregate functions. - AggregateFunctionsList::forEach(AggregateFunctionsUpdater( - aggregate_functions, offsets_of_aggregate_states, aggregate_columns, value, i, aggregates_pool)); - } -} - -#pragma GCC diagnostic pop - -template -void NO_INLINE Aggregator::executeSpecializedWithoutKey( - AggregatedDataWithoutKey & res, - size_t rows, - AggregateColumns & aggregate_columns, - Arena * arena) const -{ - for (size_t i = 0; i < rows; ++i) - { - AggregateFunctionsList::forEach(AggregateFunctionsUpdater( - aggregate_functions, offsets_of_aggregate_states, aggregate_columns, res, i, arena)); - } -} - -} - - -/** The main code is compiled with gcc 7. - * But SpecializedAggregator is compiled using clang 6 into the .so file. - * This is done because gcc can not get functions inlined, - * which were de-virtualized, in a particular case, and the performance is lower. - * And also it's easier to distribute clang for deploy to the servers. - * - * After switching from gcc 4.8 and gnu++1x to gcc 4.9 and gnu++1y (and then to gcc 5), - * an error occurred with `dlopen`: undefined symbol: __cxa_pure_virtual - * - * Most likely, this is due to the changed version of this symbol: - * gcc creates a symbol in .so - * U __cxa_pure_virtual@@CXXABI_1.3 - * but clang creates a symbol - * U __cxa_pure_virtual - * - * But it does not matter for us how the __cxa_pure_virtual function will be implemented, - * because it is not called during normal program execution, - * and if called - then the program is guaranteed buggy. - * - * Therefore, we can work around the problem this way - */ -extern "C" void __attribute__((__visibility__("default"), __noreturn__)) __cxa_pure_virtual() { abort(); } diff --git a/dbms/src/Interpreters/config_compile.h.in b/dbms/src/Interpreters/config_compile.h.in deleted file mode 100644 index e8db534a62d..00000000000 --- a/dbms/src/Interpreters/config_compile.h.in +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#cmakedefine CMAKE_LIBRARY_ARCHITECTURE "@CMAKE_LIBRARY_ARCHITECTURE@" -#cmakedefine PATH_SHARE "@PATH_SHARE@" -#cmakedefine INTERNAL_COMPILER_FLAGS "@INTERNAL_COMPILER_FLAGS@" -#cmakedefine INTERNAL_COMPILER_BIN_ROOT "@INTERNAL_COMPILER_BIN_ROOT@" -#cmakedefine INTERNAL_LINKER_EXECUTABLE "@INTERNAL_LINKER_EXECUTABLE@" -#cmakedefine INTERNAL_COMPILER_EXECUTABLE "@INTERNAL_COMPILER_EXECUTABLE@" -#cmakedefine INTERNAL_COMPILER_ENV "@INTERNAL_COMPILER_ENV@" -#if !defined(INTERNAL_COMPILER_ENV) -# define INTERNAL_COMPILER_ENV "" -#endif -#cmakedefine INTERNAL_COMPILER_HEADERS "@INTERNAL_COMPILER_HEADERS@" -#if !defined(INTERNAL_COMPILER_HEADERS) -# define INTERNAL_COMPILER_HEADERS "" -#endif -#cmakedefine INTERNAL_COMPILER_HEADERS_ROOT "@INTERNAL_COMPILER_HEADERS_ROOT@" -#if !defined(INTERNAL_COMPILER_HEADERS_ROOT) -# define INTERNAL_COMPILER_HEADERS_ROOT "" -#endif - -#cmakedefine01 INTERNAL_COMPILER_CUSTOM_ROOT -#cmakedefine INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR "@INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR@" -#cmakedefine INTERNAL_Poco_Foundation_INCLUDE_DIR "@INTERNAL_Poco_Foundation_INCLUDE_DIR@" -#cmakedefine INTERNAL_Poco_Util_INCLUDE_DIR "@INTERNAL_Poco_Util_INCLUDE_DIR@" -#cmakedefine INTERNAL_Boost_INCLUDE_DIRS "@INTERNAL_Boost_INCLUDE_DIRS@" diff --git a/dbms/src/Interpreters/tests/CMakeLists.txt b/dbms/src/Interpreters/tests/CMakeLists.txt index b4f9fff1d36..3fac5424c00 100644 --- a/dbms/src/Interpreters/tests/CMakeLists.txt +++ b/dbms/src/Interpreters/tests/CMakeLists.txt @@ -41,9 +41,6 @@ add_executable (two_level_hash_map two_level_hash_map.cpp) target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) target_link_libraries (two_level_hash_map PRIVATE dbms) -add_executable (compiler_test compiler_test.cpp) -target_link_libraries (compiler_test PRIVATE dbms) - add_executable (logical_expressions_optimizer logical_expressions_optimizer.cpp) target_link_libraries (logical_expressions_optimizer PRIVATE dbms clickhouse_parsers) diff --git a/dbms/src/Interpreters/tests/compiler_test.cpp b/dbms/src/Interpreters/tests/compiler_test.cpp deleted file mode 100644 index c56cf5775d6..00000000000 --- a/dbms/src/Interpreters/tests/compiler_test.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include - -#include - - -int main(int, char **) -{ - using namespace DB; - - Poco::AutoPtr channel = new Poco::ConsoleChannel(std::cerr); - Logger::root().setChannel(channel); - Logger::root().setLevel("trace"); - - /// Check exception handling and catching - try - { - Compiler compiler(".", 1); - - auto lib = compiler.getOrCount("catch_me_if_you_can", 0, "", []() -> std::string - { - return - "#include \n" - "void f() __attribute__((__visibility__(\"default\")));\n" - "void f()" - "{" - "try { throw std::runtime_error(\"Catch me if you can\"); }" - "catch (const std::runtime_error & e) { std::cout << \"Caught in .so: \" << e.what() << std::endl; throw; }\n" - "}" - ; - }, [](SharedLibraryPtr &){}); - - auto f = lib->template get("_Z1fv"); - - try - { - f(); - } - catch (const std::exception & e) - { - std::cout << "Caught in main(): " << e.what() << "\n"; - return 0; - } - catch (...) - { - std::cout << "Unknown exception\n"; - return -1; - } - } - catch (...) - { - std::cerr << getCurrentExceptionMessage(true) << "\n"; - return -1; - } - - return 0; -} diff --git a/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.reference b/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.reference deleted file mode 100644 index 207dc069e43..00000000000 --- a/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.reference +++ /dev/null @@ -1,2 +0,0 @@ -1 Hello -2 Hello diff --git a/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.sql b/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.sql deleted file mode 100644 index 5902b94b753..00000000000 --- a/dbms/tests/queries/0_stateless/00281_compile_sizeof_packed.sql +++ /dev/null @@ -1,2 +0,0 @@ -SET compile = 1, min_count_to_compile = 0, max_threads = 1, send_logs_level = 'none'; -SELECT arrayJoin([1, 2, 1]) AS UserID, argMax('Hello', today()) AS res GROUP BY UserID; diff --git a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.reference b/dbms/tests/queries/0_stateless/00568_compile_catch_throw.reference deleted file mode 100644 index 6ed281c757a..00000000000 --- a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.reference +++ /dev/null @@ -1,2 +0,0 @@ -1 -1 diff --git a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh b/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh deleted file mode 100755 index fbf5efcda2c..00000000000 --- a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -SETTINGS="--compile=1 --min_count_to_compile=0 --max_threads=1 --max_memory_usage=8000000 --server_logs_file=/dev/null" -output=$($CLICKHOUSE_CLIENT -q "SELECT length(groupArray(number)) FROM (SELECT * FROM system.numbers LIMIT 1000000)" $SETTINGS 2>&1) - -[[ $? -eq 0 ]] && echo "Expected non-zero RC" -if ! echo "$output" | grep -Fc -e 'Memory limit (for query) exceeded' -e 'Cannot compile code' ; then - echo -e 'There is no expected exception "Memory limit (for query) exceeded: would use..." or "Cannot compile code..."' "Whereas got:\n$output" -fi - -$CLICKHOUSE_CLIENT -q "SELECT 1" From cc0157b29378cc6af380976850a75d9c026f768c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Aug 2019 02:07:05 +0300 Subject: [PATCH 218/357] Added a test --- dbms/src/Functions/trim.cpp | 2 +- .../queries/0_stateless/00997_trim.reference | 0 dbms/tests/queries/0_stateless/00997_trim.sql | 20 +++++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/00997_trim.reference create mode 100644 dbms/tests/queries/0_stateless/00997_trim.sql diff --git a/dbms/src/Functions/trim.cpp b/dbms/src/Functions/trim.cpp index 81916604d63..46f69530005 100644 --- a/dbms/src/Functions/trim.cpp +++ b/dbms/src/Functions/trim.cpp @@ -85,7 +85,7 @@ private: char_data += num_chars; } - if constexpr (mode::trim_left) + if constexpr (mode::trim_right) { const char * found = find_last_not_symbols_or_null<' '>(char_data, char_end); if (found) diff --git a/dbms/tests/queries/0_stateless/00997_trim.reference b/dbms/tests/queries/0_stateless/00997_trim.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00997_trim.sql b/dbms/tests/queries/0_stateless/00997_trim.sql new file mode 100644 index 00000000000..7519877ec5e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_trim.sql @@ -0,0 +1,20 @@ +WITH + '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' AS x, + replaceRegexpAll(x, '.', ' ') AS spaces, + concat(substring(spaces, 1, rand(1) % 62), substring(x, 1, rand(2) % 62), substring(spaces, 1, rand(3) % 62)) AS s, + trimLeft(s) AS sl, + trimRight(s) AS sr, + trimBoth(s) AS t, + replaceRegexpOne(s, '^ +', '') AS slr, + replaceRegexpOne(s, ' +$', '') AS srr, + replaceRegexpOne(s, '^ *(.*?) *$', '\\1') AS tr +SELECT + replaceAll(s, ' ', '_'), + replaceAll(sl, ' ', '_'), + replaceAll(slr, ' ', '_'), + replaceAll(sr, ' ', '_'), + replaceAll(srr, ' ', '_'), + replaceAll(t, ' ', '_'), + replaceAll(tr, ' ', '_') +FROM numbers(100000) +WHERE NOT ((sl = slr) AND (sr = srr) AND (t = tr)) From 7703d321138c2854142f10709cb6451e4b5a024e Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 24 Aug 2019 13:53:22 +0800 Subject: [PATCH 219/357] remove symlinks --- docs/zh/database_engines/index.md | 1 - docs/zh/database_engines/mysql.md | 1 - 2 files changed, 2 deletions(-) delete mode 120000 docs/zh/database_engines/index.md delete mode 120000 docs/zh/database_engines/mysql.md diff --git a/docs/zh/database_engines/index.md b/docs/zh/database_engines/index.md deleted file mode 120000 index bbdb762a4ad..00000000000 --- a/docs/zh/database_engines/index.md +++ /dev/null @@ -1 +0,0 @@ -../../en/database_engines/index.md \ No newline at end of file diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md deleted file mode 120000 index 51ac4126e2d..00000000000 --- a/docs/zh/database_engines/mysql.md +++ /dev/null @@ -1 +0,0 @@ -../../en/database_engines/mysql.md \ No newline at end of file From 7d7c13632cb3c400e3920f06493d888da3b654e0 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 24 Aug 2019 13:54:01 +0800 Subject: [PATCH 220/357] Translate database engine documentation, update table engine documentation. --- docs/zh/database_engines/index.md | 11 +++ docs/zh/database_engines/mysql.md | 124 ++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 docs/zh/database_engines/index.md create mode 100644 docs/zh/database_engines/mysql.md diff --git a/docs/zh/database_engines/index.md b/docs/zh/database_engines/index.md new file mode 100644 index 00000000000..f8ae05e2520 --- /dev/null +++ b/docs/zh/database_engines/index.md @@ -0,0 +1,11 @@ +# 数据库引擎 + +您使用的所有表都是由数据库引擎所提供的 + +默认情况下，ClickHouse使用自己的数据库引擎，该引擎提供可配置的[表引擎](../operations/table_engines/index.md)和[所有支持的SQL语法](../query_language/syntax.md). + +除此之外，您还可以选择使用以下的数据库引擎： + +- [MySQL](mysql.md) + +[来源文章](https://clickhouse.yandex/docs/en/database_engines/) diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md new file mode 100644 index 00000000000..38dfcb5ef64 --- /dev/null +++ b/docs/zh/database_engines/mysql.md @@ -0,0 +1,124 @@ +# MySQL + +MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并允许您对表进行`INSERT`和`SELECT`查询，以方便您在ClickHouse与MySQL之间进行数据交换。 + +`MySQL`数据库引擎会将对其的查询转换为MySQL语法并发送到MySQL服务器中，因此您可以执行诸如`SHOW TABLES`或`SHOW CREATE TABLE`之类的操作。 + +但您无法对其执行以下操作： + +- `ATTACH`/`DETACH` +- `DROP` +- `RENAME` +- `CREATE TABLE` +- `ALTER` + + +## CREATE DATABASE + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +ENGINE = MySQL('host:port', 'database', 'user', 'password') +``` + +**MySQL数据库引擎参数** + +- `host:port` — 链接的MySQL地址。 +- `database` — 链接的MySQL数据库。 +- `user` — 链接的MySQL用户。 +- `password` — 链接的MySQL用户密码。 + + +## 支持的类型对应 + +MySQL | ClickHouse +------|------------ +UNSIGNED TINYINT | [UInt8](../data_types/int_uint.md) +TINYINT | [Int8](../data_types/int_uint.md) +UNSIGNED SMALLINT | [UInt16](../data_types/int_uint.md) +SMALLINT | [Int16](../data_types/int_uint.md) +UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../data_types/int_uint.md) +INT, MEDIUMINT | [Int32](../data_types/int_uint.md) +UNSIGNED BIGINT | [UInt64](../data_types/int_uint.md) +BIGINT | [Int64](../data_types/int_uint.md) +FLOAT | [Float32](../data_types/float.md) +DOUBLE | [Float64](../data_types/float.md) +DATE | [Date](../data_types/date.md) +DATETIME, TIMESTAMP | [DateTime](../data_types/datetime.md) +BINARY | [FixedString](../data_types/fixedstring.md) + +其他的MySQL数据类型将全部都转换为[String](../data_types/string.md)。 + +同时以上的所有类型都支持[Nullable](../data_types/nullable.md)。 + + +## 使用示例 + +在MySQL中创建表: + +``` +mysql> USE test; +Database changed + +mysql> CREATE TABLE `mysql_table` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `float` FLOAT NOT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from mysql_table; ++--------+-------+ +| int_id | value | ++--------+-------+ +| 1 | 2 | ++--------+-------+ +1 row in set (0,00 sec) +``` + +在ClickHouse中创建MySQL类型的数据库，同时与MySQL服务器交换数据： + +```sql +CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') +``` +```sql +SHOW DATABASES +``` +```text +┌─name─────┐ +│ default │ +│ mysql_db │ +│ system │ +└──────────┘ +``` +```sql +SHOW TABLES FROM mysql_db +``` +```text +┌─name─────────┐ +│ mysql_table │ +└──────────────┘ +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +└────────┴───────┘ +``` +```sql +INSERT INTO mysql_db.mysql_table VALUES (3,4) +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +│ 3 │ 4 │ +└────────┴───────┘ +``` + +[来源文章](https://clickhouse.yandex/docs/en/database_engines/mysql/) From 2464dd0b9fc73deb1d265b16f751e884e291af08 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Sat, 24 Aug 2019 11:51:02 +0300 Subject: [PATCH 221/357] fix --- .../queries/0_stateless/00943_materialize_index.sh | 6 +++--- .../0_stateless/00944_clear_index_in_partition.sh | 2 +- .../00975_indices_mutation_replicated_zookeeper.sh | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00943_materialize_index.sh b/dbms/tests/queries/0_stateless/00943_materialize_index.sh index feab59b368e..bc59b41b005 100755 --- a/dbms/tests/queries/0_stateless/00943_materialize_index.sh +++ b/dbms/tests/queries/0_stateless/00943_materialize_index.sh @@ -39,13 +39,13 @@ SET allow_experimental_data_skipping_indices=1; ALTER TABLE test.minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1;" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" -wait_for_mutation "minmax_idx" "mutation_2.txt" "test" +wait_for_mutation "minmax_idx" "mutation_3.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 2;" -wait_for_mutation "minmax_idx" "mutation_3.txt" "test" +wait_for_mutation "minmax_idx" "mutation_4.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" @@ -58,7 +58,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx;" -wait_for_mutation "minmax_idx" "mutation_4.txt" "test" +wait_for_mutation "minmax_idx" "mutation_5.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh index 5a7bdd8e3ae..74f15e63545 100755 --- a/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh +++ b/dbms/tests/queries/0_stateless/00944_clear_index_in_partition.sh @@ -43,7 +43,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" -wait_for_mutation "minmax_idx" "mutation_2.txt" "test" +wait_for_mutation "minmax_idx" "mutation_3.txt" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index 5e6159475f8..765dfb6abe5 100755 --- a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -45,20 +45,20 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test.indices_mutaions1 VALUES (9, 1, 2)" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" -sleep 0.5 +sleep 1 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" -wait_for_mutation "indices_mutaions1" "mutation_2.txt" "test" -wait_for_mutation "indices_mutaions2" "mutation_2.txt" "test" +wait_for_mutation "indices_mutaions1" "0000000000" "test" +wait_for_mutation "indices_mutaions2" "0000000000" "test" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="DROP TABLE test.indices_mutaions1" $CLICKHOUSE_CLIENT --query="DROP TABLE test.indices_mutaions2" From b7fdfcc7976bc31a822c98fc8d9219f626c54817 Mon Sep 17 00:00:00 2001 From: Weiqing Xu Date: Sat, 24 Aug 2019 17:27:36 +0800 Subject: [PATCH 222/357] fix HDFS HA can't work on DEBUG mode The Describe the bug when the using HDFS HA nameserivce as the uri, the port will be 0. hdfsBuilderSetNameNodePort will be called to set the port. hdfsBuilderSetNameNodePort call asset to check if the port is greater than 0. So in Release mode, it works OK. In the Debug mode, the asset will fail. How to reproduce when compiler the Clickhouse, use DEBUG mode, it will throw error when using HDFS HA nameservice url --- dbms/src/IO/HDFSCommon.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/IO/HDFSCommon.cpp b/dbms/src/IO/HDFSCommon.cpp index 0f1a58942d6..a94fbeabd60 100644 --- a/dbms/src/IO/HDFSCommon.cpp +++ b/dbms/src/IO/HDFSCommon.cpp @@ -40,7 +40,10 @@ HDFSBuilderPtr createHDFSBuilder(const Poco::URI & uri) hdfsBuilderSetUserName(builder.get(), user.c_str()); } hdfsBuilderSetNameNode(builder.get(), host.c_str()); - hdfsBuilderSetNameNodePort(builder.get(), port); + if (port != 0) + { + hdfsBuilderSetNameNodePort(builder.get(), port); + } return builder; } From 720bb3ac08f74dbc7fd42738d8835562973d1671 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Aug 2019 02:07:05 +0300 Subject: [PATCH 223/357] Added a test --- libs/libcommon/include/common/find_symbols.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/libcommon/include/common/find_symbols.h b/libs/libcommon/include/common/find_symbols.h index 920a7df04c5..162c73251fa 100644 --- a/libs/libcommon/include/common/find_symbols.h +++ b/libs/libcommon/include/common/find_symbols.h @@ -17,7 +17,7 @@ * but with the following differencies: * - works with any memory ranges, including containing zero bytes; * - doesn't require terminating zero byte: end of memory range is passed explicitly; - * - if not found, returns pointer to end instead of NULL; + * - if not found, returns pointer to end instead of nullptr; * - maximum number of symbols to search is 16. * * Uses SSE 2 in case of small number of symbols for search and SSE 4.2 in the case of large number of symbols, @@ -188,6 +188,7 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons || (num_chars >= 11 && maybe_negate(*pos == c11)) || (num_chars >= 12 && maybe_negate(*pos == c12)) || (num_chars >= 13 && maybe_negate(*pos == c13)) + || (num_chars >= 14 && maybe_negate(*pos == c14)) || (num_chars >= 15 && maybe_negate(*pos == c15)) || (num_chars >= 16 && maybe_negate(*pos == c16))) return pos; From 60fde1d29eb4f33a1abdc4eb273c278cf6a83f15 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Aug 2019 15:03:54 +0300 Subject: [PATCH 224/357] Removed useless statements from debian directory --- debian/clickhouse-common-static.install | 1 - debian/clickhouse-server.docs | 1 + debian/clickhouse-server.install | 2 -- debian/rules | 16 ---------------- 4 files changed, 1 insertion(+), 19 deletions(-) diff --git a/debian/clickhouse-common-static.install b/debian/clickhouse-common-static.install index 6666b090272..81b1dc4eb1b 100644 --- a/debian/clickhouse-common-static.install +++ b/debian/clickhouse-common-static.install @@ -1,4 +1,3 @@ usr/bin/clickhouse usr/bin/clickhouse-odbc-bridge etc/security/limits.d/clickhouse.conf -usr/share/clickhouse/* diff --git a/debian/clickhouse-server.docs b/debian/clickhouse-server.docs index 95969d08c43..e12d6533be2 100644 --- a/debian/clickhouse-server.docs +++ b/debian/clickhouse-server.docs @@ -1,3 +1,4 @@ LICENSE AUTHORS README.md +CHANGELOG.md diff --git a/debian/clickhouse-server.install b/debian/clickhouse-server.install index f69969a6084..b1475fdf162 100644 --- a/debian/clickhouse-server.install +++ b/debian/clickhouse-server.install @@ -1,6 +1,4 @@ usr/bin/clickhouse-server -usr/bin/clickhouse-clang -usr/bin/clickhouse-lld usr/bin/clickhouse-copier usr/bin/clickhouse-report etc/clickhouse-server/config.xml diff --git a/debian/rules b/debian/rules index a49ffc3f66e..c21f0999bbc 100755 --- a/debian/rules +++ b/debian/rules @@ -32,11 +32,6 @@ endif CMAKE_FLAGS += -DENABLE_UTILS=0 -#DEB_CLANG ?= $(shell which clang-6.0 || which clang-5.0 || which clang-4.0 || which clang || which clang-3.9 || which clang-3.8) - -#DEB_CC ?= gcc-7 -#DEB_CXX ?= g++-7 - ifdef DEB_CXX DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) DEB_HOST_GNU_TYPE := $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE) @@ -88,12 +83,8 @@ override_dh_auto_configure: override_dh_auto_build: # Fix for ninja. Do not add -O. $(MAKE) $(THREADS_COUNT) -C $(BUILDDIR) $(MAKE_TARGET) -# #cd $(BUILDDIR) && cmake --build . -- -j$(THREADS_COUNT) # cmake return true on error override_dh_auto_test: -# #TODO, use ENABLE_TESTS=1 -# #./debian/tests_wrapper.sh -# cd $(BUILDDIR) && ctest $(THREADS_COUNT) -V -R GLIBC_required_version cd $(BUILDDIR) && ctest $(THREADS_COUNT) -V -E with_server override_dh_clean: @@ -117,11 +108,6 @@ override_dh_install: mkdir -p $(DESTDIR)/etc/systemd/system/ cp debian/clickhouse-server.service $(DESTDIR)/etc/systemd/system/ - # fake metrika files when private dir is empty - mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika - touch $(DESTDIR)/etc/clickhouse-server/metrika/config.xml - touch $(DESTDIR)/etc/clickhouse-server/metrika/users.xml - dh_install --list-missing --sourcedir=$(DESTDIR) override_dh_auto_install: @@ -130,7 +116,5 @@ override_dh_auto_install: override_dh_shlibdeps: true # We depend only on libc and dh_shlibdeps gives us wrong (too strict) dependency. -#TODO: faster packing of non-release builds: ifdef RELEASE_COMPATIBLE override_dh_builddeb: dh_builddeb -- -Z gzip # Older systems don't have "xz", so use "gzip" instead. -#TODO: endif From cd620d2de517acff4943443a25fac71ac063b068 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Aug 2019 15:18:03 +0300 Subject: [PATCH 225/357] Fixed race condition in test (once again) --- .../queries/0_stateless/00600_replace_running_query.reference | 1 - dbms/tests/queries/0_stateless/00600_replace_running_query.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00600_replace_running_query.reference b/dbms/tests/queries/0_stateless/00600_replace_running_query.reference index 804267a1c11..a01672aae85 100644 --- a/dbms/tests/queries/0_stateless/00600_replace_running_query.reference +++ b/dbms/tests/queries/0_stateless/00600_replace_running_query.reference @@ -2,6 +2,5 @@ 1 1 1 -finished 42 readonly SELECT 2, count() FROM system.numbers 1 44 diff --git a/dbms/tests/queries/0_stateless/00600_replace_running_query.sh b/dbms/tests/queries/0_stateless/00600_replace_running_query.sh index 513f6d8440e..dbbf41dd772 100755 --- a/dbms/tests/queries/0_stateless/00600_replace_running_query.sh +++ b/dbms/tests/queries/0_stateless/00600_replace_running_query.sh @@ -30,7 +30,7 @@ ${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 43' 2>&1 | grep -cF 'is alrea # Trying to replace query of a different user $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=42&replace_running_query=1" -d 'SELECT 1' | grep -cF 'is already running by user' -$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = '42' SYNC" +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = '42' SYNC" > /dev/null wait ${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 3, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & From a6997aa83fdb8d0c9ef751337e62e6d3189176a1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Aug 2019 16:00:04 +0300 Subject: [PATCH 226/357] Modifications after merging constraints support --- .../CheckConstraintsBlockOutputStream.cpp | 93 ++++++++++--------- .../CheckConstraintsBlockOutputStream.h | 24 ++--- docker/builder/build.sh | 4 +- docs/en/query_language/alter.md | 20 ++-- docs/ru/query_language/alter.md | 22 ++--- docs/ru/query_language/create.md | 4 +- 6 files changed, 79 insertions(+), 88 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index a84885f2833..3ad72515aca 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -1,34 +1,65 @@ +#include #include -#include -#include #include #include +#include + namespace DB { +namespace ErrorCodes +{ + extern const int VIOLATED_CONSTRAINT; +} + + +CheckConstraintsBlockOutputStream::CheckConstraintsBlockOutputStream( + const String & table_, + const BlockOutputStreamPtr & output_, + const Block & header_, + const ConstraintsDescription & constraints_, + const Context & context_) + : table(table_), + output(output_), + header(header_), + constraints(constraints_), + expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())) +{ +} + + void CheckConstraintsBlockOutputStream::write(const Block & block) { - for (size_t i = 0; i < expressions.size(); ++i) + if (block.rows() > 0) { - Block res = block; - auto constraint_expr = expressions[i]; - auto res_column_uint8 = executeOnBlock(res, constraint_expr); - if (!memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1)) + for (size_t i = 0; i < expressions.size(); ++i) { - auto indices_wrong = findAllWrong(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize()); - std::string indices_str = "{"; - for (size_t j = 0; j < indices_wrong.size(); ++j) - { - indices_str += std::to_string(indices_wrong[j]); - indices_str += (j != indices_wrong.size() - 1) ? ", " : "}"; - } + Block block_to_calculate = block; + auto constraint_expr = expressions[i]; - throw Exception{"Violated constraint " + constraints.constraints[i]->name + - " in table " + table + " at indices " + indices_str + ", constraint expression: " + - serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::VIOLATED_CONSTRAINT}; + constraint_expr->execute(block_to_calculate); + ColumnWithTypeAndName res_column = block_to_calculate.getByPosition(block_to_calculate.columns() - 1); + const ColumnUInt8 & res_column_uint8 = assert_cast(*res_column.column); + + const UInt8 * data = res_column_uint8.getData().data(); + size_t size = res_column_uint8.size(); + + /// Is violated. + if (!memoryIsByte(data, size, 1)) + { + size_t row_idx = 0; + for (; row_idx < size; ++row_idx) + if (data[row_idx] != 1) + break; + + throw Exception{"Violated constraint " + constraints.constraints[i]->name + + " in table " + table + " at row " + std::to_string(row_idx) + ", constraint expression: " + + serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::VIOLATED_CONSTRAINT}; + } } } + output->write(block); rows_written += block.rows(); } @@ -48,32 +79,4 @@ void CheckConstraintsBlockOutputStream::writeSuffix() output->writeSuffix(); } -const ColumnUInt8 *CheckConstraintsBlockOutputStream::executeOnBlock( - Block & block, - const ExpressionActionsPtr & constraint) -{ - constraint->execute(block); - ColumnWithTypeAndName res_column = block.safeGetByPosition(block.columns() - 1); - return checkAndGetColumn(res_column.column.get()); -} - -std::vector CheckConstraintsBlockOutputStream::findAllWrong(const void *data, size_t size) -{ - std::vector res; - - if (size == 0) - return res; - - auto ptr = reinterpret_cast(data); - - for (size_t i = 0; i < size; ++i) - { - if (*(ptr + i) == 0x0) - { - res.push_back(i); - } - } - - return res; -} } diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index 1ce8d002062..17b30a0ec4b 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -2,16 +2,14 @@ #include #include -#include -#include + namespace DB { -namespace ErrorCodes -{ - extern const int VIOLATED_CONSTRAINT; -} +/** Check for constraints violation. If anything is found - throw an exception with detailed error message. + * Otherwise just pass block to output unchanged. + */ class CheckConstraintsBlockOutputStream : public IBlockOutputStream { @@ -21,14 +19,7 @@ public: const BlockOutputStreamPtr & output_, const Block & header_, const ConstraintsDescription & constraints_, - const Context & context_) - : table(table_), - output(output_), - header(header_), - constraints(constraints_), - expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())), - rows_written(0) - { } + const Context & context_); Block getHeader() const override { return header; } void write(const Block & block) override; @@ -39,14 +30,11 @@ public: void writeSuffix() override; private: - const ColumnUInt8* executeOnBlock(Block & block, const ExpressionActionsPtr & constraint); - std::vector findAllWrong(const void *data, size_t size); - String table; BlockOutputStreamPtr output; Block header; const ConstraintsDescription constraints; const ConstraintsExpressions expressions; - size_t rows_written; + size_t rows_written = 0; }; } diff --git a/docker/builder/build.sh b/docker/builder/build.sh index 01841f46295..8af31cb3d43 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -#ccache -s +#ccache -s # uncomment to display CCache statistics mkdir -p /server/build_docker cd /server/build_docker -cmake -G Ninja /server -DCMAKE_C_COMPILER=`which clang-8` -DCMAKE_CXX_COMPILER=`which clang++-8` -DCMAKE_BUILD_TYPE=Debug +cmake -G Ninja /server -DCMAKE_C_COMPILER=`which gcc-8` -DCMAKE_CXX_COMPILER=`which g++-8` # Set the number of build jobs to the half of number of virtual CPU cores (rounded up). # By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time. diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md index 7aa4ef89e21..84ff4f390a8 100644 --- a/docs/en/query_language/alter.md +++ b/docs/en/query_language/alter.md @@ -29,7 +29,7 @@ These actions are described in detail below. ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [AFTER name_after] ``` -Adds a new column to the table with the specified `name`, `type`, and `default_expr` (see the section [Default expressions](create.md#create-default-values)). +Adds a new column to the table with the specified `name`, `type`, and `default_expr` (see the section [Default expressions](create.md#create-default-values)). If the `IF NOT EXISTS` clause is included, the query won't return an error if the column already exists. If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. Otherwise, the column is added to the end of the table. Note that there is no way to add a column to the beginning of a table. For a chain of actions, `name_after` can be the name of a column that is added in one of the previous actions. @@ -66,7 +66,7 @@ CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name ``` Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to specify the partition expression](#alter-how-to-specify-part-expr). - + If the `IF EXISTS` clause is specified, the query won't return an error if the column doesn't exist. Example: @@ -85,7 +85,7 @@ Adds a comment to the column. If the `IF EXISTS` clause is specified, the query Each column can have one comment. If a comment already exists for the column, a new comment overwrites the previous comment. -Comments are stored in the `comment_expression` column returned by the [DESCRIBE TABLE](misc.md#misc-describe-table) query. +Comments are stored in the `comment_expression` column returned by the [DESCRIBE TABLE](misc.md#misc-describe-table) query. Example: @@ -129,7 +129,7 @@ The `ALTER` query lets you create and delete separate elements (columns) in nest There is no support for deleting columns in the primary key or the sampling key (columns that are used in the `ENGINE` expression). Changing the type for columns that are included in the primary key is only possible if this change does not cause the data to be modified (for example, you are allowed to add values to an Enum or to change a type from `DateTime` to `UInt32`). -If the `ALTER` query is not sufficient to make the table changes you need, you can create a new table, copy the data to it using the [INSERT SELECT](insert_into.md#insert_query_insert-select) query, then switch the tables using the [RENAME](misc.md#misc_operations-rename) query and delete the old table. You can use the [clickhouse-copier](../operations/utils/clickhouse-copier.md) as an alternative to the `INSERT SELECT` query. +If the `ALTER` query is not sufficient to make the table changes you need, you can create a new table, copy the data to it using the [INSERT SELECT](insert_into.md#insert_query_insert-select) query, then switch the tables using the [RENAME](misc.md#misc_operations-rename) query and delete the old table. You can use the [clickhouse-copier](../operations/utils/clickhouse-copier.md) as an alternative to the `INSERT SELECT` query. The `ALTER` query blocks all reads and writes for the table. In other words, if a long `SELECT` is running at the time of the `ALTER` query, the `ALTER` query will wait for it to complete. At the same time, all new queries to the same table will wait while this `ALTER` is running. @@ -178,9 +178,9 @@ ALTER TABLE [db].name DROP CONSTRAINT constraint_name; Queries will add or remove metadata about constraints from table so they are processed immediately. -Constraint check *will not be executed* on existing table if it was added. For now, we recommend to create new table and use `INSERT SELECT` query to fill new table. +Constraint check *will not be executed* on existing data if it was added. -All changes on distributed tables are broadcasting to ZooKeeper so will be applied on other replicas. +All changes on replicated tables are broadcasting to ZooKeeper so will be applied on other replicas. ### Manipulations With Partitions and Parts {#alter_manipulations-with-partitions} @@ -267,7 +267,7 @@ This query copies the data partition from the `table1` to `table2`. Note that da For the query to run successfully, the following conditions must be met: - Both tables must have the same structure. -- Both tables must have the same partition key. +- Both tables must have the same partition key. #### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} @@ -289,13 +289,13 @@ ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902 ALTER TABLE table_name FREEZE [PARTITION partition_expr] ``` -This query creates a local backup of a specified partition. If the `PARTITION` clause is omitted, the query creates the backup of all partitions at once. +This query creates a local backup of a specified partition. If the `PARTITION` clause is omitted, the query creates the backup of all partitions at once. Note that for old-styled tables you can specify the prefix of the partition name (for example, '2019') - then the query creates the backup for all the corresponding partitions. Read about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr). !!! note The entire backup process is performed without stopping the server. - + At the time of execution, for a data snapshot, the query creates hardlinks to a table data. Hardlinks are placed in the directory `/var/lib/clickhouse/shadow/N/...`, where: - `/var/lib/clickhouse/` is the working ClickHouse directory specified in the config. @@ -348,7 +348,7 @@ ALTER TABLE users ATTACH PARTITION 201902; ``` Note that: -- The `ALTER ... FETCH PARTITION` query isn't replicated. It places the partition to the `detached` directory only on the local server. +- The `ALTER ... FETCH PARTITION` query isn't replicated. It places the partition to the `detached` directory only on the local server. - The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas. Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas. diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md index 30fbcbc81ac..c078d35f805 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/query_language/alter.md @@ -29,7 +29,7 @@ ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [AFTER name_after] ``` -Добавляет в таблицу новый столбец с именем `name`, типом `type` и выражением для умолчания `default_expr` (смотрите раздел [Значения по умолчанию](create.md#create-default-values)). +Добавляет в таблицу новый столбец с именем `name`, типом `type` и выражением для умолчания `default_expr` (смотрите раздел [Значения по умолчанию](create.md#create-default-values)). Если указано `IF NOT EXISTS`, запрос не будет возвращать ошибку, если столбец уже существует. Если указано `AFTER name_after` (имя другого столбца), то столбец добавляется (в список столбцов таблицы) после указанного. Иначе, столбец добавляется в конец таблицы. Обратите внимание, ClickHouse не позволяет добавлять столбцы в начало таблицы. Для цепочки действий, `name_after` может быть именем столбца, который добавляется в одном из предыдущих действий. @@ -84,7 +84,7 @@ COMMENT COLUMN [IF EXISTS] name 'Text comment' Каждый столбец может содержать только один комментарий. При выполнении запроса существующий комментарий заменяется на новый. -Посмотреть комментарии можно в столбце `comment_expression` из запроса [DESCRIBE TABLE](misc.md#misc-describe-table). +Посмотреть комментарии можно в столбце `comment_expression` из запроса [DESCRIBE TABLE](misc.md#misc-describe-table). Пример: @@ -177,9 +177,9 @@ ALTER TABLE [db].name DROP CONSTRAINT constraint_name; Запросы выполняют добавление или удаление метаданных об ограничениях таблицы `[db].name`, поэтому выполняются мнгновенно. -Если ограничение появилось для непустой таблицы, то *проверка ограничения вызвана не будет*. Если же важно добавить ограничение на существующую таблицу, то рекомендуется создать новую таблицу с нужным ограничением и выполнить `INSERT SELECT` запрос для перекачки данных из одной таблицы в другую. +Если ограничение появилось для непустой таблицы, то *проверка ограничения для имеющихся данных не производится*. -Запрос на изменение ограничений так же, как и с индексами, реплицируется через ZooKeeper. +Запрос на изменение ограничений для Replicated таблиц реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. ### Манипуляции с партициями и кусками {#alter_manipulations-with-partitions} @@ -260,7 +260,7 @@ ALTER TABLE visits ATTACH PART 201901_2_2_0; ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 ``` -Копирует партицию из таблицы `table1` в таблицу `table2`. Данные из `table1` не удаляются. +Копирует партицию из таблицы `table1` в таблицу `table2`. Данные из `table1` не удаляются. Следует иметь в виду: @@ -297,19 +297,19 @@ ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr ALTER TABLE table_name FREEZE [PARTITION partition_expr] ``` -Создаёт резервную копию для заданной партиции. Если выражение `PARTITION` опущено, резервные копии будут созданы для всех партиций. +Создаёт резервную копию для заданной партиции. Если выражение `PARTITION` опущено, резервные копии будут созданы для всех партиций. !!! note - Создание резервной копии не требует остановки сервера. - + Создание резервной копии не требует остановки сервера. + Для таблиц старого стиля имя партиций можно задавать в виде префикса (например, '2019'). В этом случае резервные копии будут созданы для всех соответствующих партиций. Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - + Запрос делает следующее — для текущего состояния таблицы он формирует жесткие ссылки на данные в этой таблице. Ссылки размещаются в директории `/var/lib/clickhouse/shadow/N/...`, где: - `/var/lib/clickhouse/` — рабочая директория ClickHouse, заданная в конфигурационном файле; - `N` — инкрементальный номер резервной копии. -Структура директорий внутри резервной копии такая же, как внутри `/var/lib/clickhouse/`. Запрос выполнит 'chmod' для всех файлов, запрещая запись в них. +Структура директорий внутри резервной копии такая же, как внутри `/var/lib/clickhouse/`. Запрос выполнит 'chmod' для всех файлов, запрещая запись в них. Обратите внимание, запрос `ALTER TABLE t FREEZE PARTITION` не реплицируется. Он создает резервную копию только на локальном сервере. После создания резервной копии данные из `/var/lib/clickhouse/shadow/` можно скопировать на удалённый сервер, а локальную копию удалить. @@ -357,7 +357,7 @@ ALTER TABLE users ATTACH PARTITION 201902; #### Как задавать имя партиции в запросах ALTER {#alter-how-to-specify-part-expr} -Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать: +Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать: - Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../operations/system_tables.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. - Произвольное выражение из столбцов исходной таблицы. Также поддерживаются константы и константные выражения. Например, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. diff --git a/docs/ru/query_language/create.md b/docs/ru/query_language/create.md index e3e1c901204..2e0a21cda0b 100644 --- a/docs/ru/query_language/create.md +++ b/docs/ru/query_language/create.md @@ -119,9 +119,9 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = engine ``` -`boolean_expr_1` может быть любым булевым выражением, состоящим из операторов сравнения или функций. При наличии одного или нескольких ограничений в момент вставки данных выражения ограничений будут проверяться на истинность для каждой вставляемой строки данных. В случае, если в теле INSERT запроса придут некорректные данные — клиентов будет выкинуто исключение с нарушенным ограничением. +`boolean_expr_1` может быть любым булевым выражением, состоящим из операторов сравнения или функций. При наличии одного или нескольких ограничений в момент вставки данных выражения ограничений будут проверяться на истинность для каждой вставляемой строки данных. В случае, если в теле INSERT запроса придут некорректные данные — клиент получит исключение с описанием нарушенного ограничения. -Добавление большого числа ограничений может негативно повлиять на производительность объёмных `INSERT` запросов. +Добавление большого числа ограничений может негативно повлиять на производительность `INSERT` запросов. ### Выражение для TTL From 3fef8e92ad72c3d510e077c24838c2f103342a22 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Aug 2019 00:20:20 +0300 Subject: [PATCH 227/357] Merging CONSTRAINTS support --- dbms/programs/server/TCPHandler.cpp | 2 +- dbms/src/Core/ExternalTable.cpp | 2 +- .../CheckConstraintsBlockOutputStream.cpp | 5 +- dbms/src/Databases/DatabaseMySQL.cpp | 2 +- dbms/src/Databases/DatabasesCommon.cpp | 3 +- .../Interpreters/GlobalSubqueriesVisitor.h | 2 +- .../Interpreters/InterpreterCreateQuery.cpp | 22 ++++-- .../src/Interpreters/InterpreterCreateQuery.h | 6 +- .../Interpreters/InterpreterInsertQuery.cpp | 7 +- .../Interpreters/InterpreterSystemQuery.cpp | 2 + dbms/src/Storages/IStorage.cpp | 14 +--- dbms/src/Storages/IStorage.h | 4 +- dbms/src/Storages/Kafka/StorageKafka.cpp | 2 +- .../src/Storages/LiveView/StorageLiveView.cpp | 4 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 1 - .../MergeTree/StorageFromMergeTreeDataPart.h | 7 +- .../MergeTree/registerStorageMergeTree.cpp | 41 +++++------ dbms/src/Storages/StorageBuffer.cpp | 9 ++- dbms/src/Storages/StorageBuffer.h | 3 +- dbms/src/Storages/StorageDictionary.cpp | 4 +- dbms/src/Storages/StorageDistributed.cpp | 17 +++-- dbms/src/Storages/StorageDistributed.h | 2 + dbms/src/Storages/StorageFactory.cpp | 2 + dbms/src/Storages/StorageFactory.h | 3 + dbms/src/Storages/StorageFile.cpp | 8 ++- dbms/src/Storages/StorageFile.h | 1 + dbms/src/Storages/StorageHDFS.cpp | 8 ++- dbms/src/Storages/StorageHDFS.h | 1 + dbms/src/Storages/StorageJoin.cpp | 4 +- dbms/src/Storages/StorageJoin.h | 1 + dbms/src/Storages/StorageLog.cpp | 9 ++- dbms/src/Storages/StorageLog.h | 1 + dbms/src/Storages/StorageMaterializedView.cpp | 4 +- dbms/src/Storages/StorageMaterializedView.h | 1 + dbms/src/Storages/StorageMemory.cpp | 8 ++- dbms/src/Storages/StorageMemory.h | 2 +- dbms/src/Storages/StorageMerge.cpp | 3 +- dbms/src/Storages/StorageMySQL.cpp | 7 +- dbms/src/Storages/StorageMySQL.h | 2 +- dbms/src/Storages/StorageNull.cpp | 2 +- dbms/src/Storages/StorageNull.h | 6 +- dbms/src/Storages/StorageSet.cpp | 15 ++-- dbms/src/Storages/StorageSet.h | 6 +- dbms/src/Storages/StorageStripeLog.cpp | 9 ++- dbms/src/Storages/StorageStripeLog.h | 1 + dbms/src/Storages/StorageTinyLog.cpp | 9 ++- dbms/src/Storages/StorageTinyLog.h | 1 + dbms/src/Storages/StorageURL.cpp | 12 ++-- dbms/src/Storages/StorageURL.h | 24 ++++--- dbms/src/Storages/StorageView.cpp | 4 +- dbms/src/Storages/StorageXDBC.cpp | 3 +- dbms/src/Storages/StorageXDBC.h | 72 ++++++++++--------- dbms/src/TableFunctions/TableFunctionFile.cpp | 1 + dbms/src/TableFunctions/TableFunctionHDFS.cpp | 1 + .../src/TableFunctions/TableFunctionMySQL.cpp | 1 + dbms/src/TableFunctions/TableFunctionURL.cpp | 3 +- 56 files changed, 243 insertions(+), 153 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 5091258acaf..5e916ccbddd 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -865,7 +865,7 @@ bool TCPHandler::receiveData() if (!(storage = query_context->tryGetExternalTable(external_table_name))) { NamesAndTypesList columns = block.getNamesAndTypesList(); - storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}); + storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}, ConstraintsDescription{}); storage->startup(); query_context->addExternalTable(external_table_name, storage); } diff --git a/dbms/src/Core/ExternalTable.cpp b/dbms/src/Core/ExternalTable.cpp index e1e059a3b63..8befb4e054c 100644 --- a/dbms/src/Core/ExternalTable.cpp +++ b/dbms/src/Core/ExternalTable.cpp @@ -160,7 +160,7 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, /// Create table NamesAndTypesList columns = sample_block.getNamesAndTypesList(); - StoragePtr storage = StorageMemory::create("_external", data.second, ColumnsDescription{columns}); + StoragePtr storage = StorageMemory::create("_external", data.second, ColumnsDescription{columns}, ConstraintsDescription{}); storage->startup(); context.addExternalTable(data.second, storage); BlockOutputStreamPtr output = storage->write(ASTPtr(), context); diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 3ad72515aca..8fbb5916372 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -33,8 +33,11 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { if (block.rows() > 0) { + std::cerr << "Checking " << expressions.size() << " constraints\n"; for (size_t i = 0; i < expressions.size(); ++i) { + std::cerr << serializeAST(*(constraints.constraints[i]->expr), true) << "\n"; + Block block_to_calculate = block; auto constraint_expr = expressions[i]; @@ -54,7 +57,7 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) break; throw Exception{"Violated constraint " + constraints.constraints[i]->name + - " in table " + table + " at row " + std::to_string(row_idx) + ", constraint expression: " + + " in table " + table + " at row " + std::to_string(rows_written + row_idx + 1) + ", constraint expression: " + serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::VIOLATED_CONSTRAINT}; } } diff --git a/dbms/src/Databases/DatabaseMySQL.cpp b/dbms/src/Databases/DatabaseMySQL.cpp index e9dd5c0dacc..9fd16cc94a8 100644 --- a/dbms/src/Databases/DatabaseMySQL.cpp +++ b/dbms/src/Databases/DatabaseMySQL.cpp @@ -231,7 +231,7 @@ DatabaseMySQL::MySQLStorageInfo DatabaseMySQL::createStorageInfo( { const auto & mysql_table = StorageMySQL::create( database_name, table_name, std::move(mysql_pool), mysql_database_name, table_name, - false, "", ColumnsDescription{columns_name_and_type}, global_context); + false, "", ColumnsDescription{columns_name_and_type}, ConstraintsDescription{}, global_context); const auto & create_table_query = std::make_shared(); diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp index af43b25fbdb..1a278fbb03d 100644 --- a/dbms/src/Databases/DatabasesCommon.cpp +++ b/dbms/src/Databases/DatabasesCommon.cpp @@ -83,6 +83,7 @@ std::pair createTableFromDefinition( throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context); + ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints); return { @@ -90,7 +91,7 @@ std::pair createTableFromDefinition( StorageFactory::instance().get( ast_create_query, database_data_path, ast_create_query.table, database_name, context, context.getGlobalContext(), - columns, + columns, constraints, true, has_force_restore_data_flag) }; } diff --git a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h index 583ec026af0..926e6afd1c2 100644 --- a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h @@ -91,7 +91,7 @@ public: Block sample = interpreter->getSampleBlock(); NamesAndTypesList columns = sample.getNamesAndTypesList(); - StoragePtr external_storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}); + StoragePtr external_storage = StorageMemory::create("_external", external_table_name, ColumnsDescription{columns}, ConstraintsDescription{}); external_storage->startup(); /** We replace the subquery with the name of the temporary table. diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 555a0951f58..cb5e657bcea 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -381,26 +381,37 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres } -ColumnsDescription InterpreterCreateQuery::setColumns( +ConstraintsDescription InterpreterCreateQuery::getConstraintsDescription(const ASTExpressionList * constraints) +{ + ConstraintsDescription res; + if (constraints) + for (const auto & constraint : constraints->children) + res.constraints.push_back(std::dynamic_pointer_cast(constraint->clone())); + return res; +} + + +ColumnsDescription InterpreterCreateQuery::setProperties( ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const { ColumnsDescription columns; IndicesDescription indices; ConstraintsDescription constraints; - if (create.columns_list) { if (create.columns_list->columns) columns = getColumnsDescription(*create.columns_list->columns, context); + if (create.columns_list->indices) for (const auto & index : create.columns_list->indices->children) indices.indices.push_back( std::dynamic_pointer_cast(index->clone())); + if (create.columns_list->constraints) for (const auto & constraint : create.columns_list->constraints->children) constraints.constraints.push_back( - std::dynamic_pointer_cast(constraint->clone())); + std::dynamic_pointer_cast(constraint->clone())); } else if (!create.as_table.empty()) { @@ -567,7 +578,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) else { /// Set and retrieve list of columns. - columns = setColumns(create, as_select_sample, as_storage); + columns = setProperties(create, as_select_sample, as_storage); /// Check low cardinality types in creating table if it was not allowed in setting if (!create.attach && !context.getSettingsRef().allow_suspicious_low_cardinality_types && !create.is_materialized_view) @@ -587,6 +598,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) setEngine(create); } + ConstraintsDescription constraints = getConstraintsDescription(create.columns_list->constraints); + { std::unique_ptr guard; @@ -635,6 +648,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) context, context.getGlobalContext(), columns, + constraints, create.attach, false); } diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.h b/dbms/src/Interpreters/InterpreterCreateQuery.h index 23b1e3cdabc..2472f0321e1 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.h +++ b/dbms/src/Interpreters/InterpreterCreateQuery.h @@ -14,6 +14,7 @@ namespace DB class Context; class ASTCreateQuery; class ASTExpressionList; +class ASTConstraintDeclaration; /** Allows to create new table or database, @@ -45,13 +46,14 @@ public: /// Obtain information about columns, their types, default values and column comments, for case when columns in CREATE query is specified explicitly. static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, const Context & context); + static ConstraintsDescription getConstraintsDescription(const ASTExpressionList * constraints); private: BlockIO createDatabase(ASTCreateQuery & create); BlockIO createTable(ASTCreateQuery & create); - /// Calculate list of columns of table and return it. - ColumnsDescription setColumns(ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const; + /// Calculate list of columns, constraints, indices, etc... of table and return columns. + ColumnsDescription setProperties(ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const; void setEngine(ASTCreateQuery & create) const; void checkAccess(const ASTCreateQuery & create); diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 4033bea7f2a..549e06e0155 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -121,8 +121,11 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared( out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); - out = std::make_shared( - query.table, out, query_sample_block, table->getConstraints(), context); + std::cerr << table->getConstraints().toString() << "\n"; + + if (const auto & constraints = table->getConstraints(); !constraints.empty()) + out = std::make_shared(query.table, + out, query_sample_block, table->getConstraints(), context); auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index 0797a3deab4..6e434189c66 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -278,6 +278,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam std::string data_path = database->getDataPath(); auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context); + auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints); StoragePtr table = StorageFactory::instance().get(create, data_path, @@ -286,6 +287,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam system_context, system_context.getGlobalContext(), columns, + constraints, create.attach, false); diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 099462a66cd..b28a3b823cb 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -20,20 +20,8 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; } -IStorage::IStorage(ColumnsDescription columns_) +IStorage::IStorage(ColumnsDescription virtuals_) : virtuals(std::move(virtuals_)) { - setColumns(std::move(columns_)); -} - -IStorage::IStorage(ColumnsDescription columns_, ColumnsDescription virtuals_) : virtuals(std::move(virtuals_)) -{ - setColumns(std::move(columns_)); -} - -IStorage::IStorage(ColumnsDescription columns_, ColumnsDescription virtuals_, IndicesDescription indices_) : virtuals(std::move(virtuals_)) -{ - setColumns(std::move(columns_)); - setIndices(std::move(indices_)); } const ColumnsDescription & IStorage::getColumns() const diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 552b234eea2..201107ec62e 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -64,9 +64,7 @@ class IStorage : public std::enable_shared_from_this { public: IStorage() = default; - explicit IStorage(ColumnsDescription columns_); - IStorage(ColumnsDescription columns_, ColumnsDescription virtuals_); - IStorage(ColumnsDescription columns_, ColumnsDescription virtuals_, IndicesDescription indices_); + explicit IStorage(ColumnsDescription virtuals_); virtual ~IStorage() = default; IStorage(const IStorage &) = delete; diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index e429ef505de..db8e49ca7a9 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -85,7 +85,6 @@ StorageKafka::StorageKafka( size_t skip_broken_, bool intermediate_commit_) : IStorage( - columns_, ColumnsDescription({{"_topic", std::make_shared()}, {"_key", std::make_shared()}, {"_offset", std::make_shared()}, @@ -107,6 +106,7 @@ StorageKafka::StorageKafka( , skip_broken(skip_broken_) , intermediate_commit(intermediate_commit_) { + setColumns(columns_); task = global_context.getSchedulePool().createTask(log->name(), [this]{ threadFunc(); }); task->deactivate(); } diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 3c0d205fa3f..7704f421517 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -218,9 +218,11 @@ StorageLiveView::StorageLiveView( Context & local_context, const ASTCreateQuery & query, const ColumnsDescription & columns_) - : IStorage(columns_), table_name(table_name_), + : table_name(table_name_), database_name(database_name_), global_context(local_context.getGlobalContext()) { + setColumns(columns_); + if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 3f22e6afaeb..2e191dacfd5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -124,7 +124,6 @@ MergeTreeData::MergeTreeData( data_parts_by_state_and_info(data_parts_indexes.get()) { setProperties(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_); - setConstraints(constraints_); /// NOTE: using the same columns list as is read when performing actual merges. merging_params.check(getColumns().getAllPhysical()); diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 83582c9779f..32268499a01 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -40,8 +40,11 @@ public: protected: StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) - : IStorage(part_->storage.getColumns(), part_->storage.getVirtuals(), part_->storage.getIndices()), part(part_) - {} + : IStorage(part_->storage.getVirtuals()), part(part_) + { + setColumns(part_->storage.getColumns()); + setIndices(part_->storage.getIndices()); + } private: MergeTreeData::DataPartPtr part; diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 9e45f3352c6..41aae981a5f 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -422,21 +422,21 @@ static StoragePtr create(const StorageFactory::Arguments & args) switch (merging_params.mode) { - default: - break; - case MergeTreeData::MergingParams::Summing: - add_optional_param("list of columns to sum"); - break; - case MergeTreeData::MergingParams::Replacing: - add_optional_param("version"); - break; - case MergeTreeData::MergingParams::Collapsing: - add_mandatory_param("sign column"); - break; - case MergeTreeData::MergingParams::Graphite: - add_mandatory_param("'config_element_for_graphite_schema'"); - break; - case MergeTreeData::MergingParams::VersionedCollapsing: + default: + break; + case MergeTreeData::MergingParams::Summing: + add_optional_param("list of columns to sum"); + break; + case MergeTreeData::MergingParams::Replacing: + add_optional_param("version"); + break; + case MergeTreeData::MergingParams::Collapsing: + add_mandatory_param("sign column"); + break; + case MergeTreeData::MergingParams::Graphite: + add_mandatory_param("'config_element_for_graphite_schema'"); + break; + case MergeTreeData::MergingParams::VersionedCollapsing: { add_mandatory_param("sign column"); add_mandatory_param("version"); @@ -574,7 +574,6 @@ static StoragePtr create(const StorageFactory::Arguments & args) ASTPtr sample_by_ast; ASTPtr ttl_table_ast; IndicesDescription indices_description; - ConstraintsDescription constraints_description; MergeTreeSettings storage_settings = args.context.getMergeTreeSettings(); if (is_extended_storage_def) @@ -601,12 +600,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (args.query.columns_list && args.query.columns_list->indices) for (const auto & index : args.query.columns_list->indices->children) indices_description.indices.push_back( - std::dynamic_pointer_cast(index->clone())); + std::dynamic_pointer_cast(index->clone())); - if (args.query.columns_list && args.query.columns_list->constraints) - for (const auto & constraint : args.query.columns_list->constraints->children) - constraints_description.constraints.push_back( - std::dynamic_pointer_cast(constraint->clone())); storage_settings.loadFromQuery(*args.storage_def); } else @@ -643,14 +638,14 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (replicated) return StorageReplicatedMergeTree::create( zookeeper_path, replica_name, args.attach, args.data_path, args.database_name, args.table_name, - args.columns, indices_description, constraints_description, + args.columns, indices_description, args.constraints, args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, storage_settings, args.has_force_restore_data_flag); else return StorageMergeTree::create( args.data_path, args.database_name, args.table_name, args.columns, indices_description, - constraints_description, args.attach, args.context, date_column_name, partition_by_ast, order_by_ast, + args.constraints, args.attach, args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, storage_settings, args.has_force_restore_data_flag); } diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index cef1c966951..b78cfdf3468 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -55,11 +55,12 @@ namespace ErrorCodes } -StorageBuffer::StorageBuffer(const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, +StorageBuffer::StorageBuffer(const std::string & database_name_, const std::string & table_name_, + const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_, bool allow_materialized_) - : IStorage{columns_}, + : table_name(table_name_), database_name(database_name_), global_context(context_), num_shards(num_shards_), buffers(num_shards_), min_thresholds(min_thresholds_), max_thresholds(max_thresholds_), @@ -67,6 +68,8 @@ StorageBuffer::StorageBuffer(const std::string & database_name_, const std::stri no_destination(destination_database.empty() && destination_table.empty()), allow_materialized(allow_materialized_), log(&Logger::get("StorageBuffer (" + table_name + ")")) { + setColumns(columns_); + setConstraints(constraints_); } StorageBuffer::~StorageBuffer() @@ -743,7 +746,7 @@ void registerStorageBuffer(StorageFactory & factory) return StorageBuffer::create( args.database_name, - args.table_name, args.columns, + args.table_name, args.columns, args.constraints, args.context, num_buckets, StorageBuffer::Thresholds{min_time, min_rows, min_bytes}, diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index 4c317a7a102..8060199d20d 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -140,7 +140,8 @@ protected: /** num_shards - the level of internal parallelism (the number of independent buffers) * The buffer is flushed if all minimum thresholds or at least one of the maximum thresholds are exceeded. */ - StorageBuffer(const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, + StorageBuffer(const std::string & database_name_, const std::string & table_name_, + const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_, bool allow_materialized_); diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index 923a309f1f6..ced0025e36c 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -30,11 +30,13 @@ StorageDictionary::StorageDictionary( const Context & context, bool attach, const String & dictionary_name_) - : IStorage{columns_}, table_name(table_name_), + : table_name(table_name_), database_name(database_name_), dictionary_name(dictionary_name_), logger(&Poco::Logger::get("StorageDictionary")) { + setColumns(columns_); + if (!attach) { const auto & dictionary = context.getExternalDictionaries().getDictionary(dictionary_name); diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 7c6300bd1a6..7c20622d89e 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -209,6 +209,7 @@ StorageDistributed::StorageDistributed( const String & database_name_, const String & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, const String & remote_database_, const String & remote_table_, const String & cluster_name_, @@ -216,13 +217,16 @@ StorageDistributed::StorageDistributed( const ASTPtr & sharding_key_, const String & data_path_, bool attach_) - : IStorage{columns_}, table_name(table_name_), database_name(database_name_), + : table_name(table_name_), database_name(database_name_), remote_database(remote_database_), remote_table(remote_table_), global_context(context_), cluster_name(global_context.getMacros()->expand(cluster_name_)), has_sharding_key(sharding_key_), sharding_key_expr(sharding_key_ ? buildShardingKeyExpression(sharding_key_, global_context, getColumns().getAllPhysical(), false) : nullptr), sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/')) { + setColumns(columns_); + setConstraints(constraints_); + /// Sanity check. Skip check if the table is already created to allow the server to start. if (!attach_ && !cluster_name.empty()) { @@ -237,15 +241,16 @@ StorageDistributed::StorageDistributed( const String & database_name_, const String & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, ASTPtr remote_table_function_ptr_, const String & cluster_name_, const Context & context_, const ASTPtr & sharding_key_, const String & data_path_, bool attach) - : StorageDistributed(database_name_, table_name_, columns_, String{}, String{}, cluster_name_, context_, sharding_key_, data_path_, attach) + : StorageDistributed(database_name_, table_name_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, data_path_, attach) { - remote_table_function_ptr = remote_table_function_ptr_; + remote_table_function_ptr = remote_table_function_ptr_; } @@ -258,7 +263,7 @@ StoragePtr StorageDistributed::createWithOwnCluster( const Context & context_) { auto res = ext::shared_ptr_helper::create( - String{}, table_name_, columns_, remote_database_, remote_table_, String{}, context_, ASTPtr(), String(), false); + String{}, table_name_, columns_, ConstraintsDescription{}, remote_database_, remote_table_, String{}, context_, ASTPtr(), String(), false); res->owned_cluster = owned_cluster_; @@ -274,7 +279,7 @@ StoragePtr StorageDistributed::createWithOwnCluster( const Context & context_) { auto res = ext::shared_ptr_helper::create( - String{}, table_name_, columns_, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), false); + String{}, table_name_, columns_, ConstraintsDescription{}, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), false); res->owned_cluster = owned_cluster_; @@ -634,7 +639,7 @@ void registerStorageDistributed(StorageFactory & factory) } return StorageDistributed::create( - args.database_name, args.table_name, args.columns, + args.database_name, args.table_name, args.columns, args.constraints, remote_database, remote_table, cluster_name, args.context, sharding_key, args.data_path, args.attach); diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index 6885a758e9e..e8da6dca4a7 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -161,6 +161,7 @@ protected: const String & database_name_, const String & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, const String & remote_database_, const String & remote_table_, const String & cluster_name_, @@ -173,6 +174,7 @@ protected: const String & database_name, const String & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, ASTPtr remote_table_function_ptr_, const String & cluster_name_, const Context & context_, diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 862f76bc3ce..dc73e8a53b8 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -46,6 +46,7 @@ StoragePtr StorageFactory::get( Context & local_context, Context & context, const ColumnsDescription & columns, + const ConstraintsDescription & constraints, bool attach, bool has_force_restore_data_flag) const { @@ -154,6 +155,7 @@ StoragePtr StorageFactory::get( .local_context = local_context, .context = context, .columns = columns, + .constraints = constraints, .attach = attach, .has_force_restore_data_flag = has_force_restore_data_flag }; diff --git a/dbms/src/Storages/StorageFactory.h b/dbms/src/Storages/StorageFactory.h index 2c97fb798ee..43ffc5a638e 100644 --- a/dbms/src/Storages/StorageFactory.h +++ b/dbms/src/Storages/StorageFactory.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,7 @@ public: Context & local_context; Context & context; const ColumnsDescription & columns; + const ConstraintsDescription & constraints; bool attach; bool has_force_restore_data_flag; }; @@ -49,6 +51,7 @@ public: Context & local_context, Context & context, const ColumnsDescription & columns, + const ConstraintsDescription & constraints, bool attach, bool has_force_restore_data_flag) const; diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 5162e667133..2db24bbd610 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -72,10 +72,14 @@ StorageFile::StorageFile( const std::string & table_name_, const std::string & format_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, Context & context_) - : IStorage(columns_), + : table_name(table_name_), database_name(database_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_) { + setColumns(columns_); + setConstraints(constraints_); + if (table_fd < 0) /// Will use file { use_table_fd = false; @@ -330,7 +334,7 @@ void registerStorageFile(StorageFactory & factory) return StorageFile::create( source_path, source_fd, args.data_path, - args.database_name, args.table_name, format_name, args.columns, + args.database_name, args.table_name, format_name, args.columns, args.constraints, args.context); }); } diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index cc5878520ce..237e9a6989e 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -60,6 +60,7 @@ protected: const std::string & table_name_, const std::string & format_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, Context & context_); private: diff --git a/dbms/src/Storages/StorageHDFS.cpp b/dbms/src/Storages/StorageHDFS.cpp index 8c87f4ccd6a..aa055f7d907 100644 --- a/dbms/src/Storages/StorageHDFS.cpp +++ b/dbms/src/Storages/StorageHDFS.cpp @@ -30,14 +30,16 @@ StorageHDFS::StorageHDFS(const String & uri_, const std::string & table_name_, const String & format_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, Context & context_) - : IStorage(columns_) - , uri(uri_) + : uri(uri_) , format_name(format_name_) , table_name(table_name_) , database_name(database_name_) , context(context_) { + setColumns(columns_); + setConstraints(constraints_); } namespace @@ -175,7 +177,7 @@ void registerStorageHDFS(StorageFactory & factory) String format_name = engine_args[1]->as().value.safeGet(); - return StorageHDFS::create(url, args.database_name, args.table_name, format_name, args.columns, args.context); + return StorageHDFS::create(url, args.database_name, args.table_name, format_name, args.columns, args.constraints, args.context); }); } diff --git a/dbms/src/Storages/StorageHDFS.h b/dbms/src/Storages/StorageHDFS.h index 44f0286f97e..dae7000dd8a 100644 --- a/dbms/src/Storages/StorageHDFS.h +++ b/dbms/src/Storages/StorageHDFS.h @@ -37,6 +37,7 @@ protected: const String & table_name_, const String & format_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, Context & context_); private: diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index b9a6e28afb7..6577846efba 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -36,8 +36,9 @@ StorageJoin::StorageJoin( ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, bool overwrite) - : StorageSetOrJoinBase{path_, database_name_, table_name_, columns_} + : StorageSetOrJoinBase{path_, database_name_, table_name_, columns_, constraints_} , key_names(key_names_) , use_nulls(use_nulls_) , limits(limits_) @@ -170,6 +171,7 @@ void registerStorageJoin(StorageFactory & factory) kind, strictness, args.columns, + args.constraints, join_any_take_last_row); }); } diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index 356f4b0e7d9..c9e1283b90f 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -64,6 +64,7 @@ protected: SizeLimits limits_, ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, bool overwrite); }; diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index d17caeb0046..69e37ce2305 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -422,12 +422,15 @@ StorageLog::StorageLog( const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, size_t max_compress_block_size_) - : IStorage{columns_}, - path(path_), table_name(table_name_), database_name(database_name_), + : path(path_), table_name(table_name_), database_name(database_name_), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(table_name) + '/' + "sizes.json") { + setColumns(columns_); + setConstraints(constraints_); + if (path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); @@ -644,7 +647,7 @@ void registerStorageLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageLog::create( - args.data_path, args.database_name, args.table_name, args.columns, + args.data_path, args.database_name, args.table_name, args.columns, args.constraints, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index f612d0ab29a..e646ecd8c46 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -57,6 +57,7 @@ protected: const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, size_t max_compress_block_size_); private: diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 8a33e8af030..df39f711ab3 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -99,9 +99,11 @@ StorageMaterializedView::StorageMaterializedView( const ASTCreateQuery & query, const ColumnsDescription & columns_, bool attach_) - : IStorage{columns_}, table_name(table_name_), + : table_name(table_name_), database_name(database_name_), global_context(local_context.getGlobalContext()) { + setColumns(columns_); + if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 452ee7d51c0..03c93d8d29f 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -31,6 +31,7 @@ public: } BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; + void drop() override; void truncate(const ASTPtr &, const Context &) override; diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index 5e8f100f2f6..560da7dc2d8 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -74,9 +74,11 @@ private: }; -StorageMemory::StorageMemory(String database_name_, String table_name_, ColumnsDescription columns_description_) - : IStorage{std::move(columns_description_)}, database_name(std::move(database_name_)), table_name(std::move(table_name_)) +StorageMemory::StorageMemory(String database_name_, String table_name_, ColumnsDescription columns_description_, ConstraintsDescription constraints_) + : database_name(std::move(database_name_)), table_name(std::move(table_name_)) { + setColumns(std::move(columns_description_)); + setConstraints(std::move(constraints_)); } @@ -143,7 +145,7 @@ void registerStorageMemory(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageMemory::create(args.database_name, args.table_name, args.columns); + return StorageMemory::create(args.database_name, args.table_name, args.columns, args.constraints); }); } diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index e4a0f0cf9d4..88e9e5dcf04 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -55,7 +55,7 @@ private: std::mutex mutex; protected: - StorageMemory(String database_name_, String table_name_, ColumnsDescription columns_description_); + StorageMemory(String database_name_, String table_name_, ColumnsDescription columns_description_, ConstraintsDescription constraints_); }; } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index f362b34a515..05ff3d97943 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -53,13 +53,14 @@ StorageMerge::StorageMerge( const String & source_database_, const String & table_name_regexp_, const Context & context_) - : IStorage(columns_, ColumnsDescription({{"_table", std::make_shared()}}, true)) + : IStorage(ColumnsDescription({{"_table", std::make_shared()}}, true)) , table_name(table_name_) , database_name(database_name_) , source_database(source_database_) , table_name_regexp(table_name_regexp_) , global_context(context_) { + setColumns(columns_); } diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index 4ad00338793..ca5313e6ba8 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -45,9 +45,9 @@ StorageMySQL::StorageMySQL( const bool replace_query_, const std::string & on_duplicate_clause_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, const Context & context_) - : IStorage{columns_} - , table_name(table_name_) + : table_name(table_name_) , database_name(database_name_) , remote_database_name(remote_database_name_) , remote_table_name(remote_table_name_) @@ -56,6 +56,8 @@ StorageMySQL::StorageMySQL( , pool(std::move(pool_)) , global_context(context_) { + setColumns(columns_); + setConstraints(constraints_); } @@ -241,6 +243,7 @@ void registerStorageMySQL(StorageFactory & factory) replace_query, on_duplicate_clause, args.columns, + args.constraints, args.context); }); } diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h index 3b891566ed0..320eab9d4a9 100644 --- a/dbms/src/Storages/StorageMySQL.h +++ b/dbms/src/Storages/StorageMySQL.h @@ -28,6 +28,7 @@ public: const bool replace_query_, const std::string & on_duplicate_clause_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, const Context & context_); std::string getName() const override { return "MySQL"; } @@ -54,7 +55,6 @@ private: bool replace_query; std::string on_duplicate_clause; - mysqlxx::Pool pool; Context global_context; }; diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index 7894f8804fc..4a54995c544 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -26,7 +26,7 @@ void registerStorageNull(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageNull::create(args.database_name, args.table_name, args.columns); + return StorageNull::create(args.database_name, args.table_name, args.columns, args.constraints); }); } diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index fec7638706f..d858f738b24 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -52,9 +52,11 @@ private: String database_name; protected: - StorageNull(String database_name_, String table_name_, ColumnsDescription columns_description_) - : IStorage{std::move(columns_description_)}, table_name(std::move(table_name_)), database_name(std::move(database_name_)) + StorageNull(String database_name_, String table_name_, ColumnsDescription columns_description_, ConstraintsDescription constraints_) + : table_name(std::move(table_name_)), database_name(std::move(database_name_)) { + setColumns(std::move(columns_description_)); + setConstraints(std::move(constraints_)); } }; diff --git a/dbms/src/Storages/StorageSet.cpp b/dbms/src/Storages/StorageSet.cpp index 0d649728096..26af630ca62 100644 --- a/dbms/src/Storages/StorageSet.cpp +++ b/dbms/src/Storages/StorageSet.cpp @@ -90,9 +90,13 @@ StorageSetOrJoinBase::StorageSetOrJoinBase( const String & path_, const String & database_name_, const String & table_name_, - const ColumnsDescription & columns_) - : IStorage{columns_}, table_name(table_name_), database_name(database_name_) + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_) + : table_name(table_name_), database_name(database_name_) { + setColumns(columns_); + setConstraints(constraints_); + if (path_.empty()) throw Exception("Join and Set storages require data path", ErrorCodes::INCORRECT_FILE_NAME); @@ -105,8 +109,9 @@ StorageSet::StorageSet( const String & path_, const String & database_name_, const String & table_name_, - const ColumnsDescription & columns_) - : StorageSetOrJoinBase{path_, database_name_, table_name_, columns_}, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_) + : StorageSetOrJoinBase{path_, database_name_, table_name_, columns_, constraints_}, set(std::make_shared(SizeLimits(), false)) { Block header = getSampleBlock(); @@ -209,7 +214,7 @@ void registerStorageSet(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageSet::create(args.data_path, args.database_name, args.table_name, args.columns); + return StorageSet::create(args.data_path, args.database_name, args.table_name, args.columns, args.constraints); }); } diff --git a/dbms/src/Storages/StorageSet.h b/dbms/src/Storages/StorageSet.h index 1c2891b78e1..fe6cd332ed8 100644 --- a/dbms/src/Storages/StorageSet.h +++ b/dbms/src/Storages/StorageSet.h @@ -33,7 +33,8 @@ protected: const String & path_, const String & database_name_, const String & table_name_, - const ColumnsDescription & columns_); + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_); String path; String table_name; @@ -81,7 +82,8 @@ protected: const String & path_, const String & database_name_, const String & table_name_, - const ColumnsDescription & columns_); + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_); }; } diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp index 42745e11971..447e325095d 100644 --- a/dbms/src/Storages/StorageStripeLog.cpp +++ b/dbms/src/Storages/StorageStripeLog.cpp @@ -198,14 +198,17 @@ StorageStripeLog::StorageStripeLog( const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, bool attach, size_t max_compress_block_size_) - : IStorage{columns_}, - path(path_), table_name(table_name_), database_name(database_name_), + : path(path_), table_name(table_name_), database_name(database_name_), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(table_name) + '/' + "sizes.json"), log(&Logger::get("StorageStripeLog")) { + setColumns(columns_); + setConstraints(constraints_); + if (path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); @@ -316,7 +319,7 @@ void registerStorageStripeLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageStripeLog::create( - args.data_path, args.database_name, args.table_name, args.columns, + args.data_path, args.database_name, args.table_name, args.columns, args.constraints, args.attach, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index 6d528353dbf..22be356713b 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -74,6 +74,7 @@ protected: const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, bool attach, size_t max_compress_block_size_); }; diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 778b07503a8..45d9771822e 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -327,14 +327,17 @@ StorageTinyLog::StorageTinyLog( const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, bool attach, size_t max_compress_block_size_) - : IStorage{columns_}, - path(path_), table_name(table_name_), database_name(database_name_), + : path(path_), table_name(table_name_), database_name(database_name_), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(table_name) + '/' + "sizes.json"), log(&Logger::get("StorageTinyLog")) { + setColumns(columns_); + setConstraints(constraints_); + if (path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); @@ -450,7 +453,7 @@ void registerStorageTinyLog(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageTinyLog::create( - args.data_path, args.database_name, args.table_name, args.columns, + args.data_path, args.database_name, args.table_name, args.columns, args.constraints, args.attach, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index 9ed53f12962..7a37a42ec63 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -78,6 +78,7 @@ protected: const std::string & database_name_, const std::string & table_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, bool attach, size_t max_compress_block_size_); }; diff --git a/dbms/src/Storages/StorageURL.cpp b/dbms/src/Storages/StorageURL.cpp index 8a36c008be9..ee385af0fe8 100644 --- a/dbms/src/Storages/StorageURL.cpp +++ b/dbms/src/Storages/StorageURL.cpp @@ -24,14 +24,18 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -IStorageURLBase::IStorageURLBase(const Poco::URI & uri_, +IStorageURLBase::IStorageURLBase( + const Poco::URI & uri_, const Context & context_, const std::string & database_name_, const std::string & table_name_, const String & format_name_, - const ColumnsDescription & columns_) - : IStorage(columns_), uri(uri_), context_global(context_), format_name(format_name_), table_name(table_name_), database_name(database_name_) + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_) + : uri(uri_), context_global(context_), format_name(format_name_), table_name(table_name_), database_name(database_name_) { + setColumns(columns_); + setConstraints(constraints_); } namespace @@ -214,7 +218,7 @@ void registerStorageURL(StorageFactory & factory) String format_name = engine_args[1]->as().value.safeGet(); - return StorageURL::create(uri, args.database_name, args.table_name, format_name, args.columns, args.context); + return StorageURL::create(uri, args.database_name, args.table_name, format_name, args.columns, args.constraints, args.context); }); } } diff --git a/dbms/src/Storages/StorageURL.h b/dbms/src/Storages/StorageURL.h index 361b0f6c35e..0c4b4648223 100644 --- a/dbms/src/Storages/StorageURL.h +++ b/dbms/src/Storages/StorageURL.h @@ -2,9 +2,9 @@ #include #include -#include #include + namespace DB { /** @@ -19,7 +19,8 @@ public: String getTableName() const override { return table_name; } String getDatabaseName() const override { return database_name; } - BlockInputStreams read(const Names & column_names, + BlockInputStreams read( + const Names & column_names, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum processed_stage, @@ -31,12 +32,14 @@ public: void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; protected: - IStorageURLBase(const Poco::URI & uri_, + IStorageURLBase( + const Poco::URI & uri_, const Context & context_, const std::string & database_name_, const std::string & table_name_, const String & format_name_, - const ColumnsDescription & columns_); + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_); Poco::URI uri; const Context & context_global; @@ -48,13 +51,15 @@ private: virtual std::string getReadMethod() const; - virtual std::vector> getReadURIParams(const Names & column_names, + virtual std::vector> getReadURIParams( + const Names & column_names, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size) const; - virtual std::function getReadPOSTDataCallback(const Names & column_names, + virtual std::function getReadPOSTDataCallback( + const Names & column_names, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, @@ -63,16 +68,19 @@ private: virtual Block getHeaderBlock(const Names & column_names) const = 0; }; + class StorageURL : public ext::shared_ptr_helper, public IStorageURLBase { public: - StorageURL(const Poco::URI & uri_, + StorageURL( + const Poco::URI & uri_, const std::string & database_name_, const std::string & table_name_, const String & format_name_, const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, Context & context_) - : IStorageURLBase(uri_, context_, database_name_, table_name_, format_name_, columns_) + : IStorageURLBase(uri_, context_, database_name_, table_name_, format_name_, columns_, constraints_) { } diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 153de1ab176..912e358e3ae 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -31,8 +31,10 @@ StorageView::StorageView( const String & table_name_, const ASTCreateQuery & query, const ColumnsDescription & columns_) - : IStorage{columns_}, table_name(table_name_), database_name(database_name_) + : table_name(table_name_), database_name(database_name_) { + setColumns(columns_); + if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); diff --git a/dbms/src/Storages/StorageXDBC.cpp b/dbms/src/Storages/StorageXDBC.cpp index 57327fc4dcd..915146e1409 100644 --- a/dbms/src/Storages/StorageXDBC.cpp +++ b/dbms/src/Storages/StorageXDBC.cpp @@ -30,7 +30,8 @@ StorageXDBC::StorageXDBC( const ColumnsDescription & columns_, const Context & context_, const BridgeHelperPtr bridge_helper_) - : IStorageURLBase(Poco::URI(), context_, database_name_, table_name_, IXDBCBridgeHelper::DEFAULT_FORMAT, columns_) + /// Please add support for constraints as soon as StorageODBC or JDBC will support insertion. + : IStorageURLBase(Poco::URI(), context_, database_name_, table_name_, IXDBCBridgeHelper::DEFAULT_FORMAT, columns_, ConstraintsDescription{}) , bridge_helper(bridge_helper_) , remote_database_name(remote_database_name_) , remote_table_name(remote_table_name_) diff --git a/dbms/src/Storages/StorageXDBC.h b/dbms/src/Storages/StorageXDBC.h index 23e108519b4..c658fc5c8a1 100644 --- a/dbms/src/Storages/StorageXDBC.h +++ b/dbms/src/Storages/StorageXDBC.h @@ -1,59 +1,61 @@ #pragma once #include -#include #include + namespace DB { + /** Implements storage in the XDBC database. * Use ENGINE = xdbc(connection_string, table_name) * Example ENGINE = odbc('dsn=test', table) * Read only. */ - class StorageXDBC : public IStorageURLBase - { - public: +class StorageXDBC : public IStorageURLBase +{ +public: - BlockInputStreams read(const Names & column_names, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - unsigned num_streams) override; + BlockInputStreams read(const Names & column_names, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; - StorageXDBC(const std::string & database_name_, - const std::string & table_name_, - const std::string & remote_database_name, - const std::string & remote_table_name, - const ColumnsDescription & columns_, - const Context & context_, BridgeHelperPtr bridge_helper_); + StorageXDBC(const std::string & database_name_, + const std::string & table_name_, + const std::string & remote_database_name, + const std::string & remote_table_name, + const ColumnsDescription & columns_, + const Context & context_, BridgeHelperPtr bridge_helper_); - private: +private: - BridgeHelperPtr bridge_helper; - std::string remote_database_name; - std::string remote_table_name; + BridgeHelperPtr bridge_helper; + std::string remote_database_name; + std::string remote_table_name; - Poco::Logger * log; + Poco::Logger * log; - std::string getReadMethod() const override; + std::string getReadMethod() const override; - std::vector> getReadURIParams(const Names & column_names, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - size_t max_block_size) const override; + std::vector> getReadURIParams(const Names & column_names, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum & processed_stage, + size_t max_block_size) const override; - std::function getReadPOSTDataCallback(const Names & column_names, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - size_t max_block_size) const override; + std::function getReadPOSTDataCallback(const Names & column_names, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum & processed_stage, + size_t max_block_size) const override; - Block getHeaderBlock(const Names & column_names) const override; + Block getHeaderBlock(const Names & column_names) const override; + + std::string getName() const override; +}; - std::string getName() const override; - }; } diff --git a/dbms/src/TableFunctions/TableFunctionFile.cpp b/dbms/src/TableFunctions/TableFunctionFile.cpp index 9bd85a5bbb5..7cf2c500f1e 100644 --- a/dbms/src/TableFunctions/TableFunctionFile.cpp +++ b/dbms/src/TableFunctions/TableFunctionFile.cpp @@ -15,6 +15,7 @@ StoragePtr TableFunctionFile::getStorage( table_name, format, columns, + ConstraintsDescription{}, global_context); } diff --git a/dbms/src/TableFunctions/TableFunctionHDFS.cpp b/dbms/src/TableFunctions/TableFunctionHDFS.cpp index c4b61e178d6..4ff999d31dd 100644 --- a/dbms/src/TableFunctions/TableFunctionHDFS.cpp +++ b/dbms/src/TableFunctions/TableFunctionHDFS.cpp @@ -16,6 +16,7 @@ StoragePtr TableFunctionHDFS::getStorage( table_name, format, columns, + ConstraintsDescription{}, global_context); } diff --git a/dbms/src/TableFunctions/TableFunctionMySQL.cpp b/dbms/src/TableFunctions/TableFunctionMySQL.cpp index cee08cf2a1d..3cb9b8dea60 100644 --- a/dbms/src/TableFunctions/TableFunctionMySQL.cpp +++ b/dbms/src/TableFunctions/TableFunctionMySQL.cpp @@ -127,6 +127,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co replace_query, on_duplicate_clause, ColumnsDescription{columns}, + ConstraintsDescription{}, context); res->startup(); diff --git a/dbms/src/TableFunctions/TableFunctionURL.cpp b/dbms/src/TableFunctions/TableFunctionURL.cpp index 9991bc2f527..1f89264a422 100644 --- a/dbms/src/TableFunctions/TableFunctionURL.cpp +++ b/dbms/src/TableFunctions/TableFunctionURL.cpp @@ -4,13 +4,14 @@ #include #include + namespace DB { StoragePtr TableFunctionURL::getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name) const { Poco::URI uri(source); - return StorageURL::create(uri, getDatabaseName(), table_name, format, columns, global_context); + return StorageURL::create(uri, getDatabaseName(), table_name, format, columns, ConstraintsDescription{}, global_context); } void registerTableFunctionURL(TableFunctionFactory & factory) From 9660a4db8000981891c58cc7f9999f7c20b74fdf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Aug 2019 00:35:07 +0300 Subject: [PATCH 228/357] Better exception message --- .../CheckConstraintsBlockOutputStream.cpp | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 8fbb5916372..b50bc3da028 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -56,9 +57,29 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) if (data[row_idx] != 1) break; - throw Exception{"Violated constraint " + constraints.constraints[i]->name + - " in table " + table + " at row " + std::to_string(rows_written + row_idx + 1) + ", constraint expression: " + - serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::VIOLATED_CONSTRAINT}; + Names related_columns = constraint_expr->getRequiredColumns(); + + std::stringstream exception_message; + + exception_message << "Constraint " << backQuote(constraints.constraints[i]->name) + << " for table " << backQuote(table) + << " is violated at row " << (rows_written + row_idx + 1) + << ". Expression: (" << serializeAST(*(constraints.constraints[i]->expr), true) << ")" + << ". Column values"; + + bool first = true; + for (const auto & name : related_columns) + { + const IColumn & column = *block.getByName(name).column; + assert(row_idx < column.size()); + + exception_message << (first ? ": " : ", ") + << backQuoteIfNeed(name) << " = " << applyVisitor(FieldVisitorToString(), column[row_idx]); + + first = false; + } + + throw Exception{exception_message.str(), ErrorCodes::VIOLATED_CONSTRAINT}; } } } From 2fc177830c620a2f1926863954d8771879a7c50c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Aug 2019 01:01:01 +0300 Subject: [PATCH 229/357] Added a test --- .../00998_constraints_all_tables.reference | 14 +++++ .../00998_constraints_all_tables.sql | 53 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00998_constraints_all_tables.reference create mode 100644 dbms/tests/queries/0_stateless/00998_constraints_all_tables.sql diff --git a/dbms/tests/queries/0_stateless/00998_constraints_all_tables.reference b/dbms/tests/queries/0_stateless/00998_constraints_all_tables.reference new file mode 100644 index 00000000000..730df555af3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00998_constraints_all_tables.reference @@ -0,0 +1,14 @@ +0 +0 +3 +0 +0 +3 +0 +0 +3 +0 +0 +3 +CREATE TABLE default.constrained (`URL` String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log +CREATE TABLE default.constrained2 (`URL` String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log diff --git a/dbms/tests/queries/0_stateless/00998_constraints_all_tables.sql b/dbms/tests/queries/0_stateless/00998_constraints_all_tables.sql new file mode 100644 index 00000000000..66b93fca97b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00998_constraints_all_tables.sql @@ -0,0 +1,53 @@ +DROP TABLE IF EXISTS constrained; +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Null; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +DROP TABLE constrained; + +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Memory; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +SELECT count() FROM constrained; +DROP TABLE constrained; + +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = StripeLog; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +SELECT count() FROM constrained; +DROP TABLE constrained; + +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = TinyLog; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +SELECT count() FROM constrained; +DROP TABLE constrained; + +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), ('https://yandex.ru/te\xFFst'); -- { serverError 469 } +SELECT count() FROM constrained; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('ftp://yandex.ru/Hello'), (toValidUTF8('https://yandex.ru/te\xFFst')); +SELECT count() FROM constrained; +DROP TABLE constrained; + + +DROP TABLE IF EXISTS constrained2; +CREATE TABLE constrained (URL String, CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT is_utf8 CHECK isValidUTF8(URL)) ENGINE = Log; +CREATE TABLE constrained2 AS constrained; +SHOW CREATE TABLE constrained FORMAT TSVRaw; +SHOW CREATE TABLE constrained2 FORMAT TSVRaw; +INSERT INTO constrained VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +INSERT INTO constrained2 VALUES ('https://www.yandex.ru/?q=upyachka'), ('Hello'), ('test'); -- { serverError 469 } +DROP TABLE constrained; +DROP TABLE constrained2; From 2dbfabd08c839a54a27ed99dead67b71ed8660ff Mon Sep 17 00:00:00 2001 From: Nikita Vasilev <31595000+nikvas0@users.noreply.github.com> Date: Sun, 25 Aug 2019 01:01:36 +0300 Subject: [PATCH 230/357] fix Set index check useless (#6651) * fixed useless detection * fixed useless detection * fix * fix * fix --- .../Storages/MergeTree/MergeTreeIndexSet.cpp | 25 ++----------------- .../00997_set_index_array.reference | 1 + .../0_stateless/00997_set_index_array.sql | 24 ++++++++++++++++++ 3 files changed, 27 insertions(+), 23 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00997_set_index_array.reference create mode 100644 dbms/tests/queries/0_stateless/00997_set_index_array.sql diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp index 40aba822353..954ac774583 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -405,25 +405,6 @@ bool MergeTreeIndexConditionSet::operatorFromAST(ASTPtr & node) const return true; } -static bool checkAtomName(const String & name) -{ - static std::set atoms = { - "notEquals", - "equals", - "less", - "greater", - "lessOrEquals", - "greaterOrEquals", - "in", - "notIn", - "like", - "startsWith", - "endsWith", - "multiSearchAny" - }; - return atoms.find(name) != atoms.end(); -} - bool MergeTreeIndexConditionSet::checkASTUseless(const ASTPtr &node, bool atomic) const { if (const auto * func = node->as()) @@ -439,16 +420,14 @@ bool MergeTreeIndexConditionSet::checkASTUseless(const ASTPtr &node, bool atomic return checkASTUseless(args[0], atomic) || checkASTUseless(args[1], atomic); else if (func->name == "not") return checkASTUseless(args[0], atomic); - else if (!atomic && checkAtomName(func->name)) - return checkASTUseless(node, true); else return std::any_of(args.begin(), args.end(), - [this, &atomic](const auto & arg) { return checkASTUseless(arg, atomic); }); + [this](const auto & arg) { return checkASTUseless(arg, true); }); } else if (const auto * literal = node->as()) return !atomic && literal->value.get(); else if (const auto * identifier = node->as()) - return key_columns.find(identifier->getColumnName()) == key_columns.end(); + return key_columns.find(identifier->getColumnName()) == std::end(key_columns); else return true; } diff --git a/dbms/tests/queries/0_stateless/00997_set_index_array.reference b/dbms/tests/queries/0_stateless/00997_set_index_array.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_set_index_array.reference @@ -0,0 +1 @@ +1 diff --git a/dbms/tests/queries/0_stateless/00997_set_index_array.sql b/dbms/tests/queries/0_stateless/00997_set_index_array.sql new file mode 100644 index 00000000000..c57507ce22d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00997_set_index_array.sql @@ -0,0 +1,24 @@ +SET allow_experimental_data_skipping_indices = 1; + +DROP TABLE IF EXISTS test.set_array; + +CREATE TABLE test.set_array +( + primary_key String, + index_array Array(UInt64), + INDEX additional_index_array (index_array) TYPE set(10000) GRANULARITY 1 +) ENGINE = MergeTree() +ORDER BY (primary_key); + +INSERT INTO test.set_array +select + toString(intDiv(number, 1000000)) as primary_key, + array(number) as index_array +from system.numbers +limit 10000000; + +SET max_rows_to_read = 8192; + +select count() from test.set_array where has(index_array, 333); + +DROP TABLE test.set_array; \ No newline at end of file From 17a1938f0f1297d02a3cd438237e7ade59273810 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Aug 2019 01:04:36 +0300 Subject: [PATCH 231/357] Addition to prev. revision --- .../src/DataStreams/CheckConstraintsBlockOutputStream.cpp | 8 ++++---- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 2 -- dbms/tests/queries/0_stateless/00952_basic_constraints.sh | 5 ++--- .../queries/0_stateless/00953_constraints_operations.sh | 3 ++- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index b50bc3da028..6c04558144a 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -34,12 +34,9 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { if (block.rows() > 0) { - std::cerr << "Checking " << expressions.size() << " constraints\n"; + Block block_to_calculate = block; for (size_t i = 0; i < expressions.size(); ++i) { - std::cerr << serializeAST(*(constraints.constraints[i]->expr), true) << "\n"; - - Block block_to_calculate = block; auto constraint_expr = expressions[i]; constraint_expr->execute(block_to_calculate); @@ -54,8 +51,11 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { size_t row_idx = 0; for (; row_idx < size; ++row_idx) + { + std::cerr << row_idx << ": " << int(data[row_idx]) << "\n"; if (data[row_idx] != 1) break; + } Names related_columns = constraint_expr->getRequiredColumns(); diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 549e06e0155..45d528b0e74 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -121,8 +121,6 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared( out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); - std::cerr << table->getConstraints().toString() << "\n"; - if (const auto & constraints = table->getConstraints(); !constraints.empty()) out = std::make_shared(query.table, out, query_sample_block, table->getConstraints(), context); diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index b6aa28c46bf..8ba3b082f12 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -3,6 +3,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +EXCEPTION_TEXT=violated EXCEPTION_SUCCESS_TEXT=ok $CLICKHOUSE_CLIENT --query="CREATE DATABASE IF NOT EXISTS test;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" @@ -20,7 +21,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices {1, 3}" + $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0), (3, 4), (6, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" @@ -38,13 +39,11 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception -EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices {0}" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception -EXCEPTION_TEXT="Violated constraint a_constraint in table test_constraints at indices {1}" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11), (9, 11), (8, 12);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh index 8a563a21e02..3b415848501 100755 --- a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh @@ -3,6 +3,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +EXCEPTION_TEXT=violated EXCEPTION_SUCCESS_TEXT=ok $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" @@ -20,7 +21,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices" + $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" From ffbbb173c5a59c26988a0d6441ae7f41994ca501 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Aug 2019 01:04:58 +0300 Subject: [PATCH 232/357] Addition to prev. revision --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 6c04558144a..4bdd1a73295 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -51,11 +51,8 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { size_t row_idx = 0; for (; row_idx < size; ++row_idx) - { - std::cerr << row_idx << ": " << int(data[row_idx]) << "\n"; if (data[row_idx] != 1) break; - } Names related_columns = constraint_expr->getRequiredColumns(); From 7144a3f827a34a30f50c7818681c9fe21d40ac93 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Aug 2019 01:06:13 +0300 Subject: [PATCH 233/357] Speed up MemoryTracker by function inlining --- dbms/src/Common/CurrentThread.cpp | 6 ------ dbms/src/Common/CurrentThread.h | 7 ++++++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dbms/src/Common/CurrentThread.cpp b/dbms/src/Common/CurrentThread.cpp index 446772f218d..ca39bec414c 100644 --- a/dbms/src/Common/CurrentThread.cpp +++ b/dbms/src/Common/CurrentThread.cpp @@ -51,12 +51,6 @@ MemoryTracker * CurrentThread::getMemoryTracker() return ¤t_thread->memory_tracker; } -Int64 & CurrentThread::getUntrackedMemory() -{ - /// It assumes that (current_thread != nullptr) is already checked with getMemoryTracker() - return current_thread->untracked_memory; -} - void CurrentThread::updateProgressIn(const Progress & value) { if (unlikely(!current_thread)) diff --git a/dbms/src/Common/CurrentThread.h b/dbms/src/Common/CurrentThread.h index 01e46fbeadc..1e0140c6330 100644 --- a/dbms/src/Common/CurrentThread.h +++ b/dbms/src/Common/CurrentThread.h @@ -52,7 +52,12 @@ public: static ProfileEvents::Counters & getProfileEvents(); static MemoryTracker * getMemoryTracker(); - static Int64 & getUntrackedMemory(); + + static inline Int64 & getUntrackedMemory() + { + /// It assumes that (current_thread != nullptr) is already checked with getMemoryTracker() + return current_thread->untracked_memory; + } /// Update read and write rows (bytes) statistics (used in system.query_thread_log) static void updateProgressIn(const Progress & value); From 6d3250867c1151ebeb8fb3a078fb73a6e1f6509a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Aug 2019 01:08:45 +0300 Subject: [PATCH 234/357] Fixed build of tests --- dbms/src/Interpreters/tests/aggregate.cpp | 2 +- dbms/src/Processors/tests/processors_test_aggregation.cpp | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/dbms/src/Interpreters/tests/aggregate.cpp b/dbms/src/Interpreters/tests/aggregate.cpp index e0068e9a56b..73e71d178ea 100644 --- a/dbms/src/Interpreters/tests/aggregate.cpp +++ b/dbms/src/Interpreters/tests/aggregate.cpp @@ -79,7 +79,7 @@ int main(int argc, char ** argv) Aggregator::Params params( stream->getHeader(), {0, 1}, aggregate_descriptions, - false, 0, OverflowMode::THROW, nullptr, 0, 0, 0, 0, false, "", 1); + false, 0, OverflowMode::THROW, 0, 0, 0, false, "", 1); Aggregator aggregator(params); diff --git a/dbms/src/Processors/tests/processors_test_aggregation.cpp b/dbms/src/Processors/tests/processors_test_aggregation.cpp index a645804eba8..2306de4edc0 100644 --- a/dbms/src/Processors/tests/processors_test_aggregation.cpp +++ b/dbms/src/Processors/tests/processors_test_aggregation.cpp @@ -224,8 +224,6 @@ try overflow_row, max_rows_to_group_by, OverflowMode::THROW, - nullptr, /// No compiler - 0, /// min_count_to_compile group_by_two_level_threshold, group_by_two_level_threshold_bytes, max_bytes_before_external_group_by, @@ -298,8 +296,6 @@ try overflow_row, max_rows_to_group_by, OverflowMode::THROW, - nullptr, /// No compiler - 0, /// min_count_to_compile group_by_two_level_threshold, group_by_two_level_threshold_bytes, max_bytes_before_external_group_by, From d798cbde2fd48be636517c6e444157557d1f80c3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Aug 2019 01:28:06 +0300 Subject: [PATCH 235/357] Fixed double whitespaces --- dbms/src/Storages/StorageDistributed.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 7c20622d89e..4a54b4c5317 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -279,7 +279,7 @@ StoragePtr StorageDistributed::createWithOwnCluster( const Context & context_) { auto res = ext::shared_ptr_helper::create( - String{}, table_name_, columns_, ConstraintsDescription{}, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), false); + String{}, table_name_, columns_, ConstraintsDescription{}, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), false); res->owned_cluster = owned_cluster_; From f959c29be60b9a70eef34ecff4135e871c45201e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sat, 24 Aug 2019 21:38:50 -0400 Subject: [PATCH 236/357] Revert "Temporarily disable all LIVE VIEW tests because this feature has subtle bugs that manifestate under TSan" This reverts commit 96869d405fc132a7443cfbc13d1e583079503db8. --- .../00960_live_view_watch_events_live.py | 42 ++++ ...0960_live_view_watch_events_live.reference | 0 .../00961_temporary_live_view_watch.reference | 3 + .../00961_temporary_live_view_watch.sql | 18 ++ .../00962_temporary_live_view_watch_live.py | 42 ++++ ...2_temporary_live_view_watch_live.reference | 0 ...y_live_view_watch_live_timeout.py.disabled | 49 +++++ ...ary_live_view_watch_live_timeout.reference | 0 .../00964_live_view_watch_events_heartbeat.py | 44 ++++ ...live_view_watch_events_heartbeat.reference | 0 .../00965_live_view_watch_heartbeat.py | 45 ++++ .../00965_live_view_watch_heartbeat.reference | 0 .../00966_live_view_watch_events_http.py | 37 ++++ ...0966_live_view_watch_events_http.reference | 0 .../0_stateless/00967_live_view_watch_http.py | 37 ++++ .../00967_live_view_watch_http.reference | 0 ...t_format_jsoneachrowwithprogress.reference | 4 + ..._select_format_jsoneachrowwithprogress.sql | 12 + ...h_format_jsoneachrowwithprogress.reference | 6 + ...w_watch_format_jsoneachrowwithprogress.sql | 18 ++ ...0_live_view_watch_events_http_heartbeat.py | 43 ++++ ...view_watch_events_http_heartbeat.reference | 0 .../00971_live_view_watch_http_heartbeat.py | 43 ++++ ...1_live_view_watch_http_heartbeat.reference | 0 .../00972_live_view_select_1.reference | 1 + .../0_stateless/00972_live_view_select_1.sql | 7 + .../00973_live_view_select.reference | 4 + .../0_stateless/00973_live_view_select.sql | 18 ++ ...ive_view_select_with_aggregation.reference | 2 + ...0974_live_view_select_with_aggregation.sql | 16 ++ .../00975_live_view_create.reference | 0 .../0_stateless/00975_live_view_create.sql | 7 + .../00976_live_view_select_version.reference | 3 + .../00976_live_view_select_version.sql | 12 + .../00977_live_view_watch_events.reference | 3 + .../00977_live_view_watch_events.sql | 18 ++ .../00978_live_view_watch.reference | 3 + .../0_stateless/00978_live_view_watch.sql | 18 ++ .../0_stateless/00979_live_view_watch_live.py | 48 ++++ .../00979_live_view_watch_live.reference | 0 ...00980_create_temporary_live_view.reference | 3 + .../00980_create_temporary_live_view.sql | 15 ++ .../00991_live_view_watch_event_live.python | 81 +++++++ ...00991_live_view_watch_event_live.reference | 7 + ...991_live_view_watch_event_live.sh.disabled | 6 + .../00991_live_view_watch_http.python | 63 ++++++ .../00991_live_view_watch_http.reference | 4 + .../00991_live_view_watch_http.sh.disabled | 6 + ...ry_live_view_watch_events_heartbeat.python | 83 +++++++ ...live_view_watch_events_heartbeat.reference | 0 ...ve_view_watch_events_heartbeat.sh.disabled | 6 + ...0991_temporary_live_view_watch_live.python | 81 +++++++ ...1_temporary_live_view_watch_live.reference | 7 + ...temporary_live_view_watch_live.sh.disabled | 6 + .../queries/0_stateless/helpers/client.py | 36 +++ .../queries/0_stateless/helpers/httpclient.py | 14 ++ .../queries/0_stateless/helpers/httpexpect.py | 73 +++++++ .../queries/0_stateless/helpers/uexpect.py | 206 ++++++++++++++++++ 58 files changed, 1300 insertions(+) create mode 100755 dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py create mode 100644 dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference create mode 100644 dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference create mode 100644 dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql create mode 100755 dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py create mode 100644 dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference create mode 100755 dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled create mode 100644 dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference create mode 100755 dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py create mode 100644 dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference create mode 100755 dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py create mode 100644 dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference create mode 100755 dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py create mode 100644 dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference create mode 100755 dbms/tests/queries/0_stateless/00967_live_view_watch_http.py create mode 100644 dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference create mode 100644 dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference create mode 100644 dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql create mode 100644 dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference create mode 100644 dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql create mode 100755 dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py create mode 100644 dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference create mode 100755 dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py create mode 100644 dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference create mode 100644 dbms/tests/queries/0_stateless/00972_live_view_select_1.reference create mode 100644 dbms/tests/queries/0_stateless/00972_live_view_select_1.sql create mode 100644 dbms/tests/queries/0_stateless/00973_live_view_select.reference create mode 100644 dbms/tests/queries/0_stateless/00973_live_view_select.sql create mode 100644 dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference create mode 100644 dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql create mode 100644 dbms/tests/queries/0_stateless/00975_live_view_create.reference create mode 100644 dbms/tests/queries/0_stateless/00975_live_view_create.sql create mode 100644 dbms/tests/queries/0_stateless/00976_live_view_select_version.reference create mode 100644 dbms/tests/queries/0_stateless/00976_live_view_select_version.sql create mode 100644 dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference create mode 100644 dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql create mode 100644 dbms/tests/queries/0_stateless/00978_live_view_watch.reference create mode 100644 dbms/tests/queries/0_stateless/00978_live_view_watch.sql create mode 100755 dbms/tests/queries/0_stateless/00979_live_view_watch_live.py create mode 100644 dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference create mode 100644 dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference create mode 100644 dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql create mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python create mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference create mode 100755 dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.sh.disabled create mode 100755 dbms/tests/queries/0_stateless/00991_live_view_watch_http.python create mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference create mode 100755 dbms/tests/queries/0_stateless/00991_live_view_watch_http.sh.disabled create mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python create mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference create mode 100755 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.sh.disabled create mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python create mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference create mode 100755 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.sh.disabled create mode 100644 dbms/tests/queries/0_stateless/helpers/client.py create mode 100644 dbms/tests/queries/0_stateless/helpers/httpclient.py create mode 100644 dbms/tests/queries/0_stateless/helpers/httpexpect.py create mode 100644 dbms/tests/queries/0_stateless/helpers/uexpect.py diff --git a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py new file mode 100755 index 00000000000..b7fc3f4e3a6 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +import os +import sys +import signal + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send(' DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + client1.send('WATCH test.lv EVENTS') + client1.expect('1.*' + end_of_block) + client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect('2.*' + end_of_block) + client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') + client1.expect('3.*' + end_of_block) + # send Ctrl-C + client1.send('\x03', eol='') + match = client1.expect('(%s)|([#\$] )' % prompt) + if match.groups()[1]: + client1.send(client1.command) + client1.expect(prompt) + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference new file mode 100644 index 00000000000..6fbbedf1b21 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference @@ -0,0 +1,3 @@ +0 1 +6 2 +21 3 diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql new file mode 100644 index 00000000000..c3e2ab8d102 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; + +WATCH test.lv LIMIT 0; + +INSERT INTO test.mt VALUES (1),(2),(3); + +WATCH test.lv LIMIT 0; + +INSERT INTO test.mt VALUES (4),(5),(6); + +WATCH test.lv LIMIT 0; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py new file mode 100755 index 00000000000..f27b1213c70 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +import os +import sys +import signal + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + client1.send('WATCH test.lv') + client1.expect(r'0.*1' + end_of_block) + client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(r'6.*2' + end_of_block) + client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') + client1.expect(r'21.*3' + end_of_block) + # send Ctrl-C + client1.send('\x03', eol='') + match = client1.expect('(%s)|([#\$] )' % prompt) + if match.groups()[1]: + client1.send(client1.command) + client1.expect(prompt) + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled new file mode 100755 index 00000000000..df627c84e49 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import os +import sys +import signal + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('SET temporary_live_view_timeout=1') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + client1.send('WATCH test.lv') + client1.expect(r'0.*1' + end_of_block) + client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client2.expect(prompt) + client1.expect(r'6.*2' + end_of_block) + client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') + client2.expect(prompt) + client1.expect(r'21.*3' + end_of_block) + # send Ctrl-C + client1.send('\x03', eol='') + match = client1.expect('(%s)|([#\$] )' % prompt) + if match.groups()[1]: + client1.send(client1.command) + client1.expect(prompt) + client1.send('SELECT sleep(1)') + client1.expect(prompt) + client1.send('DROP TABLE test.lv') + client1.expect('Table test.lv doesn\'t exist') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py new file mode 100755 index 00000000000..5664c0e6c6d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +import os +import sys +import signal + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send(' DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('SET live_view_heartbeat_interval=1') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + client1.send('WATCH test.lv EVENTS') + client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect('2.*' + end_of_block) + client1.expect('Progress: 2.00 rows.*\)') + # wait for heartbeat + client1.expect('Progress: 2.00 rows.*\)') + # send Ctrl-C + client1.send('\x03', eol='') + match = client1.expect('(%s)|([#\$] )' % prompt) + if match.groups()[1]: + client1.send(client1.command) + client1.expect(prompt) + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py new file mode 100755 index 00000000000..03e22175dff --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +import os +import sys +import signal + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send(' DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('SET live_view_heartbeat_interval=1') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + client1.send('WATCH test.lv') + client1.expect(r'0.*1' + end_of_block) + client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(r'6.*2' + end_of_block) + client1.expect('Progress: 2.00 rows.*\)') + # wait for heartbeat + client1.expect('Progress: 2.00 rows.*\)') + # send Ctrl-C + client1.send('\x03', eol='') + match = client1.expect('(%s)|([#\$] )' % prompt) + if match.groups()[1]: + client1.send(client1.command) + client1.expect(prompt) + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py new file mode 100755 index 00000000000..bb9d6152200 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +import os +import sys + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block +from httpclient import client as http_client + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1: + client1.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send(' DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + + + with http_client({'method':'GET', 'url': '/?query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: + client2.expect('.*1\n') + client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(prompt) + client2.expect('.*2\n') + + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py new file mode 100755 index 00000000000..d3439431eb3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +import os +import sys + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block +from httpclient import client as http_client + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1: + client1.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send(' DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + + + with http_client({'method':'GET', 'url':'/?query=WATCH%20test.lv'}, name='client2>', log=log) as client2: + client2.expect('.*0\t1\n') + client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(prompt) + client2.expect('.*6\t2\n') + + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference new file mode 100644 index 00000000000..5ae423d90d1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference @@ -0,0 +1,4 @@ +{"row":{"a":1}} +{"row":{"a":2}} +{"row":{"a":3}} +{"progress":{"read_rows":"3","read_bytes":"36","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql new file mode 100644 index 00000000000..8c6f4197d54 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; + +INSERT INTO test.mt VALUES (1),(2),(3); + +SELECT * FROM test.lv FORMAT JSONEachRowWithProgress; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference new file mode 100644 index 00000000000..287a1ced92d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference @@ -0,0 +1,6 @@ +{"row":{"sum(a)":"0","_version":"1"}} +{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} +{"row":{"sum(a)":"6","_version":"2"}} +{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} +{"row":{"sum(a)":"21","_version":"3"}} +{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql new file mode 100644 index 00000000000..725a4ad00ed --- /dev/null +++ b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; + +WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; + +INSERT INTO test.mt VALUES (1),(2),(3); + +WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; + +INSERT INTO test.mt VALUES (4),(5),(6); + +WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py new file mode 100755 index 00000000000..63628c4a76f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +import os +import sys + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block +from httpclient import client as http_client + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1: + client1.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send(' DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + + + with http_client({'method':'GET', 'url': '/?live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: + client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}\n', escape=True) + client2.expect('{"row":{"version":"1"}', escape=True) + client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) + # heartbeat is provided by progress message + client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) + + client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(prompt) + + client2.expect('{"row":{"version":"2"}}\n', escape=True) + + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py new file mode 100755 index 00000000000..7bdb47b7caa --- /dev/null +++ b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +import os +import sys + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block +from httpclient import client as http_client + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1: + client1.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send(' DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + + with http_client({'method':'GET', 'url':'/?live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: + client2.expect('"progress".*',) + client2.expect('{"row":{"sum(a)":"0","_version":"1"}}\n', escape=True) + client2.expect('"progress".*\n') + # heartbeat is provided by progress message + client2.expect('"progress".*\n') + + client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(prompt) + + client2.expect('"progress".*"read_rows":"2".*\n') + client2.expect('{"row":{"sum(a)":"6","_version":"2"}}\n', escape=True) + + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference b/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference @@ -0,0 +1 @@ +1 diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql b/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql new file mode 100644 index 00000000000..661080b577b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS test.lv; + +CREATE LIVE VIEW test.lv AS SELECT 1; + +SELECT * FROM test.lv; + +DROP TABLE test.lv; diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.reference b/dbms/tests/queries/0_stateless/00973_live_view_select.reference new file mode 100644 index 00000000000..75236c0daf7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00973_live_view_select.reference @@ -0,0 +1,4 @@ +6 1 +6 1 +12 2 +12 2 diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.sql b/dbms/tests/queries/0_stateless/00973_live_view_select.sql new file mode 100644 index 00000000000..ff4a45ffcc1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00973_live_view_select.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; + +INSERT INTO test.mt VALUES (1),(2),(3); + +SELECT *,_version FROM test.lv; +SELECT *,_version FROM test.lv; + +INSERT INTO test.mt VALUES (1),(2),(3); + +SELECT *,_version FROM test.lv; +SELECT *,_version FROM test.lv; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference new file mode 100644 index 00000000000..6d50f0e9c3a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference @@ -0,0 +1,2 @@ +6 +21 diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql new file mode 100644 index 00000000000..3c11f855c9d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; + +INSERT INTO test.mt VALUES (1),(2),(3); + +SELECT sum(a) FROM test.lv; + +INSERT INTO test.mt VALUES (4),(5),(6); + +SELECT sum(a) FROM test.lv; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.reference b/dbms/tests/queries/0_stateless/00975_live_view_create.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.sql b/dbms/tests/queries/0_stateless/00975_live_view_create.sql new file mode 100644 index 00000000000..1c929b15b00 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00975_live_view_create.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference b/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference new file mode 100644 index 00000000000..453bd800469 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference @@ -0,0 +1,3 @@ +1 1 +2 1 +3 1 diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql b/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql new file mode 100644 index 00000000000..5f3ab1f7546 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; + +INSERT INTO test.mt VALUES (1),(2),(3); + +SELECT *,_version FROM test.lv; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference new file mode 100644 index 00000000000..01e79c32a8c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference @@ -0,0 +1,3 @@ +1 +2 +3 diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql new file mode 100644 index 00000000000..a3b84e8d4c1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; + +WATCH test.lv EVENTS LIMIT 0; + +INSERT INTO test.mt VALUES (1),(2),(3); + +WATCH test.lv EVENTS LIMIT 0; + +INSERT INTO test.mt VALUES (4),(5),(6); + +WATCH test.lv EVENTS LIMIT 0; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.reference b/dbms/tests/queries/0_stateless/00978_live_view_watch.reference new file mode 100644 index 00000000000..6fbbedf1b21 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00978_live_view_watch.reference @@ -0,0 +1,3 @@ +0 1 +6 2 +21 3 diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql b/dbms/tests/queries/0_stateless/00978_live_view_watch.sql new file mode 100644 index 00000000000..abe4a6c32ae --- /dev/null +++ b/dbms/tests/queries/0_stateless/00978_live_view_watch.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; + +WATCH test.lv LIMIT 0; + +INSERT INTO test.mt VALUES (1),(2),(3); + +WATCH test.lv LIMIT 0; + +INSERT INTO test.mt VALUES (4),(5),(6); + +WATCH test.lv LIMIT 0; + +DROP TABLE test.lv; +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py new file mode 100755 index 00000000000..948e4c93662 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +import os +import sys +import signal + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from client import client, prompt, end_of_block + +log = None +# uncomment the line below for debugging +#log=sys.stdout + +with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: + client1.expect(prompt) + client2.expect(prompt) + + client1.send('DROP TABLE IF EXISTS test.lv') + client1.expect(prompt) + client1.send(' DROP TABLE IF EXISTS test.mt') + client1.expect(prompt) + client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.expect(prompt) + client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.expect(prompt) + client1.send('WATCH test.lv') + client1.expect(r'0.*1' + end_of_block) + client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(r'6.*2' + end_of_block) + client2.expect(prompt) + client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') + client1.expect(r'21.*3' + end_of_block) + client2.expect(prompt) + for i in range(1,129): + client2.send('INSERT INTO test.mt VALUES (1)') + client1.expect(r'%d.*%d' % (21+i, 3+i) + end_of_block) + client2.expect(prompt) + # send Ctrl-C + client1.send('\x03', eol='') + match = client1.expect('(%s)|([#\$] )' % prompt) + if match.groups()[1]: + client1.send(client1.command) + client1.expect(prompt) + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') + client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference new file mode 100644 index 00000000000..7f9fcbb2e9c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference @@ -0,0 +1,3 @@ +temporary_live_view_timeout 5 +live_view_heartbeat_interval 15 +0 diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql new file mode 100644 index 00000000000..8cd6ee06ace --- /dev/null +++ b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS test.lv; +DROP TABLE IF EXISTS test.mt; + +SELECT name, value from system.settings WHERE name = 'temporary_live_view_timeout'; +SELECT name, value from system.settings WHERE name = 'live_view_heartbeat_interval'; + +SET temporary_live_view_timeout=1; +CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); +CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; + +SHOW TABLES LIKE 'lv'; +SELECT sleep(2); +SHOW TABLES LIKE 'lv'; + +DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python new file mode 100644 index 00000000000..782671cdfaf --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +import subprocess +import threading +import Queue as queue +import os +import sys +import signal + + +CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') +CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') +CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') + + +def send_query(query): + cmd = list(CLICKHOUSE_CLIENT.split()) + cmd += ['--query', query] + # print(cmd) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout + + +def send_query_in_process_group(query): + cmd = list(CLICKHOUSE_CLIENT.split()) + cmd += ['--query', query] + # print(cmd) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) + + +def read_lines_and_push_to_queue(pipe, queue): + try: + for line in iter(pipe.readline, ''): + line = line.strip() + print(line) + sys.stdout.flush() + queue.put(line) + except KeyboardInterrupt: + pass + + queue.put(None) + + +def test(): + send_query('DROP TABLE IF EXISTS test.lv').read() + send_query('DROP TABLE IF EXISTS test.mt').read() + send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() + send_query('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() + + q = queue.Queue() + p = send_query_in_process_group('WATCH test.lv') + thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) + thread.start() + + line = q.get() + print(line) + assert (line == '0\t1') + + send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() + line = q.get() + print(line) + assert (line == '6\t2') + + send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() + line = q.get() + print(line) + assert (line == '21\t3') + + # Send Ctrl+C to client. + os.killpg(os.getpgid(p.pid), signal.SIGINT) + # This insert shouldn't affect lv. + send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() + line = q.get() + print(line) + assert (line is None) + + send_query('DROP TABLE if exists test.lv').read() + send_query('DROP TABLE if exists test.lv').read() + + thread.join() + +test() diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference new file mode 100644 index 00000000000..1e94cdade41 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference @@ -0,0 +1,7 @@ +0 1 +0 1 +6 2 +6 2 +21 3 +21 3 +None diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.sh.disabled b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.sh.disabled new file mode 100755 index 00000000000..10e4e98b2e3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.sh.disabled @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +python $CURDIR/00991_live_view_watch_event_live.python diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python new file mode 100755 index 00000000000..938547ca0cb --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import subprocess +import threading +import Queue as queue +import os +import sys + + +CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') +CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') +CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') + + +def send_query(query): + cmd = list(CLICKHOUSE_CLIENT.split()) + cmd += ['--query', query] + # print(cmd) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout + + +def send_http_query(query): + cmd = list(CLICKHOUSE_CURL.split()) # list(['curl', '-sSN', '--max-time', '10']) + cmd += ['-sSN', CLICKHOUSE_URL, '-d', query] + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout + + +def read_lines_and_push_to_queue(pipe, queue): + for line in iter(pipe.readline, ''): + line = line.strip() + print(line) + sys.stdout.flush() + queue.put(line) + + queue.put(None) + + +def test(): + send_query('DROP TABLE IF EXISTS test.lv').read() + send_query('DROP TABLE IF EXISTS test.mt').read() + send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() + send_query('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() + + q = queue.Queue() + pipe = send_http_query('WATCH test.lv') + thread = threading.Thread(target=read_lines_and_push_to_queue, args=(pipe, q)) + thread.start() + + line = q.get() + print(line) + assert (line == '0\t1') + + send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() + line = q.get() + print(line) + assert (line == '6\t2') + + send_query('DROP TABLE if exists test.lv').read() + send_query('DROP TABLE if exists test.lv').read() + + thread.join() + +test() diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference new file mode 100644 index 00000000000..489457d751b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference @@ -0,0 +1,4 @@ +0 1 +0 1 +6 2 +6 2 diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.sh.disabled b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.sh.disabled new file mode 100755 index 00000000000..88cce77f595 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.sh.disabled @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +python $CURDIR/00991_live_view_watch_http.python diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python new file mode 100644 index 00000000000..70063adc6e3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +import subprocess +import threading +import Queue as queue +import os +import sys +import signal + + +CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') +CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') +CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') + + +def send_query(query): + cmd = list(CLICKHOUSE_CLIENT.split()) + cmd += ['--query', query] + # print(cmd) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout + + +def send_query_in_process_group(query): + cmd = list(CLICKHOUSE_CLIENT.split()) + cmd += ['--query', query, '--live_view_heartbeat_interval=1', '--progress'] + # print(cmd) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) + + +def read_lines_and_push_to_queue(pipe, queue): + try: + for line in iter(pipe.readline, ''): + line = line.strip() + # print(line) + sys.stdout.flush() + queue.put(line) + except KeyboardInterrupt: + pass + + queue.put(None) + + +def test(): + send_query('DROP TABLE IF EXISTS test.lv').read() + send_query('DROP TABLE IF EXISTS test.mt').read() + send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() + send_query('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() + + q = queue.Queue() + p = send_query_in_process_group('WATCH test.lv') + thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) + thread.start() + + line = q.get() + # print(line) + assert (line.endswith('0\t1')) + assert ('Progress: 0.00 rows' in line) + + send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() + line = q.get() + assert (line.endswith('6\t2')) + assert ('Progress: 1.00 rows' in line) + + # send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() + # line = q.get() + # print(line) + # assert (line.endswith('6\t2')) + # assert ('Progress: 1.00 rows' in line) + + # Send Ctrl+C to client. + os.killpg(os.getpgid(p.pid), signal.SIGINT) + # This insert shouldn't affect lv. + send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() + line = q.get() + # print(line) + # assert (line is None) + + send_query('DROP TABLE if exists test.lv').read() + send_query('DROP TABLE if exists test.lv').read() + + thread.join() + +test() diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.sh.disabled b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.sh.disabled new file mode 100755 index 00000000000..f7aa13d52b3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.sh.disabled @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +python $CURDIR/00991_temporary_live_view_watch_events_heartbeat.python diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python new file mode 100644 index 00000000000..d290018a02c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +import subprocess +import threading +import Queue as queue +import os +import sys +import signal + + +CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') +CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') +CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') + + +def send_query(query): + cmd = list(CLICKHOUSE_CLIENT.split()) + cmd += ['--query', query] + # print(cmd) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout + + +def send_query_in_process_group(query): + cmd = list(CLICKHOUSE_CLIENT.split()) + cmd += ['--query', query] + # print(cmd) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) + + +def read_lines_and_push_to_queue(pipe, queue): + try: + for line in iter(pipe.readline, ''): + line = line.strip() + print(line) + sys.stdout.flush() + queue.put(line) + except KeyboardInterrupt: + pass + + queue.put(None) + + +def test(): + send_query('DROP TABLE IF EXISTS test.lv').read() + send_query('DROP TABLE IF EXISTS test.mt').read() + send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() + send_query('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() + + q = queue.Queue() + p = send_query_in_process_group('WATCH test.lv') + thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) + thread.start() + + line = q.get() + print(line) + assert (line == '0\t1') + + send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() + line = q.get() + print(line) + assert (line == '6\t2') + + send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() + line = q.get() + print(line) + assert (line == '21\t3') + + # Send Ctrl+C to client. + os.killpg(os.getpgid(p.pid), signal.SIGINT) + # This insert shouldn't affect lv. + send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() + line = q.get() + print(line) + assert (line is None) + + send_query('DROP TABLE if exists test.lv').read() + send_query('DROP TABLE if exists test.lv').read() + + thread.join() + +test() diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference new file mode 100644 index 00000000000..1e94cdade41 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference @@ -0,0 +1,7 @@ +0 1 +0 1 +6 2 +6 2 +21 3 +21 3 +None diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.sh.disabled b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.sh.disabled new file mode 100755 index 00000000000..4d01d1c3a8e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.sh.disabled @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +python $CURDIR/00991_temporary_live_view_watch_live.python diff --git a/dbms/tests/queries/0_stateless/helpers/client.py b/dbms/tests/queries/0_stateless/helpers/client.py new file mode 100644 index 00000000000..f3938d3bf63 --- /dev/null +++ b/dbms/tests/queries/0_stateless/helpers/client.py @@ -0,0 +1,36 @@ +import os +import sys +import time + +CURDIR = os.path.dirname(os.path.realpath(__file__)) + +sys.path.insert(0, os.path.join(CURDIR)) + +import uexpect + +prompt = ':\) ' +end_of_block = r'.*\r\n.*\r\n' + +class client(object): + def __init__(self, command=None, name='', log=None): + self.client = uexpect.spawn(['/bin/bash','--noediting']) + if command is None: + command = os.environ.get('CLICKHOUSE_BINARY', 'clickhouse') + '-client' + self.client.command = command + self.client.eol('\r') + self.client.logger(log, prefix=name) + self.client.timeout(20) + self.client.expect('[#\$] ', timeout=2) + self.client.send(command) + + def __enter__(self): + return self.client.__enter__() + + def __exit__(self, type, value, traceback): + self.client.reader['kill_event'].set() + # send Ctrl-C + self.client.send('\x03', eol='') + time.sleep(0.3) + self.client.send('quit', eol='\r') + self.client.send('\x03', eol='') + return self.client.__exit__(type, value, traceback) diff --git a/dbms/tests/queries/0_stateless/helpers/httpclient.py b/dbms/tests/queries/0_stateless/helpers/httpclient.py new file mode 100644 index 00000000000..a42fad2cbc3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/helpers/httpclient.py @@ -0,0 +1,14 @@ +import os +import sys + +CURDIR = os.path.dirname(os.path.realpath(__file__)) + +sys.path.insert(0, os.path.join(CURDIR)) + +import httpexpect + +def client(request, name='', log=None): + client = httpexpect.spawn({'host':'localhost','port':8123}, request) + client.logger(log, prefix=name) + client.timeout(20) + return client diff --git a/dbms/tests/queries/0_stateless/helpers/httpexpect.py b/dbms/tests/queries/0_stateless/helpers/httpexpect.py new file mode 100644 index 00000000000..e440dafce4e --- /dev/null +++ b/dbms/tests/queries/0_stateless/helpers/httpexpect.py @@ -0,0 +1,73 @@ +# Copyright (c) 2019 Vitaliy Zakaznikov +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +import httplib + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, CURDIR) + +import uexpect + +from threading import Thread, Event +from Queue import Queue, Empty + +class IO(uexpect.IO): + def __init__(self, connection, response, queue, reader): + self.connection = connection + self.response = response + super(IO, self).__init__(None, None, queue, reader) + + def write(self, data): + raise NotImplementedError + + def close(self, force=True): + self.reader['kill_event'].set() + self.connection.close() + if self._logger: + self._logger.write('\n') + self._logger.flush() + + +def reader(response, queue, kill_event): + while True: + try: + if kill_event.is_set(): + break + data = response.read(1) + queue.put(data) + except Exception, e: + if kill_event.is_set(): + break + raise + +def spawn(connection, request): + connection = httplib.HTTPConnection(**connection) + connection.request(**request) + response = connection.getresponse() + + queue = Queue() + reader_kill_event = Event() + thread = Thread(target=reader, args=(response, queue, reader_kill_event)) + thread.daemon = True + thread.start() + + return IO(connection, response, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) + +if __name__ == '__main__': + with http({'host':'localhost','port':8123},{'method':'GET', 'url':'?query=SELECT%201'}) as client: + client.logger(sys.stdout) + client.timeout(2) + print client.response.status, client.response.reason + client.expect('1\n') diff --git a/dbms/tests/queries/0_stateless/helpers/uexpect.py b/dbms/tests/queries/0_stateless/helpers/uexpect.py new file mode 100644 index 00000000000..f71b32a53e1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/helpers/uexpect.py @@ -0,0 +1,206 @@ +# Copyright (c) 2019 Vitaliy Zakaznikov +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import pty +import time +import sys +import re + +from threading import Thread, Event +from subprocess import Popen +from Queue import Queue, Empty + +class TimeoutError(Exception): + def __init__(self, timeout): + self.timeout = timeout + + def __str__(self): + return 'Timeout %.3fs' % float(self.timeout) + +class ExpectTimeoutError(Exception): + def __init__(self, pattern, timeout, buffer): + self.pattern = pattern + self.timeout = timeout + self.buffer = buffer + + def __str__(self): + s = 'Timeout %.3fs ' % float(self.timeout) + if self.pattern: + s += 'for %s ' % repr(self.pattern.pattern) + if self.buffer: + s += 'buffer %s ' % repr(self.buffer[:]) + s += 'or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]]) + return s + +class IO(object): + class EOF(object): + pass + + class Timeout(object): + pass + + EOF = EOF + TIMEOUT = Timeout + + class Logger(object): + def __init__(self, logger, prefix=''): + self._logger = logger + self._prefix = prefix + + def write(self, data): + self._logger.write(('\n' + data).replace('\n','\n' + self._prefix)) + + def flush(self): + self._logger.flush() + + def __init__(self, process, master, queue, reader): + self.process = process + self.master = master + self.queue = queue + self.buffer = None + self.before = None + self.after = None + self.match = None + self.pattern = None + self.reader = reader + self._timeout = None + self._logger = None + self._eol = '' + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def logger(self, logger=None, prefix=''): + if logger: + self._logger = self.Logger(logger, prefix=prefix) + return self._logger + + def timeout(self, timeout=None): + if timeout: + self._timeout = timeout + return self._timeout + + def eol(self, eol=None): + if eol: + self._eol = eol + return self._eol + + def close(self, force=True): + self.reader['kill_event'].set() + os.system('pkill -TERM -P %d' % self.process.pid) + if force: + self.process.kill() + else: + self.process.terminate() + os.close(self.master) + if self._logger: + self._logger.write('\n') + self._logger.flush() + + def send(self, data, eol=None): + if eol is None: + eol = self._eol + return self.write(data + eol) + + def write(self, data): + return os.write(self.master, data) + + def expect(self, pattern, timeout=None, escape=False): + self.match = None + self.before = None + self.after = None + if escape: + pattern = re.escape(pattern) + pattern = re.compile(pattern) + if timeout is None: + timeout = self._timeout + timeleft = timeout + while True: + start_time = time.time() + if self.buffer is not None: + self.match = pattern.search(self.buffer, 0) + if self.match is not None: + self.after = self.buffer[self.match.start():self.match.end()] + self.before = self.buffer[:self.match.start()] + self.buffer = self.buffer[self.match.end():] + break + if timeleft < 0: + break + try: + data = self.read(timeout=timeleft, raise_exception=True) + except TimeoutError: + if self._logger: + self._logger.write((self.buffer or '') + '\n') + self._logger.flush() + exception = ExpectTimeoutError(pattern, timeout, self.buffer) + self.buffer = None + raise exception + timeleft -= (time.time() - start_time) + if data: + self.buffer = (self.buffer + data) if self.buffer else data + if self._logger: + self._logger.write((self.before or '') + (self.after or '')) + self._logger.flush() + if self.match is None: + exception = ExpectTimeoutError(pattern, timeout, self.buffer) + self.buffer = None + raise exception + return self.match + + def read(self, timeout=0, raise_exception=False): + data = '' + timeleft = timeout + try: + while timeleft >= 0 : + start_time = time.time() + data += self.queue.get(timeout=timeleft) + if data: + break + timeleft -= (time.time() - start_time) + except Empty: + if data: + return data + if raise_exception: + raise TimeoutError(timeout) + pass + if not data and raise_exception: + raise TimeoutError(timeout) + + return data + +def spawn(command): + master, slave = pty.openpty() + process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1) + os.close(slave) + + queue = Queue() + reader_kill_event = Event() + thread = Thread(target=reader, args=(process, master, queue, reader_kill_event)) + thread.daemon = True + thread.start() + + return IO(process, master, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) + +def reader(process, out, queue, kill_event): + while True: + try: + data = os.read(out, 65536) + queue.put(data) + except: + if kill_event.is_set(): + break + raise From 62988800e61042c131bec717b36a369f8a419ca0 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sat, 24 Aug 2019 21:40:24 -0400 Subject: [PATCH 237/357] Rewriting implementation of LIVE VIEW no users thread. --- .../LiveView/LiveViewBlockInputStream.h | 21 +++--- .../LiveView/LiveViewEventsBlockInputStream.h | 20 +++--- .../src/Storages/LiveView/StorageLiveView.cpp | 66 +++++++++---------- dbms/src/Storages/LiveView/StorageLiveView.h | 16 ++--- 4 files changed, 63 insertions(+), 60 deletions(-) diff --git a/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h b/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h index 345fceaf095..60839f3e66f 100644 --- a/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h +++ b/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h @@ -18,10 +18,13 @@ using NonBlockingResult = std::pair; public: ~LiveViewBlockInputStream() override { - /// Start storage no users thread - /// if we are the last active user - if (!storage->is_dropped && blocks_ptr.use_count() < 3) - storage->startNoUsersThread(temporary_live_view_timeout_sec); + /// Wake up no users thread + { + std::cerr << "DEBUG: live view block input stream ... send wake up thread\n"; + std::lock_guard lock(storage->no_users_thread_mutex); + storage->no_users_thread_wakeup = true; + storage->no_users_thread_condition.notify_one(); + } } LiveViewBlockInputStream(std::shared_ptr storage_, @@ -29,9 +32,12 @@ public: std::shared_ptr blocks_metadata_ptr_, std::shared_ptr active_ptr_, const bool has_limit_, const UInt64 limit_, - const UInt64 heartbeat_interval_sec_, - const UInt64 temporary_live_view_timeout_sec_) - : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), active_ptr(std::move(active_ptr_)), has_limit(has_limit_), limit(limit_), heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000), temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_) + const UInt64 heartbeat_interval_sec_) + : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), + blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), + active_ptr(std::move(active_ptr_)), + has_limit(has_limit_), limit(limit_), + heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000) { /// grab active pointer active = active_ptr.lock(); @@ -200,7 +206,6 @@ private: Int64 num_updates = -1; bool end_of_blocks = false; UInt64 heartbeat_interval_usec; - UInt64 temporary_live_view_timeout_sec; UInt64 last_event_timestamp_usec = 0; Poco::Timestamp timestamp; }; diff --git a/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h b/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h index 120d0098536..e0e6ff78d21 100644 --- a/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h +++ b/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h @@ -37,10 +37,12 @@ using NonBlockingResult = std::pair; public: ~LiveViewEventsBlockInputStream() override { - /// Start storage no users thread - /// if we are the last active user - if (!storage->is_dropped && blocks_ptr.use_count() < 3) - storage->startNoUsersThread(temporary_live_view_timeout_sec); + /// Wake up no users thread + { + std::lock_guard lock(storage->no_users_thread_mutex); + storage->no_users_thread_wakeup = true; + storage->no_users_thread_condition.notify_one(); + } } /// length default -2 because we want LIMIT to specify number of updates so that LIMIT 1 waits for 1 update /// and LIMIT 0 just returns data without waiting for any updates @@ -49,9 +51,12 @@ public: std::shared_ptr blocks_metadata_ptr_, std::shared_ptr active_ptr_, const bool has_limit_, const UInt64 limit_, - const UInt64 heartbeat_interval_sec_, - const UInt64 temporary_live_view_timeout_sec_) - : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), active_ptr(std::move(active_ptr_)), has_limit(has_limit_), limit(limit_), heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000), temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_) + const UInt64 heartbeat_interval_sec_) + : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), + blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), + active_ptr(std::move(active_ptr_)), has_limit(has_limit_), + limit(limit_), + heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000) { /// grab active pointer active = active_ptr.lock(); @@ -236,7 +241,6 @@ private: Int64 num_updates = -1; bool end_of_blocks = false; UInt64 heartbeat_interval_usec; - UInt64 temporary_live_view_timeout_sec; UInt64 last_event_timestamp_usec = 0; Poco::Timestamp timestamp; }; diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 3c0d205fa3f..d5de3b4a914 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -363,45 +363,49 @@ void StorageLiveView::checkTableCanBeDropped() const } } -void StorageLiveView::noUsersThread(const UInt64 & timeout) +void StorageLiveView::noUsersThread(std::shared_ptr storage, const UInt64 & timeout) { - if (shutdown_called) - return; - bool drop_table = false; + if (storage->shutdown_called || storage->is_dropped) + return; + { while (1) { - std::unique_lock lock(no_users_thread_mutex); - if (!no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return no_users_thread_wakeup; })) + std::unique_lock lock(storage->no_users_thread_mutex); + if(!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return storage->no_users_thread_wakeup; })) { - no_users_thread_wakeup = false; - if (shutdown_called) + storage->no_users_thread_wakeup = false; + if (storage->shutdown_called || storage->is_dropped) return; - if (hasUsers()) - return; - if (!global_context.getDependencies(database_name, table_name).empty()) + if (storage->hasUsers()) + continue; + if (!storage->global_context.getDependencies(storage->database_name, storage->table_name).empty()) continue; drop_table = true; } + else { + storage->no_users_thread_wakeup = false; + continue; + } break; } } if (drop_table) { - if (global_context.tryGetTable(database_name, table_name)) + if (storage->global_context.tryGetTable(storage->database_name, storage->table_name)) { try { /// We create and execute `drop` query for this table auto drop_query = std::make_shared(); - drop_query->database = database_name; - drop_query->table = table_name; + drop_query->database = storage->database_name; + drop_query->table = storage->table_name; drop_query->kind = ASTDropQuery::Kind::Drop; ASTPtr ast_drop_query = drop_query; - InterpreterDropQuery drop_interpreter(ast_drop_query, global_context); + InterpreterDropQuery drop_interpreter(ast_drop_query, storage->global_context); drop_interpreter.execute(); } catch (...) @@ -413,13 +417,6 @@ void StorageLiveView::noUsersThread(const UInt64 & timeout) void StorageLiveView::startNoUsersThread(const UInt64 & timeout) { - bool expected = false; - if (!start_no_users_thread_called.compare_exchange_strong(expected, true)) - return; - - if (is_dropped) - return; - if (is_temporary) { if (no_users_thread.joinable()) @@ -435,10 +432,9 @@ void StorageLiveView::startNoUsersThread(const UInt64 & timeout) std::lock_guard lock(no_users_thread_mutex); no_users_thread_wakeup = false; } - if (!is_dropped) - no_users_thread = std::thread(&StorageLiveView::noUsersThread, this, timeout); + no_users_thread = std::thread(&StorageLiveView::noUsersThread, + std::static_pointer_cast(shared_from_this()), timeout); } - start_no_users_thread_called = false; } void StorageLiveView::startup() @@ -454,19 +450,19 @@ void StorageLiveView::shutdown() if (no_users_thread.joinable()) { - std::lock_guard lock(no_users_thread_mutex); - no_users_thread_wakeup = true; - no_users_thread_condition.notify_one(); - /// Must detach the no users thread - /// as we can't join it as it will result - /// in a deadlock - no_users_thread.detach(); /// TODO Not viable at all. + { + std::lock_guard lock(no_users_thread_mutex); + no_users_thread_wakeup = true; + no_users_thread_condition.notify_one(); + } } } StorageLiveView::~StorageLiveView() { shutdown(); + if (no_users_thread.joinable()) + no_users_thread.detach(); } void StorageLiveView::drop() @@ -532,8 +528,7 @@ BlockInputStreams StorageLiveView::watch( if (query.is_watch_events) { - auto reader = std::make_shared(std::static_pointer_cast(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(), - context.getSettingsRef().temporary_live_view_timeout.totalSeconds()); + auto reader = std::make_shared(std::static_pointer_cast(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds()); if (no_users_thread.joinable()) { @@ -557,8 +552,7 @@ BlockInputStreams StorageLiveView::watch( } else { - auto reader = std::make_shared(std::static_pointer_cast(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(), - context.getSettingsRef().temporary_live_view_timeout.totalSeconds()); + auto reader = std::make_shared(std::static_pointer_cast(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds()); if (no_users_thread.joinable()) { diff --git a/dbms/src/Storages/LiveView/StorageLiveView.h b/dbms/src/Storages/LiveView/StorageLiveView.h index 9930d8d6154..a9a8985b4f8 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.h +++ b/dbms/src/Storages/LiveView/StorageLiveView.h @@ -71,11 +71,9 @@ public: { return active_ptr.use_count() > 1; } - /// Background thread for temporary tables - /// which drops this table if there are no users - void startNoUsersThread(const UInt64 & timeout); + /// No users thread mutex, predicate and wake up condition std::mutex no_users_thread_mutex; - bool no_users_thread_wakeup{false}; + bool no_users_thread_wakeup = false; std::condition_variable no_users_thread_condition; /// Get blocks hash /// must be called with mutex locked @@ -149,7 +147,7 @@ private: String database_name; ASTPtr inner_query; Context & global_context; - bool is_temporary {false}; + bool is_temporary = false; mutable Block sample_block; /// Mutex for the blocks and ready condition @@ -166,10 +164,12 @@ private: std::shared_ptr blocks_metadata_ptr; BlocksPtrs mergeable_blocks; - void noUsersThread(const UInt64 & timeout); + /// Background thread for temporary tables + /// which drops this table if there are no users + void startNoUsersThread(const UInt64 & timeout); + static void noUsersThread(std::shared_ptr storage, const UInt64 & timeout); std::thread no_users_thread; - std::atomic shutdown_called{false}; - std::atomic start_no_users_thread_called{false}; + std::atomic shutdown_called = false; UInt64 temporary_live_view_timeout; StorageLiveView( From 2342d64d1b8f2c2b7fa2946d65664432a6a890b9 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 25 Aug 2019 07:36:08 -0400 Subject: [PATCH 238/357] * Updating no users thread to sleep 3 times longer when users are present to reduce the number of times thread wakes up. * Updating startNoUsersThread to return if the thread is already running. --- .../src/Storages/LiveView/StorageLiveView.cpp | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index d5de3b4a914..6ceabf3478c 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -366,27 +366,40 @@ void StorageLiveView::checkTableCanBeDropped() const void StorageLiveView::noUsersThread(std::shared_ptr storage, const UInt64 & timeout) { bool drop_table = false; + UInt64 next_timeout = timeout; - if (storage->shutdown_called || storage->is_dropped) + if (storage->shutdown_called) return; { while (1) { std::unique_lock lock(storage->no_users_thread_mutex); - if(!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return storage->no_users_thread_wakeup; })) + if (!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(next_timeout), [&] { return storage->no_users_thread_wakeup; })) { storage->no_users_thread_wakeup = false; - if (storage->shutdown_called || storage->is_dropped) + if (storage->shutdown_called) return; if (storage->hasUsers()) + { + /// Thread woke up but there are still users so sleep for 3 times longer than + /// the original timeout to reduce the number of times thread wakes up. + /// Wait until we are explicitely woken up when a user goes away to + /// reset wait time to the original timeout. + next_timeout = timeout * 3; continue; + } if (!storage->global_context.getDependencies(storage->database_name, storage->table_name).empty()) continue; drop_table = true; } - else { + else + { + /// Thread was explicitly awaken so reset timeout to the original + next_timeout = timeout; storage->no_users_thread_wakeup = false; + if (storage->shutdown_called) + return; continue; } break; @@ -421,12 +434,12 @@ void StorageLiveView::startNoUsersThread(const UInt64 & timeout) { if (no_users_thread.joinable()) { - { - std::lock_guard lock(no_users_thread_mutex); - no_users_thread_wakeup = true; - no_users_thread_condition.notify_one(); - } - no_users_thread.join(); + /// If the thread is already running then + /// wake it up and just return + std::lock_guard lock(no_users_thread_mutex); + no_users_thread_wakeup = true; + no_users_thread_condition.notify_one(); + return; } { std::lock_guard lock(no_users_thread_mutex); From 7fb13b12f9e15eb01ecac959c4e9b0e9a5f6eb53 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 25 Aug 2019 08:27:47 -0400 Subject: [PATCH 239/357] Reverting to previous no users thread functionality to avoid keeping no users thread always alive for each live view. --- .../LiveView/LiveViewBlockInputStream.h | 18 +++--- .../LiveView/LiveViewEventsBlockInputStream.h | 17 +++--- .../src/Storages/LiveView/StorageLiveView.cpp | 56 +++++++++---------- dbms/src/Storages/LiveView/StorageLiveView.h | 3 +- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h b/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h index 60839f3e66f..f73991ddb77 100644 --- a/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h +++ b/dbms/src/Storages/LiveView/LiveViewBlockInputStream.h @@ -18,13 +18,10 @@ using NonBlockingResult = std::pair; public: ~LiveViewBlockInputStream() override { - /// Wake up no users thread - { - std::cerr << "DEBUG: live view block input stream ... send wake up thread\n"; - std::lock_guard lock(storage->no_users_thread_mutex); - storage->no_users_thread_wakeup = true; - storage->no_users_thread_condition.notify_one(); - } + /// Start storage no users thread + /// if we are the last active user + if (!storage->is_dropped && blocks_ptr.use_count() < 3) + storage->startNoUsersThread(temporary_live_view_timeout_sec); } LiveViewBlockInputStream(std::shared_ptr storage_, @@ -32,12 +29,14 @@ public: std::shared_ptr blocks_metadata_ptr_, std::shared_ptr active_ptr_, const bool has_limit_, const UInt64 limit_, - const UInt64 heartbeat_interval_sec_) + const UInt64 heartbeat_interval_sec_, + const UInt64 temporary_live_view_timeout_sec_) : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), active_ptr(std::move(active_ptr_)), has_limit(has_limit_), limit(limit_), - heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000) + heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000), + temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_) { /// grab active pointer active = active_ptr.lock(); @@ -206,6 +205,7 @@ private: Int64 num_updates = -1; bool end_of_blocks = false; UInt64 heartbeat_interval_usec; + UInt64 temporary_live_view_timeout_sec; UInt64 last_event_timestamp_usec = 0; Poco::Timestamp timestamp; }; diff --git a/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h b/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h index e0e6ff78d21..3308ff2858b 100644 --- a/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h +++ b/dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h @@ -37,12 +37,10 @@ using NonBlockingResult = std::pair; public: ~LiveViewEventsBlockInputStream() override { - /// Wake up no users thread - { - std::lock_guard lock(storage->no_users_thread_mutex); - storage->no_users_thread_wakeup = true; - storage->no_users_thread_condition.notify_one(); - } + /// Start storage no users thread + /// if we are the last active user + if (!storage->is_dropped && blocks_ptr.use_count() < 3) + storage->startNoUsersThread(temporary_live_view_timeout_sec); } /// length default -2 because we want LIMIT to specify number of updates so that LIMIT 1 waits for 1 update /// and LIMIT 0 just returns data without waiting for any updates @@ -51,12 +49,14 @@ public: std::shared_ptr blocks_metadata_ptr_, std::shared_ptr active_ptr_, const bool has_limit_, const UInt64 limit_, - const UInt64 heartbeat_interval_sec_) + const UInt64 heartbeat_interval_sec_, + const UInt64 temporary_live_view_timeout_sec_) : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), active_ptr(std::move(active_ptr_)), has_limit(has_limit_), limit(limit_), - heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000) + heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000), + temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_) { /// grab active pointer active = active_ptr.lock(); @@ -241,6 +241,7 @@ private: Int64 num_updates = -1; bool end_of_blocks = false; UInt64 heartbeat_interval_usec; + UInt64 temporary_live_view_timeout_sec; UInt64 last_event_timestamp_usec = 0; Poco::Timestamp timestamp; }; diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 6ceabf3478c..6c949424f68 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -366,7 +366,6 @@ void StorageLiveView::checkTableCanBeDropped() const void StorageLiveView::noUsersThread(std::shared_ptr storage, const UInt64 & timeout) { bool drop_table = false; - UInt64 next_timeout = timeout; if (storage->shutdown_called) return; @@ -375,33 +374,17 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co while (1) { std::unique_lock lock(storage->no_users_thread_mutex); - if (!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(next_timeout), [&] { return storage->no_users_thread_wakeup; })) + if (!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return storage->no_users_thread_wakeup; })) { storage->no_users_thread_wakeup = false; if (storage->shutdown_called) return; if (storage->hasUsers()) - { - /// Thread woke up but there are still users so sleep for 3 times longer than - /// the original timeout to reduce the number of times thread wakes up. - /// Wait until we are explicitely woken up when a user goes away to - /// reset wait time to the original timeout. - next_timeout = timeout * 3; - continue; - } + return; if (!storage->global_context.getDependencies(storage->database_name, storage->table_name).empty()) continue; drop_table = true; } - else - { - /// Thread was explicitly awaken so reset timeout to the original - next_timeout = timeout; - storage->no_users_thread_wakeup = false; - if (storage->shutdown_called) - return; - continue; - } break; } } @@ -430,24 +413,31 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co void StorageLiveView::startNoUsersThread(const UInt64 & timeout) { + bool expected = false; + if (!start_no_users_thread_called.compare_exchange_strong(expected, true)) + return; + if (is_temporary) { if (no_users_thread.joinable()) { - /// If the thread is already running then - /// wake it up and just return - std::lock_guard lock(no_users_thread_mutex); - no_users_thread_wakeup = true; - no_users_thread_condition.notify_one(); - return; + { + std::lock_guard lock(no_users_thread_mutex); + no_users_thread_wakeup = true; + no_users_thread_condition.notify_one(); + } + no_users_thread.join(); } { std::lock_guard lock(no_users_thread_mutex); no_users_thread_wakeup = false; } - no_users_thread = std::thread(&StorageLiveView::noUsersThread, - std::static_pointer_cast(shared_from_this()), timeout); + if (!is_dropped) + no_users_thread = std::thread(&StorageLiveView::noUsersThread, + std::static_pointer_cast(shared_from_this()), timeout); } + + start_no_users_thread_called = false; } void StorageLiveView::startup() @@ -541,7 +531,11 @@ BlockInputStreams StorageLiveView::watch( if (query.is_watch_events) { - auto reader = std::make_shared(std::static_pointer_cast(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds()); + auto reader = std::make_shared( + std::static_pointer_cast(shared_from_this()), + blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, + context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(), + context.getSettingsRef().temporary_live_view_timeout.totalSeconds()); if (no_users_thread.joinable()) { @@ -565,7 +559,11 @@ BlockInputStreams StorageLiveView::watch( } else { - auto reader = std::make_shared(std::static_pointer_cast(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds()); + auto reader = std::make_shared( + std::static_pointer_cast(shared_from_this()), + blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, + context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(), + context.getSettingsRef().temporary_live_view_timeout.totalSeconds()); if (no_users_thread.joinable()) { diff --git a/dbms/src/Storages/LiveView/StorageLiveView.h b/dbms/src/Storages/LiveView/StorageLiveView.h index a9a8985b4f8..f71d0758f03 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.h +++ b/dbms/src/Storages/LiveView/StorageLiveView.h @@ -72,6 +72,7 @@ public: return active_ptr.use_count() > 1; } /// No users thread mutex, predicate and wake up condition + void startNoUsersThread(const UInt64 & timeout); std::mutex no_users_thread_mutex; bool no_users_thread_wakeup = false; std::condition_variable no_users_thread_condition; @@ -166,10 +167,10 @@ private: /// Background thread for temporary tables /// which drops this table if there are no users - void startNoUsersThread(const UInt64 & timeout); static void noUsersThread(std::shared_ptr storage, const UInt64 & timeout); std::thread no_users_thread; std::atomic shutdown_called = false; + std::atomic start_no_users_thread_called = false; UInt64 temporary_live_view_timeout; StorageLiveView( From 12cb72175b8e5ad156a03ea9f8d45dd453d62a1d Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 25 Aug 2019 20:44:03 -0400 Subject: [PATCH 240/357] Updating test server config to enable live views using the allow_experimental_live_view option. --- dbms/tests/server-test.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/tests/server-test.xml b/dbms/tests/server-test.xml index d68cbca53c1..4a4e0a333ff 100644 --- a/dbms/tests/server-test.xml +++ b/dbms/tests/server-test.xml @@ -43,6 +43,7 @@ + 1 3 /tmp/clickhouse/data/ /tmp/clickhouse/tmp/ From 0599bdba25b0f0e9828634554869e2bb14492c09 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Sun, 25 Aug 2019 22:34:16 -0300 Subject: [PATCH 241/357] Described arbitrary datatype for range_hashed --- .../dicts/external_dicts_dict_layout.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/ru/query_language/dicts/external_dicts_dict_layout.md b/docs/ru/query_language/dicts/external_dicts_dict_layout.md index aafcf531860..826d9b78ae9 100644 --- a/docs/ru/query_language/dicts/external_dicts_dict_layout.md +++ b/docs/ru/query_language/dicts/external_dicts_dict_layout.md @@ -95,7 +95,7 @@ Словарь хранится в оперативной памяти в виде хэш-таблицы с упорядоченным массивом диапазонов и соответствующих им значений. -Этот способ размещения работает также как и hashed и позволяет дополнительно к ключу использовать дипазоны по дате/времени, если они указаны в словаре. +Этот способ размещения работает также как и hashed и позволяет дополнительно к ключу использовать дипазоны по дате/времени (произвольному числовому типу). Пример: таблица содержит скидки для каждого рекламодателя в виде: @@ -111,7 +111,7 @@ +---------------+---------------------+-------------------+--------+ ``` -Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure.md) определить элементы `range_min`, `range_max`. +Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure.md) определить элементы `range_min`, `range_max`. В этих элементах должны присутствовать элементы `name` и `type` (если `type` не указан, будет использован тип по умолчанию -- Date). `type` может быть любым численным типом (Date/DateTime/UInt64/Int32/др.). Пример: @@ -122,14 +122,16 @@ first + Date last + Date ... ``` -Для работы с такими словарями в функцию `dictGetT` необходимо передавать дополнительный аргумент - дату: : +Для работы с такими словарями в функцию `dictGetT` необходимо передавать дополнительный аргумент, для которого подбирается диапазон: dictGetT('dict_name', 'attr_name', id, date) @@ -158,10 +160,12 @@ Abcdef - StartDate + StartTimeStamp + UInt64 - EndDate + EndTimeStamp + UInt64 XXXType From 0a4364370a0a2c54ca9ba12d3b97ef4ca960c79b Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Sun, 25 Aug 2019 22:44:13 -0300 Subject: [PATCH 242/357] Described arbitrary datatype for range_hashed --- .../dicts/external_dicts_dict_layout.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/en/query_language/dicts/external_dicts_dict_layout.md b/docs/en/query_language/dicts/external_dicts_dict_layout.md index 03279688d6c..c3096544d25 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_layout.md +++ b/docs/en/query_language/dicts/external_dicts_dict_layout.md @@ -95,7 +95,7 @@ Configuration example: The dictionary is stored in memory in the form of a hash table with an ordered array of ranges and their corresponding values. -This storage method works the same way as hashed and allows using date/time ranges in addition to the key, if they appear in the dictionary. +This storage method works the same way as hashed and allows using date/time (arbitrary numeric type) ranges in addition to the key. Example: The table contains discounts for each advertiser in the format: @@ -111,7 +111,7 @@ Example: The table contains discounts for each advertiser in the format: +---------------+---------------------+-------------------+--------+ ``` -To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md). +To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md). These elements must contain elements `name` and` type` (if `type` is not specified, the default type will be used - Date). `type` can be any numeric type (Date / DateTime / UInt64 / Int32 / others). Example: @@ -122,14 +122,16 @@ Example: first + Date last + Date ... ``` -To work with these dictionaries, you need to pass an additional date argument to the `dictGetT` function: +To work with these dictionaries, you need to pass an additional argument to the `dictGetT` function, for which a range is selected: ``` dictGetT('dict_name', 'attr_name', id, date) @@ -160,10 +162,12 @@ Configuration example: Abcdef - StartDate + StartTimeStamp + UInt64 - EndDate + EndTimeStamp + UInt64 XXXType From c70f656d5d6e6c83f6b3e3720d14bfc3277372b8 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 25 Aug 2019 21:57:32 -0400 Subject: [PATCH 243/357] Another attempt to enable allow_experimental_live_view options during tests. --- dbms/tests/clickhouse-client.xml | 1 + dbms/tests/server-test.xml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/clickhouse-client.xml b/dbms/tests/clickhouse-client.xml index b6003ca2d09..ebce35127e5 100644 --- a/dbms/tests/clickhouse-client.xml +++ b/dbms/tests/clickhouse-client.xml @@ -1,3 +1,4 @@ + 1 100000 diff --git a/dbms/tests/server-test.xml b/dbms/tests/server-test.xml index 4a4e0a333ff..d68cbca53c1 100644 --- a/dbms/tests/server-test.xml +++ b/dbms/tests/server-test.xml @@ -43,7 +43,6 @@ - 1 3 /tmp/clickhouse/data/ /tmp/clickhouse/tmp/ From 25635d1ab0e24762a8be06908f827f2bb7dbc3b8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Aug 2019 05:46:21 +0300 Subject: [PATCH 244/357] Fixed build of tests --- .../tests/gtest_transform_query_for_external_database.cpp | 2 +- dbms/src/Storages/tests/storage_log.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index e61ab6279a2..a7d79cd23d7 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -23,7 +23,7 @@ struct State { registerFunctions(); DatabasePtr database = std::make_shared("test"); - database->attachTable("table", StorageMemory::create("test", "table", ColumnsDescription{columns})); + database->attachTable("table", StorageMemory::create("test", "table", ColumnsDescription{columns}, ConstraintsDescription{})); context.makeGlobalContext(); context.addDatabase("test", database); context.setCurrentDatabase("test"); diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp index bcf7b1d4d03..02a845cf61c 100644 --- a/dbms/src/Storages/tests/storage_log.cpp +++ b/dbms/src/Storages/tests/storage_log.cpp @@ -25,7 +25,7 @@ try names_and_types.emplace_back("a", std::make_shared()); names_and_types.emplace_back("b", std::make_shared()); - StoragePtr table = StorageLog::create("./", "test", "test", ColumnsDescription{names_and_types}, 1048576); + StoragePtr table = StorageLog::create("./", "test", "test", ColumnsDescription{names_and_types}, ConstraintsDescription{}, 1048576); table->startup(); auto context = Context::createGlobal(); From 6e8b7b9c3babde48b33916a1aace2ba079b7f0fd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Aug 2019 05:53:42 +0300 Subject: [PATCH 245/357] Fixed error --- dbms/src/Interpreters/InterpreterCreateQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index cb5e657bcea..59ae2fc2b72 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -567,6 +567,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) } ColumnsDescription columns; + ConstraintsDescription constraints; StoragePtr res; if (create.as_table_function) @@ -579,6 +580,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) { /// Set and retrieve list of columns. columns = setProperties(create, as_select_sample, as_storage); + constraints = getConstraintsDescription(create.columns_list->constraints); /// Check low cardinality types in creating table if it was not allowed in setting if (!create.attach && !context.getSettingsRef().allow_suspicious_low_cardinality_types && !create.is_materialized_view) @@ -598,8 +600,6 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) setEngine(create); } - ConstraintsDescription constraints = getConstraintsDescription(create.columns_list->constraints); - { std::unique_ptr guard; From d0d63d769efe7ea01cb01676dc5e3ab95098db2c Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 26 Aug 2019 11:00:48 +0300 Subject: [PATCH 246/357] Implemented hasTokenCaseInsensitive function And tests; --- dbms/src/Common/StringSearcher.h | 1 + dbms/src/Common/Volnitsky.h | 3 +- dbms/src/Functions/FunctionsStringSearch.cpp | 15 ++- .../queries/0_stateless/00990_hasToken.python | 100 ++++++++++++------ .../0_stateless/00990_hasToken.reference | 41 +++++++ 5 files changed, 123 insertions(+), 37 deletions(-) diff --git a/dbms/src/Common/StringSearcher.h b/dbms/src/Common/StringSearcher.h index 25287db11f5..d395e6f254a 100644 --- a/dbms/src/Common/StringSearcher.h +++ b/dbms/src/Common/StringSearcher.h @@ -775,6 +775,7 @@ using ASCIICaseInsensitiveStringSearcher = StringSearcher; using UTF8CaseSensitiveStringSearcher = StringSearcher; using UTF8CaseInsensitiveStringSearcher = StringSearcher; using ASCIICaseSensitiveTokenSearcher = TokenSearcher; +using ASCIICaseInsensitiveTokenSearcher = TokenSearcher; /** Uses functions from libc. diff --git a/dbms/src/Common/Volnitsky.h b/dbms/src/Common/Volnitsky.h index c87bdd79dab..a0273a1d858 100644 --- a/dbms/src/Common/Volnitsky.h +++ b/dbms/src/Common/Volnitsky.h @@ -652,7 +652,8 @@ using VolnitskyUTF8 = VolnitskyBase; /// ignores non-ASCII bytes using VolnitskyCaseInsensitiveUTF8 = VolnitskyBase; -using VolnitskyToken = VolnitskyBase; +using VolnitskyCaseSensitiveToken = VolnitskyBase; +using VolnitskyCaseInsensitiveToken = VolnitskyBase; using MultiVolnitsky = MultiVolnitskyBase; using MultiVolnitskyUTF8 = MultiVolnitskyBase; diff --git a/dbms/src/Functions/FunctionsStringSearch.cpp b/dbms/src/Functions/FunctionsStringSearch.cpp index 5d688232bd4..e44138bb482 100644 --- a/dbms/src/Functions/FunctionsStringSearch.cpp +++ b/dbms/src/Functions/FunctionsStringSearch.cpp @@ -436,7 +436,7 @@ struct MultiSearchFirstIndexImpl /** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation. */ -template +template struct HasTokenImpl { using ResultType = UInt8; @@ -454,7 +454,7 @@ struct HasTokenImpl /// The current index in the array of strings. size_t i = 0; - VolnitskyToken searcher(pattern.data(), pattern.size(), end - pos); + TokenSearcher searcher(pattern.data(), pattern.size(), end - pos); /// We will search for the next occurrence in all rows at once. while (pos < end && end != (pos = searcher.search(pos, end - pos))) @@ -483,7 +483,7 @@ struct HasTokenImpl static void constant_constant(const std::string & data, const std::string & pattern, UInt8 & res) { - VolnitskyToken searcher(pattern.data(), pattern.size(), data.size()); + TokenSearcher searcher(pattern.data(), pattern.size(), data.size()); const auto found = searcher.search(data.c_str(), data.size()) != data.end().base(); res = negate_result ^ found; } @@ -589,6 +589,11 @@ struct NameHasToken static constexpr auto name = "hasToken"; }; +struct NameHasTokenCaseInsensitive +{ + static constexpr auto name = "hasTokenCaseInsensitive"; +}; + using FunctionPosition = FunctionsStringSearch, NamePosition>; using FunctionPositionUTF8 = FunctionsStringSearch, NamePositionUTF8>; @@ -615,7 +620,8 @@ using FunctionMultiSearchFirstPositionUTF8 = FunctionsMultiStringSearch, NameMultiSearchFirstPositionCaseInsensitive>; using FunctionMultiSearchFirstPositionCaseInsensitiveUTF8 = FunctionsMultiStringSearch, NameMultiSearchFirstPositionCaseInsensitiveUTF8>; -using FunctionHasToken = FunctionsStringSearch, NameHasToken>; +using FunctionHasToken = FunctionsStringSearch, NameHasToken>; +using FunctionHasTokenCaseInsensitive = FunctionsStringSearch, NameHasTokenCaseInsensitive>; void registerFunctionsStringSearch(FunctionFactory & factory) { @@ -645,6 +651,7 @@ void registerFunctionsStringSearch(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); factory.registerAlias("locate", NamePosition::name, FunctionFactory::CaseInsensitive); } diff --git a/dbms/tests/queries/0_stateless/00990_hasToken.python b/dbms/tests/queries/0_stateless/00990_hasToken.python index 217d96dfe52..cd2a284655f 100755 --- a/dbms/tests/queries/0_stateless/00990_hasToken.python +++ b/dbms/tests/queries/0_stateless/00990_hasToken.python @@ -2,11 +2,12 @@ # encoding: utf-8 import re +from string import Template HAYSTACKS = [ + "hay hay hay hay hay hay hay hay hay needle hay hay hay hay hay hay hay hay hay", "hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay needle", "needle hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay hay", - "hay hay hay hay hay hay hay hay hay needle hay hay hay hay hay hay hay hay hay", ] NEEDLE = "needle" @@ -48,47 +49,77 @@ def transform_needle(query, string_transformation_func): return NEEDLE_RE.sub(replace_with_transformation, query) - -def create_cases(table_row_template, table_query_template, const_query_template): +def create_cases(case_sensitive_func, case_insensitive_func, table_row_template, table_query_template, const_query_template): const_queries = [] table_rows = [] table_queries = set() - def add_case(haystack, needle, match): + def add_case(func, haystack, needle, match): match = int(match) - const_queries.append(const_query_template.format(haystack=haystack, needle=needle, match=match)) - table_queries.add(table_query_template.format(haystack=haystack, needle=needle, match=match)) - table_rows.append(table_row_template.format(haystack=haystack, needle=needle, match=match)) + args = dict( + func = func, + haystack = haystack, + needle = needle, + match = match + ) + const_queries.append(const_query_template.substitute(args)) + table_queries.add(table_query_template.substitute(args)) + table_rows.append(table_row_template.substitute(args)) + + def add_case_sensitive(haystack, needle, match): + add_case(case_sensitive_func, haystack, needle, match) + if match: + add_case(case_sensitive_func, transform_needle(haystack, str.swapcase), transform_needle(needle, str.swapcase), match) + + def add_case_insensitive(haystack, needle, match): + add_case(case_insensitive_func, haystack, needle, match) + if match: + add_case(case_insensitive_func, transform_needle(haystack, str.swapcase), needle, match) + add_case(case_insensitive_func, haystack, transform_needle(needle, str.swapcase), match) + # Negative cases - add_case(remove_needle(HAYSTACKS[0]), NEEDLE, False) + add_case_sensitive(remove_needle(HAYSTACKS[0]), NEEDLE, False) + add_case_insensitive(remove_needle(HAYSTACKS[0]), NEEDLE, False) + for haystack in HAYSTACKS: - add_case(transform_needle(haystack, str.title), NEEDLE, False) + add_case_sensitive(transform_needle(haystack, str.swapcase), NEEDLE, False) + sep = '' h = replace_separators(haystack, sep) - add_case(h, NEEDLE, False) - add_case(small_needle(h), small_needle(NEEDLE), False) - add_case(enlarge_haystack(h, 10, sep), NEEDLE, False) + + add_case_sensitive(h, NEEDLE, False) + add_case_insensitive(h, NEEDLE, False) + + add_case_sensitive(small_needle(h), small_needle(NEEDLE), False) + add_case_insensitive(small_needle(h), small_needle(NEEDLE), False) + + add_case_sensitive(enlarge_haystack(h, 10, sep), NEEDLE, False) + add_case_insensitive(enlarge_haystack(h, 10, sep), NEEDLE, False) # positive cases for haystack in HAYSTACKS: - add_case(transform_needle(haystack, str.title), transform_needle(NEEDLE, str.title), True) - add_case(transform_needle(haystack, str.upper), transform_needle(NEEDLE, str.upper), True) + add_case_sensitive(haystack, NEEDLE, True) + add_case_insensitive(haystack, NEEDLE, True) - # Not checking all separators since some (like ' and \n) cause issues when coupled with - # re-based replacement and quoting in query - # other are rare in practice and checking all separators makes this test too lengthy. - # r'\\\\' turns into a single '\' in query - #separators = list(''' \t`~!@#$%^&*()-=+|]}[{";:/?.>,<''') + [r'\\\\'] - separators = list(''' \t;:?.,''') + [r'\\\\'] - for sep in separators: + for sep in list(''' ,'''): h = replace_separators(haystack, sep) - add_case(h, NEEDLE, True) - add_case(small_needle(h), small_needle(NEEDLE), True) - add_case(enlarge_haystack(h, 200, sep), NEEDLE, True) - add_case(replace_needle(h, 'иголка'), replace_needle(NEEDLE, 'иголка'), True) - add_case(replace_needle(h, '指针'), replace_needle(NEEDLE, '指针'), True) + add_case_sensitive(h, NEEDLE, True) + add_case_sensitive(small_needle(h), small_needle(NEEDLE), True) + add_case_sensitive(enlarge_haystack(h, 200, sep), NEEDLE, True) + + add_case_insensitive(h, NEEDLE, True) + add_case_insensitive(small_needle(h), small_needle(NEEDLE), True) + add_case_insensitive(enlarge_haystack(h, 200, sep), NEEDLE, True) + + # case insesitivity works only on ASCII strings + add_case_sensitive(replace_needle(h, 'иголка'), replace_needle(NEEDLE, 'иголка'), True) + add_case_sensitive(replace_needle(h, '指针'), replace_needle(NEEDLE, '指针'), True) + + for sep in list('''~!@$%^&*()-=+|]}[{";:/?.><\t''') + [r'\\\\']: + h = replace_separators(HAYSTACKS[0], sep) + add_case(case_sensitive_func, h, NEEDLE, True) return table_rows, table_queries, const_queries @@ -97,12 +128,14 @@ def main(): def query(x): print x - CONST_QUERY = """SELECT hasToken('{haystack}', '{needle}'), ' expecting ', {match};""" - #SELECT hasToken(haystack, '{needle}') FROM ht WHERE needle = '{needle}' AND match = {match};""" - TABLE_QUERY = """WITH '{needle}' as n SELECT haystack, needle, hasToken(haystack, n) as result FROM ht WHERE needle = n AND result != match;""" - TABLE_ROW = """('{haystack}', '{needle}', {match})""" + CONST_QUERY = Template("""SELECT ${func}('${haystack}', '${needle}'), ' expecting ', ${match};""") + TABLE_QUERY = Template("""WITH '${needle}' as n + SELECT haystack, needle, ${func}(haystack, n) as result + FROM ht + WHERE func = '${func}' AND needle = n AND result != match;""") + TABLE_ROW = Template("""('${haystack}', '${needle}', ${match}, '${func}')""") - rows, table_queries, const_queries = create_cases(TABLE_ROW, TABLE_QUERY, CONST_QUERY) + rows, table_queries, const_queries = create_cases('hasToken', 'hasTokenCaseInsensitive', TABLE_ROW, TABLE_QUERY, CONST_QUERY) for q in const_queries: query(q) @@ -112,7 +145,8 @@ def main(): ( haystack String, needle String, - match UInt8 + match UInt8, + func String ) ENGINE MergeTree() ORDER BY haystack; @@ -120,5 +154,7 @@ INSERT INTO ht VALUES {values};""".format(values=", ".join(rows))) for q in sorted(table_queries): query(q) + query("""DROP TABLE ht""") + if __name__ == '__main__': main() diff --git a/dbms/tests/queries/0_stateless/00990_hasToken.reference b/dbms/tests/queries/0_stateless/00990_hasToken.reference index 867c0c1c691..1e8c067362c 100644 --- a/dbms/tests/queries/0_stateless/00990_hasToken.reference +++ b/dbms/tests/queries/0_stateless/00990_hasToken.reference @@ -11,6 +11,47 @@ 0 expecting 0 0 expecting 0 0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +0 expecting 0 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 +1 expecting 1 1 expecting 1 1 expecting 1 1 expecting 1 From 8047aab684477c0a3bf7f78e30bdb954b011433f Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 26 Aug 2019 13:13:56 +0300 Subject: [PATCH 247/357] Add const --- dbms/src/Core/SettingsCommon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index 8fc557f1768..b8c56d50caa 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -325,7 +325,7 @@ private: /// Can be updated after first load for config/definition. /// Non updatable settings can be `changed`, /// if they were overwritten in config/definition. - bool updateable; + const bool updateable; GetStringFunction get_string; GetFieldFunction get_field; SetStringFunction set_string; From c2b6cffabc709afd1862ceb7348564d54c9f74a4 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 26 Aug 2019 15:39:35 +0300 Subject: [PATCH 248/357] Update IStorage.cpp --- dbms/src/Storages/IStorage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index eb9b4adc4f4..855336c06e5 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -399,7 +399,7 @@ void IStorage::alterSettings( { if (hasSetting(change.name)) { - auto finder = [&change] (const SettingChange & c) { return c.name == change.name;}; + auto finder = [&change] (const SettingChange & c) { return c.name == change.name; }; if (auto it = std::find_if(storage_changes.begin(), storage_changes.end(), finder); it != storage_changes.end()) it->value = change.value; else From 4f89dcc80db1b3309f07c3828a72c88c49bb0675 Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Mon, 26 Aug 2019 15:47:19 +0300 Subject: [PATCH 249/357] Try to fix --- dbms/programs/client/Client.cpp | 2 +- dbms/src/Common/TerminalSize.cpp | 8 +++----- dbms/src/Common/TerminalSize.h | 3 +-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index e1f1bd92840..6caf57a28bd 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -131,7 +131,7 @@ private: bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode. bool stdin_is_not_tty = false; /// stdin is not a terminal. - uint16_t terminal_width{}; /// Terminal width is needed to render progress bar. + uint16_t terminal_width = 0; /// Terminal width is needed to render progress bar. std::unique_ptr connection; /// Connection to DB. String query_id; /// Current query_id. diff --git a/dbms/src/Common/TerminalSize.cpp b/dbms/src/Common/TerminalSize.cpp index ffc301cf77c..7f6bb93119f 100644 --- a/dbms/src/Common/TerminalSize.cpp +++ b/dbms/src/Common/TerminalSize.cpp @@ -3,15 +3,13 @@ #include #include -namespace po = boost::program_options; - namespace DB::ErrorCodes { extern const int SYSTEM_ERROR; } -unsigned short int getTerminalWidth() +uint16_t getTerminalWidth() { if (isatty(STDIN_FILENO)) { @@ -25,13 +23,13 @@ unsigned short int getTerminalWidth() return 0; } -po::options_description createOptionsDescription(const std::string &caption, unsigned short terminal_width) +po::options_description createOptionsDescription(const std::string &caption, uint16_t terminal_width) { unsigned line_length = po::options_description::m_default_line_length; unsigned min_description_length = line_length / 2; std::string longest_option_desc = "--http_native_compression_disable_checksumming_on_decompress"; - line_length = std::max(static_cast(longest_option_desc.size()), terminal_width); + line_length = std::max(static_cast(longest_option_desc.size()), terminal_width); min_description_length = std::min(min_description_length, line_length - 2); return po::options_description(caption, line_length, min_description_length); diff --git a/dbms/src/Common/TerminalSize.h b/dbms/src/Common/TerminalSize.h index 4dd58b5c8ca..d7eee417f46 100644 --- a/dbms/src/Common/TerminalSize.h +++ b/dbms/src/Common/TerminalSize.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -8,7 +7,7 @@ namespace po = boost::program_options; -unsigned short int getTerminalWidth(); +uint16_t getTerminalWidth(); /** Creates po::options_description with name and an appropriate size for option displaying * when program is called with option --help From 2f5e6158e7a158e84e57317d8de7b66103ab3716 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 26 Aug 2019 16:27:12 +0300 Subject: [PATCH 250/357] Update TerminalSize.cpp --- dbms/src/Common/TerminalSize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/TerminalSize.cpp b/dbms/src/Common/TerminalSize.cpp index 7f6bb93119f..ea8607b93ae 100644 --- a/dbms/src/Common/TerminalSize.cpp +++ b/dbms/src/Common/TerminalSize.cpp @@ -23,7 +23,7 @@ uint16_t getTerminalWidth() return 0; } -po::options_description createOptionsDescription(const std::string &caption, uint16_t terminal_width) +po::options_description createOptionsDescription(const std::string & caption, uint16_t terminal_width) { unsigned line_length = po::options_description::m_default_line_length; unsigned min_description_length = line_length / 2; From 479b4d81f314e11badd048052fe32e4ed226e646 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Aug 2019 16:46:07 +0300 Subject: [PATCH 251/357] Fixed error --- dbms/src/Storages/StorageDistributed.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index e3952634c00..90a973ae3b9 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -220,13 +220,17 @@ StorageDistributed::StorageDistributed( : table_name(table_name_), database_name(database_name_), remote_database(remote_database_), remote_table(remote_table_), global_context(context_), cluster_name(global_context.getMacros()->expand(cluster_name_)), has_sharding_key(sharding_key_), - sharding_key_expr(sharding_key_ ? buildShardingKeyExpression(sharding_key_, global_context, getColumns().getAllPhysical(), false) : nullptr), - sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/')) { setColumns(columns_); setConstraints(constraints_); + if (sharding_key_) + { + sharding_key_expr = buildShardingKeyExpression(sharding_key_, global_context, getColumns().getAllPhysical(), false); + sharding_key_column_name = sharding_key_->getColumnName(); + } + /// Sanity check. Skip check if the table is already created to allow the server to start. if (!attach_ && !cluster_name.empty()) { From dff0cc1e1cf9f87efaac8ba734bd140bbe80d88f Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 26 Aug 2019 17:24:29 +0300 Subject: [PATCH 252/357] Remove COW and add MultiVersion --- dbms/src/Interpreters/Context.cpp | 8 ++--- .../Storages/MergeTree/DataPartsExchange.cpp | 4 +-- .../MergeTree/IMergedBlockOutputStream.cpp | 4 +-- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 31 ++++++++-------- dbms/src/Storages/MergeTree/MergeTreeData.h | 35 +++++-------------- .../MergeTree/MergeTreeDataMergerMutator.cpp | 12 +++---- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 6 ++-- .../MergeTreeIndexGranularityInfo.cpp | 2 +- .../Storages/MergeTree/MergeTreeSettings.cpp | 5 --- .../Storages/MergeTree/MergeTreeSettings.h | 13 ++----- .../MergeTreeThreadSelectBlockInputStream.cpp | 2 +- .../ReplicatedMergeTreeCleanupThread.cpp | 8 ++--- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- .../ReplicatedMergeTreeRestartingThread.cpp | 6 ++-- .../ReplicatedMergeTreeTableMetadata.cpp | 2 +- .../MergeTree/registerStorageMergeTree.cpp | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 6 ++-- dbms/src/Storages/StorageMergeTree.h | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 32 ++++++++--------- .../src/Storages/StorageReplicatedMergeTree.h | 2 +- 20 files changed, 75 insertions(+), 109 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 39b9fd40c9a..016f1fa0e49 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -142,7 +142,7 @@ struct ContextShared std::unique_ptr ddl_worker; /// Process ddl commands from zk. /// Rules for selecting the compression settings, depending on the size of the part. mutable std::unique_ptr compression_codec_selector; - MergeTreeSettingsPtr merge_tree_settings; /// Settings of MergeTree* engines. + std::optional merge_tree_settings; /// Settings of MergeTree* engines. size_t max_table_size_to_drop = 50000000000lu; /// Protects MergeTree tables from accidental DROP (50GB by default) size_t max_partition_size_to_drop = 50000000000lu; /// Protects MergeTree partitions from accidental DROP (50GB by default) String format_schema_path; /// Path to a directory that contains schema files used by input formats. @@ -1759,9 +1759,9 @@ const MergeTreeSettings & Context::getMergeTreeSettings() const if (!shared->merge_tree_settings) { auto & config = getConfigRef(); - MutableMergeTreeSettingsPtr settings_ptr = MergeTreeSettings::create(); - settings_ptr->loadFromConfig("merge_tree", config); - shared->merge_tree_settings = std::move(settings_ptr); + MergeTreeSettings mt_settings; + mt_settings.loadFromConfig("merge_tree", config); + shared->merge_tree_settings.emplace(mt_settings); } return *shared->merge_tree_settings; diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index 795cc68f1ea..8a6ceec509a 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -54,7 +54,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); String part_name = params.get("part"); - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); /// Validation of the input that may come from malicious replica. MergeTreePartInfo::fromPartName(part_name, data.format_version); @@ -175,7 +175,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( { /// Validation of the input that may come from malicious replica. MergeTreePartInfo::fromPartName(part_name, data.format_version); - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); Poco::URI uri; uri.setScheme(interserver_scheme); diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 11b64fd9788..255fb998446 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -39,7 +39,7 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( , compute_granularity(index_granularity.empty()) , codec(std::move(codec_)) , skip_indices(indices_to_recalc) - , with_final_mark(storage.getCOWSettings()->write_final_mark && can_use_adaptive_granularity) + , with_final_mark(storage.getSettings()->write_final_mark && can_use_adaptive_granularity) { if (blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); @@ -139,7 +139,7 @@ void fillIndexGranularityImpl( void IMergedBlockOutputStream::fillIndexGranularity(const Block & block) { - const auto storage_settings = storage.getCOWSettings(); + const auto storage_settings = storage.getSettings(); fillIndexGranularityImpl( block, storage_settings->index_granularity_bytes, diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 271ccaf527e..f10c2067e44 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -107,7 +107,7 @@ MergeTreeData::MergeTreeData( const ASTPtr & sample_by_ast_, const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - MergeTreeSettingsPtr settings_, + std::unique_ptr storage_settings_, bool require_part_metadata_, bool attach, BrokenPartCallback broken_part_callback_) @@ -121,11 +121,11 @@ MergeTreeData::MergeTreeData( full_path(full_path_), broken_part_callback(broken_part_callback_), log_name(database_name + "." + table_name), log(&Logger::get(log_name)), - guarded_settings(settings_), + storage_settings(std::move(storage_settings_)), data_parts_by_info(data_parts_indexes.get()), data_parts_by_state_and_info(data_parts_indexes.get()) { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); setProperties(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_); setConstraints(constraints_); @@ -733,7 +733,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { LOG_DEBUG(log, "Loading data parts"); - const auto settings = getCOWSettings(); + const auto settings = getSettings(); Strings part_file_names; Poco::DirectoryIterator end; for (Poco::DirectoryIterator it(full_path); it != end; ++it) @@ -966,7 +966,7 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life if (!lock.try_lock()) return; - const auto settings = getCOWSettings(); + const auto settings = getSettings(); time_t current_time = time(nullptr); ssize_t deadline = (custom_directories_lifetime_seconds >= 0) ? current_time - custom_directories_lifetime_seconds @@ -1021,7 +1021,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts() if (part.unique() && /// Grab only parts that are not used by anyone (SELECTs for example). part_remove_time < now && - now - part_remove_time > getCOWSettings()->old_parts_lifetime.totalSeconds()) + now - part_remove_time > getSettings()->old_parts_lifetime.totalSeconds()) { parts_to_delete.emplace_back(it); } @@ -1105,7 +1105,7 @@ void MergeTreeData::clearOldPartsFromFilesystem() void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_remove) { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); if (parts_to_remove.size() > 1 && settings->max_part_removal_threads > 1 && parts_to_remove.size() > settings->concurrent_part_removal_threshold) { /// Parallel parts removal. @@ -1342,7 +1342,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name const IndicesASTs & old_indices, const IndicesASTs & new_indices, ExpressionActionsPtr & out_expression, NameToNameMap & out_rename_map, bool & out_force_update_metadata) const { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); out_expression = nullptr; out_rename_map = {}; out_force_update_metadata = false; @@ -1508,7 +1508,7 @@ void MergeTreeData::alterDataPart( bool skip_sanity_checks, AlterDataPartTransactionPtr & transaction) { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); ExpressionActionsPtr expression; const auto & part = transaction->getDataPart(); bool force_update_metadata; @@ -1653,11 +1653,10 @@ void MergeTreeData::alterSettings( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { - std::lock_guard lock(settings_mutex); - MutableMergeTreeSettingsPtr settings = std::move(*guarded_settings.getPtr()).mutate(); - settings->updateFromChanges(new_changes); + MergeTreeSettings copy = *getSettings(); + copy.updateFromChanges(new_changes); IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); - guarded_settings.setPtr(std::move(settings)); + storage_settings.set(std::make_unique(copy)); } bool MergeTreeData::hasSetting(const String & setting_name) const @@ -2343,7 +2342,7 @@ std::optional MergeTreeData::getMinPartDataVersion() const void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); const size_t parts_count_in_total = getPartsCount(); if (parts_count_in_total >= settings->max_parts_in_total) { @@ -2381,7 +2380,7 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const void MergeTreeData::throwInsertIfNeeded() const { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); const size_t parts_count_in_total = getPartsCount(); if (parts_count_in_total >= settings->max_parts_in_total) { @@ -3076,7 +3075,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); if (!settings->enable_mixed_granularity_parts || settings->index_granularity_bytes == 0) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 9f3d1127f2c..8babd723fae 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -331,7 +331,7 @@ public: const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported. const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - MergeTreeSettingsPtr settings_, + std::unique_ptr settings_, bool require_part_metadata_, bool attach, BrokenPartCallback broken_part_callback_ = [](const String &){}); @@ -620,7 +620,7 @@ public: /// Has additional constraint in replicated version virtual bool canUseAdaptiveGranularity() const { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); return settings->index_granularity_bytes != 0 && (settings->enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts); } @@ -683,13 +683,12 @@ public: bool has_non_adaptive_index_granularity_parts = false; - /// Get copy-on-write pointer to storage settings. + /// Get constant pointer to storage settings. /// Copy this pointer into your scope and you will /// get consistent settings. - const MergeTreeSettingsPtr getCOWSettings() const + MergeTreeSettingsPtr getSettings() const { - std::shared_lock lock(settings_mutex); - return guarded_settings.copyPtr(); + return storage_settings.get(); } protected: @@ -721,26 +720,9 @@ protected: String log_name; Logger * log; - /// Just hides settings pointer from direct usage - class MergeTreeSettingsGuard - { - private: - /// Settings COW pointer. Data maybe changed at any point of time. - /// If you need consistent settings, just copy pointer to your scope. - MergeTreeSettingsPtr settings_ptr; - public: - MergeTreeSettingsGuard(MergeTreeSettingsPtr settings_ptr_) - : settings_ptr(settings_ptr_) - {} - - const MergeTreeSettingsPtr copyPtr() const { return settings_ptr; } - MergeTreeSettingsPtr getPtr() { return settings_ptr; } - void setPtr(MergeTreeSettingsPtr ptr) { settings_ptr = ptr; } - }; - - /// Storage settings. Don't use this field directly, if you - /// want readonly settings. Prefer getCOWSettings() method. - MergeTreeSettingsGuard guarded_settings; + /// Storage settings. + /// Use get and set to receive readonly versions. + MultiVersion storage_settings; /// Work with data parts @@ -829,7 +811,6 @@ protected: /// The same for clearOldTemporaryDirectories. std::mutex clear_old_temporary_directories_mutex; /// Mutex for settings usage - mutable std::shared_mutex settings_mutex; void setProperties(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, const ColumnsDescription & new_columns, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 312c5a195d8..ad489a91603 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -141,7 +141,7 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_siz throw Exception("Logical error: invalid arguments passed to getMaxSourcePartsSize: pool_used > pool_size", ErrorCodes::LOGICAL_ERROR); size_t free_entries = pool_size - pool_used; - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); UInt64 max_size = 0; if (free_entries >= data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge) @@ -159,7 +159,7 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_siz UInt64 MergeTreeDataMergerMutator::getMaxSourcePartSizeForMutation() { - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); size_t total_threads_in_pool = pool.getNumberOfThreads(); size_t busy_threads_in_pool = CurrentMetrics::values[CurrentMetrics::BackgroundPoolTask].load(std::memory_order_relaxed); @@ -179,7 +179,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( String * out_disable_reason) { MergeTreeData::DataPartsVector data_parts = data.getDataPartsVector(); - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); if (data_parts.empty()) { @@ -556,7 +556,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor Names all_column_names = data.getColumns().getNamesOfPhysical(); NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); NamesAndTypesList gathering_columns, merging_columns; Names gathering_column_names, merging_column_names; @@ -965,7 +965,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor const auto & updated_header = mutations_interpreter.getUpdatedHeader(); NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); Block in_header = in->getHeader(); @@ -1145,7 +1145,7 @@ MergeTreeDataMergerMutator::MergeAlgorithm MergeTreeDataMergerMutator::chooseMer const MergeTreeData::DataPartsVector & parts, size_t sum_rows_upper_bound, const NamesAndTypesList & gathering_columns, bool deduplicate, bool need_remove_expired_values) const { - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); if (deduplicate) return MergeAlgorithm::Horizontal; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8cbb0819d20..40dc0bf6b52 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -673,7 +673,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( size_t sum_marks = 0; size_t total_rows = 0; - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); size_t adaptive_parts = 0; for (size_t i = 0; i < parts.size(); ++i) { @@ -832,7 +832,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO SortingInfoPtr sorting_info = query_info.sorting_info; size_t adaptive_parts = 0; std::vector sum_marks_in_parts(parts.size()); - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); for (size_t i = 0; i < parts.size(); ++i) { @@ -1035,7 +1035,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal const Names & virt_columns, const Settings & settings) const { - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); size_t sum_marks = 0; size_t adaptive_parts = 0; for (size_t i = 0; i < parts.size(); ++i) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 143af37c10d..c25968480c8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -25,7 +25,7 @@ std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS( MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo( const MergeTreeData & storage) { - const auto storage_settings = storage.getCOWSettings(); + const auto storage_settings = storage.getSettings(); fixed_index_granularity = storage_settings->index_granularity; /// Granularity is fixed if (!storage.canUseAdaptiveGranularity()) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index 5bc44eee842..9eee33554ab 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -75,9 +75,4 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) #undef ADD_IF_ABSENT } - -MergeTreeSettings::MutablePtr MergeTreeSettings::clone() const -{ - return COW::create(*this); -} } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 68d3e1ec66c..6ba08fed5da 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -2,7 +2,6 @@ #include #include -#include namespace Poco @@ -22,11 +21,9 @@ class ASTStorage; /** Settings for the MergeTree family of engines. * Could be loaded from config or from a CREATE TABLE query (SETTINGS clause). */ -struct MergeTreeSettings : public SettingsCollection, public COW +struct MergeTreeSettings : public SettingsCollection { - friend class COW; - /// M (mutable) for normal settings, IM (immutable) for not updateable settings. #define LIST_OF_MERGE_TREE_SETTINGS(M, IM) \ IM(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \ @@ -101,14 +98,8 @@ struct MergeTreeSettings : public SettingsCollection, public /// NOTE: will rewrite the AST to add immutable settings. void loadFromQuery(ASTStorage & storage_def); - - MutablePtr clone() const; -private: - MergeTreeSettings() = default; - MergeTreeSettings(const MergeTreeSettings & o) = default; }; -using MergeTreeSettingsPtr = MergeTreeSettings::Ptr; -using MutableMergeTreeSettingsPtr = MergeTreeSettings::MutablePtr; +using MergeTreeSettingsPtr = std::shared_ptr; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp index 7a09bde0998..69cf173212d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp @@ -31,7 +31,7 @@ MergeTreeThreadSelectBlockInputStream::MergeTreeThreadSelectBlockInputStream( /// Maybe it will make sence to add settings `max_block_size_bytes` if (max_block_size_rows && !storage.canUseAdaptiveGranularity()) { - size_t fixed_index_granularity = storage.getCOWSettings()->index_granularity; + size_t fixed_index_granularity = storage.getSettings()->index_granularity; min_marks_to_read = (min_marks_to_read_ * fixed_index_granularity + max_block_size_rows - 1) / max_block_size_rows * max_block_size_rows / fixed_index_granularity; } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 2b03ed86895..b89fd9629c0 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -27,7 +27,7 @@ ReplicatedMergeTreeCleanupThread::ReplicatedMergeTreeCleanupThread(StorageReplic void ReplicatedMergeTreeCleanupThread::run() { - auto storage_settings = storage.getCOWSettings(); + auto storage_settings = storage.getSettings(); const auto CLEANUP_SLEEP_MS = storage_settings->cleanup_delay_period * 1000 + std::uniform_int_distribution(0, storage_settings->cleanup_delay_period_random_add * 1000)(rng); @@ -75,7 +75,7 @@ void ReplicatedMergeTreeCleanupThread::iterate() void ReplicatedMergeTreeCleanupThread::clearOldLogs() { auto zookeeper = storage.getZooKeeper(); - auto storage_settings = storage.getCOWSettings(); + auto storage_settings = storage.getSettings(); Coordination::Stat stat; if (!zookeeper->exists(storage.zookeeper_path + "/log", &stat)) @@ -287,7 +287,7 @@ struct ReplicatedMergeTreeCleanupThread::NodeWithStat void ReplicatedMergeTreeCleanupThread::clearOldBlocks() { auto zookeeper = storage.getZooKeeper(); - auto storage_settings = storage.getCOWSettings(); + auto storage_settings = storage.getSettings(); std::vector timed_blocks; getBlocksSortedByTime(*zookeeper, timed_blocks); @@ -404,7 +404,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & void ReplicatedMergeTreeCleanupThread::clearOldMutations() { - auto storage_settings = storage.getCOWSettings(); + auto storage_settings = storage.getSettings(); if (!storage_settings->finished_mutations_to_keep) return; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index d8255add55a..e8cfeb78963 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -964,7 +964,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( * Setting max_bytes_to_merge_at_max_space_in_pool still working for regular merges, * because the leader replica does not assign merges of greater size (except OPTIMIZE PARTITION and OPTIMIZE FINAL). */ - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); bool ignore_max_size = (entry.type == LogEntry::MERGE_PARTS) && (max_source_parts_size == data_settings->max_bytes_to_merge_at_max_space_in_pool); if (!ignore_max_size && sum_parts_size_in_bytes > max_source_parts_size) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 6145713492f..c766219a349 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -44,7 +44,7 @@ ReplicatedMergeTreeRestartingThread::ReplicatedMergeTreeRestartingThread(Storage , log(&Logger::get(log_name)) , active_node_identifier(generateActiveNodeIdentifier()) { - const auto storage_settings = storage.getCOWSettings(); + const auto storage_settings = storage.getSettings(); check_period_ms = storage_settings->zookeeper_session_expiration_check_period.totalSeconds() * 1000; /// Periodicity of checking lag of replica. @@ -122,7 +122,7 @@ void ReplicatedMergeTreeRestartingThread::run() } time_t current_time = time(nullptr); - const auto storage_settings = storage.getCOWSettings(); + const auto storage_settings = storage.getSettings(); if (current_time >= prev_time_of_check_delay + static_cast(storage_settings->check_delay_period)) { /// Find out lag of replicas. @@ -171,7 +171,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() activateReplica(); const auto & zookeeper = storage.getZooKeeper(); - const auto storage_settings = storage.getCOWSettings(); + const auto storage_settings = storage.getSettings(); storage.cloneReplicaIfNeeded(zookeeper); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 6b12df91546..5ada5d50b23 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -27,7 +27,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr if (data.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) date_column = data.minmax_idx_columns[data.minmax_idx_date_column_pos]; - const auto data_settings = data.getCOWSettings(); + const auto data_settings = data.getSettings(); sampling_expression = formattedAST(data.sample_by_ast); index_granularity = data_settings->index_granularity; merging_params_mode = static_cast(data.merging_params.mode); diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 8afca855dba..cebe6c3ca44 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -575,7 +575,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) ASTPtr ttl_table_ast; IndicesDescription indices_description; ConstraintsDescription constraints_description; - MutableMergeTreeSettingsPtr storage_settings = MergeTreeSettings::create(args.context.getMergeTreeSettings()); + std::unique_ptr storage_settings = std::make_unique(args.context.getMergeTreeSettings()); if (is_extended_storage_def) { diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index e7be6a35cb9..c0caa7ff1e0 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -63,14 +63,14 @@ StorageMergeTree::StorageMergeTree( const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported. const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - MergeTreeSettingsPtr settings_, + std::unique_ptr storage_settings_, bool has_force_restore_data_flag) : MergeTreeData(database_name_, table_name_, path_ + escapeForFileName(table_name_) + '/', columns_, indices_, constraints_, context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_, sample_by_ast_, ttl_table_ast_, merging_params_, - settings_, false, attach), + std::move(storage_settings_), false, attach), path(path_), background_pool(context_.getBackgroundPool()), reader(*this), writer(*this), merger_mutator(*this, global_context.getBackgroundPool()) @@ -804,7 +804,7 @@ Int64 StorageMergeTree::getCurrentMutationVersion( void StorageMergeTree::clearOldMutations() { - const auto settings = getCOWSettings(); + const auto settings = getSettings(); if (!settings->finished_mutations_to_keep) return; diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 9f51a9357ce..df1cd1682b9 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -153,7 +153,7 @@ protected: const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported. const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - MergeTreeSettingsPtr settings_, + std::unique_ptr settings_, bool has_force_restore_data_flag); }; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index ce9448dd0e7..2bf8e26112f 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -202,14 +202,14 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const ASTPtr & sample_by_ast_, const ASTPtr & ttl_table_ast_, const MergingParams & merging_params_, - MergeTreeSettingsPtr settings_, + std::unique_ptr settings_, bool has_force_restore_data_flag) : MergeTreeData(database_name_, table_name_, path_ + escapeForFileName(table_name_) + '/', columns_, indices_, constraints_, context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_, sample_by_ast_, ttl_table_ast_, merging_params_, - settings_, true, attach, + std::move(settings_), true, attach, [this] (const std::string & name) { enqueuePartForCheck(name); }), zookeeper_path(global_context.getMacros()->expand(zookeeper_path_, database_name_, table_name_)), replica_name(global_context.getMacros()->expand(replica_name_, database_name_, table_name_)), @@ -637,7 +637,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) for (const auto & part : parts) total_rows_on_filesystem += part->rows_count; - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); bool insane = unexpected_parts_rows > total_rows_on_filesystem * storage_settings->replicated_max_ratio_of_wrong_parts; if (insane && !skip_sanity_checks) @@ -781,7 +781,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (!has_been_already_added) { - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); String part_path = replica_path + "/parts/" + part_name; ops.emplace_back(zkutil::makeCheckRequest( @@ -864,7 +864,7 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd String StorageReplicatedMergeTree::getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums) const { return MinimalisticDataPartChecksums::getSerializedString(checksums, - getCOWSettings()->use_minimalistic_checksums_in_zookeeper); + getSettings()->use_minimalistic_checksums_in_zookeeper); } @@ -1035,7 +1035,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) parts.push_back(part); } - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); if (!have_all_parts) { /// If you do not have all the necessary parts, try to take some already merged part from someone. @@ -1161,7 +1161,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedMergeTree::LogEntry & entry) { const String & source_part_name = entry.source_parts.at(0); - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); LOG_TRACE(log, "Executing log entry to mutate part " << source_part_name << " to " << entry.new_part_name); DataPartPtr source_part = getActiveContainingPart(source_part_name); @@ -1276,7 +1276,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) { String replica = findReplicaHavingCoveringPart(entry, true); - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); static std::atomic_uint total_fetches {0}; if (storage_settings->replicated_max_parallel_fetches && total_fetches >= storage_settings->replicated_max_parallel_fetches) @@ -2213,7 +2213,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() if (!is_leader) return; - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); const bool deduplicate = false; /// TODO: read deduplicate option from table config const bool force_ttl = false; @@ -3029,7 +3029,7 @@ void StorageReplicatedMergeTree::assertNotReadonly() const BlockOutputStreamPtr StorageReplicatedMergeTree::write(const ASTPtr & /*query*/, const Context & context) { - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); assertNotReadonly(); const Settings & query_settings = context.getSettingsRef(); @@ -3067,7 +3067,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p }; bool force_ttl = (final && (hasTableTTL() || hasAnyColumnTTL())); - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); if (!partition && final) { @@ -3948,7 +3948,7 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) { auto zookeeper = tryGetZooKeeper(); - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); res.is_leader = is_leader; res.can_become_leader = storage_settings->replicated_can_become_leader; @@ -4141,7 +4141,7 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t out_absolute_delay = getAbsoluteDelay(); out_relative_delay = 0; - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); /** Relative delay is the maximum difference of absolute delay from any other replica, * (if this replica lags behind any other live replica, or zero, otherwise). @@ -4969,7 +4969,7 @@ void StorageReplicatedMergeTree::getCommitPartOps( const String & block_id_path) const { const String & part_name = part->name; - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); if (!block_id_path.empty()) { @@ -5016,7 +5016,7 @@ void StorageReplicatedMergeTree::updatePartHeaderInZooKeeperAndCommit( AlterDataPartTransaction & transaction) { String part_path = replica_path + "/parts/" + transaction.getPartName(); - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); bool need_delete_columns_and_checksums_nodes = false; try @@ -5201,7 +5201,7 @@ CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, const C bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const { - const auto storage_settings = getCOWSettings(); + const auto storage_settings = getSettings(); return storage_settings->index_granularity_bytes != 0 && (storage_settings->enable_mixed_granularity_parts || (!has_non_adaptive_index_granularity_parts && !other_replicas_fixed_granularity)); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 5d8090a8cbc..20af4aeb19c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -540,7 +540,7 @@ protected: const ASTPtr & sample_by_ast_, const ASTPtr & table_ttl_ast_, const MergingParams & merging_params_, - MergeTreeSettingsPtr settings_, + std::unique_ptr settings_, bool has_force_restore_data_flag); }; From d17a64c28d26cb32f84f8e62a68614c81ba88fcd Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 26 Aug 2019 17:39:49 +0300 Subject: [PATCH 253/357] fix vulnerabilities --- dbms/src/Compression/CompressionCodecDelta.cpp | 12 +++++++++--- dbms/src/Compression/CompressionCodecDoubleDelta.cpp | 12 +++++++++--- dbms/src/Compression/CompressionCodecGorilla.cpp | 12 +++++++++--- dbms/src/Compression/CompressionCodecMultiple.cpp | 6 +++++- dbms/src/Compression/ICompressionCodec.cpp | 2 +- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/dbms/src/Compression/CompressionCodecDelta.cpp b/dbms/src/Compression/CompressionCodecDelta.cpp index 9f2397f8e59..849485312c6 100644 --- a/dbms/src/Compression/CompressionCodecDelta.cpp +++ b/dbms/src/Compression/CompressionCodecDelta.cpp @@ -80,7 +80,7 @@ UInt32 CompressionCodecDelta::doCompressData(const char * source, UInt32 source_ { UInt8 bytes_to_skip = source_size % delta_bytes_size; dest[0] = delta_bytes_size; - dest[1] = bytes_to_skip; + dest[1] = bytes_to_skip; /// unused (backward compatibility) memcpy(&dest[2], source, bytes_to_skip); size_t start_pos = 2 + bytes_to_skip; switch (delta_bytes_size) @@ -101,10 +101,16 @@ UInt32 CompressionCodecDelta::doCompressData(const char * source, UInt32 source_ return 1 + 1 + source_size; } -void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 /* uncompressed_size */) const +void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const { + if (source_size < 2) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); + UInt8 bytes_size = source[0]; - UInt8 bytes_to_skip = source[1]; + UInt8 bytes_to_skip = uncompressed_size % bytes_size; + + if (2 + bytes_to_skip > source_size) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); memcpy(dest, &source[2], bytes_to_skip); UInt32 source_size_no_header = source_size - bytes_to_skip - 2; diff --git a/dbms/src/Compression/CompressionCodecDoubleDelta.cpp b/dbms/src/Compression/CompressionCodecDoubleDelta.cpp index d3830dc9fdb..d0e948f821c 100644 --- a/dbms/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/dbms/src/Compression/CompressionCodecDoubleDelta.cpp @@ -304,7 +304,7 @@ UInt32 CompressionCodecDoubleDelta::doCompressData(const char * source, UInt32 s { UInt8 bytes_to_skip = source_size % data_bytes_size; dest[0] = data_bytes_size; - dest[1] = bytes_to_skip; + dest[1] = bytes_to_skip; /// unused (backward compatibility) memcpy(&dest[2], source, bytes_to_skip); size_t start_pos = 2 + bytes_to_skip; UInt32 compressed_size = 0; @@ -328,10 +328,16 @@ UInt32 CompressionCodecDoubleDelta::doCompressData(const char * source, UInt32 s return 1 + 1 + compressed_size; } -void CompressionCodecDoubleDelta::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 /* uncompressed_size */) const +void CompressionCodecDoubleDelta::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const { + if (source_size < 2) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); + UInt8 bytes_size = source[0]; - UInt8 bytes_to_skip = source[1]; + UInt8 bytes_to_skip = uncompressed_size % bytes_size; + + if (2 + bytes_to_skip > source_size) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); memcpy(dest, &source[2], bytes_to_skip); UInt32 source_size_no_header = source_size - bytes_to_skip - 2; diff --git a/dbms/src/Compression/CompressionCodecGorilla.cpp b/dbms/src/Compression/CompressionCodecGorilla.cpp index 8af6c8bfd39..f0d15804422 100644 --- a/dbms/src/Compression/CompressionCodecGorilla.cpp +++ b/dbms/src/Compression/CompressionCodecGorilla.cpp @@ -264,7 +264,7 @@ UInt32 CompressionCodecGorilla::doCompressData(const char * source, UInt32 sourc { UInt8 bytes_to_skip = source_size % data_bytes_size; dest[0] = data_bytes_size; - dest[1] = bytes_to_skip; + dest[1] = bytes_to_skip; /// unused (backward compatibility) memcpy(&dest[2], source, bytes_to_skip); size_t start_pos = 2 + bytes_to_skip; UInt32 result_size = 0; @@ -289,10 +289,16 @@ UInt32 CompressionCodecGorilla::doCompressData(const char * source, UInt32 sourc return 1 + 1 + result_size; } -void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 /* uncompressed_size */) const +void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const { + if (source_size < 2) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); + UInt8 bytes_size = source[0]; - UInt8 bytes_to_skip = source[1]; + UInt8 bytes_to_skip = uncompressed_size % bytes_size; + + if (2 + bytes_to_skip > source_size) + throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); memcpy(dest, &source[2], bytes_to_skip); UInt32 source_size_no_header = source_size - bytes_to_skip - 2; diff --git a/dbms/src/Compression/CompressionCodecMultiple.cpp b/dbms/src/Compression/CompressionCodecMultiple.cpp index 23c244f4dcb..fad72c932c3 100644 --- a/dbms/src/Compression/CompressionCodecMultiple.cpp +++ b/dbms/src/Compression/CompressionCodecMultiple.cpp @@ -88,6 +88,9 @@ void CompressionCodecMultiple::useInfoAboutType(DataTypePtr data_type) void CompressionCodecMultiple::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 decompressed_size) const { + if (source_size < 1 || !source[0]) + throw Exception("Wrong compression methods list", ErrorCodes::CORRUPTED_DATA); + UInt8 compression_methods_size = source[0]; PODArray compressed_buf(&source[compression_methods_size + 1], &source[source_size]); @@ -103,7 +106,8 @@ void CompressionCodecMultiple::doDecompressData(const char * source, UInt32 sour UInt32 uncompressed_size = ICompressionCodec::readDecompressedBlockSize(compressed_buf.data()); if (idx == 0 && uncompressed_size != decompressed_size) - throw Exception("Wrong final decompressed size in codec Multiple, got " + toString(uncompressed_size) + ", expected " + toString(decompressed_size), ErrorCodes::CORRUPTED_DATA); + throw Exception("Wrong final decompressed size in codec Multiple, got " + toString(uncompressed_size) + + ", expected " + toString(decompressed_size), ErrorCodes::CORRUPTED_DATA); uncompressed_buf.resize(uncompressed_size + codec->getAdditionalSizeAtTheEndOfBuffer()); codec->decompress(compressed_buf.data(), source_size, uncompressed_buf.data()); diff --git a/dbms/src/Compression/ICompressionCodec.cpp b/dbms/src/Compression/ICompressionCodec.cpp index aafca2f5eb3..f9d392e5d6d 100644 --- a/dbms/src/Compression/ICompressionCodec.cpp +++ b/dbms/src/Compression/ICompressionCodec.cpp @@ -47,7 +47,7 @@ UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, ch throw Exception("Can't decompress data with codec byte " + toString(method) + " from codec with byte " + toString(method), ErrorCodes::CANNOT_DECOMPRESS); UInt8 header_size = getHeaderSize(); - UInt32 decompressed_size = unalignedLoad(&source[5]); + UInt32 decompressed_size = readDecompressedBlockSize(source); doDecompressData(&source[header_size], source_size - header_size, dest, decompressed_size); return decompressed_size; From 0d0ebf885f1e75d5a129d2a63119281313f862c0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 26 Aug 2019 17:50:34 +0300 Subject: [PATCH 254/357] Remove two redundant arguments from alter --- dbms/src/Interpreters/InterpreterAlterQuery.cpp | 2 +- dbms/src/Storages/IStorage.cpp | 11 ++++++----- dbms/src/Storages/IStorage.h | 4 +--- dbms/src/Storages/Kafka/StorageKafka.cpp | 2 -- dbms/src/Storages/Kafka/StorageKafka.h | 2 -- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 7 ++++--- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 -- dbms/src/Storages/StorageBuffer.cpp | 5 ++++- dbms/src/Storages/StorageBuffer.h | 3 +-- dbms/src/Storages/StorageDistributed.cpp | 6 ++++-- dbms/src/Storages/StorageDistributed.h | 3 +-- dbms/src/Storages/StorageMerge.cpp | 5 ++--- dbms/src/Storages/StorageMerge.h | 3 +-- dbms/src/Storages/StorageMergeTree.cpp | 7 ++++--- dbms/src/Storages/StorageMergeTree.h | 4 +--- dbms/src/Storages/StorageNull.cpp | 6 ++++-- dbms/src/Storages/StorageNull.h | 3 +-- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 8 +++++--- dbms/src/Storages/StorageReplicatedMergeTree.h | 4 +--- 19 files changed, 41 insertions(+), 46 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index a32609bd53d..5798e182b69 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -104,7 +104,7 @@ BlockIO InterpreterAlterQuery::execute() { auto table_lock_holder = table->lockAlterIntention(context.getCurrentQueryId()); alter_commands.validate(*table, context); - table->alter(alter_commands, database_name, table_name, context, table_lock_holder); + table->alter(alter_commands, context, table_lock_holder); } return {}; diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 855336c06e5..b56c20f7372 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -384,11 +384,12 @@ TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id) void IStorage::alterSettings( const SettingsChanges & new_changes, - const String & current_database_name, - const String & current_table_name, const Context & context, TableStructureWriteLockHolder & /* table_lock_holder */) { + const String current_database_name = getDatabaseName(); + const String current_table_name = getTableName(); + IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) { if (!new_changes.empty()) @@ -416,16 +417,16 @@ void IStorage::alterSettings( void IStorage::alter( const AlterCommands & params, - const String & database_name, - const String & table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { + const String database_name = getDatabaseName(); + const String table_name = getTableName(); if (params.isSettingsAlter()) { SettingsChanges new_changes; params.applyForSettingsOnly(new_changes); - alterSettings(new_changes, database_name, table_name, context, table_lock_holder); + alterSettings(new_changes, context, table_lock_holder); return; } diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index b27ab89dcbc..30aa2b41d0d 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -283,7 +283,7 @@ public: * This method must fully execute the ALTER query, taking care of the locks itself. * To update the table metadata on disk, this method should call InterpreterAlterQuery::updateMetadata. */ - virtual void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder); + virtual void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder); /** ALTER tables with regard to its partitions. * Should handle locks for each command on its own. @@ -297,8 +297,6 @@ public: */ virtual void alterSettings( const SettingsChanges & new_changes, - const String & current_database_name, - const String & current_table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder); diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index f00d273c6a4..45ca4aa6cf7 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -414,8 +414,6 @@ bool StorageKafka::hasSetting(const String & setting_name) const void StorageKafka::alterSettings( const SettingsChanges & /* new_changes */, - const String & /* current_database_name */, - const String & /* current_table_name */, const Context & /* context */, TableStructureWriteLockHolder & /* table_lock_holder */) { diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index a6039eac254..27f1e7e0ec7 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -61,8 +61,6 @@ public: void alterSettings( const SettingsChanges & new_changes, - const String & current_database_name, - const String & current_table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index f10c2067e44..448f5afcc86 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1648,14 +1648,15 @@ void MergeTreeData::alterDataPart( void MergeTreeData::alterSettings( const SettingsChanges & new_changes, - const String & current_database_name, - const String & current_table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { + const String current_database_name = getDatabaseName(); + const String current_table_name = getTableName(); + MergeTreeSettings copy = *getSettings(); copy.updateFromChanges(new_changes); - IStorage::alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); + IStorage::alterSettings(new_changes, context, table_lock_holder); storage_settings.set(std::make_unique(copy)); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 8babd723fae..f759b87f986 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -541,8 +541,6 @@ public: /// Not atomic, have to be done with alter intention lock. void alterSettings( const SettingsChanges & new_changes, - const String & current_database_name, - const String & current_table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index dd6ecf7f261..1cbf557af9b 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -692,10 +692,13 @@ void StorageBuffer::flushThread() } -void StorageBuffer::alter(const AlterCommands & params, const String & database_name_, const String & table_name_, const Context & context, TableStructureWriteLockHolder & table_lock_holder) +void StorageBuffer::alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + const String database_name_ = getDatabaseName(); + const String table_name_ = getTableName(); + /// So that no blocks of the old structure remain. optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context); diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index 4c317a7a102..8eecbaa17ec 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -91,8 +91,7 @@ public: /// The structure of the subordinate table is not checked and does not change. void alter( - const AlterCommands & params, const String & database_name, const String & table_name, - const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; + const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; private: String table_name; diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 9bb68a6b13f..83d4e4fc554 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -383,11 +383,13 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Context & c void StorageDistributed::alter( - const AlterCommands & params, const String & current_database_name, const String & current_table_name, - const Context & context, TableStructureWriteLockHolder & table_lock_holder) + const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + const String current_database_name = getDatabaseName(); + const String current_table_name = getTableName(); + auto new_columns = getColumns(); auto new_indices = getIndices(); auto new_constraints = getConstraints(); diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index 6885a758e9e..86cb0fef92c 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -85,8 +85,7 @@ public: /// in the sub-tables, you need to manually add and delete columns /// the structure of the sub-table is not checked void alter( - const AlterCommands & params, const String & database_name, const String & table_name, - const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; + const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; void startup() override; void shutdown() override; diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 3005bf7dd52..2b0e5413c26 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -396,8 +396,7 @@ DatabaseIteratorPtr StorageMerge::getDatabaseIterator(const Context & context) c void StorageMerge::alter( - const AlterCommands & params, const String & database_name_, const String & table_name_, - const Context & context, TableStructureWriteLockHolder & table_lock_holder) + const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); @@ -405,7 +404,7 @@ void StorageMerge::alter( auto new_indices = getIndices(); auto new_constraints = getConstraints(); params.applyForColumnsOnly(new_columns); - context.getDatabase(database_name_)->alterTable(context, table_name_, new_columns, new_indices, new_constraints, {}); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); setColumns(new_columns); } diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index 6d02ad029cc..7ccf072ac12 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -47,8 +47,7 @@ public: /// you need to add and remove columns in the sub-tables manually /// the structure of sub-tables is not checked void alter( - const AlterCommands & params, const String & database_name, const String & table_name, - const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; + const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const override; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index c0caa7ff1e0..1504413801c 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -245,11 +245,12 @@ std::vector StorageMergeTree::prepar void StorageMergeTree::alter( const AlterCommands & params, - const String & current_database_name, - const String & current_table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { + const String current_database_name = getDatabaseName(); + const String current_table_name = getTableName(); + if (!params.isMutable()) { SettingsChanges new_changes; @@ -257,7 +258,7 @@ void StorageMergeTree::alter( if (params.isSettingsAlter()) { params.applyForSettingsOnly(new_changes); - alterSettings(new_changes, current_database_name, current_table_name, context, table_lock_holder); + alterSettings(new_changes, context, table_lock_holder); return; } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index df1cd1682b9..bf11a9a0335 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -59,9 +59,7 @@ public: void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; - void alter( - const AlterCommands & params, const String & database_name, const String & table_name, - const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; + void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; void checkTableCanBeDropped() const override; diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index d610ff45212..b419c35f964 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -31,11 +31,13 @@ void registerStorageNull(StorageFactory & factory) } void StorageNull::alter( - const AlterCommands & params, const String & current_database_name, const String & current_table_name, - const Context & context, TableStructureWriteLockHolder & table_lock_holder) + const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + const String current_database_name = getDatabaseName(); + const String current_table_name = getTableName(); + ColumnsDescription new_columns = getColumns(); IndicesDescription new_indices = getIndices(); ConstraintsDescription new_constraints = getConstraints(); diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index fec7638706f..e4fcd7991e5 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -44,8 +44,7 @@ public: } void alter( - const AlterCommands & params, const String & database_name, const String & table_name, - const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; + const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; private: String table_name; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 2bf8e26112f..fa7ab961ff9 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3142,13 +3142,15 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p void StorageReplicatedMergeTree::alter( - const AlterCommands & params, const String & current_database_name, const String & current_table_name, - const Context & query_context, TableStructureWriteLockHolder & table_lock_holder) + const AlterCommands & params, const Context & query_context, TableStructureWriteLockHolder & table_lock_holder) { assertNotReadonly(); LOG_DEBUG(log, "Doing ALTER"); + const String current_database_name = getDatabaseName(); + const String current_table_name = getTableName(); + if (params.isSettingsAlter()) { /// We don't replicate storage_settings ALTER. It's local operation. @@ -3156,7 +3158,7 @@ void StorageReplicatedMergeTree::alter( LOG_DEBUG(log, "ALTER storage_settings only"); SettingsChanges new_changes; params.applyForSettingsOnly(new_changes); - alterSettings(new_changes, current_database_name, current_table_name, query_context, table_lock_holder); + alterSettings(new_changes, query_context, table_lock_holder); return; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 20af4aeb19c..60ce6623b1a 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -98,9 +98,7 @@ public: bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & query_context) override; - void alter( - const AlterCommands & params, const String & database_name, const String & table_name, - const Context & query_context, TableStructureWriteLockHolder & table_lock_holder) override; + void alter(const AlterCommands & params, const Context & query_context, TableStructureWriteLockHolder & table_lock_holder) override; void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) override; From 305f3ba47955670ebae21dcd44fd48799122fbe2 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 26 Aug 2019 19:09:30 +0300 Subject: [PATCH 255/357] DOCAPI-7428: domain function docs. EN review, RU translation. (#6624) * Update url_functions.md * DOCAPI-7428: RU translation * DOCAPI-7428: Fix. * Update docs/en/query_language/functions/url_functions.md Co-Authored-By: Ivan Blinkov * Update docs/en/query_language/functions/url_functions.md Co-Authored-By: Ivan Blinkov * Update docs/ru/query_language/functions/url_functions.md Co-Authored-By: Ivan Blinkov * Update docs/ru/query_language/functions/url_functions.md Co-Authored-By: Ivan Blinkov * Update docs/ru/query_language/functions/url_functions.md Co-Authored-By: Ivan Blinkov * DOCAPI-7428: Fixes. --- .../query_language/functions/url_functions.md | 32 ++++--- .../query_language/functions/url_functions.md | 87 ++++++++++++++++++- 2 files changed, 105 insertions(+), 14 deletions(-) diff --git a/docs/en/query_language/functions/url_functions.md b/docs/en/query_language/functions/url_functions.md index a21c2cfa0bf..f4ff63ce021 100644 --- a/docs/en/query_language/functions/url_functions.md +++ b/docs/en/query_language/functions/url_functions.md @@ -2,17 +2,19 @@ All these functions don't follow the RFC. They are maximally simplified for improved performance. -## Functions that extract part of a URL +## Functions that Extract Parts of a URL -If there isn't anything similar in a URL, an empty string is returned. +If the relevant part isn't present in a URL, an empty string is returned. ### protocol -Returns the protocol. Examples: http, ftp, mailto, magnet... +Extracts the protocol from a URL. + +Examples of typical returned values: http, https, ftp, mailto, tel, magnet... ### domain -Extracts the host part from URL. +Extracts the hostname from a URL. ``` domain(url) @@ -23,7 +25,7 @@ domain(url) - `url` — URL. Type: [String](../../data_types/string.md). -URL can be specified with or without scheme. Examples: +The URL can be specified with or without a scheme. Examples: ``` svn+ssh://some.svn-hosting.com:80/repo/trunk @@ -31,10 +33,18 @@ some.svn-hosting.com:80/repo/trunk https://yandex.com/time/ ``` +For these examples, the `domain` function returns the following results: + +``` +some.svn-hosting.com +some.svn-hosting.com +yandex.com +``` + **Returned values** -- Host name. If ClickHouse can parse input string as URL. -- Empty string. If ClickHouse cannot parse input string as URL. +- Host name. If ClickHouse can parse the input string as a URL. +- Empty string. If ClickHouse can't parse the input string as a URL. Type: `String`. @@ -55,7 +65,7 @@ Returns the domain and removes no more than one 'www.' from the beginning of it, ### topLevelDomain -Extracts the the top-level domain from URL. +Extracts the the top-level domain from a URL. ``` topLevelDomain(url) @@ -65,7 +75,7 @@ topLevelDomain(url) - `url` — URL. Type: [String](../../data_types/string.md). -URL can be specified with or without scheme. Examples: +The URL can be specified with or without a scheme. Examples: ``` svn+ssh://some.svn-hosting.com:80/repo/trunk @@ -75,8 +85,8 @@ https://yandex.com/time/ **Returned values** -- Domain name. If ClickHouse can parse input string as URL. -- Empty string. If ClickHouse cannot parse input string as URL. +- Domain name. If ClickHouse can parse the input string as a URL. +- Empty string. If ClickHouse cannot parse the input string as a URL. Type: `String`. diff --git a/docs/ru/query_language/functions/url_functions.md b/docs/ru/query_language/functions/url_functions.md index 1897d1b28a3..7002273d5cb 100644 --- a/docs/ru/query_language/functions/url_functions.md +++ b/docs/ru/query_language/functions/url_functions.md @@ -10,13 +10,94 @@ Возвращает протокол. Примеры: http, ftp, mailto, magnet... ### domain -Возвращает домен. Отсекает схему размером не более 16 байт. + +Извлекает имя хоста из URL. + +``` +domain(url) +``` + +**Параметры** + +- `url` — URL. Тип — [String](../../data_types/string.md). + +URL может быть указан со схемой или без неё. Примеры: + +``` +svn+ssh://some.svn-hosting.com:80/repo/trunk +some.svn-hosting.com:80/repo/trunk +https://yandex.com/time/ +``` + +Для указанных примеров функция `domain` возвращает следующие результаты: + +``` +some.svn-hosting.com +some.svn-hosting.com +yandex.com +``` + +**Возвращаемые значения** + +- Имя хоста. Если ClickHouse может распарсить входную строку как URL. +- Пустая строка. Если ClickHouse не может распарсить входную строку как URL. + +Тип — `String`. + +**Пример** + +```sql +SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk') +``` + +```text +┌─domain('svn+ssh://some.svn-hosting.com:80/repo/trunk')─┐ +│ some.svn-hosting.com │ +└────────────────────────────────────────────────────────┘ +``` ### domainWithoutWWW -Возвращает домен, удалив не более одного 'www.' с начала, если есть. + +Возвращает домен, удалив префикс 'www.', если он присутствовал. ### topLevelDomain -Возвращает домен верхнего уровня. Пример: .ru. + +Извлекает домен верхнего уровня из URL. + +``` +topLevelDomain(url) +``` + +**Параметры** + +- `url` — URL. Тип — [String](../../data_types/string.md). + +URL может быть указан со схемой или без неё. Примеры: + +``` +svn+ssh://some.svn-hosting.com:80/repo/trunk +some.svn-hosting.com:80/repo/trunk +https://yandex.com/time/ +``` + +**Возвращаемые значения** + +- Имя домена. Если ClickHouse может распарсить входную строку как URL. +- Пустая строка. Если ClickHouse не может распарсить входную строку как URL. + +Тип — `String`. + +**Пример** + +```sql +SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk') +``` + +```text +┌─topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk')─┐ +│ com │ +└────────────────────────────────────────────────────────────────────┘ +``` ### firstSignificantSubdomain Возвращает "первый существенный поддомен". Это понятие является нестандартным и специфично для Яндекс.Метрики. Первый существенный поддомен - это домен второго уровня, если он не равен одному из com, net, org, co, или домен третьего уровня, иначе. Например, firstSignificantSubdomain('') = 'yandex', firstSignificantSubdomain('') = 'yandex'. Список "несущественных" доменов второго уровня и другие детали реализации могут изменяться в будущем. From 1da261576e3d36393e8fab1a59ff21a7fd67d96c Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Mon, 26 Aug 2019 19:42:20 +0300 Subject: [PATCH 256/357] Try fix2 --- dbms/src/Common/TerminalSize.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Common/TerminalSize.cpp b/dbms/src/Common/TerminalSize.cpp index 7f6bb93119f..5f0036b86ce 100644 --- a/dbms/src/Common/TerminalSize.cpp +++ b/dbms/src/Common/TerminalSize.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB::ErrorCodes From 12084270dedecbc6e9856eaa0ae5e2f9a7e1bded Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 26 Aug 2019 19:58:40 +0300 Subject: [PATCH 257/357] fix build --- dbms/src/Compression/CompressionCodecDelta.cpp | 2 +- dbms/src/Compression/CompressionCodecDoubleDelta.cpp | 2 +- dbms/src/Compression/CompressionCodecGorilla.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Compression/CompressionCodecDelta.cpp b/dbms/src/Compression/CompressionCodecDelta.cpp index 849485312c6..abf0b79f9c8 100644 --- a/dbms/src/Compression/CompressionCodecDelta.cpp +++ b/dbms/src/Compression/CompressionCodecDelta.cpp @@ -109,7 +109,7 @@ void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_ UInt8 bytes_size = source[0]; UInt8 bytes_to_skip = uncompressed_size % bytes_size; - if (2 + bytes_to_skip > source_size) + if (UInt32(2 + bytes_to_skip) > source_size) throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); memcpy(dest, &source[2], bytes_to_skip); diff --git a/dbms/src/Compression/CompressionCodecDoubleDelta.cpp b/dbms/src/Compression/CompressionCodecDoubleDelta.cpp index d0e948f821c..9a6b551f159 100644 --- a/dbms/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/dbms/src/Compression/CompressionCodecDoubleDelta.cpp @@ -336,7 +336,7 @@ void CompressionCodecDoubleDelta::doDecompressData(const char * source, UInt32 s UInt8 bytes_size = source[0]; UInt8 bytes_to_skip = uncompressed_size % bytes_size; - if (2 + bytes_to_skip > source_size) + if (UInt32(2 + bytes_to_skip) > source_size) throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); memcpy(dest, &source[2], bytes_to_skip); diff --git a/dbms/src/Compression/CompressionCodecGorilla.cpp b/dbms/src/Compression/CompressionCodecGorilla.cpp index f0d15804422..574e40b06bf 100644 --- a/dbms/src/Compression/CompressionCodecGorilla.cpp +++ b/dbms/src/Compression/CompressionCodecGorilla.cpp @@ -297,7 +297,7 @@ void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 sourc UInt8 bytes_size = source[0]; UInt8 bytes_to_skip = uncompressed_size % bytes_size; - if (2 + bytes_to_skip > source_size) + if (UInt32(2 + bytes_to_skip) > source_size) throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS); memcpy(dest, &source[2], bytes_to_skip); From a98b91c521b1ee2c5c826df98cb10fdbfc4f2424 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 26 Aug 2019 20:48:48 +0300 Subject: [PATCH 258/357] DOCAPI-7436: Enum type docs. EN review, RU translation. (#6630) * Update enum.md (#33) * DOCAPI-7436: RU translation. * Update docs/en/data_types/enum.md Co-Authored-By: Ivan Blinkov * Update docs/ru/data_types/enum.md Co-Authored-By: Ivan Blinkov * DOCAPI-7436: Fixes. --- docs/en/data_types/enum.md | 12 ++++++---- docs/ru/data_types/enum.md | 48 +++++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md index 3fbe5b3708b..692cbad39b1 100644 --- a/docs/en/data_types/enum.md +++ b/docs/en/data_types/enum.md @@ -1,13 +1,15 @@ # Enum -Enumerated type storing pairs of the `'string' = integer` format. +Enumerated type consisting of named values. + +Named values must be delcared as `'string' = integer` pairs. ClickHouse stores only numbers, but supports operations with the values through their names. ClickHouse supports: -- 8-bit `Enum`. It can contain up to 256 values with enumeration of `[-128, 127]`. -- 16-bit `Enum`. It can contain up to 65536 values with enumeration of `[-32768, 32767]`. +- 8-bit `Enum`. It can contain up to 256 values enumerated in the `[-128, 127]` range. +- 16-bit `Enum`. It can contain up to 65536 values enumerated in the `[-32768, 32767]` range. -ClickHouse automatically chooses a type for `Enum` at data insertion. Also, you can use `Enum8` or `Enum16` types to be sure in size of storage. +ClickHouse automatically chooses the type of `Enum` when data is inserted. You can also use `Enum8` or `Enum16` types to be sure in the size of storage. ## Usage examples @@ -21,7 +23,7 @@ CREATE TABLE t_enum ENGINE = TinyLog ``` -This column `x` can only store the values that are listed in the type definition: `'hello'` or `'world'`. If you try to save any other value, ClickHouse will generate an exception. ClickHouse automatically chooses the 8-bit size for enumeration of this `Enum`. +Column `x` can only store values that are listed in the type definition: `'hello'` or `'world'`. If you try to save any other value, ClickHouse will raise an exception. 8-bit size for this `Enum` is chosen automatically. ```sql :) INSERT INTO t_enum VALUES ('hello'), ('world'), ('hello') diff --git a/docs/ru/data_types/enum.md b/docs/ru/data_types/enum.md index 9191dc5d2b0..8d32cce1648 100644 --- a/docs/ru/data_types/enum.md +++ b/docs/ru/data_types/enum.md @@ -1,27 +1,32 @@ +# Enum -# Enum8, Enum16 +Перечисляемый тип данных, содержащий именованные значения. -Включает в себя типы `Enum8` и `Enum16`. `Enum` сохраняет конечный набор пар `'строка' = целое число`. Все операции с данными типа `Enum` ClickHouse выполняет как с числами, однако пользователь при этом работает со строковыми константами. Это более эффективно с точки зрения производительности, чем работа с типом данных `String`. +Именованные значения задаются парами `'string' = integer`. ClickHouse хранить только числа, но допускает операции над ними с помощью заданных имён. -- `Enum8` описывается парами `'String' = Int8`. -- `Enum16` описывается парами `'String' = Int16`. +ClickHouse поддерживает: -## Примеры применения +- 8-битный `Enum`. Может содержать до 256 значений, пронумерованных в диапазоне `[-128, 127]`. +- 16-битный `Enum`. Может содержать до 65536 значений, пронумерованных в диапазоне `[-32768, 32767]`. + +ClickHouse автоматически выбирает размерность `Enum` при вставке данных. Чтобы точно понимать размер хранимых данных можно использовать типы `Enum8` или `Enum16`. + +## Примеры использования Создадим таблицу со столбцом типа `Enum8('hello' = 1, 'world' = 2)`. -``` +```sql CREATE TABLE t_enum ( - x Enum8('hello' = 1, 'world' = 2) + x Enum('hello' = 1, 'world' = 2) ) ENGINE = TinyLog ``` -В столбец `x` можно сохранять только значения, перечисленные при определении типа, т.е. `'hello'` или `'world'`. Если попытаться сохранить другое значение, ClickHouse сгенерирует исключение. +В столбец `x` можно сохранять только значения, перечисленные при определении типа, т.е. `'hello'` или `'world'`. Если вы попытаетесь сохранить любое другое значение, ClickHouse сгенерирует исключение. ClickHouse автоматически выберет размерность 8-bit для этого `Enum`. -``` -:) INSERT INTO t_enum Values('hello'),('world'),('hello') +```sql +:) INSERT INTO t_enum VALUES ('hello'), ('world'), ('hello') INSERT INTO t_enum VALUES @@ -35,12 +40,12 @@ INSERT INTO t_enum VALUES Exception on client: -Code: 49. DB::Exception: Unknown element 'a' for type Enum8('hello' = 1, 'world' = 2) +Code: 49. DB::Exception: Unknown element 'a' for type Enum('hello' = 1, 'world' = 2) ``` При запросе данных из таблицы ClickHouse выдаст строковые значения из `Enum`. -``` +```sql SELECT * FROM t_enum ┌─x─────┐ @@ -49,10 +54,11 @@ SELECT * FROM t_enum │ hello │ └───────┘ ``` -Если необходимо увидеть цифровые эквиваленты строкам, то необходимо привести тип. -``` -SELECT CAST(x, 'Int8') FROM t_enum +Если необходимо увидеть цифровые эквиваленты строкам, то необходимо привести тип `Enum` к целочисленному. + +```sql +SELECT CAST(x AS Int8) FROM t_enum ┌─CAST(x, 'Int8')─┐ │ 1 │ @@ -61,14 +67,14 @@ SELECT CAST(x, 'Int8') FROM t_enum └─────────────────┘ ``` -Чтобы создать значение типа Enum в запросе, также необходима функция `CAST`. +Чтобы создать значение типа `Enum` в запросе, также необходимо использовать функцию `CAST`. -``` -SELECT toTypeName(CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')) +```sql +SELECT toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)')) -┌─toTypeName(CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)'))─┐ -│ Enum8('a' = 1, 'b' = 2) │ -└──────────────────────────────────────────────────────┘ +┌─toTypeName(CAST('a', 'Enum(\'a\' = 1, \'b\' = 2)'))─┐ +│ Enum8('a' = 1, 'b' = 2) │ +└─────────────────────────────────────────────────────┘ ``` ## Общие правила и особенности использования From 4cd376f5876c528820ff8b143422da74b64291ea Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 26 Aug 2019 21:00:13 +0300 Subject: [PATCH 259/357] fix table ttl after bad merge with master --- dbms/src/DataStreams/TTLBlockInputStream.cpp | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/dbms/src/DataStreams/TTLBlockInputStream.cpp b/dbms/src/DataStreams/TTLBlockInputStream.cpp index 752f9800aa3..e2a3a7ca03b 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.cpp +++ b/dbms/src/DataStreams/TTLBlockInputStream.cpp @@ -70,7 +70,7 @@ bool TTLBlockInputStream::isTTLExpired(time_t ttl) Block TTLBlockInputStream::readImpl() { /// Skip all data if table ttl is expired for part - if (storage.hasTableTTL() && old_ttl_infos.table_ttl.max <= current_time) + if (storage.hasTableTTL() && isTTLExpired(old_ttl_infos.table_ttl.max)) { rows_removed = data_part->rows_count; return {}; @@ -80,18 +80,8 @@ Block TTLBlockInputStream::readImpl() if (!block) return block; - if (storage.hasTableTTL()) - { - /// Skip all data if table ttl is expired for part - if (isTTLExpired(old_ttl_infos.table_ttl.max)) - { - rows_removed = data_part->rows_count; - return {}; - } - - if (force || isTTLExpired(old_ttl_infos.table_ttl.min)) - removeRowsWithExpiredTableTTL(block); - } + if (storage.hasTableTTL() && (force || isTTLExpired(old_ttl_infos.table_ttl.min))) + removeRowsWithExpiredTableTTL(block); removeValuesWithExpiredColumnTTL(block); From c0bb52ca6d3af6d4e6513ed1c9df167998535b6c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 19 Aug 2019 20:03:57 +0300 Subject: [PATCH 260/357] Separate the hash table interface for ReverseIndex. It is significantly different from other uses of hash tables, and poses the main obstacle to changing the hash table interface to the one that can be easily supported by compound hash tables. Make it explicitly separate, implement it only for a particular kind of HashTable, and move this implementation to the ReverseIndex module. --- dbms/src/Columns/ReverseIndex.h | 87 ++++++++++++++++++++-- dbms/src/Common/HashTable/FixedHashTable.h | 12 +-- dbms/src/Common/HashTable/HashTable.h | 31 ++------ 3 files changed, 88 insertions(+), 42 deletions(-) diff --git a/dbms/src/Columns/ReverseIndex.h b/dbms/src/Columns/ReverseIndex.h index 8fa4e87680b..09f4f0d7f34 100644 --- a/dbms/src/Columns/ReverseIndex.h +++ b/dbms/src/Columns/ReverseIndex.h @@ -81,6 +81,8 @@ namespace template static bool isZero(const T &, const State & /*state*/) { + /// Careful: apparently this uses SFINAE to redefine isZero for all types + /// except the IndexType, for which the default ZeroTraits::isZero is used. static_assert(!std::is_same_v::type, typename std::decay::type>); return false; } @@ -121,19 +123,88 @@ namespace }; + /** + * ReverseIndexHashTableBase implements a special hash table interface for + * reverse index. + * + * The following requirements are different compared to a plain hash table: + * + * 1) Provide public access to 'hash table state' that contains + * additional data needed to calculate cell hashes. + * + * 2) Support emplace() and find() with a Key different from the resulting + * hash table key. This means emplace() accepts a different kind of object + * as a key, and then the real key can be read from the returned cell iterator. + * + * These requirements are unique to ReverseIndex and are in conflict with + * supporting hash tables that use alternative key storage, such as FixedHashMap + * or StringHashMap. Therefore, we implement an interface for ReverseIndex + * separately. + */ template - class HashTableWithPublicState : public HashTable, HashTableAllocator> + class ReverseIndexHashTableBase : public HashTable, HashTableAllocator> { using State = typename Cell::State; using Base = HashTable, HashTableAllocator>; public: using Base::Base; + using iterator = typename Base::iterator; State & getState() { return *this; } + + + template + size_t ALWAYS_INLINE reverseIndexFindCell(const ObjectToCompareWith & x, + size_t hash_value, size_t place_value) const + { + while (!this->buf[place_value].isZero(*this) + && !this->buf[place_value].keyEquals(x, hash_value, *this)) + { + place_value = this->grower.next(place_value); + } + + return place_value; + } + + template + void ALWAYS_INLINE reverseIndexEmplaceNonZero(const Key & key, iterator & it, + bool & inserted, size_t hash_value, const ObjectToCompareWith & object) + { + size_t place_value = reverseIndexFindCell(object, hash_value, + this->grower.place(hash_value)); + // emplaceNonZeroImpl() might need to re-find the cell if the table grows, + // but it will find it correctly by the key alone, so we don't have to + // pass it the 'object'. + this->emplaceNonZeroImpl(place_value, key, it, inserted, hash_value); + } + + /// Searches position by object. + template + void ALWAYS_INLINE reverseIndexEmplace(Key key, iterator & it, bool & inserted, + size_t hash_value, const ObjectToCompareWith& object) + { + if (!this->emplaceIfZero(key, it, inserted, hash_value)) + { + reverseIndexEmplaceNonZero(key, it, inserted, hash_value, object); + } + } + + template + iterator ALWAYS_INLINE reverseIndexFind(ObjectToCompareWith x, size_t hash_value) + { + if (Cell::isZero(x, *this)) + return this->hasZero() ? this->iteratorToZero() : this->end(); + + size_t place_value = reverseIndexFindCell(x, hash_value, + this->grower.place(hash_value)); + return !this->buf[place_value].isZero(*this) + ? iterator(this, &this->buf[place_value]) + : this->end(); + } }; template - class ReverseIndexStringHashTable : public HashTableWithPublicState< + class ReverseIndexStringHashTable : public ReverseIndexHashTableBase< IndexType, ReverseIndexHashTableCell< IndexType, @@ -144,7 +215,7 @@ namespace has_base_index>, ReverseIndexHash> { - using Base = HashTableWithPublicState< + using Base = ReverseIndexHashTableBase< IndexType, ReverseIndexHashTableCell< IndexType, @@ -166,7 +237,7 @@ namespace }; template - class ReverseIndexNumberHashTable : public HashTableWithPublicState< + class ReverseIndexNumberHashTable : public ReverseIndexHashTableBase< IndexType, ReverseIndexHashTableCell< IndexType, @@ -177,7 +248,7 @@ namespace has_base_index>, ReverseIndexHash> { - using Base = HashTableWithPublicState< + using Base = ReverseIndexHashTableBase< IndexType, ReverseIndexHashTableCell< IndexType, @@ -356,7 +427,7 @@ void ReverseIndex::buildIndex() else hash = getHash(column->getDataAt(row)); - index->emplace(row + base_index, iterator, inserted, hash, column->getDataAt(row)); + index->reverseIndexEmplace(row + base_index, iterator, inserted, hash, column->getDataAt(row)); if (!inserted) throw Exception("Duplicating keys found in ReverseIndex.", ErrorCodes::LOGICAL_ERROR); @@ -401,7 +472,7 @@ UInt64 ReverseIndex::insert(const StringRef & data) else column->insertData(data.data, data.size); - index->emplace(num_rows + base_index, iterator, inserted, hash, data); + index->reverseIndexEmplace(num_rows + base_index, iterator, inserted, hash, data); if constexpr (use_saved_hash) { @@ -427,7 +498,7 @@ UInt64 ReverseIndex::getInsertionPoint(const StringRef & IteratorType iterator; auto hash = getHash(data); - iterator = index->find(data, hash); + iterator = index->reverseIndexFind(data, hash); return iterator == index->end() ? size() + base_index : iterator->getValue(); } diff --git a/dbms/src/Common/HashTable/FixedHashTable.h b/dbms/src/Common/HashTable/FixedHashTable.h index 663848865da..0c3a2c1237a 100644 --- a/dbms/src/Common/HashTable/FixedHashTable.h +++ b/dbms/src/Common/HashTable/FixedHashTable.h @@ -294,26 +294,22 @@ public: void ALWAYS_INLINE emplace(Key x, iterator & it, bool & inserted) { emplaceImpl(x, it, inserted); } void ALWAYS_INLINE emplace(Key x, iterator & it, bool & inserted, size_t) { emplaceImpl(x, it, inserted); } - template - iterator ALWAYS_INLINE find(ObjectToCompareWith x) + iterator ALWAYS_INLINE find(Key x) { return !buf[x].isZero(*this) ? iterator(this, &buf[x]) : end(); } - template - const_iterator ALWAYS_INLINE find(ObjectToCompareWith x) const + const_iterator ALWAYS_INLINE find(Key x) const { return !buf[x].isZero(*this) ? const_iterator(this, &buf[x]) : end(); } - template - iterator ALWAYS_INLINE find(ObjectToCompareWith, size_t hash_value) + iterator ALWAYS_INLINE find(Key, size_t hash_value) { return !buf[hash_value].isZero(*this) ? iterator(this, &buf[hash_value]) : end(); } - template - const_iterator ALWAYS_INLINE find(ObjectToCompareWith, size_t hash_value) const + const_iterator ALWAYS_INLINE find(Key, size_t hash_value) const { return !buf[hash_value].isZero(*this) ? const_iterator(this, &buf[hash_value]) : end(); } diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index f15fe22f8fd..c5a0c812ee2 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -280,8 +280,7 @@ protected: #endif /// Find a cell with the same key or an empty cell, starting from the specified position and further along the collision resolution chain. - template - size_t ALWAYS_INLINE findCell(const ObjectToCompareWith & x, size_t hash_value, size_t place_value) const + size_t ALWAYS_INLINE findCell(const Key & x, size_t hash_value, size_t place_value) const { while (!buf[place_value].isZero(*this) && !buf[place_value].keyEquals(x, hash_value, *this)) { @@ -700,13 +699,6 @@ protected: emplaceNonZeroImpl(place_value, x, it, inserted, hash_value); } - /// Same but find place using object. Hack for ReverseIndex. - template - void ALWAYS_INLINE emplaceNonZero(Key x, iterator & it, bool & inserted, size_t hash_value, const ObjectToCompareWith & object) - { - size_t place_value = findCell(object, hash_value, grower.place(hash_value)); - emplaceNonZeroImpl(place_value, x, it, inserted, hash_value); - } public: @@ -763,14 +755,6 @@ public: emplaceNonZero(x, it, inserted, hash_value); } - /// Same, but search position by object. Hack for ReverseIndex. - template - void ALWAYS_INLINE emplace(Key x, iterator & it, bool & inserted, size_t hash_value, const ObjectToCompareWith & object) - { - if (!emplaceIfZero(x, it, inserted, hash_value)) - emplaceNonZero(x, it, inserted, hash_value, object); - } - /// Copy the cell from another hash table. It is assumed that the cell is not zero, and also that there was no such key in the table yet. void ALWAYS_INLINE insertUniqueNonZero(const Cell * cell, size_t hash_value) { @@ -783,9 +767,7 @@ public: resize(); } - - template - iterator ALWAYS_INLINE find(ObjectToCompareWith x) + iterator ALWAYS_INLINE find(Key x) { if (Cell::isZero(x, *this)) return this->hasZero() ? iteratorToZero() : end(); @@ -796,8 +778,7 @@ public: } - template - const_iterator ALWAYS_INLINE find(ObjectToCompareWith x) const + const_iterator ALWAYS_INLINE find(Key x) const { if (Cell::isZero(x, *this)) return this->hasZero() ? iteratorToZero() : end(); @@ -808,8 +789,7 @@ public: } - template - iterator ALWAYS_INLINE find(ObjectToCompareWith x, size_t hash_value) + iterator ALWAYS_INLINE find(Key x, size_t hash_value) { if (Cell::isZero(x, *this)) return this->hasZero() ? iteratorToZero() : end(); @@ -819,8 +799,7 @@ public: } - template - const_iterator ALWAYS_INLINE find(ObjectToCompareWith x, size_t hash_value) const + const_iterator ALWAYS_INLINE find(Key x, size_t hash_value) const { if (Cell::isZero(x, *this)) return this->hasZero() ? iteratorToZero() : end(); From 0c75d5f42b3bff0da65cfee2e1d5aac6ad682eb7 Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Mon, 26 Aug 2019 22:03:47 +0300 Subject: [PATCH 261/357] Function transformQueryForExternalDatabase() fixed --- dbms/src/Storages/transformQueryForExternalDatabase.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.cpp b/dbms/src/Storages/transformQueryForExternalDatabase.cpp index 55a0ef95200..7cbffc46e67 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.cpp +++ b/dbms/src/Storages/transformQueryForExternalDatabase.cpp @@ -141,16 +141,12 @@ String transformQueryForExternalDatabase( if (function->name == "and") { bool compatible_found = false; - auto new_function_and = std::make_shared(); - auto new_function_and_arguments = std::make_shared(); - new_function_and->arguments = new_function_and_arguments; - new_function_and->children.push_back(new_function_and_arguments); - + auto new_function_and = makeASTFunction("and"); for (const auto & elem : function->arguments->children) { if (isCompatible(*elem)) { - new_function_and_arguments->children.push_back(elem); + new_function_and->arguments->children.push_back(elem); compatible_found = true; } } From 0eec226c9a163ae536a0733f3a852988c90c1032 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Aug 2019 22:07:29 +0300 Subject: [PATCH 262/357] Simplify shared_ptr_helper --- .../tests/in_join_subqueries_preprocessor.cpp | 1 + dbms/src/Storages/Kafka/StorageKafka.h | 1 + .../MergeTree/StorageFromMergeTreeDataPart.h | 1 + dbms/src/Storages/StorageBuffer.h | 5 +++-- dbms/src/Storages/StorageDictionary.h | 1 + dbms/src/Storages/StorageDistributed.cpp | 10 ++------- dbms/src/Storages/StorageDistributed.h | 1 + dbms/src/Storages/StorageFile.h | 1 + dbms/src/Storages/StorageHDFS.h | 1 + dbms/src/Storages/StorageJoin.h | 1 + dbms/src/Storages/StorageLog.h | 1 + dbms/src/Storages/StorageMaterializedView.h | 1 + dbms/src/Storages/StorageMemory.h | 1 + dbms/src/Storages/StorageMerge.h | 1 + dbms/src/Storages/StorageMergeTree.h | 1 + dbms/src/Storages/StorageMySQL.h | 1 + dbms/src/Storages/StorageNull.h | 1 + .../src/Storages/StorageReplicatedMergeTree.h | 1 + dbms/src/Storages/StorageStripeLog.h | 1 + dbms/src/Storages/StorageTinyLog.h | 1 + dbms/src/Storages/StorageURL.h | 1 + dbms/src/Storages/StorageValues.h | 1 + dbms/src/Storages/StorageView.h | 1 + ...torageSystemAggregateFunctionCombinators.h | 2 ++ .../System/StorageSystemAsynchronousMetrics.h | 4 +++- .../System/StorageSystemBuildOptions.h | 1 + .../Storages/System/StorageSystemClusters.h | 1 + .../Storages/System/StorageSystemCollations.h | 1 + .../Storages/System/StorageSystemColumns.h | 1 + .../System/StorageSystemContributors.h | 1 + .../System/StorageSystemDataTypeFamilies.h | 1 + .../Storages/System/StorageSystemDatabases.h | 1 + .../System/StorageSystemDetachedParts.cpp | 1 + .../System/StorageSystemDictionaries.h | 1 + .../src/Storages/System/StorageSystemEvents.h | 1 + .../Storages/System/StorageSystemFormats.h | 1 + .../Storages/System/StorageSystemFunctions.h | 1 + .../Storages/System/StorageSystemGraphite.h | 1 + .../src/Storages/System/StorageSystemMacros.h | 1 + .../System/StorageSystemMergeTreeSettings.h | 1 + .../src/Storages/System/StorageSystemMerges.h | 1 + .../Storages/System/StorageSystemMetrics.h | 1 + .../src/Storages/System/StorageSystemModels.h | 1 + .../Storages/System/StorageSystemMutations.h | 1 + .../Storages/System/StorageSystemNumbers.h | 1 + dbms/src/Storages/System/StorageSystemOne.h | 1 + dbms/src/Storages/System/StorageSystemParts.h | 1 + .../System/StorageSystemPartsColumns.h | 1 + .../Storages/System/StorageSystemProcesses.h | 1 + .../Storages/System/StorageSystemReplicas.h | 1 + .../System/StorageSystemReplicationQueue.h | 1 + .../Storages/System/StorageSystemSettings.h | 1 + .../System/StorageSystemTableEngines.h | 1 + .../System/StorageSystemTableFunctions.h | 1 + .../src/Storages/System/StorageSystemTables.h | 1 + .../Storages/System/StorageSystemZooKeeper.h | 1 + .../libcommon/include/ext/shared_ptr_helper.h | 22 +++---------------- 57 files changed, 65 insertions(+), 30 deletions(-) diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index 7c90769b37b..43624bf16b8 100644 --- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -30,6 +30,7 @@ namespace DB /// Simplified version of the StorageDistributed class. class StorageDistributedFake : public ext::shared_ptr_helper, public DB::IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "DistributedFake"; } bool isRemote() const override { return true; } diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index a6039eac254..7dc71cbc5e4 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -20,6 +20,7 @@ namespace DB */ class StorageKafka : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Kafka"; } std::string getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 32268499a01..0e15a5660a9 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -14,6 +14,7 @@ namespace DB /// A Storage that allows reading from a single MergeTree data part. class StorageFromMergeTreeDataPart : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: String getName() const override { return "FromMergeTreeDataPart"; } String getTableName() const override { return part->storage.getTableName() + " (part " + part->name + ")"; } diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index 8060199d20d..a7683463ef5 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -39,6 +39,7 @@ class Context; */ class StorageBuffer : public ext::shared_ptr_helper, public IStorage { +friend struct ext::shared_ptr_helper; friend class BufferBlockInputStream; friend class BufferBlockOutputStream; @@ -94,6 +95,8 @@ public: const AlterCommands & params, const String & database_name, const String & table_name, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; + ~StorageBuffer() override; + private: String table_name; String database_name; @@ -145,8 +148,6 @@ protected: Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_, bool allow_materialized_); - - ~StorageBuffer() override; }; } diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index cf5fd647e74..097e81d15c6 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -21,6 +21,7 @@ class ExternalDictionaries; class StorageDictionary : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Dictionary"; } std::string getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 90a973ae3b9..add3de7cfae 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -266,11 +266,8 @@ StoragePtr StorageDistributed::createWithOwnCluster( ClusterPtr owned_cluster_, const Context & context_) { - auto res = ext::shared_ptr_helper::create( - String{}, table_name_, columns_, ConstraintsDescription{}, remote_database_, remote_table_, String{}, context_, ASTPtr(), String(), false); - + auto res = create(String{}, table_name_, columns_, ConstraintsDescription{}, remote_database_, remote_table_, String{}, context_, ASTPtr(), String(), false); res->owned_cluster = owned_cluster_; - return res; } @@ -282,11 +279,8 @@ StoragePtr StorageDistributed::createWithOwnCluster( ClusterPtr & owned_cluster_, const Context & context_) { - auto res = ext::shared_ptr_helper::create( - String{}, table_name_, columns_, ConstraintsDescription{}, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), false); - + auto res = create(String{}, table_name_, columns_, ConstraintsDescription{}, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), false); res->owned_cluster = owned_cluster_; - return res; } diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index e8da6dca4a7..4b0e939cb4b 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -29,6 +29,7 @@ class StorageDistributedDirectoryMonitor; */ class StorageDistributed : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; friend class DistributedBlockOutputStream; friend class StorageDistributedDirectoryMonitor; diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index 237e9a6989e..7268c8ddff0 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -20,6 +20,7 @@ class StorageFileBlockOutputStream; class StorageFile : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "File"; } std::string getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/StorageHDFS.h b/dbms/src/Storages/StorageHDFS.h index dae7000dd8a..30a99c9de70 100644 --- a/dbms/src/Storages/StorageHDFS.h +++ b/dbms/src/Storages/StorageHDFS.h @@ -15,6 +15,7 @@ namespace DB */ class StorageHDFS : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: String getName() const override { return "HDFS"; } String getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index c9e1283b90f..760e9eb1815 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -22,6 +22,7 @@ using JoinPtr = std::shared_ptr; */ class StorageJoin : public ext::shared_ptr_helper, public StorageSetOrJoinBase { + friend struct ext::shared_ptr_helper; public: String getName() const override { return "Join"; } diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index e646ecd8c46..70de62cb47b 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -21,6 +21,7 @@ class StorageLog : public ext::shared_ptr_helper, public IStorage { friend class LogBlockInputStream; friend class LogBlockOutputStream; +friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Log"; } diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 03c93d8d29f..b635634addf 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -11,6 +11,7 @@ namespace DB class StorageMaterializedView : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "MaterializedView"; } std::string getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index 88e9e5dcf04..92dcd4be18f 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -21,6 +21,7 @@ class StorageMemory : public ext::shared_ptr_helper, public IStor { friend class MemoryBlockInputStream; friend class MemoryBlockOutputStream; +friend struct ext::shared_ptr_helper; public: String getName() const override { return "Memory"; } diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index 6d02ad029cc..10aa90d398b 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -14,6 +14,7 @@ namespace DB */ class StorageMerge : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Merge"; } std::string getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 9f51a9357ce..050ef2982d0 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -23,6 +23,7 @@ namespace DB */ class StorageMergeTree : public ext::shared_ptr_helper, public MergeTreeData { + friend struct ext::shared_ptr_helper; public: void startup() override; void shutdown() override; diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h index 320eab9d4a9..5f369da4d25 100644 --- a/dbms/src/Storages/StorageMySQL.h +++ b/dbms/src/Storages/StorageMySQL.h @@ -18,6 +18,7 @@ namespace DB */ class StorageMySQL : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: StorageMySQL( const std::string & database_name_, diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index d858f738b24..4444604113e 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -16,6 +16,7 @@ namespace DB */ class StorageNull : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Null"; } std::string getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 5d8090a8cbc..b201d7fa3c9 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -74,6 +74,7 @@ namespace DB class StorageReplicatedMergeTree : public ext::shared_ptr_helper, public MergeTreeData { + friend struct ext::shared_ptr_helper; public: void startup() override; void shutdown() override; diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index 22be356713b..d53f000f277 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -23,6 +23,7 @@ class StorageStripeLog : public ext::shared_ptr_helper, public { friend class StripeLogBlockInputStream; friend class StripeLogBlockOutputStream; +friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "StripeLog"; } diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index 7a37a42ec63..1c148acf957 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -22,6 +22,7 @@ class StorageTinyLog : public ext::shared_ptr_helper, public ISt { friend class TinyLogBlockInputStream; friend class TinyLogBlockOutputStream; +friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "TinyLog"; } diff --git a/dbms/src/Storages/StorageURL.h b/dbms/src/Storages/StorageURL.h index 0c4b4648223..2facca8ce38 100644 --- a/dbms/src/Storages/StorageURL.h +++ b/dbms/src/Storages/StorageURL.h @@ -71,6 +71,7 @@ private: class StorageURL : public ext::shared_ptr_helper, public IStorageURLBase { + friend struct ext::shared_ptr_helper; public: StorageURL( const Poco::URI & uri_, diff --git a/dbms/src/Storages/StorageValues.h b/dbms/src/Storages/StorageValues.h index 36c3bc15301..c07cf7cbb63 100644 --- a/dbms/src/Storages/StorageValues.h +++ b/dbms/src/Storages/StorageValues.h @@ -11,6 +11,7 @@ namespace DB */ class StorageValues : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "Values"; } std::string getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h index cda128027c2..de56f120fa1 100644 --- a/dbms/src/Storages/StorageView.h +++ b/dbms/src/Storages/StorageView.h @@ -12,6 +12,7 @@ namespace DB class StorageView : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "View"; } std::string getTableName() const override { return table_name; } diff --git a/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.h b/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.h index 1d7226eda8b..86585f0ea73 100644 --- a/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.h +++ b/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.h @@ -4,11 +4,13 @@ #include #include #include + namespace DB { class StorageSystemAggregateFunctionCombinators : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; protected: void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.h b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.h index 853cb97c974..77f247f89eb 100644 --- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.h +++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.h @@ -12,8 +12,10 @@ class Context; /** Implements system table asynchronous_metrics, which allows to get values of periodically (asynchronously) updated metrics. */ -class StorageSystemAsynchronousMetrics : public ext::shared_ptr_helper, public IStorageSystemOneBlock +class StorageSystemAsynchronousMetrics : public ext::shared_ptr_helper, + public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemAsynchronousMetrics"; } diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.h b/dbms/src/Storages/System/StorageSystemBuildOptions.h index 749ffbddbaf..d81682765a2 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.h +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemBuildOptions : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; protected: void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; diff --git a/dbms/src/Storages/System/StorageSystemClusters.h b/dbms/src/Storages/System/StorageSystemClusters.h index dde9e53b626..8efb148cdb9 100644 --- a/dbms/src/Storages/System/StorageSystemClusters.h +++ b/dbms/src/Storages/System/StorageSystemClusters.h @@ -17,6 +17,7 @@ class Context; */ class StorageSystemClusters : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemClusters"; } diff --git a/dbms/src/Storages/System/StorageSystemCollations.h b/dbms/src/Storages/System/StorageSystemCollations.h index f8b7b6ee3af..eccce154047 100644 --- a/dbms/src/Storages/System/StorageSystemCollations.h +++ b/dbms/src/Storages/System/StorageSystemCollations.h @@ -8,6 +8,7 @@ namespace DB class StorageSystemCollations : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; protected: void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; diff --git a/dbms/src/Storages/System/StorageSystemColumns.h b/dbms/src/Storages/System/StorageSystemColumns.h index b9aa04b0b25..b569c2bafff 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.h +++ b/dbms/src/Storages/System/StorageSystemColumns.h @@ -13,6 +13,7 @@ class Context; */ class StorageSystemColumns : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemColumns"; } std::string getTableName() const override { return name; } diff --git a/dbms/src/Storages/System/StorageSystemContributors.h b/dbms/src/Storages/System/StorageSystemContributors.h index 4e2a47960f3..4fc91ce1fbd 100644 --- a/dbms/src/Storages/System/StorageSystemContributors.h +++ b/dbms/src/Storages/System/StorageSystemContributors.h @@ -14,6 +14,7 @@ class Context; class StorageSystemContributors : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; protected: void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; diff --git a/dbms/src/Storages/System/StorageSystemDataTypeFamilies.h b/dbms/src/Storages/System/StorageSystemDataTypeFamilies.h index 365e2790699..44dd59d471f 100644 --- a/dbms/src/Storages/System/StorageSystemDataTypeFamilies.h +++ b/dbms/src/Storages/System/StorageSystemDataTypeFamilies.h @@ -9,6 +9,7 @@ namespace DB class StorageSystemDataTypeFamilies : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; protected: void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; diff --git a/dbms/src/Storages/System/StorageSystemDatabases.h b/dbms/src/Storages/System/StorageSystemDatabases.h index c83f5a72efc..957fcae2707 100644 --- a/dbms/src/Storages/System/StorageSystemDatabases.h +++ b/dbms/src/Storages/System/StorageSystemDatabases.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemDatabases : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index ce1bfc6fc21..b95a299af68 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -21,6 +21,7 @@ class StorageSystemDetachedParts : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemDetachedParts"; } std::string getTableName() const override { return "detached_parts"; } diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.h b/dbms/src/Storages/System/StorageSystemDictionaries.h index 87df9ceada7..6b28f03d917 100644 --- a/dbms/src/Storages/System/StorageSystemDictionaries.h +++ b/dbms/src/Storages/System/StorageSystemDictionaries.h @@ -12,6 +12,7 @@ class Context; class StorageSystemDictionaries : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemDictionaries"; } diff --git a/dbms/src/Storages/System/StorageSystemEvents.h b/dbms/src/Storages/System/StorageSystemEvents.h index 5b02b7739f1..88753d3ea4f 100644 --- a/dbms/src/Storages/System/StorageSystemEvents.h +++ b/dbms/src/Storages/System/StorageSystemEvents.h @@ -13,6 +13,7 @@ class Context; */ class StorageSystemEvents : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemEvents"; } diff --git a/dbms/src/Storages/System/StorageSystemFormats.h b/dbms/src/Storages/System/StorageSystemFormats.h index 82f8303b5b0..61bd9ebeb8f 100644 --- a/dbms/src/Storages/System/StorageSystemFormats.h +++ b/dbms/src/Storages/System/StorageSystemFormats.h @@ -7,6 +7,7 @@ namespace DB { class StorageSystemFormats : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; protected: void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; diff --git a/dbms/src/Storages/System/StorageSystemFunctions.h b/dbms/src/Storages/System/StorageSystemFunctions.h index baead3d8186..1ae4483583f 100644 --- a/dbms/src/Storages/System/StorageSystemFunctions.h +++ b/dbms/src/Storages/System/StorageSystemFunctions.h @@ -15,6 +15,7 @@ class Context; */ class StorageSystemFunctions : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemFunctions"; } diff --git a/dbms/src/Storages/System/StorageSystemGraphite.h b/dbms/src/Storages/System/StorageSystemGraphite.h index b874e294782..5ba7b7bda8c 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.h +++ b/dbms/src/Storages/System/StorageSystemGraphite.h @@ -13,6 +13,7 @@ namespace DB /// Provides information about Graphite configuration. class StorageSystemGraphite : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemGraphite"; } diff --git a/dbms/src/Storages/System/StorageSystemMacros.h b/dbms/src/Storages/System/StorageSystemMacros.h index fdc091dfe1b..09cc8e3a2d9 100644 --- a/dbms/src/Storages/System/StorageSystemMacros.h +++ b/dbms/src/Storages/System/StorageSystemMacros.h @@ -15,6 +15,7 @@ class Context; */ class StorageSystemMacros : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemMacros"; } diff --git a/dbms/src/Storages/System/StorageSystemMergeTreeSettings.h b/dbms/src/Storages/System/StorageSystemMergeTreeSettings.h index 780390dd485..cb3668f13ec 100644 --- a/dbms/src/Storages/System/StorageSystemMergeTreeSettings.h +++ b/dbms/src/Storages/System/StorageSystemMergeTreeSettings.h @@ -15,6 +15,7 @@ class Context; */ class SystemMergeTreeSettings : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemMergeTreeSettings"; } diff --git a/dbms/src/Storages/System/StorageSystemMerges.h b/dbms/src/Storages/System/StorageSystemMerges.h index f45f895d661..6d970b8797f 100644 --- a/dbms/src/Storages/System/StorageSystemMerges.h +++ b/dbms/src/Storages/System/StorageSystemMerges.h @@ -15,6 +15,7 @@ class Context; class StorageSystemMerges : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemMerges"; } diff --git a/dbms/src/Storages/System/StorageSystemMetrics.h b/dbms/src/Storages/System/StorageSystemMetrics.h index f74db926126..86d95800a31 100644 --- a/dbms/src/Storages/System/StorageSystemMetrics.h +++ b/dbms/src/Storages/System/StorageSystemMetrics.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemMetrics : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemMetrics"; } diff --git a/dbms/src/Storages/System/StorageSystemModels.h b/dbms/src/Storages/System/StorageSystemModels.h index ef30bd511ea..8bcb70da35e 100644 --- a/dbms/src/Storages/System/StorageSystemModels.h +++ b/dbms/src/Storages/System/StorageSystemModels.h @@ -12,6 +12,7 @@ class Context; class StorageSystemModels : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemModels"; } diff --git a/dbms/src/Storages/System/StorageSystemMutations.h b/dbms/src/Storages/System/StorageSystemMutations.h index d2dcf99aa46..270cef13ef6 100644 --- a/dbms/src/Storages/System/StorageSystemMutations.h +++ b/dbms/src/Storages/System/StorageSystemMutations.h @@ -14,6 +14,7 @@ class Context; /// in the MergeTree tables. class StorageSystemMutations : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: String getName() const override { return "SystemMutations"; } diff --git a/dbms/src/Storages/System/StorageSystemNumbers.h b/dbms/src/Storages/System/StorageSystemNumbers.h index 5efc23a1125..76070839012 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.h +++ b/dbms/src/Storages/System/StorageSystemNumbers.h @@ -25,6 +25,7 @@ class Context; */ class StorageSystemNumbers : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemNumbers"; } std::string getTableName() const override { return name; } diff --git a/dbms/src/Storages/System/StorageSystemOne.h b/dbms/src/Storages/System/StorageSystemOne.h index 974435e99f0..3e35fdb3477 100644 --- a/dbms/src/Storages/System/StorageSystemOne.h +++ b/dbms/src/Storages/System/StorageSystemOne.h @@ -17,6 +17,7 @@ class Context; */ class StorageSystemOne : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemOne"; } std::string getTableName() const override { return name; } diff --git a/dbms/src/Storages/System/StorageSystemParts.h b/dbms/src/Storages/System/StorageSystemParts.h index eb1ded1c5d6..f7f58daad2e 100644 --- a/dbms/src/Storages/System/StorageSystemParts.h +++ b/dbms/src/Storages/System/StorageSystemParts.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemParts : public ext::shared_ptr_helper, public StorageSystemPartsBase { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemParts"; } diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.h b/dbms/src/Storages/System/StorageSystemPartsColumns.h index bd03e65557c..1ee90c7cde3 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.h +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.h @@ -16,6 +16,7 @@ class Context; class StorageSystemPartsColumns : public ext::shared_ptr_helper, public StorageSystemPartsBase { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemPartsColumns"; } diff --git a/dbms/src/Storages/System/StorageSystemProcesses.h b/dbms/src/Storages/System/StorageSystemProcesses.h index 3cbe0028af3..735315115c4 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.h +++ b/dbms/src/Storages/System/StorageSystemProcesses.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemProcesses : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemProcesses"; } diff --git a/dbms/src/Storages/System/StorageSystemReplicas.h b/dbms/src/Storages/System/StorageSystemReplicas.h index 49865ad869a..fb006f4cbde 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.h +++ b/dbms/src/Storages/System/StorageSystemReplicas.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemReplicas : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemReplicas"; } std::string getTableName() const override { return name; } diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.h b/dbms/src/Storages/System/StorageSystemReplicationQueue.h index 63dc58118cd..79e015250da 100644 --- a/dbms/src/Storages/System/StorageSystemReplicationQueue.h +++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemReplicationQueue : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemReplicationQueue"; } diff --git a/dbms/src/Storages/System/StorageSystemSettings.h b/dbms/src/Storages/System/StorageSystemSettings.h index e44e0abbcd4..f745fe32dcf 100644 --- a/dbms/src/Storages/System/StorageSystemSettings.h +++ b/dbms/src/Storages/System/StorageSystemSettings.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemSettings : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemSettings"; } diff --git a/dbms/src/Storages/System/StorageSystemTableEngines.h b/dbms/src/Storages/System/StorageSystemTableEngines.h index f0f6b62d59d..0bbb03d3898 100644 --- a/dbms/src/Storages/System/StorageSystemTableEngines.h +++ b/dbms/src/Storages/System/StorageSystemTableEngines.h @@ -10,6 +10,7 @@ namespace DB class StorageSystemTableEngines : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; protected: void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; diff --git a/dbms/src/Storages/System/StorageSystemTableFunctions.h b/dbms/src/Storages/System/StorageSystemTableFunctions.h index 413af0f5c66..504ecda4afb 100644 --- a/dbms/src/Storages/System/StorageSystemTableFunctions.h +++ b/dbms/src/Storages/System/StorageSystemTableFunctions.h @@ -9,6 +9,7 @@ namespace DB class StorageSystemTableFunctions : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; diff --git a/dbms/src/Storages/System/StorageSystemTables.h b/dbms/src/Storages/System/StorageSystemTables.h index faf419a6139..fc5a82e9a52 100644 --- a/dbms/src/Storages/System/StorageSystemTables.h +++ b/dbms/src/Storages/System/StorageSystemTables.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemTables : public ext::shared_ptr_helper, public IStorage { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemTables"; } std::string getTableName() const override { return name; } diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.h b/dbms/src/Storages/System/StorageSystemZooKeeper.h index 9644fe96162..ac0ddd29d1f 100644 --- a/dbms/src/Storages/System/StorageSystemZooKeeper.h +++ b/dbms/src/Storages/System/StorageSystemZooKeeper.h @@ -14,6 +14,7 @@ class Context; */ class StorageSystemZooKeeper : public ext::shared_ptr_helper, public IStorageSystemOneBlock { + friend struct ext::shared_ptr_helper; public: std::string getName() const override { return "SystemZooKeeper"; } diff --git a/libs/libcommon/include/ext/shared_ptr_helper.h b/libs/libcommon/include/ext/shared_ptr_helper.h index f7fd7c38ace..ca7219e6261 100644 --- a/libs/libcommon/include/ext/shared_ptr_helper.h +++ b/libs/libcommon/include/ext/shared_ptr_helper.h @@ -7,32 +7,16 @@ namespace ext /** Allows to make std::shared_ptr from T with protected constructor. * - * Derive your T class from shared_ptr_helper + * Derive your T class from shared_ptr_helper and add shared_ptr_helper as a friend * and you will have static 'create' method in your class. - * - * Downsides: - * - your class cannot be final; - * - awful compilation error messages; - * - bad code navigation. - * - different dynamic type of created object, you cannot use typeid. */ template struct shared_ptr_helper { template - static auto create(TArgs &&... args) + static std::shared_ptr create(TArgs &&... args) { - /** Local struct makes protected constructor to be accessible by std::make_shared function. - * This trick is suggested by Yurii Diachenko, - * inspired by https://habrahabr.ru/company/mailru/blog/341584/ - * that is translation of http://videocortex.io/2017/Bestiary/#-voldemort-types - */ - struct Local : T - { - Local(TArgs &&... args) : T(std::forward(args)...) {} - }; - - return std::make_shared(std::forward(args)...); + return std::shared_ptr(new T(std::forward(args)...)); } }; From 5989b19684e1b01022dc6f2e21cca252fda384bb Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Mon, 26 Aug 2019 22:16:28 +0300 Subject: [PATCH 263/357] Final --- dbms/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index b589c398238..1de9536bb56 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -240,6 +240,7 @@ target_link_libraries(clickhouse_common_io ${EXECINFO_LIBRARIES} PUBLIC ${Boost_SYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} PRIVATE apple_rt PUBLIC From 5f25c2d7b4b41500df2f4859f7db8faca5851f48 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 26 Aug 2019 22:49:52 +0300 Subject: [PATCH 264/357] merging with master --- dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 03bf45b4a59..8f07fc430e1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -213,7 +213,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( part_info.min_ttl = part->ttl_infos.part_min_ttl; part_info.max_ttl = part->ttl_infos.part_max_ttl; - time_t ttl = data.settings.ttl_only_drop_parts ? part_info.max_ttl : part_info.min_ttl; + time_t ttl = data_settings->ttl_only_drop_parts ? part_info.max_ttl : part_info.min_ttl; if (ttl && ttl <= current_time) has_part_with_expired_ttl = true; @@ -242,7 +242,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( /// NOTE Could allow selection of different merge strategy. if (can_merge_with_ttl && has_part_with_expired_ttl && !ttl_merges_blocker.isCancelled()) { - merge_selector = std::make_unique(current_time, data.settings.ttl_only_drop_parts); + merge_selector = std::make_unique(current_time, data_settings->ttl_only_drop_parts); last_merge_with_ttl = current_time; } else From 57fa4e1dd72224f36965ef82550978022cc44360 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 00:40:29 +0300 Subject: [PATCH 265/357] Minor lock improvement for ReplicatedMergeTree --- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 4 ++++ dbms/src/Storages/TableStructureLockHolder.h | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 6984ab07ee4..b3ae13ee208 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -4936,7 +4936,11 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ /// If necessary, wait until the operation is performed on all replicas. if (context.getSettingsRef().replication_alter_partitions_sync > 1) + { + lock2.release(); + lock1.release(); waitForAllReplicasToProcessLogEntry(entry); + } } void StorageReplicatedMergeTree::getCommitPartOps( diff --git a/dbms/src/Storages/TableStructureLockHolder.h b/dbms/src/Storages/TableStructureLockHolder.h index 1413ead80a8..17e960503f2 100644 --- a/dbms/src/Storages/TableStructureLockHolder.h +++ b/dbms/src/Storages/TableStructureLockHolder.h @@ -32,6 +32,11 @@ private: /// Order is important. RWLockImpl::LockHolder new_data_structure_lock; RWLockImpl::LockHolder structure_lock; + + void release() + { + *this = TableStructureReadLockHolder(); + } }; } From d654f2507eb0b1a1c60fb266b0b5812c79922330 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 00:40:56 +0300 Subject: [PATCH 266/357] Fixed typo in test --- .../0_stateless/00626_replace_partition_from_table_zookeeper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 5d1a7338e46..0d0bbae1402 100755 --- a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -101,7 +101,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE test.src;" $CLICKHOUSE_CLIENT --query="CREATE TABLE test.src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;" $CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '0', 1);" $CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '1', 1);" -$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (1, '1', 2);" -- trash part to be +$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (1, '1', 2); -- trash part to be deleted" # Stop replication at the second replica and remove source table to use fetch instead of copying $CLICKHOUSE_CLIENT --query="SYSTEM STOP REPLICATION QUEUES test.dst_r2;" From 7985270624e877de15f958f0e3f19a23365f34de Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 00:41:10 +0300 Subject: [PATCH 267/357] Disable processors by default --- dbms/src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 3adae6d9e93..f12efe6548e 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -332,7 +332,7 @@ struct Settings : public SettingsCollection M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only for 'mysql' table function.") \ M(SettingBool, allow_experimental_data_skipping_indices, false, "If it is set to true, data skipping indices can be used in CREATE TABLE/ALTER TABLE queries.") \ \ - M(SettingBool, experimental_use_processors, true, "Use processors pipeline.") \ + M(SettingBool, experimental_use_processors, false, "Use processors pipeline.") \ \ M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.") \ M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.") \ From 5fb5c8dffcbc773d46d96d3ec43fc4febca4d468 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 26 Aug 2019 19:50:37 -0400 Subject: [PATCH 268/357] Updating all live view tests to set the allow_experimental_live_view option. --- .../0_stateless/00960_live_view_watch_events_live.py | 5 +++++ .../queries/0_stateless/00961_temporary_live_view_watch.sql | 2 ++ .../0_stateless/00962_temporary_live_view_watch_live.py | 5 +++++ ...00963_temporary_live_view_watch_live_timeout.py.disabled | 5 +++++ .../0_stateless/00964_live_view_watch_events_heartbeat.py | 5 +++++ .../queries/0_stateless/00965_live_view_watch_heartbeat.py | 5 +++++ .../0_stateless/00966_live_view_watch_events_http.py | 5 ++++- .../tests/queries/0_stateless/00967_live_view_watch_http.py | 5 ++++- ...0968_live_view_select_format_jsoneachrowwithprogress.sql | 2 ++ ...00969_live_view_watch_format_jsoneachrowwithprogress.sql | 2 ++ .../00970_live_view_watch_events_http_heartbeat.py | 6 ++++-- .../0_stateless/00971_live_view_watch_http_heartbeat.py | 5 ++++- dbms/tests/queries/0_stateless/00972_live_view_select_1.sql | 2 ++ dbms/tests/queries/0_stateless/00973_live_view_select.sql | 2 ++ .../0_stateless/00974_live_view_select_with_aggregation.sql | 2 ++ dbms/tests/queries/0_stateless/00975_live_view_create.sql | 2 ++ .../queries/0_stateless/00976_live_view_select_version.sql | 2 ++ .../queries/0_stateless/00977_live_view_watch_events.sql | 2 ++ dbms/tests/queries/0_stateless/00978_live_view_watch.sql | 2 ++ .../tests/queries/0_stateless/00979_live_view_watch_live.py | 5 +++++ .../0_stateless/00980_create_temporary_live_view.sql | 2 ++ 21 files changed, 68 insertions(+), 5 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py index b7fc3f4e3a6..2095683720e 100755 --- a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py +++ b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py @@ -16,6 +16,11 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client2.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client2.send('SET allow_experimental_live_view = 1') + client2.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send(' DROP TABLE IF EXISTS test.mt') diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql index c3e2ab8d102..7992da92f97 100644 --- a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql +++ b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py index f27b1213c70..3dbec01b29a 100755 --- a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py +++ b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py @@ -16,6 +16,11 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client2.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client2.send('SET allow_experimental_live_view = 1') + client2.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send('DROP TABLE IF EXISTS test.mt') diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled index df627c84e49..b324c1b90cc 100755 --- a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled +++ b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled @@ -16,6 +16,11 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client2.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client2.send('SET allow_experimental_live_view = 1') + client2.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send('DROP TABLE IF EXISTS test.mt') diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py index 5664c0e6c6d..528f18839bb 100755 --- a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py +++ b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py @@ -16,6 +16,11 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client2.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client2.send('SET allow_experimental_live_view = 1') + client2.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send(' DROP TABLE IF EXISTS test.mt') diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py index 03e22175dff..2723936f876 100755 --- a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py +++ b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py @@ -16,6 +16,11 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client2.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client2.send('SET allow_experimental_live_view = 1') + client2.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send(' DROP TABLE IF EXISTS test.mt') diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py index bb9d6152200..72ab3ea8818 100755 --- a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py +++ b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py @@ -15,6 +15,9 @@ log = None with client(name='client1>', log=log) as client1: client1.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send(' DROP TABLE IF EXISTS test.mt') @@ -25,7 +28,7 @@ with client(name='client1>', log=log) as client1: client1.expect(prompt) - with http_client({'method':'GET', 'url': '/?query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: + with http_client({'method':'GET', 'url': '/?allow_experimental_live_view=1&query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: client2.expect('.*1\n') client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py index d3439431eb3..e2f33971c3d 100755 --- a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py +++ b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py @@ -15,6 +15,9 @@ log = None with client(name='client1>', log=log) as client1: client1.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send(' DROP TABLE IF EXISTS test.mt') @@ -25,7 +28,7 @@ with client(name='client1>', log=log) as client1: client1.expect(prompt) - with http_client({'method':'GET', 'url':'/?query=WATCH%20test.lv'}, name='client2>', log=log) as client2: + with http_client({'method':'GET', 'url':'/?allow_experimental_live_view=1&query=WATCH%20test.lv'}, name='client2>', log=log) as client2: client2.expect('.*0\t1\n') client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql index 8c6f4197d54..1023cdf6b29 100644 --- a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql +++ b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql index 725a4ad00ed..3e46d55c014 100644 --- a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql +++ b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py index 63628c4a76f..8435cdc147a 100755 --- a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py +++ b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py @@ -15,6 +15,9 @@ log = None with client(name='client1>', log=log) as client1: client1.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send(' DROP TABLE IF EXISTS test.mt') @@ -24,8 +27,7 @@ with client(name='client1>', log=log) as client1: client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') client1.expect(prompt) - - with http_client({'method':'GET', 'url': '/?live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: + with http_client({'method':'GET', 'url': '/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}\n', escape=True) client2.expect('{"row":{"version":"1"}', escape=True) client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py index 7bdb47b7caa..2317d705efe 100755 --- a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py +++ b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py @@ -15,6 +15,9 @@ log = None with client(name='client1>', log=log) as client1: client1.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send(' DROP TABLE IF EXISTS test.mt') @@ -24,7 +27,7 @@ with client(name='client1>', log=log) as client1: client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') client1.expect(prompt) - with http_client({'method':'GET', 'url':'/?live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: + with http_client({'method':'GET', 'url':'/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: client2.expect('"progress".*',) client2.expect('{"row":{"sum(a)":"0","_version":"1"}}\n', escape=True) client2.expect('"progress".*\n') diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql b/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql index 661080b577b..135516b0cd3 100644 --- a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql +++ b/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; CREATE LIVE VIEW test.lv AS SELECT 1; diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.sql b/dbms/tests/queries/0_stateless/00973_live_view_select.sql index ff4a45ffcc1..4b5ca0a2dd7 100644 --- a/dbms/tests/queries/0_stateless/00973_live_view_select.sql +++ b/dbms/tests/queries/0_stateless/00973_live_view_select.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql index 3c11f855c9d..3faaec8f623 100644 --- a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql +++ b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.sql b/dbms/tests/queries/0_stateless/00975_live_view_create.sql index 1c929b15b00..02c1644d193 100644 --- a/dbms/tests/queries/0_stateless/00975_live_view_create.sql +++ b/dbms/tests/queries/0_stateless/00975_live_view_create.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.mt; CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql b/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql index 5f3ab1f7546..ae1c59a92d7 100644 --- a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql +++ b/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql index a3b84e8d4c1..3e0d066fb8d 100644 --- a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql +++ b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql b/dbms/tests/queries/0_stateless/00978_live_view_watch.sql index abe4a6c32ae..b8d0d93ccab 100644 --- a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql +++ b/dbms/tests/queries/0_stateless/00978_live_view_watch.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py index 948e4c93662..8c5bc5b8eb2 100755 --- a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py +++ b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py @@ -16,6 +16,11 @@ with client(name='client1>', log=log) as client1, client(name='client2>', log=lo client1.expect(prompt) client2.expect(prompt) + client1.send('SET allow_experimental_live_view = 1') + client1.expect(prompt) + client2.send('SET allow_experimental_live_view = 1') + client2.expect(prompt) + client1.send('DROP TABLE IF EXISTS test.lv') client1.expect(prompt) client1.send(' DROP TABLE IF EXISTS test.mt') diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql index 8cd6ee06ace..037c2a9e587 100644 --- a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql +++ b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql @@ -1,3 +1,5 @@ +SET allow_experimental_live_view = 1; + DROP TABLE IF EXISTS test.lv; DROP TABLE IF EXISTS test.mt; From 2fbcd607f3b10ee49cecd796fc75594b62edca46 Mon Sep 17 00:00:00 2001 From: Weiqing Xu Date: Tue, 27 Aug 2019 08:23:07 +0800 Subject: [PATCH 269/357] check free space when use external sort/aggerator --- dbms/src/Core/Settings.h | 1 + .../src/DataStreams/MergeSortingBlockInputStream.cpp | 10 ++++++++-- dbms/src/DataStreams/MergeSortingBlockInputStream.h | 8 +++++++- dbms/src/Interpreters/Aggregator.cpp | 5 +++++ dbms/src/Interpreters/Aggregator.h | 10 +++++++--- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 12 ++++++------ dbms/src/Interpreters/tests/aggregate.cpp | 2 +- .../Processors/Transforms/MergeSortingTransform.cpp | 9 ++++++++- .../Processors/Transforms/MergeSortingTransform.h | 8 +++++++- .../Processors/tests/processors_test_aggregation.cpp | 6 ++++-- .../processors_test_merge_sorting_transform.cpp | 2 +- 11 files changed, 55 insertions(+), 18 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index cb12a969b76..43e22f78149 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -352,6 +352,7 @@ struct Settings : public SettingsCollection \ M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ M(SettingBool, compile, false, "Whether query compilation is enabled. Will be removed after 2020-03-13") \ + M(SettingUInt64, min_free_disk_space, 0, "The minimum disk space to keep") \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index e30dd4ae1de..86fcc4ffff9 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -21,10 +21,11 @@ namespace DB MergeSortingBlockInputStream::MergeSortingBlockInputStream( const BlockInputStreamPtr & input, SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_) + size_t max_bytes_before_external_sort_, const std::string & tmp_path_, size_t min_free_disk_space_) : description(description_), max_merged_block_size(max_merged_block_size_), limit(limit_), max_bytes_before_remerge(max_bytes_before_remerge_), - max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_path(tmp_path_) + max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_path(tmp_path_), + min_free_disk_space(min_free_disk_space_) { children.push_back(input); header = children.at(0)->getHeader(); @@ -77,6 +78,11 @@ Block MergeSortingBlockInputStream::readImpl() */ if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort) { + auto freeSpace = Poco::File(tmp_path).freeSpace(); + if (min_free_disk_space > freeSpace - sum_bytes_in_blocks) + { + throw Exception("Not enough space.", ErrorCodes::NOT_ENOUGH_SPACE); + } Poco::File(tmp_path).createDirectories(); temporary_files.emplace_back(std::make_unique(tmp_path)); const std::string & path = temporary_files.back()->path(); diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.h b/dbms/src/DataStreams/MergeSortingBlockInputStream.h index 9f257b82260..4cd9315bc3c 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.h +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.h @@ -18,6 +18,10 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_ENOUGH_SPACE; +} /** Merges stream of sorted each-separately blocks to sorted as-a-whole stream of blocks. * If data to sort is too much, could use external sorting, with temporary files. */ @@ -73,7 +77,8 @@ public: MergeSortingBlockInputStream(const BlockInputStreamPtr & input, SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_); + size_t max_bytes_before_external_sort_, const std::string & tmp_path_, + size_t min_free_disk_space_); String getName() const override { return "MergeSorting"; } @@ -93,6 +98,7 @@ private: size_t max_bytes_before_remerge; size_t max_bytes_before_external_sort; const std::string tmp_path; + size_t min_free_disk_space; Logger * log = &Logger::get("MergeSortingBlockInputStream"); diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 373b47f7315..f2018e443f1 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -639,6 +639,11 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re && current_memory_usage > static_cast(params.max_bytes_before_external_group_by) && worth_convert_to_two_level) { + auto freeSpace = Poco::File(params.tmp_path).freeSpace(); + if (params.min_free_disk_space > freeSpace - current_memory_usage) + { + throw Exception("Not enough space.", ErrorCodes::NOT_ENOUGH_SPACE); + } writeToTemporaryFile(result); } diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index b48663ff689..c3d1d5df8fd 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -39,6 +39,7 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_AGGREGATED_DATA_VARIANT; + extern const int NOT_ENOUGH_SPACE; } class IBlockOutputStream; @@ -796,6 +797,7 @@ public: /// Settings is used to determine cache size. No threads are created. size_t max_threads; + const size_t min_free_disk_space; Params( const Block & src_header_, const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, @@ -803,21 +805,23 @@ public: size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_, size_t max_bytes_before_external_group_by_, bool empty_result_for_aggregation_by_empty_set_, - const std::string & tmp_path_, size_t max_threads_) + const std::string & tmp_path_, size_t max_threads_, + size_t min_free_disk_space_) : src_header(src_header_), keys(keys_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()), overflow_row(overflow_row_), max_rows_to_group_by(max_rows_to_group_by_), group_by_overflow_mode(group_by_overflow_mode_), group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_), max_bytes_before_external_group_by(max_bytes_before_external_group_by_), empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_), - tmp_path(tmp_path_), max_threads(max_threads_) + tmp_path(tmp_path_), max_threads(max_threads_), + min_free_disk_space(min_free_disk_space_) { } /// Only parameters that matter during merge. Params(const Block & intermediate_header_, const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_) - : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, "", max_threads_) + : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, "", max_threads_, 0) { intermediate_header = intermediate_header_; } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 2d583c3c353..6c8769aa3d1 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -1657,7 +1657,7 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context.getTemporaryPath(), settings.max_threads); + context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space); /// If there are several sources, then we perform parallel aggregation if (pipeline.streams.size() > 1) @@ -1723,7 +1723,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context.getTemporaryPath(), settings.max_threads); + context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space); auto transform_params = std::make_shared(params, final); @@ -1943,7 +1943,7 @@ void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificato false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, SettingUInt64(0), SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context.getTemporaryPath(), settings.max_threads); + context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space); if (modificator == Modificator::ROLLUP) pipeline.firstStream() = std::make_shared(pipeline.firstStream(), params); @@ -1972,7 +1972,7 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPipeline & pipeline, Modif false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, SettingUInt64(0), SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context.getTemporaryPath(), settings.max_threads); + context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space); auto transform_params = std::make_shared(params, true); @@ -2073,7 +2073,7 @@ void InterpreterSelectQuery::executeOrder(Pipeline & pipeline, SortingInfoPtr so pipeline.firstStream() = std::make_shared( pipeline.firstStream(), order_descr, settings.max_block_size, limit, settings.max_bytes_before_remerge_sort, - settings.max_bytes_before_external_sort, context.getTemporaryPath()); + settings.max_bytes_before_external_sort, context.getTemporaryPath(), settings.min_free_disk_space); } } @@ -2111,7 +2111,7 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, SortingInfoP return std::make_shared( header, order_descr, settings.max_block_size, limit, settings.max_bytes_before_remerge_sort, - settings.max_bytes_before_external_sort, context.getTemporaryPath()); + settings.max_bytes_before_external_sort, context.getTemporaryPath(), settings.min_free_disk_space); }); } diff --git a/dbms/src/Interpreters/tests/aggregate.cpp b/dbms/src/Interpreters/tests/aggregate.cpp index 73e71d178ea..4d4d964aa9a 100644 --- a/dbms/src/Interpreters/tests/aggregate.cpp +++ b/dbms/src/Interpreters/tests/aggregate.cpp @@ -79,7 +79,7 @@ int main(int argc, char ** argv) Aggregator::Params params( stream->getHeader(), {0, 1}, aggregate_descriptions, - false, 0, OverflowMode::THROW, 0, 0, 0, false, "", 1); + false, 0, OverflowMode::THROW, 0, 0, 0, false, "", 1, 0); Aggregator aggregator(params); diff --git a/dbms/src/Processors/Transforms/MergeSortingTransform.cpp b/dbms/src/Processors/Transforms/MergeSortingTransform.cpp index 8591f5447f7..93e9acf9ebc 100644 --- a/dbms/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/dbms/src/Processors/Transforms/MergeSortingTransform.cpp @@ -236,11 +236,13 @@ MergeSortingTransform::MergeSortingTransform( SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_) + size_t max_bytes_before_external_sort_, const std::string & tmp_path_, + size_t min_free_disk_space_) : IProcessor({header}, {header}) , description(description_), max_merged_block_size(max_merged_block_size_), limit(limit_) , max_bytes_before_remerge(max_bytes_before_remerge_) , max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_path(tmp_path_) + , min_free_disk_space(min_free_disk_space_) { auto & sample = inputs.front().getHeader(); @@ -504,6 +506,11 @@ void MergeSortingTransform::consume(Chunk chunk) */ if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort) { + auto freeSpace = Poco::File(tmp_path).freeSpace(); + if (min_free_disk_space > freeSpace - sum_bytes_in_blocks) + { + throw Exception("Not enough space.", ErrorCodes::NOT_ENOUGH_SPACE); + } Poco::File(tmp_path).createDirectories(); temporary_files.emplace_back(std::make_unique(tmp_path)); const std::string & path = temporary_files.back()->path(); diff --git a/dbms/src/Processors/Transforms/MergeSortingTransform.h b/dbms/src/Processors/Transforms/MergeSortingTransform.h index 0ab517fc5d4..eec249296ef 100644 --- a/dbms/src/Processors/Transforms/MergeSortingTransform.h +++ b/dbms/src/Processors/Transforms/MergeSortingTransform.h @@ -14,6 +14,10 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_ENOUGH_SPACE; +} class MergeSorter; class MergeSortingTransform : public IProcessor @@ -24,7 +28,8 @@ public: SortDescription & description_, size_t max_merged_block_size_, UInt64 limit_, size_t max_bytes_before_remerge_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_); + size_t max_bytes_before_external_sort_, const std::string & tmp_path_, + size_t min_free_disk_space_); ~MergeSortingTransform() override; @@ -44,6 +49,7 @@ private: size_t max_bytes_before_remerge; size_t max_bytes_before_external_sort; const std::string tmp_path; + size_t min_free_disk_space; Logger * log = &Logger::get("MergeSortingBlockInputStream"); diff --git a/dbms/src/Processors/tests/processors_test_aggregation.cpp b/dbms/src/Processors/tests/processors_test_aggregation.cpp index 2306de4edc0..ed868d08762 100644 --- a/dbms/src/Processors/tests/processors_test_aggregation.cpp +++ b/dbms/src/Processors/tests/processors_test_aggregation.cpp @@ -229,7 +229,8 @@ try max_bytes_before_external_group_by, false, /// empty_result_for_aggregation_by_empty_set cur_path, /// tmp_path - 1 /// max_threads + 1, /// max_threads + 0 ); auto agg_params = std::make_shared(params, /* final =*/ false); @@ -301,7 +302,8 @@ try max_bytes_before_external_group_by, false, /// empty_result_for_aggregation_by_empty_set cur_path, /// tmp_path - 1 /// max_threads + 1, /// max_threads + 0 ); auto agg_params = std::make_shared(params, /* final =*/ false); diff --git a/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp b/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp index a5059011e9b..8e6b4655127 100644 --- a/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp +++ b/dbms/src/Processors/tests/processors_test_merge_sorting_transform.cpp @@ -133,7 +133,7 @@ try SortDescription description = {{0, 1, 1}}; auto transform = std::make_shared( source->getPort().getHeader(), description, - max_merged_block_size, limit, max_bytes_before_remerge, max_bytes_before_external_sort, "."); + max_merged_block_size, limit, max_bytes_before_remerge, max_bytes_before_external_sort, ".", 0); auto sink = std::make_shared(); connect(source->getPort(), transform->getInputs().front()); From 7851d8fe0a6dfc98278fcda40f939a6fd898efd8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Aug 2019 12:34:53 +0300 Subject: [PATCH 270/357] Add ability to alter settings with other types of alter --- dbms/src/Storages/IStorage.cpp | 40 +++++++++---------- dbms/src/Storages/IStorage.h | 11 ++--- dbms/src/Storages/Kafka/KafkaSettings.h | 20 +++++----- dbms/src/Storages/Kafka/StorageKafka.cpp | 9 ----- dbms/src/Storages/Kafka/StorageKafka.h | 5 --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 18 ++++----- dbms/src/Storages/MergeTree/MergeTreeData.h | 8 ++-- dbms/src/Storages/StorageMergeTree.cpp | 27 ++++++++----- .../Storages/StorageReplicatedMergeTree.cpp | 19 ++++++++- .../00980_merge_alter_settings.reference | 1 + .../00980_merge_alter_settings.sql | 4 ++ ...keeper_merge_tree_alter_settings.reference | 2 + ...80_zookeeper_merge_tree_alter_settings.sql | 5 +++ 13 files changed, 90 insertions(+), 79 deletions(-) diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 58974010c27..2f3a48d90b6 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -370,15 +370,9 @@ TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id) } -void IStorage::alterSettings( - const SettingsChanges & new_changes, - const Context & context, - TableStructureWriteLockHolder & /* table_lock_holder */) +IDatabase::ASTModifier IStorage::getSettingsModifier(const SettingsChanges & new_changes) const { - const String current_database_name = getDatabaseName(); - const String current_table_name = getTableName(); - - IDatabase::ASTModifier storage_modifier = [&] (IAST & ast) + return [&] (IAST & ast) { if (!new_changes.empty()) { @@ -399,7 +393,6 @@ void IStorage::alterSettings( } } }; - context.getDatabase(current_database_name)->alterTable(context, current_table_name, getColumns(), getIndices(), getConstraints(), storage_modifier); } @@ -408,26 +401,29 @@ void IStorage::alter( const Context & context, TableStructureWriteLockHolder & table_lock_holder) { + if (params.isMutable()) + throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); + const String database_name = getDatabaseName(); const String table_name = getTableName(); + if (params.isSettingsAlter()) { SettingsChanges new_changes; params.applyForSettingsOnly(new_changes); - alterSettings(new_changes, context, table_lock_holder); - return; + IDatabase::ASTModifier settings_modifier = getSettingsModifier(new_changes); + context.getDatabase(database_name)->alterTable(context, table_name, getColumns(), getIndices(), getConstraints(), settings_modifier); + } + else + { + lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); + auto new_columns = getColumns(); + auto new_indices = getIndices(); + auto new_constraints = getConstraints(); + params.applyForColumnsOnly(new_columns); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); + setColumns(std::move(new_columns)); } - - if (params.isMutable()) - throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); - - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - auto new_columns = getColumns(); - auto new_indices = getIndices(); - auto new_constraints = getConstraints(); - params.applyForColumnsOnly(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); - setColumns(std::move(new_columns)); } } diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 5672a1ba3fb..35c1439964a 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -147,6 +147,10 @@ protected: /// still thread-unsafe part. /// Returns whether the column is virtual - by default all columns are real. /// Initially reserved virtual column name may be shadowed by real column. virtual bool isVirtualColumn(const String & column_name) const; + + /// Returns modifier of settings in storage definition + IDatabase::ASTModifier getSettingsModifier(const SettingsChanges & new_changes) const; + private: ColumnsDescription columns; /// combined real and virtual columns const ColumnsDescription virtuals = {}; @@ -291,13 +295,6 @@ public: throw Exception("Partition operations are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } - /** ALTER table settings if possible. Otherwise throws exception. - */ - virtual void alterSettings( - const SettingsChanges & new_changes, - const Context & context, - TableStructureWriteLockHolder & table_lock_holder); - /** Perform any background work. For example, combining parts in a MergeTree type table. * Returns whether any work has been done. */ diff --git a/dbms/src/Storages/Kafka/KafkaSettings.h b/dbms/src/Storages/Kafka/KafkaSettings.h index e43ea7cd70e..bc453238b51 100644 --- a/dbms/src/Storages/Kafka/KafkaSettings.h +++ b/dbms/src/Storages/Kafka/KafkaSettings.h @@ -17,16 +17,16 @@ struct KafkaSettings : public SettingsCollection /// M (mutable) for normal settings, IM (immutable) for not updateable settings. #define LIST_OF_KAFKA_SETTINGS(M, IM) \ - M(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ - M(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ - M(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \ - M(SettingString, kafka_format, "", "The message format for Kafka engine.") \ - M(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.") \ - M(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine") \ - M(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.") \ - M(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \ - M(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block") \ - M(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block") + IM(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ + IM(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ + IM(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \ + IM(SettingString, kafka_format, "", "The message format for Kafka engine.") \ + IM(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.") \ + IM(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine") \ + IM(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.") \ + IM(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \ + IM(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block") \ + IM(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block") DECLARE_SETTINGS_COLLECTION(LIST_OF_KAFKA_SETTINGS) diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 1694c8e3ade..d53469259ab 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -412,15 +412,6 @@ bool StorageKafka::hasSetting(const String & setting_name) const return KafkaSettings::findIndex(setting_name) != KafkaSettings::npos; } -void StorageKafka::alterSettings( - const SettingsChanges & /* new_changes */, - const Context & /* context */, - TableStructureWriteLockHolder & /* table_lock_holder */) -{ - throw Exception("Storage '" + getName() + "' doesn't support settings alter", ErrorCodes::UNSUPPORTED_METHOD); -} - - void registerStorageKafka(StorageFactory & factory) { factory.registerStorage("Kafka", [](const StorageFactory::Arguments & args) diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index 27f1e7e0ec7..c2ff3562116 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -59,11 +59,6 @@ public: bool hasSetting(const String & setting_name) const override; - void alterSettings( - const SettingsChanges & new_changes, - const Context & context, - TableStructureWriteLockHolder & table_lock_holder) override; - protected: StorageKafka( const std::string & table_name_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 0767fb35326..b2d4a4b9d73 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1645,18 +1645,16 @@ void MergeTreeData::alterDataPart( return; } -void MergeTreeData::alterSettings( +void MergeTreeData::changeSettings( const SettingsChanges & new_changes, - const Context & context, - TableStructureWriteLockHolder & table_lock_holder) + TableStructureWriteLockHolder & /* table_lock_holder */) { - const String current_database_name = getDatabaseName(); - const String current_table_name = getTableName(); - - MergeTreeSettings copy = *getSettings(); - copy.updateFromChanges(new_changes); - IStorage::alterSettings(new_changes, context, table_lock_holder); - storage_settings.set(std::make_unique(copy)); + if (!new_changes.empty()) + { + MergeTreeSettings copy = *getSettings(); + copy.updateFromChanges(new_changes); + storage_settings.set(std::make_unique(copy)); + } } bool MergeTreeData::hasSetting(const String & setting_name) const diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index f759b87f986..0440a3181c8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -537,12 +537,10 @@ public: bool skip_sanity_checks, AlterDataPartTransactionPtr& transaction); - /// Performs ALTER of table settings (MergeTreeSettings). Lightweight operation, affects metadata only. - /// Not atomic, have to be done with alter intention lock. - void alterSettings( + /// Change MergeTreeSettings + void changeSettings( const SettingsChanges & new_changes, - const Context & context, - TableStructureWriteLockHolder & table_lock_holder) override; + TableStructureWriteLockHolder & table_lock_holder); /// All MergeTreeData children have settings. bool hasSetting(const String & setting_name) const override; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 1504413801c..4b7b2c446f6 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -253,15 +253,6 @@ void StorageMergeTree::alter( if (!params.isMutable()) { - SettingsChanges new_changes; - /// We don't need to lock table structure exclusively to ALTER settings. - if (params.isSettingsAlter()) - { - params.applyForSettingsOnly(new_changes); - alterSettings(new_changes, context, table_lock_holder); - return; - } - lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); @@ -269,8 +260,14 @@ void StorageMergeTree::alter( ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; + SettingsChanges new_changes; + params.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, {}); + + changeSettings(new_changes, table_lock_holder); + + IDatabase::ASTModifier settings_modifier = getSettingsModifier(new_changes); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, settings_modifier); setColumns(std::move(new_columns)); return; } @@ -305,9 +302,19 @@ void StorageMergeTree::alter( if (new_ttl_table_ast.get() != ttl_table_ast.get()) storage_ast.set(storage_ast.ttl_table, new_ttl_table_ast); + + if (!new_changes.empty()) + { + auto settings_modifier = getSettingsModifier(new_changes); + settings_modifier(ast); + } }; + + changeSettings(new_changes, table_lock_holder); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier); + /// Reinitialize primary key because primary key column types might have changed. setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index cb3828f5817..5c1924c322c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3158,7 +3158,12 @@ void StorageReplicatedMergeTree::alter( LOG_DEBUG(log, "ALTER storage_settings_ptr only"); SettingsChanges new_changes; params.applyForSettingsOnly(new_changes); - alterSettings(new_changes, query_context, table_lock_holder); + + changeSettings(new_changes, table_lock_holder); + + IDatabase::ASTModifier settings_modifier = getSettingsModifier(new_changes); + global_context.getDatabase(current_database_name)->alterTable( + query_context, current_table_name, getColumns(), getIndices(), getConstraints(), settings_modifier); return; } @@ -3231,6 +3236,18 @@ void StorageReplicatedMergeTree::alter( if (new_metadata_str != ReplicatedMergeTreeTableMetadata(*this).toString()) changed_nodes.emplace_back(zookeeper_path, "metadata", new_metadata_str); + /// Perform settings update locally + if (!new_changes.empty()) + { + IDatabase::ASTModifier settings_modifier = getSettingsModifier(new_changes); + + changeSettings(new_changes, table_lock_holder); + + global_context.getDatabase(current_database_name)->alterTable( + query_context, current_table_name, getColumns(), getIndices(), getConstraints(), settings_modifier); + + } + /// Modify shared metadata nodes in ZooKeeper. Coordination::Requests ops; for (const auto & node : changed_nodes) diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference index c7f912ddc79..ee3818d25dc 100644 --- a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.reference @@ -3,3 +3,4 @@ CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = Merge CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100 2 CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 +CREATE TABLE default.table_for_alter (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 diff --git a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql index 43838b8a727..ed42a79ebbf 100644 --- a/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql +++ b/dbms/tests/queries/0_stateless/00980_merge_alter_settings.sql @@ -47,5 +47,9 @@ ALTER TABLE table_for_alter MODIFY SETTING check_delay_period=10, check_delay_pe SHOW CREATE TABLE table_for_alter; +ALTER TABLE table_for_alter ADD COLUMN Data2 UInt64, MODIFY SETTING check_delay_period=5, check_delay_period=10, check_delay_period=15; + +SHOW CREATE TABLE table_for_alter; + DROP TABLE IF EXISTS table_for_alter; diff --git a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference index e55bfadd538..159102e1ca7 100644 --- a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference +++ b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference @@ -8,3 +8,5 @@ CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) EN 6 CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 CREATE TABLE default.replicated_table_for_alter2 (`id` UInt64, `Data` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\') ORDER BY id SETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.replicated_table_for_alter1 (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\') ORDER BY id SETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1, check_delay_period = 15 +CREATE TABLE default.replicated_table_for_alter2 (`id` UInt64, `Data` String, `Data2` UInt64) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\') ORDER BY id SETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 diff --git a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql index 792a704b6a1..f2e453c99d2 100644 --- a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql +++ b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql @@ -58,5 +58,10 @@ ATTACH TABLE replicated_table_for_alter1; SHOW CREATE TABLE replicated_table_for_alter1; SHOW CREATE TABLE replicated_table_for_alter2; +ALTER TABLE replicated_table_for_alter1 ADD COLUMN Data2 UInt64, MODIFY SETTING check_delay_period=5, check_delay_period=10, check_delay_period=15; + +SHOW CREATE TABLE replicated_table_for_alter1; +SHOW CREATE TABLE replicated_table_for_alter2; + DROP TABLE IF EXISTS replicated_table_for_alter2; DROP TABLE IF EXISTS replicated_table_for_alter1; From 36472b14f96762a997205d3653adf23ac7ce4353 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Aug 2019 12:48:20 +0300 Subject: [PATCH 271/357] Defend kafka storage from alters --- dbms/src/Storages/IStorage.h | 2 +- dbms/src/Storages/Kafka/StorageKafka.cpp | 5 +++++ dbms/src/Storages/Kafka/StorageKafka.h | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 35c1439964a..6c23a638ddf 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -149,7 +149,7 @@ protected: /// still thread-unsafe part. virtual bool isVirtualColumn(const String & column_name) const; /// Returns modifier of settings in storage definition - IDatabase::ASTModifier getSettingsModifier(const SettingsChanges & new_changes) const; + virtual IDatabase::ASTModifier getSettingsModifier(const SettingsChanges & new_changes) const; private: ColumnsDescription columns; /// combined real and virtual columns diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index d53469259ab..835ce43b1a4 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -412,6 +412,11 @@ bool StorageKafka::hasSetting(const String & setting_name) const return KafkaSettings::findIndex(setting_name) != KafkaSettings::npos; } +IDatabase::ASTModifier StorageKafka::getSettingsModifier(const SettingsChanges & /* new_changes */) const +{ + throw Exception("Storage '" + getName() + "' doesn't support settings alter", ErrorCodes::UNSUPPORTED_METHOD); +} + void registerStorageKafka(StorageFactory & factory) { factory.registerStorage("Kafka", [](const StorageFactory::Arguments & args) diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index c2ff3562116..b1eac57dca1 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -59,6 +59,7 @@ public: bool hasSetting(const String & setting_name) const override; + protected: StorageKafka( const std::string & table_name_, @@ -70,6 +71,7 @@ protected: size_t num_consumers_, UInt64 max_block_size_, size_t skip_broken, bool intermediate_commit_); + IDatabase::ASTModifier getSettingsModifier(const SettingsChanges & new_changes) const override; private: // Configuration and state String table_name; From 9c0b3778687d8b9bef5bcccadd2b086bd3451909 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Tue, 27 Aug 2019 13:24:42 +0300 Subject: [PATCH 272/357] DOCAPI-7431: Queries with parameters docs. EN review. RU translation. (#6632) * Update cli.md (#36) * Update http.md (#37) * DOCAPI-7431: RU Translation. * DOCAPI-7431: Fix. * Update docs/en/interfaces/cli.md Co-Authored-By: Ivan Blinkov * DOCAPI-7431: Fixes. --- docs/en/interfaces/cli.md | 12 ++++--- docs/en/interfaces/http.md | 2 +- docs/ru/interfaces/cli.md | 71 ++++++++++++++++++++++++++------------ docs/ru/interfaces/http.md | 10 ++++++ 4 files changed, 66 insertions(+), 29 deletions(-) diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index 9f9448f27c8..b6e59c4aa50 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -67,22 +67,22 @@ The command-line client allows passing external data (external temporary tables) ### Queries with Parameters {#cli-queries-with-parameters} -You can create a query with parameters, and pass values for these parameters with the parameters of the client app. For example: +You can create a query with parameters and pass values to them from client application. This allows to avoid formatting query with specific dynamic values on client side. For example: ```bash clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" ``` -#### Syntax of a Query {#cli-queries-with-parameters-syntax} +#### Query Syntax {#cli-queries-with-parameters-syntax} -Format a query by the standard method. Values that you want to put into the query from the app parameters place in braces and format as follows: +Format a query as usual, then place the values that you want to pass from the app parameters to the query in braces in the following format: ``` {:} ``` -- `name` — Identifier of a placeholder that should be used in app parameter as `--param_name = value`. -- `data type` — A data type of app parameter value. For example, data structure like `(integer, ('string', integer))` can have a data type `Tuple(UInt8, Tuple(String, UInt8))` (also you can use another [integer](../data_types/int_uint.md) types). +- `name` — Placeholder identifier. In the console client it should be used in app parameters as `--param_ = value`. +- `data type` — [Data type](../data_types/index.md) of the app parameter value. For example, a data structure like `(integer, ('string', integer))` can have the `Tuple(UInt8, Tuple(String, UInt8))` data type (you can also use another [integer](../data_types/int_uint.md) types). #### Example @@ -118,6 +118,8 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va - `--stacktrace` – If specified, also print the stack trace if an exception occurs. - `--config-file` – The name of the configuration file. - `--secure` – If specified, will connect to server over secure connection. +- `--param_` — Value for a [query with parameters](#cli-queries-with-parameters). + ### Configuration Files diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index ef32b101b71..80cf72ec0e2 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -246,7 +246,7 @@ Use buffering to avoid situations where a query processing error occurred after ### Queries with Parameters {#cli-queries-with-parameters} -You can create a query with parameters, and pass values for these parameters with the parameters of the HTTP request. For more information, see [CLI Formatted Queries](cli.md#cli-queries-with-parameters). +You can create a query with parameters and pass values for them from the corresponding HTTP request parameters. For more information, see [Queries with Parameters for CLI](cli.md#cli-queries-with-parameters). ### Example diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index 0f3ce84345e..59980109240 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -54,8 +54,7 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA По умолчанию, в качестве формата, используется формат PrettyCompact (красивые таблички). Вы можете изменить формат с помощью секции FORMAT запроса, или с помощью указания `\G` на конце запроса, с помощью аргумента командной строки `--format` или `--vertical`, или с помощью конфигурационного файла клиента. -Чтобы выйти из клиента, нажмите Ctrl+D (или Ctrl+C), или наберите вместо запроса одно из: -"exit", "quit", "logout", "учше", "йгше", "дщпщге", "exit;", "quit;", "logout;", "учшеж", "йгшеж", "дщпщгеж", "q", "й", "q", "Q", ":q", "й", "Й", "Жй" +Чтобы выйти из клиента, нажмите Ctrl+D (или Ctrl+C), или наберите вместо запроса одно из: "exit", "quit", "logout", "учше", "йгше", "дщпщге", "exit;", "quit;", "logout;", "учшеж", "йгшеж", "дщпщгеж", "q", "й", "q", "Q", ":q", "й", "Й", "Жй" При выполнении запроса, клиент показывает: @@ -68,38 +67,64 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA Клиент командной строки позволяет передать внешние данные (внешние временные таблицы) для использования запроса. Подробнее смотрите раздел "Внешние данные для обработки запроса" +### Запросы с параметрами {#cli-queries-with-parameters} + +Вы можете создать запрос с параметрами и передавать в них значения из приложения. Это позволяет избежать форматирования запросов на стороне клиента, если известно, какие из параметров запроса динамически меняются. Например: + +```bash +clickhouse-client --param_parName="[1, 2]" -q "SELECT * FROM table WHERE a = {parName:Array(UInt16)}" +``` + +#### Cинтаксис запроса {#cli-queries-with-parameters-syntax} + +Отформатируйте запрос обычным способом. Представьте значения, которые вы хотите передать из параметров приложения в запрос в следующем формате: + +``` +{:} +``` + +- `name` — идентификатор подстановки. В консольном клиенте его следует использовать как часть имени параметра `--param_ = value`. +- `data type` — [тип данных](../data_types/index.md) значения. Например, структура данных `(integer, ('string', integer))` может иметь тип данных `Tuple(UInt8, Tuple(String, UInt8))` ([целочисленный](../data_types/int_uint.md) тип может быть и другим). + +#### Пример + +```bash +clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM table WHERE val = {tuple_in_tuple:Tuple(UInt8, Tuple(String, UInt8))}" +``` + ## Конфигурирование {#interfaces_cli_configuration} В `clickhouse-client` можно передавать различные параметры (все параметры имеют значения по умолчанию) с помощью: - - Командной строки. - - Параметры командной строки переопределяют значения по умолчанию и параметры конфигурационных файлов. - - - Конфигурационных файлов. - +- Командной строки. + + Параметры командной строки переопределяют значения по умолчанию и параметры конфигурационных файлов. + +- Конфигурационных файлов. + Параметры в конфигурационных файлах переопределяют значения по умолчанию. ### Параметры командной строки -- `--host, -h` - имя сервера, по умолчанию - localhost. Вы можете использовать как имя, так и IPv4 или IPv6 адрес. -- `--port` - порт, к которому соединяться, по умолчанию - 9000. Замечу, что для HTTP и родного интерфейса используются разные порты. -- `--user, -u` - имя пользователя, по умолчанию - default. -- `--password` - пароль, по умолчанию - пустая строка. -- `--query, -q` - запрос для выполнения, при использовании в неинтерактивном режиме. -- `--database, -d` - выбрать текущую БД, по умолчанию - текущая БД из настроек сервера (по умолчанию - БД default). -- `--multiline, -m` - если указано - разрешить многострочные запросы, не отправлять запрос по нажатию Enter. -- `--multiquery, -n` - если указано - разрешить выполнять несколько запросов, разделённых точкой с запятой. Работает только в неинтерактивном режиме. -- `--format, -f` - использовать указанный формат по умолчанию для вывода результата. -- `--vertical, -E` - если указано, использовать формат Vertical по умолчанию для вывода результата. То же самое, что --format=Vertical. В этом формате каждое значение выводится на отдельной строке, что удобно для отображения широких таблиц. -- `--time, -t` - если указано, в неинтерактивном режиме вывести время выполнения запроса в stderr. -- `--stacktrace` - если указано, в случае исключения, выводить также его стек трейс. -- `--config-file` - имя конфигурационного файла. -- `--secure` - если указано, будет использован безопасный канал. +- `--host, -h` — имя сервера, по умолчанию — localhost. Вы можете использовать как имя, так и IPv4 или IPv6 адрес. +- `--port` — порт, к которому соединяться, по умолчанию — 9000. Замечу, что для HTTP и родного интерфейса используются разные порты. +- `--user, -u` — имя пользователя, по умолчанию — default. +- `--password` — пароль, по умолчанию — пустая строка. +- `--query, -q` — запрос для выполнения, при использовании в неинтерактивном режиме. +- `--database, -d` — выбрать текущую БД, по умолчанию — текущая БД из настроек сервера (по умолчанию — БД default). +- `--multiline, -m` — если указано — разрешить многострочные запросы, не отправлять запрос по нажатию Enter. +- `--multiquery, -n` — если указано — разрешить выполнять несколько запросов, разделённых точкой с запятой. Работает только в неинтерактивном режиме. +- `--format, -f` — использовать указанный формат по умолчанию для вывода результата. +- `--vertical, -E` — если указано, использовать формат Vertical по умолчанию для вывода результата. То же самое, что --format=Vertical. В этом формате каждое значение выводится на отдельной строке, что удобно для отображения широких таблиц. +- `--time, -t` — если указано, в неинтерактивном режиме вывести время выполнения запроса в stderr. +- `--stacktrace` — если указано, в случае исключения, выводить также его стек трейс. +- `--config-file` — имя конфигурационного файла. +- `--secure` — если указано, будет использован безопасный канал. +- `--param_` — значение параметра для [запроса с параметрами](#cli-queries-with-parameters). ### Конфигурационные файлы -`clickhouse-client` использует первый существующий файл из: +`clickhouse—client` использует первый существующий файл из: - Определенного параметром `--config-file`. - `./clickhouse-client.xml` diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index 20067942dd8..77eb984e8f4 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -245,5 +245,15 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa Буферизация позволяет избежать ситуации когда код ответа и HTTP-заголовки были отправлены клиенту, после чего возникла ошибка выполнения запроса. В такой ситуации сообщение об ошибке записывается в конце тела ответа, и на стороне клиента ошибка может быть обнаружена только на этапе парсинга. +### Запросы с параметрами {#cli-queries-with-parameters} + +Можно создать запрос с параметрами и передать для них значения из соответствующих параметров HTTP-запроса. Дополнительную информацию смотрите в [Запросы с параметрами для консольного клиента](cli.md#cli-queries-with-parameters). + +### Пример + +```bash +curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}" +``` + [Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/http_interface/) From cbc78ffc43a08e94c9a07a62f87da8f01d46db33 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Tue, 27 Aug 2019 14:04:52 +0300 Subject: [PATCH 273/357] DOCAPI-7443: Virtual columns docs update. EN review. RU translation. (#6640) * Update index.md (#35) * DOCAPI-7443: RU translation. * Link fix. * DOCAPI-7443: Fixes. * DOCAPI-7443: Fixes. --- docs/en/operations/table_engines/index.md | 30 +++++------ docs/en/query_language/misc.md | 2 +- docs/ru/operations/table_engines/index.md | 65 ++++++++++++++++++++++- docs/ru/operations/table_engines/kafka.md | 13 +++++ docs/ru/operations/table_engines/merge.md | 14 ++--- docs/ru/query_language/misc.md | 2 +- 6 files changed, 96 insertions(+), 30 deletions(-) diff --git a/docs/en/operations/table_engines/index.md b/docs/en/operations/table_engines/index.md index 41680a5b3af..ce8cf5b4678 100644 --- a/docs/en/operations/table_engines/index.md +++ b/docs/en/operations/table_engines/index.md @@ -2,7 +2,7 @@ The table engine (type of table) determines: -- How and where data is stored, where to write it to, and from where to read it. +- How and where data is stored, where to write it to, and where to read it from. - Which queries are supported, and how. - Concurrent data access. - Use of indexes, if present. @@ -11,13 +11,13 @@ The table engine (type of table) determines: ## Engine Families -### *MergeTree +### MergeTree -The most universal and functional table engines for high-load tasks. The common property of these engines is quick data insertion with subsequent data processing in the background. The `*MergeTree` engines support data replication (with [Replicated*](replication.md) versions of engines), partitioning and other features not supported in other engines. +The most universal and functional table engines for high-load tasks. The property shared by these engines is quick data insertion with subsequent background data processing. `MergeTree` family engines support data replication (with [Replicated*](replication.md) versions of engines), partitioning, and other features not supported in other engines. -Engines of the family: +Engines in the family: -- [MergTree](mergetree.md) +- [MergeTree](mergetree.md) - [ReplacingMergeTree](replacingmergetree.md) - [SummingMergeTree](summingmergetree.md) - [AggregatingMergeTree](aggregatingmergetree.md) @@ -25,11 +25,11 @@ Engines of the family: - [VersionedCollapsingMergeTree](versionedcollapsingmergetree.md) - [GraphiteMergeTree](graphitemergetree.md) -### *Log +### Log -Lightweight [engines](log_family.md) with minimum functionality. They are the most effective in scenarios when you need to quickly write many small tables (up to about 1 million rows) and read them later as a whole. +Lightweight [engines](log_family.md) with minimum functionality. They're the most effective when you need to quickly write many small tables (up to approximately 1 million rows) and read them later as a whole. -Engines of the family: +Engines in the family: - [TinyLog](tinylog.md) - [StripeLog](stripelog.md) @@ -39,7 +39,7 @@ Engines of the family: Engines for communicating with other data storage and processing systems. -Engines of the family: +Engines in the family: - [Kafka](kafka.md) - [MySQL](mysql.md) @@ -48,9 +48,7 @@ Engines of the family: ### Special engines -Engines solving special tasks. - -Engines of the family: +Engines in the family: - [Distributed](distributed.md) - [MaterializedView](materializedview.md) @@ -67,12 +65,12 @@ Engines of the family: ## Virtual columns {#table_engines-virtual_columns} -Virtual column is an integral attribute of a table engine that is defined in the source code of the engine. +Virtual column is an integral table engine attribute that is defined in the engine source code. -You should not specify virtual columns in the `CREATE TABLE` query, and you cannot see them in the results of `SHOW CREATE TABLE` and `DESCRIBE TABLE` queries. Also, virtual columns are read-only, so you can't insert data into virtual columns. +You shouldn't specify virtual columns in the `CREATE TABLE` query and you can't see them in `SHOW CREATE TABLE` and `DESCRIBE TABLE` query results. Virtual columns are also read-only, so you can't insert data into virtual columns. -To select data from a virtual column, you must specify its name in the `SELECT` query. The `SELECT *` doesn't return values from virtual columns. +To select data from a virtual column, you must specify its name in the `SELECT` query. `SELECT *` doesn't return values from virtual columns. -If you create a table with a column that has the same name as one of the table virtual columns, the virtual column becomes inaccessible. Doing so is not recommended. To help avoiding conflicts virtual column names are usually prefixed with an underscore. +If you create a table with a column that has the same name as one of the table virtual columns, the virtual column becomes inaccessible. We don't recommend doing this. To help avoid conflicts, virtual column names are usually prefixed with an underscore. [Original article](https://clickhouse.yandex/docs/en/operations/table_engines/) diff --git a/docs/en/query_language/misc.md b/docs/en/query_language/misc.md index 514f5d9f823..08e8f819b8c 100644 --- a/docs/en/query_language/misc.md +++ b/docs/en/query_language/misc.md @@ -201,7 +201,7 @@ All tables are renamed under global locking. Renaming tables is a light operatio SET param = value ``` -Assigns `value` to the `param` configurations settings for the current session. You cannot change [server settings](../operations/server_settings/index.md) this way. +Assigns `value` to the `param` [setting](../operations/settings/index.md) for the current session. You cannot change [server settings](../operations/server_settings/index.md) this way. You can also set all the values from the specified settings profile in a single query. diff --git a/docs/ru/operations/table_engines/index.md b/docs/ru/operations/table_engines/index.md index cf38d90b55f..ce414a3c0d1 100644 --- a/docs/ru/operations/table_engines/index.md +++ b/docs/ru/operations/table_engines/index.md @@ -9,8 +9,69 @@ - Возможно ли многопоточное выполнение запроса. - Параметры репликации данных. -При чтении, движок обязан лишь выдать запрошенные столбцы, но в некоторых случаях движок может частично обрабатывать данные при ответе на запрос. +## Семейства движков -Для большинства серьёзных задач, следует использовать движки семейства `MergeTree`. +### MergeTree + +Наиболее универсальные и функциональные движки таблиц для задач с высокой загрузкой. Общим свойством этих движков является быстрая вставка данных с последующей фоновой обработкой данных. Движки `*MergeTree` поддерживают репликацию данных (в [Replicated*](replication.md) версиях движков), партиционирование, и другие возможности не поддержанные для других движков. + +Движки семейства: + +- [MergeTree](mergetree.md) +- [ReplacingMergeTree](replacingmergetree.md) +- [SummingMergeTree](summingmergetree.md) +- [AggregatingMergeTree](aggregatingmergetree.md) +- [CollapsingMergeTree](collapsingmergetree.md) +- [VersionedCollapsingMergeTree](versionedcollapsingmergetree.md) +- [GraphiteMergeTree](graphitemergetree.md) + +### Log + +Простые [движки](log_family.md) с минимальной функциональностью. Они наиболее эффективны, когда вам нужно быстро записать много небольших таблиц (до примерно 1 миллиона строк) и прочитать их позже целиком. + +Движки семейства: + +- [TinyLog](tinylog.md) +- [StripeLog](stripelog.md) +- [Log](log.md) + +### Движки для интергации + +Движки для связи с другими системами хранения и обработки данных. + +Движки семейства: + +- [Kafka](kafka.md) +- [MySQL](mysql.md) +- [ODBC](odbc.md) +- [JDBC](jdbc.md) + +### Специальные движки + +Движки семейства: + +- [Distributed](distributed.md) +- [MaterializedView](materializedview.md) +- [Dictionary](dictionary.md) +- [Merge](merge.md) +- [File](file.md) +- [Null](null.md) +- [Set](set.md) +- [Join](join.md) +- [URL](url.md) +- [View](view.md) +- [Memory](memory.md) +- [Buffer](buffer.md) + +## Виртуальные столбцы {#table_engines-virtual_columns} + +Виртуальный столбец — это неотъемлемый атрибут движка таблиц, определенный в исходном коде движка. + +Виртуальные столбцы не надо указывать в запросе `CREATE TABLE` и их не отображаются в результатах запросов `SHOW CREATE TABLE` и `DESCRIBE TABLE`. Также виртуальные столбцы доступны только для чтения, поэтому вы не можете вставлять в них данные. + +Чтобы получить данные из виртуального столбца, необходимо указать его название в запросе `SELECT`. `SELECT *` не отображает данные из виртуальных столбцов. + +При создании таблицы со столбцом, имя которого совпадает с именем одного из виртуальных столбцов таблицы, виртуальный столбец становится недоступным. Не делайте так. Чтобы помочь избежать конфликтов, имена виртуальных столбцов обычно предваряются подчеркиванием. [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/) + diff --git a/docs/ru/operations/table_engines/kafka.md b/docs/ru/operations/table_engines/kafka.md index 086d4fb4f08..f2318d824e2 100644 --- a/docs/ru/operations/table_engines/kafka.md +++ b/docs/ru/operations/table_engines/kafka.md @@ -152,4 +152,17 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format В документе [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) можно увидеть список возможных опций конфигурации. Используйте подчеркивание (`_`) вместо точки в конфигурации ClickHouse. Например, `check.crcs=true` будет соответствовать `true`. +## Виртуальные столбцы + +- `_topic` — топик Kafka. +- `_key` — ключ сообщения. +- `_offset` — оффсет сообщения. +- `_timestamp` — временная метка сообщения. +- `_partition` — секция топика Kafka. + +**Смотрите также** + +- [Виртуальные столбцы](index.md#table_engines-virtual_columns) + [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/kafka/) + diff --git a/docs/ru/operations/table_engines/merge.md b/docs/ru/operations/table_engines/merge.md index 28065e5d120..b87bfdafd75 100644 --- a/docs/ru/operations/table_engines/merge.md +++ b/docs/ru/operations/table_engines/merge.md @@ -52,18 +52,12 @@ FROM WatchLog ## Виртуальные столбцы -Виртуальные столбцы — столбцы, предоставляемые движком таблиц независимо от определения таблицы. То есть, такие столбцы не указываются в `CREATE TABLE`, но доступны для `SELECT`. +- `_table` — содержит имя таблицы, из которой данные были прочитаны. Тип — [String](../../data_types/string.md). -Виртуальные столбцы отличаются от обычных следующими особенностями: + В секции `WHERE/PREWHERE` можно установить константное условие на столбец `_table` (например, `WHERE _table='xyz'`). В этом случае операции чтения выполняются только для тех таблиц, для которых выполняется условие на значение `_table`, таким образом, столбец `_table` работает как индекс. -- они не указываются в определении таблицы; -- в них нельзя вставить данные при `INSERT`; -- при `INSERT` без указания списка столбцов виртуальные столбцы не учитываются; -- они не выбираются при использовании звёздочки (`SELECT *`); -- виртуальные столбцы не показываются в запросах `SHOW CREATE TABLE` и `DESC TABLE`; +**Смотрите также** -Таблица типа `Merge` содержит виртуальный столбец `_table` типа `String`. (Если в таблице уже есть столбец `_table`, то виртуальный столбец называется `_table1`; если уже есть `_table1`, то `_table2` и т. п.) Он содержит имя таблицы, из которой были прочитаны данные. - -Если секция `WHERE/PREWHERE` содержит (в качестве одного из элементов конъюнкции или в качестве всего выражения) условия на столбец `_table`, не зависящие от других столбцов таблицы, то эти условия используются как индекс: условия выполняются над множеством имён таблиц, из которых нужно читать данные, и чтение будет производиться только из тех таблиц, для которых условия сработали. +- [Виртуальные столбцы](index.md#table_engines-virtual_columns) [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/merge/) diff --git a/docs/ru/query_language/misc.md b/docs/ru/query_language/misc.md index e366abf2f3d..ab19e559649 100644 --- a/docs/ru/query_language/misc.md +++ b/docs/ru/query_language/misc.md @@ -199,7 +199,7 @@ RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... SET param = value ``` -Устанавливает значение `value` для настройки `param` в текущей сессии. [Конфигурационные параметры сервера](../operations/server_settings/index.md) нельзя изменить подобным образом. +Устанавливает значение `value` для [настройки](../operations/settings/index.md) `param` в текущей сессии. [Конфигурационные параметры сервера](../operations/server_settings/index.md) нельзя изменить подобным образом. Можно одним запросом установить все настройки из заданного профиля настроек. From f1f85a5a2452d2d566972054a590593fef68cbb2 Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Tue, 27 Aug 2019 16:09:58 +0300 Subject: [PATCH 274/357] Fixed case with single subquery and added tests. --- .../gtest_transform_query_for_external_database.cpp | 11 +++++++++++ .../Storages/transformQueryForExternalDatabase.cpp | 2 ++ 2 files changed, 13 insertions(+) diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index e61ab6279a2..b07605cb7ee 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -76,3 +76,14 @@ TEST(TransformQueryForExternalDatabase, Substring) "SELECT \"column\" FROM \"test\".\"table\"", state().context, state().columns); } + +TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) +{ + check("SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(column, 10) = RIGHT(column, 10) AND column IN (1, 42) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello' AND column != 4", + "SELECT \"column\" FROM \"test\".\"table\" WHERE 1 AND (\"column\" = 42) AND (\"column\" IN (1, 42)) AND (\"column\" != 4)", + state().context, state().columns); + check("SELECT column FROM test.table WHERE toString(column) = '42' AND left(column, 10) = RIGHT(column, 10) AND column = 42", + "SELECT \"column\" FROM \"test\".\"table\" WHERE (\"column\" = 42)", + state().context, state().columns); + +} \ No newline at end of file diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.cpp b/dbms/src/Storages/transformQueryForExternalDatabase.cpp index 7cbffc46e67..b6e48836efa 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.cpp +++ b/dbms/src/Storages/transformQueryForExternalDatabase.cpp @@ -150,6 +150,8 @@ String transformQueryForExternalDatabase( compatible_found = true; } } + if (new_function_and->arguments->children.size() == 1) + new_function_and->name = ""; if (compatible_found) select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(new_function_and)); From 17f18e42c0f03c00746875285142bcfd140a3829 Mon Sep 17 00:00:00 2001 From: Dmitry Rubashkin Date: Tue, 27 Aug 2019 16:13:40 +0300 Subject: [PATCH 275/357] new line --- .../tests/gtest_transform_query_for_external_database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index b07605cb7ee..67cd21b3cfb 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -86,4 +86,4 @@ TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) "SELECT \"column\" FROM \"test\".\"table\" WHERE (\"column\" = 42)", state().context, state().columns); -} \ No newline at end of file +} From 106b9717cd32833926a9bc1115fa51a3d35209f0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Aug 2019 16:14:19 +0300 Subject: [PATCH 276/357] Refactoring of immutable settings --- .../performance-test/PerformanceTestInfo.cpp | 2 +- dbms/src/Core/SettingsCommon.h | 91 +++++++++---------- dbms/src/Interpreters/Context.cpp | 2 +- dbms/src/Storages/Kafka/KafkaSettings.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../Storages/MergeTree/MergeTreeSettings.cpp | 2 +- .../MergeTree/registerStorageMergeTree.cpp | 3 + 7 files changed, 51 insertions(+), 53 deletions(-) diff --git a/dbms/programs/performance-test/PerformanceTestInfo.cpp b/dbms/programs/performance-test/PerformanceTestInfo.cpp index f335a16e0ff..40a066aa0a7 100644 --- a/dbms/programs/performance-test/PerformanceTestInfo.cpp +++ b/dbms/programs/performance-test/PerformanceTestInfo.cpp @@ -97,7 +97,7 @@ void PerformanceTestInfo::applySettings(XMLConfigurationPtr config) } extractSettings(config, "settings", config_settings, settings_to_apply); - settings.loadFromChanges(settings_to_apply); + settings.applyChanges(settings_to_apply); if (settings_contain("average_rows_speed_precision")) TestStats::avg_rows_speed_precision = diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index b8c56d50caa..de30f43c7c8 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -34,6 +34,7 @@ struct SettingNumber { Type value; bool changed = false; + bool immutable = false; SettingNumber(Type x = 0) : value(x) {} @@ -76,6 +77,7 @@ struct SettingMaxThreads UInt64 value; bool is_auto; bool changed = false; + bool immutable = false; SettingMaxThreads(UInt64 x = 0) : value(x ? x : getAutoValue()), is_auto(x == 0) {} @@ -104,6 +106,7 @@ struct SettingTimespan { Poco::Timespan value; bool changed = false; + bool immutable = false; SettingTimespan(UInt64 x = 0) : value(x * microseconds_per_io_unit) {} @@ -136,6 +139,7 @@ struct SettingString { String value; bool changed = false; + bool immutable = false; SettingString(const String & x = String{}) : value(x) {} @@ -158,6 +162,7 @@ struct SettingChar public: char value; bool changed = false; + bool immutable = false; SettingChar(char x = '\0') : value(x) {} @@ -182,6 +187,7 @@ struct SettingEnum { EnumType value; bool changed = false; + bool immutable = false; SettingEnum(EnumType x) : value(x) {} @@ -316,16 +322,17 @@ private: using SerializeFunction = void (*)(const Derived &, WriteBuffer & buf); using DeserializeFunction = void (*)(Derived &, ReadBuffer & buf); using CastValueWithoutApplyingFunction = Field (*)(const Field &); + using SetImmutable = void(*)(Derived &); + using IsImmutable = bool(*)(const Derived &); + struct MemberInfo { IsChangedFunction is_changed; StringRef name; StringRef description; - /// Can be updated after first load for config/definition. - /// Non updatable settings can be `changed`, - /// if they were overwritten in config/definition. - const bool updateable; + /// At one moment this setting can became immutable + const bool can_be_immutable; GetStringFunction get_string; GetFieldFunction get_field; SetStringFunction set_string; @@ -333,6 +340,8 @@ private: SerializeFunction serialize; DeserializeFunction deserialize; CastValueWithoutApplyingFunction cast_value_without_applying; + SetImmutable set_immutable; + IsImmutable is_immutable; bool isChanged(const Derived & collection) const { return is_changed(collection); } }; @@ -405,7 +414,6 @@ public: const_reference(const const_reference & src) = default; const StringRef & getName() const { return member->name; } const StringRef & getDescription() const { return member->description; } - bool isUpdateable() const { return member->updateable; } bool isChanged() const { return member->isChanged(*collection); } Field getValue() const { return member->get_field(*collection); } String getValueAsString() const { return member->get_string(*collection); } @@ -423,20 +431,20 @@ public: public: reference(Derived & collection_, const MemberInfo & member_) : const_reference(collection_, member_) {} reference(const const_reference & src) : const_reference(src) {} - void setValue(const Field & value) { this->member->set_field(*const_cast(this->collection), value); } - void setValue(const String & value) { this->member->set_string(*const_cast(this->collection), value); } - void updateValue(const Field & value) + void setValue(const Field & value) { - if (!this->member->updateable) + if (this->member->is_immutable(*this->collection)) throw Exception("Setting '" + this->member->name.toString() + "' is restricted for updates.", ErrorCodes::IMMUTABLE_SETTING); - setValue(value); + this->member->set_field(*const_cast(this->collection), value); } - void updateValue(const String & value) + void setValue(const String & value) { - if (!this->member->updateable) + if (this->member->is_immutable(*this->collection)) throw Exception("Setting '" + this->member->name.toString() + "' is restricted for updates.", ErrorCodes::IMMUTABLE_SETTING); - setValue(value); + this->member->set_string(*const_cast(this->collection), value); } + bool canBeImmutable() const { return this->member->can_be_immutable; } + void makeImmutableForever() { this->member->set_immutable(*const_cast(this->collection)); } }; /// Iterator to iterating through all the settings. @@ -519,15 +527,6 @@ public: void set(size_t index, const String & value) { (*this)[index].setValue(value); } void set(const String & name, const String & value) { (*this)[name].setValue(value); } - /// Updates setting's value. Checks it' mutability. - void update(size_t index, const Field & value) { (*this)[index].updateValue(value); } - - void update(const String & name, const Field & value) { (*this)[name].updateValue(value); } - - void update(size_t index, const String & value) { (*this)[index].updateValue(value); } - - void update(const String & name, const String & value) { (*this)[name].updateValue(value); } - /// Returns value of a setting. Field get(size_t index) const { return (*this)[index].getValue(); } Field get(const String & name) const { return (*this)[name].getValue(); } @@ -591,35 +590,19 @@ public: return found_changes; } - /// Applies change to the settings. Doesn't check settings mutability. - void loadFromChange(const SettingChange & change) + /// Applies change to concrete setting. + void applyChange(const SettingChange & change) { set(change.name, change.value); } - /// Applies changes to the settings. Should be used in initial settings loading. - /// (on table creation or loading from config) - void loadFromChanges(const SettingsChanges & changes) + /// Applies changes to the settings. + void applyChanges(const SettingsChanges & changes) { for (const SettingChange & change : changes) - loadFromChange(change); + applyChange(change); } - /// Applies change to the settings, checks settings mutability. - void updateFromChange(const SettingChange & change) - { - update(change.name, change.value); - } - - /// Applies changes to the settings. Should be used for settigns update. - /// (ALTER MODIFY SETTINGS) - void updateFromChanges(const SettingsChanges & changes) - { - for (const SettingChange & change : changes) - updateFromChange(change); - } - - void copyChangesFrom(const Derived & src) { for (const auto & member : members()) @@ -632,6 +615,14 @@ public: dest.copyChangesFrom(castToDerived()); } + /// Make all possible immutable settings (can_be_immutable) immutable forever + void finishSettingsInitialization() + { + for (auto & member : *this) + if (member.canBeImmutable()) + member.makeImmutableForever(); + } + /// Writes the settings to buffer (e.g. to be sent to remote server). /// Only changed settings are written. They are written as list of contiguous name-value pairs, /// finished with empty name. @@ -690,22 +681,26 @@ public: static void NAME##_setField(Derived & collection, const Field & value) { collection.NAME.set(value); } \ static void NAME##_serialize(const Derived & collection, WriteBuffer & buf) { collection.NAME.serialize(buf); } \ static void NAME##_deserialize(Derived & collection, ReadBuffer & buf) { collection.NAME.deserialize(buf); } \ - static Field NAME##_castValueWithoutApplying(const Field & value) { TYPE temp{DEFAULT}; temp.set(value); return temp.toField(); } + static Field NAME##_castValueWithoutApplying(const Field & value) { TYPE temp{DEFAULT}; temp.set(value); return temp.toField(); } \ + static void NAME##_setImmutable(Derived & collection) { collection.NAME.immutable = true; } \ + static bool NAME##_isImmutable(const Derived & collection) { return collection.NAME.immutable; } #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ add({[](const Derived & d) { return d.NAME.changed; }, \ - StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ + StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ - &Functions::NAME##_castValueWithoutApplying }); + &Functions::NAME##_castValueWithoutApplying, \ + &Functions::NAME##_setImmutable, &Functions::NAME##_isImmutable }); #define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ add({[](const Derived & d) { return d.NAME.changed; }, \ - StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ + StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ - &Functions::NAME##_castValueWithoutApplying }); + &Functions::NAME##_castValueWithoutApplying, \ + &Functions::NAME##_setImmutable, &Functions::NAME##_isImmutable }); } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 016f1fa0e49..9aad83b4623 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1132,7 +1132,7 @@ void Context::updateSettingsChanges(const SettingsChanges & changes) if (change.name == "profile") setProfile(change.value.safeGet()); else - settings.updateFromChange(change); + settings.applyChange(change); } } diff --git a/dbms/src/Storages/Kafka/KafkaSettings.cpp b/dbms/src/Storages/Kafka/KafkaSettings.cpp index b08d45780bb..d08282a9794 100644 --- a/dbms/src/Storages/Kafka/KafkaSettings.cpp +++ b/dbms/src/Storages/Kafka/KafkaSettings.cpp @@ -22,7 +22,7 @@ void KafkaSettings::loadFromQuery(ASTStorage & storage_def) { try { - loadFromChanges(storage_def.settings->changes); + applyChanges(storage_def.settings->changes); } catch (Exception & e) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b2d4a4b9d73..f8f915cadef 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1652,7 +1652,7 @@ void MergeTreeData::changeSettings( if (!new_changes.empty()) { MergeTreeSettings copy = *getSettings(); - copy.updateFromChanges(new_changes); + copy.applyChanges(new_changes); storage_settings.set(std::make_unique(copy)); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp index 9eee33554ab..224c094dbfb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -46,7 +46,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) { try { - loadFromChanges(storage_def.settings->changes); + applyChanges(storage_def.settings->changes); } catch (Exception & e) { diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 674116a54bc..596ea4f5d0b 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -637,6 +637,9 @@ static StoragePtr create(const StorageFactory::Arguments & args) throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \ "before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS); + /// Finalize settings and disable updates + storage_settings->finishSettingsInitialization(); + if (replicated) return StorageReplicatedMergeTree::create( zookeeper_path, replica_name, args.attach, args.data_path, args.database_name, args.table_name, From b0769330092c3c047c7ab9a704b14d347c821c88 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Tue, 27 Aug 2019 16:49:56 +0300 Subject: [PATCH 277/357] DOCAPI-7400: Gorilla codecs. EN review. RU translation. (#6598) * Update create.md * DOCAPI-7400: RU translation. * DOCAPI-7400: Fixes. * Update docs/ru/query_language/create.md Co-Authored-By: Ivan Blinkov * Update docs/ru/query_language/create.md Co-Authored-By: Ivan Blinkov * DOCAPI-7400: Clarifications. * DOCAPI-7400: Link fix. --- .../agg_functions/parametric_functions.md | 2 +- docs/en/query_language/create.md | 21 +++-- docs/ru/query_language/create.md | 35 ++++---- docs/zh/query_language/create.md | 90 ++++++++++++++++++- .../functions/hash_functions.md | 2 +- .../functions/other_functions.md | 2 +- 6 files changed, 128 insertions(+), 24 deletions(-) diff --git a/docs/en/query_language/agg_functions/parametric_functions.md b/docs/en/query_language/agg_functions/parametric_functions.md index 84898a61133..3ea26de5937 100644 --- a/docs/en/query_language/agg_functions/parametric_functions.md +++ b/docs/en/query_language/agg_functions/parametric_functions.md @@ -45,7 +45,7 @@ FROM ( └─────────────────────────────────────────────────────────────────────────┘ ``` -You can visualize a histogram with the [bar](../other_functions.md#function-bar) function, for example: +You can visualize a histogram with the [bar](../functions/other_functions.md#function-bar) function, for example: ```sql WITH histogram(5)(rand() % 100) AS hist diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md index 54db63fdaef..8060dfbe09d 100644 --- a/docs/en/query_language/create.md +++ b/docs/en/query_language/create.md @@ -164,19 +164,30 @@ These codecs are designed to make compression more effective using specifities o Specialized codecs: -- `Delta(delta_bytes)` — Compression approach, when raw values are replaced with the difference of two neighbor values. Up to `delta_bytes` are used for storing delta value, so `delta_bytes` is a maximum size of raw values. Possible `delta_bytes` values: 1, 2, 4, 8. Default value for `delta_bytes` is `sizeof(type)`, if it is equals to 1, 2, 4, 8. Otherwise it equals 1. -- `DoubleDelta` — Compresses values down to 1 bit (in the best case), using deltas calculation. Best compression rates are achieved on monotonic sequences with constant stride, for example, time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64 bit types. Uses 1 extra bit for 32 byte deltas: 5 bit prefix instead of 4 bit prefix. For additional information, see the "Compressing time stamps" section of the [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) document. -- `Gorilla` — Compresses values down to 1 bit (in the best case). The codec is efficient when storing series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64 bit types. For additional information, see the "Compressing values" section of the [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) document. +- `Delta(delta_bytes)` — Compression approach in which raw values are replaced by the difference of two neighboring values, except for the first value that stays unchanged. Up to `delta_bytes` are used for storing delta values, so `delta_bytes` is the maximum size of raw values. Possible `delta_bytes` values: 1, 2, 4, 8. The default value for `delta_bytes` is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it's 1. +- `DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). - `T64` — Compression approach that crops unused high bits of values in integer data types (including `Enum`, `Date` and `DateTime`). At each step of its algorithm, codec takes a block of 64 values, puts them into 64x64 bit matrix, transposes it, crops the unused bits of values and returns the rest as a sequence. Unused bits are the bits, that don't differ between maximum and minimum values in the whole data part for which the compression is used. +`DoubleDelta` and `Gorilla` codecs are used in Gorilla TSDB as the components of its compressing algorithm. Gorilla approach is effective in scenarios when there is a sequence of slowly changing values with their timestamps. Timestamps are effectively compressed by the `DoubleDelta` codec, and values are effectively compressed by the `Gorilla` codec. For example, to get an effectively stored table, you can create it in the following configuration: + +```sql +CREATE TABLE codec_example +( + timestamp DateTime CODEC(DoubleDelta), + slow_values Float32 CODEC(Gorilla) +) +ENGINE = MergeTree() +``` + ### Common purpose codecs {#create-query-common-purpose-codecs} Codecs: - `NONE` — No compression. - `LZ4` — Lossless [data compression algorithm](https://github.com/lz4/lz4) used by default. Applies LZ4 fast compression. -- `LZ4HC[(level)]` — LZ4 CH (high compression) algorithm with configurable level. Default level: 9. If you set `level <= 0`, the default level is applied. Possible levels: [1, 12]. Recommended levels are in range: [4, 9]. -- `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default level: 1. +- `LZ4HC[(level)]` — LZ4 HC (high compression) algorithm with configurable level. Default level: 9. Setting `level <= 0` applies the default level. Possible levels: [1, 12]. Recommended level range: [4, 9]. +- `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default value: 1. High compression levels useful for asymmetric scenarios, like compress once, decompress a lot of times. Greater levels stands for better compression and higher CPU usage. diff --git a/docs/ru/query_language/create.md b/docs/ru/query_language/create.md index 2e0a21cda0b..d32afb7b9d9 100644 --- a/docs/ru/query_language/create.md +++ b/docs/ru/query_language/create.md @@ -127,24 +127,28 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Определяет время хранения значений. Может быть указано только для таблиц семейства MergeTree. Подробнее смотрите в [TTL для столбцов и таблиц](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). -## Форматы сжатия для колонок +## Кодеки сжатия столбцов -Помимо сжатия для колонок по умолчанию, определяемого в [настройках сервера](../operations/server_settings/settings.md#compression), -существует возможность указать формат сжатия индивидуально для каждой колонки. +Помимо сжатия данных по умолчанию, определяемого [конфигурационными параметрами сервера](../operations/server_settings/settings.md#compression), можно задать сжатие для каждого отдельного столбца. -Поддерживаемые форматы: +Поддерживаемые алгоритмы сжатия: -- `NONE` - сжатие отсутствует -- `LZ4` -- `LZ4HC(level)` - алгоритм сжатия LZ4\_HC с указанным уровнем компрессии `level`. -Возможный диапазон значений `level`: \[3, 12\]. Значение по умолчанию: 9. Чем выше уровень, тем лучше сжатие, но тратится больше времени. Рекомендованный диапазон \[4, 9\]. -- `ZSTD(level)` - алгоритм сжатия ZSTD с указанным уровнем компрессии `level`. Возможный диапазон значений `level`: \[1, 22\]. Значение по умолчанию: 1. -Чем выше уровень, тем лучше сжатие, но тратится больше времени. -- `Delta(delta_bytes)` - способ сжатия, при котором вместо числовых значений поля сохраняется разность между двумя соседними значениями. Значение `delta_bytes` - число байт для хранения дельты. -Возможные значения: 1, 2, 4, 8. Значение по умолчанию: если `sizeof(type)` равен 1, 2, 4, 8 - `sizeof(type)`, иначе - 1. +- `NONE` — без сжатия. +- `LZ4` — [алгоритм сжатия данных](https://github.com/lz4/lz4) без потерь, используемый по умолчанию. Применяет быстрое сжатие LZ4. +- `LZ4HC[(level)]` — алгоритм сильного сжатия LZ4 HC с настраиваемым уровнем. Уровень по умолчанию — 9. Настройка `level <= 0` устанавливает уровень по умолчанию. Возможные уровни: [1, 12]. Рекомендуемый диапазон уровней: [4, 9]. +- `ZSTD[(level)]` — [Алгоритм сжатия ZSTD](https://en.wikipedia.org/wiki/Zstandard) с настаиваемым уровнем `level`. Возможные уровни: [1, 22]. Значение по умолчанию — 1. +- `Delta(delta_bytes)` — способ сжатия, при котором исходные значения заменяются разностью двух соседних значений. Для хранение разностей используется до `delta_bytes` байтов, т.е. `delta_bytes` — это максимальный размер исходных значений. Возможные значения `delta_bytes` — 1, 2, 4, 8. Значение `delta_bytes` по умолчанию равно `sizeof(type)`, если вычисленный размер равен 1, 2, 4 или 8. Во всех остальных случаях — 1. +- `DoubleDelta` — Сжимает значения вплоть до размера в 1 бит благодаря сохранению разностей. Оптимальные уровни сжатия достигаются для монотонных последовательностей с постоянным шагом, например, временные ряды. Может использоваться с любым типом данных фиксированного размера. Реализует алгоритм, используемый в Gorilla TSDB, расширяя его для поддержки 64-битных типов. Использует 1 дополнительный бит для 32-байтовых значений: 5-битные префиксы вместо 4-битных префиксов. Подробнее смотрите в разделе "Compressing Time Stamps" в [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `Gorilla` — Сжимает значения вплоть до размера в 1 bit. Эффективен при хранении рядов медленно изменяющихся чисел с плавающей запятой, потому, что лучшее сжатие достигается, когда соседние значения бинарно равны. Реализует алгоритм, используемый в Gorilla TSDB, расширяя его для поддержки 64-битных типов. Подробнее смотрите в разделе "Compressing Values" в [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). + +Высокие уровни сжатия полезны для асимметричных сценариев, например, для таких, в которых требуется однократное сжатие и многократная распаковка. Более высокие уровни обеспечивают лучшее сжатие, но более высокое потребление вычислительных ресурсов. + +!!! warning "Предупреждение" + Базу данных ClickHouse не получится распаковать с помощью внешних утилит типа `lz4`. Используйте специальную программу [clickhouse-compressor](https://github.com/yandex/ClickHouse/tree/master/dbms/programs/compressor). Пример использования: -``` + +```sql CREATE TABLE codec_example ( dt Date CODEC(ZSTD), /* используется уровень сжатия по умолчанию */ @@ -157,9 +161,10 @@ PARTITION BY tuple() ORDER BY dt ``` -Кодеки могут комбинироваться между собой. Если для колонки указана своя последовательность кодеков, то общий табличный кодек не применяется (должен быть указан в последовательности принудительно, если нужен). В примере ниже - оптимизация для хранения timeseries метрик. +Кодеки можно комбинировать. Если для колонки указана своя последовательность кодеков, то общий табличный кодек не применяется (должен быть указан в последовательности принудительно, если нужен). В примере ниже - оптимизация для хранения timeseries метрик. Как правило, значения одной и той же метрики `path` не сильно различаются между собой, и выгоднее использовать дельта-компрессию вместо записи всего числа: -``` + +```sql CREATE TABLE timeseries_example ( dt Date, diff --git a/docs/zh/query_language/create.md b/docs/zh/query_language/create.md index 1b1abef47db..d3a6c2e841b 100644 --- a/docs/zh/query_language/create.md +++ b/docs/zh/query_language/create.md @@ -80,7 +80,95 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... 不能够为nested类型的列设置默认值。 -### 临时表 +### Constraints {#constraints} + +WARNING: This feature is experimental. Correct work is not guaranteed on non-MergeTree family engines. + +Along with columns descriptions constraints could be defined: + +``sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` could by any boolean expression. If constraints are defined for the table, each of them will be checked for every row in `INSERT` query. If any constraint is not satisfied — server will raise an exception with constraint name and checking expression. + +Adding large amount of constraints can negatively affect performance of big `INSERT` queries. + +### TTL expression + +Defines storage time for values. Can be specified only for MergeTree-family tables. For the detailed description, see [TTL for columns and tables](../operations/table_engines/mergetree.md#table_engine-mergetree-ttl). + +## Column Compression Codecs + +By default, ClickHouse applies to columns the compression method, defined in [server settings](../operations/server_settings/settings.md#compression). Also, you can define compression method for each individual column in the `CREATE TABLE` query. + +``` +CREATE TABLE codec_example +( + dt Date CODEC(ZSTD), + ts DateTime CODEC(LZ4HC), + float_value Float32 CODEC(NONE), + double_value Float64 CODEC(LZ4HC(9)) + value Float32 CODEC(Delta, ZSTD) +) +ENGINE = +... +``` + +If a codec is specified, the default codec doesn't apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codecs combination for you project, pass benchmarks, similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. + +!!!warning + You cannot decompress ClickHouse database files with external utilities, for example, `lz4`. Use the special utility, [clickhouse-compressor](https://github.com/yandex/ClickHouse/tree/master/dbms/programs/compressor). + +Compression is supported for the table engines: + +- [*MergeTree](../operations/table_engines/mergetree.md) family +- [*Log](../operations/table_engines/log_family.md) family +- [Set](../operations/table_engines/set.md) +- [Join](../operations/table_engines/join.md) + +ClickHouse supports common purpose codecs and specialized codecs. + +### Specialized codecs {#create-query-specialized-codecs} + +These codecs are designed to make compression more effective using specifities of the data. Some of this codecs don't compress data by itself, but they prepare data to be compressed better by common purpose codecs. + +Specialized codecs: + +- `Delta(delta_bytes)` — Compression approach in which raw values are replaced by the difference of two neighboring values, except for the first value that stays unchanged. Up to `delta_bytes` are used for storing delta values, so `delta_bytes` is the maximum size of raw values. Possible `delta_bytes` values: 1, 2, 4, 8. The default value for `delta_bytes` is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it's 1. +- `DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +- `T64` — Compression approach that crops unused high bits of values in integer data types (including `Enum`, `Date` and `DateTime`). At each step of its algorithm, codec takes a block of 64 values, puts them into 64x64 bit matrix, transposes it, crops the unused bits of values and returns the rest as a sequence. Unused bits are the bits, that don't differ between maximum and minimum values in the whole data part for which the compression is used. + +`DoubleDelta` and `Gorilla` codecs are used in Gorilla TSDB as the components of its compressing algorithm. Gorilla approach is effective in scenarios when there is a sequence of slowly changing values with their timestamps. Timestamps are effectively compressed by the `DoubleDelta` codec, and values are effectively compressed by the `Gorilla` codec. For example, to get an effectively stored table, you can create it in the following configuration: + +```sql +CREATE TABLE codec_example +( + timestamp DateTime CODEC(DoubleDelta), + slow_values Float32 CODEC(Gorilla) +) +ENGINE = MergeTree() +``` + +### Common purpose codecs {#create-query-common-purpose-codecs} + +Codecs: + +- `NONE` — No compression. +- `LZ4` — Lossless [data compression algorithm](https://github.com/lz4/lz4) used by default. Applies LZ4 fast compression. +- `LZ4HC[(level)]` — LZ4 HC (high compression) algorithm with configurable level. Default level: 9. Setting `level <= 0` applies the default level. Possible levels: [1, 12]. Recommended level range: [4, 9]. +- `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default value: 1. + +High compression levels useful for asymmetric scenarios, like compress once, decompress a lot of times. Greater levels stands for better compression and higher CPU usage. + +## 临时表 ClickHouse支持临时表，其具有以下特征： diff --git a/docs/zh/query_language/functions/hash_functions.md b/docs/zh/query_language/functions/hash_functions.md index 57af83a4a5e..9fc6c79b0b3 100644 --- a/docs/zh/query_language/functions/hash_functions.md +++ b/docs/zh/query_language/functions/hash_functions.md @@ -21,7 +21,7 @@ Hash函数可以用于将元素不可逆的伪随机打乱。 SipHash是一种加密哈希函数。它的处理性能至少比MD5快三倍。 有关详细信息，请参阅链接： -## sipHash128 +## sipHash128 {#hash_functions-siphash128} 计算字符串的SipHash。 接受String类型的参数，返回FixedString(16)。 diff --git a/docs/zh/query_language/functions/other_functions.md b/docs/zh/query_language/functions/other_functions.md index 84fbdaeb3ca..329db222b48 100644 --- a/docs/zh/query_language/functions/other_functions.md +++ b/docs/zh/query_language/functions/other_functions.md @@ -122,7 +122,7 @@ Accepts constant strings: database name, table name, and column name. Returns a The function throws an exception if the table does not exist. For elements in a nested data structure, the function checks for the existence of a column. For the nested data structure itself, the function returns 0. -## bar +## bar {#function-bar} 使用unicode构建图表。 From 8ca00a26f6e2f72a6cae6a94e0c14e9363d71f68 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 17:31:34 +0300 Subject: [PATCH 278/357] Fixed build --- dbms/src/Storages/TableStructureLockHolder.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/TableStructureLockHolder.h b/dbms/src/Storages/TableStructureLockHolder.h index 17e960503f2..3f2f8e1fa32 100644 --- a/dbms/src/Storages/TableStructureLockHolder.h +++ b/dbms/src/Storages/TableStructureLockHolder.h @@ -26,17 +26,17 @@ private: struct TableStructureReadLockHolder { + void release() + { + *this = TableStructureReadLockHolder(); + } + private: friend class IStorage; /// Order is important. RWLockImpl::LockHolder new_data_structure_lock; RWLockImpl::LockHolder structure_lock; - - void release() - { - *this = TableStructureReadLockHolder(); - } }; } From b21fdff77e79c84b9e1672a9ce233201f5904960 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 21:59:21 +0300 Subject: [PATCH 279/357] Merging #6678 --- .../src/DataStreams/MergeSortingBlockInputStream.cpp | 12 +++++++----- dbms/src/Interpreters/Aggregator.cpp | 12 +++++++----- .../Processors/Transforms/MergeSortingTransform.cpp | 12 +++++++----- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index 86fcc4ffff9..377c1fee4e0 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -78,11 +79,12 @@ Block MergeSortingBlockInputStream::readImpl() */ if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort) { - auto freeSpace = Poco::File(tmp_path).freeSpace(); - if (min_free_disk_space > freeSpace - sum_bytes_in_blocks) - { - throw Exception("Not enough space.", ErrorCodes::NOT_ENOUGH_SPACE); - } +#if !UNBUNDLED + auto free_space = Poco::File(tmp_path).freeSpace(); + if (sum_bytes_in_blocks + min_free_disk_space > free_space) + throw Exception("Not enough space for external sort in " + tmp_path, ErrorCodes::NOT_ENOUGH_SPACE); +#endif + Poco::File(tmp_path).createDirectories(); temporary_files.emplace_back(std::make_unique(tmp_path)); const std::string & path = temporary_files.back()->path(); diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index f2018e443f1..f38573d3d34 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -24,6 +24,7 @@ #include #include #include +#include namespace ProfileEvents @@ -639,11 +640,12 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re && current_memory_usage > static_cast(params.max_bytes_before_external_group_by) && worth_convert_to_two_level) { - auto freeSpace = Poco::File(params.tmp_path).freeSpace(); - if (params.min_free_disk_space > freeSpace - current_memory_usage) - { - throw Exception("Not enough space.", ErrorCodes::NOT_ENOUGH_SPACE); - } +#if !UNBUNDLED + auto free_space = Poco::File(params.tmp_path).freeSpace(); + if (current_memory_usage + params.min_free_disk_space > free_space) + throw Exception("Not enough space for external aggregation in " + params.tmp_path, ErrorCodes::NOT_ENOUGH_SPACE); +#endif + writeToTemporaryFile(result); } diff --git a/dbms/src/Processors/Transforms/MergeSortingTransform.cpp b/dbms/src/Processors/Transforms/MergeSortingTransform.cpp index 93e9acf9ebc..c59fc7cc745 100644 --- a/dbms/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/dbms/src/Processors/Transforms/MergeSortingTransform.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -506,11 +507,12 @@ void MergeSortingTransform::consume(Chunk chunk) */ if (max_bytes_before_external_sort && sum_bytes_in_blocks > max_bytes_before_external_sort) { - auto freeSpace = Poco::File(tmp_path).freeSpace(); - if (min_free_disk_space > freeSpace - sum_bytes_in_blocks) - { - throw Exception("Not enough space.", ErrorCodes::NOT_ENOUGH_SPACE); - } +#if !UNBUNDLED + auto free_space = Poco::File(tmp_path).freeSpace(); + if (sum_bytes_in_blocks + min_free_disk_space > free_space) + throw Exception("Not enough space for external sort in " + tmp_path, ErrorCodes::NOT_ENOUGH_SPACE); +#endif + Poco::File(tmp_path).createDirectories(); temporary_files.emplace_back(std::make_unique(tmp_path)); const std::string & path = temporary_files.back()->path(); From 25c53ccd9e315a10b60569cff0d0fb63b45b9c46 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 22:05:35 +0300 Subject: [PATCH 280/357] Rename setting --- dbms/src/Core/Settings.h | 2 +- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 3794548628c..88a81027cb6 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -347,12 +347,12 @@ struct Settings : public SettingsCollection M(SettingSeconds, live_view_heartbeat_interval, DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC, "The heartbeat interval in seconds to indicate live query is alive.") \ M(SettingSeconds, temporary_live_view_timeout, DEFAULT_TEMPORARY_LIVE_VIEW_TIMEOUT_SEC, "Timeout after which temporary live view is deleted.") \ M(SettingUInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.") \ + M(SettingUInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.") \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ M(SettingBool, compile, false, "Whether query compilation is enabled. Will be removed after 2020-03-13") \ - M(SettingUInt64, min_free_disk_space, 0, "The minimum disk space to keep") \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 6c8769aa3d1..79fbcf44323 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -1657,7 +1657,7 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space); + context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); /// If there are several sources, then we perform parallel aggregation if (pipeline.streams.size() > 1) @@ -1723,7 +1723,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0), allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space); + context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); auto transform_params = std::make_shared(params, final); @@ -1943,7 +1943,7 @@ void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificato false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, SettingUInt64(0), SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space); + context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); if (modificator == Modificator::ROLLUP) pipeline.firstStream() = std::make_shared(pipeline.firstStream(), params); @@ -1972,7 +1972,7 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPipeline & pipeline, Modif false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, SettingUInt64(0), SettingUInt64(0), settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space); + context.getTemporaryPath(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); auto transform_params = std::make_shared(params, true); @@ -2073,7 +2073,7 @@ void InterpreterSelectQuery::executeOrder(Pipeline & pipeline, SortingInfoPtr so pipeline.firstStream() = std::make_shared( pipeline.firstStream(), order_descr, settings.max_block_size, limit, settings.max_bytes_before_remerge_sort, - settings.max_bytes_before_external_sort, context.getTemporaryPath(), settings.min_free_disk_space); + settings.max_bytes_before_external_sort, context.getTemporaryPath(), settings.min_free_disk_space_for_temporary_data); } } @@ -2111,7 +2111,7 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, SortingInfoP return std::make_shared( header, order_descr, settings.max_block_size, limit, settings.max_bytes_before_remerge_sort, - settings.max_bytes_before_external_sort, context.getTemporaryPath(), settings.min_free_disk_space); + settings.max_bytes_before_external_sort, context.getTemporaryPath(), settings.min_free_disk_space_for_temporary_data); }); } From 0add95b7d8f0add0e02b6956ae0d2bcaa87f96c6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 22:21:53 +0300 Subject: [PATCH 281/357] Set default value of "queue_max_wait_ms" to zero, because current value makes no sense --- dbms/src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 881ea83f30b..9f780392466 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -61,7 +61,7 @@ struct Settings : public SettingsCollection M(SettingSeconds, receive_timeout, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "") \ M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "") \ M(SettingSeconds, tcp_keep_alive_timeout, 0, "") \ - M(SettingMilliseconds, queue_max_wait_ms, 5000, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \ + M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \ M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.") \ M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.") \ M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.") \ From 63c0070cd50aaa463cc00eedee4c43868ba8c69f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 22:32:33 +0300 Subject: [PATCH 282/357] Fixed flacky test --- .../00626_replace_partition_from_table_zookeeper.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 0d0bbae1402..75d6a0dd59b 100755 --- a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +# Because REPLACE PARTITION does not forces immediate removal of replaced data parts from local filesystem +# (it tries to do it as quick as possible, but it still performed in separate thread asynchronously) +# and when we do DETACH TABLE / ATTACH TABLE or SYSTEM RESTART REPLICA, these files may be discovered +# and discarded after restart with Warning/Error messages in log. This is Ok. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh From 5208a70203e6acd8593733b9e329fb161f84f05d Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 27 Aug 2019 22:41:51 +0300 Subject: [PATCH 283/357] one less place with changed nullability columns --- dbms/src/Interpreters/AnalyzedJoin.cpp | 17 ---------- dbms/src/Interpreters/AnalyzedJoin.h | 3 -- dbms/src/Interpreters/ExpressionActions.cpp | 4 +-- dbms/src/Interpreters/SyntaxAnalyzer.cpp | 37 +++++++++++++-------- dbms/src/Interpreters/SyntaxAnalyzer.h | 4 +-- 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/dbms/src/Interpreters/AnalyzedJoin.cpp b/dbms/src/Interpreters/AnalyzedJoin.cpp index 4176b0b8012..6a3b9b8ac1b 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.cpp +++ b/dbms/src/Interpreters/AnalyzedJoin.cpp @@ -7,7 +7,6 @@ #include #include -#include namespace DB { @@ -102,22 +101,6 @@ std::unordered_map AnalyzedJoin::getOriginalColumnsMap(const Nam return out; } -void AnalyzedJoin::calculateAvailableJoinedColumns(bool make_nullable) -{ - if (!make_nullable) - { - available_joined_columns = columns_from_joined_table; - return; - } - - for (auto & column : columns_from_joined_table) - { - auto type = column.type->canBeInsideNullable() ? makeNullable(column.type) : column.type; - available_joined_columns.emplace_back(NameAndTypePair(column.name, std::move(type))); - } -} - - NamesAndTypesList getNamesAndTypeListFromTableExpression(const ASTTableExpression & table_expression, const Context & context) { NamesAndTypesList names_and_type_list; diff --git a/dbms/src/Interpreters/AnalyzedJoin.h b/dbms/src/Interpreters/AnalyzedJoin.h index af010aaca11..1ce11da95e0 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.h +++ b/dbms/src/Interpreters/AnalyzedJoin.h @@ -42,8 +42,6 @@ private: /// All columns which can be read from joined table. Duplicating names are qualified. NamesAndTypesList columns_from_joined_table; - /// Columns from joined table which may be added to block. It's columns_from_joined_table with possibly modified types. - NamesAndTypesList available_joined_columns; /// Name -> original name. Names are the same as in columns_from_joined_table list. std::unordered_map original_names; /// Original name -> name. Only ranamed columns. @@ -61,7 +59,6 @@ public: std::unordered_map getOriginalColumnsMap(const NameSet & required_columns) const; void deduplicateAndQualifyColumnNames(const NameSet & left_table_columns, const String & right_table_prefix); - void calculateAvailableJoinedColumns(bool make_nullable); size_t rightKeyInclusion(const String & name) const; }; diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index c7b510abcf0..5ef05569f91 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -278,8 +278,8 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings, case JOIN: { bool is_null_used_as_default = settings.join_use_nulls; - bool right_or_full_join = join_kind == ASTTableJoin::Kind::Right || join_kind == ASTTableJoin::Kind::Full; - bool left_or_full_join = join_kind == ASTTableJoin::Kind::Left || join_kind == ASTTableJoin::Kind::Full; + bool right_or_full_join = isRightOrFull(join_kind); + bool left_or_full_join = isLeftOrFull(join_kind); for (auto & col : sample_block) { diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index c7f89153b44..0c73beeef16 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -488,13 +489,14 @@ void getArrayJoinedColumns(ASTPtr & query, SyntaxAnalyzerResult & result, const } } -void setJoinStrictness(ASTSelectQuery & select_query, JoinStrictness join_default_strictness) +void setJoinStrictness(ASTSelectQuery & select_query, JoinStrictness join_default_strictness, ASTTableJoin::Kind & join_kind) { const ASTTablesInSelectQueryElement * node = select_query.join(); if (!node) return; auto & table_join = const_cast(node)->table_join->as(); + join_kind = table_join.kind; if (table_join.strictness == ASTTableJoin::Strictness::Unspecified && table_join.kind != ASTTableJoin::Kind::Cross) @@ -511,7 +513,7 @@ void setJoinStrictness(ASTSelectQuery & select_query, JoinStrictness join_defaul /// Find the columns that are obtained by JOIN. void collectJoinedColumns(AnalyzedJoin & analyzed_join, const ASTSelectQuery & select_query, const NameSet & source_columns, - const Aliases & aliases, bool join_use_nulls) + const Aliases & aliases) { const ASTTablesInSelectQueryElement * node = select_query.join(); if (!node) @@ -537,10 +539,6 @@ void collectJoinedColumns(AnalyzedJoin & analyzed_join, const ASTSelectQuery & s if (is_asof) data.asofToJoinKeys(); } - - bool make_nullable = join_use_nulls && isLeftOrFull(table_join.kind); - - analyzed_join.calculateAvailableJoinedColumns(make_nullable); } void replaceJoinedTable(const ASTTablesInSelectQueryElement* join) @@ -611,7 +609,8 @@ std::vector getAggregates(const ASTPtr & query) /// Calculate which columns are required to execute the expression. /// Then, delete all other columns from the list of available columns. /// After execution, columns will only contain the list of columns needed to read from the table. -void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query, const NamesAndTypesList & additional_source_columns) +void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query, const NamesAndTypesList & additional_source_columns, + bool make_joined_columns_nullable) { /// We caclulate required_source_columns with source_columns modifications and swap them on exit required_source_columns = source_columns; @@ -639,7 +638,7 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query, const NamesA /// Add columns obtained by JOIN (if needed). columns_added_by_join.clear(); - for (const auto & joined_column : analyzed_join.available_joined_columns) + for (const auto & joined_column : analyzed_join.columns_from_joined_table) { auto & name = joined_column.name; if (avaliable_columns.count(name)) @@ -649,7 +648,15 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query, const NamesA { /// Optimisation: do not add columns needed only in JOIN ON section. if (columns_context.nameInclusion(name) > analyzed_join.rightKeyInclusion(name)) - columns_added_by_join.push_back(joined_column); + { + if (make_joined_columns_nullable) + { + auto type = joined_column.type->canBeInsideNullable() ? makeNullable(joined_column.type) : joined_column.type; + columns_added_by_join.emplace_back(NameAndTypePair(joined_column.name, std::move(type))); + } + else + columns_added_by_join.push_back(joined_column); + } required.erase(name); } } @@ -759,7 +766,7 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query, const NamesA if (columns_context.has_table_join) { ss << ", joined columns:"; - for (const auto & column : analyzed_join.available_joined_columns) + for (const auto & column : analyzed_join.columns_from_joined_table) ss << " '" << column.name << "'"; } @@ -865,6 +872,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( /// Optimize if with constant condition after constants was substituted instead of scalar subqueries. OptimizeIfWithConstantConditionVisitor(result.aliases).visit(query); + bool make_joined_columns_nullable = false; if (select_query) { /// GROUP BY injective function elimination. @@ -885,12 +893,15 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( /// Push the predicate expression down to the subqueries. result.rewrite_subqueries = PredicateExpressionsOptimizer(select_query, settings, context).optimize(); - setJoinStrictness(*select_query, settings.join_default_strictness); - collectJoinedColumns(result.analyzed_join, *select_query, source_columns_set, result.aliases, settings.join_use_nulls); + ASTTableJoin::Kind join_kind = ASTTableJoin::Kind::Comma; + setJoinStrictness(*select_query, settings.join_default_strictness, join_kind); + make_joined_columns_nullable = settings.join_use_nulls && isLeftOrFull(join_kind); + + collectJoinedColumns(result.analyzed_join, *select_query, source_columns_set, result.aliases); } result.aggregates = getAggregates(query); - result.collectUsedColumns(query, additional_source_columns); + result.collectUsedColumns(query, additional_source_columns, make_joined_columns_nullable); return std::make_shared(result); } diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.h b/dbms/src/Interpreters/SyntaxAnalyzer.h index a31dfef7e82..e95d7354e8a 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.h +++ b/dbms/src/Interpreters/SyntaxAnalyzer.h @@ -20,7 +20,7 @@ struct SyntaxAnalyzerResult NamesAndTypesList source_columns; /// Set of columns that are enough to read from the table to evaluate the expression. It does not include joined columns. NamesAndTypesList required_source_columns; - /// Columns will be added to block by JOIN. It's a subset of analyzed_join.available_joined_columns + /// Columns will be added to block by JOIN. It's a subset of analyzed_join.columns_from_joined_table with corrected Nullability NamesAndTypesList columns_added_by_join; Aliases aliases; @@ -42,7 +42,7 @@ struct SyntaxAnalyzerResult /// Predicate optimizer overrides the sub queries bool rewrite_subqueries = false; - void collectUsedColumns(const ASTPtr & query, const NamesAndTypesList & additional_source_columns); + void collectUsedColumns(const ASTPtr & query, const NamesAndTypesList & additional_source_columns, bool make_joined_columns_nullable); Names requiredSourceColumns() const { return required_source_columns.getNames(); } }; From e80ff65a0f10ab1abe6792a79b56196aad74a661 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 27 Aug 2019 16:23:12 -0400 Subject: [PATCH 284/357] Removing allow_experimental_live_view option from clickhouse-client.xml. --- dbms/tests/clickhouse-client.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/tests/clickhouse-client.xml b/dbms/tests/clickhouse-client.xml index ebce35127e5..b6003ca2d09 100644 --- a/dbms/tests/clickhouse-client.xml +++ b/dbms/tests/clickhouse-client.xml @@ -1,4 +1,3 @@ - 1 100000 From 285d66cee15e8746129230372d1a3d8444527d3a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 23:43:08 +0300 Subject: [PATCH 285/357] Improve table locks (incomplete) --- dbms/src/Databases/DatabaseDictionary.cpp | 20 -------- dbms/src/Databases/DatabaseDictionary.h | 14 ------ dbms/src/Databases/DatabaseFactory.cpp | 1 + dbms/src/Databases/DatabaseMemory.cpp | 20 -------- dbms/src/Databases/DatabaseMemory.h | 14 ------ dbms/src/Databases/DatabaseMySQL.h | 12 +---- dbms/src/Databases/DatabaseOrdinary.cpp | 5 +- dbms/src/Databases/DatabaseOrdinary.h | 4 +- dbms/src/Databases/IDatabase.h | 46 +++++++++++-------- .../src/Interpreters/InterpreterDropQuery.cpp | 8 ++-- .../Interpreters/InterpreterRenameQuery.cpp | 19 ++++---- dbms/src/Storages/IStorage.h | 12 +++-- .../src/Storages/LiveView/StorageLiveView.cpp | 2 +- dbms/src/Storages/LiveView/StorageLiveView.h | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 +- dbms/src/Storages/MergeTree/MergeTreeData.h | 4 +- dbms/src/Storages/StorageDictionary.h | 1 - dbms/src/Storages/StorageDistributed.cpp | 2 +- dbms/src/Storages/StorageDistributed.h | 11 +++-- dbms/src/Storages/StorageJoin.cpp | 2 +- dbms/src/Storages/StorageJoin.h | 2 +- dbms/src/Storages/StorageLog.cpp | 4 +- dbms/src/Storages/StorageLog.h | 4 +- dbms/src/Storages/StorageMaterializedView.cpp | 3 +- dbms/src/Storages/StorageMaterializedView.h | 6 +-- dbms/src/Storages/StorageMemory.cpp | 4 +- dbms/src/Storages/StorageMemory.h | 10 ++-- dbms/src/Storages/StorageMergeTree.cpp | 10 ++-- dbms/src/Storages/StorageMergeTree.h | 6 +-- .../Storages/StorageReplicatedMergeTree.cpp | 14 +++--- .../src/Storages/StorageReplicatedMergeTree.h | 6 +-- dbms/src/Storages/StorageSet.cpp | 5 +- dbms/src/Storages/StorageSet.h | 4 +- dbms/src/Storages/StorageStripeLog.cpp | 4 +- dbms/src/Storages/StorageStripeLog.h | 4 +- dbms/src/Storages/StorageTinyLog.cpp | 4 +- dbms/src/Storages/StorageTinyLog.h | 4 +- 37 files changed, 123 insertions(+), 174 deletions(-) diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index bfc0f6e89d0..2bb9bd30238 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -115,26 +115,6 @@ void DatabaseDictionary::removeTable( throw Exception("DatabaseDictionary: removeTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); } -void DatabaseDictionary::renameTable( - const Context &, - const String &, - IDatabase &, - const String &) -{ - throw Exception("DatabaseDictionary: renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - -void DatabaseDictionary::alterTable( - const Context &, - const String &, - const ColumnsDescription &, - const IndicesDescription &, - const ConstraintsDescription &, - const ASTModifier &) -{ - throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - time_t DatabaseDictionary::getTableMetadataModificationTime( const Context &, const String &) diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index 650a6986722..1e1af7ef581 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -60,20 +60,6 @@ public: void attachTable(const String & table_name, const StoragePtr & table) override; StoragePtr detachTable(const String & table_name) override; - void renameTable( - const Context & context, - const String & table_name, - IDatabase & to_database, - const String & to_table_name) override; - - void alterTable( - const Context & context, - const String & name, - const ColumnsDescription & columns, - const IndicesDescription & indices, - const ConstraintsDescription & constraints, - const ASTModifier & engine_modifier) override; - time_t getTableMetadataModificationTime( const Context & context, const String & table_name) override; diff --git a/dbms/src/Databases/DatabaseFactory.cpp b/dbms/src/Databases/DatabaseFactory.cpp index 1dcf41dc4d6..af2a00830e1 100644 --- a/dbms/src/Databases/DatabaseFactory.cpp +++ b/dbms/src/Databases/DatabaseFactory.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "config_core.h" #if USE_MYSQL diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index a7f8460366c..1356a28d245 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -39,26 +39,6 @@ void DatabaseMemory::removeTable( detachTable(table_name); } -void DatabaseMemory::renameTable( - const Context &, - const String &, - IDatabase &, - const String &) -{ - throw Exception("DatabaseMemory: renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - -void DatabaseMemory::alterTable( - const Context &, - const String &, - const ColumnsDescription &, - const IndicesDescription &, - const ConstraintsDescription &, - const ASTModifier &) -{ - throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); -} - time_t DatabaseMemory::getTableMetadataModificationTime( const Context &, const String &) diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 32d3045612b..33bb8787168 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -37,20 +37,6 @@ public: const Context & context, const String & table_name) override; - void renameTable( - const Context & context, - const String & table_name, - IDatabase & to_database, - const String & to_table_name) override; - - void alterTable( - const Context & context, - const String & name, - const ColumnsDescription & columns, - const IndicesDescription & indices, - const ConstraintsDescription & constraints, - const ASTModifier & engine_modifier) override; - time_t getTableMetadataModificationTime( const Context & context, const String & table_name) override; diff --git a/dbms/src/Databases/DatabaseMySQL.h b/dbms/src/Databases/DatabaseMySQL.h index 6c277f1e9f8..7044a594b4c 100644 --- a/dbms/src/Databases/DatabaseMySQL.h +++ b/dbms/src/Databases/DatabaseMySQL.h @@ -5,6 +5,8 @@ #include #include +#include + namespace DB { @@ -61,21 +63,11 @@ public: throw Exception("MySQL database engine does not support attach table.", ErrorCodes::NOT_IMPLEMENTED); } - void renameTable(const Context &, const String &, IDatabase &, const String &) override - { - throw Exception("MySQL database engine does not support rename table.", ErrorCodes::NOT_IMPLEMENTED); - } - void createTable(const Context &, const String &, const StoragePtr &, const ASTPtr &) override { throw Exception("MySQL database engine does not support create table.", ErrorCodes::NOT_IMPLEMENTED); } - void alterTable(const Context &, const String &, const ColumnsDescription &, const IndicesDescription &, const ConstraintsDescription &, const ASTModifier &) override - { - throw Exception("MySQL database engine does not support alter table.", ErrorCodes::NOT_IMPLEMENTED); - } - private: struct MySQLStorageInfo { diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 4748bd0d792..06e355892e8 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -355,7 +355,8 @@ void DatabaseOrdinary::renameTable( const Context & context, const String & table_name, IDatabase & to_database, - const String & to_table_name) + const String & to_table_name, + TableStructureWriteLockHolder & lock) { DatabaseOrdinary * to_database_concrete = typeid_cast(&to_database); @@ -372,7 +373,7 @@ void DatabaseOrdinary::renameTable( { table->rename(context.getPath() + "/data/" + escapeForFileName(to_database_concrete->name) + "/", to_database_concrete->name, - to_table_name); + to_table_name, lock); } catch (const Exception &) { diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index a71029f6495..e8895075768 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -35,7 +36,8 @@ public: const Context & context, const String & table_name, IDatabase & to_database, - const String & to_table_name) override; + const String & to_table_name, + TableStructureWriteLockHolder &) override; void alterTable( const Context & context, diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index b9d4c6ce4cb..549d795b66d 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -1,16 +1,9 @@ #pragma once -#include #include -#include #include -#include -#include -#include #include -#include -#include -#include +#include #include #include @@ -21,8 +14,16 @@ namespace DB { class Context; - struct Settings; +struct ConstraintsDescription; +class ColumnsDescription; +struct IndicesDescription; +struct TableStructureWriteLockHolder; + +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} /** Allows to iterate over tables. @@ -102,22 +103,29 @@ public: /// Rename the table and possibly move the table to another database. virtual void renameTable( - const Context & context, - const String & name, - IDatabase & to_database, - const String & to_name) = 0; + const Context & /*context*/, + const String & /*name*/, + IDatabase & /*to_database*/, + const String & /*to_name*/, + TableStructureWriteLockHolder &) + { + throw Exception(getEngineName() + ": renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); + } using ASTModifier = std::function; /// Change the table structure in metadata. /// You must call under the TableStructureLock of the corresponding table . If engine_modifier is empty, then engine does not change. virtual void alterTable( - const Context & context, - const String & name, - const ColumnsDescription & columns, - const IndicesDescription & indices, - const ConstraintsDescription & constraints, - const ASTModifier & engine_modifier) = 0; + const Context & /*context*/, + const String & /*name*/, + const ColumnsDescription & /*columns*/, + const IndicesDescription & /*indices*/, + const ConstraintsDescription & /*constraints*/, + const ASTModifier & /*engine_modifier*/) + { + throw Exception(getEngineName() + ": renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); + } /// Returns time of table's metadata change, 0 if there is no corresponding metadata file. virtual time_t getTableMetadataModificationTime( diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 91213b6100e..d601580a044 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -80,7 +80,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t /// If table was already dropped by anyone, an exception will be thrown auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId()); /// Drop table data, don't touch metadata - database_and_table.second->truncate(query_ptr, context); + database_and_table.second->truncate(query_ptr, context, table_lock); } else if (kind == ASTDropQuery::Kind::Drop) { @@ -94,7 +94,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t database_and_table.first->removeTable(context, database_and_table.second->getTableName()); /// Delete table data - database_and_table.second->drop(); + database_and_table.second->drop(table_lock); database_and_table.second->is_dropped = true; String database_data_path = database_and_table.first->getDataPath(); @@ -128,7 +128,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr /// If table was already dropped by anyone, an exception will be thrown auto table_lock = table->lockExclusively(context.getCurrentQueryId()); /// Drop table data, don't touch metadata - table->truncate(query_ptr, context); + table->truncate(query_ptr, context, table_lock); } else if (kind == ASTDropQuery::Kind::Drop) { @@ -137,7 +137,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr /// If table was already dropped by anyone, an exception will be thrown auto table_lock = table->lockExclusively(context.getCurrentQueryId()); /// Delete table data - table->drop(); + table->drop(table_lock); table->is_dropped = true; } } diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index 360adf45194..97b17c6afd7 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -26,6 +26,8 @@ struct RenameDescription to_table_name(elem.to.table) {} + TableStructureWriteLockHolder from_table_lock; + String from_database_name; String from_table_name; @@ -75,7 +77,7 @@ BlockIO InterpreterRenameQuery::execute() } }; - std::set unique_tables_from; + std::map tables_from_locks; /// Don't allow to drop tables (that we are renaming); don't allow to create tables in places where tables will be renamed. std::map> table_guards; @@ -87,7 +89,11 @@ BlockIO InterpreterRenameQuery::execute() UniqueTableName from(descriptions.back().from_database_name, descriptions.back().from_table_name); UniqueTableName to(descriptions.back().to_database_name, descriptions.back().to_table_name); - unique_tables_from.emplace(from); + if (!tables_from_locks.count(from)) + if (auto table = context.tryGetTable(from.database_name, from.table_name)) + tables_from_locks.emplace(from, table->lockExclusively(context.getCurrentQueryId())); + + descriptions.back().table_lock = tables_from_locks[from]; if (!table_guards.count(from)) table_guards.emplace(from, context.getDDLGuard(from.database_name, from.table_name)); @@ -96,13 +102,6 @@ BlockIO InterpreterRenameQuery::execute() table_guards.emplace(to, context.getDDLGuard(to.database_name, to.table_name)); } - std::vector locks; - locks.reserve(unique_tables_from.size()); - - for (const auto & names : unique_tables_from) - if (auto table = context.tryGetTable(names.database_name, names.table_name)) - locks.emplace_back(table->lockExclusively(context.getCurrentQueryId())); - /** All tables are locked. If there are more than one rename in chain, * we need to hold global lock while doing all renames. Order matters to avoid deadlocks. * It provides atomicity of all RENAME chain as a whole, from the point of view of DBMS client, @@ -119,7 +118,7 @@ BlockIO InterpreterRenameQuery::execute() context.assertTableDoesntExist(elem.to_database_name, elem.to_table_name); context.getDatabase(elem.from_database_name)->renameTable( - context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name); + context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name, elem.table_lock); } return {}; diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 6c23a638ddf..ffe4ed1b775 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -9,11 +9,13 @@ #include #include #include +#include +#include +#include #include #include #include #include -#include #include #include @@ -24,7 +26,6 @@ namespace DB namespace ErrorCodes { - extern const int TABLE_IS_DROPPED; extern const int NOT_IMPLEMENTED; } @@ -261,12 +262,12 @@ public: * The table is not usable during and after call to this method. * If you do not need any action other than deleting the directory with data, you can leave this method blank. */ - virtual void drop() {} + virtual void drop(TableStructureWriteLockHolder &) {} /** Clear the table data and leave it empty. * Must be called under lockForAlter. */ - virtual void truncate(const ASTPtr & /*query*/, const Context & /* context */) + virtual void truncate(const ASTPtr & /*query*/, const Context & /* context */, TableStructureWriteLockHolder &) { throw Exception("Truncate is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } @@ -276,7 +277,8 @@ public: * In this function, you need to rename the directory with the data, if any. * Called when the table structure is locked for write. */ - virtual void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/) + virtual void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/, + TableStructureWriteLockHolder &) { throw Exception("Method rename is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 7704f421517..9468a180772 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -471,7 +471,7 @@ StorageLiveView::~StorageLiveView() shutdown(); } -void StorageLiveView::drop() +void StorageLiveView::drop(TableStructureWriteLockHolder &) { global_context.removeDependency( DatabaseAndTableName(select_database_name, select_table_name), diff --git a/dbms/src/Storages/LiveView/StorageLiveView.h b/dbms/src/Storages/LiveView/StorageLiveView.h index 9930d8d6154..79ac259f912 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.h +++ b/dbms/src/Storages/LiveView/StorageLiveView.h @@ -105,7 +105,7 @@ public: } void checkTableCanBeDropped() const override; - void drop() override; + void drop(TableStructureWriteLockHolder &) override; void startup() override; void shutdown() override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b2d4a4b9d73..70ae2414418 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1690,7 +1690,7 @@ void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr empty_columns.clear(); } -void MergeTreeData::freezeAll(const String & with_name, const Context & context) +void MergeTreeData::freezeAll(const String & with_name, const Context & context, TableStructureReadLockHolder &) { freezePartitionsByMatcher([] (const DataPartPtr &){ return true; }, with_name, context); } @@ -2550,7 +2550,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part } -void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & with_name, const Context & context) +void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & with_name, const Context & context, TableStructureReadLockHolder &) { std::optional prefix; String partition_id; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 0440a3181c8..ea9db0920a3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -549,7 +549,7 @@ public: void removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part); /// Freezes all parts. - void freezeAll(const String & with_name, const Context & context); + void freezeAll(const String & with_name, const Context & context, TableStructureReadLockHolder & table_lock_holder); /// Should be called if part data is suspected to be corrupted. void reportBrokenPart(const String & name) const @@ -577,7 +577,7 @@ public: * Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number, * or if 'with_name' is specified - backup is created in directory with specified name. */ - void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context); + void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context, TableStructureReadLockHolder & table_lock_holder); size_t getColumnCompressedSize(const std::string & name) const { diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index 097e81d15c6..9539240e75d 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -34,7 +34,6 @@ public: size_t max_block_size = DEFAULT_BLOCK_SIZE, unsigned threads = 1) override; - void drop() override {} static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure); template diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index b2e632f9414..2c289dd714e 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -415,7 +415,7 @@ void StorageDistributed::shutdown() } -void StorageDistributed::truncate(const ASTPtr &, const Context &) +void StorageDistributed::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { std::lock_guard lock(cluster_nodes_mutex); diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index 3261f5a0beb..153ada5d284 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -77,12 +77,17 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void drop() override {} + void drop(TableStructureWriteLockHolder &) override {} /// Removes temporary data in local filesystem. - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; + + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override + { + table_name = new_table_name; + database_name = new_database_name; + } - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; } /// in the sub-tables, you need to manually add and delete columns /// the structure of the sub-table is not checked void alter( diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index e2cbe542e11..54effdcd4fa 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -55,7 +55,7 @@ StorageJoin::StorageJoin( } -void StorageJoin::truncate(const ASTPtr &, const Context &) +void StorageJoin::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { Poco::File(path).remove(true); Poco::File(path).createDirectories(); diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index 760e9eb1815..bdc50b9d767 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -26,7 +26,7 @@ class StorageJoin : public ext::shared_ptr_helper, public StorageSe public: String getName() const override { return "Join"; } - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; /// Access the innards. JoinPtr & getJoin() { return join; } diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 69e37ce2305..3811b226357 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -512,7 +512,7 @@ void StorageLog::loadMarks() } -void StorageLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) +void StorageLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { std::unique_lock lock(rwlock); @@ -530,7 +530,7 @@ void StorageLog::rename(const String & new_path_to_db, const String & new_databa marks_file = Poco::File(path + escapeForFileName(table_name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); } -void StorageLog::truncate(const ASTPtr &, const Context &) +void StorageLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { std::shared_lock lock(rwlock); diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index 70de62cb47b..7f792337abc 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -38,11 +38,11 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override; - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; std::string full_path() const { return path + escapeForFileName(table_name) + '/';} diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index df39f711ab3..c3e52f8a32e 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -299,7 +299,8 @@ static void executeRenameQuery(Context & global_context, const String & database } -void StorageMaterializedView::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) +void StorageMaterializedView::rename( + const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { if (has_inner_table && tryGetTargetTable()) { diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index b635634addf..327e8feed79 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -33,9 +33,9 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void drop() override; + void drop(TableStructureWriteLockHolder &) override; - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; @@ -43,7 +43,7 @@ public: void mutate(const MutationCommands & commands, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; void shutdown() override; diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index 560da7dc2d8..1b820e55c5e 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -123,13 +123,13 @@ BlockOutputStreamPtr StorageMemory::write( } -void StorageMemory::drop() +void StorageMemory::drop(TableStructureWriteLockHolder &) { std::lock_guard lock(mutex); data.clear(); } -void StorageMemory::truncate(const ASTPtr &, const Context &) +void StorageMemory::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { std::lock_guard lock(mutex); data.clear(); diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index 92dcd4be18f..eb2d6ff2e21 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -40,11 +40,15 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void drop() override; + void drop(TableStructureWriteLockHolder &) override; - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; } + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override + { + table_name = new_table_name; + database_name = new_database_name; + } private: String database_name; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 4b7b2c446f6..46fcc977a60 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -157,13 +157,13 @@ void StorageMergeTree::checkPartitionCanBeDropped(const ASTPtr & partition) global_context.checkPartitionCanBeDropped(database_name, table_name, partition_size); } -void StorageMergeTree::drop() +void StorageMergeTree::drop(TableStructureWriteLockHolder &) { shutdown(); dropAllData(); } -void StorageMergeTree::truncate(const ASTPtr &, const Context &) +void StorageMergeTree::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { { /// Asks to complete merges and does not allow them to start. @@ -181,7 +181,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const Context &) clearOldPartsFromFilesystem(); } -void StorageMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) +void StorageMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { std::string new_full_path = new_path_to_db + escapeForFileName(new_table_name) + '/'; @@ -995,7 +995,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma case PartitionCommand::FREEZE_PARTITION: { auto lock = lockStructureForShare(false, context.getCurrentQueryId()); - freezePartition(command.partition, command.with_name, context); + freezePartition(command.partition, command.with_name, context, lock); } break; @@ -1020,7 +1020,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma case PartitionCommand::FREEZE_ALL_PARTITIONS: { auto lock = lockStructureForShare(false, context.getCurrentQueryId()); - freezeAll(command.with_name, context); + freezeAll(command.with_name, context, lock); } break; diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index d135ffd6e1a..06ffa94c8ec 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -55,10 +55,10 @@ public: std::vector getMutationsStatus() const override; CancellationCode killMutation(const String & mutation_id) override; - void drop() override; - void truncate(const ASTPtr &, const Context &) override; + void drop(TableStructureWriteLockHolder &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index afcc5d968af..e5821c1bcaf 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3132,7 +3132,6 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p if (query_context.getSettingsRef().replication_alter_partitions_sync != 0) { /// NOTE Table lock must not be held while waiting. Some combination of R-W-R locks from different threads will yield to deadlock. - /// TODO Check all other "wait" places. for (auto & merge_entry : merge_entries) waitForAllReplicasToProcessLogEntry(merge_entry); } @@ -3484,7 +3483,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part case PartitionCommand::FREEZE_PARTITION: { auto lock = lockStructureForShare(false, query_context.getCurrentQueryId()); - freezePartition(command.partition, command.with_name, query_context); + freezePartition(command.partition, command.with_name, query_context, lock); } break; @@ -3509,7 +3508,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part case PartitionCommand::FREEZE_ALL_PARTITIONS: { auto lock = lockStructureForShare(false, query_context.getCurrentQueryId()); - freezeAll(command.with_name, query_context); + freezeAll(command.with_name, query_context, lock); } break; } @@ -3633,8 +3632,10 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPt } -void StorageReplicatedMergeTree::truncate(const ASTPtr & query, const Context & query_context) +void StorageReplicatedMergeTree::truncate(const ASTPtr & query, const Context & query_context, TableStructureWriteLockHolder & table_lock) { + table_lock.release(); /// Truncate is done asynchronously. + assertNotReadonly(); zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); @@ -3701,7 +3702,7 @@ void StorageReplicatedMergeTree::checkPartitionCanBeDropped(const ASTPtr & parti } -void StorageReplicatedMergeTree::drop() +void StorageReplicatedMergeTree::drop(TableStructureWriteLockHolder &) { { auto zookeeper = tryGetZooKeeper(); @@ -3731,7 +3732,8 @@ void StorageReplicatedMergeTree::drop() } -void StorageReplicatedMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) +void StorageReplicatedMergeTree::rename( + const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { std::string new_full_path = new_path_to_db + escapeForFileName(new_table_name) + '/'; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 0fa2e3631e2..c5000944439 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -109,11 +109,11 @@ public: /** Removes a replica from ZooKeeper. If there are no other replicas, it deletes the entire table from ZooKeeper. */ - void drop() override; + void drop(TableStructureWriteLockHolder &) override; - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; bool supportsIndexForIn() const override { return true; } diff --git a/dbms/src/Storages/StorageSet.cpp b/dbms/src/Storages/StorageSet.cpp index 26af630ca62..c76857bf610 100644 --- a/dbms/src/Storages/StorageSet.cpp +++ b/dbms/src/Storages/StorageSet.cpp @@ -126,7 +126,7 @@ void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block); size_t StorageSet::getSize() const { return set->getTotalRowCount(); } -void StorageSet::truncate(const ASTPtr &, const Context &) +void StorageSet::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { Poco::File(path).remove(true); Poco::File(path).createDirectories(); @@ -193,7 +193,8 @@ void StorageSetOrJoinBase::restoreFromFile(const String & file_path) } -void StorageSetOrJoinBase::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) +void StorageSetOrJoinBase::rename( + const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { /// Rename directory with data. String new_path = new_path_to_db + escapeForFileName(new_table_name); diff --git a/dbms/src/Storages/StorageSet.h b/dbms/src/Storages/StorageSet.h index fe6cd332ed8..671bc78897b 100644 --- a/dbms/src/Storages/StorageSet.h +++ b/dbms/src/Storages/StorageSet.h @@ -22,7 +22,7 @@ public: String getTableName() const override { return table_name; } String getDatabaseName() const override { return database_name; } - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; @@ -69,7 +69,7 @@ public: /// Access the insides. SetPtr & getSet() { return set; } - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; private: SetPtr set; diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp index 447e325095d..00c359f9797 100644 --- a/dbms/src/Storages/StorageStripeLog.cpp +++ b/dbms/src/Storages/StorageStripeLog.cpp @@ -223,7 +223,7 @@ StorageStripeLog::StorageStripeLog( } -void StorageStripeLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) +void StorageStripeLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { std::unique_lock lock(rwlock); @@ -294,7 +294,7 @@ CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Conte return file_checker.check(); } -void StorageStripeLog::truncate(const ASTPtr &, const Context &) +void StorageStripeLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { if (table_name.empty()) throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index d53f000f277..31da4d9d502 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -40,7 +40,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override; @@ -55,7 +55,7 @@ public: String getDataPath() const override { return full_path(); } - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; private: String path; diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 45d9771822e..da7013a62c3 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -378,7 +378,7 @@ void StorageTinyLog::addFiles(const String & column_name, const IDataType & type } -void StorageTinyLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) +void StorageTinyLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { std::unique_lock lock(rwlock); @@ -424,7 +424,7 @@ CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context return file_checker.check(); } -void StorageTinyLog::truncate(const ASTPtr &, const Context &) +void StorageTinyLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { if (table_name.empty()) throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index 1c148acf957..505edd7c556 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -39,7 +39,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override; @@ -54,7 +54,7 @@ public: String getDataPath() const override { return full_path(); } - void truncate(const ASTPtr &, const Context &) override; + void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override; private: String path; From e7cbd34a29ff31d5861d44d0045d9e6749fe26a5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Aug 2019 23:43:35 +0300 Subject: [PATCH 286/357] Improve table locks (incomplete) --- dbms/src/Interpreters/InterpreterRenameQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index 97b17c6afd7..a3b21063ad7 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -93,7 +93,7 @@ BlockIO InterpreterRenameQuery::execute() if (auto table = context.tryGetTable(from.database_name, from.table_name)) tables_from_locks.emplace(from, table->lockExclusively(context.getCurrentQueryId())); - descriptions.back().table_lock = tables_from_locks[from]; + descriptions.back().from_table_lock = tables_from_locks[from]; if (!table_guards.count(from)) table_guards.emplace(from, context.getDDLGuard(from.database_name, from.table_name)); @@ -118,7 +118,7 @@ BlockIO InterpreterRenameQuery::execute() context.assertTableDoesntExist(elem.to_database_name, elem.to_table_name); context.getDatabase(elem.from_database_name)->renameTable( - context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name, elem.table_lock); + context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name, elem.from_table_lock); } return {}; From c8798b2ffac4c8d5c36cb39ae2d80e9f4edf93b1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 02:47:30 +0300 Subject: [PATCH 287/357] Slightly improve table locks --- dbms/src/Interpreters/InterpreterRenameQuery.cpp | 2 +- dbms/src/Storages/IStorage.cpp | 1 + dbms/src/Storages/Kafka/StorageKafka.cpp | 2 +- dbms/src/Storages/Kafka/StorageKafka.h | 11 ++++------- dbms/src/Storages/StorageBuffer.h | 6 +++++- dbms/src/Storages/StorageFile.cpp | 8 +------- dbms/src/Storages/StorageFile.h | 4 +--- dbms/src/Storages/StorageHDFS.cpp | 2 +- dbms/src/Storages/StorageHDFS.h | 2 +- dbms/src/Storages/StorageMaterializedView.cpp | 4 ++-- dbms/src/Storages/StorageMerge.h | 7 +++++-- dbms/src/Storages/StorageNull.h | 2 +- dbms/src/Storages/StorageURL.cpp | 2 +- dbms/src/Storages/StorageURL.h | 2 +- dbms/src/Storages/StorageView.h | 2 +- dbms/src/Storages/System/StorageSystemColumns.cpp | 1 + dbms/src/Storages/System/StorageSystemPartsBase.cpp | 5 +++++ 17 files changed, 33 insertions(+), 30 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index a3b21063ad7..e763c002209 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -113,7 +113,7 @@ BlockIO InterpreterRenameQuery::execute() if (descriptions.size() > 1) lock = context.getLock(); - for (const auto & elem : descriptions) + for (auto & elem : descriptions) { context.assertTableDoesntExist(elem.to_database_name, elem.to_table_name); diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 2f3a48d90b6..9091008a38b 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -22,6 +22,7 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; extern const int SETTINGS_ARE_NOT_SUPPORTED; extern const int UNKNOWN_SETTING; + extern const int TABLE_IS_DROPPED; } IStorage::IStorage(ColumnsDescription virtuals_) : virtuals(std::move(virtuals_)) diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 835ce43b1a4..2d55eb42f1e 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -189,7 +189,7 @@ void StorageKafka::shutdown() } -void StorageKafka::rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name) +void StorageKafka::rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { table_name = new_table_name; database_name = new_database_name; diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index cd55f28820e..2353d1abe44 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -1,16 +1,14 @@ #pragma once #include -#include -#include #include -#include -#include #include #include #include +#include + namespace DB { @@ -40,10 +38,9 @@ public: BlockOutputStreamPtr write( const ASTPtr & query, - const Context & context - ) override; + const Context & context) override; - void rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name) override; + void rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; void updateDependencies() override; diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index b81ca42a0eb..1c565a7d8f0 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -73,7 +73,11 @@ public: void shutdown() override; bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override; - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; } + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override + { + table_name = new_table_name; + database_name = new_database_name; + } bool supportsSampling() const override { return true; } bool supportsPrewhere() const override diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 2db24bbd610..cfd14c58a2d 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -264,13 +264,7 @@ BlockOutputStreamPtr StorageFile::write( } -void StorageFile::drop() -{ - /// Extra actions are not required. -} - - -void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) +void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { if (!is_db_table) throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED); diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index 7268c8ddff0..1410cc5f215 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -38,9 +38,7 @@ public: const ASTPtr & query, const Context & context) override; - void drop() override; - - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; String getDataPath() const override { return path; } diff --git a/dbms/src/Storages/StorageHDFS.cpp b/dbms/src/Storages/StorageHDFS.cpp index aa055f7d907..cb25580248f 100644 --- a/dbms/src/Storages/StorageHDFS.cpp +++ b/dbms/src/Storages/StorageHDFS.cpp @@ -148,7 +148,7 @@ BlockInputStreams StorageHDFS::read( max_block_size)}; } -void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) +void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { table_name = new_table_name; database_name = new_database_name; diff --git a/dbms/src/Storages/StorageHDFS.h b/dbms/src/Storages/StorageHDFS.h index 30a99c9de70..8361916e0e2 100644 --- a/dbms/src/Storages/StorageHDFS.h +++ b/dbms/src/Storages/StorageHDFS.h @@ -30,7 +30,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; protected: StorageHDFS(const String & uri_, diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index c3e52f8a32e..87008fce5bf 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -232,7 +232,7 @@ static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, } -void StorageMaterializedView::drop() +void StorageMaterializedView::drop(TableStructureWriteLockHolder &) { global_context.removeDependency( DatabaseAndTableName(select_database_name, select_table_name), @@ -242,7 +242,7 @@ void StorageMaterializedView::drop() executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_database_name, target_table_name); } -void StorageMaterializedView::truncate(const ASTPtr &, const Context &) +void StorageMaterializedView::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) { if (has_inner_table) executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_database_name, target_table_name); diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index 6708a92c3b0..dbf5d219957 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -42,8 +42,11 @@ public: size_t max_block_size, unsigned num_streams) override; - void drop() override {} - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; } + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override + { + table_name = new_table_name; + database_name = new_database_name; + } /// you need to add and remove columns in the sub-tables manually /// the structure of sub-tables is not checked diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index 04cd5f25e8f..e1a80f3fbaf 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -38,7 +38,7 @@ public: return std::make_shared(getSampleBlock()); } - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override { table_name = new_table_name; database_name = new_database_name; diff --git a/dbms/src/Storages/StorageURL.cpp b/dbms/src/Storages/StorageURL.cpp index ee385af0fe8..4f3d41604f5 100644 --- a/dbms/src/Storages/StorageURL.cpp +++ b/dbms/src/Storages/StorageURL.cpp @@ -187,7 +187,7 @@ BlockInputStreams IStorageURLBase::read(const Names & column_names, return {std::make_shared(block_input, column_defaults, context)}; } -void IStorageURLBase::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) +void IStorageURLBase::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) { table_name = new_table_name; database_name = new_database_name; diff --git a/dbms/src/Storages/StorageURL.h b/dbms/src/Storages/StorageURL.h index 2facca8ce38..cdd78c7b60f 100644 --- a/dbms/src/Storages/StorageURL.h +++ b/dbms/src/Storages/StorageURL.h @@ -29,7 +29,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; protected: IStorageURLBase( diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h index de56f120fa1..6d2e1d04e6f 100644 --- a/dbms/src/Storages/StorageView.h +++ b/dbms/src/Storages/StorageView.h @@ -30,7 +30,7 @@ public: size_t max_block_size, unsigned num_streams) override; - void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override + void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override { table_name = new_table_name; database_name = new_database_name; diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 30b673ddbbb..e4c84de23da 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -18,6 +18,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int TABLE_IS_DROPPED; } StorageSystemColumns::StorageSystemColumns(const std::string & name_) diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index bced500a072..69d11891198 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -17,6 +17,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int TABLE_IS_DROPPED; +} + bool StorageSystemPartsBase::hasStateColumn(const Names & column_names) const { bool has_state_column = false; From 3db38c690e5c012afa2a5623c50d4b10cbc8fbdf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 03:15:08 +0300 Subject: [PATCH 288/357] Changes to Benchmark after merge --- dbms/programs/benchmark/Benchmark.cpp | 4 +- .../src/Common/{T_test.h => StudentTTest.cpp} | 145 ++++++------------ dbms/src/Common/StudentTTest.h | 59 +++++++ 3 files changed, 111 insertions(+), 97 deletions(-) rename dbms/src/Common/{T_test.h => StudentTTest.cpp} (64%) create mode 100644 dbms/src/Common/StudentTTest.h diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp index cb034a258af..fedb7f778a1 100644 --- a/dbms/programs/benchmark/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -32,7 +32,7 @@ #include #include #include -#include +#include /** A tool for evaluating ClickHouse performance. @@ -192,7 +192,7 @@ private: using MultiStats = std::vector>; MultiStats comparison_info_per_interval; MultiStats comparison_info_total; - T_test t_test; + StudentTTest t_test; Stopwatch total_watch; Stopwatch delay_watch; diff --git a/dbms/src/Common/T_test.h b/dbms/src/Common/StudentTTest.cpp similarity index 64% rename from dbms/src/Common/T_test.h rename to dbms/src/Common/StudentTTest.cpp index 1160fd0805e..170f06c2877 100644 --- a/dbms/src/Common/T_test.h +++ b/dbms/src/Common/StudentTTest.cpp @@ -1,55 +1,14 @@ -#pragma once +#include "StudentTTest.h" -#include #include #include #include #include +#include -/** - * About: - * This is implementation of Independent two-sample t-test - * Read about it on https://en.wikipedia.org/wiki/Student%27s_t-test (Equal or unequal sample sizes, equal variance) - * - * Usage: - * It's it used to assume with some level of confidence that two distributions don't differ. - * Values can be added with T_test.add(0/1, value) and after compared and reported with compareAndReport(). - */ -struct T_test + +namespace { - struct DistributionData - { - size_t size = 0; - double sum = 0; - double squares_sum = 0; - - void add(double value) - { - ++size; - sum += value; - squares_sum += value * value; - } - - double avg() const - { - return sum / size; - } - - double var() const - { - return (squares_sum - (sum * sum / size)) / static_cast(size - 1); - } - - void clear() - { - size = 0; - sum = 0; - squares_sum = 0; - } - }; - - std::vector data; - /// First row corresponds to infinity size of distributions case const double students_table[101][6] = { @@ -156,59 +115,55 @@ struct T_test { 1.290, 1.660, 1.984, 2.364, 2.626, 3.174 }, }; - const std::vector confidence_level = { 80, 90, 95, 98, 99, 99.5 }; + const double confidence_level[6] = { 80, 90, 95, 98, 99, 99.5 }; +} - T_test() + +void StudentTTest::clear() +{ + data[0].clear(); + data[1].clear(); +} + +void StudentTTest::add(size_t distribution, double value) +{ + if (distribution > 1) + throw std::logic_error("Distribution number for Student's T-Test must be eigther 0 or 1"); + data[distribution].add(value); +} + +/// Confidence_level_index can be set in range [0, 5]. Corresponding values can be found above. +std::pair StudentTTest::compareAndReport(size_t confidence_level_index) const +{ + if (confidence_level_index > 5) + confidence_level_index = 5; + + if (data[0].size == 0 || data[1].size == 0) + return {true, ""}; + + size_t degrees_of_freedom = (data[0].size - 1) + (data[1].size - 1); + + double table_value = students_table[degrees_of_freedom > 100 ? 0 : degrees_of_freedom][confidence_level_index]; + + double pooled_standard_deviation = sqrt(((data[0].size - 1) * data[0].var() + (data[1].size - 1) * data[1].var()) / degrees_of_freedom); + + double t_statistic = pooled_standard_deviation * sqrt(1.0 / data[0].size + 1.0 / data[1].size); + + double mean_difference = fabs(data[0].avg() - data[1].avg()); + + double mean_confidence_interval = table_value * t_statistic; + + std::stringstream ss; + if (mean_difference > mean_confidence_interval && (mean_difference - mean_confidence_interval > 0.0001)) /// difference must be more than 0.0001, to take into account connection latency. { - data.resize(2); + ss << "Difference at " << confidence_level[confidence_level_index] << "% confidence : "; + ss << std::fixed << std::setprecision(8) << "mean difference is " << mean_difference << ", but confidence interval is " << mean_confidence_interval; + return {false, ss.str()}; } - - void clear() + else { - data[0].clear(); - data[1].clear(); + ss << "No difference proven at " << confidence_level[confidence_level_index] << "% confidence"; + return {true, ss.str()}; } +} - void add(size_t distribution, double value) - { - if (distribution > 1) - return; - data[distribution].add(value); - } - - /// Confidence_level_index can be set in range [0, 5]. Corresponding values can be found above. - std::pair compareAndReport(size_t confidence_level_index = 5) const - { - if (confidence_level_index > 5) - confidence_level_index = 5; - - if (data[0].size == 0 || data[1].size == 0) - return {true, ""}; - - size_t degrees_of_freedom = (data[0].size - 1) + (data[1].size - 1); - - double table_value = students_table[degrees_of_freedom > 100 ? 0 : degrees_of_freedom][confidence_level_index]; - - double pooled_standard_deviation = sqrt(((data[0].size - 1) * data[0].var() + (data[1].size - 1) * data[1].var()) / degrees_of_freedom); - - double t_statistic = pooled_standard_deviation * sqrt(1.0 / data[0].size + 1.0 / data[1].size); - - double mean_difference = fabs(data[0].avg() - data[1].avg()); - - double mean_confidence_interval = table_value * t_statistic; - - std::stringstream ss; - if (mean_difference > mean_confidence_interval && (mean_difference - mean_confidence_interval > 0.0001)) /// difference must be more than 0.0001, to take into account connection latency. - { - ss << "Difference at " << confidence_level[confidence_level_index] << "% confidence : "; - ss << std::fixed << std::setprecision(8) << "mean difference is " << mean_difference << ", but confidence interval is " << mean_confidence_interval; - return {false, ss.str()}; - } - else - { - ss << "No difference proven at " << confidence_level[confidence_level_index] << "% confidence"; - return {true, ss.str()}; - } - } - -}; diff --git a/dbms/src/Common/StudentTTest.h b/dbms/src/Common/StudentTTest.h new file mode 100644 index 00000000000..0816268fba7 --- /dev/null +++ b/dbms/src/Common/StudentTTest.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include + +/** + * About: + * This is implementation of Independent two-sample t-test + * Read about it on https://en.wikipedia.org/wiki/Student%27s_t-test (Equal or unequal sample sizes, equal variance) + * + * Usage: + * It's it used to assume with some level of confidence that two distributions don't differ. + * Values can be added with t_test.add(0/1, value) and after compared and reported with compareAndReport(). + */ +class StudentTTest +{ +private: + struct DistributionData + { + size_t size = 0; + double sum = 0; + double squares_sum = 0; + + void add(double value) + { + ++size; + sum += value; + squares_sum += value * value; + } + + double avg() const + { + return sum / size; + } + + double var() const + { + return (squares_sum - (sum * sum / size)) / static_cast(size - 1); + } + + void clear() + { + size = 0; + sum = 0; + squares_sum = 0; + } + }; + + std::array data; + +public: + void clear(); + + void add(size_t distribution, double value); + + /// Confidence_level_index can be set in range [0, 5]. Corresponding values can be found above. TODO: Trash - no separation of concepts in code. + std::pair compareAndReport(size_t confidence_level_index = 5) const; +}; From f2d081a7857064e3af74e8e41f076810e302361b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 03:42:45 +0300 Subject: [PATCH 289/357] Addition to prev. revision --- dbms/src/Common/StudentTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/StudentTTest.h b/dbms/src/Common/StudentTTest.h index 0816268fba7..b09190050b5 100644 --- a/dbms/src/Common/StudentTTest.h +++ b/dbms/src/Common/StudentTTest.h @@ -47,7 +47,7 @@ private: } }; - std::array data; + std::array data {}; public: void clear(); From 32c7f9688aa5f6bbc60164c82795bd5aed04c304 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 13:13:22 +0300 Subject: [PATCH 290/357] Removed extra quoting from Settings --- dbms/src/Core/Settings.h | 2 +- dbms/src/Core/SettingsCommon.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 88a81027cb6..aa6893d6d85 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -302,7 +302,7 @@ struct Settings : public SettingsCollection M(SettingChar, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.") \ M(SettingBool, format_csv_allow_single_quotes, 1, "If it is set to true, allow strings in single quotes.") \ M(SettingBool, format_csv_allow_double_quotes, 1, "If it is set to true, allow strings in double quotes.") \ - M(SettingBool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \N") \ + M(SettingBool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \\N") \ \ M(SettingDateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.") \ M(SettingBool, log_profile_events, true, "Log query performance statistics into the query_log and query_thread_log.") \ diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index b8c56d50caa..97edfbe9934 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -695,7 +695,7 @@ public: #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ add({[](const Derived & d) { return d.NAME.changed; }, \ - StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ + StringRef(#NAME, strlen(#NAME)), StringRef(DESCRIPTION, strlen(DESCRIPTION)), true, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ @@ -703,7 +703,7 @@ public: #define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ add({[](const Derived & d) { return d.NAME.changed; }, \ - StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \ + StringRef(#NAME, strlen(#NAME)), StringRef(DESCRIPTION, strlen(DESCRIPTION)), false, \ &Functions::NAME##_getString, &Functions::NAME##_getField, \ &Functions::NAME##_setString, &Functions::NAME##_setField, \ &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ From 24a8755b828a42c00aa0bbc501e02bb792c1c548 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 2 Aug 2019 19:30:09 +0300 Subject: [PATCH 291/357] Formatting changes for StringHashMap PR #5417. --- .../AggregateFunctionGroupUniqArray.h | 2 +- dbms/src/Columns/ColumnLowCardinality.cpp | 2 +- dbms/src/Common/HashTable/FixedHashMap.h | 2 +- dbms/src/Common/HashTable/HashMap.h | 18 ++++++------------ dbms/src/Common/HashTable/HashTable.h | 1 - dbms/src/Interpreters/Aggregator.h | 8 -------- 6 files changed, 9 insertions(+), 24 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h index 4b52f1e6fd9..38b67efd6dc 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h @@ -90,7 +90,7 @@ public: auto & set = this->data(place).value; size_t size = set.size(); writeVarUInt(size, buf); - for (auto & elem : set) + for (const auto & elem : set) writeIntBinary(elem, buf); } diff --git a/dbms/src/Columns/ColumnLowCardinality.cpp b/dbms/src/Columns/ColumnLowCardinality.cpp index fa713f76e5e..32ba2378100 100644 --- a/dbms/src/Columns/ColumnLowCardinality.cpp +++ b/dbms/src/Columns/ColumnLowCardinality.cpp @@ -34,7 +34,7 @@ namespace auto & data = res_col->getData(); data.resize(hash_map.size()); - for (auto val : hash_map) + for (const auto & val : hash_map) data[val.getSecond()] = val.getFirst(); for (auto & ind : index) diff --git a/dbms/src/Common/HashTable/FixedHashMap.h b/dbms/src/Common/HashTable/FixedHashMap.h index ae076ddb877..d50c87a6583 100644 --- a/dbms/src/Common/HashTable/FixedHashMap.h +++ b/dbms/src/Common/HashTable/FixedHashMap.h @@ -11,8 +11,8 @@ struct FixedHashMapCell using State = TState; using value_type = PairNoInit; - bool full; Mapped mapped; + bool full; FixedHashMapCell() {} FixedHashMapCell(const Key &, const State &) : full(true) {} diff --git a/dbms/src/Common/HashTable/HashMap.h b/dbms/src/Common/HashTable/HashMap.h index f82563c4449..98669619d3d 100644 --- a/dbms/src/Common/HashTable/HashMap.h +++ b/dbms/src/Common/HashTable/HashMap.h @@ -128,14 +128,12 @@ struct HashMapCellWithSavedHash : public HashMapCell }; -template -< +template < typename Key, typename Cell, typename Hash = DefaultHash, typename Grower = HashTableGrower<>, - typename Allocator = HashTableAllocator -> + typename Allocator = HashTableAllocator> class HashMapTable : public HashTable { public: @@ -173,23 +171,19 @@ public: }; -template -< +template < typename Key, typename Mapped, typename Hash = DefaultHash, typename Grower = HashTableGrower<>, - typename Allocator = HashTableAllocator -> + typename Allocator = HashTableAllocator> using HashMap = HashMapTable, Hash, Grower, Allocator>; -template -< +template < typename Key, typename Mapped, typename Hash = DefaultHash, typename Grower = HashTableGrower<>, - typename Allocator = HashTableAllocator -> + typename Allocator = HashTableAllocator> using HashMapWithSavedHash = HashMapTable, Hash, Grower, Allocator>; diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index c5a0c812ee2..d29459a90d5 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -95,7 +95,6 @@ struct HashTableCell /// Create a cell with the given key / key and value. HashTableCell(const Key & key_, const State &) : key(key_) {} -/// HashTableCell(const value_type & value_, const State & state) : key(value_) {} /// Get what the value_type of the container will be. value_type & getValueMutable() { return key; } diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index c3d1d5df8fd..29e24ab282a 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -196,8 +196,6 @@ struct AggregationMethodString using Data = TData; using Key = typename Data::key_type; using Mapped = typename Data::mapped_type; - using iterator = typename Data::iterator; - using const_iterator = typename Data::const_iterator; Data data; @@ -224,8 +222,6 @@ struct AggregationMethodFixedString using Data = TData; using Key = typename Data::key_type; using Mapped = typename Data::mapped_type; - using iterator = typename Data::iterator; - using const_iterator = typename Data::const_iterator; Data data; @@ -254,8 +250,6 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod using Data = typename Base::Data; using Key = typename Base::Key; using Mapped = typename Base::Mapped; - using iterator = typename Base::iterator; - using const_iterator = typename Base::const_iterator; using Base::data; @@ -365,8 +359,6 @@ struct AggregationMethodSerialized using Data = TData; using Key = typename Data::key_type; using Mapped = typename Data::mapped_type; - using iterator = typename Data::iterator; - using const_iterator = typename Data::const_iterator; Data data; From 0b5a7f5f4a0aa0b42b9890160d49444eaed8f13d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 28 Aug 2019 14:03:29 +0300 Subject: [PATCH 292/357] Disable consecutive key optimization for UInt8/16 LowCardinality columns. A follow-up to #6298, this is as well a part of StringHashMap PR #5417 by Amos Bird. --- dbms/src/Interpreters/Aggregator.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index c3d1d5df8fd..6a27c426283 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -460,8 +460,8 @@ struct AggregatedDataVariants : private boost::noncopyable std::unique_ptr> nullable_keys256_two_level; /// Support for low cardinality. - std::unique_ptr>> low_cardinality_key8; - std::unique_ptr>> low_cardinality_key16; + std::unique_ptr>> low_cardinality_key8; + std::unique_ptr>> low_cardinality_key16; std::unique_ptr>> low_cardinality_key32; std::unique_ptr>> low_cardinality_key64; std::unique_ptr>> low_cardinality_key_string; From 99d62efb4d543f265b91258cc53f3c1ab228e966 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 15:26:13 +0300 Subject: [PATCH 293/357] Added a test --- .../queries/0_stateless/00999_settings_no_extra_quotes.reference | 1 + .../tests/queries/0_stateless/00999_settings_no_extra_quotes.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.reference create mode 100644 dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.sql diff --git a/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.reference b/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.reference @@ -0,0 +1 @@ +0 diff --git a/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.sql b/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.sql new file mode 100644 index 00000000000..55d9ff2780d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_settings_no_extra_quotes.sql @@ -0,0 +1 @@ +SELECT DISTINCT description LIKE '"%"' FROM system.settings; From a1101e5278b24781e97fded4251ffe5a04e7b9c5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 16:21:19 +0300 Subject: [PATCH 294/357] Fixed build --- dbms/src/Databases/DatabasesCommon.h | 1 + dbms/src/Storages/Kafka/KafkaBlockInputStream.h | 2 ++ dbms/src/Storages/Kafka/StorageKafka.h | 2 ++ 3 files changed, 5 insertions(+) diff --git a/dbms/src/Databases/DatabasesCommon.h b/dbms/src/Databases/DatabasesCommon.h index 00190d89b1e..734708e4c95 100644 --- a/dbms/src/Databases/DatabasesCommon.h +++ b/dbms/src/Databases/DatabasesCommon.h @@ -4,6 +4,7 @@ #include #include #include +#include /// General functionality for several different database engines. diff --git a/dbms/src/Storages/Kafka/KafkaBlockInputStream.h b/dbms/src/Storages/Kafka/KafkaBlockInputStream.h index fef7f8d0469..011ed5fe046 100644 --- a/dbms/src/Storages/Kafka/KafkaBlockInputStream.h +++ b/dbms/src/Storages/Kafka/KafkaBlockInputStream.h @@ -4,6 +4,8 @@ #include #include +#include + namespace DB { diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index 2353d1abe44..51a06a890db 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include From 9d0e5b925bb2376d5e0a548452eb49a6fce14bd6 Mon Sep 17 00:00:00 2001 From: akonyaev Date: Wed, 28 Aug 2019 16:26:38 +0300 Subject: [PATCH 295/357] ADQM-40 test for orc input format --- .../0_stateless/00900_orc_load.reference | 2 ++ .../queries/0_stateless/00900_orc_load.sh | 17 +++++++++++++++++ .../queries/0_stateless/data_orc/test.orc | Bin 0 -> 771 bytes 3 files changed, 19 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00900_orc_load.reference create mode 100755 dbms/tests/queries/0_stateless/00900_orc_load.sh create mode 100644 dbms/tests/queries/0_stateless/data_orc/test.orc diff --git a/dbms/tests/queries/0_stateless/00900_orc_load.reference b/dbms/tests/queries/0_stateless/00900_orc_load.reference new file mode 100644 index 00000000000..fe79e37ee18 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00900_orc_load.reference @@ -0,0 +1,2 @@ +0 0 0 0 0 2019-01-01 test1 +2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 diff --git a/dbms/tests/queries/0_stateless/00900_orc_load.sh b/dbms/tests/queries/0_stateless/00900_orc_load.sh new file mode 100755 index 00000000000..cd553f6d234 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00900_orc_load.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CUR_DIR/../shell_config.sh + +CB_DIR=$(dirname "$CLICKHOUSE_CLIENT_BINARY") +[ "$CB_DIR" == "." ] && ROOT_DIR=$CUR_DIR/../../../.. +[ "$CB_DIR" != "." ] && BUILD_DIR=$CB_DIR/../.. +[ -z "$ROOT_DIR" ] && ROOT_DIR=$CB_DIR/../../.. + +DATA_FILE=$CUR_DIR/data_orc/test.orc + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_load" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (int Int32, smallint Int8, bigint Int64, float Float32, double Float64, date Date, y String) ENGINE = Memory" +cat $DATA_FILE | ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" +${CLICKHOUSE_CLIENT} --query="select * from orc_load" + diff --git a/dbms/tests/queries/0_stateless/data_orc/test.orc b/dbms/tests/queries/0_stateless/data_orc/test.orc new file mode 100644 index 0000000000000000000000000000000000000000..1b2c9aa492203b672ef40a48424ed1128e615544 GIT binary patch literal 771 zcmeYdau#G@;9?VE;b012uwY<#Jn30dT0(+=(8C0wM~@Q(etiG`zrK#sK=J?sgE3I0 zI2SVm1A~wN2a^yN2ZO-B|I89V9Kc`()Ee{@taZ}k1ffX}fLcGxb4jQQoMB*NumQ*kLL-n4Hxu6zS?BtD4vYG(sO}xhTYP zJ`N@;4vtIHnFN41fWZpn>gP$R42+6~d}^wUdeA%1Sb_QS=fWXBK-<+fv7?Kkj8eo<)f;Dts>b?Xt zrX;nv#1KRnu`@8ZF^0}#VDLTT&DCJQ!*W4wZRT2zqgNFkz5my4!WUa&XfsK1W!M^4 zov+~qfkz#c{bqf*z^QNg#^iC?(f47iW@Nr8eiZ+4x7*t9GxzS=mdPn6D|GBG`wyUB zZXG^x^w2?(0|!(O9aK4T{Gh^z_du8b5lRr!&|(mI%G7Pdd~`;mQjvoo=YkXS{|6k< zXiQ*wDrz}n$r4M+1r7ooqD$5twe&b7F~h5GrLI?ss9@=o7Q+Uw9~Lv#SXk)!rVBA7 zDa7|QdwMo>Z&lg4<46F9kECY@iva_}21bU*31Tx2=o~$JQ0K#hKqW6lk7m!4UCr(t zb0>DrY@OO0+hu9I-lc1?g}I?R`;jE?Q$Fk$*JN7$>~zTUs>=-WTP5P{mE^ffBzdWj zFuSpUF{2^3z-A^xX@SqohPxRji*mCEGc-@-V4o}?&)$4sKhR^Jy&F6itY8S3!mV?P zMja~#N`q4UKWE0pEKQtK!VxJZY{D+F=L($330juRwrNZZb-5BKbc{jk;p2o& zj~?tWnd`~W(O6@+;5F-x2=Ab7Z=TizQzo`}Ff<=#v1j18%_zVq(a<2E#Kgd(QE$N3 M^o5z(Kgd}E06(l5t^fc4 literal 0 HcmV?d00001 From 90d5410aaa6e7e641efe7b67e0496ab53f2a64ab Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 28 Aug 2019 17:14:09 +0300 Subject: [PATCH 296/357] join_use_nulls with not nullable types --- dbms/src/Interpreters/ExpressionActions.cpp | 8 ++--- dbms/src/Interpreters/Join.cpp | 4 +-- .../00999_join_not_nullable_types.reference | 8 +++++ .../00999_join_not_nullable_types.sql | 34 +++++++++++++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00999_join_not_nullable_types.reference create mode 100644 dbms/tests/queries/0_stateless/00999_join_not_nullable_types.sql diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 5ef05569f91..0083820d6e9 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -291,8 +291,8 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings, bool make_nullable = is_null_used_as_default && right_or_full_join; - if (make_nullable && !col.type->isNullable()) - col.type = std::make_shared(col.type); + if (make_nullable && col.type->canBeInsideNullable()) + col.type = makeNullable(col.type); } for (const auto & col : columns_added_by_join) @@ -316,8 +316,8 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings, } } - if (make_nullable && !res_type->isNullable()) - res_type = std::make_shared(res_type); + if (make_nullable && res_type->canBeInsideNullable()) + res_type = makeNullable(res_type); sample_block.insert(ColumnWithTypeAndName(nullptr, res_type, col.name)); } diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 8e91424bc21..63bf88a8437 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -50,7 +50,7 @@ static std::unordered_map requiredRightKeys(const Names & k static void convertColumnToNullable(ColumnWithTypeAndName & column) { - if (column.type->isNullable()) + if (column.type->isNullable() || !column.type->canBeInsideNullable()) return; column.type = makeNullable(column.type); @@ -71,7 +71,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, if (nullable) { convertColumnToNullable(column); - if (negative_null_map.size()) + if (column.type->isNullable() && negative_null_map.size()) { MutableColumnPtr mutable_column = (*std::move(column.column)).mutate(); assert_cast(*mutable_column).applyNegatedNullMap(negative_null_map); diff --git a/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.reference b/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.reference new file mode 100644 index 00000000000..7b6947fa9a2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.reference @@ -0,0 +1,8 @@ +0 ['left'] 0 ['left'] \N +1 ['left'] 1 ['left'] 1 +2 [] \N [] 2 +['left'] 0 ['left'] \N +['left'] 1 ['left'] 1 +[] \N [] 2 +['left'] 42 \N +['right'] \N 42 diff --git a/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.sql b/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.sql new file mode 100644 index 00000000000..2a24c6dd296 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_join_not_nullable_types.sql @@ -0,0 +1,34 @@ +SET join_use_nulls = 1; + +SELECT * FROM +( + SELECT number, ['left'] as ar, number AS left_number FROM system.numbers LIMIT 2 +) +FULL JOIN +( + SELECT number, ['right'] as ar, number AS right_number FROM system.numbers LIMIT 1, 2 +) +USING (number) +ORDER BY number; + +SELECT * FROM +( + SELECT ['left'] as ar, number AS left_number FROM system.numbers LIMIT 2 +) +FULL JOIN +( + SELECT ['right'] as ar, number AS right_number FROM system.numbers LIMIT 1, 2 +) +ON left_number = right_number +ORDER BY left_number; + +SELECT * FROM +( + SELECT ['left'] as ar, 42 AS left_number +) +FULL JOIN +( + SELECT ['right'] as ar, 42 AS right_number +) +USING(ar) +ORDER BY left_number; From 10b878b30b9ef7237b269487d843f6b662036878 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 18:05:38 +0300 Subject: [PATCH 297/357] Disable Poco::AbstractConfiguration substitutions in query in clickhouse-client --- dbms/programs/client/Client.cpp | 2 +- dbms/programs/local/LocalServer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 6caf57a28bd..0bb6cf62f90 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -672,7 +672,7 @@ private: String text; if (config().has("query")) - text = config().getString("query"); + text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query. else { /// If 'query' parameter is not set, read a query from stdin. diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index 54383050b6c..1844c037784 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -268,7 +268,7 @@ void LocalServer::attachSystemTables() void LocalServer::processQueries() { String initial_create_query = getInitialCreateTableQuery(); - String queries_str = initial_create_query + config().getString("query"); + String queries_str = initial_create_query + config().getRawString("query"); std::vector queries; auto parse_res = splitMultipartQuery(queries_str, queries); From f519234908a585895a6103f8b1307ab3479c6b05 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 18:13:19 +0300 Subject: [PATCH 298/357] Added a test --- .../01000_unneeded_substitutions_client.reference | 1 + .../0_stateless/01000_unneeded_substitutions_client.sh | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.reference create mode 100755 dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh diff --git a/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.reference b/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.reference new file mode 100644 index 00000000000..13a393df666 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.reference @@ -0,0 +1 @@ +${} diff --git a/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh b/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh new file mode 100755 index 00000000000..f6517fc2a42 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01000_unneeded_substitutions_client.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "SELECT '\${}'" From d06e3503123f616f0a95b36aff4d7406db476625 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 18:20:22 +0300 Subject: [PATCH 299/357] Added setting "replace_running_query_max_wait_ms" --- dbms/src/Core/Settings.h | 1 + dbms/src/Interpreters/ProcessList.cpp | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 92f882e4fc5..f6b65ed19fe 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -62,6 +62,7 @@ struct Settings : public SettingsCollection M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "") \ M(SettingSeconds, tcp_keep_alive_timeout, 0, "") \ M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \ + M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.") \ M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.") \ M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.") \ M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.") \ diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp index 5a13477147c..71376c6d129 100644 --- a/dbms/src/Interpreters/ProcessList.cpp +++ b/dbms/src/Interpreters/ProcessList.cpp @@ -87,10 +87,10 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as { std::unique_lock lock(mutex); - const auto max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds(); + const auto queue_max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds(); if (!is_unlimited_query && max_size && processes.size() >= max_size) { - if (!max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(max_wait_ms), [&]{ return processes.size() < max_size; })) + if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), [&]{ return processes.size() < max_size; })) throw Exception("Too many simultaneous queries. Maximum: " + toString(max_size), ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES); } @@ -127,7 +127,9 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as /// Ask queries to cancel. They will check this flag. running_query->second->is_killed.store(true, std::memory_order_relaxed); - if (!max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(max_wait_ms), [&] + const auto replace_running_query_max_wait_ms = settings.replace_running_query_max_wait_ms.totalMilliseconds(); + if (!replace_running_query_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(replace_running_query_max_wait_ms), + [&] { running_query = user_process_list->second.queries.find(client_info.current_query_id); if (running_query == user_process_list->second.queries.end()) @@ -135,8 +137,10 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as running_query->second->is_killed.store(true, std::memory_order_relaxed); return false; })) + { throw Exception("Query with id = " + client_info.current_query_id + " is already running and can't be stopped", ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING); + } } } } From 06c0e35e46b4cebcb545112573801fb61dc2a2c1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 18:21:57 +0300 Subject: [PATCH 300/357] Added setting "replace_running_query_max_wait_ms" --- dbms/tests/queries/0_stateless/00600_replace_running_query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00600_replace_running_query.sh b/dbms/tests/queries/0_stateless/00600_replace_running_query.sh index dbbf41dd772..9fc25291548 100755 --- a/dbms/tests/queries/0_stateless/00600_replace_running_query.sh +++ b/dbms/tests/queries/0_stateless/00600_replace_running_query.sh @@ -35,6 +35,6 @@ wait ${CLICKHOUSE_CLIENT} --query_id=42 --query='SELECT 3, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & wait_for_query_to_start '42' -${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --queue_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null +${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --replace_running_query_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null ${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --query='SELECT 44' wait From d1c268c2ed030c3346b54dd9cdf680032e6ac612 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 18:24:23 +0300 Subject: [PATCH 301/357] Added setting "kafka_max_wait_ms" --- dbms/src/Core/Settings.h | 1 + dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index f6b65ed19fe..a1e5ad73c46 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -63,6 +63,7 @@ struct Settings : public SettingsCollection M(SettingSeconds, tcp_keep_alive_timeout, 0, "") \ M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \ M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.") \ + M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.") \ M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.") \ M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.") \ M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.") \ diff --git a/dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp b/dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp index 1962e4fbc63..09f97f8c836 100644 --- a/dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -43,7 +43,7 @@ Block KafkaBlockInputStream::getHeader() const void KafkaBlockInputStream::readPrefixImpl() { - auto timeout = std::chrono::milliseconds(context.getSettingsRef().queue_max_wait_ms.totalMilliseconds()); + auto timeout = std::chrono::milliseconds(context.getSettingsRef().kafka_max_wait_ms.totalMilliseconds()); buffer = storage.popReadBuffer(timeout); claimed = !!buffer; From cced091881743354abbd0a05626a6bff3552484e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 18:24:35 +0300 Subject: [PATCH 302/357] Addition to prev. revision --- dbms/src/Interpreters/ClusterProxy/executeQuery.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index 25d5a6eb0d4..d2c11fd0feb 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -18,6 +18,7 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin { Settings new_settings = settings; new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time); + new_settings.replace_running_query_max_wait_ms = Cluster::saturate(new_settings.replace_running_query_max_wait_ms, settings.max_execution_time); /// Does not matter on remote servers, because queries are sent under different user. new_settings.max_concurrent_queries_for_user = 0; From b6a0dba2df192f5f434c76e0bbcb0314c13ad808 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 18:27:26 +0300 Subject: [PATCH 303/357] Added a setting "connection_pool_max_wait_ms" --- dbms/src/Client/ConnectionPool.h | 2 +- dbms/src/Core/Settings.h | 1 + dbms/src/Interpreters/ClusterProxy/executeQuery.cpp | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/src/Client/ConnectionPool.h b/dbms/src/Client/ConnectionPool.h index 322bad04794..1ecb432c827 100644 --- a/dbms/src/Client/ConnectionPool.h +++ b/dbms/src/Client/ConnectionPool.h @@ -74,7 +74,7 @@ public: { Entry entry; if (settings) - entry = Base::get(settings->queue_max_wait_ms.totalMilliseconds()); + entry = Base::get(settings->connection_pool_max_wait_ms.totalMilliseconds()); else entry = Base::get(-1); diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index a1e5ad73c46..0be8279d3e0 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -62,6 +62,7 @@ struct Settings : public SettingsCollection M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "") \ M(SettingSeconds, tcp_keep_alive_timeout, 0, "") \ M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \ + M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when connection pool is full.") \ M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.") \ M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.") \ M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.") \ diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index d2c11fd0feb..dc0d3ef27b1 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -18,6 +18,7 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin { Settings new_settings = settings; new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time); + new_settings.connection_pool_max_wait_ms = Cluster::saturate(new_settings.connection_pool_max_wait_ms, settings.max_execution_time); new_settings.replace_running_query_max_wait_ms = Cluster::saturate(new_settings.replace_running_query_max_wait_ms, settings.max_execution_time); /// Does not matter on remote servers, because queries are sent under different user. From 5e27937227b211865cdb3e169495665fc2c02e23 Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 28 Aug 2019 18:44:18 +0300 Subject: [PATCH 304/357] fix RIGHT JOIN for Tuples --- dbms/src/Columns/ColumnTuple.cpp | 10 +++++ dbms/src/Columns/ColumnTuple.h | 1 + ...00999_nullable_nested_types_4877.reference | 26 ++++++++++++ .../00999_nullable_nested_types_4877.sql | 41 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.reference create mode 100644 dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.sql diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp index 07599b3456f..3e3e311270f 100644 --- a/dbms/src/Columns/ColumnTuple.cpp +++ b/dbms/src/Columns/ColumnTuple.cpp @@ -81,6 +81,16 @@ MutableColumnPtr ColumnTuple::cloneEmpty() const return ColumnTuple::create(std::move(new_columns)); } +MutableColumnPtr ColumnTuple::cloneResized(size_t new_size) const +{ + const size_t tuple_size = columns.size(); + MutableColumns new_columns(tuple_size); + for (size_t i = 0; i < tuple_size; ++i) + new_columns[i] = columns[i]->cloneResized(new_size); + + return ColumnTuple::create(std::move(new_columns)); +} + Field ColumnTuple::operator[](size_t n) const { return Tuple{ext::map(columns, [n] (const auto & column) { return (*column)[n]; })}; diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h index 65dd19fc6da..e5e47ac74db 100644 --- a/dbms/src/Columns/ColumnTuple.h +++ b/dbms/src/Columns/ColumnTuple.h @@ -42,6 +42,7 @@ public: const char * getFamilyName() const override { return "Tuple"; } MutableColumnPtr cloneEmpty() const override; + MutableColumnPtr cloneResized(size_t size) const override; size_t size() const override { diff --git a/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.reference b/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.reference new file mode 100644 index 00000000000..1e2036c94c7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.reference @@ -0,0 +1,26 @@ +a ('b','c') ('b','c') +d ('e','f') ('','') +a +x +a ('b','c') ('b','c') +x ('','') ('y','z') +a +d +a +x +a ('b','c') ('b','c') +d ('e','f') ('','') +a ('b','c') ('b','c') +x ('','') ('y','z') +a b ['b','c'] +d e [] +a b ['b','c'] +x ['y','z'] +a +d +a +x +a b ['b','c'] +d e [] +a b ['b','c'] +x \N ['y','z'] diff --git a/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.sql b/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.sql new file mode 100644 index 00000000000..ca523d77235 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_nullable_nested_types_4877.sql @@ -0,0 +1,41 @@ +DROP TABLE IF EXISTS l; +DROP TABLE IF EXISTS r; + +CREATE TABLE l (a String, b Tuple(String, String)) ENGINE = Memory(); +CREATE TABLE r (a String, c Tuple(String, String)) ENGINE = Memory(); + +INSERT INTO l (a, b) VALUES ('a', ('b', 'c')), ('d', ('e', 'f')); +INSERT INTO r (a, c) VALUES ('a', ('b', 'c')), ('x', ('y', 'z')); + +SET join_use_nulls = 0; +SELECT * from l LEFT JOIN r USING a ORDER BY a; +SELECT a from l RIGHT JOIN r USING a ORDER BY a; +SELECT * from l RIGHT JOIN r USING a ORDER BY a; + +SET join_use_nulls = 1; +SELECT a from l LEFT JOIN r USING a ORDER BY a; +SELECT a from l RIGHT JOIN r USING a ORDER BY a; +SELECT * from l LEFT JOIN r USING a ORDER BY a; +SELECT * from l RIGHT JOIN r USING a ORDER BY a; + +DROP TABLE l; +DROP TABLE r; + +CREATE TABLE l (a String, b String) ENGINE = Memory(); +CREATE TABLE r (a String, c Array(String)) ENGINE = Memory(); + +INSERT INTO l (a, b) VALUES ('a', 'b'), ('d', 'e'); +INSERT INTO r (a, c) VALUES ('a', ['b', 'c']), ('x', ['y', 'z']); + +SET join_use_nulls = 0; +SELECT * from l LEFT JOIN r USING a ORDER BY a; +SELECT * from l RIGHT JOIN r USING a ORDER BY a; + +SET join_use_nulls = 1; +SELECT a from l LEFT JOIN r USING a ORDER BY a; +SELECT a from l RIGHT JOIN r USING a ORDER BY a; +SELECT * from l LEFT JOIN r USING a ORDER BY a; +SELECT * from l RIGHT JOIN r USING a ORDER BY a; + +DROP TABLE l; +DROP TABLE r; From 6d6c53d42bb251e185594ab971c23645ad35ca3b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 19:15:57 +0300 Subject: [PATCH 305/357] Style --- libs/libcommon/include/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/libcommon/include/common/DateLUTImpl.h b/libs/libcommon/include/common/DateLUTImpl.h index 2258620eb26..ef50d6ede3f 100644 --- a/libs/libcommon/include/common/DateLUTImpl.h +++ b/libs/libcommon/include/common/DateLUTImpl.h @@ -28,7 +28,7 @@ enum class WeekModeFlag : UInt8 FIRST_WEEKDAY = 4, NEWYEAR_DAY = 8 }; -typedef std::pair YearWeek; +using YearWeek = std::pair; /** Lookup table to conversion of time to date, and to month / year / day of week / day of month and so on. * First time was implemented for OLAPServer, that needed to do billions of such transformations. From 91bc0eca11f35cb6a1dc84bb7eaf3d71ad534cb3 Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 28 Aug 2019 20:00:20 +0300 Subject: [PATCH 306/357] add test for #4858 --- .../0_stateless/00881_unknown_identifier_in_in.reference | 1 + .../queries/0_stateless/00881_unknown_identifier_in_in.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.reference create mode 100644 dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.sql diff --git a/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.reference b/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.reference @@ -0,0 +1 @@ +1 diff --git a/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.sql b/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.sql new file mode 100644 index 00000000000..2ce709c45be --- /dev/null +++ b/dbms/tests/queries/0_stateless/00881_unknown_identifier_in_in.sql @@ -0,0 +1,4 @@ +SELECT toUInt64(1) x FROM (select 1) +GROUP BY 1 +HAVING x +IN ( SELECT countIf(y, z == 1) FROM (SELECT 1 y, 1 z) ); From b4d98f6e13ac83c10ee73e38008a816e01d6c97d Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 28 Aug 2019 21:23:20 +0300 Subject: [PATCH 307/357] Trying to fix vertical merge --- dbms/src/Common/ErrorCodes.cpp | 1 + dbms/src/Interpreters/ExpressionActions.cpp | 2 +- .../MergeTree/IMergedBlockOutputStream.cpp | 4 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 38 +++++++-- dbms/src/Storages/MergeTree/MergeTreeData.h | 16 ++-- .../MergeTree/MergeTreeDataMergerMutator.cpp | 17 +++- .../Storages/MergeTree/MergeTreeIndices.cpp | 1 + .../src/Storages/MergeTree/MergeTreeIndices.h | 14 ++++ ...kip_indices_with_alter_and_merge.reference | 2 + ...test_skip_indices_with_alter_and_merge.sql | 79 +++++++++++++++++++ 10 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.reference create mode 100644 dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 87ab252c583..977e1f4425d 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -447,6 +447,7 @@ namespace ErrorCodes extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW = 470; extern const int SETTINGS_ARE_NOT_SUPPORTED = 471; extern const int IMMUTABLE_SETTING = 472; + extern const int UNSUPPORTED_SKIP_INDEX_EXPRESSION = 473; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index c7b510abcf0..88cd69fe015 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -726,7 +726,7 @@ void ExpressionActions::addImpl(ExpressionAction action, Names & new_names) new_names.push_back(action.result_name); new_names.insert(new_names.end(), action.array_joined_columns.begin(), action.array_joined_columns.end()); - /// Compiled functions are custom functions and them don't need building + /// Compiled functions are custom functions and they don't need building if (action.type == ExpressionAction::APPLY_FUNCTION && !action.is_function_compiled) { if (sample_block.has(action.result_name)) diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 255fb998446..e673fd4759a 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -336,9 +336,9 @@ void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( size_t skip_index_current_mark = 0; /// Filling and writing skip indices like in IMergedBlockOutputStream::writeColumn - for (size_t i = 0; i < storage.skip_indices.size(); ++i) + for (size_t i = 0; i < skip_indices.size(); ++i) { - const auto index = storage.skip_indices[i]; + const auto index = skip_indices[i]; auto & stream = *skip_indices_streams[i]; size_t prev_pos = 0; skip_index_current_mark = skip_index_mark; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b2d4a4b9d73..bc34a81c6c2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -91,6 +91,7 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int BAD_DATA_PART_NAME; extern const int UNKNOWN_SETTING; + extern const int UNSUPPORTED_SKIP_INDEX_EXPRESSION; } @@ -349,6 +350,7 @@ void MergeTreeData::setProperties( MergeTreeIndices new_indices; + auto settings_ptr = getSettings(); if (!indices_description.indices.empty()) { std::set indices_names; @@ -357,11 +359,18 @@ void MergeTreeData::setProperties( { const auto & index_decl = std::dynamic_pointer_cast(index_ast); - new_indices.push_back( - MergeTreeIndexFactory::instance().get( - all_columns, - std::dynamic_pointer_cast(index_decl->clone()), - global_context)); + auto index_ptr = MergeTreeIndexFactory::instance().get( + all_columns, + std::dynamic_pointer_cast(index_decl->clone()), + global_context); + + if (index_ptr->getColumnsRequiredForIndexCalc().size() > 1 && settings_ptr->enable_vertical_merge_algorithm) + throw Exception("Index '" + index_ptr->name + "' contains expression with multiple columns and " + + "'enable_vertical_merge_algorithm' is set to true in storage settings. " + + "Disable vertical merge or use only one column in index expression.", + ErrorCodes::UNSUPPORTED_SKIP_INDEX_EXPRESSION); + + new_indices.push_back(std::move(index_ptr)); if (indices_names.find(new_indices.back()->name) != indices_names.end()) throw Exception( @@ -1293,7 +1302,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c } if (columns_alter_forbidden.count(command.column_name)) - throw Exception("trying to ALTER key column " + command.column_name, ErrorCodes::ILLEGAL_COLUMN); + throw Exception("Trying to ALTER key column " + command.column_name, ErrorCodes::ILLEGAL_COLUMN); if (columns_alter_metadata_only.count(command.column_name)) { @@ -1600,7 +1609,7 @@ void MergeTreeData::alterDataPart( true /* sync */, compression_codec, true /* skip_offsets */, - {}, + {}, /// currently restricted unused_written_offsets, part->index_granularity, &part->index_granularity_info); @@ -3085,4 +3094,19 @@ bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const return true; } + +std::vector MergeTreeData::getIndicesForColumn(const String & column_name) const +{ + std::vector result; + + for (size_t i = 0; i < skip_indices.size(); ++i) + { + const auto & index_columns = skip_indices[i]->getColumnsRequiredForIndexCalc(); + if (std::find(index_columns.begin(), index_columns.end(), column_name) != index_columns.end()) + result.emplace_back(skip_indices[i]); + } + + return result; +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 0440a3181c8..6cf1f50dabb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -621,6 +621,15 @@ public: (settings->enable_mixed_granularity_parts || !has_non_adaptive_index_granularity_parts); } + /// Get constant pointer to storage settings. + /// Copy this pointer into your scope and you will + /// get consistent settings. + MergeTreeSettingsPtr getSettings() const + { + return storage_settings.get(); + } + + std::vector getIndicesForColumn(const String & column_name) const; MergeTreeDataFormatVersion format_version; @@ -679,13 +688,6 @@ public: bool has_non_adaptive_index_granularity_parts = false; - /// Get constant pointer to storage settings. - /// Copy this pointer into your scope and you will - /// get consistent settings. - MergeTreeSettingsPtr getSettings() const - { - return storage_settings.get(); - } protected: diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index ad489a91603..1742a4463b2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -825,6 +825,17 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor rows_sources_read_buf.seek(0, 0); ColumnGathererStream column_gathered_stream(column_name, column_part_streams, rows_sources_read_buf); + + std::vector skip_idx_to_recalc = data.getIndicesForColumn(column_name); + for (const auto & idx : skip_idx_to_recalc) + if (idx->getColumnsRequiredForIndexCalc().size() > 1) + throw Exception("Skip index '" + idx->name + "' has expression on multiple columns. " + + "Vertical merge is not supported for tables with skip indices with expressions on multiple columns. " + + "It's better to avoid indices with multiple columns in expression. " + + "Also you can disable vertical merges with setting enable_vertical_merge_algorithm=0, " + + "but it will lead to additional memory consuption for big merges.", + ErrorCodes::LOGICAL_ERROR); + MergedColumnOnlyOutputStream column_to( data, column_gathered_stream.getHeader(), @@ -832,7 +843,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor false, compression_codec, false, - {}, + skip_idx_to_recalc, written_offset_columns, to.getIndexGranularity() ); @@ -1017,8 +1028,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor { const auto & index = data.skip_indices[i]; const auto & index_cols = index->expr->getRequiredColumns(); - auto it = find(cbegin(index_cols), cend(index_cols), col); - if (it != cend(index_cols) && indices_to_recalc.insert(index).second) + auto it = std::find(std::cbegin(index_cols), std::cend(index_cols), col); + if (it != std::cend(index_cols) && indices_to_recalc.insert(index).second) { ASTPtr expr_list = MergeTreeData::extractKeyExpressionList( storage_from_source_part->getIndices().indices[i]->expr->clone()); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp index e19aafbd25d..a799cc5cffb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp @@ -51,6 +51,7 @@ std::unique_ptr MergeTreeIndexFactory::get( return lft + ", " + rht.first; }), ErrorCodes::INCORRECT_QUERY); + return it->second(columns, node, context); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndices.h b/dbms/src/Storages/MergeTree/MergeTreeIndices.h index c430d1e8135..0b626d5e10a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndices.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndices.h @@ -104,11 +104,25 @@ public: virtual MergeTreeIndexConditionPtr createIndexCondition( const SelectQueryInfo & query_info, const Context & context) const = 0; + Names getColumnsRequiredForIndexCalc() const { return expr->getRequiredColumns(); } + + /// Index name String name; + + /// Index expression with columns arguments ExpressionActionsPtr expr; + + /// Names of columns which are used + /// to calculate expression for index Names columns; + + /// Data types of columns DataTypes data_types; + + /// Block with columns and data_types Block header; + + /// Skip index granularity size_t granularity; }; diff --git a/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.reference b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.reference new file mode 100644 index 00000000000..5aad65487ae --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.reference @@ -0,0 +1,2 @@ +201 +201 diff --git a/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql new file mode 100644 index 00000000000..c67f6c594ff --- /dev/null +++ b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql @@ -0,0 +1,79 @@ +SET allow_experimental_data_skipping_indices=1; +DROP TABLE IF EXISTS table_test_creation; + +CREATE TABLE table_test_creation ( + k UInt64, + val1 UInt64, + val2 UInt64, + INDEX idx1 val1 * val2 TYPE minmax GRANULARITY 1 +) ENGINE MergeTree() +ORDER BY k; -- { serverError 473 } + +CREATE TABLE table_test_creation ( + k UInt64, + val1 UInt64, + val2 UInt64 +) ENGINE MergeTree() +ORDER BY k; + +ALTER TABLE table_test_creation ADD INDEX idx1 val1 * val2 TYPE minmax GRANULARITY 1; -- { serverError 473 } + +ALTER TABLE table_test_creation ADD INDEX idx1 val1 TYPE minmax GRANULARITY 1; + +ALTER TABLE table_test_creation MODIFY SETTING enable_vertical_merge_algorithm=0; + +ALTER TABLE table_test_creation ADD INDEX idx2 val1 * val2 TYPE minmax GRANULARITY 1; + +DROP TABLE IF EXISTS table_test_creation; + +CREATE TABLE table_test_creation ( + k UInt64, + val1 UInt64, + val2 UInt64, + INDEX idx1 val1 * val2 TYPE minmax GRANULARITY 1 +) ENGINE MergeTree() +ORDER BY k SETTINGS enable_vertical_merge_algorithm=0; + +DROP TABLE IF EXISTS table_test_creation; + +DROP TABLE IF EXISTS test_vertical_merge; + +CREATE TABLE test_vertical_merge ( + k UInt64, + val1 UInt64, + val2 UInt64, + INDEX idx1 val2 TYPE minmax GRANULARITY 1 +) ENGINE MergeTree() +ORDER BY k +SETTINGS vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1; + +INSERT INTO test_vertical_merge SELECT number, number + 5, number * 12 from numbers(1000); + +SELECT COUNT() from test_vertical_merge WHERE val2 <= 2400; + +OPTIMIZE TABLE test_vertical_merge FINAL; + +SELECT COUNT() from test_vertical_merge WHERE val2 <= 2400; + +DROP TABLE IF EXISTS test_vertical_merge; + +--DROP TABLE IF EXISTS test_alter_multiple_columns; +-- +--CREATE TABLE test_alter_multiple_columns ( +-- k UInt64, +-- val1 UInt64, +-- val2 UInt64, +-- INDEX idx1 val2 * val1 TYPE minmax GRANULARITY 1 +--) ENGINE MergeTree() +--ORDER BY k +--SETTINGS enable_vertical_merge_algorithm=0; +-- +--INSERT INTO test_alter_multiple_columns SELECT number, number + 5, number * 12 from numbers(1000); +-- +--SELECT COUNT() from test_alter_multiple_columns WHERE val2 <= 2400; +-- +--ALTER TABLE test_alter_multiple_columns MODIFY COLUMN val2 UInt16; +-- +--SELECT COUNT() from test_alter_multiple_columns WHERE val2 <= 2400; +-- +--DROP TABLE IF EXISTS test_alter_multiple_columns; From 48dce81e61699f956d1650a99056b53f2534027c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 21:54:30 +0300 Subject: [PATCH 308/357] Minor modifications after #6413 --- dbms/src/Databases/DatabaseOrdinary.cpp | 13 ++++++++----- dbms/src/Interpreters/InterpreterDropQuery.cpp | 10 +++++++--- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 9039cb83cdc..b988329127e 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -136,14 +136,14 @@ void DatabaseOrdinary::loadTables( continue; // There are files that we tried to delete previously - const std::string tmp_drop_ext = ".sql.tmp_drop"; - if (endsWith(dir_it.name(), ".sql.tmp_drop")) + static const char * tmp_drop_ext = ".sql.tmp_drop"; + if (endsWith(dir_it.name(), tmp_drop_ext)) { - const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - tmp_drop_ext.size()); + const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext)); if (Poco::File(data_path + '/' + table_name).exists()) { Poco::File(dir_it->path()).renameTo(table_name + ".sql"); - LOG_WARNING(log, "Table was not dropped previously"); + LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously"); } else { @@ -325,7 +325,10 @@ void DatabaseOrdinary::removeTable( Poco::File(table_metadata_path + ".tmp_drop").remove(); return; } - catch (...) {} + catch (...) + { + LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); + } attachTable(table_name, res); throw; } diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index caabbc20e3b..226a93aff88 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -91,13 +91,17 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId()); - const auto prev_metadata_name = database_and_table.first->getMetadataPath() + escapeForFileName(database_and_table.second->getTableName()) + ".sql"; - const auto drop_metadata_name = database_and_table.first->getMetadataPath() + escapeForFileName(database_and_table.second->getTableName()) + ".sql.tmp_drop"; + const std::string metadata_file_without_extension = + database_and_table.first->getMetadataPath() + + escapeForFileName(database_and_table.second->getTableName()); + + const auto prev_metadata_name = metadata_file_without_extension + ".sql"; + const auto drop_metadata_name = metadata_file_without_extension + ".sql.tmp_drop"; /// Try to rename metadata file and delete the data try { - //There some kind of tables that have no metadata - ignore renaming + /// There some kind of tables that have no metadata - ignore renaming if (Poco::File(prev_metadata_name).exists()) Poco::File(prev_metadata_name).renameTo(drop_metadata_name); /// Delete table data From 39d50b5144b7d3f11eed2fb9b883e0f4c38d8253 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 22:01:52 +0300 Subject: [PATCH 309/357] Remove mimalloc --- .gitmodules | 3 - CMakeLists.txt | 1 - cmake/find_mimalloc.cmake | 17 --- dbms/CMakeLists.txt | 5 - dbms/src/Common/MiAllocator.cpp | 70 ------------ dbms/src/Common/MiAllocator.h | 27 ----- dbms/src/Common/config.h.in | 1 - dbms/src/Common/tests/CMakeLists.txt | 3 - dbms/src/Common/tests/mi_malloc_test.cpp | 118 -------------------- dbms/src/DataStreams/MarkInCompressedFile.h | 9 +- dbms/src/IO/UncompressedCache.h | 9 -- 11 files changed, 1 insertion(+), 262 deletions(-) delete mode 100644 cmake/find_mimalloc.cmake delete mode 100644 dbms/src/Common/MiAllocator.cpp delete mode 100644 dbms/src/Common/MiAllocator.h delete mode 100644 dbms/src/Common/tests/mi_malloc_test.cpp diff --git a/.gitmodules b/.gitmodules index f6990fed41f..e5be5438cc7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -97,9 +97,6 @@ [submodule "contrib/rapidjson"] path = contrib/rapidjson url = https://github.com/Tencent/rapidjson -[submodule "contrib/mimalloc"] - path = contrib/mimalloc - url = https://github.com/ClickHouse-Extras/mimalloc [submodule "contrib/fastops"] path = contrib/fastops url = https://github.com/ClickHouse-Extras/fastops diff --git a/CMakeLists.txt b/CMakeLists.txt index f84a181a39c..e181bdbc2af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -476,7 +476,6 @@ include (cmake/find_consistent-hashing.cmake) include (cmake/find_base64.cmake) include (cmake/find_parquet.cmake) include (cmake/find_hyperscan.cmake) -include (cmake/find_mimalloc.cmake) include (cmake/find_simdjson.cmake) include (cmake/find_rapidjson.cmake) include (cmake/find_fastops.cmake) diff --git a/cmake/find_mimalloc.cmake b/cmake/find_mimalloc.cmake deleted file mode 100644 index 1820421379f..00000000000 --- a/cmake/find_mimalloc.cmake +++ /dev/null @@ -1,17 +0,0 @@ -if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT ARCH_PPC64LE) - option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" FALSE) -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h") - message (WARNING "submodule contrib/mimalloc is missing. to fix try run: \n git submodule update --init --recursive") - return() -endif () - -if (ENABLE_MIMALLOC) - message (FATAL_ERROR "Mimalloc is not production ready. (Disable with cmake -D ENABLE_MIMALLOC=0). If you want to use mimalloc, you must manually remove this message.") - - set (MIMALLOC_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include) - set (USE_MIMALLOC 1) - set (MIMALLOC_LIBRARY mimalloc-static) - message (STATUS "Using mimalloc: ${MIMALLOC_INCLUDE_DIR} : ${MIMALLOC_LIBRARY}") -endif () diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 355c66902a8..af59af7642b 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -266,11 +266,6 @@ if(RE2_INCLUDE_DIR) target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR}) endif() -if (USE_MIMALLOC) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${MIMALLOC_INCLUDE_DIR}) - target_link_libraries (clickhouse_common_io PRIVATE ${MIMALLOC_LIBRARY}) -endif () - if(CPUID_LIBRARY) target_link_libraries(clickhouse_common_io PRIVATE ${CPUID_LIBRARY}) endif() diff --git a/dbms/src/Common/MiAllocator.cpp b/dbms/src/Common/MiAllocator.cpp deleted file mode 100644 index 04e61a5de16..00000000000 --- a/dbms/src/Common/MiAllocator.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "MiAllocator.h" - -#if USE_MIMALLOC -#include - -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int CANNOT_ALLOCATE_MEMORY; -} - -void * MiAllocator::alloc(size_t size, size_t alignment) -{ - void * ptr; - if (alignment == 0) - { - ptr = mi_malloc(size); - if (!ptr) - DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - else - { - ptr = mi_malloc_aligned(size, alignment); - if (!ptr) - DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc (mi_malloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - return ptr; -} - -void MiAllocator::free(void * buf, size_t) -{ - mi_free(buf); -} - -void * MiAllocator::realloc(void * old_ptr, size_t, size_t new_size, size_t alignment) -{ - if (old_ptr == nullptr) - return alloc(new_size, alignment); - - if (new_size == 0) - { - mi_free(old_ptr); - return nullptr; - } - - void * ptr; - - if (alignment == 0) - { - ptr = mi_realloc(old_ptr, alignment); - if (!ptr) - DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - else - { - ptr = mi_realloc_aligned(old_ptr, new_size, alignment); - if (!ptr) - DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc (mi_realloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - return ptr; -} - -} - -#endif diff --git a/dbms/src/Common/MiAllocator.h b/dbms/src/Common/MiAllocator.h deleted file mode 100644 index 127be82434b..00000000000 --- a/dbms/src/Common/MiAllocator.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include - -#if USE_MIMALLOC -#include - -namespace DB -{ - -/* - * This is a different allocator that is based on mimalloc (Microsoft malloc). - * It can be used separately from main allocator to catch heap corruptions and vulnerabilities (for example, for caches). - * We use MI_SECURE mode in mimalloc to achieve such behaviour. - */ -struct MiAllocator -{ - static void * alloc(size_t size, size_t alignment = 0); - - static void free(void * buf, size_t); - - static void * realloc(void * old_ptr, size_t, size_t new_size, size_t alignment = 0); -}; - -} - -#endif diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index b6a5b6de2b8..7804068e5c4 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -8,6 +8,5 @@ #cmakedefine01 USE_CPUID #cmakedefine01 USE_CPUINFO #cmakedefine01 USE_BROTLI -#cmakedefine01 USE_MIMALLOC #cmakedefine01 USE_UNWIND #cmakedefine01 CLICKHOUSE_SPLIT_BINARY diff --git a/dbms/src/Common/tests/CMakeLists.txt b/dbms/src/Common/tests/CMakeLists.txt index 2c99c85baec..67c0e376f74 100644 --- a/dbms/src/Common/tests/CMakeLists.txt +++ b/dbms/src/Common/tests/CMakeLists.txt @@ -76,8 +76,5 @@ target_link_libraries (cow_compositions PRIVATE clickhouse_common_io) add_executable (stopwatch stopwatch.cpp) target_link_libraries (stopwatch PRIVATE clickhouse_common_io) -add_executable (mi_malloc_test mi_malloc_test.cpp) -target_link_libraries (mi_malloc_test PRIVATE clickhouse_common_io) - add_executable (symbol_index symbol_index.cpp) target_link_libraries (symbol_index PRIVATE clickhouse_common_io) diff --git a/dbms/src/Common/tests/mi_malloc_test.cpp b/dbms/src/Common/tests/mi_malloc_test.cpp deleted file mode 100644 index ce1e4a3a770..00000000000 --- a/dbms/src/Common/tests/mi_malloc_test.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/** In addition to ClickHouse (Apache 2) license, this file can be also used under MIT license: - -MIT License - -Copyright (c) 2019 Yandex LLC, Alexey Milovidov - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -*/ - -#include -#include -#include -#include -#include -#include - -#include - -//#undef USE_MIMALLOC -//#define USE_MIMALLOC 0 - -#if USE_MIMALLOC - -#include -#define malloc mi_malloc -#define free mi_free - -#else - -#include - -#endif - - -size_t total_size{0}; - -struct Allocation -{ - void * ptr = nullptr; - size_t size = 0; - - Allocation() {} - - Allocation(size_t size_) - : size(size_) - { - ptr = malloc(size); - if (!ptr) - throw std::runtime_error("Cannot allocate memory"); - total_size += size; - } - - ~Allocation() - { - if (ptr) - { - free(ptr); - total_size -= size; - } - ptr = nullptr; - } - - Allocation(const Allocation &) = delete; - - Allocation(Allocation && rhs) - { - ptr = rhs.ptr; - size = rhs.size; - rhs.ptr = nullptr; - rhs.size = 0; - } -}; - - -int main(int, char **) -{ - std::vector allocations; - - constexpr size_t limit = 100000000; - constexpr size_t min_alloc_size = 65536; - constexpr size_t max_alloc_size = 10000000; - - std::mt19937 rng; - auto distribution = std::uniform_int_distribution(min_alloc_size, max_alloc_size); - - size_t total_allocations = 0; - - while (true) - { - size_t size = distribution(rng); - - while (total_size + size > limit) - allocations.pop_back(); - - allocations.emplace_back(size); - - ++total_allocations; - if (total_allocations % (1ULL << 20) == 0) - std::cerr << "Total allocations: " << total_allocations << "\n"; - } -} diff --git a/dbms/src/DataStreams/MarkInCompressedFile.h b/dbms/src/DataStreams/MarkInCompressedFile.h index a5970a89738..62fe8eedf76 100644 --- a/dbms/src/DataStreams/MarkInCompressedFile.h +++ b/dbms/src/DataStreams/MarkInCompressedFile.h @@ -6,10 +6,6 @@ #include #include -#include -#if USE_MIMALLOC -#include -#endif namespace DB { @@ -43,9 +39,6 @@ struct MarkInCompressedFile } }; -#if USE_MIMALLOC -using MarksInCompressedFile = PODArray; -#else + using MarksInCompressedFile = PODArray; -#endif } diff --git a/dbms/src/IO/UncompressedCache.h b/dbms/src/IO/UncompressedCache.h index 1f17c5e61b6..86f1530e5b3 100644 --- a/dbms/src/IO/UncompressedCache.h +++ b/dbms/src/IO/UncompressedCache.h @@ -6,11 +6,6 @@ #include #include -#include -#if USE_MIMALLOC -#include -#endif - namespace ProfileEvents { @@ -25,11 +20,7 @@ namespace DB struct UncompressedCacheCell { -#if USE_MIMALLOC - Memory data; -#else Memory<> data; -#endif size_t compressed_size; UInt32 additional_bytes; }; From e9875950a488eaae51488f69e8f2274ed4343b61 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Aug 2019 22:36:04 +0300 Subject: [PATCH 310/357] Make test timeout to be more significant --- .../00816_long_concurrent_alter_column.sh | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/dbms/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index 17779b73add..965408065cf 100755 --- a/dbms/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/dbms/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -8,14 +8,58 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo "DROP TABLE IF EXISTS concurrent_alter_column" | ${CLICKHOUSE_CLIENT} echo "CREATE TABLE concurrent_alter_column (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple()" | ${CLICKHOUSE_CLIENT} -for i in {1..500}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n +function thread1() +{ + while true; do + for i in {1..500}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n --query_id=alter1 + done +} -for i in {1..100}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE concurrent_alter_column DROP COLUMN d" | ${CLICKHOUSE_CLIENT} -n; done & -for i in {1..100}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE concurrent_alter_column DROP COLUMN e" | ${CLICKHOUSE_CLIENT} -n; done & -for i in {1..100}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN f DOUBLE" | ${CLICKHOUSE_CLIENT}; sleep `echo 0.0$RANDOM`; echo "ALTER TABLE concurrent_alter_column DROP COLUMN f" | ${CLICKHOUSE_CLIENT} -n; done & +function thread2() +{ + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter2; + sleep `echo 0.0$RANDOM`; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN d" | ${CLICKHOUSE_CLIENT} --query_id=alter2; + done +} + +function thread3() +{ + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter3; + sleep `echo 0.0$RANDOM`; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN e" | ${CLICKHOUSE_CLIENT} --query_id=alter3; + done +} + +function thread4() +{ + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN f DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter4; + sleep `echo 0.0$RANDOM`; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN f" | ${CLICKHOUSE_CLIENT} --query_id=alter4; + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; + +TIMEOUT=30 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & wait echo "DROP TABLE concurrent_alter_column" | ${CLICKHOUSE_CLIENT} +# Check for deadlocks +echo "SELECT * FROM system.processes WHERE query_id LIKE 'alter%'" | ${CLICKHOUSE_CLIENT} + echo 'did not crash' From b4339f266df17c0cd873ef10249647409e66f229 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Wed, 28 Aug 2019 23:49:37 +0300 Subject: [PATCH 311/357] Make a better build scheme (#6500) * Fix shared build * Major default libs refactor * Fix build with gcc_eh * Link all libraries as a big group. * Use global interface library as a group * Build capnproto using our cmake * Use only internal libunwind --- CMakeLists.txt | 258 ++---------------- cmake/default_libs.cmake | 48 ++++ cmake/find_capnp.cmake | 64 ++--- cmake/find_cxx.cmake | 47 +++- cmake/find_unwind.cmake | 64 +---- cmake/test_compiler.cmake | 47 ---- contrib/CMakeLists.txt | 19 -- contrib/arrow-cmake/CMakeLists.txt | 3 +- contrib/capnproto-cmake/CMakeLists.txt | 68 +++++ contrib/jemalloc-cmake/CMakeLists.txt | 3 +- contrib/libcxx-cmake/CMakeLists.txt | 20 +- contrib/libcxxabi-cmake/CMakeLists.txt | 25 +- contrib/librdkafka-cmake/CMakeLists.txt | 2 +- contrib/libunwind-cmake/CMakeLists.txt | 16 +- .../mariadb-connector-c-cmake/CMakeLists.txt | 5 - dbms/CMakeLists.txt | 19 +- dbms/programs/server/Server.cpp | 2 +- dbms/src/Common/Config/CMakeLists.txt | 2 +- dbms/src/Common/QueryProfiler.cpp | 4 +- dbms/src/Common/QueryProfiler.h | 3 +- dbms/src/Common/StackTrace.cpp | 27 +- dbms/src/Common/ZooKeeper/CMakeLists.txt | 2 +- dbms/src/Dictionaries/CMakeLists.txt | 2 +- dbms/src/Interpreters/tests/CMakeLists.txt | 2 +- dbms/tests/tsan_suppressions.txt | 3 + docker/packager/packager | 4 +- libs/CMakeLists.txt | 4 - libs/libcommon/CMakeLists.txt | 2 - .../include/common/config_common.h.in | 1 - libs/libcommon/src/tests/CMakeLists.txt | 2 +- libs/libglibc-compatibility/CMakeLists.txt | 56 ++-- utils/compressor/CMakeLists.txt | 2 +- 32 files changed, 318 insertions(+), 508 deletions(-) create mode 100644 cmake/default_libs.cmake delete mode 100644 cmake/test_compiler.cmake create mode 100644 contrib/capnproto-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f84a181a39c..fb4ca18126d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,14 +1,22 @@ +foreach(policy + CMP0023 + CMP0048 # CMake 3.0 + CMP0074 # CMake 3.12 + CMP0077 + CMP0079 + ) + if(POLICY ${policy}) + cmake_policy(SET ${policy} NEW) + endif() +endforeach() + project(ClickHouse) cmake_minimum_required(VERSION 3.3) -foreach(policy - CMP0023 - CMP0074 # CMake 3.12 - ) - if(POLICY ${policy}) - cmake_policy(SET ${policy} NEW) - endif() -endforeach() +# Ignore export() since we don't use it, +# but it gets broken with a global targets via link_libraries() +macro (export) +endmacro () set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/") set(CMAKE_EXPORT_COMPILE_COMMANDS 1) # Write compile_commands.json @@ -128,12 +136,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64") endif () endif () -if (GLIBC_COMPATIBILITY) - set (USE_INTERNAL_MEMCPY ON) -else () - message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.") -endif () - string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER}) find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld") @@ -172,20 +174,15 @@ if (ARCH_NATIVE) set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native") endif () -# Special options for better optimized code with clang -#if (COMPILER_CLANG) -# set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -Wno-unused-command-line-argument -mllvm -inline-threshold=10000") -#endif () - if (CMAKE_VERSION VERSION_LESS "3.8.0") if (NOT MSVC) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") endif () else () set (CMAKE_CXX_STANDARD 17) set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS set (CMAKE_CXX_STANDARD_REQUIRED ON) - set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++1z") + set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++17") endif () if (COMPILER_GCC OR COMPILER_CLANG) @@ -207,17 +204,13 @@ endif() set (CMAKE_BUILD_COLOR_MAKEFILE ON) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}") -#set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}") -#set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}") -# Uses MAKE_STATIC_LIBRARIES - option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF) if (UNBUNDLED) @@ -225,149 +218,28 @@ if (UNBUNDLED) else () set(NOT_UNBUNDLED 1) endif () + # Using system libs can cause lot of warnings in includes. if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () - -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package (Threads) - -include (cmake/find_cxx.cmake) - -include (cmake/test_compiler.cmake) - -if (OS_LINUX AND COMPILER_CLANG AND USE_STATIC_LIBRARIES) - option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux)" ${HAVE_LIBCXX}) - - if (USE_LIBCXX) - set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. - endif () -endif () - -if (USE_LIBCXX) - set (STATIC_STDLIB_FLAGS "") -else () - set (STATIC_STDLIB_FLAGS "-static-libgcc -static-libstdc++") -endif () - -if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD)) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}") - - # Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained. - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}") -endif () - -if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}") -endif () - # Make this extra-checks for correct library dependencies. if (NOT SANITIZE) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") endif () -include (cmake/find_unwind.cmake) +include(cmake/dbms_glob_sources.cmake) +include(cmake/default_libs.cmake) -if (USE_INTERNAL_UNWIND_LIBRARY) - option (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING "Use internal unwind library for exception handling" ${USE_STATIC_LIBRARIES}) -endif () - - -# Set standard, system and compiler libraries explicitly. -# This is intended for more control of what we are linking. +###################################### +### Add targets below this comment ### +###################################### string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") -set (DEFAULT_LIBS "") -if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING OR USE_LIBCXX)) - # Note: this probably has no effect, but I'm not an expert in CMake. - set (CMAKE_C_IMPLICIT_LINK_LIBRARIES "") - set (CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "") - - # Disable default linked libraries. - set (DEFAULT_LIBS "-nodefaultlibs") - - # We need builtins from Clang's RT even without libcxx - for ubsan+int128. See https://bugs.llvm.org/show_bug.cgi?id=16404 - set (BUILTINS_LIB_PATH "") - if (COMPILER_CLANG) - execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) - else () - set (BUILTINS_LIB_PATH "-lgcc") - endif () - - string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) - set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") - - # Add C++ libraries. - # - # This consist of: - # - C++ standard library (like implementation of std::string); - # - C++ ABI implementation (functions for exceptions like __cxa_throw, RTTI, etc); - # - functions for internal implementation of exception handling (stack unwinding based on DWARF info; TODO replace with bundled libunwind); - # - compiler builtins (example: functions for implementation of __int128 operations); - # - # There are two variants of C++ library: libc++ (from LLVM compiler infrastructure) and libstdc++ (from GCC). - - if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - if (USE_STATIC_LIBRARIES) - set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a") - else () - set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_shared${${CMAKE_POSTFIX_VARIABLE}}.so") - endif () - else () - set (EXCEPTION_HANDLING_LIBRARY "-lgcc_eh") - endif () - - message (STATUS "Using exception handling library: ${EXCEPTION_HANDLING_LIBRARY}") - - if (USE_LIBCXX) - if (USE_INTERNAL_LIBCXX_LIBRARY) - set (LIBCXX_LIBS "${ClickHouse_BINARY_DIR}/contrib/libcxx-cmake/libcxx_static${${CMAKE_POSTFIX_VARIABLE}}.a ${ClickHouse_BINARY_DIR}/contrib/libcxxabi-cmake/libcxxabi_static${${CMAKE_POSTFIX_VARIABLE}}.a") - else () - set (LIBCXX_LIBS "-lc++ -lc++abi -lc++fs") - endif () - - set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic ${LIBCXX_LIBS} ${EXCEPTION_HANDLING_LIBRARY} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") - else () - set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lstdc++ -lstdc++fs ${EXCEPTION_HANDLING_LIBRARY} ${COVERAGE_OPTION} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic") - endif () - - # Linking with GLIBC prevents portability of binaries to older systems. - # We overcome this behaviour by statically linking with our own implementation of all new symbols (that don't exist in older Libc or have infamous "symbol versioning"). - # The order of linking is important: 'glibc-compatibility' must be before libc but after all other libraries. - if (GLIBC_COMPATIBILITY) - message (STATUS "Some symbols from glibc will be replaced for compatibility") - - string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) - set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") - - # FIXME: glibc-compatibility may be non-static in some builds! - set (DEFAULT_LIBS "${DEFAULT_LIBS} ${ClickHouse_BINARY_DIR}/libs/libglibc-compatibility/libglibc-compatibility${${CMAKE_POSTFIX_VARIABLE}}.a") - endif () - - # Add Libc. GLIBC is actually a collection of interdependent libraries. - set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc") - - # Note: we'd rather use Musl libc library, but it's little bit more difficult to use. - - message(STATUS "Default libraries: ${DEFAULT_LIBS}") -endif () - -if (NOT GLIBC_COMPATIBILITY) - set (M_LIBRARY m) -endif () - -if (DEFAULT_LIBS) - # Add default libs to all targets as the last dependency. - set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) - set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) -endif () - if (NOT MAKE_STATIC_LIBRARIES) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif () @@ -420,20 +292,12 @@ if (UNBUNDLED) else () set(NOT_UNBUNDLED 1) endif () + # Using system libs can cause lot of warnings in includes. if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () -if (USE_LIBCXX) - set (HAVE_LIBCXX 1) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") -endif() - -if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${LIBCXX_INCLUDE_DIR} -isystem ${LIBCXXABI_INCLUDE_DIR}") -endif () - message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}") include(GNUInstallDirs) @@ -499,79 +363,11 @@ include (libs/libmysqlxx/cmake/find_mysqlclient.cmake) include (cmake/print_flags.cmake) +install (EXPORT global DESTINATION cmake) + add_subdirectory (contrib EXCLUDE_FROM_ALL) add_subdirectory (libs) add_subdirectory (utils) add_subdirectory (dbms) include (cmake/print_include_directories.cmake) - -if (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - # FIXME: actually glibc-compatibility should always be built first, - # because it's unconditionally linked via $DEFAULT_LIBS, - # and these looks like the first places that get linked. - function (add_default_dependencies target_name) - if (TARGET ${target_name}) - if (GLIBC_COMPATIBILITY) - add_dependencies(${target_name} glibc-compatibility) - endif () - - if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) - add_dependencies(${target_name} cxx_static cxxabi_static) - endif () - - if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - add_dependencies(${target_name} unwind_static) - endif () - endif () - endfunction () - - add_default_dependencies(ltdl) - add_default_dependencies(zlibstatic) - add_default_dependencies(jemalloc) - add_default_dependencies(memcpy) - add_default_dependencies(Foundation) - add_default_dependencies(common) - add_default_dependencies(gtest) - add_default_dependencies(lz4) - add_default_dependencies(zstd) - add_default_dependencies(snappy) - add_default_dependencies(arrow) - add_default_dependencies(protoc) - add_default_dependencies(thrift_static) - add_default_dependencies(cityhash) - add_default_dependencies(farmhash) - add_default_dependencies(murmurhash) - add_default_dependencies(metrohash) - add_default_dependencies(metrohash128) - add_default_dependencies(consistent-hashing) - add_default_dependencies(double-conversion) - add_default_dependencies(cctz) - add_default_dependencies(kj) - add_default_dependencies(simdjson) - add_default_dependencies(apple_rt) - add_default_dependencies(h3) - add_default_dependencies(re2) - add_default_dependencies(re2_st) - add_default_dependencies(hs_compile_shared) - add_default_dependencies(hs_exec_shared) - add_default_dependencies(hs_shared) - add_default_dependencies(widechar_width) - add_default_dependencies(string_utils) - add_default_dependencies(consistent-hashing-sumbur) - add_default_dependencies(boost_program_options_internal) - add_default_dependencies(boost_system_internal) - add_default_dependencies(boost_regex_internal) - add_default_dependencies(roaring) - add_default_dependencies(btrie) - add_default_dependencies(cpuid) - add_default_dependencies(mysqlclient) - add_default_dependencies(zlib) - add_default_dependencies(thrift) - add_default_dependencies(brotli) - add_default_dependencies(libprotobuf) - add_default_dependencies(base64) - add_default_dependencies(readpassphrase) - add_default_dependencies(unwind_static) - add_default_dependencies(fastops) -endif () diff --git a/cmake/default_libs.cmake b/cmake/default_libs.cmake new file mode 100644 index 00000000000..54a01042558 --- /dev/null +++ b/cmake/default_libs.cmake @@ -0,0 +1,48 @@ +# Set standard, system and compiler libraries explicitly. +# This is intended for more control of what we are linking. + +set (DEFAULT_LIBS "-nodefaultlibs") + +if (OS_LINUX) + # We need builtins from Clang's RT even without libcxx - for ubsan+int128. + # See https://bugs.llvm.org/show_bug.cgi?id=16404 + if (COMPILER_CLANG) + execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) + else () + set (BUILTINS_LIBRARY "-lgcc") + endif () + + set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl") + + message(STATUS "Default libraries: ${DEFAULT_LIBS}") +endif () + +set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) +set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) + +# Global libraries + +add_library(global-libs INTERFACE) + +# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'. +# Just make sure we have pthreads at all. +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +add_subdirectory(libs/libglibc-compatibility) +include (cmake/find_unwind.cmake) +include (cmake/find_cxx.cmake) + +add_library(global-group INTERFACE) +target_link_libraries(global-group INTERFACE + -Wl,--start-group + $ + -Wl,--end-group +) + +link_libraries(global-group) + +install( + TARGETS global-group global-libs + EXPORT global +) diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index ec591afdc38..572fc1b3341 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -1,50 +1,20 @@ -option (ENABLE_CAPNP "Enable Cap'n Proto" ON) - -if (ENABLE_CAPNP) - # cmake 3.5.1 bug: - # capnproto uses this cmake feature: - # target_compile_features(kj PUBLIC cxx_constexpr) - # old cmake adds -std=gnu++11 to end of all compile commands (even if -std=gnu++17 already present in compile string) - # cmake 3.9.1 (ubuntu artful) have no this bug (c++17 support added to cmake 3.8.2) - if (CMAKE_VERSION VERSION_LESS "3.8.0") - set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT 0) - set (MISSING_INTERNAL_CAPNP_LIBRARY 1) - else () - set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT ${NOT_UNBUNDLED}) - endif () - - option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${USE_INTERNAL_CAPNP_LIBRARY_DEFAULT}) - - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/CMakeLists.txt") - if (USE_INTERNAL_CAPNP_LIBRARY) - message (WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive") - endif () - set (USE_INTERNAL_CAPNP_LIBRARY 0) - set (MISSING_INTERNAL_CAPNP_LIBRARY 1) - endif () - - if (NOT USE_INTERNAL_CAPNP_LIBRARY) - set (CAPNP_PATHS "/usr/local/opt/capnp/lib") - set (CAPNP_INCLUDE_PATHS "/usr/local/opt/capnp/include") - find_library (CAPNP capnp PATHS ${CAPNP_PATHS}) - find_library (CAPNPC capnpc PATHS ${CAPNP_PATHS}) - find_library (KJ kj PATHS ${CAPNP_PATHS}) - set (CAPNP_LIBRARY ${CAPNPC} ${CAPNP} ${KJ}) - find_path (CAPNP_INCLUDE_DIR NAMES capnp/schema-parser.h PATHS ${CAPNP_INCLUDE_PATHS}) - endif () - - if (CAPNP_INCLUDE_DIR AND CAPNP_LIBRARY) - set(USE_CAPNP 1) - elseif (NOT MISSING_INTERNAL_CAPNP_LIBRARY) - set (USE_INTERNAL_CAPNP_LIBRARY 1) - set (CAPNP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src") - set (CAPNP_LIBRARY capnpc) - set (USE_CAPNP 1) - endif () -endif () +option (USE_CAPNP "Enable Cap'n Proto" ON) if (USE_CAPNP) - message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_INCLUDE_DIR} : ${CAPNP_LIBRARY}") -else () - message (STATUS "Build without capnp (support for Cap'n Proto format will be disabled)") + option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED}) + + # FIXME: refactor to use `add_library(… IMPORTED)` if possible. + if (NOT USE_INTERNAL_CAPNP_LIBRARY) + find_library (KJ kj) + find_library (CAPNP capnp) + find_library (CAPNPC capnpc) + + set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) + else () + add_subdirectory(contrib/capnproto-cmake) + + set (CAPNP_LIBRARIES capnpc) + endif () + + message (STATUS "Using capnp: ${CAPNP_LIBRARIES}") endif () diff --git a/cmake/find_cxx.cmake b/cmake/find_cxx.cmake index 2b2952f6efd..35a0b9d0927 100644 --- a/cmake/find_cxx.cmake +++ b/cmake/find_cxx.cmake @@ -1,26 +1,49 @@ -if (NOT APPLE) +if (OS_LINUX AND COMPILER_CLANG) + option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${HAVE_LIBCXX}) option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED}) +endif() + +if (USE_LIBCXX) + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. endif () +# FIXME: make better check for submodule presence if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/include/vector") message (WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive") set (USE_INTERNAL_LIBCXX_LIBRARY 0) endif () +# FIXME: make better check for submodule presence if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/src") message (WARNING "submodule contrib/libcxxabi is missing. to fix try run: \n git submodule update --init --recursive") - set (USE_INTERNAL_LIBCXXABI_LIBRARY 0) + set (USE_INTERNAL_LIBCXX_LIBRARY 0) endif () -if (NOT USE_INTERNAL_LIBCXX_LIBRARY) - find_library (LIBCXX_LIBRARY c++) - find_library (LIBCXXABI_LIBRARY c++abi) +if (USE_LIBCXX) + if (NOT USE_INTERNAL_LIBCXX_LIBRARY) + find_library (LIBCXX_LIBRARY c++) + find_library (LIBCXXFS_LIBRARY c++fs) + find_library (LIBCXXABI_LIBRARY c++abi) + + target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) + else () + set (LIBCXX_LIBRARY cxx) + set (LIBCXXABI_LIBRARY cxxabi) + add_subdirectory(contrib/libcxxabi-cmake) + add_subdirectory(contrib/libcxx-cmake) + + # Exception handling library is embedded into libcxxabi. + endif () + + target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY}) + + set (HAVE_LIBCXX 1) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + + message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}") + message (STATUS "Using libcxxfs: ${LIBCXXFS_LIBRARY}") + message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}") else () - set (LIBCXX_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx/include) - set (LIBCXXABI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/include) - set (LIBCXX_LIBRARY cxx_static) - set (LIBCXXABI_LIBRARY cxxabi_static) + target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static + target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) endif () - -message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}") -message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}") diff --git a/cmake/find_unwind.cmake b/cmake/find_unwind.cmake index 25e088e8deb..ea6e1d4bacb 100644 --- a/cmake/find_unwind.cmake +++ b/cmake/find_unwind.cmake @@ -1,59 +1,17 @@ -include (CMakePushCheckState) -cmake_push_check_state () +option (USE_UNWIND "Enable libunwind (better stacktraces)" ON) -option (ENABLE_UNWIND "Enable libunwind (better stacktraces)" ON) +if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32) + set (USE_UNWIND OFF) +endif () -if (ENABLE_UNWIND) +if (USE_UNWIND) + add_subdirectory(contrib/libunwind-cmake) + set (UNWIND_LIBRARIES unwind) + set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES}) -if (CMAKE_SYSTEM MATCHES "Linux" AND NOT ARCH_ARM AND NOT ARCH_32) - option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" ${NOT_UNBUNDLED}) + message (STATUS "Using libunwind: ${UNWIND_LIBRARIES}") else () - option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" OFF) + set (EXCEPTION_HANDLING_LIBRARY gcc_eh) endif () -if (NOT USE_INTERNAL_UNWIND_LIBRARY) - find_library (UNWIND_LIBRARY unwind) - find_path (UNWIND_INCLUDE_DIR NAMES unwind.h PATHS ${UNWIND_INCLUDE_PATHS}) - - include (CheckCXXSourceCompiles) - set(CMAKE_REQUIRED_INCLUDES ${UNWIND_INCLUDE_DIR}) - set(CMAKE_REQUIRED_LIBRARIES ${UNWIND_LIBRARY}) - check_cxx_source_compiles(" - #include - #define UNW_LOCAL_ONLY - #include - int main () { - ucontext_t context; - unw_cursor_t cursor; - unw_init_local(&cursor, &context); - return 0; - } - " HAVE_UNW_INIT_LOCAL) - if (NOT HAVE_UNW_INIT_LOCAL) - set(UNWIND_LIBRARY "") - set(UNWIND_INCLUDE_DIR "") - endif () - -endif () - -if (UNWIND_LIBRARY AND UNWIND_INCLUDE_DIR) - set (USE_UNWIND 1) -elseif (CMAKE_SYSTEM MATCHES "Linux" AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT UNBUNDLED) - set (USE_INTERNAL_UNWIND_LIBRARY 1) - - set (PACKAGE_VERSION "9.0.0svn" CACHE STRING "") - - set (UNWIND_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libunwind/include") - - set (LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "") - set (LIBUNWIND_ENABLE_STATIC ON CACHE BOOL "") - set (UNWIND_LIBRARY unwind_static) - - set (USE_UNWIND 1) -endif () - -endif () - -message (STATUS "Using unwind=${USE_UNWIND}: ${UNWIND_INCLUDE_DIR} : ${UNWIND_LIBRARY}") - -cmake_pop_check_state () +message (STATUS "Using exception handler: ${EXCEPTION_HANDLING_LIBRARY}") diff --git a/cmake/test_compiler.cmake b/cmake/test_compiler.cmake deleted file mode 100644 index 570c058b9f7..00000000000 --- a/cmake/test_compiler.cmake +++ /dev/null @@ -1,47 +0,0 @@ -include (CheckCXXSourceCompiles) -include (CMakePushCheckState) - -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads) - -cmake_push_check_state () - -if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") -# clang4 : -no-pie cause error -# clang6 : -no-pie cause warning - - if (MAKE_STATIC_LIBRARIES) - set (TEST_FLAG "-Wl,-Bstatic -stdlib=libc++ -lc++ -lc++abi -Wl,-Bdynamic") - else () - set (TEST_FLAG "-stdlib=libc++ -lc++ -lc++abi") - endif () - - set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") - set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} Threads::Threads) - - check_cxx_source_compiles(" - #include - int main() { - std::cerr << std::endl; - return 0; - } - " HAVE_LIBCXX) - -else () - - set (TEST_FLAG "-no-pie") - set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") - - check_cxx_source_compiles(" - int main() { - return 0; - } - " HAVE_NO_PIE) - - if (HAVE_NO_PIE) - set (FLAG_NO_PIE ${TEST_FLAG}) - endif () - -endif () - -cmake_pop_check_state () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e652c393141..96462de0190 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -23,16 +23,6 @@ if (USE_INTERNAL_ORC_LIBRARY) add_subdirectory(orc) endif() -if (USE_INTERNAL_UNWIND_LIBRARY) - add_subdirectory (libunwind-cmake) -endif () - -if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) - add_subdirectory(libcxx-cmake) - add_subdirectory(libcxxabi-cmake) -endif() - - if (USE_INTERNAL_BOOST_LIBRARY) add_subdirectory (boost-cmake) endif () @@ -172,15 +162,6 @@ if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY) add_library(ODBC::ODBC ALIAS ${ODBC_LIBRARIES}) endif () -if (ENABLE_CAPNP AND USE_INTERNAL_CAPNP_LIBRARY) - set (BUILD_TESTING 0 CACHE INTERNAL "") - set (_save ${CMAKE_CXX_EXTENSIONS}) - set (CMAKE_CXX_EXTENSIONS) - add_subdirectory (capnproto/c++) - set (CMAKE_CXX_EXTENSIONS ${_save}) - target_include_directories(${CAPNP_LIBRARY} PUBLIC $) -endif () - if (USE_INTERNAL_PARQUET_LIBRARY) if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) # We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 843ff9cd8af..ba1ddc2414a 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -44,7 +44,6 @@ set( thriftcpp_threads_SOURCES add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES}) set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641 target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS}) -target_link_libraries(${THRIFT_LIBRARY} PRIVATE Threads::Threads) # === orc @@ -219,7 +218,7 @@ endif() add_library(${ARROW_LIBRARY} ${ARROW_SRCS}) add_dependencies(${ARROW_LIBRARY} protoc) target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS}) -target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY} Threads::Threads) +target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY}) if (ARROW_WITH_LZ4) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY}) endif() diff --git a/contrib/capnproto-cmake/CMakeLists.txt b/contrib/capnproto-cmake/CMakeLists.txt new file mode 100644 index 00000000000..275007c145f --- /dev/null +++ b/contrib/capnproto-cmake/CMakeLists.txt @@ -0,0 +1,68 @@ +set (CAPNPROTO_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src) + +set (KJ_SRCS + ${CAPNPROTO_SOURCE_DIR}/kj/array.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/common.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/debug.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/exception.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/io.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/memory.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/mutex.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/string.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/hash.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/table.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/thread.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/main.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/arena.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/test-helpers.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/units.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/encoding.c++ + + ${CAPNPROTO_SOURCE_DIR}/kj/refcount.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/string-tree.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/time.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/filesystem.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/filesystem-disk-unix.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/filesystem-disk-win32.c++ + ${CAPNPROTO_SOURCE_DIR}/kj/parse/char.c++ +) + +add_library(kj ${KJ_SRCS}) +target_include_directories(kj INTERFACE ${CAPNPROTO_SOURCE_DIR}) + +set (CAPNP_SRCS + ${CAPNPROTO_SOURCE_DIR}/capnp/c++.capnp.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/blob.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/arena.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/layout.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/list.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/any.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/message.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/schema.capnp.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/serialize.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/serialize-packed.c++ + + ${CAPNPROTO_SOURCE_DIR}/capnp/schema.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/schema-loader.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/dynamic.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/stringify.c++ +) + +add_library(capnp ${CAPNP_SRCS}) +target_link_libraries(capnp PUBLIC kj) + +set (CAPNPC_SRCS + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/type-id.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/error-reporter.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/lexer.capnp.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/lexer.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/grammar.capnp.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/parser.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/node-translator.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/compiler.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/schema-parser.c++ + ${CAPNPROTO_SOURCE_DIR}/capnp/serialize-text.c++ +) + +add_library(capnpc ${CAPNPC_SRCS}) +target_link_libraries(capnpc PUBLIC capnp) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 47f057c0559..e44c54d2b37 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -59,7 +59,6 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") if (USE_UNWIND) target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1) - target_include_directories (jemalloc BEFORE PRIVATE ${UNWIND_INCLUDE_DIR}) - target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARY}) + target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARIES}) endif () endif () diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index e9ca5e1e7cd..07fa70b9869 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -1,5 +1,4 @@ set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx) -#set(LIBCXX_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxx) set(SRCS ${LIBCXX_SOURCE_DIR}/src/optional.cpp @@ -16,10 +15,6 @@ ${LIBCXX_SOURCE_DIR}/src/condition_variable.cpp ${LIBCXX_SOURCE_DIR}/src/hash.cpp ${LIBCXX_SOURCE_DIR}/src/string.cpp ${LIBCXX_SOURCE_DIR}/src/debug.cpp -#${LIBCXX_SOURCE_DIR}/src/support/win32/support.cpp -#${LIBCXX_SOURCE_DIR}/src/support/win32/locale_win32.cpp -#${LIBCXX_SOURCE_DIR}/src/support/win32/thread_win32.cpp -#${LIBCXX_SOURCE_DIR}/src/support/solaris/xlocale.cpp ${LIBCXX_SOURCE_DIR}/src/stdexcept.cpp ${LIBCXX_SOURCE_DIR}/src/utility.cpp ${LIBCXX_SOURCE_DIR}/src/any.cpp @@ -43,9 +38,16 @@ ${LIBCXX_SOURCE_DIR}/src/system_error.cpp ${LIBCXX_SOURCE_DIR}/src/random.cpp ) -add_library(cxx_static ${SRCS}) +add_library(cxx ${SRCS}) -target_include_directories(cxx_static PUBLIC ${LIBCXX_SOURCE_DIR}/include) -target_compile_definitions(cxx_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) -target_compile_options(cxx_static PRIVATE -nostdinc++) +target_include_directories(cxx SYSTEM BEFORE PUBLIC $) +target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) +target_compile_options(cxx PRIVATE -nostdinc++) +target_link_libraries(cxx PUBLIC cxxabi) +install( + TARGETS cxx + EXPORT global + ARCHIVE DESTINATION lib + RUNTIME DESTINATION lib +) diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt index 2abece86691..546d39933af 100644 --- a/contrib/libcxxabi-cmake/CMakeLists.txt +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -1,13 +1,10 @@ set(LIBCXXABI_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi) -set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx) -#set(LIBCXXABI_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxxabi) set(SRCS ${LIBCXXABI_SOURCE_DIR}/src/stdlib_stdexcept.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_virtual.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_thread_atexit.cpp ${LIBCXXABI_SOURCE_DIR}/src/fallback_malloc.cpp -#${LIBCXXABI_SOURCE_DIR}/src/cxa_noexception.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_guard.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_default_handlers.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_personality.cpp @@ -25,10 +22,24 @@ ${LIBCXXABI_SOURCE_DIR}/src/cxa_vector.cpp ${LIBCXXABI_SOURCE_DIR}/src/stdlib_new_delete.cpp ) -add_library(cxxabi_static ${SRCS}) +add_library(cxxabi ${SRCS}) -target_include_directories(cxxabi_static PUBLIC ${LIBCXXABI_SOURCE_DIR}/include ${LIBCXX_SOURCE_DIR}/include) -target_compile_definitions(cxxabi_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY) -target_compile_options(cxxabi_static PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. +target_include_directories(cxxabi SYSTEM BEFORE + PUBLIC $ + PRIVATE $ +) +target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY) +target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. +if (USE_UNWIND) + target_link_libraries(cxxabi PRIVATE ${UNWIND_LIBRARIES}) +else () + target_link_libraries(cxxabi PRIVATE gcc_eh) +endif () +install( + TARGETS cxxabi + EXPORT global + ARCHIVE DESTINATION lib + RUNTIME DESTINATION lib +) diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 75cd3968204..64dc83fa8b6 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -65,7 +65,7 @@ add_library(rdkafka ${SRCS}) target_include_directories(rdkafka SYSTEM PUBLIC include) target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common) # Because wrong path to "zstd_errors.h" is used. -target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY} Threads::Threads) +target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY}) if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY) target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) endif() diff --git a/contrib/libunwind-cmake/CMakeLists.txt b/contrib/libunwind-cmake/CMakeLists.txt index 4f24fe249f5..f09d0979692 100644 --- a/contrib/libunwind-cmake/CMakeLists.txt +++ b/contrib/libunwind-cmake/CMakeLists.txt @@ -24,9 +24,15 @@ set(LIBUNWIND_SOURCES ${LIBUNWIND_C_SOURCES} ${LIBUNWIND_ASM_SOURCES}) -add_library(unwind_static ${LIBUNWIND_SOURCES}) +add_library(unwind ${LIBUNWIND_SOURCES}) -target_include_directories(unwind_static SYSTEM BEFORE PUBLIC ${LIBUNWIND_SOURCE_DIR}/include) -target_compile_definitions(unwind_static PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY) -target_compile_options(unwind_static PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti) -target_link_libraries(unwind_static PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) +target_include_directories(unwind SYSTEM BEFORE PUBLIC $) +target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY) +target_compile_options(unwind PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti) + +install( + TARGETS unwind + EXPORT global + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt index 1f453a7f6d1..2e80b0c325f 100644 --- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -62,11 +62,6 @@ endif() add_library(mysqlclient ${SRCS}) -target_link_libraries(mysqlclient PRIVATE ${CMAKE_DL_LIBS} Threads::Threads) -if(M_LIBRARY) - target_link_libraries(mysqlclient PRIVATE ${M_LIBRARY}) -endif() - if(OPENSSL_LIBRARIES) target_link_libraries(mysqlclient PRIVATE ${OPENSSL_LIBRARIES}) target_compile_definitions(mysqlclient PRIVATE -D HAVE_OPENSSL -D HAVE_TLS) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 355c66902a8..98eb23809da 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -97,8 +97,6 @@ add_subdirectory (src) set(dbms_headers) set(dbms_sources) -include(../cmake/dbms_glob_sources.cmake) - add_headers_and_sources(clickhouse_common_io src/Common) add_headers_and_sources(clickhouse_common_io src/Common/HashTable) add_headers_and_sources(clickhouse_common_io src/IO) @@ -163,9 +161,7 @@ if (OS_FREEBSD) endif () if (USE_UNWIND) - if (NOT USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING) - target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARY}) - endif () + target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARIES}) endif () add_subdirectory(src/Common/ZooKeeper) @@ -245,12 +241,6 @@ target_link_libraries(clickhouse_common_io PRIVATE apple_rt PUBLIC - Threads::Threads - PRIVATE - ${CMAKE_DL_LIBS} - PRIVATE - rt - PUBLIC roaring ) @@ -298,7 +288,6 @@ target_link_libraries (dbms ${Boost_FILESYSTEM_LIBRARY} PUBLIC ${Boost_SYSTEM_LIBRARY} - Threads::Threads ) target_include_directories(dbms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include) @@ -365,10 +354,7 @@ if (USE_ICU) endif () if (USE_CAPNP) - target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY}) - if (NOT USE_INTERNAL_CAPNP_LIBRARY) - target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR}) - endif () + target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARIES}) endif () if (USE_PARQUET) @@ -381,7 +367,6 @@ endif () if(OPENSSL_CRYPTO_LIBRARY) target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) endif () -target_link_libraries(dbms PRIVATE Threads::Threads) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index ef61537e38d..5f5e464eb01 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -520,7 +520,7 @@ int Server::main(const std::vector & /*args*/) /// Init trace collector only after trace_log system table was created /// Disable it if we collect test coverage information, because it will work extremely slow. -#if USE_INTERNAL_UNWIND_LIBRARY && !WITH_COVERAGE +#if USE_UNWIND && !WITH_COVERAGE /// QueryProfiler cannot work reliably with any other libunwind or without PHDR cache. if (hasPHDRCache()) global_context->initializeTraceCollector(); diff --git a/dbms/src/Common/Config/CMakeLists.txt b/dbms/src/Common/Config/CMakeLists.txt index e1f78e2c213..c2869763f1b 100644 --- a/dbms/src/Common/Config/CMakeLists.txt +++ b/dbms/src/Common/Config/CMakeLists.txt @@ -4,5 +4,5 @@ add_headers_and_sources(clickhouse_common_config .) add_library(clickhouse_common_config ${clickhouse_common_config_headers} ${clickhouse_common_config_sources}) -target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY} Threads::Threads) +target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY}) target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR}) diff --git a/dbms/src/Common/QueryProfiler.cpp b/dbms/src/Common/QueryProfiler.cpp index b39cdfb4ef5..a0b75c567a9 100644 --- a/dbms/src/Common/QueryProfiler.cpp +++ b/dbms/src/Common/QueryProfiler.cpp @@ -100,7 +100,7 @@ QueryProfilerBase::QueryProfilerBase(const Int32 thread_id, const : log(&Logger::get("QueryProfiler")) , pause_signal(pause_signal_) { -#if USE_INTERNAL_UNWIND_LIBRARY +#if USE_UNWIND /// Sanity check. if (!hasPHDRCache()) throw Exception("QueryProfiler cannot be used without PHDR cache, that is not available for TSan build", ErrorCodes::NOT_IMPLEMENTED); @@ -173,7 +173,7 @@ QueryProfilerBase::~QueryProfilerBase() template void QueryProfilerBase::tryCleanup() { -#if USE_INTERNAL_UNWIND_LIBRARY +#if USE_UNWIND if (timer_id != nullptr && timer_delete(timer_id)) LOG_ERROR(log, "Failed to delete query profiler timer " + errnoToString(ErrorCodes::CANNOT_DELETE_TIMER)); diff --git a/dbms/src/Common/QueryProfiler.h b/dbms/src/Common/QueryProfiler.h index b6420ccc703..abbff6f6c0e 100644 --- a/dbms/src/Common/QueryProfiler.h +++ b/dbms/src/Common/QueryProfiler.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -43,7 +44,7 @@ private: Poco::Logger * log; -#if USE_INTERNAL_UNWIND_LIBRARY +#if USE_UNWIND /// Timer id from timer_create(2) timer_t timer_id = nullptr; #endif diff --git a/dbms/src/Common/StackTrace.cpp b/dbms/src/Common/StackTrace.cpp index a642ec2b73a..9981d0941aa 100644 --- a/dbms/src/Common/StackTrace.cpp +++ b/dbms/src/Common/StackTrace.cpp @@ -1,15 +1,20 @@ -#include -#include -#include #include -#include + #include #include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if USE_UNWIND +# include +#endif std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext_t & context) { @@ -215,12 +220,6 @@ StackTrace::StackTrace(NoCapture) { } - -#if USE_UNWIND -extern "C" int unw_backtrace(void **, int); -#endif - - void StackTrace::tryCapture() { size = 0; diff --git a/dbms/src/Common/ZooKeeper/CMakeLists.txt b/dbms/src/Common/ZooKeeper/CMakeLists.txt index 6a611886e65..aa6efcd3ca1 100644 --- a/dbms/src/Common/ZooKeeper/CMakeLists.txt +++ b/dbms/src/Common/ZooKeeper/CMakeLists.txt @@ -4,7 +4,7 @@ add_headers_and_sources(clickhouse_common_zookeeper .) add_library(clickhouse_common_zookeeper ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources}) -target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY} Threads::Threads) +target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY}) target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR}) if (ENABLE_TESTS) diff --git a/dbms/src/Dictionaries/CMakeLists.txt b/dbms/src/Dictionaries/CMakeLists.txt index af858e6b26d..4d066d1f59b 100644 --- a/dbms/src/Dictionaries/CMakeLists.txt +++ b/dbms/src/Dictionaries/CMakeLists.txt @@ -15,7 +15,7 @@ list(REMOVE_ITEM clickhouse_dictionaries_sources DictionaryFactory.cpp Dictionar list(REMOVE_ITEM clickhouse_dictionaries_headers DictionaryFactory.h DictionarySourceFactory.h DictionaryStructure.h) add_library(clickhouse_dictionaries ${clickhouse_dictionaries_sources}) -target_link_libraries(clickhouse_dictionaries PRIVATE dbms clickhouse_common_io ${BTRIE_LIBRARIES} PUBLIC Threads::Threads) +target_link_libraries(clickhouse_dictionaries PRIVATE dbms clickhouse_common_io ${BTRIE_LIBRARIES}) if(Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY) target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) diff --git a/dbms/src/Interpreters/tests/CMakeLists.txt b/dbms/src/Interpreters/tests/CMakeLists.txt index 3fac5424c00..03c06eb7257 100644 --- a/dbms/src/Interpreters/tests/CMakeLists.txt +++ b/dbms/src/Interpreters/tests/CMakeLists.txt @@ -57,5 +57,5 @@ target_link_libraries (users PRIVATE dbms clickhouse_common_config stdc++fs) if (OS_LINUX) add_executable (internal_iotop internal_iotop.cpp) - target_link_libraries (internal_iotop PRIVATE dbms Threads::Threads) + target_link_libraries (internal_iotop PRIVATE dbms) endif () diff --git a/dbms/tests/tsan_suppressions.txt b/dbms/tests/tsan_suppressions.txt index 476e135de14..3dc306ee133 100644 --- a/dbms/tests/tsan_suppressions.txt +++ b/dbms/tests/tsan_suppressions.txt @@ -1,2 +1,5 @@ # libc++ race:locale + +# Too many mutexes: https://github.com/google/sanitizers/issues/950 +deadlock:DB::MergeTreeReadPool::fillPerPartInfo diff --git a/docker/packager/packager b/docker/packager/packager index 0e8bf6ea98d..c132f514569 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -143,10 +143,10 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") if unbundled: - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0') + cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DUSE_CAPNP=0') if split_binary: - cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 -DGLIBC_COMPATIBILITY=ON') + cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') if with_coverage: cmake_flags.append('-DWITH_COVERAGE=1') diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index d526a662dc0..b5bcbd804be 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -15,10 +15,6 @@ if (USE_INTERNAL_MEMCPY) add_subdirectory (libmemcpy) endif() -if (GLIBC_COMPATIBILITY) - add_subdirectory (libglibc-compatibility) -endif () - if (USE_MYSQL) add_subdirectory (libmysqlxx) endif () diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 2744714a9c4..c78473890dc 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -123,9 +123,7 @@ target_link_libraries (common PUBLIC ${Boost_SYSTEM_LIBRARY} PRIVATE - ${CMAKE_DL_LIBS} ${MALLOC_LIBRARIES} - Threads::Threads ${MEMCPY_LIBRARIES}) if (RT_LIBRARY) diff --git a/libs/libcommon/include/common/config_common.h.in b/libs/libcommon/include/common/config_common.h.in index 1301049b24b..810cf0b87f9 100644 --- a/libs/libcommon/include/common/config_common.h.in +++ b/libs/libcommon/include/common/config_common.h.in @@ -8,5 +8,4 @@ #cmakedefine01 USE_LIBEDIT #cmakedefine01 HAVE_READLINE_HISTORY #cmakedefine01 UNBUNDLED -#cmakedefine01 USE_INTERNAL_UNWIND_LIBRARY #cmakedefine01 WITH_COVERAGE diff --git a/libs/libcommon/src/tests/CMakeLists.txt b/libs/libcommon/src/tests/CMakeLists.txt index 2bb8afe6fa1..15d872ac49d 100644 --- a/libs/libcommon/src/tests/CMakeLists.txt +++ b/libs/libcommon/src/tests/CMakeLists.txt @@ -16,7 +16,7 @@ target_link_libraries (date_lut3 common ${PLATFORM_LIBS}) target_link_libraries (date_lut4 common ${PLATFORM_LIBS}) target_link_libraries (date_lut_default_timezone common ${PLATFORM_LIBS}) target_link_libraries (local_date_time_comparison common) -target_link_libraries (realloc-perf common Threads::Threads) +target_link_libraries (realloc-perf common) add_check(local_date_time_comparison) if(USE_GTEST) diff --git a/libs/libglibc-compatibility/CMakeLists.txt b/libs/libglibc-compatibility/CMakeLists.txt index fe98ae9bf0d..a62f5e75e17 100644 --- a/libs/libglibc-compatibility/CMakeLists.txt +++ b/libs/libglibc-compatibility/CMakeLists.txt @@ -1,25 +1,45 @@ -enable_language(ASM) -include(CheckIncludeFile) +if (GLIBC_COMPATIBILITY) + set (USE_INTERNAL_MEMCPY ON) -check_include_file("sys/random.h" HAVE_SYS_RANDOM_H) + enable_language(ASM) + include(CheckIncludeFile) -if(COMPILER_CLANG) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-builtin-requires-header") -endif() + check_include_file("sys/random.h" HAVE_SYS_RANDOM_H) -add_headers_and_sources(glibc_compatibility .) -add_headers_and_sources(glibc_compatibility musl) -list(APPEND glibc_compatibility_sources musl/syscall.s musl/longjmp.s) + if(COMPILER_CLANG) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-builtin-requires-header") + endif() -list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c) -if(HAVE_SYS_RANDOM_H) - list(APPEND glibc_compatibility_sources musl/getentropy.c) -endif() + add_headers_and_sources(glibc_compatibility .) + add_headers_and_sources(glibc_compatibility musl) + list(APPEND glibc_compatibility_sources musl/syscall.s musl/longjmp.s) -if(MAKE_STATIC_LIBRARIES) - list(APPEND glibc_compatibility_sources libcxxabi/cxa_thread_atexit.cpp) -endif() + list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c) + if(HAVE_SYS_RANDOM_H) + list(APPEND glibc_compatibility_sources musl/getentropy.c) + endif() -add_library(glibc-compatibility STATIC ${glibc_compatibility_sources}) + if(MAKE_STATIC_LIBRARIES) + list(APPEND glibc_compatibility_sources libcxxabi/cxa_thread_atexit.cpp) + endif() -target_include_directories(glibc-compatibility PRIVATE libcxxabi) + add_library(glibc-compatibility STATIC ${glibc_compatibility_sources}) + + target_include_directories(glibc-compatibility PRIVATE libcxxabi) + + if (USE_STATIC_LIBRARIES=0 AND MAKE_STATIC_LIBRARIES=OFF) + target_compile_options(PRIVATE -fPIC) + endif () + + target_link_libraries(global-libs INTERFACE glibc-compatibility) + + install( + TARGETS glibc-compatibility + EXPORT global + ARCHIVE DESTINATION lib + ) + + message (STATUS "Some symbols from glibc will be replaced for compatibility") +elseif (YANDEX_OFFICIAL_BUILD) + message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.") +endif () diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt index 3fdf8aa5eaf..c032054187b 100644 --- a/utils/compressor/CMakeLists.txt +++ b/utils/compressor/CMakeLists.txt @@ -4,7 +4,7 @@ add_executable (zstd_test zstd_test.cpp) if(ZSTD_LIBRARY) target_link_libraries(zstd_test PRIVATE ${ZSTD_LIBRARY}) endif() -target_link_libraries (zstd_test PRIVATE common Threads::Threads) +target_link_libraries (zstd_test PRIVATE common) add_executable (mutator mutator.cpp) target_link_libraries(mutator PRIVATE clickhouse_common_io) From d1a980fcddc9609d35270b6b18b03d7f15bb794f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 29 Aug 2019 01:18:26 +0300 Subject: [PATCH 312/357] Fix for data race in StorageMerge --- dbms/src/Storages/StorageMerge.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 6215550e413..56ab949f30c 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -366,8 +366,8 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr if (storage.get() != this) { - virtual_column->insert(storage->getTableName()); selected_tables.emplace_back(storage, get_lock ? storage->lockStructureForShare(false, query_id) : TableStructureReadLockHolder{}); + virtual_column->insert(storage->getTableName()); } iterator->next(); From 6cbb52128bec45ddcb9a3d47719406689d6c9e7d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 29 Aug 2019 01:51:38 +0300 Subject: [PATCH 313/357] Fix extracting tuple from JSON. --- dbms/src/Functions/FunctionsJSON.h | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dbms/src/Functions/FunctionsJSON.h b/dbms/src/Functions/FunctionsJSON.h index cae88a40b59..b9fddf57d39 100644 --- a/dbms/src/Functions/FunctionsJSON.h +++ b/dbms/src/Functions/FunctionsJSON.h @@ -733,16 +733,15 @@ struct JSONExtractTree if (!JSONParser::firstArrayElement(it2)) return false; - size_t index = 0; - do + for (size_t index = 0; index != nested.size(); ++index) { if (nested[index]->addValueToColumn(tuple.getColumn(index), it2)) were_valid_elements = true; else tuple.getColumn(index).insertDefault(); - ++index; + if (!JSONParser::nextArrayElement(it2)) + break; } - while (JSONParser::nextArrayElement(it2)); set_size(old_size + static_cast(were_valid_elements)); return were_valid_elements; @@ -756,16 +755,15 @@ struct JSONExtractTree if (!JSONParser::firstObjectMember(it2)) return false; - size_t index = 0; - do + for (size_t index = 0; index != nested.size(); ++index) { if (nested[index]->addValueToColumn(tuple.getColumn(index), it2)) were_valid_elements = true; else tuple.getColumn(index).insertDefault(); - ++index; + if (!JSONParser::nextObjectMember(it2)) + break; } - while (JSONParser::nextObjectMember(it2)); } else { From 4ec12c3d24fb7254255392bc84bafd93ccf7076b Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 29 Aug 2019 01:51:02 +0300 Subject: [PATCH 314/357] Add more tests for JSON functions. --- .../0_stateless/00918_json_functions.reference | 14 ++++++++++++++ .../queries/0_stateless/00918_json_functions.sql | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/dbms/tests/queries/0_stateless/00918_json_functions.reference b/dbms/tests/queries/0_stateless/00918_json_functions.reference index 44d4dc1c9bf..a23b177d468 100644 --- a/dbms/tests/queries/0_stateless/00918_json_functions.reference +++ b/dbms/tests/queries/0_stateless/00918_json_functions.reference @@ -46,6 +46,13 @@ hello 1 Thursday Friday +(3,5) +(7,3) +(5,0) +(3,5) +(3,0) +(3,5) +(3,0) --JSONExtractKeysAndValues-- [('a','hello')] [('b',[-100,200,300])] @@ -121,6 +128,13 @@ hello 1 Thursday Friday +(3,5) +(7,3) +(5,0) +(3,5) +(3,0) +(3,5) +(3,0) --JSONExtractKeysAndValues-- [('a','hello')] [('b',[-100,200,300])] diff --git a/dbms/tests/queries/0_stateless/00918_json_functions.sql b/dbms/tests/queries/0_stateless/00918_json_functions.sql index 83f6d1578f9..38bf0a7ffec 100644 --- a/dbms/tests/queries/0_stateless/00918_json_functions.sql +++ b/dbms/tests/queries/0_stateless/00918_json_functions.sql @@ -54,6 +54,13 @@ SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable( SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8'); SELECT JSONExtract('{"day": "Thursday"}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(a Int, b Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(c Int, a Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(b Int, d Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(Int, Int)'); +SELECT JSONExtract('{"a":3}', 'Tuple(Int, Int)'); +SELECT JSONExtract('[3,5,7]', 'Tuple(Int, Int)'); +SELECT JSONExtract('[3]', 'Tuple(Int, Int)'); SELECT '--JSONExtractKeysAndValues--'; SELECT JSONExtractKeysAndValues('{"a": "hello", "b": [-100, 200.0, 300]}', 'String'); @@ -138,6 +145,13 @@ SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable( SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8'); SELECT JSONExtract('{"day": "Thursday"}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); SELECT JSONExtract('{"day": 5}', 'day', 'Enum8(\'Sunday\' = 0, \'Monday\' = 1, \'Tuesday\' = 2, \'Wednesday\' = 3, \'Thursday\' = 4, \'Friday\' = 5, \'Saturday\' = 6)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(a Int, b Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(c Int, a Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(b Int, d Int)'); +SELECT JSONExtract('{"a":3,"b":5,"c":7}', 'Tuple(Int, Int)'); +SELECT JSONExtract('{"a":3}', 'Tuple(Int, Int)'); +SELECT JSONExtract('[3,5,7]', 'Tuple(Int, Int)'); +SELECT JSONExtract('[3]', 'Tuple(Int, Int)'); SELECT '--JSONExtractKeysAndValues--'; SELECT JSONExtractKeysAndValues('{"a": "hello", "b": [-100, 200.0, 300]}', 'String'); From d3b378ea10bb5df25e7f83dce9311c9c4c30e961 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 29 Aug 2019 11:02:50 +0300 Subject: [PATCH 315/357] Fix flappy test (descrease number of iterations) --- dbms/tests/queries/0_stateless/00980_alter_settings_race.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh b/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh index 4a948841ed7..3a9e854210d 100755 --- a/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh +++ b/dbms/tests/queries/0_stateless/00980_alter_settings_race.sh @@ -8,7 +8,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_for_concurrent_alter" $CLICKHOUSE_CLIENT --query="CREATE TABLE table_for_concurrent_alter (id UInt64, Data String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096;"; n=0 -while [ "$n" -lt 100 ]; +while [ "$n" -lt 50 ]; do n=$(( n + 1 )) $CLICKHOUSE_CLIENT --query="INSERT INTO table_for_concurrent_alter VALUES(1, 'Hello')" > /dev/null 2> /dev/null & @@ -17,7 +17,7 @@ done & q=0 -while [ "$q" -lt 100 ]; +while [ "$q" -lt 50 ]; do q=$(( q + 1 )) counter=$(( 100 + q )) From 85afd61c019e7e98fecc31c90cff67ca6b549f60 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Thu, 29 Aug 2019 13:44:22 +0300 Subject: [PATCH 316/357] Fix build with internal libcxx --- cmake/find_cxx.cmake | 2 +- contrib/capnproto-cmake/CMakeLists.txt | 3 ++- contrib/libcxx-cmake/CMakeLists.txt | 4 ++-- contrib/libcxxabi-cmake/CMakeLists.txt | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmake/find_cxx.cmake b/cmake/find_cxx.cmake index 35a0b9d0927..da6e3d48cab 100644 --- a/cmake/find_cxx.cmake +++ b/cmake/find_cxx.cmake @@ -1,5 +1,5 @@ if (OS_LINUX AND COMPILER_CLANG) - option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${HAVE_LIBCXX}) + option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ON) option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED}) endif() diff --git a/contrib/capnproto-cmake/CMakeLists.txt b/contrib/capnproto-cmake/CMakeLists.txt index 275007c145f..d92a5a282ce 100644 --- a/contrib/capnproto-cmake/CMakeLists.txt +++ b/contrib/capnproto-cmake/CMakeLists.txt @@ -28,7 +28,8 @@ set (KJ_SRCS ) add_library(kj ${KJ_SRCS}) -target_include_directories(kj INTERFACE ${CAPNPROTO_SOURCE_DIR}) +target_include_directories(kj PUBLIC ${CAPNPROTO_SOURCE_DIR}) +target_compile_options(kj PUBLIC -Wno-non-virtual-dtor) set (CAPNP_SRCS ${CAPNPROTO_SOURCE_DIR}/capnp/c++.capnp.c++ diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index 07fa70b9869..e74b4a8364f 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -40,9 +40,9 @@ ${LIBCXX_SOURCE_DIR}/src/random.cpp add_library(cxx ${SRCS}) -target_include_directories(cxx SYSTEM BEFORE PUBLIC $) +target_include_directories(cxx BEFORE PUBLIC $) target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) -target_compile_options(cxx PRIVATE -nostdinc++) +target_compile_options(cxx PUBLIC -nostdinc++) target_link_libraries(cxx PUBLIC cxxabi) install( diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt index 546d39933af..4996c4f2d7b 100644 --- a/contrib/libcxxabi-cmake/CMakeLists.txt +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -29,7 +29,7 @@ target_include_directories(cxxabi SYSTEM BEFORE PRIVATE $ ) target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY) -target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. +target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined -Wno-macro-redefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. if (USE_UNWIND) target_link_libraries(cxxabi PRIVATE ${UNWIND_LIBRARIES}) From 0899b3818ddf5310e63aa303f3b7913c9e890a42 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 29 Aug 2019 13:49:26 +0300 Subject: [PATCH 317/357] Remove redundant changes --- dbms/src/Common/ErrorCodes.cpp | 1 - dbms/src/Parsers/ASTIndexDeclaration.h | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 12 ++-- .../MergeTree/MergeTreeDataMergerMutator.cpp | 23 +++---- ...test_skip_indices_with_alter_and_merge.sql | 61 +------------------ 5 files changed, 15 insertions(+), 84 deletions(-) diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 977e1f4425d..87ab252c583 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -447,7 +447,6 @@ namespace ErrorCodes extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW = 470; extern const int SETTINGS_ARE_NOT_SUPPORTED = 471; extern const int IMMUTABLE_SETTING = 472; - extern const int UNSUPPORTED_SKIP_INDEX_EXPRESSION = 473; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Parsers/ASTIndexDeclaration.h b/dbms/src/Parsers/ASTIndexDeclaration.h index feb3f59b9c6..48deffc2c3a 100644 --- a/dbms/src/Parsers/ASTIndexDeclaration.h +++ b/dbms/src/Parsers/ASTIndexDeclaration.h @@ -41,7 +41,7 @@ public: return res; } - void formatImpl(const FormatSettings & s, FormatState &state, FormatStateStacked frame) const override + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override { frame.need_parens = false; std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index af1b8dee450..1ad84274c29 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -364,12 +364,6 @@ void MergeTreeData::setProperties( std::dynamic_pointer_cast(index_decl->clone()), global_context); - if (index_ptr->getColumnsRequiredForIndexCalc().size() > 1 && settings_ptr->enable_vertical_merge_algorithm) - throw Exception("Index '" + index_ptr->name + "' contains expression with multiple columns and " - + "'enable_vertical_merge_algorithm' is set to true in storage settings. " - + "Disable vertical merge or use only one column in index expression.", - ErrorCodes::UNSUPPORTED_SKIP_INDEX_EXPRESSION); - new_indices.push_back(std::move(index_ptr)); if (indices_names.find(new_indices.back()->name) != indices_names.end()) @@ -1587,7 +1581,8 @@ void MergeTreeData::alterDataPart( if (expression) { BlockInputStreamPtr part_in = std::make_shared( - *this, part, expression->getRequiredColumns(), false, /* take_column_types_from_storage = */ false); + *this, part, expression->getRequiredColumns(), false, /* take_column_types_from_storage = */ false); + auto compression_codec = global_context.chooseCompressionCodec( part->bytes_on_disk, @@ -1609,7 +1604,8 @@ void MergeTreeData::alterDataPart( true /* sync */, compression_codec, true /* skip_offsets */, - {}, /// currently restricted + /// Don't recalc indices because indices alter is restricted + std::vector{}, unused_written_offsets, part->index_granularity, &part->index_granularity_info); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 940bd04377c..b6ecd39e713 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -379,7 +379,7 @@ static void extractMergingAndGatheringColumns( std::set key_columns(sort_key_columns_vec.cbegin(), sort_key_columns_vec.cend()); for (const auto & index : indexes) { - Names index_columns_vec = index->expr->getRequiredColumns(); + Names index_columns_vec = index->getColumnsRequiredForIndexCalc(); std::copy(index_columns_vec.cbegin(), index_columns_vec.cend(), std::inserter(key_columns, key_columns.end())); } @@ -829,16 +829,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor rows_sources_read_buf.seek(0, 0); ColumnGathererStream column_gathered_stream(column_name, column_part_streams, rows_sources_read_buf); - std::vector skip_idx_to_recalc = data.getIndicesForColumn(column_name); - for (const auto & idx : skip_idx_to_recalc) - if (idx->getColumnsRequiredForIndexCalc().size() > 1) - throw Exception("Skip index '" + idx->name + "' has expression on multiple columns. " - + "Vertical merge is not supported for tables with skip indices with expressions on multiple columns. " - + "It's better to avoid indices with multiple columns in expression. " - + "Also you can disable vertical merges with setting enable_vertical_merge_algorithm=0, " - + "but it will lead to additional memory consuption for big merges.", - ErrorCodes::LOGICAL_ERROR); - MergedColumnOnlyOutputStream column_to( data, column_gathered_stream.getHeader(), @@ -846,10 +836,13 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor false, compression_codec, false, - skip_idx_to_recalc, + /// we don't need to recalc indices here + /// because all of them were already recalculated and written + /// as key part of vertical merge + std::vector{}, written_offset_columns, - to.getIndexGranularity() - ); + to.getIndexGranularity()); + size_t column_elems_written = 0; column_to.writePrefix(); @@ -1030,7 +1023,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor for (size_t i = 0; i < data.skip_indices.size(); ++i) { const auto & index = data.skip_indices[i]; - const auto & index_cols = index->expr->getRequiredColumns(); + const auto & index_cols = index->getColumnsRequiredForIndexCalc(); auto it = std::find(std::cbegin(index_cols), std::cend(index_cols), col); if (it != std::cend(index_cols) && indices_to_recalc.insert(index).second) { diff --git a/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql index c67f6c594ff..55b2f21dc32 100644 --- a/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql +++ b/dbms/tests/queries/0_stateless/00999_test_skip_indices_with_alter_and_merge.sql @@ -1,48 +1,12 @@ SET allow_experimental_data_skipping_indices=1; -DROP TABLE IF EXISTS table_test_creation; - -CREATE TABLE table_test_creation ( - k UInt64, - val1 UInt64, - val2 UInt64, - INDEX idx1 val1 * val2 TYPE minmax GRANULARITY 1 -) ENGINE MergeTree() -ORDER BY k; -- { serverError 473 } - -CREATE TABLE table_test_creation ( - k UInt64, - val1 UInt64, - val2 UInt64 -) ENGINE MergeTree() -ORDER BY k; - -ALTER TABLE table_test_creation ADD INDEX idx1 val1 * val2 TYPE minmax GRANULARITY 1; -- { serverError 473 } - -ALTER TABLE table_test_creation ADD INDEX idx1 val1 TYPE minmax GRANULARITY 1; - -ALTER TABLE table_test_creation MODIFY SETTING enable_vertical_merge_algorithm=0; - -ALTER TABLE table_test_creation ADD INDEX idx2 val1 * val2 TYPE minmax GRANULARITY 1; - -DROP TABLE IF EXISTS table_test_creation; - -CREATE TABLE table_test_creation ( - k UInt64, - val1 UInt64, - val2 UInt64, - INDEX idx1 val1 * val2 TYPE minmax GRANULARITY 1 -) ENGINE MergeTree() -ORDER BY k SETTINGS enable_vertical_merge_algorithm=0; - -DROP TABLE IF EXISTS table_test_creation; - DROP TABLE IF EXISTS test_vertical_merge; CREATE TABLE test_vertical_merge ( k UInt64, val1 UInt64, val2 UInt64, - INDEX idx1 val2 TYPE minmax GRANULARITY 1 + INDEX idx1 val1 * val2 TYPE minmax GRANULARITY 1, + INDEX idx2 val1 * k TYPE minmax GRANULARITY 1 ) ENGINE MergeTree() ORDER BY k SETTINGS vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1; @@ -56,24 +20,3 @@ OPTIMIZE TABLE test_vertical_merge FINAL; SELECT COUNT() from test_vertical_merge WHERE val2 <= 2400; DROP TABLE IF EXISTS test_vertical_merge; - ---DROP TABLE IF EXISTS test_alter_multiple_columns; --- ---CREATE TABLE test_alter_multiple_columns ( --- k UInt64, --- val1 UInt64, --- val2 UInt64, --- INDEX idx1 val2 * val1 TYPE minmax GRANULARITY 1 ---) ENGINE MergeTree() ---ORDER BY k ---SETTINGS enable_vertical_merge_algorithm=0; --- ---INSERT INTO test_alter_multiple_columns SELECT number, number + 5, number * 12 from numbers(1000); --- ---SELECT COUNT() from test_alter_multiple_columns WHERE val2 <= 2400; --- ---ALTER TABLE test_alter_multiple_columns MODIFY COLUMN val2 UInt16; --- ---SELECT COUNT() from test_alter_multiple_columns WHERE val2 <= 2400; --- ---DROP TABLE IF EXISTS test_alter_multiple_columns; From 702b47ae5b801c03e43a123162e53e436ed759bc Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 29 Aug 2019 13:56:43 +0300 Subject: [PATCH 318/357] Remove redundant changes --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 28 ++++--------------- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 -- .../src/Storages/MergeTree/MergeTreeIndices.h | 6 ++-- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 1ad84274c29..10c9465f9ae 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -91,7 +91,6 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int BAD_DATA_PART_NAME; extern const int UNKNOWN_SETTING; - extern const int UNSUPPORTED_SKIP_INDEX_EXPRESSION; } @@ -350,7 +349,6 @@ void MergeTreeData::setProperties( MergeTreeIndices new_indices; - auto settings_ptr = getSettings(); if (!indices_description.indices.empty()) { std::set indices_names; @@ -359,12 +357,11 @@ void MergeTreeData::setProperties( { const auto & index_decl = std::dynamic_pointer_cast(index_ast); - auto index_ptr = MergeTreeIndexFactory::instance().get( - all_columns, - std::dynamic_pointer_cast(index_decl->clone()), - global_context); - - new_indices.push_back(std::move(index_ptr)); + new_indices.push_back( + MergeTreeIndexFactory::instance().get( + all_columns, + std::dynamic_pointer_cast(index_decl->clone()), + global_context)); if (indices_names.find(new_indices.back()->name) != indices_names.end()) throw Exception( @@ -3090,19 +3087,4 @@ bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const return true; } - -std::vector MergeTreeData::getIndicesForColumn(const String & column_name) const -{ - std::vector result; - - for (size_t i = 0; i < skip_indices.size(); ++i) - { - const auto & index_columns = skip_indices[i]->getColumnsRequiredForIndexCalc(); - if (std::find(index_columns.begin(), index_columns.end(), column_name) != index_columns.end()) - result.emplace_back(skip_indices[i]); - } - - return result; -} - } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index d4c6cbf7c0a..c135c91ddf7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -629,8 +629,6 @@ public: return storage_settings.get(); } - std::vector getIndicesForColumn(const String & column_name) const; - MergeTreeDataFormatVersion format_version; Context global_context; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndices.h b/dbms/src/Storages/MergeTree/MergeTreeIndices.h index 0b626d5e10a..22a392ca9c5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndices.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndices.h @@ -109,11 +109,11 @@ public: /// Index name String name; - /// Index expression with columns arguments + /// Index expression (x * y) + /// with columns arguments ExpressionActionsPtr expr; - /// Names of columns which are used - /// to calculate expression for index + /// Names of columns for index Names columns; /// Data types of columns From 36abd1e6fca9a6f128bb77a3160ea7d64ba8c3bb Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Thu, 29 Aug 2019 14:09:01 +0300 Subject: [PATCH 319/357] Fix build --- contrib/libcxx-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index e74b4a8364f..16aafa7e862 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -42,7 +42,7 @@ add_library(cxx ${SRCS}) target_include_directories(cxx BEFORE PUBLIC $) target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) -target_compile_options(cxx PUBLIC -nostdinc++) +target_compile_options(cxx PUBLIC -nostdinc++ -Wno-reserved-id-macro) target_link_libraries(cxx PUBLIC cxxabi) install( From dac55278adcab1c7509624616c3374ef6f1dc300 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 29 Aug 2019 14:15:56 +0300 Subject: [PATCH 320/357] Better comments --- dbms/src/Interpreters/MutationsInterpreter.cpp | 4 +++- dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 27fb48c7b5b..2069d2372e2 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -289,13 +289,15 @@ void MutationsInterpreter::prepare(bool dry_run) throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); } + /// We cares about affected indices because we also need to rewrite them + /// when one of index columns updated or filtered with delete if (!affected_indices_columns.empty()) { if (!stages.empty()) { std::vector stages_copy; /// Copy all filled stages except index calculation stage. - for (const auto &stage : stages) + for (const auto & stage : stages) { stages_copy.emplace_back(context); stages_copy.back().column_to_updated = stage.column_to_updated; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index b6ecd39e713..7079996af80 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1042,6 +1042,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor auto indices_recalc_expr = ExpressionAnalyzer( indices_recalc_expr_list, indices_recalc_syntax, context).getActions(false); + + /// We can update only one column, but some skip idx expression may depend on several + /// columns (c1 + c2 * c3). It works because in stream was created with help of + /// MutationsInterpreter which knows about skip indices and stream 'in' already has + /// all required columns. + /// TODO move this logic to single place. in = std::make_shared( std::make_shared(in, indices_recalc_expr)); } From 6d6851e747058c9fb2f0180900d6d0b735e2c1c3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 29 Aug 2019 15:36:41 +0300 Subject: [PATCH 321/357] Add capnp-library do build image --- docker/packager/deb/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index f08c2dc3eab..7410920a462 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -71,7 +71,9 @@ RUN apt-get --allow-unauthenticated update -y \ odbcinst \ tzdata \ gperf \ - alien + alien \ + libcapnp-dev + COPY build.sh / CMD ["/bin/bash", "/build.sh"] From 549c61cad9c9df3566ce70d5a02b209c6af51856 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 29 Aug 2019 15:38:20 +0300 Subject: [PATCH 322/357] Update MarkInCompressedFile.h --- dbms/src/DataStreams/MarkInCompressedFile.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/DataStreams/MarkInCompressedFile.h b/dbms/src/DataStreams/MarkInCompressedFile.h index 62fe8eedf76..46d078f2b76 100644 --- a/dbms/src/DataStreams/MarkInCompressedFile.h +++ b/dbms/src/DataStreams/MarkInCompressedFile.h @@ -41,4 +41,5 @@ struct MarkInCompressedFile }; using MarksInCompressedFile = PODArray; + } From 0e606c790ed2af32b12bc3379f0bd9a31f17a89f Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Thu, 29 Aug 2019 16:38:18 +0300 Subject: [PATCH 323/357] Fix install of libcxx and libcxxabi --- contrib/libcxx-cmake/CMakeLists.txt | 1 + contrib/libcxxabi-cmake/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index 16aafa7e862..8759786c3e8 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -50,4 +50,5 @@ install( EXPORT global ARCHIVE DESTINATION lib RUNTIME DESTINATION lib + LIBRARY DESTINATION lib ) diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt index 4996c4f2d7b..2abe5702132 100644 --- a/contrib/libcxxabi-cmake/CMakeLists.txt +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -42,4 +42,5 @@ install( EXPORT global ARCHIVE DESTINATION lib RUNTIME DESTINATION lib + LIBRARY DESTINATION lib ) From 260b8c7fa7baa21f4283206cf59dd5670b233759 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 29 Aug 2019 18:32:25 +0300 Subject: [PATCH 324/357] Removed obsolete directory --- contrib/mimalloc | 1 - 1 file changed, 1 deletion(-) delete mode 160000 contrib/mimalloc diff --git a/contrib/mimalloc b/contrib/mimalloc deleted file mode 160000 index a787bdebce9..00000000000 --- a/contrib/mimalloc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a787bdebce94bf3776dc0d1ad597917f479ab8d5 From 5851316742794665b6c559d0f886cbe302782eb5 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Thu, 29 Aug 2019 18:36:07 +0300 Subject: [PATCH 325/357] Don't interrupt copyData() used inside Kafka materialized views (#6597) * Print better offsets on commit * Stop consumption on the read buffer level * Don't use cancellation in the middle of copyData() * Add test * Disable squashing stream for Kafka materialized views * Commit last read message, since rdkafka auto-modifies offset --- .../Interpreters/InterpreterInsertQuery.cpp | 6 +- .../src/Interpreters/InterpreterInsertQuery.h | 5 +- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 57 +++++++++++--- .../Kafka/ReadBufferFromKafkaConsumer.h | 9 ++- dbms/src/Storages/Kafka/StorageKafka.cpp | 9 ++- .../integration/test_storage_kafka/test.py | 74 +++++++++++++++++++ 6 files changed, 140 insertions(+), 20 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 45d528b0e74..f7edca14089 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -38,8 +38,8 @@ namespace ErrorCodes InterpreterInsertQuery::InterpreterInsertQuery( - const ASTPtr & query_ptr_, const Context & context_, bool allow_materialized_) - : query_ptr(query_ptr_), context(context_), allow_materialized(allow_materialized_) + const ASTPtr & query_ptr_, const Context & context_, bool allow_materialized_, bool no_squash_) + : query_ptr(query_ptr_), context(context_), allow_materialized(allow_materialized_), no_squash(no_squash_) { checkStackSize(); } @@ -109,7 +109,7 @@ BlockIO InterpreterInsertQuery::execute() /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side. /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks). - if (!(context.getSettingsRef().insert_distributed_sync && table->isRemote())) + if (!(context.getSettingsRef().insert_distributed_sync && table->isRemote()) && !no_squash) { out = std::make_shared( out, out->getHeader(), context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.h b/dbms/src/Interpreters/InterpreterInsertQuery.h index 9cde2c274fe..37cb3110984 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.h +++ b/dbms/src/Interpreters/InterpreterInsertQuery.h @@ -15,7 +15,7 @@ namespace DB class InterpreterInsertQuery : public IInterpreter { public: - InterpreterInsertQuery(const ASTPtr & query_ptr_, const Context & context_, bool allow_materialized_ = false); + InterpreterInsertQuery(const ASTPtr & query_ptr_, const Context & context_, bool allow_materialized_ = false, bool no_squash_ = false); /** Prepare a request for execution. Return block streams * - the stream into which you can write data to execute the query, if INSERT; @@ -33,7 +33,8 @@ private: ASTPtr query_ptr; const Context & context; - bool allow_materialized; + const bool allow_materialized; + const bool no_squash; }; diff --git a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 01fd09db7e3..960a998932d 100644 --- a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -4,14 +4,21 @@ namespace DB { using namespace std::chrono_literals; + ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( - ConsumerPtr consumer_, Poco::Logger * log_, size_t max_batch_size, size_t poll_timeout_, bool intermediate_commit_) + ConsumerPtr consumer_, + Poco::Logger * log_, + size_t max_batch_size, + size_t poll_timeout_, + bool intermediate_commit_, + const std::atomic & stopped_) : ReadBuffer(nullptr, 0) , consumer(consumer_) , log(log_) , batch_size(max_batch_size) , poll_timeout(poll_timeout_) , intermediate_commit(intermediate_commit_) + , stopped(stopped_) , current(messages.begin()) { } @@ -26,11 +33,46 @@ ReadBufferFromKafkaConsumer::~ReadBufferFromKafkaConsumer() void ReadBufferFromKafkaConsumer::commit() { + auto PrintOffsets = [this] (const char * prefix, const cppkafka::TopicPartitionList & offsets) + { + for (const auto & topic_part : offsets) + { + auto print_special_offset = [&topic_part] + { + switch (topic_part.get_offset()) + { + case cppkafka::TopicPartition::OFFSET_BEGINNING: return "BEGINNING"; + case cppkafka::TopicPartition::OFFSET_END: return "END"; + case cppkafka::TopicPartition::OFFSET_STORED: return "STORED"; + case cppkafka::TopicPartition::OFFSET_INVALID: return "INVALID"; + default: return ""; + } + }; + + if (topic_part.get_offset() < 0) + { + LOG_TRACE( + log, + prefix << " " << print_special_offset() << " (topic: " << topic_part.get_topic() + << ", partition: " << topic_part.get_partition() << ")"); + } + else + { + LOG_TRACE( + log, + prefix << " " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() + << ", partition: " << topic_part.get_partition() << ")"); + } + } + }; + + PrintOffsets("Polled offset", consumer->get_offsets_position(consumer->get_assignment())); + if (current != messages.end()) { /// Since we can poll more messages than we already processed, /// commit only processed messages. - consumer->async_commit(*current); + consumer->async_commit(*std::prev(current)); } else { @@ -41,14 +83,7 @@ void ReadBufferFromKafkaConsumer::commit() consumer->async_commit(); } - const auto & offsets = consumer->get_offsets_committed(consumer->get_assignment()); - for (const auto & topic_part : offsets) - { - LOG_TRACE( - log, - "Committed offset " << topic_part.get_offset() << " (topic: " << topic_part.get_topic() - << ", partition: " << topic_part.get_partition() << ")"); - } + PrintOffsets("Committed offset", consumer->get_offsets_committed(consumer->get_assignment())); stalled = false; } @@ -114,7 +149,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() /// NOTE: ReadBuffer was implemented with an immutable underlying contents in mind. /// If we failed to poll any message once - don't try again. /// Otherwise, the |poll_timeout| expectations get flawn. - if (stalled) + if (stalled || stopped) return false; if (current == messages.end()) diff --git a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index 8f78eae23e1..c489ce6b5f0 100644 --- a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -17,7 +17,12 @@ class ReadBufferFromKafkaConsumer : public ReadBuffer { public: ReadBufferFromKafkaConsumer( - ConsumerPtr consumer_, Poco::Logger * log_, size_t max_batch_size, size_t poll_timeout_, bool intermediate_commit_); + ConsumerPtr consumer_, + Poco::Logger * log_, + size_t max_batch_size, + size_t poll_timeout_, + bool intermediate_commit_, + const std::atomic & stopped_); ~ReadBufferFromKafkaConsumer() override; void commit(); // Commit all processed messages. @@ -43,6 +48,8 @@ private: bool stalled = false; bool intermediate_commit = true; + const std::atomic & stopped; + Messages messages; Messages::const_iterator current; diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 2d55eb42f1e..b71ee5c8a18 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -274,8 +274,10 @@ ConsumerBufferPtr StorageKafka::createReadBuffer() batch_size = settings.max_block_size.value; size_t poll_timeout = settings.stream_poll_timeout_ms.totalMilliseconds(); + /// NOTE: we pass |stream_cancelled| by reference here, so the buffers should not outlive the storage. return std::make_shared( - std::make_unique(consumer, log, batch_size, poll_timeout, intermediate_commit), row_delimiter); + std::make_unique(consumer, log, batch_size, poll_timeout, intermediate_commit, stream_cancelled), + row_delimiter); } @@ -371,7 +373,7 @@ bool StorageKafka::streamToViews() block_size = settings.max_block_size; // Create a stream for each consumer and join them in a union stream - InterpreterInsertQuery interpreter{insert, global_context}; + InterpreterInsertQuery interpreter(insert, global_context, false, true); auto block_io = interpreter.execute(); // Create a stream for each consumer and join them in a union stream @@ -396,7 +398,8 @@ bool StorageKafka::streamToViews() else in = streams[0]; - copyData(*in, *block_io.out, &stream_cancelled); + std::atomic stub; + copyData(*in, *block_io.out, &stub); // Check whether the limits were applied during query execution bool limits_applied = false; diff --git a/dbms/tests/integration/test_storage_kafka/test.py b/dbms/tests/integration/test_storage_kafka/test.py index 0f0cf6996ef..47ba957cdef 100644 --- a/dbms/tests/integration/test_storage_kafka/test.py +++ b/dbms/tests/integration/test_storage_kafka/test.py @@ -504,6 +504,7 @@ def test_kafka_virtual_columns_with_materialized_view(kafka_cluster): kafka_check_result(result, True, 'test_kafka_virtual2.reference') +@pytest.mark.timeout(60) def test_kafka_insert(kafka_cluster): instance.query(''' CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -540,6 +541,7 @@ def test_kafka_insert(kafka_cluster): kafka_check_result(result, True) +@pytest.mark.timeout(60) def test_kafka_produce_consume(kafka_cluster): instance.query(''' CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -603,6 +605,78 @@ def test_kafka_produce_consume(kafka_cluster): assert int(result) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) +@pytest.mark.timeout(120) +def test_kafka_commit_on_block_write(kafka_cluster): + instance.query(''' + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'block', + kafka_group_name = 'block', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 100, + kafka_row_delimiter = '\\n'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.kafka; + ''') + + cancel = threading.Event() + + i = [0] + def produce(): + while not cancel.is_set(): + messages = [] + for _ in range(101): + messages.append(json.dumps({'key': i[0], 'value': i[0]})) + i[0] += 1 + kafka_produce('block', messages) + + kafka_thread = threading.Thread(target=produce) + kafka_thread.start() + + while int(instance.query('SELECT count() FROM test.view')) == 0: + time.sleep(1) + + cancel.set() + + instance.query(''' + DROP TABLE test.kafka; + ''') + + while int(instance.query("SELECT count() FROM system.tables WHERE database='test' AND name='kafka'")) == 1: + time.sleep(1) + + instance.query(''' + CREATE TABLE test.kafka (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'block', + kafka_group_name = 'block', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 100, + kafka_row_delimiter = '\\n'; + ''') + + while int(instance.query('SELECT uniqExact(key) FROM test.view')) < i[0]: + time.sleep(1) + + result = int(instance.query('SELECT count() == uniqExact(key) FROM test.view')) + + instance.query(''' + DROP TABLE test.consumer; + DROP TABLE test.view; + ''') + + kafka_thread.join() + + assert result == 1, 'Messages from kafka get duplicated!' + + if __name__ == '__main__': cluster.start() raw_input("Cluster created, press any key to destroy...") From a894288fa059ad06c325780aa780c49ae57b49da Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 29 Aug 2019 18:48:00 +0300 Subject: [PATCH 326/357] Fallback from O_DIRECT. --- dbms/src/Common/ProfileEvents.cpp | 2 + dbms/src/IO/createReadBufferFromFileBase.cpp | 40 ++++++++++-------- dbms/src/IO/createWriteBufferFromFileBase.cpp | 41 ++++++++++--------- 3 files changed, 46 insertions(+), 37 deletions(-) diff --git a/dbms/src/Common/ProfileEvents.cpp b/dbms/src/Common/ProfileEvents.cpp index c73e5422439..67303b085f4 100644 --- a/dbms/src/Common/ProfileEvents.cpp +++ b/dbms/src/Common/ProfileEvents.cpp @@ -36,8 +36,10 @@ M(MarkCacheMisses, "") \ M(CreatedReadBufferOrdinary, "") \ M(CreatedReadBufferAIO, "") \ + M(CreatedReadBufferAIOFailed, "") \ M(CreatedWriteBufferOrdinary, "") \ M(CreatedWriteBufferAIO, "") \ + M(CreatedWriteBufferAIOFailed, "") \ M(DiskReadElapsedMicroseconds, "Total time spent waiting for read syscall. This include reads from page cache.") \ M(DiskWriteElapsedMicroseconds, "Total time spent waiting for write syscall. This include writes to page cache.") \ M(NetworkReceiveElapsedMicroseconds, "") \ diff --git a/dbms/src/IO/createReadBufferFromFileBase.cpp b/dbms/src/IO/createReadBufferFromFileBase.cpp index 512f3dfd3ed..8fc5923e6ff 100644 --- a/dbms/src/IO/createReadBufferFromFileBase.cpp +++ b/dbms/src/IO/createReadBufferFromFileBase.cpp @@ -10,34 +10,38 @@ namespace ProfileEvents { extern const Event CreatedReadBufferOrdinary; extern const Event CreatedReadBufferAIO; + extern const Event CreatedReadBufferAIOFailed; } namespace DB { -#if !defined(__linux__) -namespace ErrorCodes -{ - extern const int NOT_IMPLEMENTED; -} -#endif std::unique_ptr createReadBufferFromFileBase(const std::string & filename_, size_t estimated_size, size_t aio_threshold, size_t buffer_size_, int flags_, char * existing_memory_, size_t alignment) { - if ((aio_threshold == 0) || (estimated_size < aio_threshold)) - { - ProfileEvents::increment(ProfileEvents::CreatedReadBufferOrdinary); - return std::make_unique(filename_, buffer_size_, flags_, existing_memory_, alignment); - } - else - { #if defined(__linux__) || defined(__FreeBSD__) - ProfileEvents::increment(ProfileEvents::CreatedReadBufferAIO); - return std::make_unique(filename_, buffer_size_, flags_, existing_memory_); -#else - throw Exception("AIO is implemented only on Linux and FreeBSD", ErrorCodes::NOT_IMPLEMENTED); -#endif + if (aio_threshold && estimated_size >= aio_threshold) + { + /// Attempt to open a file with O_DIRECT + try + { + auto res = std::make_unique(filename_, buffer_size_, flags_, existing_memory_); + ProfileEvents::increment(ProfileEvents::CreatedReadBufferAIO); + return res; + } + catch (const ErrnoException &) + { + /// Fallback to cached IO if O_DIRECT is not supported. + ProfileEvents::increment(ProfileEvents::CreatedReadBufferAIOFailed); + } } +#else + (void)aio_threshold; + (void)estimated_size; +#endif + + ProfileEvents::increment(ProfileEvents::CreatedReadBufferOrdinary); + return std::make_unique(filename_, buffer_size_, flags_, existing_memory_, alignment); } } diff --git a/dbms/src/IO/createWriteBufferFromFileBase.cpp b/dbms/src/IO/createWriteBufferFromFileBase.cpp index 32cbcb1b12e..d20af3ede76 100644 --- a/dbms/src/IO/createWriteBufferFromFileBase.cpp +++ b/dbms/src/IO/createWriteBufferFromFileBase.cpp @@ -10,36 +10,39 @@ namespace ProfileEvents { extern const Event CreatedWriteBufferOrdinary; extern const Event CreatedWriteBufferAIO; + extern const Event CreatedWriteBufferAIOFailed; } namespace DB { -#if !defined(__linux__) -namespace ErrorCodes -{ - extern const int NOT_IMPLEMENTED; -} -#endif - std::unique_ptr createWriteBufferFromFileBase(const std::string & filename_, size_t estimated_size, size_t aio_threshold, size_t buffer_size_, int flags_, mode_t mode, char * existing_memory_, size_t alignment) { - if ((aio_threshold == 0) || (estimated_size < aio_threshold)) - { - ProfileEvents::increment(ProfileEvents::CreatedWriteBufferOrdinary); - return std::make_unique(filename_, buffer_size_, flags_, mode, existing_memory_, alignment); - } - else - { #if defined(__linux__) || defined(__FreeBSD__) - ProfileEvents::increment(ProfileEvents::CreatedWriteBufferAIO); - return std::make_unique(filename_, buffer_size_, flags_, mode, existing_memory_); -#else - throw Exception("AIO is implemented only on Linux and FreeBSD", ErrorCodes::NOT_IMPLEMENTED); -#endif + if (aio_threshold && estimated_size >= aio_threshold) + { + /// Attempt to open a file with O_DIRECT + try + { + auto res = std::make_unique(filename_, buffer_size_, flags_, mode, existing_memory_); + ProfileEvents::increment(ProfileEvents::CreatedWriteBufferAIO); + return res; + } + catch (const ErrnoException &) + { + /// Fallback to cached IO if O_DIRECT is not supported. + ProfileEvents::increment(ProfileEvents::CreatedWriteBufferAIOFailed); + } } +#else + (void)aio_threshold; + (void)estimated_size; +#endif + + ProfileEvents::increment(ProfileEvents::CreatedWriteBufferOrdinary); + return std::make_unique(filename_, buffer_size_, flags_, mode, existing_memory_, alignment); } } From 0bca68e50b540608832b17202ceec6a2d7a5cba4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 29 Aug 2019 21:55:20 +0300 Subject: [PATCH 327/357] Style --- dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 3 ++- dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 7079996af80..fdeb8c3ea96 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -561,7 +561,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); const auto data_settings = data.getSettings(); - NamesAndTypesList gathering_columns, merging_columns; + NamesAndTypesList gathering_columns; + NamesAndTypesList merging_columns; Names gathering_column_names, merging_column_names; extractMergingAndGatheringColumns( all_columns, data.sorting_key_expr, data.skip_indices, diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 3c15bd54df2..0a852e68b74 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -103,7 +103,6 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG serialize_settings.getter = createStreamGetter(column.name, already_written_offset_columns, skip_offsets); column.type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); - if (with_final_mark) writeFinalMark(column.name, column.type, offset_columns, skip_offsets, serialize_settings.path); } From b66485a1d2e6f3bf76086ba99876fce58f37016e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 30 Aug 2019 11:09:13 +0300 Subject: [PATCH 328/357] Reduced children_mutex lock scope in IBlockInputStream This is to fix TSan warning 'lock-order-inversion'. Thread locks IBlockInputStream::children_mutex (A) and then subsequently locks MergeTreeDataPart::columns_lock mutex (B), holding it for extended period of time, surviving the unlock of the A. Then, while B is still locked, A is locked again, causing a TSan warning. --- dbms/src/DataStreams/IBlockInputStream.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index 309eecbeebf..f33c4534a3f 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -314,7 +314,11 @@ private: /// NOTE: Acquire a read lock, therefore f() should be thread safe std::shared_lock lock(children_mutex); - for (auto & child : children) + // Reduce lock scope and avoid recursive locking since that is undefined for shared_mutex. + const auto children_copy = children; + lock.unlock(); + + for (auto & child : children_copy) if (f(*child)) return; } From da8f67123f8661dd33eb324ed9163b37f8719516 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 30 Aug 2019 12:50:38 +0300 Subject: [PATCH 329/357] Improve error handling in cache dictionaries: allow using expired values while the source of a cache dictionary doesn't respond; clone the source after an error to reset connections; don't ask the source for a little time after error; show the last exception in system.dictionaries for a cache dictionary too. --- dbms/src/Dictionaries/CacheDictionary.cpp | 7 + dbms/src/Dictionaries/CacheDictionary.h | 10 +- dbms/src/Dictionaries/CacheDictionary.inc.h | 165 +++++++++++------- dbms/src/Dictionaries/IDictionary.h | 2 + dbms/src/Interpreters/ExternalLoader.cpp | 18 +- dbms/src/Interpreters/ExternalLoader.h | 15 +- dbms/src/Interpreters/IExternalLoadable.cpp | 11 +- dbms/src/Interpreters/IExternalLoadable.h | 12 ++ .../System/StorageSystemDictionaries.cpp | 14 +- 9 files changed, 163 insertions(+), 91 deletions(-) diff --git a/dbms/src/Dictionaries/CacheDictionary.cpp b/dbms/src/Dictionaries/CacheDictionary.cpp index 53fc746e565..c3a78150f05 100644 --- a/dbms/src/Dictionaries/CacheDictionary.cpp +++ b/dbms/src/Dictionaries/CacheDictionary.cpp @@ -70,6 +70,7 @@ CacheDictionary::CacheDictionary( , dict_struct(dict_struct_) , source_ptr{std::move(source_ptr_)} , dict_lifetime(dict_lifetime_) + , log(&Logger::get("ExternalDictionaries")) , size{roundUpToPowerOfTwoOrZero(std::max(size_, size_t(max_collision_length)))} , size_overlap_mask{this->size - 1} , cells{this->size} @@ -575,6 +576,12 @@ BlockInputStreamPtr CacheDictionary::getBlockInputStream(const Names & column_na return std::make_shared(shared_from_this(), max_block_size, getCachedIds(), column_names); } +std::exception_ptr CacheDictionary::getLastException() const +{ + const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs}; + return last_exception; +} + void registerDictionaryCache(DictionaryFactory & factory) { auto create_layout = [=](const std::string & name, diff --git a/dbms/src/Dictionaries/CacheDictionary.h b/dbms/src/Dictionaries/CacheDictionary.h index 7e1cec6ffe9..750c51a7cf3 100644 --- a/dbms/src/Dictionaries/CacheDictionary.h +++ b/dbms/src/Dictionaries/CacheDictionary.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +75,8 @@ public: void isInVectorConstant(const PaddedPODArray & child_ids, const Key ancestor_id, PaddedPODArray & out) const override; void isInConstantVector(const Key child_id, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const override; + std::exception_ptr getLastException() const override; + template using ResultArrayType = std::conditional_t, DecimalPaddedPODArray, PaddedPODArray>; @@ -253,8 +256,9 @@ private: const std::string name; const DictionaryStructure dict_struct; - const DictionarySourcePtr source_ptr; + mutable DictionarySourcePtr source_ptr; const DictionaryLifetime dict_lifetime; + Logger * const log; mutable std::shared_mutex rw_lock; @@ -274,6 +278,10 @@ private: Attribute * hierarchical_attribute = nullptr; std::unique_ptr string_arena; + mutable std::exception_ptr last_exception; + mutable size_t error_count = 0; + mutable std::chrono::system_clock::time_point backoff_end_time; + mutable pcg64 rnd_engine; mutable size_t bytes_allocated = 0; diff --git a/dbms/src/Dictionaries/CacheDictionary.inc.h b/dbms/src/Dictionaries/CacheDictionary.inc.h index 39e1a4b00df..51d515a63dd 100644 --- a/dbms/src/Dictionaries/CacheDictionary.inc.h +++ b/dbms/src/Dictionaries/CacheDictionary.inc.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -243,77 +244,102 @@ template void CacheDictionary::update( const std::vector & requested_ids, PresentIdHandler && on_cell_updated, AbsentIdHandler && on_id_not_found) const { + CurrentMetrics::Increment metric_increment{CurrentMetrics::DictCacheRequests}; + ProfileEvents::increment(ProfileEvents::DictCacheKeysRequested, requested_ids.size()); + std::unordered_map remaining_ids{requested_ids.size()}; for (const auto id : requested_ids) remaining_ids.insert({id, 0}); - std::uniform_int_distribution distribution{dict_lifetime.min_sec, dict_lifetime.max_sec}; + const auto now = std::chrono::system_clock::now(); const ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs}; + if (now > backoff_end_time) { - CurrentMetrics::Increment metric_increment{CurrentMetrics::DictCacheRequests}; - Stopwatch watch; - auto stream = source_ptr->loadIds(requested_ids); - stream->readPrefix(); - - const auto now = std::chrono::system_clock::now(); - - while (const auto block = stream->read()) + try { - const auto id_column = typeid_cast(block.safeGetByPosition(0).column.get()); - if (!id_column) - throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH}; - - const auto & ids = id_column->getData(); - - /// cache column pointers - const auto column_ptrs = ext::map( - ext::range(0, attributes.size()), [&block](size_t i) { return block.safeGetByPosition(i + 1).column.get(); }); - - for (const auto i : ext::range(0, ids.size())) + if (error_count) { - const auto id = ids[i]; - - const auto find_result = findCellIdx(id, now); - const auto & cell_idx = find_result.cell_idx; - - auto & cell = cells[cell_idx]; - - for (const auto attribute_idx : ext::range(0, attributes.size())) - { - const auto & attribute_column = *column_ptrs[attribute_idx]; - auto & attribute = attributes[attribute_idx]; - - setAttributeValue(attribute, cell_idx, attribute_column[i]); - } - - /// if cell id is zero and zero does not map to this cell, then the cell is unused - if (cell.id == 0 && cell_idx != zero_cell_idx) - element_count.fetch_add(1, std::memory_order_relaxed); - - cell.id = id; - if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0) - cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}); - else - cell.setExpiresAt(std::chrono::time_point::max()); - - /// inform caller - on_cell_updated(id, cell_idx); - /// mark corresponding id as found - remaining_ids[id] = 1; + /// Recover after error: we have to clone the source here because + /// it could keep connections which should be reset after error. + source_ptr = source_ptr->clone(); } + + Stopwatch watch; + auto stream = source_ptr->loadIds(requested_ids); + stream->readPrefix(); + + while (const auto block = stream->read()) + { + const auto id_column = typeid_cast(block.safeGetByPosition(0).column.get()); + if (!id_column) + throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH}; + + const auto & ids = id_column->getData(); + + /// cache column pointers + const auto column_ptrs = ext::map( + ext::range(0, attributes.size()), [&block](size_t i) { return block.safeGetByPosition(i + 1).column.get(); }); + + for (const auto i : ext::range(0, ids.size())) + { + const auto id = ids[i]; + + const auto find_result = findCellIdx(id, now); + const auto & cell_idx = find_result.cell_idx; + + auto & cell = cells[cell_idx]; + + for (const auto attribute_idx : ext::range(0, attributes.size())) + { + const auto & attribute_column = *column_ptrs[attribute_idx]; + auto & attribute = attributes[attribute_idx]; + + setAttributeValue(attribute, cell_idx, attribute_column[i]); + } + + /// if cell id is zero and zero does not map to this cell, then the cell is unused + if (cell.id == 0 && cell_idx != zero_cell_idx) + element_count.fetch_add(1, std::memory_order_relaxed); + + cell.id = id; + if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0) + { + std::uniform_int_distribution distribution{dict_lifetime.min_sec, dict_lifetime.max_sec}; + cell.setExpiresAt(now + std::chrono::seconds{distribution(rnd_engine)}); + } + else + cell.setExpiresAt(std::chrono::time_point::max()); + + /// inform caller + on_cell_updated(id, cell_idx); + /// mark corresponding id as found + remaining_ids[id] = 1; + } + } + + stream->readSuffix(); + + error_count = 0; + last_exception = std::exception_ptr{}; + backoff_end_time = std::chrono::system_clock::time_point{}; + + ProfileEvents::increment(ProfileEvents::DictCacheRequestTimeNs, watch.elapsed()); } + catch (...) + { + ++error_count; + last_exception = std::current_exception(); + backoff_end_time = now + std::chrono::seconds(ExternalLoadableBackoff{}.calculateDuration(rnd_engine, error_count)); - stream->readSuffix(); - - ProfileEvents::increment(ProfileEvents::DictCacheKeysRequested, requested_ids.size()); - ProfileEvents::increment(ProfileEvents::DictCacheRequestTimeNs, watch.elapsed()); + tryLogException(last_exception, log, "Could not update cache dictionary '" + getName() + + "', next update is scheduled at " + DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(backoff_end_time))); + } } size_t not_found_num = 0, found_num = 0; - const auto now = std::chrono::system_clock::now(); /// Check which ids have not been found and require setting null_value for (const auto & id_found_pair : remaining_ids) { @@ -328,24 +354,45 @@ void CacheDictionary::update( const auto find_result = findCellIdx(id, now); const auto & cell_idx = find_result.cell_idx; - auto & cell = cells[cell_idx]; - /// Set null_value for each attribute - for (auto & attribute : attributes) - setDefaultAttributeValue(attribute, cell_idx); + if (error_count) + { + if (find_result.outdated) + { + /// We have expired data for that `id` so we can continue using it. + bool was_default = cell.isDefault(); + cell.setExpiresAt(backoff_end_time); + if (was_default) + cell.setDefault(); + if (was_default) + on_id_not_found(id, cell_idx); + else + on_cell_updated(id, cell_idx); + continue; + } + /// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`. + std::rethrow_exception(last_exception); + } /// Check if cell had not been occupied before and increment element counter if it hadn't if (cell.id == 0 && cell_idx != zero_cell_idx) element_count.fetch_add(1, std::memory_order_relaxed); cell.id = id; + if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0) - cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}); + { + std::uniform_int_distribution distribution{dict_lifetime.min_sec, dict_lifetime.max_sec}; + cell.setExpiresAt(now + std::chrono::seconds{distribution(rnd_engine)}); + } else cell.setExpiresAt(std::chrono::time_point::max()); + /// Set null_value for each attribute cell.setDefault(); + for (auto & attribute : attributes) + setDefaultAttributeValue(attribute, cell_idx); /// inform caller that the cell has not been found on_id_not_found(id, cell_idx); diff --git a/dbms/src/Dictionaries/IDictionary.h b/dbms/src/Dictionaries/IDictionary.h index 05f9806dc04..a1c080ca6eb 100644 --- a/dbms/src/Dictionaries/IDictionary.h +++ b/dbms/src/Dictionaries/IDictionary.h @@ -56,6 +56,8 @@ struct IDictionaryBase : public IExternalLoadable return source && source->isModified(); } + virtual std::exception_ptr getLastException() const { return {}; } + std::shared_ptr shared_from_this() { return std::static_pointer_cast(IExternalLoadable::shared_from_this()); diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index cc8466bebc1..6e16fd37cba 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -1,6 +1,5 @@ #include "ExternalLoader.h" -#include #include #include #include @@ -933,6 +932,8 @@ private: class ExternalLoader::PeriodicUpdater : private boost::noncopyable { public: + static constexpr UInt64 check_period_sec = 5; + PeriodicUpdater(ConfigFilesReader & config_files_reader_, LoadingDispatcher & loading_dispatcher_) : config_files_reader(config_files_reader_), loading_dispatcher(loading_dispatcher_) { @@ -940,11 +941,10 @@ public: ~PeriodicUpdater() { enable(false); } - void enable(bool enable_, const ExternalLoaderUpdateSettings & settings_ = {}) + void enable(bool enable_) { std::unique_lock lock{mutex}; enabled = enable_; - settings = settings_; if (enable_) { @@ -985,9 +985,7 @@ public: return std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; } - std::uniform_int_distribution distribution(0, static_cast(std::exp2(error_count - 1))); - std::chrono::seconds delay(std::min(settings.backoff_max_sec, settings.backoff_initial_sec + distribution(rnd_engine))); - return std::chrono::system_clock::now() + delay; + return std::chrono::system_clock::now() + std::chrono::seconds(ExternalLoadableBackoff{}.calculateDuration(rnd_engine, error_count)); } private: @@ -996,9 +994,8 @@ private: setThreadName("ExterLdrReload"); std::unique_lock lock{mutex}; - auto timeout = [this] { return std::chrono::seconds(settings.check_period_sec); }; auto pred = [this] { return !enabled; }; - while (!event.wait_for(lock, timeout(), pred)) + while (!event.wait_for(lock, std::chrono::seconds(check_period_sec), pred)) { lock.unlock(); loading_dispatcher.setConfiguration(config_files_reader.read()); @@ -1012,7 +1009,6 @@ private: mutable std::mutex mutex; bool enabled = false; - ExternalLoaderUpdateSettings settings; ThreadFromGlobalPool thread; std::condition_variable event; mutable pcg64 rnd_engine{randomSeed()}; @@ -1051,9 +1047,9 @@ void ExternalLoader::enableAsyncLoading(bool enable) loading_dispatcher->enableAsyncLoading(enable); } -void ExternalLoader::enablePeriodicUpdates(bool enable_, const ExternalLoaderUpdateSettings & settings_) +void ExternalLoader::enablePeriodicUpdates(bool enable_) { - periodic_updater->enable(enable_, settings_); + periodic_updater->enable(enable_); } bool ExternalLoader::hasCurrentlyLoadedObjects() const diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index 8a52d991759..ecfc43c2dd9 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -11,19 +11,6 @@ namespace DB { -struct ExternalLoaderUpdateSettings -{ - UInt64 check_period_sec = 5; - UInt64 backoff_initial_sec = 5; - /// 10 minutes - UInt64 backoff_max_sec = 10 * 60; - - ExternalLoaderUpdateSettings() = default; - ExternalLoaderUpdateSettings(UInt64 check_period_sec_, UInt64 backoff_initial_sec_, UInt64 backoff_max_sec_) - : check_period_sec(check_period_sec_), backoff_initial_sec(backoff_initial_sec_), backoff_max_sec(backoff_max_sec_) {} -}; - - /* External configuration structure. * * @@ -105,7 +92,7 @@ public: void enableAsyncLoading(bool enable); /// Sets settings for periodic updates. - void enablePeriodicUpdates(bool enable, const ExternalLoaderUpdateSettings & settings = {}); + void enablePeriodicUpdates(bool enable); /// Returns the status of the object. /// If the object has not been loaded yet then the function returns Status::NOT_LOADED. diff --git a/dbms/src/Interpreters/IExternalLoadable.cpp b/dbms/src/Interpreters/IExternalLoadable.cpp index 37855490a9f..e8bf8cbaf3c 100644 --- a/dbms/src/Interpreters/IExternalLoadable.cpp +++ b/dbms/src/Interpreters/IExternalLoadable.cpp @@ -1,7 +1,7 @@ #include #include - +#include namespace DB { @@ -16,4 +16,13 @@ ExternalLoadableLifetime::ExternalLoadableLifetime(const Poco::Util::AbstractCon max_sec = has_min ? config.getUInt64(config_prefix + ".max") : min_sec; } + +UInt64 ExternalLoadableBackoff::calculateDuration(pcg64 & rnd_engine, size_t error_count) const +{ + if (error_count < 1) + error_count = 1; + std::uniform_int_distribution distribution(0, static_cast(std::exp2(error_count - 1))); + return std::min(backoff_max_sec, backoff_initial_sec + distribution(rnd_engine)); +} + } diff --git a/dbms/src/Interpreters/IExternalLoadable.h b/dbms/src/Interpreters/IExternalLoadable.h index f8725a67989..e842fdb8573 100644 --- a/dbms/src/Interpreters/IExternalLoadable.h +++ b/dbms/src/Interpreters/IExternalLoadable.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -25,6 +26,17 @@ struct ExternalLoadableLifetime }; +/// Delay before trying to load again after error. +struct ExternalLoadableBackoff +{ + UInt64 backoff_initial_sec = 5; + UInt64 backoff_max_sec = 10 * 60; /// 10 minutes + + /// Calculates time to try loading again after error. + UInt64 calculateDuration(pcg64 & rnd_engine, size_t error_count = 1) const; +}; + + /// Basic interface for external loadable objects. Is used in ExternalLoader. class IExternalLoadable : public std::enable_shared_from_this, private boost::noncopyable { diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp index 6f85e1ea84b..826bb601609 100644 --- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp +++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp @@ -50,10 +50,11 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con res_columns[i++]->insert(static_cast(load_result.status)); res_columns[i++]->insert(load_result.origin); - if (load_result.object) - { - const auto dict_ptr = std::static_pointer_cast(load_result.object); + std::exception_ptr last_exception = load_result.exception; + const auto dict_ptr = std::dynamic_pointer_cast(load_result.object); + if (dict_ptr) + { res_columns[i++]->insert(dict_ptr->getTypeName()); const auto & dict_struct = dict_ptr->getStructure(); @@ -66,6 +67,9 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con res_columns[i++]->insert(dict_ptr->getElementCount()); res_columns[i++]->insert(dict_ptr->getLoadFactor()); res_columns[i++]->insert(dict_ptr->getSource()->toString()); + + if (!last_exception) + last_exception = dict_ptr->getLastException(); } else { @@ -76,8 +80,8 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con res_columns[i++]->insert(static_cast(std::chrono::system_clock::to_time_t(load_result.loading_start_time))); res_columns[i++]->insert(std::chrono::duration_cast>(load_result.loading_duration).count()); - if (load_result.exception) - res_columns[i++]->insert(getExceptionMessage(load_result.exception, false)); + if (last_exception) + res_columns[i++]->insert(getExceptionMessage(last_exception, false)); else res_columns[i++]->insertDefault(); } From f6120558dfd4aa9d61b2beff7c7513fc168d909a Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 30 Aug 2019 17:29:08 +0300 Subject: [PATCH 330/357] Fix bad size of marks --- .../MergeTree/IMergedBlockOutputStream.cpp | 12 ++++---- .../MergeTree/IMergedBlockOutputStream.h | 5 +++- .../MergeTree/MergedBlockOutputStream.cpp | 4 ++- .../MergedColumnOnlyOutputStream.cpp | 5 +++- ...01000_bad_size_of_marks_skip_idx.reference | 2 ++ .../01000_bad_size_of_marks_skip_idx.sql | 28 +++++++++++++++++++ 6 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.reference create mode 100644 dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.sql diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index e673fd4759a..71e55015d77 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -333,7 +333,7 @@ void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( { /// Creating block for update Block indices_update_block(skip_indexes_columns); - size_t skip_index_current_mark = 0; + size_t skip_index_current_data_mark = 0; /// Filling and writing skip indices like in IMergedBlockOutputStream::writeColumn for (size_t i = 0; i < skip_indices.size(); ++i) @@ -341,7 +341,7 @@ void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( const auto index = skip_indices[i]; auto & stream = *skip_indices_streams[i]; size_t prev_pos = 0; - skip_index_current_mark = skip_index_mark; + skip_index_current_data_mark = skip_index_data_mark; while (prev_pos < rows) { UInt64 limit = 0; @@ -351,7 +351,7 @@ void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( } else { - limit = index_granularity.getMarkRows(skip_index_current_mark); + limit = index_granularity.getMarkRows(skip_index_current_data_mark); if (skip_indices_aggregators[i]->empty()) { skip_indices_aggregators[i] = index->createIndexAggregator(); @@ -366,9 +366,9 @@ void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( /// to be compatible with normal .mrk2 file format if (can_use_adaptive_granularity) writeIntBinary(1UL, stream.marks); - - ++skip_index_current_mark; } + /// this mark is aggregated, go to the next one + skip_index_current_data_mark++; } size_t pos = prev_pos; @@ -388,7 +388,7 @@ void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( prev_pos = pos; } } - skip_index_mark = skip_index_current_mark; + skip_index_data_mark = skip_index_current_data_mark; } void IMergedBlockOutputStream::finishSkipIndicesSerialization( diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index 97c7922042d..6b58f95020f 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -141,7 +141,10 @@ protected: size_t aio_threshold; size_t current_mark = 0; - size_t skip_index_mark = 0; + + /// Number of mark in data from which skip indicies have to start + /// aggregation. I.e. it's data mark number, not skip indices mark. + size_t skip_index_data_mark = 0; const bool can_use_adaptive_granularity; const std::string marks_file_extension; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index b923337d71a..9e33b4594f3 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -332,7 +332,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm else if (skip_indexes_column_name_to_position.end() != skip_index_column_it) { const auto & index_column = *skip_indexes_columns[skip_index_column_it->second].column; - writeColumn(column.name, *column.type, index_column, offset_columns, false, serialization_states[i], current_mark); + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, index_column, offset_columns, false, serialization_states[i], current_mark); } else { @@ -349,6 +349,8 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm rows_count += rows; + /// Should be written before index offset update, because we calculate, + /// indices of currently written granules calculateAndSerializeSkipIndices(skip_indexes_columns, rows); { diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 3c15bd54df2..931cdc67afe 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -68,7 +68,6 @@ void MergedColumnOnlyOutputStream::write(const Block & block) if (!rows) return; - calculateAndSerializeSkipIndices(skip_indexes_columns, rows); size_t new_index_offset = 0; size_t new_current_mark = 0; @@ -79,6 +78,10 @@ void MergedColumnOnlyOutputStream::write(const Block & block) std::tie(new_current_mark, new_index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, skip_offsets, serialization_states[i], current_mark); } + /// Should be written before index offset update, because we calculate, + /// indices of currently written granules + calculateAndSerializeSkipIndices(skip_indexes_columns, rows); + index_offset = new_index_offset; current_mark = new_current_mark; } diff --git a/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.reference b/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.sql b/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.sql new file mode 100644 index 00000000000..7af19fec695 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01000_bad_size_of_marks_skip_idx.sql @@ -0,0 +1,28 @@ +SET allow_experimental_data_skipping_indices=1; + +DROP TABLE IF EXISTS bad_skip_idx; + +CREATE TABLE bad_skip_idx +( + id UInt64, + value String +) ENGINE MergeTree() +ORDER BY id SETTINGS index_granularity_bytes = 64, vertical_merge_algorithm_min_rows_to_activate = 0, vertical_merge_algorithm_min_columns_to_activate = 0; -- actually vertical merge is not required condition for this bug, but it's more easy to reproduce (becuse we don't recalc granularities) + +-- 7 rows per granule +INSERT INTO bad_skip_idx SELECT number, concat('x', toString(number)) FROM numbers(1000); + +-- 3 rows per granule +INSERT INTO bad_skip_idx SELECT number, concat('xxxxxxxxxx', toString(number)) FROM numbers(1000,1000); + +SELECT COUNT(*) from bad_skip_idx WHERE value = 'xxxxxxxxxx1015'; -- check no exception + +INSERT INTO bad_skip_idx SELECT number, concat('x', toString(number)) FROM numbers(1000); + +ALTER TABLE bad_skip_idx ADD INDEX idx value TYPE bloom_filter(0.01) GRANULARITY 4; + +OPTIMIZE TABLE bad_skip_idx FINAL; + +SELECT COUNT(*) from bad_skip_idx WHERE value = 'xxxxxxxxxx1015'; -- check no exception + +DROP TABLE IF EXISTS bad_skip_idx; From d4ea6a52343fc741f7c3d8c4b2e765be867ed09c Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 30 Aug 2019 17:30:28 +0300 Subject: [PATCH 331/357] Fix comment --- dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index 6b58f95020f..37aa2203a72 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -142,7 +142,7 @@ protected: size_t current_mark = 0; - /// Number of mark in data from which skip indicies have to start + /// Number of mark in data from which skip indices have to start /// aggregation. I.e. it's data mark number, not skip indices mark. size_t skip_index_data_mark = 0; From 83c75ca2adfa34c256628077232f8131199accf5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 30 Aug 2019 19:21:05 +0300 Subject: [PATCH 332/357] Added a test (but it doesn't reproduce the issue #6746) --- ...er_nullable_adaptive_granularity.reference | 0 ...002_alter_nullable_adaptive_granularity.sh | 58 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.reference create mode 100755 dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh diff --git a/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.reference b/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh b/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh new file mode 100755 index 00000000000..85fc847f3f3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test (x UInt8, s String MATERIALIZED toString(rand64())) ENGINE = MergeTree ORDER BY s"; + +function thread1() +{ + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO test SELECT rand() FROM numbers(1000)"; + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x Nullable(UInt8);"; + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x UInt8;"; + sleep 0.0$RANDOM + done +} + +function thread3() +{ + while true; do + $CLICKHOUSE_CLIENT -n --query "SELECT count() FROM test FORMAT Null"; + done +} + +function thread4() +{ + while true; do + $CLICKHOUSE_CLIENT -n --query "OPTIMIZE TABLE test FINAL"; + sleep 0.1$RANDOM + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT -q "DROP TABLE test" From 808f4d0b8a9c290f5b794b1f33f00db4b46bd769 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 30 Aug 2019 19:50:59 +0300 Subject: [PATCH 333/357] Intermediate refactoring --- dbms/src/Core/Settings.h | 3 +- dbms/src/Core/SettingsCommon.h | 55 +------------------ dbms/src/Storages/AlterCommands.cpp | 9 +-- dbms/src/Storages/IStorage.cpp | 18 +++--- dbms/src/Storages/IStorage.h | 6 +- dbms/src/Storages/Kafka/KafkaSettings.h | 23 ++++---- dbms/src/Storages/Kafka/StorageKafka.cpp | 45 ++++++++++----- dbms/src/Storages/Kafka/StorageKafka.h | 4 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 14 +++-- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 +- .../Storages/MergeTree/MergeTreeSettings.h | 14 +++-- .../MergeTree/registerStorageMergeTree.cpp | 3 - 12 files changed, 78 insertions(+), 118 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index cb12a969b76..e03efa3f56f 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -42,8 +42,7 @@ struct Settings : public SettingsCollection * but we are not going to do it, because settings is used everywhere as static struct fields. */ -/// M (mutable) for normal settings, IM (immutable) for not updateable settings. -#define LIST_OF_SETTINGS(M, IM) \ +#define LIST_OF_SETTINGS(M) \ M(SettingUInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.") \ M(SettingUInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.") \ M(SettingUInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading") \ diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h index de30f43c7c8..07badbdddc0 100644 --- a/dbms/src/Core/SettingsCommon.h +++ b/dbms/src/Core/SettingsCommon.h @@ -17,11 +17,6 @@ class Field; class ReadBuffer; class WriteBuffer; -namespace ErrorCodes -{ - extern const int IMMUTABLE_SETTING; -} - /** One setting for any type. * Stores a value within itself, as well as a flag - whether the value was changed. * This is done so that you can send to the remote servers only changed settings (or explicitly specified in the config) values. @@ -34,7 +29,6 @@ struct SettingNumber { Type value; bool changed = false; - bool immutable = false; SettingNumber(Type x = 0) : value(x) {} @@ -77,7 +71,6 @@ struct SettingMaxThreads UInt64 value; bool is_auto; bool changed = false; - bool immutable = false; SettingMaxThreads(UInt64 x = 0) : value(x ? x : getAutoValue()), is_auto(x == 0) {} @@ -106,7 +99,6 @@ struct SettingTimespan { Poco::Timespan value; bool changed = false; - bool immutable = false; SettingTimespan(UInt64 x = 0) : value(x * microseconds_per_io_unit) {} @@ -139,7 +131,6 @@ struct SettingString { String value; bool changed = false; - bool immutable = false; SettingString(const String & x = String{}) : value(x) {} @@ -162,7 +153,6 @@ struct SettingChar public: char value; bool changed = false; - bool immutable = false; SettingChar(char x = '\0') : value(x) {} @@ -187,7 +177,6 @@ struct SettingEnum { EnumType value; bool changed = false; - bool immutable = false; SettingEnum(EnumType x) : value(x) {} @@ -322,8 +311,6 @@ private: using SerializeFunction = void (*)(const Derived &, WriteBuffer & buf); using DeserializeFunction = void (*)(Derived &, ReadBuffer & buf); using CastValueWithoutApplyingFunction = Field (*)(const Field &); - using SetImmutable = void(*)(Derived &); - using IsImmutable = bool(*)(const Derived &); struct MemberInfo @@ -331,8 +318,6 @@ private: IsChangedFunction is_changed; StringRef name; StringRef description; - /// At one moment this setting can became immutable - const bool can_be_immutable; GetStringFunction get_string; GetFieldFunction get_field; SetStringFunction set_string; @@ -340,8 +325,6 @@ private: SerializeFunction serialize; DeserializeFunction deserialize; CastValueWithoutApplyingFunction cast_value_without_applying; - SetImmutable set_immutable; - IsImmutable is_immutable; bool isChanged(const Derived & collection) const { return is_changed(collection); } }; @@ -431,20 +414,8 @@ public: public: reference(Derived & collection_, const MemberInfo & member_) : const_reference(collection_, member_) {} reference(const const_reference & src) : const_reference(src) {} - void setValue(const Field & value) - { - if (this->member->is_immutable(*this->collection)) - throw Exception("Setting '" + this->member->name.toString() + "' is restricted for updates.", ErrorCodes::IMMUTABLE_SETTING); - this->member->set_field(*const_cast(this->collection), value); - } - void setValue(const String & value) - { - if (this->member->is_immutable(*this->collection)) - throw Exception("Setting '" + this->member->name.toString() + "' is restricted for updates.", ErrorCodes::IMMUTABLE_SETTING); - this->member->set_string(*const_cast(this->collection), value); - } - bool canBeImmutable() const { return this->member->can_be_immutable; } - void makeImmutableForever() { this->member->set_immutable(*const_cast(this->collection)); } + void setValue(const Field & value) { this->member->set_field(*const_cast(this->collection), value); } + void setValue(const String & value) { this->member->set_string(*const_cast(this->collection), value); } }; /// Iterator to iterating through all the settings. @@ -615,14 +586,6 @@ public: dest.copyChangesFrom(castToDerived()); } - /// Make all possible immutable settings (can_be_immutable) immutable forever - void finishSettingsInitialization() - { - for (auto & member : *this) - if (member.canBeImmutable()) - member.makeImmutableForever(); - } - /// Writes the settings to buffer (e.g. to be sent to remote server). /// Only changed settings are written. They are written as list of contiguous name-value pairs, /// finished with empty name. @@ -666,7 +629,7 @@ public: { \ LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_, IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \ }; \ - LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_, IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_) \ + LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_) \ } @@ -682,8 +645,6 @@ public: static void NAME##_serialize(const Derived & collection, WriteBuffer & buf) { collection.NAME.serialize(buf); } \ static void NAME##_deserialize(Derived & collection, ReadBuffer & buf) { collection.NAME.deserialize(buf); } \ static Field NAME##_castValueWithoutApplying(const Field & value) { TYPE temp{DEFAULT}; temp.set(value); return temp.toField(); } \ - static void NAME##_setImmutable(Derived & collection) { collection.NAME.immutable = true; } \ - static bool NAME##_isImmutable(const Derived & collection) { return collection.NAME.immutable; } #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ @@ -693,14 +654,4 @@ public: &Functions::NAME##_setString, &Functions::NAME##_setField, \ &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ &Functions::NAME##_castValueWithoutApplying, \ - &Functions::NAME##_setImmutable, &Functions::NAME##_isImmutable }); - -#define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \ - add({[](const Derived & d) { return d.NAME.changed; }, \ - StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \ - &Functions::NAME##_getString, &Functions::NAME##_getField, \ - &Functions::NAME##_setString, &Functions::NAME##_setField, \ - &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \ - &Functions::NAME##_castValueWithoutApplying, \ - &Functions::NAME##_setImmutable, &Functions::NAME##_isImmutable }); } diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index edac6b3b88b..2a311461f00 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -543,15 +543,8 @@ void AlterCommands::validate(const IStorage & table, const Context & context) } } else if (command.type == AlterCommand::MODIFY_SETTING) - { for (const auto & change : command.settings_changes) - { - if (!table.hasSetting(change.name)) - { - throw Exception{"Storage '" + table.getName() + "' doesn't have setting '" + change.name + "'", ErrorCodes::UNKNOWN_SETTING}; - } - } - } + table.checkSetting(change.name); } /** Existing defaulted columns may require default expression extensions with a type conversion, diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 2f3a48d90b6..a26af1637fa 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -308,11 +308,10 @@ bool IStorage::isVirtualColumn(const String & column_name) const return getColumns().get(column_name).is_virtual; } -bool IStorage::hasSetting(const String & /* setting_name */) const +void IStorage::checkSetting(const String & /* setting_name */) const { if (!supportsSettings()) throw Exception("Storage '" + getName() + "' doesn't support settings.", ErrorCodes::SETTINGS_ARE_NOT_SUPPORTED); - return false; } TableStructureReadLockHolder IStorage::lockStructureForShare(bool will_add_new_data, const String & query_id) @@ -380,16 +379,13 @@ IDatabase::ASTModifier IStorage::getSettingsModifier(const SettingsChanges & new /// Make storage settings unique for (const auto & change : new_changes) { - if (hasSetting(change.name)) - { - auto finder = [&change] (const SettingChange & c) { return c.name == change.name; }; - if (auto it = std::find_if(storage_changes.begin(), storage_changes.end(), finder); it != storage_changes.end()) - it->value = change.value; - else - storage_changes.push_back(change); - } + checkSetting(change.name); + + auto finder = [&change] (const SettingChange & c) { return c.name == change.name; }; + if (auto it = std::find_if(storage_changes.begin(), storage_changes.end(), finder); it != storage_changes.end()) + it->value = change.value; else - throw Exception{"Storage '" + getName() + "' doesn't have setting '" + change.name + "'", ErrorCodes::UNKNOWN_SETTING}; + storage_changes.push_back(change); } } }; diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 6c23a638ddf..dc3f4905310 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -138,8 +138,8 @@ public: /// thread-unsafe part. lockStructure must be acquired /// If |need_all| is set, then checks that all the columns of the table are in the block. void check(const Block & block, bool need_all = false) const; - /// Check storage has setting. Exception will be thrown if it doesn't support settings at all. - virtual bool hasSetting(const String & setting_name) const; + /// Check storage has setting and setting can be modified. + virtual bool checkSetting(const String & setting_name) const; protected: /// still thread-unsafe part. void setIndices(IndicesDescription indices_); @@ -149,7 +149,7 @@ protected: /// still thread-unsafe part. virtual bool isVirtualColumn(const String & column_name) const; /// Returns modifier of settings in storage definition - virtual IDatabase::ASTModifier getSettingsModifier(const SettingsChanges & new_changes) const; + IDatabase::ASTModifier getSettingsModifier(const SettingsChanges & new_changes) const; private: ColumnsDescription columns; /// combined real and virtual columns diff --git a/dbms/src/Storages/Kafka/KafkaSettings.h b/dbms/src/Storages/Kafka/KafkaSettings.h index bc453238b51..6ff62f30411 100644 --- a/dbms/src/Storages/Kafka/KafkaSettings.h +++ b/dbms/src/Storages/Kafka/KafkaSettings.h @@ -15,18 +15,17 @@ struct KafkaSettings : public SettingsCollection { -/// M (mutable) for normal settings, IM (immutable) for not updateable settings. -#define LIST_OF_KAFKA_SETTINGS(M, IM) \ - IM(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ - IM(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ - IM(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \ - IM(SettingString, kafka_format, "", "The message format for Kafka engine.") \ - IM(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.") \ - IM(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine") \ - IM(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.") \ - IM(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \ - IM(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block") \ - IM(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block") +#define LIST_OF_KAFKA_SETTINGS(M) \ + M(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \ + M(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \ + M(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \ + M(SettingString, kafka_format, "", "The message format for Kafka engine.") \ + M(SettingChar, kafka_row_delimiter, '\0', "The character to be considered as a delimiter in Kafka message.") \ + M(SettingString, kafka_schema, "", "Schema identifier (used by schema-based formats) for Kafka engine") \ + M(SettingUInt64, kafka_num_consumers, 1, "The number of consumers per table for Kafka engine.") \ + M(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \ + M(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block") \ + M(SettingUInt64, kafka_commit_every_batch, 0, "Commit every consumed and handled batch instead of a single commit after writing a whole block") DECLARE_SETTINGS_COLLECTION(LIST_OF_KAFKA_SETTINGS) diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 835ce43b1a4..956a19a8ba8 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -34,16 +34,37 @@ namespace DB { +namespace ProfileEvents +{ + extern const Event RejectedInserts; + extern const Event DelayedInserts; + extern const Event DelayedInsertsMilliseconds; +} + +namespace CurrentMetrics +{ + extern const Metric DelayedInserts; +} + namespace ErrorCodes { - extern const int INCORRECT_DATA; - extern const int UNKNOWN_EXCEPTION; - extern const int CANNOT_READ_FROM_ISTREAM; - extern const int INVALID_CONFIG_PARAMETER; - extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int UNSUPPORTED_METHOD; + extern const int MEMORY_LIMIT_EXCEEDED; + extern const int SYNTAX_ERROR; + extern const int INVALID_PARTITION_VALUE; + extern const int METADATA_MISMATCH; + extern const int PART_IS_TEMPORARILY_LOCKED; + extern const int TOO_MANY_PARTS; + extern const int INCOMPATIBLE_COLUMNS; + extern const int CANNOT_UPDATE_COLUMN; + extern const int CANNOT_ALLOCATE_MEMORY; + extern const int CANNOT_MUNMAP; + extern const int CANNOT_MREMAP; + extern const int BAD_TTL_EXPRESSION; + extern const int INCORRECT_FILE_NAME; + extern const int BAD_DATA_PART_NAME; + extern const int UNKNOWN_SETTING; + extern const int IMMUTABLE_SETTING; } namespace @@ -407,14 +428,12 @@ bool StorageKafka::streamToViews() } -bool StorageKafka::hasSetting(const String & setting_name) const +void StorageKafka::checkSetting(const String & setting_name) const { - return KafkaSettings::findIndex(setting_name) != KafkaSettings::npos; -} + if (KafkaSettings::findIndex(setting_name) == KafkaSettings::npos) + throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting_name + "'", ErrorCodes::UNKNOWN_SETTING}; -IDatabase::ASTModifier StorageKafka::getSettingsModifier(const SettingsChanges & /* new_changes */) const -{ - throw Exception("Storage '" + getName() + "' doesn't support settings alter", ErrorCodes::UNSUPPORTED_METHOD); + throw Exception{"Setting '" + setting_name + "' is immutable for storage '" + getName() + "'", ErrorCodes::IMMUTABLE_SETTING}; } void registerStorageKafka(StorageFactory & factory) diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index b1eac57dca1..fc336eabafb 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -57,8 +57,7 @@ public: const auto & getSchemaName() const { return schema_name; } const auto & skipBroken() const { return skip_broken; } - bool hasSetting(const String & setting_name) const override; - + void checkSetting(const String & setting_name) const override; protected: StorageKafka( @@ -71,7 +70,6 @@ protected: size_t num_consumers_, UInt64 max_block_size_, size_t skip_broken, bool intermediate_commit_); - IDatabase::ASTModifier getSettingsModifier(const SettingsChanges & new_changes) const override; private: // Configuration and state String table_name; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index f8f915cadef..aceedec78bc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -91,6 +91,7 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int BAD_DATA_PART_NAME; extern const int UNKNOWN_SETTING; + extern const int IMMUTABLE_SETTING; } @@ -1324,10 +1325,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast, /* only_check = */ true); for (const auto & setting : new_changes) - { - if (!hasSetting(setting.name)) - throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting.name + "'", ErrorCodes::UNKNOWN_SETTING}; - } + checkSetting(setting.name); /// Check that type conversions are possible. ExpressionActionsPtr unused_expression; @@ -1657,9 +1655,13 @@ void MergeTreeData::changeSettings( } } -bool MergeTreeData::hasSetting(const String & setting_name) const +void MergeTreeData::checkSetting(const String & setting_name) const { - return MergeTreeSettings::findIndex(setting_name) != MergeTreeSettings::npos; + if (MergeTreeSettings::findIndex(setting_name) == MergeTreeSettings::npos) + throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting_name + "'", ErrorCodes::UNKNOWN_SETTING}; + if (MergeTreeSettings::isImmutableSetting(setting_name)) + throw Exception{"Setting '" + setting_name + "' is immutable for storage '" + getName() + "'", ErrorCodes::IMMUTABLE_SETTING}; + } void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 0440a3181c8..5a82ee407a4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -543,7 +543,7 @@ public: TableStructureWriteLockHolder & table_lock_holder); /// All MergeTreeData children have settings. - bool hasSetting(const String & setting_name) const override; + void checkSetting(const String & setting_name) const override; /// Remove columns, that have been markedd as empty after zeroing values with expired ttl void removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 6ba08fed5da..20d81771f5e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -2,6 +2,7 @@ #include #include +#include namespace Poco @@ -24,9 +25,8 @@ class ASTStorage; struct MergeTreeSettings : public SettingsCollection { -/// M (mutable) for normal settings, IM (immutable) for not updateable settings. -#define LIST_OF_MERGE_TREE_SETTINGS(M, IM) \ - IM(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \ +#define LIST_OF_MERGE_TREE_SETTINGS(M) \ + M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \ \ /** Merge settings. */ \ M(SettingUInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024, "Maximum in total size of parts to merge, when there are maximum free threads in background pool (or entries in replication queue).") \ @@ -80,7 +80,7 @@ struct MergeTreeSettings : public SettingsCollection M(SettingBool, use_minimalistic_part_header_in_zookeeper, false, "Store part header (checksums and columns) in a compact format and a single part znode instead of separate znodes (/columns and /checksums). This can dramatically reduce snapshot size in ZooKeeper. Before enabling check that all replicas support new format.") \ M(SettingUInt64, finished_mutations_to_keep, 100, "How many records about mutations that are done to keep. If zero, then keep all of them.") \ M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024, "Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled).") \ - IM(SettingUInt64, index_granularity_bytes, 10 * 1024 * 1024, "Approximate amount of bytes in single granule (0 - disabled).") \ + M(SettingUInt64, index_granularity_bytes, 10 * 1024 * 1024, "Approximate amount of bytes in single granule (0 - disabled).") \ M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "Minimal time in seconds, when merge with TTL can be repeated.") \ M(SettingBool, write_final_mark, 1, "Write final mark after end of column (0 - disabled, do nothing if index_granularity_bytes=0)") \ M(SettingBool, enable_mixed_granularity_parts, 0, "Enable parts with adaptive and non adaptive granularity") \ @@ -98,6 +98,12 @@ struct MergeTreeSettings : public SettingsCollection /// NOTE: will rewrite the AST to add immutable settings. void loadFromQuery(ASTStorage & storage_def); + + /// We check settings after storage creation + static bool isImmutableSetting(const String & name) + { + return name == "index_granularity" || name == "index_granularity_bytes"; + } }; using MergeTreeSettingsPtr = std::shared_ptr; diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 596ea4f5d0b..674116a54bc 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -637,9 +637,6 @@ static StoragePtr create(const StorageFactory::Arguments & args) throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \ "before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS); - /// Finalize settings and disable updates - storage_settings->finishSettingsInitialization(); - if (replicated) return StorageReplicatedMergeTree::create( zookeeper_path, replica_name, args.attach, args.data_path, args.database_name, args.table_name, From cd5c0fc9ac29af25632ab50f902497db737ddfa7 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Fri, 30 Aug 2019 20:40:27 +0300 Subject: [PATCH 334/357] Fix build issues (#6744) * libcxxabi uses exception handling library as public * Don't set -stdlib for internal libc++ - it poisons the checks. * Enable capnproto in unbundled build back --- cmake/find_ccache.cmake | 21 ++++++++++--------- cmake/find_cxx.cmake | 3 ++- contrib/libcxx-cmake/CMakeLists.txt | 2 +- contrib/libcxxabi-cmake/CMakeLists.txt | 7 +------ .../integration/test_storage_kafka/test.py | 2 +- docker/packager/packager | 2 +- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/cmake/find_ccache.cmake b/cmake/find_ccache.cmake index c2d467ca6ba..95d6b208cfa 100644 --- a/cmake/find_ccache.cmake +++ b/cmake/find_ccache.cmake @@ -1,13 +1,14 @@ find_program (CCACHE_FOUND ccache) -if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache") - execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION) - string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION}) - if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - #message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}") - set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) - set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) - else () - message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118") - endif () +if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache") + execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION) + string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION}) + + if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + #message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}") + set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) + set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) + else () + message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118") + endif () endif () diff --git a/cmake/find_cxx.cmake b/cmake/find_cxx.cmake index da6e3d48cab..f84a76183ec 100644 --- a/cmake/find_cxx.cmake +++ b/cmake/find_cxx.cmake @@ -25,6 +25,8 @@ if (USE_LIBCXX) find_library (LIBCXXFS_LIBRARY c++fs) find_library (LIBCXXABI_LIBRARY c++abi) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) else () set (LIBCXX_LIBRARY cxx) @@ -38,7 +40,6 @@ if (USE_LIBCXX) target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY}) set (HAVE_LIBCXX 1) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}") message (STATUS "Using libcxxfs: ${LIBCXXFS_LIBRARY}") diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index 8759786c3e8..9609c7ca9e7 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -40,7 +40,7 @@ ${LIBCXX_SOURCE_DIR}/src/random.cpp add_library(cxx ${SRCS}) -target_include_directories(cxx BEFORE PUBLIC $) +target_include_directories(cxx SYSTEM BEFORE PUBLIC $) target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI) target_compile_options(cxx PUBLIC -nostdinc++ -Wno-reserved-id-macro) target_link_libraries(cxx PUBLIC cxxabi) diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt index 2abe5702132..68bb5606689 100644 --- a/contrib/libcxxabi-cmake/CMakeLists.txt +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -30,12 +30,7 @@ target_include_directories(cxxabi SYSTEM BEFORE ) target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY) target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined -Wno-macro-redefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast. - -if (USE_UNWIND) - target_link_libraries(cxxabi PRIVATE ${UNWIND_LIBRARIES}) -else () - target_link_libraries(cxxabi PRIVATE gcc_eh) -endif () +target_link_libraries(cxxabi PUBLIC ${EXCEPTION_HANDLING_LIBRARY}) install( TARGETS cxxabi diff --git a/dbms/tests/integration/test_storage_kafka/test.py b/dbms/tests/integration/test_storage_kafka/test.py index 47ba957cdef..7d17c1cf362 100644 --- a/dbms/tests/integration/test_storage_kafka/test.py +++ b/dbms/tests/integration/test_storage_kafka/test.py @@ -605,7 +605,7 @@ def test_kafka_produce_consume(kafka_cluster): assert int(result) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) -@pytest.mark.timeout(120) +@pytest.mark.timeout(180) def test_kafka_commit_on_block_write(kafka_cluster): instance.query(''' DROP TABLE IF EXISTS test.view; diff --git a/docker/packager/packager b/docker/packager/packager index c132f514569..d0d46fd5f51 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -143,7 +143,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") if unbundled: - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DUSE_CAPNP=0') + cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') From 2647d4ca158b56546b335757fd39fb1297e253d1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 30 Aug 2019 23:12:26 +0300 Subject: [PATCH 335/357] Rename immutable to readonly --- dbms/src/Common/ErrorCodes.cpp | 2 +- dbms/src/Storages/Kafka/StorageKafka.cpp | 4 ++-- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 6 +++--- dbms/src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 87ab252c583..76622cfa43b 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -446,7 +446,7 @@ namespace ErrorCodes extern const int VIOLATED_CONSTRAINT = 469; extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW = 470; extern const int SETTINGS_ARE_NOT_SUPPORTED = 471; - extern const int IMMUTABLE_SETTING = 472; + extern const int READONLY_SETTING = 472; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 22238a0a15e..fc43e7046a7 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -45,7 +45,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int UNSUPPORTED_METHOD; extern const int UNKNOWN_SETTING; - extern const int IMMUTABLE_SETTING; + extern const int READONLY_SETTING; } namespace @@ -417,7 +417,7 @@ void StorageKafka::checkSetting(const String & setting_name) const if (KafkaSettings::findIndex(setting_name) == KafkaSettings::npos) throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting_name + "'", ErrorCodes::UNKNOWN_SETTING}; - throw Exception{"Setting '" + setting_name + "' is immutable for storage '" + getName() + "'", ErrorCodes::IMMUTABLE_SETTING}; + throw Exception{"Setting '" + setting_name + "' is readonly for storage '" + getName() + "'", ErrorCodes::READONLY_SETTING}; } void registerStorageKafka(StorageFactory & factory) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index cf2dfc63ac8..a95c526bf20 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -91,7 +91,7 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int BAD_DATA_PART_NAME; extern const int UNKNOWN_SETTING; - extern const int IMMUTABLE_SETTING; + extern const int READONLY_SETTING; } @@ -1661,8 +1661,8 @@ void MergeTreeData::checkSetting(const String & setting_name) const { if (MergeTreeSettings::findIndex(setting_name) == MergeTreeSettings::npos) throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting_name + "'", ErrorCodes::UNKNOWN_SETTING}; - if (MergeTreeSettings::isImmutableSetting(setting_name)) - throw Exception{"Setting '" + setting_name + "' is immutable for storage '" + getName() + "'", ErrorCodes::IMMUTABLE_SETTING}; + if (MergeTreeSettings::isReadonlySetting(setting_name)) + throw Exception{"Setting '" + setting_name + "' is readonly for storage '" + getName() + "'", ErrorCodes::READONLY_SETTING}; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 6d6157bcc25..8ab7965de04 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -101,7 +101,7 @@ struct MergeTreeSettings : public SettingsCollection void loadFromQuery(ASTStorage & storage_def); /// We check settings after storage creation - static bool isImmutableSetting(const String & name) + static bool isReadonlySetting(const String & name) { return name == "index_granularity" || name == "index_granularity_bytes"; } From cc118b3fed1e8e6cb172aabb1503d84c4a8bfa8a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 01:43:15 +0300 Subject: [PATCH 336/357] Added a test for RENAME / Merge table race condition --- ...1001_rename_merge_race_condition.reference | 0 .../01001_rename_merge_race_condition.sh | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.reference create mode 100755 dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.sh diff --git a/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.reference b/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.sh b/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.sh new file mode 100755 index 00000000000..b0f1dda7c45 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01001_rename_merge_race_condition.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt64) ENGINE = Memory"; + + +function thread1() +{ + while true; do + seq 1 1000 | sed -r -e 's/.+/RENAME TABLE test1 TO test2; RENAME TABLE test2 TO test1;/' | $CLICKHOUSE_CLIENT -n + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM merge(currentDatabase(), '^test[12]$')" + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; From 2fd8f5c32495da2cc20851a2ad6b68f22c381316 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 02:09:08 +0300 Subject: [PATCH 337/357] Removed code that I don't understand and that has no comments --- dbms/src/Interpreters/InterpreterKillQueryQuery.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp b/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp index 4e4120c5580..89339668088 100644 --- a/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -265,11 +265,6 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S if (where_expression) select_query += " WHERE " + queryToString(where_expression); - auto use_processors = context.getSettingsRef().experimental_use_processors; - context.getSettingsRef().experimental_use_processors = false; - - SCOPE_EXIT(context.getSettingsRef().experimental_use_processors = use_processors); - BlockIO block_io = executeQuery(select_query, context, true); Block res = block_io.in->read(); From fd85a862f039c659d9d699c6ea42aa463658147b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 02:19:05 +0300 Subject: [PATCH 338/357] Added a test just in case --- .../01003_kill_query_race_condition.reference | 0 .../01003_kill_query_race_condition.sh | 50 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01003_kill_query_race_condition.reference create mode 100755 dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh diff --git a/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.reference b/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh b/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh new file mode 100755 index 00000000000..9eeafaa8442 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +function thread1() +{ + while true; do + $CLICKHOUSE_CLIENT --query_id=hello --query "SELECT count() FROM system.numbers"; + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id = 'hello'" --format Null; + sleep 0.$RANDOM + done +} + +function thread3() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SHOW PROCESSLIST" --format Null; + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.processes" --format Null; + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & + +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +wait From bb0ca310ab9406ab6a110657554b9a53cf6307a8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 02:38:03 +0300 Subject: [PATCH 339/357] Allow to ATTACH live views --- dbms/src/Storages/LiveView/StorageLiveView.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 1726fe0fba1..b6877c52752 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -593,7 +593,7 @@ void registerStorageLiveView(StorageFactory & factory) { factory.registerStorage("LiveView", [](const StorageFactory::Arguments & args) { - if (!args.local_context.getSettingsRef().allow_experimental_live_view) + if (!args.attach && !args.local_context.getSettingsRef().allow_experimental_live_view) throw Exception("Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')", ErrorCodes::SUPPORT_IS_DISABLED); return StorageLiveView::create(args.table_name, args.database_name, args.local_context, args.query, args.columns); From 5fcdd6f20b58ad2b0b5da4d4ab0373e4ceefe8eb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 03:19:10 +0300 Subject: [PATCH 340/357] Added stress test variant that is as simple to run as ./stress --- dbms/tests/stress | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 dbms/tests/stress diff --git a/dbms/tests/stress b/dbms/tests/stress new file mode 100755 index 00000000000..9099b5a9b91 --- /dev/null +++ b/dbms/tests/stress @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# https://stackoverflow.com/questions/360201/how-do-i-kill-background-processes-jobs-when-my-shell-script-exits +trap 'kill -9 $(jobs -p)' EXIT + +function thread() +{ + while true; do + ./clickhouse-test --order random 2>&1 | awk '{ printf "'$1' " }' + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread; + +NUM_THREADS=16 +TIMEOUT=300 + +for i in $(seq 1 $NUM_THREADS); do + timeout $TIMEOUT bash -c "thread $i" 2> /dev/null & +done + +wait From 57c8091e5b680ab1d9e32b147e4abf0f2b17a273 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 03:30:12 +0300 Subject: [PATCH 341/357] Better stress test script --- dbms/tests/stress | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/stress b/dbms/tests/stress index 9099b5a9b91..90728f7e990 100755 --- a/dbms/tests/stress +++ b/dbms/tests/stress @@ -13,8 +13,8 @@ function thread() # https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout export -f thread; -NUM_THREADS=16 -TIMEOUT=300 +NUM_THREADS=${1:-"16"} +TIMEOUT=${2:-"300"} for i in $(seq 1 $NUM_THREADS); do timeout $TIMEOUT bash -c "thread $i" 2> /dev/null & From fbd616b6a4ae49108ddf3dc54aea769b1dbac786 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 31 Aug 2019 03:39:38 +0300 Subject: [PATCH 342/357] Add integration test for handling errors by a cache dictionary. --- .../dictionary_preset_cache_xypairs.xml | 31 ++++++++++ .../integration/test_dictionaries/test.py | 59 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml new file mode 100644 index 00000000000..4142706259a --- /dev/null +++ b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml @@ -0,0 +1,31 @@ + + + cache_xypairs + + + localhost + 9000 + default + + test + xypairs
+ + + 1 + + + 5 + + + + + x + + + y + UInt64 + 0 + + + + diff --git a/dbms/tests/integration/test_dictionaries/test.py b/dbms/tests/integration/test_dictionaries/test.py index 251fe7f31ee..95f82f65c0d 100644 --- a/dbms/tests/integration/test_dictionaries/test.py +++ b/dbms/tests/integration/test_dictionaries/test.py @@ -17,6 +17,10 @@ def get_status(dictionary_name): return instance.query("SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") +def get_last_exception(dictionary_name): + return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n").replace("\\'", "'") + + def get_loading_start_time(dictionary_name): s = instance.query("SELECT loading_start_time FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") if s == "0000-00-00 00:00:00": @@ -350,3 +354,58 @@ def test_reload_after_fail_by_timer(started_cluster): time.sleep(6); query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" assert get_status("no_file_2") == "LOADED" + + +def test_reload_after_fail_in_cache_dictionary(started_cluster): + query = instance.query + query_and_get_error = instance.query_and_get_error + + # Can't get a value from the cache dictionary because the source (table `test.xypairs`) doesn't respond. + expected_error = "Table test.xypairs doesn't exist" + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(1))") + assert get_status("cache_xypairs") == "LOADED" + assert expected_error in get_last_exception("cache_xypairs") + + # Create table `test.xypairs`. + query(''' + drop table if exists test.xypairs; + create table test.xypairs (x UInt64, y UInt64) engine=Log; + insert into test.xypairs values (1, 56), (3, 78); + ''') + + # Cache dictionary now works. + assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "56", ignore_error=True) + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert get_last_exception("cache_xypairs") == "" + + # Drop table `test.xypairs`. + query('drop table if exists test.xypairs') + + # Values are cached so we can get them. + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert get_last_exception("cache_xypairs") == "" + + # But we can't get a value from the source table which isn't cached. + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") + assert expected_error in get_last_exception("cache_xypairs") + + # Passed time should not spoil the cache. + time.sleep(5); + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") + assert expected_error in get_last_exception("cache_xypairs") + + # Create table `test.xypairs` again with changed values. + query(''' + drop table if exists test.xypairs; + create table test.xypairs (x UInt64, y UInt64) engine=Log; + insert into test.xypairs values (1, 57), (3, 79); + ''') + + # The cache dictionary returns new values now. + assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "57") + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(3))") == "79" + assert get_last_exception("cache_xypairs") == "" From e4376a3f6f42ed2a9069f5f8242695461b8d9bf4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 04:21:10 +0300 Subject: [PATCH 343/357] Addition to prev. revision --- .../queries/0_stateless/01003_kill_query_race_condition.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh b/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh index 9eeafaa8442..d8a73ac24a4 100755 --- a/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh +++ b/dbms/tests/queries/0_stateless/01003_kill_query_race_condition.sh @@ -8,7 +8,7 @@ set -e function thread1() { while true; do - $CLICKHOUSE_CLIENT --query_id=hello --query "SELECT count() FROM system.numbers"; + $CLICKHOUSE_CLIENT --query_id=hello --query "SELECT count() FROM numbers(1000000000)" --format Null; done } From d879bcb010fdf27f5d467e2f658daeed2fd67b1e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 05:32:42 +0300 Subject: [PATCH 344/357] Added a test for deadlock in RENAME TABLE --- .../01004_rename_deadlock.reference | 0 .../0_stateless/01004_rename_deadlock.sh | 60 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01004_rename_deadlock.reference create mode 100755 dbms/tests/queries/0_stateless/01004_rename_deadlock.sh diff --git a/dbms/tests/queries/0_stateless/01004_rename_deadlock.reference b/dbms/tests/queries/0_stateless/01004_rename_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh b/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh new file mode 100755 index 00000000000..b09fabe3d9e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt8) ENGINE = MergeTree ORDER BY x"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test2 (x UInt8) ENGINE = MergeTree ORDER BY x"; + +function thread1() +{ + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE test1 TO test_tmp, test2 TO test1, test_tmp TO test2" + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM test1 UNION ALL SELECT * FROM test2" --format Null + done +} + +function thread3() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.tables" --format Null + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; + +TIMEOUT=1000 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT -q "DROP TABLE test1" +$CLICKHOUSE_CLIENT -q "DROP TABLE test2" From 202673e3bd40c57dd738bb4a2adf19f973bd53e7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 13:37:28 +0300 Subject: [PATCH 345/357] Avoid deadlock in multiple tables RENAME --- .../Interpreters/InterpreterRenameQuery.cpp | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index e763c002209..3e8bc3a88fb 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -26,8 +26,6 @@ struct RenameDescription to_table_name(elem.to.table) {} - TableStructureWriteLockHolder from_table_lock; - String from_database_name; String from_table_name; @@ -77,8 +75,6 @@ BlockIO InterpreterRenameQuery::execute() } }; - std::map tables_from_locks; - /// Don't allow to drop tables (that we are renaming); don't allow to create tables in places where tables will be renamed. std::map> table_guards; @@ -89,36 +85,26 @@ BlockIO InterpreterRenameQuery::execute() UniqueTableName from(descriptions.back().from_database_name, descriptions.back().from_table_name); UniqueTableName to(descriptions.back().to_database_name, descriptions.back().to_table_name); - if (!tables_from_locks.count(from)) - if (auto table = context.tryGetTable(from.database_name, from.table_name)) - tables_from_locks.emplace(from, table->lockExclusively(context.getCurrentQueryId())); - - descriptions.back().from_table_lock = tables_from_locks[from]; - - if (!table_guards.count(from)) - table_guards.emplace(from, context.getDDLGuard(from.database_name, from.table_name)); - - if (!table_guards.count(to)) - table_guards.emplace(to, context.getDDLGuard(to.database_name, to.table_name)); + table_guards[from]; + table_guards[to]; } - /** All tables are locked. If there are more than one rename in chain, - * we need to hold global lock while doing all renames. Order matters to avoid deadlocks. - * It provides atomicity of all RENAME chain as a whole, from the point of view of DBMS client, - * but only in cases when there was no exceptions during this process and server does not fall. - */ - - decltype(context.getLock()) lock; - - if (descriptions.size() > 1) - lock = context.getLock(); + /// Must do it in consistent order. + for (auto & table_guard : table_guards) + table_guard.second = context.getDDLGuard(table_guard.first.database_name, table_guard.first.table_name); for (auto & elem : descriptions) { context.assertTableDoesntExist(elem.to_database_name, elem.to_table_name); + auto from_table = context.getTable(elem.from_database_name, elem.from_table_name); + auto from_table_lock = from_table->lockExclusively(context.getCurrentQueryId()); context.getDatabase(elem.from_database_name)->renameTable( - context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name, elem.from_table_lock); + context, + elem.from_table_name, + *context.getDatabase(elem.to_database_name), + elem.to_table_name, + from_table_lock); } return {}; From 3568d3d890c6d3e7f657e397b6882d7aad1985ba Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 13:38:20 +0300 Subject: [PATCH 346/357] Updated test --- dbms/tests/queries/0_stateless/01004_rename_deadlock.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh b/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh index b09fabe3d9e..5d5726bb001 100755 --- a/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh +++ b/dbms/tests/queries/0_stateless/01004_rename_deadlock.sh @@ -36,7 +36,7 @@ export -f thread1; export -f thread2; export -f thread3; -TIMEOUT=1000 +TIMEOUT=10 timeout $TIMEOUT bash -c thread1 2> /dev/null & timeout $TIMEOUT bash -c thread2 2> /dev/null & From c0e465f9f00116ce636f5e52932767c8290fcd65 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 14:20:09 +0300 Subject: [PATCH 347/357] Stress test: more beautiful --- dbms/tests/stress | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/stress b/dbms/tests/stress index 90728f7e990..4a6ad411298 100755 --- a/dbms/tests/stress +++ b/dbms/tests/stress @@ -6,7 +6,7 @@ trap 'kill -9 $(jobs -p)' EXIT function thread() { while true; do - ./clickhouse-test --order random 2>&1 | awk '{ printf "'$1' " }' + ./clickhouse-test --order random 2>&1 | awk '/^\w+:/ { printf("\033[0;%s%sm \033[0m", ('$1' % 2 ? "4" : "10"), (int('$1' / 2) % 8)) }' done } From 783df7a5c541d1ceec50cd562cd99ffee0e16726 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 14:32:14 +0300 Subject: [PATCH 348/357] Added a test that prooves that our locking model is non-viable --- .../01005_rwr_shard_deadlock.reference | 0 .../0_stateless/01005_rwr_shard_deadlock.sh | 46 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.reference create mode 100755 dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh diff --git a/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.reference b/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh b/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh new file mode 100755 index 00000000000..1afd3acd324 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt8) ENGINE = MergeTree ORDER BY tuple()"; + +function thread1() +{ + while true; do + $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x Nullable(UInt8)" + $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x UInt8" + done +} + +function thread2() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT x FROM test1 WHERE x IN (SELECT x FROM remote('127.0.0.2', currentDatabase(), test1))" --format Null + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT -q "DROP TABLE test1" From aac0b27daaca3af5bb163d8b6fc2eb07ee6867fc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 31 Aug 2019 15:18:14 +0300 Subject: [PATCH 349/357] Fixed possible deadlock in distributed queries --- dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp | 2 +- dbms/src/Functions/FunctionJoinGet.cpp | 2 +- dbms/src/Interpreters/Context.cpp | 6 ++++++ dbms/src/Interpreters/Context.h | 4 ++++ dbms/src/Interpreters/InterpreterDescribeQuery.cpp | 2 +- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 2 +- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 7 files changed, 15 insertions(+), 5 deletions(-) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 807a9129a75..6c3012a481e 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -26,7 +26,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( * Although now any insertion into the table is done via PushingToViewsBlockOutputStream, * but it's clear that here is not the best place for this functionality. */ - addTableLock(storage->lockStructureForShare(true, context.getCurrentQueryId())); + addTableLock(storage->lockStructureForShare(true, context.getInitialQueryId())); /// If the "root" table deduplactes blocks, there are no need to make deduplication for children /// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks diff --git a/dbms/src/Functions/FunctionJoinGet.cpp b/dbms/src/Functions/FunctionJoinGet.cpp index 5201a0ba5c2..0aad01c62f3 100644 --- a/dbms/src/Functions/FunctionJoinGet.cpp +++ b/dbms/src/Functions/FunctionJoinGet.cpp @@ -65,7 +65,7 @@ FunctionBasePtr FunctionBuilderJoinGet::buildImpl(const ColumnsWithTypeAndName & auto join = storage_join->getJoin(); DataTypes data_types(arguments.size()); - auto table_lock = storage_join->lockStructureForShare(false, context.getCurrentQueryId()); + auto table_lock = storage_join->lockStructureForShare(false, context.getInitialQueryId()); for (size_t i = 0; i < arguments.size(); ++i) data_types[i] = arguments[i].type; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 016f1fa0e49..d7c713abb3d 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1162,6 +1162,12 @@ String Context::getCurrentQueryId() const } +String Context::getInitialQueryId() const +{ + return client_info.initial_query_id; +} + + void Context::setCurrentDatabase(const String & name) { auto lock = getLock(); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index f60f16e5b29..f7ba0a7dbaa 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -264,6 +264,10 @@ public: String getCurrentDatabase() const; String getCurrentQueryId() const; + + /// Id of initiating query for distributed queries; or current query id if it's not a distributed query. + String getInitialQueryId() const; + void setCurrentDatabase(const String & name); void setCurrentQueryId(const String & query_id); diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index faec6f96045..b7f51674a75 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -92,7 +92,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() table = context.getTable(database_name, table_name); } - auto table_lock = table->lockStructureForShare(false, context.getCurrentQueryId()); + auto table_lock = table->lockStructureForShare(false, context.getInitialQueryId()); columns = table->getColumns(); } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index f7edca14089..6fa55b2d8e0 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -100,7 +100,7 @@ BlockIO InterpreterInsertQuery::execute() checkAccess(query); StoragePtr table = getTable(query); - auto table_lock = table->lockStructureForShare(true, context.getCurrentQueryId()); + auto table_lock = table->lockStructureForShare(true, context.getInitialQueryId()); /// We create a pipeline of several streams, into which we will write data. BlockOutputStreamPtr out; diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 79fbcf44323..69613c73705 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -294,7 +294,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( } if (storage) - table_lock = storage->lockStructureForShare(false, context.getCurrentQueryId()); + table_lock = storage->lockStructureForShare(false, context.getInitialQueryId()); syntax_analyzer_result = SyntaxAnalyzer(context, options).analyze( query_ptr, source_header.getNamesAndTypesList(), required_result_column_names, storage, NamesAndTypesList()); From 7382a9f3c3b5502f440cf5be1788c36dfd597e94 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 1 Sep 2019 00:15:40 +0300 Subject: [PATCH 350/357] Rename method for settings check --- dbms/src/Storages/AlterCommands.cpp | 2 +- dbms/src/Storages/IStorage.cpp | 4 ++-- dbms/src/Storages/IStorage.h | 2 +- dbms/src/Storages/Kafka/StorageKafka.cpp | 2 +- dbms/src/Storages/Kafka/StorageKafka.h | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 2a311461f00..bcfd852a628 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -544,7 +544,7 @@ void AlterCommands::validate(const IStorage & table, const Context & context) } else if (command.type == AlterCommand::MODIFY_SETTING) for (const auto & change : command.settings_changes) - table.checkSetting(change.name); + table.checkSettingCanBeChanged(change.name); } /** Existing defaulted columns may require default expression extensions with a type conversion, diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 2fb737de71b..cbd14666006 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -309,7 +309,7 @@ bool IStorage::isVirtualColumn(const String & column_name) const return getColumns().get(column_name).is_virtual; } -void IStorage::checkSetting(const String & /* setting_name */) const +void IStorage::checkSettingCanBeChanged(const String & /* setting_name */) const { if (!supportsSettings()) throw Exception("Storage '" + getName() + "' doesn't support settings.", ErrorCodes::SETTINGS_ARE_NOT_SUPPORTED); @@ -380,7 +380,7 @@ IDatabase::ASTModifier IStorage::getSettingsModifier(const SettingsChanges & new /// Make storage settings unique for (const auto & change : new_changes) { - checkSetting(change.name); + checkSettingCanBeChanged(change.name); auto finder = [&change] (const SettingChange & c) { return c.name == change.name; }; if (auto it = std::find_if(storage_changes.begin(), storage_changes.end(), finder); it != storage_changes.end()) diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index f0dcef1f629..d92b06029d8 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -140,7 +140,7 @@ public: /// thread-unsafe part. lockStructure must be acquired void check(const Block & block, bool need_all = false) const; /// Check storage has setting and setting can be modified. - virtual void checkSetting(const String & setting_name) const; + virtual void checkSettingCanBeChanged(const String & setting_name) const; protected: /// still thread-unsafe part. void setIndices(IndicesDescription indices_); diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index fc43e7046a7..10b4381dd2d 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -412,7 +412,7 @@ bool StorageKafka::streamToViews() } -void StorageKafka::checkSetting(const String & setting_name) const +void StorageKafka::checkSettingCanBeChanged(const String & setting_name) const { if (KafkaSettings::findIndex(setting_name) == KafkaSettings::npos) throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting_name + "'", ErrorCodes::UNKNOWN_SETTING}; diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index 76983ac4b9e..e8799983705 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -57,7 +57,7 @@ public: const auto & getSchemaName() const { return schema_name; } const auto & skipBroken() const { return skip_broken; } - void checkSetting(const String & setting_name) const override; + void checkSettingCanBeChanged(const String & setting_name) const override; protected: StorageKafka( diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index a95c526bf20..e5550ce91b4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1325,7 +1325,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast, /* only_check = */ true); for (const auto & setting : new_changes) - checkSetting(setting.name); + checkSettingCanBeChanged(setting.name); /// Check that type conversions are possible. ExpressionActionsPtr unused_expression; @@ -1657,7 +1657,7 @@ void MergeTreeData::changeSettings( } } -void MergeTreeData::checkSetting(const String & setting_name) const +void MergeTreeData::checkSettingCanBeChanged(const String & setting_name) const { if (MergeTreeSettings::findIndex(setting_name) == MergeTreeSettings::npos) throw Exception{"Storage '" + getName() + "' doesn't have setting '" + setting_name + "'", ErrorCodes::UNKNOWN_SETTING}; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 69c5449a751..ffca5de1a16 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -543,7 +543,7 @@ public: TableStructureWriteLockHolder & table_lock_holder); /// All MergeTreeData children have settings. - void checkSetting(const String & setting_name) const override; + void checkSettingCanBeChanged(const String & setting_name) const override; /// Remove columns, that have been markedd as empty after zeroing values with expired ttl void removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part); From dcc6163d326965811f759b42032d9735afd3f998 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 1 Sep 2019 00:39:17 +0300 Subject: [PATCH 351/357] Added function "trap" --- .../registerFunctionsIntrospection.cpp | 2 + dbms/src/Functions/trap.cpp | 143 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 dbms/src/Functions/trap.cpp diff --git a/dbms/src/Functions/registerFunctionsIntrospection.cpp b/dbms/src/Functions/registerFunctionsIntrospection.cpp index 448400b37ab..700a568d822 100644 --- a/dbms/src/Functions/registerFunctionsIntrospection.cpp +++ b/dbms/src/Functions/registerFunctionsIntrospection.cpp @@ -6,12 +6,14 @@ class FunctionFactory; void registerFunctionAddressToSymbol(FunctionFactory & factory); void registerFunctionDemangle(FunctionFactory & factory); void registerFunctionAddressToLine(FunctionFactory & factory); +void registerFunctionTrap(FunctionFactory & factory); void registerFunctionsIntrospection(FunctionFactory & factory) { registerFunctionAddressToSymbol(factory); registerFunctionDemangle(factory); registerFunctionAddressToLine(factory); + registerFunctionTrap(factory); } } diff --git a/dbms/src/Functions/trap.cpp b/dbms/src/Functions/trap.cpp new file mode 100644 index 00000000000..e05d5efa4f7 --- /dev/null +++ b/dbms/src/Functions/trap.cpp @@ -0,0 +1,143 @@ +#if 0 + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + + +/// Various illegal actions to test diagnostic features of ClickHouse itself. Should not be enabled in production builds. +class FunctionTrap : public IFunction +{ +public: + static constexpr auto name = "trap"; + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!isString(arguments[0])) + throw Exception("The only argument for function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + if (const ColumnConst * column = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get())) + { + String mode = column->getValue(); + + if (mode == "read nullptr c++") + { + volatile int x = *reinterpret_cast(0); + (void)x; + } + else if (mode == "read nullptr asm") + { + __asm__ volatile ("movq $0, %rax"); + __asm__ volatile ("movq (%rax), %rax"); + } + else if (mode == "illegal instruction") + { + __asm__ volatile ("ud2a"); + } + else if (mode == "abort") + { + abort(); + } + else if (mode == "use after free") + { + int * x_ptr; + { + auto x = std::make_unique(); + x_ptr = x.get(); + } + *x_ptr = 1; + (void)x_ptr; + } + else if (mode == "use after scope") + { + volatile int * x_ptr; + [&]{ + volatile int x = 0; + x_ptr = &x; + (void)x; + }(); + [&]{ + volatile int y = 1; + *x_ptr = 2; + (void)y; + }(); + (void)x_ptr; + } + else if (mode == "uninitialized memory") + { + int x; + (void)write(2, &x, sizeof(x)); + } + else if (mode == "data race") + { + int x = 0; + std::thread t1([&]{ ++x; }); + std::thread t2([&]{ ++x; }); + t1.join(); + t2.join(); + } + else + throw Exception("Unknown trap mode", ErrorCodes::BAD_ARGUMENTS); + } + else + throw Exception("The only argument for function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_COLUMN); + + block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(input_rows_count, 0ULL); + } +}; + + +void registerFunctionTrap(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#else + +namespace DB +{ + class FunctionFactory; + void registerFunctionTrap(FunctionFactory &) {} +} + +#endif From cb79e2371ed1899b7ee0fa3c9647dddada218993 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 1 Sep 2019 00:47:15 +0300 Subject: [PATCH 352/357] Add "os_thread_ids" column to system tables --- dbms/src/Common/ThreadStatus.h | 1 + dbms/src/Interpreters/ProcessList.cpp | 1 + dbms/src/Interpreters/ProcessList.h | 1 + dbms/src/Interpreters/QueryLog.cpp | 9 +++++++++ dbms/src/Interpreters/QueryLog.h | 1 + dbms/src/Interpreters/ThreadStatusExt.cpp | 2 ++ dbms/src/Interpreters/executeQuery.cpp | 2 ++ dbms/src/Storages/System/StorageSystemProcesses.cpp | 9 +++++++++ 8 files changed, 26 insertions(+) diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index f175db74771..2ba55fa07d0 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -61,6 +61,7 @@ public: InternalTextLogsQueueWeakPtr logs_queue_ptr; std::vector thread_numbers; + std::vector os_thread_ids; /// The first thread created this thread group UInt32 master_thread_number = 0; diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp index 71376c6d129..100ecc00dc1 100644 --- a/dbms/src/Interpreters/ProcessList.cpp +++ b/dbms/src/Interpreters/ProcessList.cpp @@ -444,6 +444,7 @@ QueryStatusInfo QueryStatus::getInfo(bool get_thread_list, bool get_profile_even { std::lock_guard lock(thread_group->mutex); res.thread_numbers = thread_group->thread_numbers; + res.os_thread_ids = thread_group->os_thread_ids; } if (get_profile_events) diff --git a/dbms/src/Interpreters/ProcessList.h b/dbms/src/Interpreters/ProcessList.h index 4cdf7c18fea..d5631abdb0c 100644 --- a/dbms/src/Interpreters/ProcessList.h +++ b/dbms/src/Interpreters/ProcessList.h @@ -66,6 +66,7 @@ struct QueryStatusInfo /// Optional fields, filled by request std::vector thread_numbers; + std::vector os_thread_ids; std::shared_ptr profile_counters; std::shared_ptr query_settings; }; diff --git a/dbms/src/Interpreters/QueryLog.cpp b/dbms/src/Interpreters/QueryLog.cpp index 52e552d833f..7cca320b04b 100644 --- a/dbms/src/Interpreters/QueryLog.cpp +++ b/dbms/src/Interpreters/QueryLog.cpp @@ -78,6 +78,7 @@ Block QueryLogElement::createBlock() {std::make_shared(), "revision"}, {std::make_shared(std::make_shared()), "thread_numbers"}, + {std::make_shared(std::make_shared()), "os_thread_ids"}, {std::make_shared(std::make_shared()), "ProfileEvents.Names"}, {std::make_shared(std::make_shared()), "ProfileEvents.Values"}, {std::make_shared(std::make_shared()), "Settings.Names"}, @@ -123,6 +124,14 @@ void QueryLogElement::appendToBlock(Block & block) const columns[i++]->insert(threads_array); } + { + Array threads_array; + threads_array.reserve(os_thread_ids.size()); + for (const UInt32 thread_number : os_thread_ids) + threads_array.emplace_back(UInt64(thread_number)); + columns[i++]->insert(threads_array); + } + if (profile_counters) { auto column_names = columns[i++].get(); diff --git a/dbms/src/Interpreters/QueryLog.h b/dbms/src/Interpreters/QueryLog.h index 95d563fd21e..2b8f8120050 100644 --- a/dbms/src/Interpreters/QueryLog.h +++ b/dbms/src/Interpreters/QueryLog.h @@ -60,6 +60,7 @@ struct QueryLogElement ClientInfo client_info; std::vector thread_numbers; + std::vector os_thread_ids; std::shared_ptr profile_counters; std::shared_ptr query_settings; diff --git a/dbms/src/Interpreters/ThreadStatusExt.cpp b/dbms/src/Interpreters/ThreadStatusExt.cpp index 28740417b71..1407d0d2073 100644 --- a/dbms/src/Interpreters/ThreadStatusExt.cpp +++ b/dbms/src/Interpreters/ThreadStatusExt.cpp @@ -61,6 +61,7 @@ void ThreadStatus::initializeQuery() thread_group->memory_tracker.setDescription("(for query)"); thread_group->thread_numbers.emplace_back(thread_number); + thread_group->os_thread_ids.emplace_back(os_thread_id); thread_group->master_thread_number = thread_number; thread_group->master_thread_os_id = os_thread_id; @@ -99,6 +100,7 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool /// NOTE: A thread may be attached multiple times if it is reused from a thread pool. thread_group->thread_numbers.emplace_back(thread_number); + thread_group->os_thread_ids.emplace_back(os_thread_id); } if (query_context) diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 36bdcc27634..07445aac646 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -400,6 +400,7 @@ static std::tuple executeQueryImpl( } elem.thread_numbers = std::move(info.thread_numbers); + elem.os_thread_ids = std::move(info.os_thread_ids); elem.profile_counters = std::move(info.profile_counters); if (log_queries) @@ -437,6 +438,7 @@ static std::tuple executeQueryImpl( elem.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0; elem.thread_numbers = std::move(info.thread_numbers); + elem.os_thread_ids = std::move(info.os_thread_ids); elem.profile_counters = std::move(info.profile_counters); } diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp index 2450ec9296e..56905b29349 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.cpp +++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp @@ -58,6 +58,7 @@ NamesAndTypesList StorageSystemProcesses::getNamesAndTypes() {"query", std::make_shared()}, {"thread_numbers", std::make_shared(std::make_shared())}, + {"os_thread_ids", std::make_shared(std::make_shared())}, {"ProfileEvents.Names", std::make_shared(std::make_shared())}, {"ProfileEvents.Values", std::make_shared(std::make_shared())}, {"Settings.Names", std::make_shared(std::make_shared())}, @@ -120,6 +121,14 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, const Contex res_columns[i++]->insert(threads_array); } + { + Array threads_array; + threads_array.reserve(process.os_thread_ids.size()); + for (const UInt32 thread_number : process.os_thread_ids) + threads_array.emplace_back(thread_number); + res_columns[i++]->insert(threads_array); + } + { IColumn * column_profile_events_names = res_columns[i++].get(); IColumn * column_profile_events_values = res_columns[i++].get(); From 4d21f97d6aeefeafa92b76370e06d7a3ff48f9db Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Sun, 1 Sep 2019 16:03:38 +0300 Subject: [PATCH 353/357] Increase test timeout --- dbms/tests/integration/test_storage_kafka/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/integration/test_storage_kafka/test.py b/dbms/tests/integration/test_storage_kafka/test.py index 7d17c1cf362..24c5a3ac3bf 100644 --- a/dbms/tests/integration/test_storage_kafka/test.py +++ b/dbms/tests/integration/test_storage_kafka/test.py @@ -605,7 +605,7 @@ def test_kafka_produce_consume(kafka_cluster): assert int(result) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) -@pytest.mark.timeout(180) +@pytest.mark.timeout(300) def test_kafka_commit_on_block_write(kafka_cluster): instance.query(''' DROP TABLE IF EXISTS test.view; From 55f11b367575bb9f174649afbf0180c4172a0883 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 1 Sep 2019 18:25:46 +0300 Subject: [PATCH 354/357] Speed up test for "text_log" in cases when text_log is large (such as on production servers) --- .../tests/queries/0_stateless/00974_text_log_table_not_empty.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh index 149f0668bd1..c3cde4c08bb 100755 --- a/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh +++ b/dbms/tests/queries/0_stateless/00974_text_log_table_not_empty.sh @@ -10,7 +10,7 @@ do ${CLICKHOUSE_CLIENT} --query="SYSTEM FLUSH LOGS" sleep 0.1; -if [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0") == 1 ]]; then echo 1; exit; fi; +if [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() > 0 FROM system.text_log WHERE position(system.text_log.message, 'SELECT 6103') > 0 AND event_date >= yesterday()") == 1 ]]; then echo 1; exit; fi; done; From 8917572087db020a2c9c4ec180fb34c28b4714de Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 1 Sep 2019 18:28:43 +0300 Subject: [PATCH 355/357] Speed up another test in cases when query_log is large (such as on production servers) --- .../00933_test_fix_extra_seek_on_compressed_cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/dbms/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index b0fd9a70bd4..1f7571a2404 100755 --- a/dbms/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/dbms/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --query_id="test-query-uncompresse sleep 1 $CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" -$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) ORDER BY event_time DESC LIMIT 1" +$CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) AND event_date >= yesterday() ORDER BY event_time DESC LIMIT 1" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" From 4ebf610808c2bbc1bf15ffd56a4eff914163e929 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 1 Sep 2019 19:21:54 +0300 Subject: [PATCH 356/357] Disable query profiler with sanitizers --- dbms/programs/server/Server.cpp | 13 ++++++++++++- dbms/src/Core/Defines.h | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 5f5e464eb01..f10dc07ab56 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -520,7 +520,18 @@ int Server::main(const std::vector & /*args*/) /// Init trace collector only after trace_log system table was created /// Disable it if we collect test coverage information, because it will work extremely slow. -#if USE_UNWIND && !WITH_COVERAGE + /// + /// It also cannot work with sanitizers. + /// Sanitizers are using quick "frame walking" stack unwinding (this implies -fno-omit-frame-pointer) + /// And they do unwiding frequently (on every malloc/free, thread/mutex operations, etc). + /// They change %rbp during unwinding and it confuses libunwind if signal comes during sanitizer unwiding + /// and query profiler decide to unwind stack with libunwind at this moment. + /// + /// Symptoms: you'll get silent Segmentation Fault - without sanitizer message and without usual ClickHouse diagnostics. + /// + /// Look at compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h + /// +#if USE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER) /// QueryProfiler cannot work reliably with any other libunwind or without PHDR cache. if (hasPHDRCache()) global_context->initializeTraceCollector(); diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index a172cf6e243..33c65081e40 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -124,6 +124,8 @@ #endif #endif +/// TODO Strange enough, there is no way to detect UB sanitizer. + /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute. /// It is useful in case when compiler cannot see (and exploit) it, but UBSan can. /// Example: multiplication of signed integers with possibility of overflow when both sides are from user input. From 7d1b1de89585c4185010993245b93707775f5b75 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 2 Sep 2019 00:49:11 +0300 Subject: [PATCH 357/357] Temporarily remove all LIVE VIEW tests --- .../00960_live_view_watch_events_live.py | 47 ----------- ...0960_live_view_watch_events_live.reference | 0 .../00961_temporary_live_view_watch.reference | 3 - .../00961_temporary_live_view_watch.sql | 20 ----- .../00962_temporary_live_view_watch_live.py | 47 ----------- ...2_temporary_live_view_watch_live.reference | 0 ...y_live_view_watch_live_timeout.py.disabled | 54 ------------ ...ary_live_view_watch_live_timeout.reference | 0 .../00964_live_view_watch_events_heartbeat.py | 49 ----------- ...live_view_watch_events_heartbeat.reference | 0 .../00965_live_view_watch_heartbeat.py | 50 ----------- .../00965_live_view_watch_heartbeat.reference | 0 .../00966_live_view_watch_events_http.py | 40 --------- ...0966_live_view_watch_events_http.reference | 0 .../0_stateless/00967_live_view_watch_http.py | 40 --------- .../00967_live_view_watch_http.reference | 0 ...t_format_jsoneachrowwithprogress.reference | 4 - ..._select_format_jsoneachrowwithprogress.sql | 14 ---- ...h_format_jsoneachrowwithprogress.reference | 6 -- ...w_watch_format_jsoneachrowwithprogress.sql | 20 ----- ...0_live_view_watch_events_http_heartbeat.py | 45 ---------- ...view_watch_events_http_heartbeat.reference | 0 .../00971_live_view_watch_http_heartbeat.py | 46 ---------- ...1_live_view_watch_http_heartbeat.reference | 0 .../00972_live_view_select_1.reference | 1 - .../0_stateless/00972_live_view_select_1.sql | 9 -- .../00973_live_view_select.reference | 4 - .../0_stateless/00973_live_view_select.sql | 20 ----- ...ive_view_select_with_aggregation.reference | 2 - ...0974_live_view_select_with_aggregation.sql | 18 ---- .../00975_live_view_create.reference | 0 .../0_stateless/00975_live_view_create.sql | 9 -- .../00976_live_view_select_version.reference | 3 - .../00976_live_view_select_version.sql | 14 ---- .../00977_live_view_watch_events.reference | 3 - .../00977_live_view_watch_events.sql | 20 ----- .../00978_live_view_watch.reference | 3 - .../0_stateless/00978_live_view_watch.sql | 20 ----- .../0_stateless/00979_live_view_watch_live.py | 53 ------------ .../00979_live_view_watch_live.reference | 0 ...00980_create_temporary_live_view.reference | 3 - .../00980_create_temporary_live_view.sql | 17 ---- .../00991_live_view_watch_event_live.python | 81 ------------------ ...00991_live_view_watch_event_live.reference | 7 -- .../00991_live_view_watch_http.python | 63 -------------- .../00991_live_view_watch_http.reference | 4 - ...ry_live_view_watch_events_heartbeat.python | 83 ------------------- ...live_view_watch_events_heartbeat.reference | 0 ...0991_temporary_live_view_watch_live.python | 81 ------------------ ...1_temporary_live_view_watch_live.reference | 7 -- 50 files changed, 1010 deletions(-) delete mode 100755 dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py delete mode 100644 dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference delete mode 100644 dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference delete mode 100644 dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql delete mode 100755 dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py delete mode 100644 dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference delete mode 100755 dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled delete mode 100644 dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference delete mode 100755 dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py delete mode 100644 dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference delete mode 100755 dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py delete mode 100644 dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference delete mode 100755 dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py delete mode 100644 dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference delete mode 100755 dbms/tests/queries/0_stateless/00967_live_view_watch_http.py delete mode 100644 dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference delete mode 100644 dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference delete mode 100644 dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql delete mode 100644 dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference delete mode 100644 dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql delete mode 100755 dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py delete mode 100644 dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference delete mode 100755 dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py delete mode 100644 dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference delete mode 100644 dbms/tests/queries/0_stateless/00972_live_view_select_1.reference delete mode 100644 dbms/tests/queries/0_stateless/00972_live_view_select_1.sql delete mode 100644 dbms/tests/queries/0_stateless/00973_live_view_select.reference delete mode 100644 dbms/tests/queries/0_stateless/00973_live_view_select.sql delete mode 100644 dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference delete mode 100644 dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql delete mode 100644 dbms/tests/queries/0_stateless/00975_live_view_create.reference delete mode 100644 dbms/tests/queries/0_stateless/00975_live_view_create.sql delete mode 100644 dbms/tests/queries/0_stateless/00976_live_view_select_version.reference delete mode 100644 dbms/tests/queries/0_stateless/00976_live_view_select_version.sql delete mode 100644 dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference delete mode 100644 dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql delete mode 100644 dbms/tests/queries/0_stateless/00978_live_view_watch.reference delete mode 100644 dbms/tests/queries/0_stateless/00978_live_view_watch.sql delete mode 100755 dbms/tests/queries/0_stateless/00979_live_view_watch_live.py delete mode 100644 dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference delete mode 100644 dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference delete mode 100644 dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql delete mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python delete mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference delete mode 100755 dbms/tests/queries/0_stateless/00991_live_view_watch_http.python delete mode 100644 dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference delete mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python delete mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference delete mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python delete mode 100644 dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference diff --git a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py deleted file mode 100755 index 2095683720e..00000000000 --- a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv EVENTS') - client1.expect('1.*' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect('2.*' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect('3.*' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference b/dbms/tests/queries/0_stateless/00960_live_view_watch_events_live.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference deleted file mode 100644 index 6fbbedf1b21..00000000000 --- a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.reference +++ /dev/null @@ -1,3 +0,0 @@ -0 1 -6 2 -21 3 diff --git a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql b/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql deleted file mode 100644 index 7992da92f97..00000000000 --- a/dbms/tests/queries/0_stateless/00961_temporary_live_view_watch.sql +++ /dev/null @@ -1,20 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py deleted file mode 100755 index 3dbec01b29a..00000000000 --- a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00962_temporary_live_view_watch_live.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled deleted file mode 100755 index b324c1b90cc..00000000000 --- a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.py.disabled +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET temporary_live_view_timeout=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client2.expect(prompt) - client1.expect(r'6.*2' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client2.expect(prompt) - client1.expect(r'21.*3' + end_of_block) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('SELECT sleep(1)') - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect('Table test.lv doesn\'t exist') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference b/dbms/tests/queries/0_stateless/00963_temporary_live_view_watch_live_timeout.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py deleted file mode 100755 index 528f18839bb..00000000000 --- a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET live_view_heartbeat_interval=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv EVENTS') - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect('2.*' + end_of_block) - client1.expect('Progress: 2.00 rows.*\)') - # wait for heartbeat - client1.expect('Progress: 2.00 rows.*\)') - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference b/dbms/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py deleted file mode 100755 index 2723936f876..00000000000 --- a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('SET live_view_heartbeat_interval=1') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client1.expect('Progress: 2.00 rows.*\)') - # wait for heartbeat - client1.expect('Progress: 2.00 rows.*\)') - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference b/dbms/tests/queries/0_stateless/00965_live_view_watch_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py deleted file mode 100755 index 72ab3ea8818..00000000000 --- a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - - with http_client({'method':'GET', 'url': '/?allow_experimental_live_view=1&query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: - client2.expect('.*1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - client2.expect('.*2\n') - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference b/dbms/tests/queries/0_stateless/00966_live_view_watch_events_http.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py deleted file mode 100755 index e2f33971c3d..00000000000 --- a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - - with http_client({'method':'GET', 'url':'/?allow_experimental_live_view=1&query=WATCH%20test.lv'}, name='client2>', log=log) as client2: - client2.expect('.*0\t1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - client2.expect('.*6\t2\n') - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference b/dbms/tests/queries/0_stateless/00967_live_view_watch_http.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference deleted file mode 100644 index 5ae423d90d1..00000000000 --- a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.reference +++ /dev/null @@ -1,4 +0,0 @@ -{"row":{"a":1}} -{"row":{"a":2}} -{"row":{"a":3}} -{"progress":{"read_rows":"3","read_bytes":"36","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} diff --git a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql deleted file mode 100644 index 1023cdf6b29..00000000000 --- a/dbms/tests/queries/0_stateless/00968_live_view_select_format_jsoneachrowwithprogress.sql +++ /dev/null @@ -1,14 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT * FROM test.lv FORMAT JSONEachRowWithProgress; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference deleted file mode 100644 index 287a1ced92d..00000000000 --- a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.reference +++ /dev/null @@ -1,6 +0,0 @@ -{"row":{"sum(a)":"0","_version":"1"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} -{"row":{"sum(a)":"6","_version":"2"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} -{"row":{"sum(a)":"21","_version":"3"}} -{"progress":{"read_rows":"1","read_bytes":"16","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} diff --git a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql b/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql deleted file mode 100644 index 3e46d55c014..00000000000 --- a/dbms/tests/queries/0_stateless/00969_live_view_watch_format_jsoneachrowwithprogress.sql +++ /dev/null @@ -1,20 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0 FORMAT JSONEachRowWithProgress; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py deleted file mode 100755 index 8435cdc147a..00000000000 --- a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - with http_client({'method':'GET', 'url': '/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}\n', escape=True) - client2.expect('{"row":{"version":"1"}', escape=True) - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) - # heartbeat is provided by progress message - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) - - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - - client2.expect('{"row":{"version":"2"}}\n', escape=True) - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference b/dbms/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py deleted file mode 100755 index 2317d705efe..00000000000 --- a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block -from httpclient import client as http_client - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1: - client1.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - - with http_client({'method':'GET', 'url':'/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: - client2.expect('"progress".*',) - client2.expect('{"row":{"sum(a)":"0","_version":"1"}}\n', escape=True) - client2.expect('"progress".*\n') - # heartbeat is provided by progress message - client2.expect('"progress".*\n') - - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(prompt) - - client2.expect('"progress".*"read_rows":"2".*\n') - client2.expect('{"row":{"sum(a)":"6","_version":"2"}}\n', escape=True) - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference b/dbms/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference b/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/dbms/tests/queries/0_stateless/00972_live_view_select_1.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql b/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql deleted file mode 100644 index 135516b0cd3..00000000000 --- a/dbms/tests/queries/0_stateless/00972_live_view_select_1.sql +++ /dev/null @@ -1,9 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; - -CREATE LIVE VIEW test.lv AS SELECT 1; - -SELECT * FROM test.lv; - -DROP TABLE test.lv; diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.reference b/dbms/tests/queries/0_stateless/00973_live_view_select.reference deleted file mode 100644 index 75236c0daf7..00000000000 --- a/dbms/tests/queries/0_stateless/00973_live_view_select.reference +++ /dev/null @@ -1,4 +0,0 @@ -6 1 -6 1 -12 2 -12 2 diff --git a/dbms/tests/queries/0_stateless/00973_live_view_select.sql b/dbms/tests/queries/0_stateless/00973_live_view_select.sql deleted file mode 100644 index 4b5ca0a2dd7..00000000000 --- a/dbms/tests/queries/0_stateless/00973_live_view_select.sql +++ /dev/null @@ -1,20 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; -SELECT *,_version FROM test.lv; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; -SELECT *,_version FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference deleted file mode 100644 index 6d50f0e9c3a..00000000000 --- a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.reference +++ /dev/null @@ -1,2 +0,0 @@ -6 -21 diff --git a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql b/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql deleted file mode 100644 index 3faaec8f623..00000000000 --- a/dbms/tests/queries/0_stateless/00974_live_view_select_with_aggregation.sql +++ /dev/null @@ -1,18 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT sum(a) FROM test.lv; - -INSERT INTO test.mt VALUES (4),(5),(6); - -SELECT sum(a) FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.reference b/dbms/tests/queries/0_stateless/00975_live_view_create.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00975_live_view_create.sql b/dbms/tests/queries/0_stateless/00975_live_view_create.sql deleted file mode 100644 index 02c1644d193..00000000000 --- a/dbms/tests/queries/0_stateless/00975_live_view_create.sql +++ /dev/null @@ -1,9 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference b/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference deleted file mode 100644 index 453bd800469..00000000000 --- a/dbms/tests/queries/0_stateless/00976_live_view_select_version.reference +++ /dev/null @@ -1,3 +0,0 @@ -1 1 -2 1 -3 1 diff --git a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql b/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql deleted file mode 100644 index ae1c59a92d7..00000000000 --- a/dbms/tests/queries/0_stateless/00976_live_view_select_version.sql +++ /dev/null @@ -1,14 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT * FROM test.mt; - -INSERT INTO test.mt VALUES (1),(2),(3); - -SELECT *,_version FROM test.lv; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference deleted file mode 100644 index 01e79c32a8c..00000000000 --- a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.reference +++ /dev/null @@ -1,3 +0,0 @@ -1 -2 -3 diff --git a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql b/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql deleted file mode 100644 index 3e0d066fb8d..00000000000 --- a/dbms/tests/queries/0_stateless/00977_live_view_watch_events.sql +++ /dev/null @@ -1,20 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv EVENTS LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv EVENTS LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv EVENTS LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.reference b/dbms/tests/queries/0_stateless/00978_live_view_watch.reference deleted file mode 100644 index 6fbbedf1b21..00000000000 --- a/dbms/tests/queries/0_stateless/00978_live_view_watch.reference +++ /dev/null @@ -1,3 +0,0 @@ -0 1 -6 2 -21 3 diff --git a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql b/dbms/tests/queries/0_stateless/00978_live_view_watch.sql deleted file mode 100644 index b8d0d93ccab..00000000000 --- a/dbms/tests/queries/0_stateless/00978_live_view_watch.sql +++ /dev/null @@ -1,20 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (1),(2),(3); - -WATCH test.lv LIMIT 0; - -INSERT INTO test.mt VALUES (4),(5),(6); - -WATCH test.lv LIMIT 0; - -DROP TABLE test.lv; -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py deleted file mode 100755 index 8c5bc5b8eb2..00000000000 --- a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import signal - -CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) - -from client import client, prompt, end_of_block - -log = None -# uncomment the line below for debugging -#log=sys.stdout - -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send('SET allow_experimental_live_view = 1') - client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') - client2.expect(prompt) - - client1.send('DROP TABLE IF EXISTS test.lv') - client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') - client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') - client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') - client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client2.expect(prompt) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) - client2.expect(prompt) - for i in range(1,129): - client2.send('INSERT INTO test.mt VALUES (1)') - client1.expect(r'%d.*%d' % (21+i, 3+i) + end_of_block) - client2.expect(prompt) - # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) - if match.groups()[1]: - client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00979_live_view_watch_live.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference deleted file mode 100644 index 7f9fcbb2e9c..00000000000 --- a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.reference +++ /dev/null @@ -1,3 +0,0 @@ -temporary_live_view_timeout 5 -live_view_heartbeat_interval 15 -0 diff --git a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql b/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql deleted file mode 100644 index 037c2a9e587..00000000000 --- a/dbms/tests/queries/0_stateless/00980_create_temporary_live_view.sql +++ /dev/null @@ -1,17 +0,0 @@ -SET allow_experimental_live_view = 1; - -DROP TABLE IF EXISTS test.lv; -DROP TABLE IF EXISTS test.mt; - -SELECT name, value from system.settings WHERE name = 'temporary_live_view_timeout'; -SELECT name, value from system.settings WHERE name = 'live_view_heartbeat_interval'; - -SET temporary_live_view_timeout=1; -CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple(); -CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt; - -SHOW TABLES LIKE 'lv'; -SELECT sleep(2); -SHOW TABLES LIKE 'lv'; - -DROP TABLE test.mt; diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python deleted file mode 100644 index 782671cdfaf..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.python +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - line = q.get() - print(line) - assert (line == '21\t3') - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - print(line) - assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference b/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference deleted file mode 100644 index 1e94cdade41..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_event_live.reference +++ /dev/null @@ -1,7 +0,0 @@ -0 1 -0 1 -6 2 -6 2 -21 3 -21 3 -None diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python deleted file mode 100755 index 938547ca0cb..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.python +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_http_query(query): - cmd = list(CLICKHOUSE_CURL.split()) # list(['curl', '-sSN', '--max-time', '10']) - cmd += ['-sSN', CLICKHOUSE_URL, '-d', query] - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def read_lines_and_push_to_queue(pipe, queue): - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - pipe = send_http_query('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(pipe, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference b/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference deleted file mode 100644 index 489457d751b..00000000000 --- a/dbms/tests/queries/0_stateless/00991_live_view_watch_http.reference +++ /dev/null @@ -1,4 +0,0 @@ -0 1 -0 1 -6 2 -6 2 diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python deleted file mode 100644 index 70063adc6e3..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.python +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query, '--live_view_heartbeat_interval=1', '--progress'] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - # print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - # print(line) - assert (line.endswith('0\t1')) - assert ('Progress: 0.00 rows' in line) - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - assert (line.endswith('6\t2')) - assert ('Progress: 1.00 rows' in line) - - # send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - # line = q.get() - # print(line) - # assert (line.endswith('6\t2')) - # assert ('Progress: 1.00 rows' in line) - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - # print(line) - # assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_events_heartbeat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python deleted file mode 100644 index d290018a02c..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.python +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import threading -import Queue as queue -import os -import sys -import signal - - -CLICKHOUSE_CLIENT = os.environ.get('CLICKHOUSE_CLIENT') -CLICKHOUSE_CURL = os.environ.get('CLICKHOUSE_CURL') -CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') - - -def send_query(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout - - -def send_query_in_process_group(query): - cmd = list(CLICKHOUSE_CLIENT.split()) - cmd += ['--query', query] - # print(cmd) - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=os.setsid) - - -def read_lines_and_push_to_queue(pipe, queue): - try: - for line in iter(pipe.readline, ''): - line = line.strip() - print(line) - sys.stdout.flush() - queue.put(line) - except KeyboardInterrupt: - pass - - queue.put(None) - - -def test(): - send_query('DROP TABLE IF EXISTS test.lv').read() - send_query('DROP TABLE IF EXISTS test.mt').read() - send_query('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()').read() - send_query('CREATE TEMPORARY LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt').read() - - q = queue.Queue() - p = send_query_in_process_group('WATCH test.lv') - thread = threading.Thread(target=read_lines_and_push_to_queue, args=(p.stdout, q)) - thread.start() - - line = q.get() - print(line) - assert (line == '0\t1') - - send_query('INSERT INTO test.mt VALUES (1),(2),(3)').read() - line = q.get() - print(line) - assert (line == '6\t2') - - send_query('INSERT INTO test.mt VALUES (4),(5),(6)').read() - line = q.get() - print(line) - assert (line == '21\t3') - - # Send Ctrl+C to client. - os.killpg(os.getpgid(p.pid), signal.SIGINT) - # This insert shouldn't affect lv. - send_query('INSERT INTO test.mt VALUES (7),(8),(9)').read() - line = q.get() - print(line) - assert (line is None) - - send_query('DROP TABLE if exists test.lv').read() - send_query('DROP TABLE if exists test.lv').read() - - thread.join() - -test() diff --git a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference b/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference deleted file mode 100644 index 1e94cdade41..00000000000 --- a/dbms/tests/queries/0_stateless/00991_temporary_live_view_watch_live.reference +++ /dev/null @@ -1,7 +0,0 @@ -0 1 -0 1 -6 2 -6 2 -21 3 -21 3 -None