From bb40d4729c234b9a3b7eb62bcf060a320a5e84ec Mon Sep 17 00:00:00 2001 From: Maxim Fridental Date: Fri, 12 Jul 2019 13:17:38 +0200 Subject: [PATCH 01/17] Implement COLUMNS clause --- dbms/src/Interpreters/AsteriskSemantic.h | 3 ++ .../PredicateExpressionsOptimizer.cpp | 19 +++++-- .../TranslateQualifiedNamesVisitor.cpp | 20 +++++++- dbms/src/Parsers/ASTColumnsClause.cpp | 22 ++++++++ dbms/src/Parsers/ASTColumnsClause.h | 51 +++++++++++++++++++ dbms/src/Parsers/ExpressionElementParsers.cpp | 21 ++++++++ dbms/src/Parsers/ExpressionElementParsers.h | 10 +++- .../00969_columns_clause.reference | 2 + .../0_stateless/00969_columns_clause.sql | 3 ++ 9 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 dbms/src/Parsers/ASTColumnsClause.cpp create mode 100644 dbms/src/Parsers/ASTColumnsClause.h create mode 100644 dbms/tests/queries/0_stateless/00969_columns_clause.reference create mode 100644 dbms/tests/queries/0_stateless/00969_columns_clause.sql diff --git a/dbms/src/Interpreters/AsteriskSemantic.h b/dbms/src/Interpreters/AsteriskSemantic.h index c0928a42691..1dcfd344a72 100644 --- a/dbms/src/Interpreters/AsteriskSemantic.h +++ b/dbms/src/Interpreters/AsteriskSemantic.h @@ -4,6 +4,7 @@ #include #include +#include "../Parsers/ASTColumnsClause.h" namespace DB { @@ -24,9 +25,11 @@ struct AsteriskSemantic static void setAliases(ASTAsterisk & node, const RevertedAliasesPtr & aliases) { node.semantic = makeSemantic(aliases); } static void setAliases(ASTQualifiedAsterisk & node, const RevertedAliasesPtr & aliases) { node.semantic = makeSemantic(aliases); } + static void setAliases(ASTColumnsClause & node, const RevertedAliasesPtr & aliases) { node.semantic = makeSemantic(aliases); } static RevertedAliasesPtr getAliases(const ASTAsterisk & node) { return node.semantic ? node.semantic->aliases : nullptr; } static RevertedAliasesPtr getAliases(const ASTQualifiedAsterisk & node) { return node.semantic ? node.semantic->aliases : nullptr; } + static RevertedAliasesPtr getAliases(const ASTColumnsClause & node) { return node.semantic ? node.semantic->aliases : nullptr; } private: static std::shared_ptr makeSemantic(const RevertedAliasesPtr & aliases) diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp index 3d2f465bde8..1e92f7b348b 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -24,6 +24,7 @@ #include #include #include +#include "../Parsers/ASTColumnsClause.h" namespace DB { @@ -412,7 +413,7 @@ ASTs PredicateExpressionsOptimizer::getSelectQueryProjectionColumns(ASTPtr & ast for (const auto & projection_column : select_query->select()->children) { - if (projection_column->as() || projection_column->as()) + if (projection_column->as() || projection_column->as() || projection_column->as()) { ASTs evaluated_columns = evaluateAsterisk(select_query, projection_column); @@ -483,8 +484,20 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que throw Exception("Logical error: unexpected table expression", ErrorCodes::LOGICAL_ERROR); const auto block = storage->getSampleBlock(); - for (size_t idx = 0; idx < block.columns(); idx++) - projection_columns.emplace_back(std::make_shared(block.getByPosition(idx).name)); + if (const auto * asterisk_pattern = asterisk->as()) + { + for (size_t idx = 0; idx < block.columns(); idx++) + { + auto & col = block.getByPosition(idx); + if (asterisk_pattern->isColumnMatching(col.name)) + projection_columns.emplace_back(std::make_shared(col.name)); + } + } + else + { + for (size_t idx = 0; idx < block.columns(); idx++) + projection_columns.emplace_back(std::make_shared(block.getByPosition(idx).name)); + } } } return projection_columns; diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index e2d38422ef1..a418708f1dd 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -17,6 +17,7 @@ #include #include #include +#include "../Parsers/ASTColumnsClause.h" namespace DB @@ -166,7 +167,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt bool has_asterisk = false; for (const auto & child : node.children) { - if (child->as()) + if (child->as() || child->as()) { if (tables_with_columns.empty()) throw Exception("An asterisk cannot be replaced with empty columns.", ErrorCodes::LOGICAL_ERROR); @@ -207,6 +208,23 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt first_table = false; } } + else if (const auto * asterisk_pattern = child->as()) + { + bool first_table = true; + for (const auto & [table, table_columns] : tables_with_columns) + { + for (const auto & column_name : table_columns) + { + if (asterisk_pattern->isColumnMatching(column_name) && (first_table || !data.join_using_columns.count(column_name))) + { + String table_name = table.getQualifiedNamePrefix(false); + addIdentifier(node.children, table_name, column_name, AsteriskSemantic::getAliases(*asterisk_pattern)); + } + } + + first_table = false; + } + } else if (const auto * qualified_asterisk = child->as()) { DatabaseAndTableWithAlias ident_db_and_name(qualified_asterisk->children[0]); diff --git a/dbms/src/Parsers/ASTColumnsClause.cpp b/dbms/src/Parsers/ASTColumnsClause.cpp new file mode 100644 index 00000000000..2f2dca471b7 --- /dev/null +++ b/dbms/src/Parsers/ASTColumnsClause.cpp @@ -0,0 +1,22 @@ +#include +#include +#include "ASTColumnsClause.h" + +namespace DB +{ + +ASTPtr ASTColumnsClause::clone() const +{ + auto clone = std::make_shared(*this); + clone->cloneChildren(); + return clone; +} + +void ASTColumnsClause::appendColumnName(WriteBuffer & ostr) const { writeString(originalPattern, ostr); } + +void ASTColumnsClause::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << " '" << originalPattern << "'"; +} + +} diff --git a/dbms/src/Parsers/ASTColumnsClause.h b/dbms/src/Parsers/ASTColumnsClause.h new file mode 100644 index 00000000000..67f400b305c --- /dev/null +++ b/dbms/src/Parsers/ASTColumnsClause.h @@ -0,0 +1,51 @@ +#pragma once +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_COMPILE_REGEXP; +} + +struct AsteriskSemantic; +struct AsteriskSemanticImpl; + + +class ASTColumnsClause : public IAST +{ +public: + + String getID(char) const override { return "ColumnsClause"; } + ASTPtr clone() const override; + void appendColumnName(WriteBuffer & ostr) const override; + void setPattern(String pattern) + { + originalPattern = pattern; + columnMatcher = std::make_shared(pattern, RE2::Quiet); + if (!columnMatcher->ok()) + throw DB::Exception("COLUMNS pattern " + originalPattern + " cannot be compiled: " + columnMatcher->error(), DB::ErrorCodes::CANNOT_COMPILE_REGEXP); + } + bool isColumnMatching(String columnName) const + { + return RE2::FullMatch(columnName, *columnMatcher); + } + +protected: + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + +private: + std::shared_ptr columnMatcher; + String originalPattern; + std::shared_ptr semantic; /// pimpl + + friend struct AsteriskSemantic; +}; + + +} diff --git a/dbms/src/Parsers/ExpressionElementParsers.cpp b/dbms/src/Parsers/ExpressionElementParsers.cpp index 9c0071c64e8..4669c37edae 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.cpp +++ b/dbms/src/Parsers/ExpressionElementParsers.cpp @@ -29,6 +29,7 @@ #include #include +#include "ASTColumnsClause.h" namespace DB @@ -1168,6 +1169,25 @@ bool ParserAlias::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } +bool ParserColumnsClause::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword columns("COLUMNS"); + ParserStringLiteral regex; + + if (!columns.ignore(pos, expected)) + return false; + + ASTPtr regex_node; + if (!regex.parse(pos, regex_node, expected)) + return false; + + auto res = std::make_shared(); + res->setPattern(regex_node->as().value.get()); + res->children.push_back(regex_node); + node = std::move(res); + return true; +} + bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected &) { if (pos->type == TokenType::Asterisk) @@ -1265,6 +1285,7 @@ bool ParserExpressionElement::parseImpl(Pos & pos, ASTPtr & node, Expected & exp || ParserFunction().parse(pos, node, expected) || ParserQualifiedAsterisk().parse(pos, node, expected) || ParserAsterisk().parse(pos, node, expected) + || ParserColumnsClause().parse(pos, node, expected) || ParserCompoundIdentifier().parse(pos, node, expected) || ParserSubstitution().parse(pos, node, expected); } diff --git a/dbms/src/Parsers/ExpressionElementParsers.h b/dbms/src/Parsers/ExpressionElementParsers.h index b4fe77e8bb3..d3a9f97f0b8 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.h +++ b/dbms/src/Parsers/ExpressionElementParsers.h @@ -56,7 +56,6 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); }; - /// Just * class ParserAsterisk : public IParserBase { @@ -65,7 +64,6 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); }; - /** Something like t.* or db.table.* */ class ParserQualifiedAsterisk : public IParserBase @@ -75,6 +73,14 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); }; +/** COLUMNS '' + */ +class ParserColumnsClause : public IParserBase +{ +protected: + const char * getName() const { return "COLUMNS clause"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); +}; /** A function, for example, f(x, y + 1, g(z)). * Or an aggregate function: sum(x + f(y)), corr(x, y). The syntax is the same as the usual function. diff --git a/dbms/tests/queries/0_stateless/00969_columns_clause.reference b/dbms/tests/queries/0_stateless/00969_columns_clause.reference new file mode 100644 index 00000000000..fcfd7d0919e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00969_columns_clause.reference @@ -0,0 +1,2 @@ +100 10 +120 8 diff --git a/dbms/tests/queries/0_stateless/00969_columns_clause.sql b/dbms/tests/queries/0_stateless/00969_columns_clause.sql new file mode 100644 index 00000000000..63d041984c1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00969_columns_clause.sql @@ -0,0 +1,3 @@ +CREATE TABLE IF NOT EXISTS ColumnsClauseTest (product_price Int64, product_weight Int16, amount Int64) Engine=TinyLog; +INSERT INTO ColumnsClauseTest VALUES (100, 10, 324), (120, 8, 23); +SELECT COLUMNS 'product.*' from ColumnsClauseTest ORDER BY product_price; From ef6c7ea5be41c608044fefce961a6f2473f45e52 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jul 2019 23:05:00 +0300 Subject: [PATCH 02/17] Merge COLUMNS matcher (incomplete) --- dbms/src/Interpreters/AsteriskSemantic.h | 2 +- .../PredicateExpressionsOptimizer.cpp | 9 ++--- .../TranslateQualifiedNamesVisitor.cpp | 3 +- dbms/src/Parsers/ASTColumnsClause.cpp | 29 ++++++++++++++-- dbms/src/Parsers/ASTColumnsClause.h | 33 +++++++++---------- dbms/src/Parsers/ExpressionElementParsers.cpp | 11 ++++++- dbms/src/Parsers/ExpressionElementParsers.h | 4 +-- .../0_stateless/00969_columns_clause.sql | 2 +- 8 files changed, 62 insertions(+), 31 deletions(-) diff --git a/dbms/src/Interpreters/AsteriskSemantic.h b/dbms/src/Interpreters/AsteriskSemantic.h index 1dcfd344a72..9b0939f6001 100644 --- a/dbms/src/Interpreters/AsteriskSemantic.h +++ b/dbms/src/Interpreters/AsteriskSemantic.h @@ -4,7 +4,7 @@ #include #include -#include "../Parsers/ASTColumnsClause.h" +#include namespace DB { diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp index 1e92f7b348b..eb7c34be9b8 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +25,7 @@ #include #include #include -#include "../Parsers/ASTColumnsClause.h" + namespace DB { @@ -267,7 +268,7 @@ std::vector PredicateExpressionsOptimizer::splitConjunctionPredicate(con continue; } } - idx++; + ++idx; } } return predicate_expressions; @@ -486,7 +487,7 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que const auto block = storage->getSampleBlock(); if (const auto * asterisk_pattern = asterisk->as()) { - for (size_t idx = 0; idx < block.columns(); idx++) + for (size_t idx = 0; idx < block.columns(); ++idx) { auto & col = block.getByPosition(idx); if (asterisk_pattern->isColumnMatching(col.name)) @@ -495,7 +496,7 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que } else { - for (size_t idx = 0; idx < block.columns(); idx++) + for (size_t idx = 0; idx < block.columns(); ++idx) projection_columns.emplace_back(std::make_shared(block.getByPosition(idx).name)); } } diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index a418708f1dd..1ca5fa69ce5 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -16,8 +16,7 @@ #include #include #include -#include -#include "../Parsers/ASTColumnsClause.h" +#include namespace DB diff --git a/dbms/src/Parsers/ASTColumnsClause.cpp b/dbms/src/Parsers/ASTColumnsClause.cpp index 2f2dca471b7..8fbcfce471f 100644 --- a/dbms/src/Parsers/ASTColumnsClause.cpp +++ b/dbms/src/Parsers/ASTColumnsClause.cpp @@ -1,6 +1,12 @@ +#include "ASTColumnsClause.h" + #include #include -#include "ASTColumnsClause.h" +#include + +#include + + namespace DB { @@ -12,11 +18,28 @@ ASTPtr ASTColumnsClause::clone() const return clone; } -void ASTColumnsClause::appendColumnName(WriteBuffer & ostr) const { writeString(originalPattern, ostr); } +void ASTColumnsClause::appendColumnName(WriteBuffer & ostr) const { writeString(original_pattern, ostr); } void ASTColumnsClause::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << " '" << originalPattern << "'"; + WriteBufferFromOwnString pattern_quoted; + writeQuotedString(original_pattern, pattern_quoted); + + settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(" << pattern_quoted.str() << ")"; } +void ASTColumnsClause::setPattern(String pattern) +{ + original_pattern = std::move(pattern); + column_matcher = std::make_shared(pattern, RE2::Quiet); + if (!column_matcher->ok()) + throw DB::Exception("COLUMNS pattern " + original_pattern + " cannot be compiled: " + column_matcher->error(), DB::ErrorCodes::CANNOT_COMPILE_REGEXP); +} + +bool ASTColumnsClause::isColumnMatching(const String & column_name) const +{ + return RE2::FullMatch(column_name, *column_matcher); +} + + } diff --git a/dbms/src/Parsers/ASTColumnsClause.h b/dbms/src/Parsers/ASTColumnsClause.h index 67f400b305c..1cf13941420 100644 --- a/dbms/src/Parsers/ASTColumnsClause.h +++ b/dbms/src/Parsers/ASTColumnsClause.h @@ -1,13 +1,19 @@ #pragma once -#include -#include #include -#include + + +namespace re2 +{ + class RE2; +} + namespace DB { +class WriteBuffer; + namespace ErrorCodes { extern const int CANNOT_COMPILE_REGEXP; @@ -17,31 +23,24 @@ struct AsteriskSemantic; struct AsteriskSemanticImpl; +/** SELECT COLUMNS('regexp') is expanded to multiple columns like * (asterisk). + */ class ASTColumnsClause : public IAST { public: - String getID(char) const override { return "ColumnsClause"; } ASTPtr clone() const override; + void appendColumnName(WriteBuffer & ostr) const override; - void setPattern(String pattern) - { - originalPattern = pattern; - columnMatcher = std::make_shared(pattern, RE2::Quiet); - if (!columnMatcher->ok()) - throw DB::Exception("COLUMNS pattern " + originalPattern + " cannot be compiled: " + columnMatcher->error(), DB::ErrorCodes::CANNOT_COMPILE_REGEXP); - } - bool isColumnMatching(String columnName) const - { - return RE2::FullMatch(columnName, *columnMatcher); - } + void setPattern(String pattern); + bool isColumnMatching(const String & column_name) const; protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: - std::shared_ptr columnMatcher; - String originalPattern; + std::shared_ptr column_matcher; + String original_pattern; std::shared_ptr semantic; /// pimpl friend struct AsteriskSemantic; diff --git a/dbms/src/Parsers/ExpressionElementParsers.cpp b/dbms/src/Parsers/ExpressionElementParsers.cpp index 4669c37edae..80ec890b5d5 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.cpp +++ b/dbms/src/Parsers/ExpressionElementParsers.cpp @@ -1177,10 +1177,18 @@ bool ParserColumnsClause::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte if (!columns.ignore(pos, expected)) return false; + if (pos->type != TokenType::OpeningRoundBracket) + return false; + ++pos; + ASTPtr regex_node; if (!regex.parse(pos, regex_node, expected)) return false; + if (pos->type != TokenType::ClosingRoundBracket) + return false; + ++pos; + auto res = std::make_shared(); res->setPattern(regex_node->as().value.get()); res->children.push_back(regex_node); @@ -1188,6 +1196,7 @@ bool ParserColumnsClause::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte return true; } + bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected &) { if (pos->type == TokenType::Asterisk) @@ -1282,10 +1291,10 @@ bool ParserExpressionElement::parseImpl(Pos & pos, ASTPtr & node, Expected & exp || ParserLeftExpression().parse(pos, node, expected) || ParserRightExpression().parse(pos, node, expected) || ParserCase().parse(pos, node, expected) + || ParserColumnsClause().parse(pos, node, expected) /// before ParserFunction because it can be also parsed as a function. || ParserFunction().parse(pos, node, expected) || ParserQualifiedAsterisk().parse(pos, node, expected) || ParserAsterisk().parse(pos, node, expected) - || ParserColumnsClause().parse(pos, node, expected) || ParserCompoundIdentifier().parse(pos, node, expected) || ParserSubstitution().parse(pos, node, expected); } diff --git a/dbms/src/Parsers/ExpressionElementParsers.h b/dbms/src/Parsers/ExpressionElementParsers.h index d3a9f97f0b8..bcf7be49e4e 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.h +++ b/dbms/src/Parsers/ExpressionElementParsers.h @@ -73,12 +73,12 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); }; -/** COLUMNS '' +/** COLUMNS('') */ class ParserColumnsClause : public IParserBase { protected: - const char * getName() const { return "COLUMNS clause"; } + const char * getName() const { return "COLUMNS matcher"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); }; diff --git a/dbms/tests/queries/0_stateless/00969_columns_clause.sql b/dbms/tests/queries/0_stateless/00969_columns_clause.sql index 63d041984c1..b1d7949b8ee 100644 --- a/dbms/tests/queries/0_stateless/00969_columns_clause.sql +++ b/dbms/tests/queries/0_stateless/00969_columns_clause.sql @@ -1,3 +1,3 @@ CREATE TABLE IF NOT EXISTS ColumnsClauseTest (product_price Int64, product_weight Int16, amount Int64) Engine=TinyLog; INSERT INTO ColumnsClauseTest VALUES (100, 10, 324), (120, 8, 23); -SELECT COLUMNS 'product.*' from ColumnsClauseTest ORDER BY product_price; +SELECT COLUMNS('product.*') from ColumnsClauseTest ORDER BY product_price; From 5b879e143f510c09c538de6d4aa9be88199e36de Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 20 Jul 2019 00:03:25 +0300 Subject: [PATCH 03/17] Fix segfault in ExternalLoader::reloadOutdated(). --- dbms/src/Interpreters/ExternalLoader.cpp | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index c7b3d202a28..3b6f9c21b7d 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -536,7 +536,7 @@ public: for (const auto & name_and_info : infos) { const auto & info = name_and_info.second; - if ((now >= info.next_update_time) && !info.loading() && info.was_loading()) + if ((now >= info.next_update_time) && !info.loading() && info.loaded()) is_modified_map.emplace(info.object, true); } } @@ -558,16 +558,21 @@ public: std::lock_guard lock{mutex}; TimePoint now = std::chrono::system_clock::now(); for (auto & [name, info] : infos) - if ((now >= info.next_update_time) && !info.loading() && info.was_loading()) + if ((now >= info.next_update_time) && !info.loading()) { - auto it = is_modified_map.find(info.object); - if (it == is_modified_map.end()) - continue; /// Object has been just added, it can be simply omitted from this update of outdated. - bool is_modified_flag = it->second; - if (info.loaded() && !is_modified_flag) - info.next_update_time = calculate_next_update_time(info.object, info.error_count); - else - startLoading(name, info); + if (info.loaded()) + { + auto it = is_modified_map.find(info.object); + if (it == is_modified_map.end()) + continue; /// Object has just been added, we can simply omit it from this update of outdated. + bool is_modified_flag = it->second; + if (!is_modified_flag) + { + info.next_update_time = calculate_next_update_time(info.object, info.error_count); + continue; + } + } + startLoading(name, info); } } } From f18e9592a1d3a422a56166dfc33cb9a714d3b5c9 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 19 Jul 2019 22:37:48 +0300 Subject: [PATCH 04/17] Update ExternalLoader.cpp --- dbms/src/Interpreters/ExternalLoader.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 3b6f9c21b7d..a5863adb92d 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -511,12 +511,14 @@ public: { std::lock_guard lock{mutex}; for (auto & [name, info] : infos) + { if ((info.was_loading() || load_never_loading) && filter_by_name(name)) { cancelLoading(info); info.forced_to_reload = true; startLoading(name, info); } + } } /// Starts reloading of all the objects. @@ -558,6 +560,7 @@ public: std::lock_guard lock{mutex}; TimePoint now = std::chrono::system_clock::now(); for (auto & [name, info] : infos) + { if ((now >= info.next_update_time) && !info.loading()) { if (info.loaded()) @@ -574,6 +577,7 @@ public: } startLoading(name, info); } + } } } From e64f8e606d62dadbe5c9a9fd20a6607709317497 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 20 Jul 2019 00:01:20 +0300 Subject: [PATCH 05/17] Add comments. --- dbms/src/Interpreters/ExternalLoader.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index a5863adb92d..31a69086f5c 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -530,8 +530,8 @@ public: /// The function doesn't touch the objects which were never tried to load. void reloadOutdated() { + /// Iterate through all the objects and find loaded ones which should be checked if they were modified. std::unordered_map is_modified_map; - { std::lock_guard lock{mutex}; TimePoint now = std::chrono::system_clock::now(); @@ -543,7 +543,9 @@ public: } } - /// The `mutex` should be unlocked while we're calling the function is_object_modified(). + /// Find out which of the loaded objects were modified. + /// We couldn't perform these checks while we were building `is_modified_map` because + /// the `mutex` should be unlocked while we're calling the function is_object_modified(). for (auto & [object, is_modified_flag] : is_modified_map) { try @@ -556,6 +558,7 @@ public: } } + /// Iterate through all the objects again and either start loading or just set `next_update_time`. { std::lock_guard lock{mutex}; TimePoint now = std::chrono::system_clock::now(); @@ -567,15 +570,24 @@ public: { auto it = is_modified_map.find(info.object); if (it == is_modified_map.end()) - continue; /// Object has just been added, we can simply omit it from this update of outdated. + continue; /// Object has been just loaded (it wasn't loaded while we were building the map `is_modified_map`), so we don't have to reload it right now. + bool is_modified_flag = it->second; if (!is_modified_flag) { + /// Object wasn't modified so we only have to set `next_update_time`. info.next_update_time = calculate_next_update_time(info.object, info.error_count); continue; } + + /// Object was modified and should be reloaded. + startLoading(name, info); + } + else if (info.failed()) + { + /// Object was never loaded successfully and should be reloaded. + startLoading(name, info); } - startLoading(name, info); } } } From da50663e2c701a08da84b8ac5be8a9da40d134c2 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 26 Jun 2019 21:33:48 +0200 Subject: [PATCH 06/17] Pass a proper message to exception --- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index b51da168192..089c8bb72d8 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3042,8 +3042,9 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p if (!selected) { - LOG_INFO(log, "Cannot select parts for optimization" + (disable_reason.empty() ? "" : ": " + disable_reason)); - return handle_noop(disable_reason); + String message = "Cannot select parts for optimization" + (disable_reason.empty() ? "" : ": " + disable_reason) + LOG_INFO(log, message); + return handle_noop(message); } ReplicatedMergeTreeLogEntryData merge_entry; From 8008ce6e9e2fd85cab5820496cee37cfc87e1b3d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 26 Jun 2019 23:04:39 +0300 Subject: [PATCH 07/17] Update StorageReplicatedMergeTree.cpp --- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 089c8bb72d8..934d651fead 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3042,9 +3042,12 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p if (!selected) { - String message = "Cannot select parts for optimization" + (disable_reason.empty() ? "" : ": " + disable_reason) - LOG_INFO(log, message); - return handle_noop(message); + std::stringstream message; + message << "Cannot select parts for optimization"; + if (!disable_reason.empty()) + message << ": " << disable_reason; + LOG_INFO(log, message.rdbuf()); + return handle_noop(message.str()); } ReplicatedMergeTreeLogEntryData merge_entry; From 6ae38d94732458fa9f381cad29e2bbc151450a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Podlipsk=C3=BD?= Date: Sat, 20 Jul 2019 11:45:56 +0300 Subject: [PATCH 08/17] Fix TTL link in MergeTree docs --- docs/en/operations/table_engines/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 523b442319a..8610397be7c 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -72,7 +72,7 @@ For a description of request parameters, see [request description](../../query_l It must depends on `Date` or `DateTime` column and has one `Date` or `DateTime` column as a result. Example: `TTL date + INTERVAL 1 DAY` - For more details, see [TTL for columns and tables](mergetree.md) + For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl) - `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: - `index_granularity` — The granularity of an index. The number of data rows between the "marks" of an index. By default, 8192. The list of all available parameters you can see in [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). From 15fd980d0b06cdbfcba0ff1dd752e67fbaec7902 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 12 Jul 2019 15:39:16 +0200 Subject: [PATCH 09/17] Add description "unable to select parts" and logging --- dbms/src/Storages/StorageMergeTree.cpp | 30 ++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index d021866487c..1419049ae64 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -530,7 +530,11 @@ bool StorageMergeTree::merge( } if (!selected) + { + if (out_disable_reason) + *out_disable_reason = "Cannot select parts for optimization"; return false; + } merging_tagger.emplace(future_part, MergeTreeDataMergerMutator::estimateNeededDiskSpace(future_part.parts), *this); } @@ -886,8 +890,19 @@ bool StorageMergeTree::optimize( { if (!merge(true, partition_id, true, deduplicate, &disable_reason)) { + std::stringstream message; + message << "Cannot OPTIMIZE table"; + if (! disable_reason.empty()) + { + message << ": " << disable_reason; + } else + { + message << " by some reason."; + } + LOG_INFO(log, message.rdbuf()); + if (context.getSettingsRef().optimize_throw_if_noop) - throw Exception(disable_reason.empty() ? "Can't OPTIMIZE by some reason" : disable_reason, ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); + throw Exception(message.str(), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); return false; } } @@ -900,8 +915,19 @@ bool StorageMergeTree::optimize( if (!merge(true, partition_id, final, deduplicate, &disable_reason)) { + std::stringstream message; + message << "Cannot OPTIMIZE table"; + if (! disable_reason.empty()) + { + message << ": " << disable_reason; + } else + { + message << " by some reason."; + } + LOG_INFO(log, message.rdbuf()); + if (context.getSettingsRef().optimize_throw_if_noop) - throw Exception(disable_reason.empty() ? "Can't OPTIMIZE by some reason" : disable_reason, ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); + throw Exception(message.str(), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); return false; } } From 47058e8e11c93d883ecdba591bd420ac26449271 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Jul 2019 05:13:42 +0300 Subject: [PATCH 10/17] Fixed error --- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 1 - dbms/src/Interpreters/QueryNormalizer.cpp | 1 - dbms/src/Parsers/ASTColumnsClause.cpp | 4 ++-- dbms/src/Parsers/ASTSelectQuery.cpp | 1 - dbms/src/Storages/StorageView.cpp | 8 ++++---- dbms/tests/queries/0_stateless/00969_columns_clause.sql | 6 +++++- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index e8143f3c41a..055f5c8f3ec 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Interpreters/QueryNormalizer.cpp b/dbms/src/Interpreters/QueryNormalizer.cpp index c35c47179c6..cea801c7c2f 100644 --- a/dbms/src/Interpreters/QueryNormalizer.cpp +++ b/dbms/src/Interpreters/QueryNormalizer.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Parsers/ASTColumnsClause.cpp b/dbms/src/Parsers/ASTColumnsClause.cpp index 8fbcfce471f..2bd8670480e 100644 --- a/dbms/src/Parsers/ASTColumnsClause.cpp +++ b/dbms/src/Parsers/ASTColumnsClause.cpp @@ -31,14 +31,14 @@ void ASTColumnsClause::formatImpl(const FormatSettings & settings, FormatState & void ASTColumnsClause::setPattern(String pattern) { original_pattern = std::move(pattern); - column_matcher = std::make_shared(pattern, RE2::Quiet); + column_matcher = std::make_shared(original_pattern, RE2::Quiet); if (!column_matcher->ok()) throw DB::Exception("COLUMNS pattern " + original_pattern + " cannot be compiled: " + column_matcher->error(), DB::ErrorCodes::CANNOT_COMPILE_REGEXP); } bool ASTColumnsClause::isColumnMatching(const String & column_name) const { - return RE2::FullMatch(column_name, *column_matcher); + return RE2::PartialMatch(column_name, *column_matcher); } diff --git a/dbms/src/Parsers/ASTSelectQuery.cpp b/dbms/src/Parsers/ASTSelectQuery.cpp index 44012cd071c..16396095ce9 100644 --- a/dbms/src/Parsers/ASTSelectQuery.cpp +++ b/dbms/src/Parsers/ASTSelectQuery.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 9d4855b586a..153de1ab176 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -1,9 +1,12 @@ #include #include +#include + #include #include #include #include +#include #include #include @@ -11,10 +14,7 @@ #include #include -#include -#include -#include -#include + namespace DB { diff --git a/dbms/tests/queries/0_stateless/00969_columns_clause.sql b/dbms/tests/queries/0_stateless/00969_columns_clause.sql index b1d7949b8ee..1ea344f0bcd 100644 --- a/dbms/tests/queries/0_stateless/00969_columns_clause.sql +++ b/dbms/tests/queries/0_stateless/00969_columns_clause.sql @@ -1,3 +1,7 @@ -CREATE TABLE IF NOT EXISTS ColumnsClauseTest (product_price Int64, product_weight Int16, amount Int64) Engine=TinyLog; +DROP TABLE IF EXISTS ColumnsClauseTest; + +CREATE TABLE ColumnsClauseTest (product_price Int64, product_weight Int16, amount Int64) Engine=TinyLog; INSERT INTO ColumnsClauseTest VALUES (100, 10, 324), (120, 8, 23); SELECT COLUMNS('product.*') from ColumnsClauseTest ORDER BY product_price; + +DROP TABLE ColumnsClauseTest; From 75de4eec70dde559966a807cc90cafac362e3be5 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 12 Jul 2019 15:42:49 +0200 Subject: [PATCH 11/17] Add tests for optimization exceptions --- ...est_zookeeper_optimize_exception.reference | 2 ++ ...00029_test_zookeeper_optimize_exception.sh | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.reference create mode 100755 dbms/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh diff --git a/dbms/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.reference b/dbms/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/dbms/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh b/dbms/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh new file mode 100755 index 00000000000..ef33f037690 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CUR_DIR/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_optimize_exception" + +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception (date Date) ENGINE=MergeTree() PARTITION BY toYYYYMM(date) ORDER BY date" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception_replicated (date Date) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test/optimize', 'r1') PARTITION BY toYYYYMM(date) ORDER BY date" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO test_optimize_exception VALUES (toDate('2017-09-09')), (toDate('2017-09-10'))" +${CLICKHOUSE_CLIENT} --query="INSERT INTO test_optimize_exception VALUES (toDate('2017-09-09')), (toDate('2017-09-10'))" +${CLICKHOUSE_CLIENT} --query="INSERT INTO test_optimize_exception_replicated VALUES (toDate('2017-09-09')), (toDate('2017-09-10'))" +${CLICKHOUSE_CLIENT} --query="INSERT INTO test_optimize_exception_replicated VALUES (toDate('2017-09-09')), (toDate('2017-09-10'))" + +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="OPTIMIZE TABLE test_optimize_exception PARTITION 201709 FINAL" +${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --query="OPTIMIZE TABLE test_optimize_exception_replicated PARTITION 201709 FINAL" + +echo `${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null --query="OPTIMIZE TABLE test_optimize_exception PARTITION 201710" 2>&1` \ + | grep -c 'Code: 388. DB::Exception: .* DB::Exception: .* Cannot select parts for optimization' +echo `${CLICKHOUSE_CLIENT} --optimize_throw_if_noop 1 --server_logs_file=/dev/null --query="OPTIMIZE TABLE test_optimize_exception_replicated PARTITION 201710" 2>&1` \ + | grep -c 'Code: 388. DB::Exception: .* DB::Exception:.* Cannot select parts for optimization' From 28a0481f1ae3ea9cdaea4c046522dc9686b12ed6 Mon Sep 17 00:00:00 2001 From: Yuriy Date: Sun, 21 Jul 2019 15:09:41 +0300 Subject: [PATCH 12/17] php mysql client test --- dbms/programs/server/MySQLHandler.cpp | 4 +-- .../clients/php-mysqlnd/Dockerfile | 7 +++++ .../clients/php-mysqlnd/client.crt | 18 ++++++++++++ .../clients/php-mysqlnd/client.key | 28 +++++++++++++++++++ .../clients/php-mysqlnd/docker_compose.yml | 7 +++++ .../clients/php-mysqlnd/test.php | 27 ++++++++++++++++++ .../integration/test_mysql_protocol/test.py | 14 ++++++++++ 7 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/Dockerfile create mode 100644 dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/client.crt create mode 100644 dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/client.key create mode 100644 dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml create mode 100644 dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/test.php diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index a935f13e82a..af2cfd366d2 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -299,9 +299,9 @@ void MySQLHandler::authenticate(const HandshakeResponse & handshake_response, co LOG_TRACE(log, "Received empty password"); } - if (!password.empty()) + if (!password.empty() && password.back() == 0) { - password.pop_back(); /// terminating null byte + password.pop_back(); } try diff --git a/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/Dockerfile b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/Dockerfile new file mode 100644 index 00000000000..85e256dfed6 --- /dev/null +++ b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/Dockerfile @@ -0,0 +1,7 @@ +FROM php:7.3-cli + +COPY ./client.crt client.crt +COPY ./client.key client.key +COPY ./test.php test.php + +RUN docker-php-ext-install pdo pdo_mysql diff --git a/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/client.crt b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/client.crt new file mode 100644 index 00000000000..6f4deca038f --- /dev/null +++ b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/client.crt @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC+zCCAeOgAwIBAgIJAIhI9ozZJ+TWMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAeFw0xOTA0MjIwNDMyNTJaFw0yMDA0MjEwNDMyNTJaMBQx +EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAK+wVUEdqF2uXvN0MJBgnAHyXi6JTi4p/F6igsrCjSNjJWzHH0vQmK8ujfcF +CkifW88i+W5eHctuEtQqNHK+t9x9YiZtXrj6m/XkOXs20mYgENSmbbbHbriTPnZB +zZrq6UqMlwIHNNAa+I3NMORQxVRaI0ybXnGVO5elr70xHpk03xL0JWKHpEqYp4db +2aBQgF6y3Ww4khxjIYqpUYXWXGFnVIRU7FKVEAM1xyKqvQzXjQ5sVM/wyHknveEF +3b/X4ggN+KNl5KOc0cWDh1/XaatJAPaUUPqZcq76tynLbP64Xm3dxHcj+gtRkO67 +ef6MSg6l63m3XQP6Qb+MIkd06OsCAwEAAaNQME4wHQYDVR0OBBYEFDmODTO8QLDN +ykR3x0LIOnjNhrKhMB8GA1UdIwQYMBaAFDmODTO8QLDNykR3x0LIOnjNhrKhMAwG +A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAwaiJc7uqEpnH3aukbftDwX +m8GfEnj1HVdgg+9GGNq+9rvUYBF6gdPmjRCX9dO0cclLFx8jc2org0rTSq9WoOhX +E6qL4Eqrmc5SE3Y9jZM0h6GRD4oXK014FmtZ3T6ddZU3dQLj3BS2r1XrvmubTvGN +ZuTJNY8nx8Hh6H5XINmsEjUF9E5hog+PwCE03xt2adIdYL+gsbxASeNYyeUFpZv5 +zcXR3VoakBWnAaOVgCHq2qh96QAnL7ZKzFkGf/MdwV10KU3dmb+ICbQUUdf9Gc17 +aaDCIRws312F433FdXBkGs2UkB7ZZme9dfn6O1QbeTNvex2VLMqYx/CTkfFbOQA= +-----END CERTIFICATE----- diff --git a/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/client.key b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/client.key new file mode 100644 index 00000000000..6eddb3295db --- /dev/null +++ b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCvsFVBHahdrl7z +dDCQYJwB8l4uiU4uKfxeooLKwo0jYyVsxx9L0JivLo33BQpIn1vPIvluXh3LbhLU +KjRyvrfcfWImbV64+pv15Dl7NtJmIBDUpm22x264kz52Qc2a6ulKjJcCBzTQGviN +zTDkUMVUWiNMm15xlTuXpa+9MR6ZNN8S9CVih6RKmKeHW9mgUIBest1sOJIcYyGK +qVGF1lxhZ1SEVOxSlRADNcciqr0M140ObFTP8Mh5J73hBd2/1+IIDfijZeSjnNHF +g4df12mrSQD2lFD6mXKu+rcpy2z+uF5t3cR3I/oLUZDuu3n+jEoOpet5t10D+kG/ +jCJHdOjrAgMBAAECggEARF66zrxb6RkSmmt8+rKeA6PuQu3sHsr4C1vyyjUr97l9 +tvdGlpp20LWtSZQMjHZ3pARYTTsTHTeY3DgQcRcHNicVKx8k3ZepWeeW9vw+pL+V +zSt3RsoVrH6gsCSrfr4sS3aqzX9AbjwQvh48CJ3mLQ1m70kHV+xbZIh1+4pB/hyP +1wKyUE18ZkOptXvO/TtoHzLQCecpkXtWzmry1Eh2isvXA+NMrAtLibGsyM1mtm7i +5ozevzHabvvCDBEe+KgZdONgVhhhvm2eOd+/s4w3rw4ETud4fI/ZAJyWXhiIKFnA +VJbElWruSAoVBW7p2bsF5PbmVzvo8vXL+VylxYD+AQKBgQDhLoRKTVhNkn/QjKxq +sdOh+QZra0LzjVpAmkQzu7wZMSHEz9qePQciDQQrYKrmRF1vNcIRCVUTqWYheJ/1 +lKRrCGa0ab6k96zkWMqLHD5u+UeJV7r1dJIx08ME9kNJ+x/XtB8klRIji16NiQUS +qc6p8z0M2AnbJzsRfWZRH8FeYwKBgQDHu8dzdtVGI7MtxfPOE/bfajiopDg8BdTC +pdug2T8XofRHRq7Q+0vYjTAZFT/slib91Pk6VvvPdo9VBZiL4omv4dAq6mOOdX/c +U14mJe1X5GCrr8ExZ8BfNJ3t/6sV1fcxyJwAw7iBguqxA2JqdM/wFk10K8XqvzVn +CD6O9yGt2QKBgFX1BMi8N538809vs41S7l9hCQNOQZNo/O+2M5yv6ECRkbtoQKKw +1x03bMUGNJaLuELweXE5Z8GGo5bZTe5X3F+DKHlr+DtO1C+ieUaa9HY2MAmMdLCn +2/qrREGLo+oEs4YKmuzC/taUp/ZNPKOAMISNdluFyFVg51pozPrgrVbTAoGBAKkE +LBl3O67o0t0vH8sJdeVFG8EJhlS0koBMnfgVHqC++dm+5HwPyvTrNQJkyv1HaqNt +r6FArkG3ED9gRuBIyT6+lctbIPgSUip9mbQqcBfqOCvQxGksZMur2ODncz09HLtS +CUFUXjOqNzOnq4ZuZu/Bz7U4vXiSaXxQq6+LTUKxAoGAFZU/qrI06XxnrE9A1X0W +l7DSkpZaDcu11NrZ473yONih/xOZNh4SSBpX8a7F6Pmh9BdtGqphML8NFPvQKcfP +b9H2iid2tc292uyrUEb5uTMmv61zoTwtitqLzO0+tS6PT3fXobX+eyeEWKzPBljL +HFtxG5CCXpkdnWRmaJnhTzA= +-----END PRIVATE KEY----- diff --git a/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml new file mode 100644 index 00000000000..28145864024 --- /dev/null +++ b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/docker_compose.yml @@ -0,0 +1,7 @@ +version: '2.2' +services: + php1: + build: + context: ./ + # to keep container running + command: sleep infinity diff --git a/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/test.php b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/test.php new file mode 100644 index 00000000000..86354381835 --- /dev/null +++ b/dbms/tests/integration/test_mysql_protocol/clients/php-mysqlnd/test.php @@ -0,0 +1,27 @@ + PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => false, + PDO::MYSQL_ATTR_DIRECT_QUERY => true, + PDO::MYSQL_ATTR_SSL_CERT => "client.crt", + PDO::MYSQL_ATTR_SSL_KEY => "client.key", + PDO::MYSQL_ATTR_SSL_VERIFY_SERVER_CERT => 0, +]; +$pdo = new PDO($dsn, $user, $pass, $options); + +$stmt = $pdo->query("SELECT name FROM tables WHERE name = 'tables'"); + +foreach ($stmt as $row) +{ + echo $row["name"] . "\n"; +} +?> diff --git a/dbms/tests/integration/test_mysql_protocol/test.py b/dbms/tests/integration/test_mysql_protocol/test.py index 04e6ac1ee8b..f0c696b4dc7 100644 --- a/dbms/tests/integration/test_mysql_protocol/test.py +++ b/dbms/tests/integration/test_mysql_protocol/test.py @@ -43,6 +43,13 @@ def golang_container(): yield docker.from_env().containers.get(cluster.project_name + '_golang1_1') +@pytest.fixture(scope='module') +def php_container(): + docker_compose = os.path.join(SCRIPT_DIR, 'clients', 'php-mysqlnd', 'docker_compose.yml') + subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build']) + yield docker.from_env().containers.get(cluster.project_name + '_php1_1') + + def test_mysql_client(mysql_client, server_address): # type: (Container, str) -> None code, (stdout, stderr) = mysql_client.exec_run(''' @@ -138,3 +145,10 @@ def test_golang_client(server_address, golang_container): with open(os.path.join(SCRIPT_DIR, 'clients', 'golang', '0.reference')) as fp: reference = fp.read() assert stdout == reference + + +def test_php_client(server_address, php_container): + # type: (str, Container) -> None + code, (stdout, stderr) = php_container.exec_run('php -f test.php {host} {port} default 123 '.format(host=server_address, port=server_port), demux=True) + assert code == 0 + assert stdout == 'tables\n' From 9f9dd040dd835432f3d1cd876098a53c3288786f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 21 Jul 2019 17:14:55 +0300 Subject: [PATCH 13/17] Update StorageMergeTree.cpp --- dbms/src/Storages/StorageMergeTree.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 1419049ae64..0536423101d 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -892,13 +892,10 @@ bool StorageMergeTree::optimize( { std::stringstream message; message << "Cannot OPTIMIZE table"; - if (! disable_reason.empty()) - { + if (!disable_reason.empty()) message << ": " << disable_reason; - } else - { + else message << " by some reason."; - } LOG_INFO(log, message.rdbuf()); if (context.getSettingsRef().optimize_throw_if_noop) @@ -917,13 +914,10 @@ bool StorageMergeTree::optimize( { std::stringstream message; message << "Cannot OPTIMIZE table"; - if (! disable_reason.empty()) - { + if (!disable_reason.empty()) message << ": " << disable_reason; - } else - { + else message << " by some reason."; - } LOG_INFO(log, message.rdbuf()); if (context.getSettingsRef().optimize_throw_if_noop) From fb0d09c5d359887c1e5687b83b024b01197c3466 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Jul 2019 20:03:58 +0300 Subject: [PATCH 14/17] Rename files --- dbms/src/Interpreters/AsteriskSemantic.h | 6 +++--- .../Interpreters/PredicateExpressionsOptimizer.cpp | 6 +++--- .../TranslateQualifiedNamesVisitor.cpp | 6 +++--- ...{ASTColumnsClause.cpp => ASTColumnsMatcher.cpp} | 14 +++++++------- .../{ASTColumnsClause.h => ASTColumnsMatcher.h} | 4 ++-- dbms/src/Parsers/ExpressionElementParsers.cpp | 8 ++++---- dbms/src/Parsers/ExpressionElementParsers.h | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) rename dbms/src/Parsers/{ASTColumnsClause.cpp => ASTColumnsMatcher.cpp} (63%) rename dbms/src/Parsers/{ASTColumnsClause.h => ASTColumnsMatcher.h} (88%) diff --git a/dbms/src/Interpreters/AsteriskSemantic.h b/dbms/src/Interpreters/AsteriskSemantic.h index 9b0939f6001..1bd9ecedddd 100644 --- a/dbms/src/Interpreters/AsteriskSemantic.h +++ b/dbms/src/Interpreters/AsteriskSemantic.h @@ -4,7 +4,7 @@ #include #include -#include +#include namespace DB { @@ -25,11 +25,11 @@ struct AsteriskSemantic static void setAliases(ASTAsterisk & node, const RevertedAliasesPtr & aliases) { node.semantic = makeSemantic(aliases); } static void setAliases(ASTQualifiedAsterisk & node, const RevertedAliasesPtr & aliases) { node.semantic = makeSemantic(aliases); } - static void setAliases(ASTColumnsClause & node, const RevertedAliasesPtr & aliases) { node.semantic = makeSemantic(aliases); } + static void setAliases(ASTColumnsMatcher & node, const RevertedAliasesPtr & aliases) { node.semantic = makeSemantic(aliases); } static RevertedAliasesPtr getAliases(const ASTAsterisk & node) { return node.semantic ? node.semantic->aliases : nullptr; } static RevertedAliasesPtr getAliases(const ASTQualifiedAsterisk & node) { return node.semantic ? node.semantic->aliases : nullptr; } - static RevertedAliasesPtr getAliases(const ASTColumnsClause & node) { return node.semantic ? node.semantic->aliases : nullptr; } + static RevertedAliasesPtr getAliases(const ASTColumnsMatcher & node) { return node.semantic ? node.semantic->aliases : nullptr; } private: static std::shared_ptr makeSemantic(const RevertedAliasesPtr & aliases) diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp index eb7c34be9b8..b6e6545e475 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -414,7 +414,7 @@ ASTs PredicateExpressionsOptimizer::getSelectQueryProjectionColumns(ASTPtr & ast for (const auto & projection_column : select_query->select()->children) { - if (projection_column->as() || projection_column->as() || projection_column->as()) + if (projection_column->as() || projection_column->as() || projection_column->as()) { ASTs evaluated_columns = evaluateAsterisk(select_query, projection_column); @@ -485,7 +485,7 @@ ASTs PredicateExpressionsOptimizer::evaluateAsterisk(ASTSelectQuery * select_que throw Exception("Logical error: unexpected table expression", ErrorCodes::LOGICAL_ERROR); const auto block = storage->getSampleBlock(); - if (const auto * asterisk_pattern = asterisk->as()) + if (const auto * asterisk_pattern = asterisk->as()) { for (size_t idx = 0; idx < block.columns(); ++idx) { diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 1ca5fa69ce5..df27a21c9b4 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include namespace DB @@ -166,7 +166,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt bool has_asterisk = false; for (const auto & child : node.children) { - if (child->as() || child->as()) + if (child->as() || child->as()) { if (tables_with_columns.empty()) throw Exception("An asterisk cannot be replaced with empty columns.", ErrorCodes::LOGICAL_ERROR); @@ -207,7 +207,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt first_table = false; } } - else if (const auto * asterisk_pattern = child->as()) + else if (const auto * asterisk_pattern = child->as()) { bool first_table = true; for (const auto & [table, table_columns] : tables_with_columns) diff --git a/dbms/src/Parsers/ASTColumnsClause.cpp b/dbms/src/Parsers/ASTColumnsMatcher.cpp similarity index 63% rename from dbms/src/Parsers/ASTColumnsClause.cpp rename to dbms/src/Parsers/ASTColumnsMatcher.cpp index 2bd8670480e..e9cdb822c6e 100644 --- a/dbms/src/Parsers/ASTColumnsClause.cpp +++ b/dbms/src/Parsers/ASTColumnsMatcher.cpp @@ -1,4 +1,4 @@ -#include "ASTColumnsClause.h" +#include "ASTColumnsMatcher.h" #include #include @@ -11,16 +11,16 @@ namespace DB { -ASTPtr ASTColumnsClause::clone() const +ASTPtr ASTColumnsMatcher::clone() const { - auto clone = std::make_shared(*this); + auto clone = std::make_shared(*this); clone->cloneChildren(); return clone; } -void ASTColumnsClause::appendColumnName(WriteBuffer & ostr) const { writeString(original_pattern, ostr); } +void ASTColumnsMatcher::appendColumnName(WriteBuffer & ostr) const { writeString(original_pattern, ostr); } -void ASTColumnsClause::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTColumnsMatcher::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { WriteBufferFromOwnString pattern_quoted; writeQuotedString(original_pattern, pattern_quoted); @@ -28,7 +28,7 @@ void ASTColumnsClause::formatImpl(const FormatSettings & settings, FormatState & settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(" << pattern_quoted.str() << ")"; } -void ASTColumnsClause::setPattern(String pattern) +void ASTColumnsMatcher::setPattern(String pattern) { original_pattern = std::move(pattern); column_matcher = std::make_shared(original_pattern, RE2::Quiet); @@ -36,7 +36,7 @@ void ASTColumnsClause::setPattern(String pattern) throw DB::Exception("COLUMNS pattern " + original_pattern + " cannot be compiled: " + column_matcher->error(), DB::ErrorCodes::CANNOT_COMPILE_REGEXP); } -bool ASTColumnsClause::isColumnMatching(const String & column_name) const +bool ASTColumnsMatcher::isColumnMatching(const String & column_name) const { return RE2::PartialMatch(column_name, *column_matcher); } diff --git a/dbms/src/Parsers/ASTColumnsClause.h b/dbms/src/Parsers/ASTColumnsMatcher.h similarity index 88% rename from dbms/src/Parsers/ASTColumnsClause.h rename to dbms/src/Parsers/ASTColumnsMatcher.h index 1cf13941420..e5ec9a2873d 100644 --- a/dbms/src/Parsers/ASTColumnsClause.h +++ b/dbms/src/Parsers/ASTColumnsMatcher.h @@ -25,10 +25,10 @@ struct AsteriskSemanticImpl; /** SELECT COLUMNS('regexp') is expanded to multiple columns like * (asterisk). */ -class ASTColumnsClause : public IAST +class ASTColumnsMatcher : public IAST { public: - String getID(char) const override { return "ColumnsClause"; } + String getID(char) const override { return "ColumnsMatcher"; } ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; diff --git a/dbms/src/Parsers/ExpressionElementParsers.cpp b/dbms/src/Parsers/ExpressionElementParsers.cpp index 80ec890b5d5..8cd017f0710 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.cpp +++ b/dbms/src/Parsers/ExpressionElementParsers.cpp @@ -29,7 +29,7 @@ #include #include -#include "ASTColumnsClause.h" +#include "ASTColumnsMatcher.h" namespace DB @@ -1169,7 +1169,7 @@ bool ParserAlias::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } -bool ParserColumnsClause::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +bool ParserColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword columns("COLUMNS"); ParserStringLiteral regex; @@ -1189,7 +1189,7 @@ bool ParserColumnsClause::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte return false; ++pos; - auto res = std::make_shared(); + auto res = std::make_shared(); res->setPattern(regex_node->as().value.get()); res->children.push_back(regex_node); node = std::move(res); @@ -1291,7 +1291,7 @@ bool ParserExpressionElement::parseImpl(Pos & pos, ASTPtr & node, Expected & exp || ParserLeftExpression().parse(pos, node, expected) || ParserRightExpression().parse(pos, node, expected) || ParserCase().parse(pos, node, expected) - || ParserColumnsClause().parse(pos, node, expected) /// before ParserFunction because it can be also parsed as a function. + || ParserColumnsMatcher().parse(pos, node, expected) /// before ParserFunction because it can be also parsed as a function. || ParserFunction().parse(pos, node, expected) || ParserQualifiedAsterisk().parse(pos, node, expected) || ParserAsterisk().parse(pos, node, expected) diff --git a/dbms/src/Parsers/ExpressionElementParsers.h b/dbms/src/Parsers/ExpressionElementParsers.h index bcf7be49e4e..dca82f72f12 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.h +++ b/dbms/src/Parsers/ExpressionElementParsers.h @@ -75,7 +75,7 @@ protected: /** COLUMNS('') */ -class ParserColumnsClause : public IParserBase +class ParserColumnsMatcher : public IParserBase { protected: const char * getName() const { return "COLUMNS matcher"; } From 8060993ca20a16a0bf25102d78fd8f45d0aade51 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Jul 2019 23:40:34 +0300 Subject: [PATCH 15/17] Removed obsolete comment --- dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index df27a21c9b4..1b063bc4625 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -124,7 +124,6 @@ void TranslateQualifiedNamesMatcher::visit(ASTSelectQuery & select, const ASTPtr if (auto join = select.join()) extractJoinUsingColumns(join->table_join, data); -#if 1 /// TODO: legacy? /// If the WHERE clause or HAVING consists of a single qualified column, the reference must be translated not only in children, /// but also in where_expression and having_expression. if (select.prewhere()) @@ -133,7 +132,6 @@ void TranslateQualifiedNamesMatcher::visit(ASTSelectQuery & select, const ASTPtr Visitor(data).visit(select.refWhere()); if (select.having()) Visitor(data).visit(select.refHaving()); -#endif } static void addIdentifier(ASTs & nodes, const String & table_name, const String & column_name, AsteriskSemantic::RevertedAliasesPtr aliases) From b2961bcc31450f75b249d512bfde7150e1397712 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Jul 2019 00:19:42 +0300 Subject: [PATCH 16/17] Added check for empty number of columns; improved test --- dbms/src/Interpreters/SyntaxAnalyzer.cpp | 5 +++++ .../00969_columns_clause.reference | 22 +++++++++++++++++++ .../0_stateless/00969_columns_clause.sql | 20 +++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index 62982ea1e59..83cacc94692 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -41,6 +41,7 @@ namespace ErrorCodes extern const int EMPTY_NESTED_TABLE; extern const int LOGICAL_ERROR; extern const int INVALID_JOIN_ON_EXPRESSION; + extern const int EMPTY_LIST_OF_COLUMNS_QUERIED; } NameSet removeDuplicateColumns(NamesAndTypesList & columns) @@ -110,6 +111,10 @@ void translateQualifiedNames(ASTPtr & query, const ASTSelectQuery & select_query TranslateQualifiedNamesVisitor::Data visitor_data(source_columns_set, tables_with_columns); TranslateQualifiedNamesVisitor visitor(visitor_data, log.stream()); visitor.visit(query); + + /// This may happen after expansion of COLUMNS('regexp'). + if (select_query.select()->children.empty()) + throw Exception("Empty list of columns in SELECT query", ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED); } bool hasArrayJoin(const ASTPtr & ast) diff --git a/dbms/tests/queries/0_stateless/00969_columns_clause.reference b/dbms/tests/queries/0_stateless/00969_columns_clause.reference index fcfd7d0919e..125a49fae0d 100644 --- a/dbms/tests/queries/0_stateless/00969_columns_clause.reference +++ b/dbms/tests/queries/0_stateless/00969_columns_clause.reference @@ -1,2 +1,24 @@ 100 10 120 8 +0 0 +1 1 +0 0 +1 1 +0 +1 +0 +1 +0 +1 +0 +1 +0 +1 +0 +1 +0 +2 +3 +2 +4 +3 diff --git a/dbms/tests/queries/0_stateless/00969_columns_clause.sql b/dbms/tests/queries/0_stateless/00969_columns_clause.sql index 1ea344f0bcd..fc5b72d3913 100644 --- a/dbms/tests/queries/0_stateless/00969_columns_clause.sql +++ b/dbms/tests/queries/0_stateless/00969_columns_clause.sql @@ -5,3 +5,23 @@ INSERT INTO ColumnsClauseTest VALUES (100, 10, 324), (120, 8, 23); SELECT COLUMNS('product.*') from ColumnsClauseTest ORDER BY product_price; DROP TABLE ColumnsClauseTest; + +SELECT number, COLUMNS('') FROM numbers(2); +SELECT number, COLUMNS('ber') FROM numbers(2); -- It works for unanchored regular expressions. +SELECT number, COLUMNS('x') FROM numbers(2); +SELECT COLUMNS('') FROM numbers(2); + +SELECT COLUMNS('x') FROM numbers(10) WHERE number > 5; -- { serverError 51 } + +SELECT * FROM numbers(2) WHERE NOT ignore(); +SELECT * FROM numbers(2) WHERE NOT ignore(*); +SELECT * FROM numbers(2) WHERE NOT ignore(COLUMNS('.+')); +SELECT * FROM numbers(2) WHERE NOT ignore(COLUMNS('x')); +SELECT COLUMNS('n') + COLUMNS('u') FROM system.numbers LIMIT 2; + +SELECT COLUMNS('n') + COLUMNS('u') FROM (SELECT 1 AS a, 2 AS b); -- { serverError 42 } +SELECT COLUMNS('a') + COLUMNS('b') FROM (SELECT 1 AS a, 2 AS b); +SELECT COLUMNS('a') + COLUMNS('a') FROM (SELECT 1 AS a, 2 AS b); +SELECT COLUMNS('b') + COLUMNS('b') FROM (SELECT 1 AS a, 2 AS b); +SELECT COLUMNS('a|b') + COLUMNS('b') FROM (SELECT 1 AS a, 2 AS b); -- { serverError 42 } +SELECT plus(COLUMNS('^(a|b)$')) FROM (SELECT 1 AS a, 2 AS b); From 9b5cba81748026ad183bcbd3ac99cf65cac15409 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Jul 2019 00:35:43 +0300 Subject: [PATCH 17/17] Fixed bad code --- dbms/tests/queries/0_stateless/00386_long_in_pk.python | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00386_long_in_pk.python b/dbms/tests/queries/0_stateless/00386_long_in_pk.python index d1db1a32d1e..247e0fe1e61 100644 --- a/dbms/tests/queries/0_stateless/00386_long_in_pk.python +++ b/dbms/tests/queries/0_stateless/00386_long_in_pk.python @@ -8,16 +8,16 @@ def gen_queries(): columns = tuple('a b c d'.split()) order_by_columns = tuple('a b c'.split()) partition_by_columns = tuple(' tuple() a'.split()) - + for partition in partition_by_columns: for key_mask in range(1, 1 << len(order_by_columns)): key = ','.join(order_by_columns[i] for i in range(len(order_by_columns)) if (1 << i) & key_mask != 0) create_query = create_template.format(key, partition) for q in (drop_query, create_query, insert_query): yield q - + for column, value in zip(columns, values): - yield 'select {} in {} from tab_00386'.format(column, value) + yield 'select {} in {} from tab_00386'.format(column, value) yield 'select {} in tuple({}) from tab_00386'.format(column, value) yield 'select {} in (select {} from tab_00386) from tab_00386'.format(column, column) @@ -26,7 +26,7 @@ def gen_queries(): yield 'select ({}, {}) in tuple({}, {}) from tab_00386'.format(columns[i], columns[j], values[i], values[j]) yield 'select ({}, {}) in (select {}, {} from tab_00386) from tab_00386'.format(columns[i], columns[j], columns[i], columns[j]) yield 'select ({}, {}) in (select ({}, {}) from tab_00386) from tab_00386'.format(columns[i], columns[j], columns[i], columns[j]) - + yield "select e in (1, 'a') from tab_00386" yield "select f in tuple((1, 'a')) from tab_00386" yield "select f in tuple(tuple((1, 'a'))) from tab_00386" @@ -41,7 +41,7 @@ import os def main(): url = os.environ['CLICKHOUSE_URL_PARAMS'] - + for q in gen_queries(): resp = requests.post(url, data=q) if resp.status_code != 200 or resp.text.strip() not in ('1', ''):