From f24bedd2b8e98d44aa8a5b49c22891cbf78e0e51 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 12 May 2019 14:36:02 +0300 Subject: [PATCH 001/133] Added constraints to parsers --- dbms/src/Parsers/ASTConstraintDeclaration.h | 38 ++++++++++++++ dbms/src/Parsers/ASTCreateQuery.cpp | 12 +++++ dbms/src/Parsers/ASTCreateQuery.h | 1 + dbms/src/Parsers/ParserCreateQuery.cpp | 55 ++++++++++++++++++--- dbms/src/Parsers/ParserCreateQuery.h | 22 +++++++-- 5 files changed, 118 insertions(+), 10 deletions(-) create mode 100644 dbms/src/Parsers/ASTConstraintDeclaration.h diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.h b/dbms/src/Parsers/ASTConstraintDeclaration.h new file mode 100644 index 00000000000..cfa4f8f98ab --- /dev/null +++ b/dbms/src/Parsers/ASTConstraintDeclaration.h @@ -0,0 +1,38 @@ +#pragma once + +#include + +namespace DB +{ + +/** name CHECK logical_expr + */ +class ASTConstraintDeclaration : public IAST { +public: + String name; + IAST *expr; + + String getID(char) const override { return "Constraint"; } + + ASTPtr clone() const override { + auto res = std::make_shared(); + + res->name = name; + + if (expr) + res->set(res->expr, expr->clone()); + + return res; + } + + void formatImpl(const FormatSettings &s, FormatState &state, FormatStateStacked frame) const override { + frame.need_parens = false; + std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); + + s.ostr << s.nl_or_ws << indent_str; + s.ostr << backQuoteIfNeed(name); + s.ostr << (s.hilite ? hilite_keyword : "") << " CHECK " << (s.hilite ? hilite_none : ""); + expr->formatImpl(s, state, frame); + } +}; +} diff --git a/dbms/src/Parsers/ASTCreateQuery.cpp b/dbms/src/Parsers/ASTCreateQuery.cpp index e99c543f5ec..b60eceb5167 100644 --- a/dbms/src/Parsers/ASTCreateQuery.cpp +++ b/dbms/src/Parsers/ASTCreateQuery.cpp @@ -128,6 +128,8 @@ ASTPtr ASTColumns::clone() const res->set(res->columns, columns->clone()); if (indices) res->set(res->indices, indices->clone()); + if (constraints) + res->set(res->constraints, constraints->clone()); return res; } @@ -156,6 +158,16 @@ void ASTColumns::formatImpl(const FormatSettings & s, FormatState & state, Forma list.children.push_back(elem); } } + if (constraints) + { + for (const auto & constraint : constraints->children) + { + auto elem = std::make_shared(); + elem->prefix = "CONSTRAINT"; + elem->set(elem->elem, constraint->clone()); + list.children.push_back(elem); + } + } if (!list.children.empty()) list.formatImpl(s, state, frame); diff --git a/dbms/src/Parsers/ASTCreateQuery.h b/dbms/src/Parsers/ASTCreateQuery.h index 2755e1a3d78..b6948c19146 100644 --- a/dbms/src/Parsers/ASTCreateQuery.h +++ b/dbms/src/Parsers/ASTCreateQuery.h @@ -36,6 +36,7 @@ class ASTColumns : public IAST public: ASTExpressionList * columns = nullptr; ASTExpressionList * indices = nullptr; + ASTExpressionList * constraints = nullptr; String getID(char) const override { return "Columns definition"; } diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index fd6665a5a2c..9f584dbbf8c 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -137,12 +138,41 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return true; } +bool ParserConstraintDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_check("CHECK"); -bool ParserColumnAndIndexDeclaraion::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) + ParserIdentifier name_p; + ParserLogicalOrExpression expression_p; + + ASTPtr name; + ASTPtr expr; + + if (!name_p.parse(pos, name, expected)) + return false; + + if (!s_check.ignore(pos, expected)) + return false; + + if (!expression_p.parse(pos, expr, expected)) + return false; + + auto constraint = std::make_shared(); + constraint->name = name->as().name; + constraint->set(constraint->expr, expr); + node = constraint; + + return true; +} + + +bool ParserTablePropertyDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_index("INDEX"); + ParserKeyword s_constraint("CONSTRAINT"); ParserIndexDeclaration index_p; + ParserConstraintDeclaration constraint_p; ParserColumnDeclaration column_p; ASTPtr new_node = nullptr; @@ -152,6 +182,11 @@ bool ParserColumnAndIndexDeclaraion::parseImpl(Pos & pos, ASTPtr & node, Expecte if (!index_p.parse(pos, new_node, expected)) return false; } + else if (s_constraint.ignore(pos, expected)) + { + if (!constraint_p.parse(pos, new_node, expected)) + return false; + } else { if (!column_p.parse(pos, new_node, expected)) @@ -168,16 +203,18 @@ bool ParserIndexDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & .parse(pos, node, expected); } - -bool ParserColumnsOrIndicesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr list; - if (!ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + if (!ParserList( + std::make_unique(), + std::make_unique(TokenType::Comma), false) .parse(pos, list, expected)) return false; ASTPtr columns = std::make_shared(); ASTPtr indices = std::make_shared(); + ASTPtr constraints = std::make_shared(); for (const auto & elem : list->children) { @@ -185,6 +222,8 @@ bool ParserColumnsOrIndicesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, columns->children.push_back(elem); else if (elem->as()) indices->children.push_back(elem); + else if (elem->as()) + constraints->children.push_back(elem); else return false; } @@ -195,6 +234,8 @@ bool ParserColumnsOrIndicesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, res->set(res->columns, columns); if (!indices->children.empty()) res->set(res->indices, indices); + if (!constraints->children.empty()) + res->set(res->constraints, constraints); node = res; @@ -317,7 +358,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserToken s_rparen(TokenType::ClosingRoundBracket); ParserStorage storage_p; ParserIdentifier name_p; - ParserColumnsOrIndicesDeclarationList columns_or_indices_p; + ParserTablePropertiesDeclarationList table_properties_p; ParserSelectWithUnionQuery select_p; ASTPtr database; @@ -391,7 +432,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) /// List of columns. if (s_lparen.ignore(pos, expected)) { - if (!columns_or_indices_p.parse(pos, columns_list, expected)) + if (!table_properties_p.parse(pos, columns_list, expected)) return false; if (!s_rparen.ignore(pos, expected)) @@ -498,7 +539,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) /// Optional - a list of columns can be specified. It must fully comply with SELECT. if (s_lparen.ignore(pos, expected)) { - if (!columns_or_indices_p.parse(pos, columns_list, expected)) + if (!table_properties_p.parse(pos, columns_list, expected)) return false; if (!s_rparen.ignore(pos, expected)) diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index bd3c8f671f0..42583d8dd19 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -244,11 +244,20 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserConstraintDeclaration : public IParserBase +{ +public: + ParserConstraintDeclaration() {} -class ParserColumnAndIndexDeclaraion : public IParserBase +protected: + const char * getName() const override { return "constraint declaration"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +class ParserTablePropertyDeclaration : public IParserBase { protected: - const char * getName() const override { return "column or index declaration"; } + const char * getName() const override { return "table propery (column, index, constraint) declaration"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -260,8 +269,15 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserConstraintDeclarationList : public IParserBase +{ +protected: + const char * getName() const override { return "constraint declaration list"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; -class ParserColumnsOrIndicesDeclarationList : public IParserBase + +class ParserTablePropertiesDeclarationList : public IParserBase { protected: const char * getName() const override { return "columns or indices declaration list"; } From 502d86bd022bc53f8f4f9452cd4d1201e849aa4c Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 07:08:03 +0300 Subject: [PATCH 002/133] Added constraints to InterpreterCreateQuery and storages --- .../Interpreters/InterpreterCreateQuery.cpp | 23 +++++++++++ .../src/Interpreters/InterpreterCreateQuery.h | 2 + dbms/src/Storages/ConstraintsDescription.cpp | 39 +++++++++++++++++++ dbms/src/Storages/ConstraintsDescription.h | 23 +++++++++++ dbms/src/Storages/ITableDeclaration.cpp | 5 +++ dbms/src/Storages/ITableDeclaration.h | 5 +++ dbms/src/Storages/StorageMergeTree.h | 3 ++ 7 files changed, 100 insertions(+) create mode 100644 dbms/src/Storages/ConstraintsDescription.cpp create mode 100644 dbms/src/Storages/ConstraintsDescription.h diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 4bc35f1e378..e4e5a8e7f83 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -252,6 +252,16 @@ ASTPtr InterpreterCreateQuery::formatIndices(const IndicesDescription & indices) return res; } +ASTPtr InterpreterCreateQuery::formatConstraints(const ConstraintsDescription & constraints) +{ + auto res = std::make_shared(); + + for (const auto & constraint : constraints.constraints) + res->children.push_back(constraint->clone()); + + return res; +} + ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns_ast, const Context & context) { /// First, deduce implicit types. @@ -370,6 +380,8 @@ ColumnsDescription InterpreterCreateQuery::setColumns( { ColumnsDescription columns; IndicesDescription indices; + ConstraintsDescription constraints; + if (create.columns_list) { @@ -379,11 +391,16 @@ ColumnsDescription InterpreterCreateQuery::setColumns( for (const auto & index : create.columns_list->indices->children) indices.indices.push_back( std::dynamic_pointer_cast(index->clone())); + if (create.columns_list->constraints) + for (const auto & constraint : create.columns_list->constraints->children) + constraints.constraints.push_back( + std::dynamic_pointer_cast(constraint->clone())); } else if (!create.as_table.empty()) { columns = as_storage->getColumns(); indices = as_storage->getIndicesDescription(); + constraints = as_storage->getConstraintsDescription(); } else if (create.select) { @@ -395,6 +412,7 @@ ColumnsDescription InterpreterCreateQuery::setColumns( /// Even if query has list of columns, canonicalize it (unfold Nested columns). ASTPtr new_columns = formatColumns(columns); ASTPtr new_indices = formatIndices(indices); + ASTPtr new_constraints = formatConstraints(constraints); if (!create.columns_list) { @@ -412,6 +430,11 @@ ColumnsDescription InterpreterCreateQuery::setColumns( else if (new_indices) create.columns_list->set(create.columns_list->indices, new_indices); + if (new_constraints && create.columns_list->constraints) + create.columns_list->replace(create.columns_list->constraints, new_constraints); + else if (new_constraints) + create.columns_list->set(create.columns_list->constraints, new_constraints); + /// Check for duplicates std::set all_columns; for (const auto & column : columns) diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.h b/dbms/src/Interpreters/InterpreterCreateQuery.h index 2f124e7df9b..a7886f644ad 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.h +++ b/dbms/src/Interpreters/InterpreterCreateQuery.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -31,6 +32,7 @@ public: static ASTPtr formatColumns(const ColumnsDescription & columns); static ASTPtr formatIndices(const IndicesDescription & indices); + static ASTPtr formatConstraints(const ConstraintsDescription & constraints); void setDatabaseLoadingThreadpool(ThreadPool & thread_pool_) { diff --git a/dbms/src/Storages/ConstraintsDescription.cpp b/dbms/src/Storages/ConstraintsDescription.cpp new file mode 100644 index 00000000000..042ee06ff59 --- /dev/null +++ b/dbms/src/Storages/ConstraintsDescription.cpp @@ -0,0 +1,39 @@ +#include + +#include +#include +#include +#include + + +namespace DB +{ + +String ConstraintsDescription::toString() const +{ + if (constraints.empty()) + return {}; + + ASTExpressionList list; + for (const auto & constraint : constraints) + list.children.push_back(constraint); + + return serializeAST(list, true); +} + +ConstraintsDescription ConstraintsDescription::parse(const String & str) +{ + if (str.empty()) + return {}; + + ConstraintsDescription res; + ParserConstraintDeclarationList parser; + ASTPtr list = parseQuery(parser, str, 0); + + for (const auto & constraint : list->children) + res.constraints.push_back(std::dynamic_pointer_cast(constraint)); + + return res; +} + +} diff --git a/dbms/src/Storages/ConstraintsDescription.h b/dbms/src/Storages/ConstraintsDescription.h new file mode 100644 index 00000000000..c2954d94428 --- /dev/null +++ b/dbms/src/Storages/ConstraintsDescription.h @@ -0,0 +1,23 @@ +#pragma once + +#include + + +namespace DB +{ + +using ConstraintsASTs = std::vector>; + +struct ConstraintsDescription +{ + ConstraintsASTs constraints; + + ConstraintsDescription() = default; + + bool empty() const { return constraints.empty(); } + String toString() const; + + static ConstraintsDescription parse(const String & str); +}; + +} diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index c9385c24cbe..b24072f0de8 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -36,6 +36,11 @@ void ITableDeclaration::setIndicesDescription(IndicesDescription indices_) indices = std::move(indices_); } +void ITableDeclaration::setConstraintsDescription(ConstraintsDescription constraints_) +{ + constraints = std::move(constraints_); +} + bool ITableDeclaration::hasColumn(const String & column_name) const { diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h index e2ac3b1d6c4..e1e6d88a68f 100644 --- a/dbms/src/Storages/ITableDeclaration.h +++ b/dbms/src/Storages/ITableDeclaration.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB @@ -19,6 +20,9 @@ public: virtual const IndicesDescription & getIndicesDescription() const { return indices; } virtual void setIndicesDescription(IndicesDescription indices_); + virtual const ConstraintsDescription & getConstraintsDescription() const { return constraints; } + virtual void setConstraintsDescription(ConstraintsDescription constraints_); + /// NOTE: These methods should include virtual columns, but should NOT include ALIAS columns /// (they are treated separately). virtual NameAndTypePair getColumn(const String & column_name) const; @@ -57,6 +61,7 @@ public: private: ColumnsDescription columns; IndicesDescription indices; + ConstraintsDescription constraints; }; } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index c5ea2d8c3a0..0b4b00b7cb7 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -46,6 +46,9 @@ public: virtual const IndicesDescription & getIndicesDescription() const override { return data.getIndicesDescription(); } virtual void setIndicesDescription(IndicesDescription indices_) override { data.setIndicesDescription(std::move(indices_)); } + virtual const ConstraintsDescription & getConstraintsDescription() const override { return data.getConstraintsDescription(); } + virtual void setConstraintsDescription(ConstraintsDescription constraints_) override { data.setConstraintsDescription(constraints_; )} + NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); } bool hasColumn(const String & column_name) const override { return data.hasColumn(column_name); } From 2f8864a6ea96f124af79914fd351fe998d47b538 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 07:14:13 +0300 Subject: [PATCH 003/133] Some style fixes --- dbms/src/Parsers/ASTConstraintDeclaration.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.h b/dbms/src/Parsers/ASTConstraintDeclaration.h index cfa4f8f98ab..d72358be498 100644 --- a/dbms/src/Parsers/ASTConstraintDeclaration.h +++ b/dbms/src/Parsers/ASTConstraintDeclaration.h @@ -7,14 +7,16 @@ namespace DB /** name CHECK logical_expr */ -class ASTConstraintDeclaration : public IAST { +class ASTConstraintDeclaration : public IAST +{ public: String name; IAST *expr; String getID(char) const override { return "Constraint"; } - ASTPtr clone() const override { + ASTPtr clone() const override + { auto res = std::make_shared(); res->name = name; @@ -25,7 +27,8 @@ public: return res; } - void formatImpl(const FormatSettings &s, FormatState &state, FormatStateStacked frame) const override { + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override + { frame.need_parens = false; std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); From 72d5a4634c10d4d1e846867c22da3061fe20417e Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 07:27:09 +0300 Subject: [PATCH 004/133] Syntax fix --- dbms/src/Storages/StorageMergeTree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 0b4b00b7cb7..184db1a858b 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -47,7 +47,7 @@ public: virtual void setIndicesDescription(IndicesDescription indices_) override { data.setIndicesDescription(std::move(indices_)); } virtual const ConstraintsDescription & getConstraintsDescription() const override { return data.getConstraintsDescription(); } - virtual void setConstraintsDescription(ConstraintsDescription constraints_) override { data.setConstraintsDescription(constraints_; )} + virtual void setConstraintsDescription(ConstraintsDescription constraints_) override { data.setConstraintsDescription(constraints_); } NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); } bool hasColumn(const String & column_name) const override { return data.hasColumn(column_name); } From 07abed6d31b2a024527f34601c457de40bdcc220 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 08:05:19 +0300 Subject: [PATCH 005/133] Renamed constraints getter and setter --- dbms/src/Interpreters/InterpreterCreateQuery.cpp | 2 +- dbms/src/Storages/ITableDeclaration.cpp | 2 +- dbms/src/Storages/ITableDeclaration.h | 4 ++-- dbms/src/Storages/StorageMergeTree.h | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index e9027f473e5..15968d58ac5 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -400,7 +400,7 @@ ColumnsDescription InterpreterCreateQuery::setColumns( { columns = as_storage->getColumns(); indices = as_storage->getIndices(); - constraints = as_storage->getConstraintsDescription(); + constraints = as_storage->getConstraints(); } else if (create.select) { diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index cb9ba9daf9a..47d24300452 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -46,7 +46,7 @@ void ITableDeclaration::setIndices(IndicesDescription indices_) indices = std::move(indices_); } -void ITableDeclaration::setConstraintsDescription(ConstraintsDescription constraints_) +void ITableDeclaration::setConstraints(ConstraintsDescription constraints_) { constraints = std::move(constraints_); } diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h index 6d50f8acc0f..52db660b19b 100644 --- a/dbms/src/Storages/ITableDeclaration.h +++ b/dbms/src/Storages/ITableDeclaration.h @@ -20,8 +20,8 @@ public: const IndicesDescription & getIndices() const; void setIndices(IndicesDescription indices_); - virtual const ConstraintsDescription & getConstraintsDescription() const { return constraints; } - virtual void setConstraintsDescription(ConstraintsDescription constraints_); + virtual const ConstraintsDescription & getConstraints() const { return constraints; } + virtual void setConstraints(ConstraintsDescription constraints_); /// NOTE: These methods should include virtual columns, but should NOT include ALIAS columns /// (they are treated separately). diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 2c42bbd35c5..18386d78b8c 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -32,8 +32,8 @@ public: std::string getDatabaseName() const override { return database_name; } bool supportsIndexForIn() const override { return true; } - virtual const ConstraintsDescription & getConstraintsDescription() const override { return data.getConstraintsDescription(); } - virtual void setConstraintsDescription(ConstraintsDescription constraints_) override { data.setConstraintsDescription(constraints_); } + virtual const ConstraintsDescription & getConstraints() const override { return data.getConstraints(); } + virtual void setConstraints(ConstraintsDescription constraints_) override { data.setConstraints(constraints_); } BlockInputStreams read( const Names & column_names, From 7919a62198ce97493f1f151914ea69765423037c Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 17 May 2019 09:21:29 +0300 Subject: [PATCH 006/133] Removed constraints getter and setter from StorageMergeTree --- dbms/src/Storages/StorageMergeTree.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 18386d78b8c..b5156ce7137 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -32,8 +32,6 @@ public: std::string getDatabaseName() const override { return database_name; } bool supportsIndexForIn() const override { return true; } - virtual const ConstraintsDescription & getConstraints() const override { return data.getConstraints(); } - virtual void setConstraints(ConstraintsDescription constraints_) override { data.setConstraints(constraints_); } BlockInputStreams read( const Names & column_names, From 1f5715b985125d8820d06c5a23a98d290d278cdb Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 18 May 2019 08:16:33 +0300 Subject: [PATCH 007/133] Removed double whitespace --- dbms/src/Parsers/ParserCreateQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index 9f584dbbf8c..76150b95b07 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -143,7 +143,7 @@ bool ParserConstraintDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & ParserKeyword s_check("CHECK"); ParserIdentifier name_p; - ParserLogicalOrExpression expression_p; + ParserLogicalOrExpression expression_p; ASTPtr name; ASTPtr expr; From ce9389660853759cb7566c7bf2a932894ccd06de Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 18 May 2019 11:05:52 +0300 Subject: [PATCH 008/133] Added constraints description to MergeTree and related storages --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 ++ dbms/src/Storages/MergeTree/MergeTreeData.h | 1 + .../Storages/MergeTree/registerStorageMergeTree.cpp | 10 +++++++--- dbms/src/Storages/StorageMergeTree.cpp | 3 ++- dbms/src/Storages/StorageMergeTree.h | 1 + dbms/src/Storages/StorageReplicatedMergeTree.cpp | 3 ++- dbms/src/Storages/StorageReplicatedMergeTree.h | 1 + 7 files changed, 16 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ea51159d9ba..4f09547a578 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -93,6 +93,7 @@ MergeTreeData::MergeTreeData( const String & database_, const String & table_, const String & full_path_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, Context & context_, const String & date_column_name, const ASTPtr & partition_by_ast_, @@ -121,6 +122,7 @@ MergeTreeData::MergeTreeData( data_parts_by_state_and_info(data_parts_indexes.get()) { setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_); + setConstraints(constraints_); /// NOTE: using the same columns list as is read when performing actual merges. merging_params.check(getColumns().getAllPhysical()); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index fecddb28540..294588721ae 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -332,6 +332,7 @@ public: const String & full_path_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, Context & context_, const String & date_column_name, const ASTPtr & partition_by_ast_, diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index b23a2eedc0e..b255f16c327 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -574,6 +574,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) ASTPtr sample_by_ast; ASTPtr ttl_table_ast; IndicesDescription indices_description; + ConstraintsDescription constraints_description; MergeTreeSettings storage_settings = args.context.getMergeTreeSettings(); if (is_extended_storage_def) @@ -602,7 +603,10 @@ static StoragePtr create(const StorageFactory::Arguments & args) indices_description.indices.push_back( std::dynamic_pointer_cast(index->clone())); - + if (args.query.columns_list && args.query.columns_list->constraints) + for (const auto & constraint : args.query.columns_list->constraints->children) + constraints_description.constraints.push_back( + std::dynamic_pointer_cast(constraint->clone())); storage_settings.loadFromQuery(*args.storage_def); } else @@ -639,14 +643,14 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (replicated) return StorageReplicatedMergeTree::create( zookeeper_path, replica_name, args.attach, args.data_path, args.database_name, args.table_name, - args.columns, indices_description, + args.columns, indices_description, constraints_description, args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, storage_settings, args.has_force_restore_data_flag); else return StorageMergeTree::create( args.data_path, args.database_name, args.table_name, args.columns, indices_description, - args.attach, args.context, date_column_name, partition_by_ast, order_by_ast, + constraints_description, args.attach, args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, storage_settings, args.has_force_restore_data_flag); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 8feb2d1fe81..12e2ffc77af 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -53,6 +53,7 @@ StorageMergeTree::StorageMergeTree( const String & table_name_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, bool attach, Context & context_, const String & date_column_name, @@ -66,7 +67,7 @@ StorageMergeTree::StorageMergeTree( bool has_force_restore_data_flag) : MergeTreeData(database_name_, table_name_, path_ + escapeForFileName(table_name_) + '/', - columns_, indices_, + columns_, indices_, constraints_, context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_, sample_by_ast_, ttl_table_ast_, merging_params_, settings_, false, attach), diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index b5156ce7137..74e0d85bfb8 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -139,6 +139,7 @@ protected: const String & table_name_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, bool attach, Context & context_, const String & date_column_name, diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index daa1b0d10e2..8d467418f12 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -199,6 +199,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const String & table_name_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, Context & context_, const String & date_column_name, const ASTPtr & partition_by_ast_, @@ -211,7 +212,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( bool has_force_restore_data_flag) : MergeTreeData(database_name_, table_name_, path_ + escapeForFileName(table_name_) + '/', - columns_, indices_, + columns_, indices_, constraints_, context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_, sample_by_ast_, ttl_table_ast_, merging_params_, settings_, true, attach, diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index eba0511e15e..cd7d043a54a 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -513,6 +513,7 @@ protected: const String & path_, const String & database_name_, const String & name_, const ColumnsDescription & columns_, const IndicesDescription & indices_, + const ConstraintsDescription & constraints_, Context & context_, const String & date_column_name, const ASTPtr & partition_by_ast_, From 300ec160f40fffb48bf160fe82a11781819ba30e Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 08:27:00 +0300 Subject: [PATCH 009/133] Constraints MVP --- .../CheckConstraintsBlockOutputStream.cpp | 43 +++++++++++++++++ .../CheckConstraintsBlockOutputStream.h | 48 +++++++++++++++++++ .../Interpreters/InterpreterInsertQuery.cpp | 4 ++ dbms/src/Storages/ConstraintsDescription.h | 14 +++++- 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp create mode 100644 dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp new file mode 100644 index 00000000000..99f9f9bc90d --- /dev/null +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -0,0 +1,43 @@ +#include + + +namespace DB +{ + +void CheckConstraintsBlockOutputStream::write(const Block & block) +{ + for (auto & constraint_expr: expressions) + if (!checkConstraintOnBlock(block, constraint_expr)) + throw Exception("Some constraints are not satisfied", ErrorCodes::QUERY_WAS_CANCELLED); + output->write(block); +} + +void CheckConstraintsBlockOutputStream::flush() +{ + output->flush(); +} + +void CheckConstraintsBlockOutputStream::writePrefix() +{ + output->writePrefix(); +} + +void CheckConstraintsBlockOutputStream::writeSuffix() +{ + output->writeSuffix(); +} + +bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) +{ + Block res = block; + constraint->execute(res); + assert(block.columns() == res.columns() - 1); + ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); + size_t column_size = res_column.column->size(); + for (size_t i = 0; i < column_size; ++i) + if (!res_column.column->getBool(i)) + return false; + return true; +} + +} diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h new file mode 100644 index 00000000000..623eccc8172 --- /dev/null +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CONSTRAINTS_ARE_NOT_SATISFIED; +} + +class CheckConstraintsBlockOutputStream : public IBlockOutputStream +{ +public: + CheckConstraintsBlockOutputStream( + const BlockOutputStreamPtr & output_, + const Block & header_, + const ConstraintsDescription & constraints_, + const Context & context_) + : output(output_), + header(header_), + constraints(constraints_), + expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())), + context(context_) + { } + + Block getHeader() const override { return header; } + void write(const Block & block) override; + + void flush() override; + + void writePrefix() override; + void writeSuffix() override; + + bool checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint); + +private: + BlockOutputStreamPtr output; + Block header; + const ConstraintsDescription constraints; + const ConstraintsExpressions expressions; + const Context & context; +}; +} diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index e4391f52247..fa6df1599ea 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -117,6 +118,9 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared( out, query_sample_block, table->getSampleBlock(), table->getColumns().getDefaults(), context); + out = std::make_shared( + out, query_sample_block, table->getConstraints(), context); + auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); out = std::move(out_wrapper); diff --git a/dbms/src/Storages/ConstraintsDescription.h b/dbms/src/Storages/ConstraintsDescription.h index c2954d94428..fbb0f5167fc 100644 --- a/dbms/src/Storages/ConstraintsDescription.h +++ b/dbms/src/Storages/ConstraintsDescription.h @@ -1,12 +1,13 @@ #pragma once #include - +#include namespace DB { using ConstraintsASTs = std::vector>; +using ConstraintsExpressions = std::vector; struct ConstraintsDescription { @@ -18,6 +19,17 @@ struct ConstraintsDescription String toString() const; static ConstraintsDescription parse(const String & str); + + ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const { + ConstraintsExpressions res; + res.reserve(constraints.size()); + for (const auto & constraint : constraints) { + ASTPtr expr = constraint->expr->clone(); + auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); + res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); + } + return res; + } }; } From 60a5b94ba47ee4050dbb5d38822ceb2d15a479f2 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 08:44:31 +0300 Subject: [PATCH 010/133] Style fix --- dbms/src/Storages/ConstraintsDescription.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/ConstraintsDescription.h b/dbms/src/Storages/ConstraintsDescription.h index fbb0f5167fc..0f565379204 100644 --- a/dbms/src/Storages/ConstraintsDescription.h +++ b/dbms/src/Storages/ConstraintsDescription.h @@ -20,10 +20,12 @@ struct ConstraintsDescription static ConstraintsDescription parse(const String & str); - ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const { + ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const + { ConstraintsExpressions res; res.reserve(constraints.size()); - for (const auto & constraint : constraints) { + for (const auto & constraint : constraints) + { ASTPtr expr = constraint->expr->clone(); auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); From 19d099f90a147fc385721a525a39249bfb682304 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 09:03:18 +0300 Subject: [PATCH 011/133] Removed ITableDeclaration --- dbms/src/Storages/ITableDeclaration.cpp | 0 dbms/src/Storages/ITableDeclaration.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dbms/src/Storages/ITableDeclaration.cpp delete mode 100644 dbms/src/Storages/ITableDeclaration.h diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h deleted file mode 100644 index e69de29bb2d..00000000000 From 773849a43d9ab25df77c38e00de5d6b914c0b0f2 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 09:08:25 +0300 Subject: [PATCH 012/133] Added constraints getter and setter to IStorage --- dbms/src/Storages/IStorage.cpp | 10 ++++++++++ dbms/src/Storages/IStorage.h | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 06320cc1f30..7c19fd94aea 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -47,6 +47,16 @@ void IStorage::setIndices(IndicesDescription indices_) indices = std::move(indices_); } +const ConstraintsDescription & IStorage::getConstraints() const +{ + return constraints; +} + +void IStorage::setConstraints(ConstraintsDescription constraints_) +{ + constraints = std::move(constraints_); +} + NameAndTypePair IStorage::getColumn(const String & column_name) const { /// By default, we assume that there are no virtual columns in the storage. diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index f18592ebce5..b01244ba111 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -88,6 +89,9 @@ public: /// thread-unsafe part. lockStructure must be acquired const IndicesDescription & getIndices() const; void setIndices(IndicesDescription indices_); + const ConstraintsDescription & getConstraints() const; + void setConstraints(ConstraintsDescription constraints_); + /// NOTE: these methods should include virtual columns, /// but should NOT include ALIAS columns (they are treated separately). virtual NameAndTypePair getColumn(const String & column_name) const; @@ -115,6 +119,7 @@ public: /// thread-unsafe part. lockStructure must be acquired private: ColumnsDescription columns; IndicesDescription indices; + ConstraintsDescription constraints; public: /// Acquire this lock if you need the table structure to remain constant during the execution of From d1492fc05d534034d1a0ed1322a9e86f1020cc62 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 10:17:06 +0300 Subject: [PATCH 013/133] Removed context from CheckConstraintsBlockOutputStream --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h | 7 ++----- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index 623eccc8172..e1e15f8e454 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -19,13 +19,11 @@ public: CheckConstraintsBlockOutputStream( const BlockOutputStreamPtr & output_, const Block & header_, - const ConstraintsDescription & constraints_, - const Context & context_) + const ConstraintsDescription & constraints_) : output(output_), header(header_), constraints(constraints_), - expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())), - context(context_) + expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())) { } Block getHeader() const override { return header; } @@ -43,6 +41,5 @@ private: Block header; const ConstraintsDescription constraints; const ConstraintsExpressions expressions; - const Context & context; }; } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index fa6df1599ea..d47dd978b3a 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -118,8 +118,7 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared( out, query_sample_block, table->getSampleBlock(), table->getColumns().getDefaults(), context); - out = std::make_shared( - out, query_sample_block, table->getConstraints(), context); + out = std::make_shared(out, query_sample_block, table->getConstraints()); auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); From 9e6625441c936a4fef236c34cefa0b0654645364 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 19 May 2019 10:19:44 +0300 Subject: [PATCH 014/133] Returned context to CheckConstraintsBlockOutputStream constructor --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h | 3 ++- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index e1e15f8e454..16b240eb758 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -19,7 +19,8 @@ public: CheckConstraintsBlockOutputStream( const BlockOutputStreamPtr & output_, const Block & header_, - const ConstraintsDescription & constraints_) + const ConstraintsDescription & constraints_, + const Context & context_) : output(output_), header(header_), constraints(constraints_), diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index d47dd978b3a..fa6df1599ea 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -118,7 +118,8 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared( out, query_sample_block, table->getSampleBlock(), table->getColumns().getDefaults(), context); - out = std::make_shared(out, query_sample_block, table->getConstraints()); + out = std::make_shared( + out, query_sample_block, table->getConstraints(), context); auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); From c926cf65627dfbbc724b8b469c254399e6fb8486 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 25 May 2019 17:07:45 +0300 Subject: [PATCH 015/133] Minor review fixes --- dbms/src/Parsers/ASTConstraintDeclaration.cpp | 28 +++++++++++++++++++ dbms/src/Parsers/ASTConstraintDeclaration.h | 25 ++--------------- dbms/src/Parsers/ParserCreateQuery.cpp | 6 ++++ dbms/src/Parsers/ParserCreateQuery.h | 3 -- dbms/src/Storages/ConstraintsDescription.cpp | 15 ++++++++++ dbms/src/Storages/ConstraintsDescription.h | 13 +-------- 6 files changed, 53 insertions(+), 37 deletions(-) create mode 100644 dbms/src/Parsers/ASTConstraintDeclaration.cpp diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.cpp b/dbms/src/Parsers/ASTConstraintDeclaration.cpp new file mode 100644 index 00000000000..834ac81891b --- /dev/null +++ b/dbms/src/Parsers/ASTConstraintDeclaration.cpp @@ -0,0 +1,28 @@ +#include + +namespace DB { + +ASTPtr ASTConstraintDeclaration::clone() const +{ + auto res = std::make_shared(); + + res->name = name; + + if (expr) + res->set(res->expr, expr->clone()); + + return res; +} + +void ASTConstraintDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +{ + frame.need_parens = false; + std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); + + s.ostr << s.nl_or_ws << indent_str; + s.ostr << backQuoteIfNeed(name); + s.ostr << (s.hilite ? hilite_keyword : "") << " CHECK " << (s.hilite ? hilite_none : ""); + expr->formatImpl(s, state, frame); +} + +} \ No newline at end of file diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.h b/dbms/src/Parsers/ASTConstraintDeclaration.h index d72358be498..3a8ad75f54b 100644 --- a/dbms/src/Parsers/ASTConstraintDeclaration.h +++ b/dbms/src/Parsers/ASTConstraintDeclaration.h @@ -11,31 +11,12 @@ class ASTConstraintDeclaration : public IAST { public: String name; - IAST *expr; + IAST * expr; String getID(char) const override { return "Constraint"; } - ASTPtr clone() const override - { - auto res = std::make_shared(); + ASTPtr clone() const override; - res->name = name; - - if (expr) - res->set(res->expr, expr->clone()); - - return res; - } - - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override - { - frame.need_parens = false; - std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); - - s.ostr << s.nl_or_ws << indent_str; - s.ostr << backQuoteIfNeed(name); - s.ostr << (s.hilite ? hilite_keyword : "") << " CHECK " << (s.hilite ? hilite_none : ""); - expr->formatImpl(s, state, frame); - } + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index 76150b95b07..c828cd0d780 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -203,6 +203,12 @@ bool ParserIndexDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & .parse(pos, node, expected); } +bool ParserConstraintDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + .parse(pos, node, expected); +} + bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr list; diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index 42583d8dd19..bc921773605 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -246,9 +246,6 @@ protected: class ParserConstraintDeclaration : public IParserBase { -public: - ParserConstraintDeclaration() {} - protected: const char * getName() const override { return "constraint declaration"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; diff --git a/dbms/src/Storages/ConstraintsDescription.cpp b/dbms/src/Storages/ConstraintsDescription.cpp index 042ee06ff59..ad0cd76733a 100644 --- a/dbms/src/Storages/ConstraintsDescription.cpp +++ b/dbms/src/Storages/ConstraintsDescription.cpp @@ -36,4 +36,19 @@ ConstraintsDescription ConstraintsDescription::parse(const String & str) return res; } +ConstraintsExpressions ConstraintsDescription::getExpressions(const DB::Context & context, + const DB::NamesAndTypesList & source_columns_) const +{ + ConstraintsExpressions res; + res.reserve(constraints.size()); + for (const auto & constraint : constraints) + { + // SyntaxAnalyzer::analyze has query as non-const argument so to avoid accidental query changes we clone it + ASTPtr expr = constraint->expr->clone(); + auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); + res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); + } + return res; +} + } diff --git a/dbms/src/Storages/ConstraintsDescription.h b/dbms/src/Storages/ConstraintsDescription.h index 0f565379204..3ced0e8ddc9 100644 --- a/dbms/src/Storages/ConstraintsDescription.h +++ b/dbms/src/Storages/ConstraintsDescription.h @@ -20,18 +20,7 @@ struct ConstraintsDescription static ConstraintsDescription parse(const String & str); - ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const - { - ConstraintsExpressions res; - res.reserve(constraints.size()); - for (const auto & constraint : constraints) - { - ASTPtr expr = constraint->expr->clone(); - auto syntax_result = SyntaxAnalyzer(context).analyze(expr, source_columns_); - res.push_back(ExpressionAnalyzer(constraint->expr->clone(), syntax_result, context).getActions(false)); - } - return res; - } + ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const; }; } From 94db808cd7f73c9ad5ec3dfcd2f348061c38b8e3 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 25 May 2019 18:54:49 +0300 Subject: [PATCH 016/133] Basic test for constraints parsers and failures --- .../0_stateless/00951_basic_constraints.sh | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00951_basic_constraints.sh diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh new file mode 100644 index 00000000000..43bf274a82b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh @@ -0,0 +1,45 @@ +exec 2>&1 + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT b_constraint CHECK b > 0 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw and exception +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" + +# Test two constraints on one table +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT b_constraint CHECK b > 10, + CONSTRAINT a_constraint CHECK a < 10 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must throw an exception +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw an exception +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (7, 18), (0, 11);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" \ No newline at end of file From a3535c69b23aff266dcbdc8ed856024efff7fa69 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 25 May 2019 18:57:35 +0300 Subject: [PATCH 017/133] Minor style fix --- dbms/src/Parsers/ASTConstraintDeclaration.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Parsers/ASTConstraintDeclaration.cpp b/dbms/src/Parsers/ASTConstraintDeclaration.cpp index 834ac81891b..a1b063fc44a 100644 --- a/dbms/src/Parsers/ASTConstraintDeclaration.cpp +++ b/dbms/src/Parsers/ASTConstraintDeclaration.cpp @@ -1,6 +1,7 @@ #include -namespace DB { +namespace DB +{ ASTPtr ASTConstraintDeclaration::clone() const { @@ -25,4 +26,4 @@ void ASTConstraintDeclaration::formatImpl(const FormatSettings & s, FormatState expr->formatImpl(s, state, frame); } -} \ No newline at end of file +} From c185f5741c116f6038979e56f220e8f2b632ab2a Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 25 May 2019 21:13:43 +0300 Subject: [PATCH 018/133] Fixed test --- dbms/tests/queries/0_stateless/00951_basic_constraints.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh index 43bf274a82b..cf622293688 100644 --- a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + exec 2>&1 $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" From b0e3315ac7855bae85dbb8adc21b826e971f6ce0 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 26 May 2019 08:32:22 +0300 Subject: [PATCH 019/133] Added tests reference --- .../0_stateless/00951_basic_constraints.reference | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00951_basic_constraints.reference diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.reference b/dbms/tests/queries/0_stateless/00951_basic_constraints.reference new file mode 100644 index 00000000000..28bf65f8d48 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00951_basic_constraints.reference @@ -0,0 +1,10 @@ +1 2 +Received exception from server (version 19.8.1): +Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +1 2 +Received exception from server (version 19.8.1): +Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +Received exception from server (version 19.8.1): +Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +0 11 +7 18 From f87bb846586a848d5c24a793d8474783008f4cd8 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 26 May 2019 11:43:45 +0300 Subject: [PATCH 020/133] Added curdir to 00951 test --- dbms/tests/queries/0_stateless/00951_basic_constraints.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh index cf622293688..49bf6771ab4 100644 --- a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00951_basic_constraints.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + exec 2>&1 $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" From 94e7521beae08dafcc700bf8c9991f786a6b39ec Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 26 May 2019 12:37:35 +0300 Subject: [PATCH 021/133] Changed number of tests --- ...ic_constraints.reference => 00952_basic_constraints.reference} | 0 .../{00951_basic_constraints.sh => 00952_basic_constraints.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename dbms/tests/queries/0_stateless/{00951_basic_constraints.reference => 00952_basic_constraints.reference} (100%) rename dbms/tests/queries/0_stateless/{00951_basic_constraints.sh => 00952_basic_constraints.sh} (100%) diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.reference b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00951_basic_constraints.reference rename to dbms/tests/queries/0_stateless/00952_basic_constraints.reference diff --git a/dbms/tests/queries/0_stateless/00951_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh similarity index 100% rename from dbms/tests/queries/0_stateless/00951_basic_constraints.sh rename to dbms/tests/queries/0_stateless/00952_basic_constraints.sh From af50b1ff5cef7bea4c35d25067c914174acc7b89 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 26 May 2019 14:08:37 +0300 Subject: [PATCH 022/133] Made 00952 test executable --- dbms/tests/queries/0_stateless/00952_basic_constraints.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 dbms/tests/queries/0_stateless/00952_basic_constraints.sh diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh old mode 100644 new mode 100755 From 1aaab0745969277b9e9fab643d10832a571ff469 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 27 May 2019 02:57:18 +0300 Subject: [PATCH 023/133] Fixed exception handling in 00952 test --- .../CheckConstraintsBlockOutputStream.cpp | 2 +- .../0_stateless/00952_basic_constraints.reference | 9 +++------ .../queries/0_stateless/00952_basic_constraints.sh | 14 +++++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 99f9f9bc90d..1cc271e5578 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -8,7 +8,7 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { for (auto & constraint_expr: expressions) if (!checkConstraintOnBlock(block, constraint_expr)) - throw Exception("Some constraints are not satisfied", ErrorCodes::QUERY_WAS_CANCELLED); + throw Exception{"Some constraints are not satisfied", ErrorCodes::QUERY_WAS_CANCELLED}; output->write(block); } diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference index 28bf65f8d48..1bede18351d 100644 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference @@ -1,10 +1,7 @@ 1 2 -Received exception from server (version 19.8.1): -Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +Exception ok 1 2 -Received exception from server (version 19.8.1): -Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. -Received exception from server (version 19.8.1): -Code: 394. DB::Exception: Received from localhost:9001, ::1. DB::Exception: Some constraints are not satisfied. +Exception ok +Exception ok 0 11 7 18 diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index 49bf6771ab4..b214982da5e 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -3,8 +3,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -exec 2>&1 - $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints @@ -20,7 +18,9 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" +EXCEPTION_TEXT="Some constraints are not satisfied" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" $CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" @@ -36,11 +36,15 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +EXCEPTION_TEXT="Some constraints are not satisfied" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" +EXCEPTION_TEXT="Some constraints are not satisfied" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must succeed From ff6cdaeb9846ae63ddba399e215809f0be641793 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 27 May 2019 09:30:18 +0300 Subject: [PATCH 024/133] Removed word "exception" from test reference --- .../queries/0_stateless/00952_basic_constraints.reference | 6 +++--- dbms/tests/queries/0_stateless/00952_basic_constraints.sh | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference index 1bede18351d..4d98efd8939 100644 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.reference +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.reference @@ -1,7 +1,7 @@ 1 2 -Exception ok +ok 1 2 -Exception ok -Exception ok +ok +ok 0 11 7 18 diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index b214982da5e..93fa16ce4af 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -3,6 +3,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +EXCEPTION_SUCCESS_TEXT=ok + $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints @@ -20,7 +22,7 @@ $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception EXCEPTION_TEXT="Some constraints are not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" 2>&1 \ - | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" $CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" @@ -38,13 +40,13 @@ ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception EXCEPTION_TEXT="Some constraints are not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ - | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception EXCEPTION_TEXT="Some constraints are not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" 2>&1 \ - | grep -q "$EXCEPTION_TEXT" && echo "Exception ok" || echo "Did not thrown an exception" + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must succeed From e7293486bd39103a25d4d51ca4e8e8597c1c11ce Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 2 Jun 2019 17:41:12 +0300 Subject: [PATCH 025/133] Added ALTER TABLE support to constraints (ADD CONSTRAINT, DROP CONSTRAINT) --- dbms/src/Databases/DatabaseDictionary.cpp | 1 + dbms/src/Databases/DatabaseDictionary.h | 1 + dbms/src/Databases/DatabaseMemory.cpp | 1 + dbms/src/Databases/DatabaseMemory.h | 1 + dbms/src/Databases/DatabaseOrdinary.cpp | 7 ++ dbms/src/Databases/DatabaseOrdinary.h | 1 + dbms/src/Databases/IDatabase.h | 2 + dbms/src/Parsers/ASTAlterQuery.cpp | 11 +++ dbms/src/Parsers/ASTAlterQuery.h | 13 ++- dbms/src/Parsers/ParserAlterQuery.cpp | 25 ++++++ dbms/src/Storages/AlterCommands.cpp | 80 ++++++++++++++++++- dbms/src/Storages/AlterCommands.h | 17 +++- dbms/src/Storages/IStorage.cpp | 3 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 13 +-- dbms/src/Storages/MergeTree/MergeTreeData.h | 3 +- .../ReplicatedMergeTreeTableMetadata.cpp | 16 ++++ .../ReplicatedMergeTreeTableMetadata.h | 6 +- dbms/src/Storages/StorageBuffer.cpp | 3 +- dbms/src/Storages/StorageDistributed.cpp | 3 +- dbms/src/Storages/StorageMerge.cpp | 3 +- dbms/src/Storages/StorageMergeTree.cpp | 16 ++-- dbms/src/Storages/StorageNull.cpp | 3 +- .../Storages/StorageReplicatedMergeTree.cpp | 14 +++- .../00953_constraints_operations.reference | 4 + .../00953_constraints_operations.sh | 40 ++++++++++ 25 files changed, 255 insertions(+), 32 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00953_constraints_operations.reference create mode 100755 dbms/tests/queries/0_stateless/00953_constraints_operations.sh diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 195dcea5287..8add0fa8911 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -131,6 +131,7 @@ void DatabaseDictionary::alterTable( const String &, const ColumnsDescription &, const IndicesDescription &, + const ConstraintsDescription &, const ASTModifier &) { throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index 3bff84c36b8..d0ea33c6e4e 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -72,6 +72,7 @@ public: const String & name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 3eea0bc666a..c53309ca6c1 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -54,6 +54,7 @@ void DatabaseMemory::alterTable( const String &, const ColumnsDescription &, const IndicesDescription &, + const ConstraintsDescription &, const ASTModifier &) { throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index fe7cc783ba3..dc770373360 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -49,6 +49,7 @@ public: const String & name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 9fa7d1b1196..1ef67dfd9f4 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -516,6 +516,7 @@ void DatabaseOrdinary::alterTable( const String & table_name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & storage_modifier) { /// Read the definition of the table and replace the necessary parts with new ones. @@ -538,6 +539,7 @@ void DatabaseOrdinary::alterTable( ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns); ASTPtr new_indices = InterpreterCreateQuery::formatIndices(indices); + ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(constraints); ast_create_query.columns_list->replace(ast_create_query.columns_list->columns, new_columns); @@ -546,6 +548,11 @@ void DatabaseOrdinary::alterTable( else ast_create_query.columns_list->set(ast_create_query.columns_list->indices, new_indices); + if (ast_create_query.columns_list->constraints) + ast_create_query.columns_list->replace(ast_create_query.columns_list->constraints, new_constraints); + else + ast_create_query.columns_list->set(ast_create_query.columns_list->constraints, new_constraints); + if (storage_modifier) storage_modifier(*ast_create_query.storage); diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index 887bf101d62..2ed1a426d64 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -43,6 +43,7 @@ public: const String & name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index d53de1dfcb5..37ee1e676e9 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -114,6 +115,7 @@ public: const String & name, const ColumnsDescription & columns, const IndicesDescription & indices, + const ConstraintsDescription & constraints, const ASTModifier & engine_modifier) = 0; /// Returns time of table's metadata change, 0 if there is no corresponding metadata file. diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index e614f64d208..a85890d3cd0 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -105,6 +105,17 @@ void ASTAlterCommand::formatImpl( << "DROP INDEX " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); index->formatImpl(settings, state, frame); } + else if (type == ASTAlterCommand::ADD_CONSTRAINT) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ADD CONSTRAINT" << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); + constraint_decl->formatImpl(settings, state, frame); + } + else if (type == ASTAlterCommand::DROP_CONSTRAINT) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str + << "DROP CONSTRAINT " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); + constraint->formatImpl(settings, state, frame); + } else if (type == ASTAlterCommand::DROP_PARTITION) { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << (detach ? "DETACH" : "DROP") << " PARTITION " diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 2c4b3ddbaf1..2ca88f3145d 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -32,6 +32,9 @@ public: ADD_INDEX, DROP_INDEX, + ADD_CONSTRAINT, + DROP_CONSTRAINT, + DROP_PARTITION, ATTACH_PARTITION, REPLACE_PARTITION, @@ -69,7 +72,15 @@ public: /** The ADD INDEX query stores the name of the index following AFTER. * The DROP INDEX query stores the name for deletion. */ - ASTPtr index; + ASTPtr index; + + /** The ADD CONSTRAINT query stores the ConstraintDeclaration there. + */ + ASTPtr constraint_decl; + + /** The DROP CONSTRAINT query stores the name for deletion. + */ + ASTPtr constraint; /** Used in DROP PARTITION and ATTACH PARTITION FROM queries. * The value or ID of the partition is stored here. diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 98891bbdf5f..e4220fda868 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -32,6 +32,9 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_add_index("ADD INDEX"); ParserKeyword s_drop_index("DROP INDEX"); + ParserKeyword s_add_constraint("ADD CONSTRAINT"); + ParserKeyword s_drop_constraint("DROP CONSTRAINT"); + ParserKeyword s_attach_partition("ATTACH PARTITION"); ParserKeyword s_detach_partition("DETACH PARTITION"); ParserKeyword s_drop_partition("DROP PARTITION"); @@ -57,6 +60,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserStringLiteral parser_string_literal; ParserCompoundColumnDeclaration parser_col_decl; ParserIndexDeclaration parser_idx_decl; + ParserConstraintDeclaration parser_constraint_decl; ParserCompoundColumnDeclaration parser_modify_col_decl(false); ParserPartition parser_partition; ParserExpression parser_exp_elem; @@ -125,6 +129,27 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->type = ASTAlterCommand::DROP_INDEX; command->detach = false; } + else if (s_add_constraint.ignore(pos, expected)) + { + if (s_if_not_exists.ignore(pos, expected)) + command->if_not_exists = true; + + if (!parser_constraint_decl.parse(pos, command->constraint_decl, expected)) + return false; + + command->type = ASTAlterCommand::ADD_CONSTRAINT; + } + else if (s_drop_constraint.ignore(pos, expected)) + { + if (s_if_exists.ignore(pos, expected)) + command->if_exists = true; + + if (!parser_name.parse(pos, command->constraint, expected)) + return false; + + command->type = ASTAlterCommand::DROP_CONSTRAINT; + command->detach = false; + } else if (s_clear_column.ignore(pos, expected)) { if (s_if_exists.ignore(pos, expected)) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 88f3e909f49..22095ec4ae3 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -165,6 +166,32 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } + else if (command_ast->type == ASTAlterCommand::ADD_CONSTRAINT) + { + AlterCommand command; + command.constraint_decl = command_ast->constraint_decl; + command.type = AlterCommand::ADD_CONSTRAINT; + + const auto & ast_constraint_decl = command_ast->constraint_decl->as(); + + command.constraint_name = ast_constraint_decl.name; + + command.if_not_exists = command_ast->if_not_exists; + + return command; + } + else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) + { + if (command_ast->clear_column) + throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); + + AlterCommand command; + command.type = AlterCommand::DROP_CONSTRAINT; + command.constraint_name = command_ast->constraint->as().name; + command.if_exists = command_ast->if_exists; + + return command; + } else if (command_ast->type == ASTAlterCommand::MODIFY_TTL) { AlterCommand command; @@ -178,7 +205,8 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const + ConstraintsDescription & constraints_description, + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const { if (type == ADD_COLUMN) { @@ -298,6 +326,44 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri indices_description.indices.erase(erase_it); } + else if (type == ADD_CONSTRAINT) + { + if (std::any_of( + constraints_description.constraints.cbegin(), + constraints_description.constraints.cend(), + [this](const ASTPtr & constraint_ast) + { + return constraint_ast->as().name == constraint_name; + })) + { + if (if_not_exists) + return; + throw Exception("Cannot add constraint " + constraint_name + ": constraint with this name already exists", + ErrorCodes::ILLEGAL_COLUMN); + } + + auto insert_it = constraints_description.constraints.end(); + + constraints_description.constraints.emplace(insert_it, std::dynamic_pointer_cast(constraint_decl)); + } + else if (type == DROP_CONSTRAINT) + { + auto erase_it = std::find_if( + constraints_description.constraints.begin(), + constraints_description.constraints.end(), + [this](const ASTPtr & constraint_ast) + { + return constraint_ast->as().name == constraint_name; + }); + + if (erase_it == constraints_description.constraints.end()) { + if (if_exists) + return; + throw Exception("Wrong constraint name. Cannot find constraint `" + constraint_name + "` to drop.", + ErrorCodes::LOGICAL_ERROR); + } + constraints_description.constraints.erase(erase_it); + } else if (type == MODIFY_TTL) { ttl_table_ast = ttl; @@ -317,20 +383,23 @@ bool AlterCommand::isMutable() const } void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const + ConstraintsDescription & constraints_description, + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const { auto new_columns_description = columns_description; auto new_indices_description = indices_description; + auto new_constraints_description = constraints_description; auto new_order_by_ast = order_by_ast; auto new_primary_key_ast = primary_key_ast; auto new_ttl_table_ast = ttl_table_ast; for (const AlterCommand & command : *this) if (!command.ignore) - command.apply(new_columns_description, new_indices_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + command.apply(new_columns_description, new_indices_description, new_constraints_description, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); columns_description = std::move(new_columns_description); indices_description = std::move(new_indices_description); + constraints_description = std::move(new_constraints_description); order_by_ast = std::move(new_order_by_ast); primary_key_ast = std::move(new_primary_key_ast); ttl_table_ast = std::move(new_ttl_table_ast); @@ -518,10 +587,11 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const { auto out_columns_description = columns_description; IndicesDescription indices_description; + ConstraintsDescription constraints_description; ASTPtr out_order_by; ASTPtr out_primary_key; ASTPtr out_ttl_table; - apply(out_columns_description, indices_description, out_order_by, out_primary_key, out_ttl_table); + apply(out_columns_description, indices_description, constraints_description, out_order_by, out_primary_key, out_ttl_table); if (out_order_by) throw Exception("Storage doesn't support modifying ORDER BY expression", ErrorCodes::NOT_IMPLEMENTED); @@ -529,6 +599,8 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const throw Exception("Storage doesn't support modifying PRIMARY KEY expression", ErrorCodes::NOT_IMPLEMENTED); if (!indices_description.indices.empty()) throw Exception("Storage doesn't support modifying indices", ErrorCodes::NOT_IMPLEMENTED); + if (!constraints_description.constraints.empty()) + throw Exception("Storage doesn't support modifying constraints", ErrorCodes::NOT_IMPLEMENTED); if (out_ttl_table) throw Exception("Storage doesn't support modifying TTL expression", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index 4905b80f92f..1dfd46f9617 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -25,6 +26,8 @@ struct AlterCommand MODIFY_ORDER_BY, ADD_INDEX, DROP_INDEX, + ADD_CONSTRAINT, + DROP_CONSTRAINT, MODIFY_TTL, UKNOWN_TYPE, }; @@ -62,6 +65,12 @@ struct AlterCommand /// For ADD/DROP INDEX String index_name; + // For ADD CONSTRAINT + ASTPtr constraint_decl; + + // For ADD/DROP CONSTRAINT + String constraint_name; + /// For MODIFY TTL ASTPtr ttl; @@ -84,7 +93,8 @@ struct AlterCommand static std::optional parse(const ASTAlterCommand * command); void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, - ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; + ConstraintsDescription & constraints_description, + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; /// Checks that not only metadata touched by that command bool isMutable() const; @@ -95,8 +105,9 @@ class Context; class AlterCommands : public std::vector { public: - void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, ASTPtr & order_by_ast, - ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; + void apply(ColumnsDescription & columns_description, IndicesDescription & indices_description, + ConstraintsDescription & constraints_description, + ASTPtr & order_by_ast, ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast) const; /// For storages that don't support MODIFY_ORDER_BY. void apply(ColumnsDescription & columns_description) const; diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 7c19fd94aea..07f52749d2d 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -346,8 +346,9 @@ void IStorage::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 8427982efd7..144c6591ed9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -121,7 +121,7 @@ MergeTreeData::MergeTreeData( data_parts_by_info(data_parts_indexes.get()), data_parts_by_state_and_info(data_parts_indexes.get()) { - setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_); + setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_); setConstraints(constraints_); /// NOTE: using the same columns list as is read when performing actual merges. @@ -255,7 +255,8 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam void MergeTreeData::setPrimaryKeyIndicesAndColumns( const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, - const ColumnsDescription & new_columns, const IndicesDescription & indices_description, bool only_check) + const ColumnsDescription & new_columns, const IndicesDescription & indices_description, + const ConstraintsDescription & constraints_description, bool only_check) { if (!new_order_by_ast) throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -425,6 +426,8 @@ void MergeTreeData::setPrimaryKeyIndicesAndColumns( setIndices(indices_description); skip_indices = std::move(new_indices); + setConstraints(constraints_description); + primary_key_and_skip_indices_expr = new_indices_with_primary_key_expr; sorting_key_and_skip_indices_expr = new_indices_with_sorting_key_expr; } @@ -1180,11 +1183,11 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c /// Check that needed transformations can be applied to the list of columns without considering type conversions. auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - commands.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); - + commands.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); if (getIndices().empty() && !new_indices.empty() && !context.getSettingsRef().allow_experimental_data_skipping_indices) throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \ @@ -1267,7 +1270,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c } setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, - new_columns, new_indices, /* only_check = */ true); + new_columns, new_indices, new_constraints, /* only_check = */ true); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast, /* only_check = */ true); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index d589bb77013..bfcfbaba9a1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -778,7 +778,8 @@ protected: void setPrimaryKeyIndicesAndColumns(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, const ColumnsDescription & new_columns, - const IndicesDescription & indices_description, bool only_check = false); + const IndicesDescription & indices_description, + const ConstraintsDescription & constraints_description, bool only_check = false); void initPartitionKey(); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index b122785c5fd..8549264e2c2 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -46,6 +46,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr partition_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.partition_by_ast)); skip_indices = data.getIndices().toString(); + constraints = data.getConstraints().toString(); index_granularity_bytes = data.index_granularity_info.index_granularity_bytes; ttl_table = formattedAST(data.ttl_table_ast); } @@ -229,6 +230,21 @@ ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTabl ErrorCodes::METADATA_MISMATCH); } + if (constraints != from_zk.constraints) + { + if (allow_alter) + { + diff.constraints_changed = true; + diff.new_constraints = from_zk.constraints; + } + else + throw Exception( + "Existing table metadata in ZooKeeper differs in constraints." + " Stored in ZooKeeper: " + from_zk.constraints + + ", local: " + constraints, + ErrorCodes::METADATA_MISMATCH); + } + if (index_granularity_bytes != from_zk.index_granularity_bytes) throw Exception("Existing table metadata in ZooKeeper differs in index granularity bytes." " Stored in ZooKeeper: " + DB::toString(from_zk.index_granularity_bytes) + diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h index 55cfdb1494d..b28a7306e33 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h @@ -26,6 +26,7 @@ struct ReplicatedMergeTreeTableMetadata String partition_key; String sorting_key; String skip_indices; + String constraints; UInt64 index_granularity_bytes; String ttl_table; @@ -46,10 +47,13 @@ struct ReplicatedMergeTreeTableMetadata bool skip_indices_changed = false; String new_skip_indices; + bool constraints_changed = false; + String new_constraints; + bool ttl_table_changed = false; String new_ttl_table; - bool empty() const { return !sorting_key_changed && !skip_indices_changed; } + bool empty() const { return !sorting_key_changed && !skip_indices_changed && !constraints_changed; } }; Diff checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, bool allow_alter) const; diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 24e1ecef2e3..5a4409600d8 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -701,8 +701,9 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_ auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 4440d2b96ee..42e769019d6 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -346,8 +346,9 @@ void StorageDistributed::alter( auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); + context.getDatabase(database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 1fbde64fd85..9f3331a033e 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -417,8 +417,9 @@ void StorageMerge::alter( auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, {}); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, new_indices, new_constraints, {}); setColumns(new_columns); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 01049bfbf76..10709cd8181 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -242,8 +242,9 @@ void StorageMergeTree::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); return; } @@ -252,15 +253,14 @@ void StorageMergeTree::alter( auto merge_blocker = merger_mutator.actions_blocker.cancel(); lockNewDataStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - checkAlter(params, context); - auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + params.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); auto transactions = prepareAlterTransactions(new_columns, new_indices, context); @@ -279,11 +279,10 @@ void StorageMergeTree::alter( if (new_ttl_table_ast.get() != ttl_table_ast.get()) storage_ast.set(storage_ast.ttl_table, new_ttl_table_ast); }; - - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, storage_modifier); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier); /// Reinitialize primary key because primary key column types might have changed. - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices); + setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast); @@ -834,10 +833,11 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr ignored_order_by_ast; ASTPtr ignored_primary_key_ast; ASTPtr ignored_ttl_table_ast; - alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); + alter_command.apply(new_columns, new_indices, new_constraints, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); auto columns_for_parts = new_columns.getAllPhysical(); for (const auto & part : parts) diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index 1762c8372f5..73cb0243c11 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -38,8 +38,9 @@ void StorageNull::alter( ColumnsDescription new_columns = getColumns(); IndicesDescription new_indices = getIndices(); + ConstraintsDescription new_constraints = getConstraints(); params.apply(new_columns); - context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, {}); + context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, {}); setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index aaf8f18b65b..250a9dc96e8 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -422,6 +422,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_order_by_ast = order_by_ast; auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr new_ttl_table_ast = ttl_table_ast; IDatabase::ASTModifier storage_modifier; if (!metadata_diff.empty()) @@ -451,6 +452,9 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column if (metadata_diff.skip_indices_changed) new_indices = IndicesDescription::parse(metadata_diff.new_skip_indices); + if (metadata_diff.constraints_changed) + new_constraints = ConstraintsDescription::parse(metadata_diff.new_constraints); + if (metadata_diff.ttl_table_changed) { ParserExpression parser; @@ -476,11 +480,11 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column }; } - global_context.getDatabase(database_name)->alterTable(global_context, table_name, new_columns, new_indices, storage_modifier); + global_context.getDatabase(database_name)->alterTable(global_context, table_name, new_columns, new_indices, new_constraints, storage_modifier); /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices); + setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast); } @@ -1507,10 +1511,11 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & auto new_columns = getColumns(); auto new_indices = getIndices(); + auto new_constraints = getConstraints(); ASTPtr ignored_order_by_ast; ASTPtr ignored_primary_key_ast; ASTPtr ignored_ttl_table_ast; - alter_command.apply(new_columns, new_indices, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); + alter_command.apply(new_columns, new_indices, new_constraints, ignored_order_by_ast, ignored_primary_key_ast, ignored_ttl_table_ast); size_t modified_parts = 0; auto parts = getDataParts(); @@ -3114,10 +3119,11 @@ void StorageReplicatedMergeTree::alter( ColumnsDescription new_columns = getColumns(); IndicesDescription new_indices = getIndices(); + ConstraintsDescription new_constraints = getConstraints(); ASTPtr new_order_by_ast = order_by_ast; ASTPtr new_primary_key_ast = primary_key_ast; ASTPtr new_ttl_table_ast = ttl_table_ast; - params.apply(new_columns, new_indices, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); + params.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast); String new_columns_str = new_columns.toString(); if (new_columns_str != getColumns().toString()) diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.reference b/dbms/tests/queries/0_stateless/00953_constraints_operations.reference new file mode 100644 index 00000000000..5713da9fef5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.reference @@ -0,0 +1,4 @@ +1 2 +ok +1 2 +ok diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh new file mode 100755 index 00000000000..917719c3e46 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +EXCEPTION_SUCCESS_TEXT=ok + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints +( + a UInt32, + b UInt32, + CONSTRAINT b_constraint CHECK b > 0 +) +ENGINE = MergeTree ORDER BY (a);" + +# This one must succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +# This one must throw and exception +EXCEPTION_TEXT="Some constraints are not satisfied" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" +$CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE test_constraints DROP CONSTRAINT b_constraint;" + +# This one must suceed now +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE test_constraints ADD CONSTRAINT b_constraint CHECK b > 10;" + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 10);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 11);" + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_constraints;" \ No newline at end of file From 33f0ebd8ab83ef61990a66f7fb04d424f25cff5f Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 2 Jun 2019 18:08:28 +0300 Subject: [PATCH 026/133] Brace style fix --- dbms/src/Storages/AlterCommands.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 22095ec4ae3..0d20847727a 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -356,7 +356,8 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri return constraint_ast->as().name == constraint_name; }); - if (erase_it == constraints_description.constraints.end()) { + if (erase_it == constraints_description.constraints.end()) + { if (if_exists) return; throw Exception("Wrong constraint name. Cannot find constraint `" + constraint_name + "` to drop.", From b63623d0146965191cc5dcec728d25ec7d84ab46 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Wed, 5 Jun 2019 10:33:34 +0300 Subject: [PATCH 027/133] Extended constraint exception with constraint name and expression --- .../DataStreams/CheckConstraintsBlockOutputStream.cpp | 10 +++++++--- .../queries/0_stateless/00952_basic_constraints.sh | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 1cc271e5578..da77e4a1c2e 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -1,14 +1,18 @@ #include - +#include namespace DB { void CheckConstraintsBlockOutputStream::write(const Block & block) { - for (auto & constraint_expr: expressions) + for (size_t i = 0; i < expressions.size(); ++i) + { + auto constraint_expr = expressions[i]; if (!checkConstraintOnBlock(block, constraint_expr)) - throw Exception{"Some constraints are not satisfied", ErrorCodes::QUERY_WAS_CANCELLED}; + throw Exception{"Constraint " + constraints.constraints[i]->name + " is not satisfied at, constraint expression: " + + serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::LOGICAL_ERROR}; + } output->write(block); } diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index 93fa16ce4af..1d2a46dae61 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh EXCEPTION_SUCCESS_TEXT=ok - +$CLICKHOUSE_CLIENT --query="CREATE DATABASE IF NOT EXISTS test;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_constraints;" $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints @@ -20,7 +20,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Some constraints are not satisfied" +EXCEPTION_TEXT="Constraint b_constraint is not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" @@ -38,13 +38,13 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception -EXCEPTION_TEXT="Some constraints are not satisfied" +EXCEPTION_TEXT="Constraint b_constraint is not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception -EXCEPTION_TEXT="Some constraints are not satisfied" +EXCEPTION_TEXT="Constraint a_constraint is not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" From f413b1e346270e732988826e84e548857dc8174e Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Wed, 5 Jun 2019 11:05:46 +0300 Subject: [PATCH 028/133] Implemented memoryIsByte, replaced memoryIsZero with it, implemented memory check in CheckConstraintsBlockOutputStream --- dbms/src/Columns/ColumnsCommon.cpp | 43 ++++--------------- dbms/src/Columns/ColumnsCommon.h | 2 +- .../CheckConstraintsBlockOutputStream.cpp | 21 ++++++++- .../CheckConstraintsBlockOutputStream.h | 2 + 4 files changed, 32 insertions(+), 36 deletions(-) diff --git a/dbms/src/Columns/ColumnsCommon.cpp b/dbms/src/Columns/ColumnsCommon.cpp index 6ad3d0907ab..0745a3d5b9f 100644 --- a/dbms/src/Columns/ColumnsCommon.cpp +++ b/dbms/src/Columns/ColumnsCommon.cpp @@ -61,43 +61,18 @@ std::vector countColumnsSizeInSelector(IColumn::ColumnIndex num_columns, return counts; } -/** clang 4 generates better code than gcc 6. - * And both gcc and clang could not vectorize trivial loop by bytes automatically. - */ -bool memoryIsZero(const void * data, size_t size) +bool memoryIsByte(const void * data, size_t size, uint8_t byte) { - const Int8 * pos = reinterpret_cast(data); - const Int8 * end = pos + size; - -#ifdef __SSE2__ - const __m128 zero16 = _mm_setzero_ps(); - const Int8 * end64 = pos + size / 64 * 64; - - for (; pos < end64; pos += 64) - if (_mm_movemask_ps(_mm_cmpneq_ps( - _mm_loadu_ps(reinterpret_cast(pos)), - zero16)) - | _mm_movemask_ps(_mm_cmpneq_ps( - _mm_loadu_ps(reinterpret_cast(pos + 16)), - zero16)) - | _mm_movemask_ps(_mm_cmpneq_ps( - _mm_loadu_ps(reinterpret_cast(pos + 32)), - zero16)) - | _mm_movemask_ps(_mm_cmpneq_ps( - _mm_loadu_ps(reinterpret_cast(pos + 48)), - zero16))) - return false; - - /// TODO Add duff device for tail? -#endif - - for (; pos < end; ++pos) - if (*pos) - return false; - - return true; + if (size == 0) + return true; + auto ptr = reinterpret_cast(data); + return *ptr == byte && memcmp(ptr, ptr + 1, size - 1) == 0; } +bool memoryIsZero(const void * data, size_t size) +{ + return memoryIsByte(data, size, 0x0); +} namespace ErrorCodes { diff --git a/dbms/src/Columns/ColumnsCommon.h b/dbms/src/Columns/ColumnsCommon.h index 0b14b76ad39..46c6c0e9df3 100644 --- a/dbms/src/Columns/ColumnsCommon.h +++ b/dbms/src/Columns/ColumnsCommon.h @@ -22,7 +22,7 @@ std::vector countColumnsSizeInSelector(IColumn::ColumnIndex num_columns, /// Returns true, if the memory contains only zeros. bool memoryIsZero(const void * data, size_t size); - +bool memoryIsByte(const void * data, size_t size, uint8_t byte); /// The general implementation of `filter` function for ColumnArray and ColumnString. template diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index da77e4a1c2e..ac432694d83 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -1,5 +1,8 @@ #include +#include +#include #include +#include namespace DB { @@ -31,17 +34,33 @@ void CheckConstraintsBlockOutputStream::writeSuffix() output->writeSuffix(); } -bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) +bool CheckConstraintsBlockOutputStream::checkImplMemory(const Block & block, const ExpressionActionsPtr & constraint) +{ + Block res = block; + constraint->execute(res); + assert(block.columns() == res.columns() - 1); + ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); + auto res_column_uint8 = checkAndGetColumn(res_column.column.get()); + return memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1); +} + +bool CheckConstraintsBlockOutputStream::checkImplBool(const Block & block, const ExpressionActionsPtr & constraint) { Block res = block; constraint->execute(res); assert(block.columns() == res.columns() - 1); ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); size_t column_size = res_column.column->size(); + // std::cerr << "Sizes of constraints: " << res_column.column->size() << ' ' << res_column.column->get << '\n'; for (size_t i = 0; i < column_size; ++i) if (!res_column.column->getBool(i)) return false; return true; } +bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) +{ + return checkImplMemory(block, constraint); +} + } diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index 16b240eb758..6ea42cf44af 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -35,6 +35,8 @@ public: void writePrefix() override; void writeSuffix() override; + bool checkImplMemory(const Block & block, const ExpressionActionsPtr & constraint); + bool checkImplBool(const Block & block, const ExpressionActionsPtr & constraint); bool checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint); private: From bba3b33bdca89da53a8ac1707234b02959e8486c Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Wed, 5 Jun 2019 15:17:53 +0300 Subject: [PATCH 029/133] Fixed exception text in 00953_constraints_operations --- dbms/tests/queries/0_stateless/00953_constraints_operations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh index 917719c3e46..f0fc5b71fbf 100755 --- a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh @@ -20,7 +20,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Some constraints are not satisfied" +EXCEPTION_TEXT="Constraint b_constraint is not satisfied" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" From 375e4640867c0318b47fbab3149a35a53d0d46bd Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Thu, 6 Jun 2019 00:25:48 +0300 Subject: [PATCH 030/133] Fixed exception text --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index ac432694d83..75067b6afa7 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -13,7 +13,7 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { auto constraint_expr = expressions[i]; if (!checkConstraintOnBlock(block, constraint_expr)) - throw Exception{"Constraint " + constraints.constraints[i]->name + " is not satisfied at, constraint expression: " + + throw Exception{"Constraint " + constraints.constraints[i]->name + " is not satisfied, constraint expression: " + serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::LOGICAL_ERROR}; } output->write(block); From 76772d1de095ff3c41174a0a914d544162d60c02 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Thu, 6 Jun 2019 01:25:57 +0300 Subject: [PATCH 031/133] Documentation on constraints (RU, EN) --- docs/en/query_language/alter.md | 16 ++++++++++++++++ docs/en/query_language/create.md | 20 ++++++++++++++++++++ docs/en/query_language/insert_into.md | 3 +++ docs/ru/query_language/alter.md | 16 ++++++++++++++++ docs/ru/query_language/create.md | 20 ++++++++++++++++++++ docs/ru/query_language/insert_into.md | 4 ++++ 6 files changed, 79 insertions(+) diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md index 85941987ce9..fc42fc636e7 100644 --- a/docs/en/query_language/alter.md +++ b/docs/en/query_language/alter.md @@ -166,6 +166,22 @@ are available: These commands are lightweight in a sense that they only change metadata or remove files. Also, they are replicated (syncing indices metadata through ZooKeeper). +### Manipulations with constraints + +See more on [constraints](create.md#constraints) + +Constraints could be added or deleted using following syntax: +``` +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +Queries will add or remove metadata about constraints from table so they are processed immediately. + +Constraint check *will not be executed* on existing table if it was added. For now, we recommend to create new table and use `INSERT SELECT` query to fill new table. + +All changes on distributed tables are broadcasting to ZooKeeper so will be applied on other replicas. + ### Manipulations With Partitions and Parts {#alter_manipulations-with-partitions} The following operations with [partitions](../operations/table_engines/custom_partitioning_key.md) are available: diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md index 9ebd50839d3..573388195e3 100644 --- a/docs/en/query_language/create.md +++ b/docs/en/query_language/create.md @@ -80,6 +80,26 @@ If you add a new column to a table but later change its default expression, the It is not possible to set default values for elements in nested data structures. +### Constraints {#constraints} + +WARNING: This feature is experimental. Correct work is not guaranteed on non-MergeTree family engines. + +Along with columns descriptions constraints could be defined: + +``sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` could by any boolean expression. If constraints are defined for the table, each of them will be checked for every row in `INSERT` query. If any constraint is not satisfied — server will raise an exception with constraint name and checking expression. + +Adding large amount of constraints can negatively affect performance of big `INSERT` queries. + ### TTL expression Can be specified only for MergeTree-family tables. An expression for setting storage time for values. It must depends on `Date` or `DateTime` column and has one `Date` or `DateTime` column as a result. Example: diff --git a/docs/en/query_language/insert_into.md b/docs/en/query_language/insert_into.md index 914c3b2917f..c0cb9f8c3b1 100644 --- a/docs/en/query_language/insert_into.md +++ b/docs/en/query_language/insert_into.md @@ -40,6 +40,9 @@ INSERT INTO t FORMAT TabSeparated You can insert data separately from the query by using the command-line client or the HTTP interface. For more information, see the section "[Interfaces](../interfaces/index.md#interfaces)". +### Constraints + +If table has [constraints](create.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. ### Inserting The Results of `SELECT` {#insert_query_insert-select} diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md index 2367386172a..5e847abce4d 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/query_language/alter.md @@ -165,6 +165,22 @@ ALTER TABLE [db].name DROP INDEX name Запрос на изменение индексов реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. +### Манипуляции с ограничениями (constraints) + +Про ограничения подробнее написано [тут](create.md#constraints). + +Добавить или удалить ограничение можно с помощью запросов +``` +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +Запросы выполняют добавление или удаление метаданных об ограничениях таблицы `[db].name`, поэтому выполняются мнгновенно. + +Если ограничение появилось для непустой таблицы, то *проверка ограничения вызвана не будет*. Если же важно добавить ограничение на существующую таблицу, то рекомендуется создать новую таблицу с нужным ограничением и выполнить `INSERT SELECT` запрос для перекачки данных из одной таблицы в другую. + +Запрос на изменение ограничений так же, как и с индексами, реплицируется через ZooKeeper. + ### Манипуляции с партициями и кусками {#alter_manipulations-with-partitions} Для работы с [партициями](../operations/table_engines/custom_partitioning_key.md) доступны следующие операции: diff --git a/docs/ru/query_language/create.md b/docs/ru/query_language/create.md index ee6dc3c7820..0b842e22e47 100644 --- a/docs/ru/query_language/create.md +++ b/docs/ru/query_language/create.md @@ -80,6 +80,26 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... Отсутствует возможность задать значения по умолчанию для элементов вложенных структур данных. +### Ограничения (constraints) {#constraints} + +WARNING: Находится в экспериментальном режиме, поддержано в MergeTree (работоспособность на других типах движков таблиц не гарантируется). + +Наряду с объявлением столбцов можно объявить ограчения на значения в столбцах таблицы: + +```sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], + ... + CONSTRAINT constraint_name_1 CHECK boolean_expr_1, + ... +) ENGINE = engine +``` + +`boolean_expr_1` может быть любым булевым выражением, состоящим из операторов сравнения или функций. При наличии одного или нескольких ограничений в момент вставки данных выражения ограничений будут проверяться на истинность для каждой вставляемой строки данных. В случае, если в теле INSERT запроса придут некорректные данные — клиентов будет выкинуто исключение с нарушенным ограничением. + +Добавление большого числа ограничений может негативно повлиять на производительность объёмных `INSERT` запросов. + ### Выражение для TTL Может быть указано только для таблиц семейства MergeTree. Выражение для указания времени хранения значений. Оно должно зависеть от стобца типа `Date` или `DateTime` и в качестве результата вычислять столбец типа `Date` или `DateTime`. Пример: diff --git a/docs/ru/query_language/insert_into.md b/docs/ru/query_language/insert_into.md index 356b720e157..454339ebcdb 100644 --- a/docs/ru/query_language/insert_into.md +++ b/docs/ru/query_language/insert_into.md @@ -40,6 +40,10 @@ INSERT INTO t FORMAT TabSeparated С помощью консольного клиента или HTTP интерфейса можно вставлять данные отдельно от запроса. Как это сделать, читайте в разделе "[Интерфейсы](../interfaces/index.md#interfaces)". +### Ограничения (constraints) + +Если в таблице объявлены [ограничения](create.md#constraints), то их выполнимость будет проверена для каждой вставляемой строки. Если для хотя бы одной строки ограничения не будут выполнены, запрос будет остановлен. + ### Вставка результатов `SELECT` {#insert_query_insert-select} ``` sql From bb78012cf91e21d7394d001eba6480918ab4fb01 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 30 Jun 2019 12:09:06 +0200 Subject: [PATCH 032/133] Removed unused method and assertions --- .../CheckConstraintsBlockOutputStream.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 75067b6afa7..ec4a7bd45b8 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -38,26 +38,11 @@ bool CheckConstraintsBlockOutputStream::checkImplMemory(const Block & block, con { Block res = block; constraint->execute(res); - assert(block.columns() == res.columns() - 1); ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); auto res_column_uint8 = checkAndGetColumn(res_column.column.get()); return memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1); } -bool CheckConstraintsBlockOutputStream::checkImplBool(const Block & block, const ExpressionActionsPtr & constraint) -{ - Block res = block; - constraint->execute(res); - assert(block.columns() == res.columns() - 1); - ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); - size_t column_size = res_column.column->size(); - // std::cerr << "Sizes of constraints: " << res_column.column->size() << ' ' << res_column.column->get << '\n'; - for (size_t i = 0; i < column_size; ++i) - if (!res_column.column->getBool(i)) - return false; - return true; -} - bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) { return checkImplMemory(block, constraint); From ea4d42c3aca78eb6d114ea3703042649cc0db110 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 30 Jun 2019 12:09:58 +0200 Subject: [PATCH 033/133] Merged DROP_(COLUMN|INDEX|CONSTRAINT) AST parsing into one block --- dbms/src/Storages/AlterCommands.cpp | 39 ++++++++++------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 80c01a0028e..1ce70723238 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -74,17 +74,6 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } - else if (command_ast->type == ASTAlterCommand::DROP_COLUMN && !command_ast->partition) - { - if (command_ast->clear_column) - throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); - - AlterCommand command; - command.type = AlterCommand::DROP_COLUMN; - command.column_name = *getIdentifierName(command_ast->column); - command.if_exists = command_ast->if_exists; - return command; - } else if (command_ast->type == ASTAlterCommand::MODIFY_COLUMN) { AlterCommand command; @@ -154,18 +143,6 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } - else if (command_ast->type == ASTAlterCommand::DROP_INDEX) - { - if (command_ast->clear_column) - throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); - - AlterCommand command; - command.type = AlterCommand::DROP_INDEX; - command.index_name = command_ast->index->as().name; - command.if_exists = command_ast->if_exists; - - return command; - } else if (command_ast->type == ASTAlterCommand::ADD_CONSTRAINT) { AlterCommand command; @@ -180,15 +157,25 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } - else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) + else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT + || command_ast->type == ASTAlterCommand::DROP_INDEX + || (command_ast->type == ASTAlterCommand::DROP_COLUMN && !command_ast->partition)) { if (command_ast->clear_column) throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; - command.type = AlterCommand::DROP_CONSTRAINT; - command.constraint_name = command_ast->constraint->as().name; command.if_exists = command_ast->if_exists; + if (command_ast->type == ASTAlterCommand::DROP_INDEX) { + command.type = AlterCommand::DROP_INDEX; + command.index_name = command_ast->index->as().name; + } else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) { + command.type = AlterCommand::DROP_CONSTRAINT; + command.constraint_name = command_ast->constraint->as().name; + } else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) { + command.type = AlterCommand::DROP_COLUMN; + command.column_name = *getIdentifierName(command_ast->column); + } return command; } From 06e92e14b828c1457410d04aa8682acbda11d182 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 5 Jul 2019 10:10:15 +0300 Subject: [PATCH 034/133] alterTable fix in MySQL --- dbms/src/Databases/DatabaseMySQL.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Databases/DatabaseMySQL.h b/dbms/src/Databases/DatabaseMySQL.h index 7ce836d6a64..01a8c1df0d2 100644 --- a/dbms/src/Databases/DatabaseMySQL.h +++ b/dbms/src/Databases/DatabaseMySQL.h @@ -71,7 +71,7 @@ public: throw Exception("MySQL database engine does not support create table.", ErrorCodes::NOT_IMPLEMENTED); } - void alterTable(const Context &, const String &, const ColumnsDescription &, const IndicesDescription &, const ASTModifier &) override + void alterTable(const Context &, const String &, const ColumnsDescription &, const IndicesDescription &, const ConstraintsDescription &, const ASTModifier &) override { throw Exception("MySQL database engine does not support alter table.", ErrorCodes::NOT_IMPLEMENTED); } From 430400c3c5206c383a1e0870f00a77f5923147a3 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 5 Jul 2019 10:16:34 +0300 Subject: [PATCH 035/133] Style fix --- dbms/src/Storages/AlterCommands.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 8b3713ae31f..c90d5ca2c25 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -166,13 +166,16 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ AlterCommand command; command.if_exists = command_ast->if_exists; - if (command_ast->type == ASTAlterCommand::DROP_INDEX) { + if (command_ast->type == ASTAlterCommand::DROP_INDEX) + { command.type = AlterCommand::DROP_INDEX; command.index_name = command_ast->index->as().name; - } else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) { + } else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) + { command.type = AlterCommand::DROP_CONSTRAINT; command.constraint_name = command_ast->constraint->as().name; - } else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) { + } else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) + { command.type = AlterCommand::DROP_COLUMN; command.column_name = *getIdentifierName(command_ast->column); } From 3757aa9d020046d0a825a27546b43066ced2ffec Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 6 Jul 2019 23:30:48 +0300 Subject: [PATCH 036/133] More style fix --- dbms/src/Storages/AlterCommands.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index c90d5ca2c25..b250452c683 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -170,11 +170,13 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ { command.type = AlterCommand::DROP_INDEX; command.index_name = command_ast->index->as().name; - } else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) + } + else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT) { command.type = AlterCommand::DROP_CONSTRAINT; command.constraint_name = command_ast->constraint->as().name; - } else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) + } + else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) { command.type = AlterCommand::DROP_COLUMN; command.column_name = *getIdentifierName(command_ast->column); From c1d91222d50e368529e0c6675b4a6a6bb6681103 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sat, 13 Jul 2019 13:42:52 +0300 Subject: [PATCH 037/133] Ranamed setPrimaryKeyIndicesAndColumns to setProperties --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 6 +++--- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 2 +- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 3c30e7ce778..c2aa0a61855 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -121,7 +121,7 @@ MergeTreeData::MergeTreeData( data_parts_by_info(data_parts_indexes.get()), data_parts_by_state_and_info(data_parts_indexes.get()) { - setPrimaryKeyIndicesAndColumns(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_); + setProperties(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_); setConstraints(constraints_); /// NOTE: using the same columns list as is read when performing actual merges. @@ -232,7 +232,7 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam } -void MergeTreeData::setPrimaryKeyIndicesAndColumns( +void MergeTreeData::setProperties( const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, const ColumnsDescription & new_columns, const IndicesDescription & indices_description, const ConstraintsDescription & constraints_description, bool only_check) @@ -1262,7 +1262,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c } } - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, + setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints, /* only_check = */ true); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast, /* only_check = */ true); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 7572e9f2856..d4848462df0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -757,7 +757,7 @@ protected: /// The same for clearOldTemporaryDirectories. std::mutex clear_old_temporary_directories_mutex; - void setPrimaryKeyIndicesAndColumns(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, + void setProperties(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast, const ColumnsDescription & new_columns, const IndicesDescription & indices_description, const ConstraintsDescription & constraints_description, bool only_check = false); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 7cf8facb5f2..b7adc8c5bff 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -282,7 +282,7 @@ void StorageMergeTree::alter( context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier); /// Reinitialize primary key because primary key column types might have changed. - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); + setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index f1f4a4049f8..ae0ccfaaf1e 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -484,7 +484,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. - setPrimaryKeyIndicesAndColumns(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); + setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast); } From 3c2172b750989b72d9290615aeb669a6b90dc096 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 22 Jul 2019 14:23:11 +0300 Subject: [PATCH 038/133] parse and interpret query --- dbms/src/Core/Settings.h | 3 ++- dbms/src/Interpreters/InterpreterAlterQuery.cpp | 7 +++++++ dbms/src/Parsers/ASTAlterQuery.cpp | 6 ++++++ dbms/src/Parsers/ASTAlterQuery.h | 3 ++- dbms/src/Parsers/ParserAlterQuery.cpp | 17 +++++++++++++++++ dbms/src/Storages/PartitionCommands.cpp | 11 +++++++++++ dbms/src/Storages/PartitionCommands.h | 3 ++- dbms/src/Storages/StorageMergeTree.cpp | 4 ++++ .../src/Storages/StorageReplicatedMergeTree.cpp | 4 ++++ 9 files changed, 55 insertions(+), 3 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index b1182cae9bf..4bb76039cab 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -336,7 +336,8 @@ struct Settings : public SettingsCollection \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ - M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") + M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ + M(SettingBool, allow_drop_detached_part, false, "Allow ALTER TABLE ... DROP DETACHED PART ... queries") DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 8751ff067b1..074fbb7d4c2 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int ILLEGAL_COLUMN; + extern const int SUPPORT_IS_DISABLED; } @@ -53,7 +54,13 @@ BlockIO InterpreterAlterQuery::execute() if (auto alter_command = AlterCommand::parse(command_ast)) alter_commands.emplace_back(std::move(*alter_command)); else if (auto partition_command = PartitionCommand::parse(command_ast)) + { + if (partition_command->type == PartitionCommand::DROP_DETACHED_PARTITION + && !context.getSettingsRef().allow_drop_detached_part) + throw DB::Exception("Cannot execute query: DROP DETACHED PART is disabled " + "(see allow_drop_detached setting)", ErrorCodes::SUPPORT_IS_DISABLED); partition_commands.emplace_back(std::move(*partition_command)); + } else if (auto mut_command = MutationCommand::parse(command_ast)) mutation_commands.emplace_back(std::move(*mut_command)); else diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index c7cd100b415..6d87156a19b 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -118,6 +118,12 @@ void ASTAlterCommand::formatImpl( << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); } + else if (type == ASTAlterCommand::DROP_DETACHED_PARTITION) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DROP DETACHED" << (part ? " PART " : " PARTITION ") + << (settings.hilite ? hilite_none : ""); + partition->formatImpl(settings, state, frame); + } else if (type == ASTAlterCommand::ATTACH_PARTITION) { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ATTACH " diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 2c4b3ddbaf1..d6a54812960 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -33,6 +33,7 @@ public: DROP_INDEX, DROP_PARTITION, + DROP_DETACHED_PARTITION, ATTACH_PARTITION, REPLACE_PARTITION, FETCH_PARTITION, @@ -90,7 +91,7 @@ public: bool detach = false; /// true for DETACH PARTITION - bool part = false; /// true for ATTACH PART + bool part = false; /// true for ATTACH PART and DROP DETACHED PART bool clear_column = false; /// for CLEAR COLUMN (do not drop column from metadata) diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 98891bbdf5f..75c6f6291a8 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -35,6 +35,8 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_attach_partition("ATTACH PARTITION"); ParserKeyword s_detach_partition("DETACH PARTITION"); ParserKeyword s_drop_partition("DROP PARTITION"); + ParserKeyword s_drop_detached_partition("DROP DETACHED PARTITION"); + ParserKeyword s_drop_detached_part("DROP DETACHED PART"); ParserKeyword s_attach_part("ATTACH PART"); ParserKeyword s_fetch_partition("FETCH PARTITION"); ParserKeyword s_replace_partition("REPLACE PARTITION"); @@ -87,6 +89,21 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->type = ASTAlterCommand::DROP_PARTITION; } + else if (s_drop_detached_partition.ignore(pos, expected)) + { + if (!parser_partition.parse(pos, command->partition, expected)) + return false; + + command->type = ASTAlterCommand::DROP_DETACHED_PARTITION; + } + else if (s_drop_detached_part.ignore(pos, expected)) + { + if (!parser_string_literal.parse(pos, command->partition, expected)) + return false; + + command->type = ASTAlterCommand::DROP_DETACHED_PARTITION; + command->part = true; + } else if (s_drop_column.ignore(pos, expected)) { if (s_if_exists.ignore(pos, expected)) diff --git a/dbms/src/Storages/PartitionCommands.cpp b/dbms/src/Storages/PartitionCommands.cpp index f6aaee4c70e..bab3f6ced24 100644 --- a/dbms/src/Storages/PartitionCommands.cpp +++ b/dbms/src/Storages/PartitionCommands.cpp @@ -23,6 +23,17 @@ std::optional PartitionCommand::parse(const ASTAlterCommand * res.detach = command_ast->detach; return res; } + else if (command_ast->type == ASTAlterCommand::DROP_DETACHED_PARTITION) + { + if (!command_ast->part) // TODO + throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + + PartitionCommand res; + res.type = DROP_DETACHED_PARTITION; + res.partition = command_ast->partition; + res.part = command_ast->part; + return res; + } else if (command_ast->type == ASTAlterCommand::ATTACH_PARTITION) { PartitionCommand res; diff --git a/dbms/src/Storages/PartitionCommands.h b/dbms/src/Storages/PartitionCommands.h index 1f66c3f0c30..cb71a02548c 100644 --- a/dbms/src/Storages/PartitionCommands.h +++ b/dbms/src/Storages/PartitionCommands.h @@ -21,6 +21,7 @@ struct PartitionCommand ATTACH_PARTITION, CLEAR_COLUMN, DROP_PARTITION, + DROP_DETACHED_PARTITION, FETCH_PARTITION, FREEZE_ALL_PARTITIONS, FREEZE_PARTITION, @@ -35,7 +36,7 @@ struct PartitionCommand /// true for DETACH PARTITION. bool detach = false; - /// true for ATTACH PART (and false for PARTITION) + /// true for ATTACH PART and DROP DETACHED PART (and false for PARTITION) bool part = false; /// For ATTACH PARTITION partition FROM db.table diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index d021866487c..6e527c0c6c1 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -920,6 +920,10 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma dropPartition(command.partition, command.detach, context); break; + case PartitionCommand::DROP_DETACHED_PARTITION: + // TODO + throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + case PartitionCommand::ATTACH_PARTITION: attachPartition(command.partition, command.part, context); break; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index b51da168192..5f91c304e98 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3348,6 +3348,10 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part dropPartition(query, command.partition, command.detach, query_context); break; + case PartitionCommand::DROP_DETACHED_PARTITION: + // TODO + throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + case PartitionCommand::ATTACH_PARTITION: attachPartition(command.partition, command.part, query_context); break; From ad787938f5fa247901da7003c5717fed6a838445 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Jul 2019 22:43:33 +0300 Subject: [PATCH 039/133] better detached part name parsing --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 17 +------------- .../Storages/MergeTree/MergeTreePartInfo.cpp | 22 +++++++++++++++++++ .../Storages/MergeTree/MergeTreePartInfo.h | 6 +++++ .../System/StorageSystemDetachedParts.cpp | 1 + 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b32470f9f77..6a7b6d5405e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2579,22 +2579,7 @@ MergeTreeData::getDetachedParts() const res.emplace_back(); auto & part = res.back(); - /// First, try to parse as . - if (MergeTreePartInfo::tryParsePartName(dir_name, &part, format_version)) - continue; - - /// Next, as _. Use entire name as prefix if it fails. - part.prefix = dir_name; - const auto first_separator = dir_name.find_first_of('_'); - if (first_separator == String::npos) - continue; - - const auto part_name = dir_name.substr(first_separator + 1, - dir_name.size() - first_separator - 1); - if (!MergeTreePartInfo::tryParsePartName(part_name, &part, format_version)) - continue; - - part.prefix = dir_name.substr(0, first_separator); + DetachedPartInfo::tryParseDetachedPartName(dir_name, &part, format_version); } return res; } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 19f77448110..732cc3436f4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -188,4 +188,26 @@ String MergeTreePartInfo::getPartNameV0(DayNum left_date, DayNum right_date) con return wb.str(); } +bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, + MergeTreeDataFormatVersion format_version) +{ + /// First, try to parse as . + if (MergeTreePartInfo::tryParsePartName(dir_name, part_info, format_version)) + return part_info->valid_name = true; + + /// Next, as _. Use entire name as prefix if it fails. + part_info->prefix = dir_name; + const auto first_separator = dir_name.find_first_of('_'); + if (first_separator == String::npos) + return part_info->valid_name = false; + + // TODO what if contains '_'? + const auto part_name = dir_name.substr(first_separator + 1, + dir_name.size() - first_separator - 1); + if (!MergeTreePartInfo::tryParsePartName(part_name, part_info, format_version)) + return part_info->valid_name = false; + + part_info->prefix = dir_name.substr(0, first_separator); + return part_info->valid_name = true; +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index e80664c3dd9..2a168086a1c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -93,6 +93,12 @@ struct MergeTreePartInfo struct DetachedPartInfo : public MergeTreePartInfo { String prefix; + + /// If false, prefix contains full directory name and MergeTreePartInfo may be in invalid state + /// (directory name was not successfully parsed). + bool valid_name; + + static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, MergeTreeDataFormatVersion format_version); }; } diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index 9ae6f7b607a..9f33a60b84a 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -28,6 +28,7 @@ public: protected: explicit StorageSystemDetachedParts() { + // TODO add column "directory_name" or "is_valid_name" setColumns(ColumnsDescription{{ {"database", std::make_shared()}, {"table", std::make_shared()}, From 6e4aabbb1a4074d1a979190b063ffbe959894ebc Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Jul 2019 23:18:18 +0300 Subject: [PATCH 040/133] draft for StorageMergeTree --- dbms/src/Storages/StorageMergeTree.cpp | 33 ++++++++++++++++++++++++-- dbms/src/Storages/StorageMergeTree.h | 1 + 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 6e527c0c6c1..bd09588981b 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -921,8 +921,8 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma break; case PartitionCommand::DROP_DETACHED_PARTITION: - // TODO - throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + dropDetached(command.partition, command.part, context); + break; case PartitionCommand::ATTACH_PARTITION: attachPartition(command.partition, command.part, context); @@ -993,6 +993,34 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } +void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const Context & /*context*/) +{ + if (!part) // TODO + throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); + + String part_id = partition->as().value.safeGet(); + Poco::Path part_path(part_id); + const bool file_zero_depth = part_path.isFile() && part_path.depth() == 0 && part_path.getFileName() != ".."; + const bool dir_zero_depth = part_path.isDirectory() && part_path.depth() == 1 && part_path.directory(0) != ".."; + const bool zero_depth = file_zero_depth || dir_zero_depth; + if (!part_path.isRelative() || !zero_depth) + throw DB::Exception("Part name must contain exactly one path component: name of detached part", ErrorCodes::INCORRECT_FILE_NAME); + + part_id = part_path.isFile() ? part_path.getFileName() : part_path.directory(0); + Poco::Path base_dir(full_path + "detached"); + Poco::File detached_part_dir(Poco::Path(base_dir, part_id)); + if (!detached_part_dir.exists()) + throw DB::Exception("Detached part \"" + part_id + "\" not found" , ErrorCodes::INCORRECT_FILE_NAME); + + DetachedPartInfo info; + DetachedPartInfo::tryParseDetachedPartName(part_id, &info, format_version); + MergeTreeDataPart detached_part(*this, part_id, info); + detached_part.relative_path = "detached/" + part_id; + + // TODO make sure it's ok + detached_part.remove(); +} + void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) { // TODO: should get some locks to prevent race with 'alter … modify column' @@ -1039,6 +1067,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par LOG_DEBUG(log, "Checking data"); MutableDataPartPtr part = loadPartAndFixMetadata(source_path); + // TODO fix race with DROP DETACHED PARTITION LOG_INFO(log, "Attaching part " << source_part_name << " from " << source_path); renameTempPartAndAdd(part, &increment); diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 0de9618d915..fa2561e4ab2 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -120,6 +120,7 @@ private: // Partition helpers void dropPartition(const ASTPtr & partition, bool detach, const Context & context); + void dropDetached(const ASTPtr & partition, bool part, const Context & context); void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); From 856c8ef0e8dedf6a0166673c2a1b292b4648c655 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 24 Jul 2019 18:53:41 +0300 Subject: [PATCH 041/133] test for bug in ATTACH PART --- .../0_stateless/00974_attach_active_part.reference | 5 +++++ .../0_stateless/00974_attach_active_part.sh | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00974_attach_active_part.reference create mode 100755 dbms/tests/queries/0_stateless/00974_attach_active_part.sh diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference new file mode 100644 index 00000000000..3a90499810c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference @@ -0,0 +1,5 @@ +0_1_1_0 +1_2_2_0 +2_3_3_0 +3_4_4_0 +16 \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh new file mode 100755 index 00000000000..a3b2505f197 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_bug"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_bug (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; +$CLICKHOUSE_CLIENT --query="INSERT INTO attach_bug SELECT number FROM system.numbers LIMIT 16"; +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_bug ATTACH PART '../1_2_2_0'" 2> /dev/null; # | grep "" +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_bug' ORDER BY name FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT count() FROM attach_bug FORMAT TSV"; # will fail +$CLICKHOUSE_CLIENT --query="DROP TABLE attach_bug"; + + From 1805ab5736ec6c934af98dbda8c50a00fb6bb165 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Jul 2019 13:46:07 +0300 Subject: [PATCH 042/133] attach --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 8 ++- dbms/src/Storages/MergeTree/MergeTreeData.h | 1 + .../Storages/MergeTree/MergeTreePartInfo.cpp | 9 +++ .../Storages/MergeTree/MergeTreePartInfo.h | 2 + dbms/src/Storages/StorageMergeTree.cpp | 63 +++++++++++++------ dbms/src/Storages/StorageMergeTree.h | 1 + 6 files changed, 62 insertions(+), 22 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 6a7b6d5405e..94e42c34d0f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2336,6 +2336,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const St { MutableDataPartPtr part = std::make_shared(*this, Poco::Path(relative_path).getFileName()); part->relative_path = relative_path; + loadPartAndFixMetadata(part); + return part; +} + +void MergeTreeData::loadPartAndFixMetadata(MutableDataPartPtr part) +{ String full_part_path = part->getFullPath(); /// Earlier the list of columns was written incorrectly. Delete it and re-create. @@ -2357,8 +2363,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const St Poco::File(full_part_path + "checksums.txt.tmp").renameTo(full_part_path + "checksums.txt"); } - - return part; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 29962382749..cec3651652b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -533,6 +533,7 @@ public: /// Check that the part is not broken and calculate the checksums for it if they are not present. MutableDataPartPtr loadPartAndFixMetadata(const String & relative_path); + void loadPartAndFixMetadata(MutableDataPartPtr part); /** Create local backup (snapshot) for parts with specified prefix. * Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number, diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 732cc3436f4..45a0e1d488c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -210,4 +210,13 @@ bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, Detache part_info->prefix = dir_name.substr(0, first_separator); return part_info->valid_name = true; } + +String DetachedPartInfo::fullDirName() const +{ + if (!valid_name) + return prefix; + if (prefix.empty()) + return getPartName(); + return prefix + "_" + fullDirName(); +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index 2a168086a1c..7d0fb446ee3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -98,6 +98,8 @@ struct DetachedPartInfo : public MergeTreePartInfo /// (directory name was not successfully parsed). bool valid_name; + String fullDirName() const; + static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, MergeTreeDataFormatVersion format_version); }; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index bd09588981b..2f437e6d46b 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -36,6 +36,7 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int CANNOT_ASSIGN_OPTIMIZE; extern const int INCOMPATIBLE_COLUMNS; + extern const int BAD_DATA_PART_NAME; } namespace ActionLocks @@ -999,25 +1000,13 @@ void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const C throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); String part_id = partition->as().value.safeGet(); - Poco::Path part_path(part_id); - const bool file_zero_depth = part_path.isFile() && part_path.depth() == 0 && part_path.getFileName() != ".."; - const bool dir_zero_depth = part_path.isDirectory() && part_path.depth() == 1 && part_path.directory(0) != ".."; - const bool zero_depth = file_zero_depth || dir_zero_depth; - if (!part_path.isRelative() || !zero_depth) - throw DB::Exception("Part name must contain exactly one path component: name of detached part", ErrorCodes::INCORRECT_FILE_NAME); - - part_id = part_path.isFile() ? part_path.getFileName() : part_path.directory(0); - Poco::Path base_dir(full_path + "detached"); - Poco::File detached_part_dir(Poco::Path(base_dir, part_id)); - if (!detached_part_dir.exists()) - throw DB::Exception("Detached part \"" + part_id + "\" not found" , ErrorCodes::INCORRECT_FILE_NAME); + validateDetachedPartName(part_id); DetachedPartInfo info; DetachedPartInfo::tryParseDetachedPartName(part_id, &info, format_version); MergeTreeDataPart detached_part(*this, part_id, info); detached_part.relative_path = "detached/" + part_id; - // TODO make sure it's ok detached_part.remove(); } @@ -1038,6 +1027,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par Strings parts; if (attach_part) { + validateDetachedPartName(partition_id); parts.push_back(partition_id); } else @@ -1048,6 +1038,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par { const String & name = it.name(); MergeTreePartInfo part_info; + /// Parts with prefix in name (e.g. attaching_1_3_3_0, delete_tmp_1_3_3_0) will be ignored if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version) || part_info.partition_id != partition_id) { @@ -1062,16 +1053,38 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par for (const auto & source_part_name : parts) { - String source_path = source_dir + source_part_name; + MutableDataPartPtr part; + try + { + part = std::make_shared(*this, source_part_name); + part->relative_path = "detached/" + source_part_name; + part->renameTo("detached/attaching_" + source_part_name, false); - LOG_DEBUG(log, "Checking data"); - MutableDataPartPtr part = loadPartAndFixMetadata(source_path); + LOG_DEBUG(log, "Checking data in " << part->relative_path); + loadPartAndFixMetadata(part); - // TODO fix race with DROP DETACHED PARTITION - LOG_INFO(log, "Attaching part " << source_part_name << " from " << source_path); - renameTempPartAndAdd(part, &increment); + LOG_INFO(log, "Attaching part " << source_part_name << " from " << part->relative_path); + renameTempPartAndAdd(part, &increment); - LOG_INFO(log, "Finished attaching part"); + LOG_INFO(log, "Finished attaching part"); + } + catch (...) + { + tryLogCurrentException(log, String(__PRETTY_FUNCTION__) + ": cannot attach part " + source_part_name); + + if (part->relative_path == "detached/attaching_" + source_part_name) + { + try + { + part->renameTo("detached/" + source_part_name, false); + } + catch (...) + { + tryLogCurrentException(log, __PRETTY_FUNCTION__); + } + } + + } } /// New parts with other data may appear in place of deleted parts. @@ -1150,6 +1163,16 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } } +void StorageMergeTree::validateDetachedPartName(const String & name) const +{ + if (name.find('/') != std::string::npos || name == "." || name == "..") + throw DB::Exception("Invalid part name", ErrorCodes::INCORRECT_FILE_NAME); + + Poco::File detached_part_dir(full_path + "detached/" + name); + if (!detached_part_dir.exists()) + throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); +} + ActionLock StorageMergeTree::getActionLock(StorageActionBlockType action_type) { if (action_type == ActionLocks::PartsMerge) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index fa2561e4ab2..42061894a8e 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -124,6 +124,7 @@ private: void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); + void validateDetachedPartName(const String & name) const; friend class MergeTreeBlockOutputStream; friend class MergeTreeData; From 6ac950c6dc710fed7e1daf0a4bbdb09d3c0feb86 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Jul 2019 14:09:01 +0300 Subject: [PATCH 043/133] test attach active part fails --- .../queries/0_stateless/00974_attach_active_part.reference | 3 ++- dbms/tests/queries/0_stateless/00974_attach_active_part.sh | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference index 3a90499810c..fc0fce0a541 100644 --- a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference +++ b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference @@ -1,5 +1,6 @@ +OK 0_1_1_0 1_2_2_0 2_3_3_0 3_4_4_0 -16 \ No newline at end of file +16 diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh index a3b2505f197..32e2b21608f 100755 --- a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh +++ b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh @@ -6,9 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_bug"; $CLICKHOUSE_CLIENT --query="CREATE TABLE attach_bug (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; $CLICKHOUSE_CLIENT --query="INSERT INTO attach_bug SELECT number FROM system.numbers LIMIT 16"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_bug ATTACH PART '../1_2_2_0'" 2> /dev/null; # | grep "" +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_bug ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' $CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_bug' ORDER BY name FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="SElECT count() FROM attach_bug FORMAT TSV"; # will fail +$CLICKHOUSE_CLIENT --query="SElECT count() FROM attach_bug FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE attach_bug"; - - From e4212bfe593f3e28b42603c4dfeffcb2f60702ce Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Jul 2019 14:57:16 +0300 Subject: [PATCH 044/133] add full part name to detached_parts --- dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp | 2 +- dbms/src/Storages/System/StorageSystemDetachedParts.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 45a0e1d488c..de7150e4cea 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -217,6 +217,6 @@ String DetachedPartInfo::fullDirName() const return prefix; if (prefix.empty()) return getPartName(); - return prefix + "_" + fullDirName(); + return prefix + "_" + getPartName(); } } diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index 9f33a60b84a..9b32f1fb29b 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -28,7 +28,6 @@ public: protected: explicit StorageSystemDetachedParts() { - // TODO add column "directory_name" or "is_valid_name" setColumns(ColumnsDescription{{ {"database", std::make_shared()}, {"table", std::make_shared()}, @@ -37,7 +36,8 @@ protected: {"reason", std::make_shared()}, {"min_block_number", std::make_shared()}, {"max_block_number", std::make_shared()}, - {"level", std::make_shared()} + {"level", std::make_shared()}, + {"directory_name", std::make_shared()} }}); } @@ -63,12 +63,13 @@ protected: int i = 0; columns[i++]->insert(info.database); columns[i++]->insert(info.table); - columns[i++]->insert(p.partition_id); - columns[i++]->insert(p.getPartName()); + columns[i++]->insert(p.valid_name ? p.partition_id : ""); + columns[i++]->insert(p.valid_name ? p.getPartName() : ""); columns[i++]->insert(p.prefix); columns[i++]->insert(p.min_block); columns[i++]->insert(p.max_block); columns[i++]->insert(p.level); + columns[i++]->insert(p.fullDirName()); } } From 3ba26aba43e30aa92fed80fb6f842dc3eb064ef1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Jul 2019 19:28:08 +0300 Subject: [PATCH 045/133] tests --- .../Storages/MergeTree/MergeTreeDataPart.cpp | 9 +++- .../Storages/MergeTree/MergeTreeDataPart.h | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 6 +-- .../00974_attach_active_part.reference | 6 --- .../0_stateless/00974_attach_active_part.sh | 12 ------ .../00974_attach_invalid_parts.reference | 17 ++++++++ .../0_stateless/00974_attach_invalid_parts.sh | 41 +++++++++++++++++++ .../0_stateless/00975_drop_detached.reference | 2 + .../0_stateless/00975_drop_detached.sh | 30 ++++++++++++++ 9 files changed, 102 insertions(+), 23 deletions(-) delete mode 100644 dbms/tests/queries/0_stateless/00974_attach_active_part.reference delete mode 100755 dbms/tests/queries/0_stateless/00974_attach_active_part.sh create mode 100644 dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference create mode 100755 dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh create mode 100644 dbms/tests/queries/0_stateless/00975_drop_detached.reference create mode 100755 dbms/tests/queries/0_stateless/00975_drop_detached.sh diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 7b8be970e1d..865aaf80ed1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -347,7 +347,7 @@ UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) return res; } -void MergeTreeDataPart::remove() const +void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const { if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); @@ -398,6 +398,13 @@ void MergeTreeDataPart::remove() const return; } + if (force_recursive) + { + /// Part is not loaded (we don't know which files are there), so remove dir recursively. + to_dir.remove(true); + return; + } + try { /// Remove each expected file in directory, then remove directory itself. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h index f41ea8af424..98af00c071a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h @@ -241,7 +241,7 @@ struct MergeTreeDataPart /// Calculate the total size of the entire directory with all the files static UInt64 calculateTotalSizeOnDisk(const String & from); - void remove() const; + void remove(bool force_recursive = false) const; /// Makes checks and move part to new directory /// Changes only relative_dir_name, you need to update other metadata (name, is_temp) explicitly diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 2f437e6d46b..0c1503347c0 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -1007,7 +1007,7 @@ void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const C MergeTreeDataPart detached_part(*this, part_id, info); detached_part.relative_path = "detached/" + part_id; - detached_part.remove(); + detached_part.remove(true); } void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) @@ -1070,9 +1070,9 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par } catch (...) { - tryLogCurrentException(log, String(__PRETTY_FUNCTION__) + ": cannot attach part " + source_part_name); + LOG_INFO(log, "Cannot attach part " << source_part_name << " :" << getCurrentExceptionMessage(false)); - if (part->relative_path == "detached/attaching_" + source_part_name) + if (part && part->relative_path == "detached/attaching_" + source_part_name) { try { diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference b/dbms/tests/queries/0_stateless/00974_attach_active_part.reference deleted file mode 100644 index fc0fce0a541..00000000000 --- a/dbms/tests/queries/0_stateless/00974_attach_active_part.reference +++ /dev/null @@ -1,6 +0,0 @@ -OK -0_1_1_0 -1_2_2_0 -2_3_3_0 -3_4_4_0 -16 diff --git a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh b/dbms/tests/queries/0_stateless/00974_attach_active_part.sh deleted file mode 100755 index 32e2b21608f..00000000000 --- a/dbms/tests/queries/0_stateless/00974_attach_active_part.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_bug"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_bug (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; -$CLICKHOUSE_CLIENT --query="INSERT INTO attach_bug SELECT number FROM system.numbers LIMIT 16"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_bug ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_bug' ORDER BY name FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="SElECT count() FROM attach_bug FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="DROP TABLE attach_bug"; diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference new file mode 100644 index 00000000000..d44f46779ca --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference @@ -0,0 +1,17 @@ +=== cannot attach active === +OK +0_1_1_0 +1_2_2_0 +2_3_3_0 +3_4_4_0 +16 120 +=== attach all valid parts === +0_5_5_0 +0_6_6_0 +1_2_2_0 +1_4_4_0 +16 120 +=== detached === +0_5_5_0 +delete_tmp_0_7_7 +attaching_0_6_6 diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh new file mode 100755 index 00000000000..89a6be183d2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` +cur_db=`${CLICKHOUSE_CLIENT} --query "SELECT currentDatabase()"` + +echo '=== cannot attach active ==='; +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_active"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; +$CLICKHOUSE_CLIENT --query="INSERT INTO attach_active SELECT number FROM system.numbers LIMIT 16"; +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_active ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_active' AND database='${cur_db}' ORDER BY name FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_active FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="DROP TABLE attach_active"; + + + +echo '=== attach all valid parts ==='; +$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_partitions"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_partitions (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; +$CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8"; +$CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; + +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; +mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part +cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ +cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/delete_tmp_0_7_7_0/ +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; + +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_partitions FORMAT TSV"; +echo '=== detached ==='; +$CLICKHOUSE_CLIENT --query="SELECT directory_name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' FORMAT TSV"; + +$CLICKHOUSE_CLIENT --query="DROP TABLE attach_partitions"; +$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.reference b/dbms/tests/queries/0_stateless/00975_drop_detached.reference new file mode 100644 index 00000000000..40732c908ab --- /dev/null +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.reference @@ -0,0 +1,2 @@ +OK +0_3_3_0 diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh new file mode 100755 index 00000000000..9f831560bdc --- /dev/null +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` +cur_db=`${CLICKHOUSE_CLIENT} --query "SELECT currentDatabase()"` + +$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS drop_detached"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; +$CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8"; +$CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; + +$CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; +mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ +mkdir $ch_dir/data/$cur_db/drop_detached/detached/delete_tmp_0_7_7_0/ +mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ + +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'delete_tmp_0_7_7_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" + +$CLICKHOUSE_CLIENT --query="SElECT directory_name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; +$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From 8f4883b0d2d84f324c530068116dabc6f5c05146 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 28 Jul 2019 15:33:40 +0300 Subject: [PATCH 046/133] Better constraint exception --- .../CheckConstraintsBlockOutputStream.cpp | 46 +++++++++++++++---- .../CheckConstraintsBlockOutputStream.h | 16 ++++--- .../Interpreters/InterpreterInsertQuery.cpp | 2 +- .../0_stateless/00952_basic_constraints.sh | 14 +++--- .../00953_constraints_operations.sh | 2 +- 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index ec4a7bd45b8..cb9b8871a68 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -12,11 +12,23 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) for (size_t i = 0; i < expressions.size(); ++i) { auto constraint_expr = expressions[i]; - if (!checkConstraintOnBlock(block, constraint_expr)) - throw Exception{"Constraint " + constraints.constraints[i]->name + " is not satisfied, constraint expression: " + - serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::LOGICAL_ERROR}; + auto res_column_uint8 = executeOnBlock(block, constraint_expr); + if (!memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1)) + { + auto indices_wrong = findAllWrong(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize()); + std::string indices_str = "{"; + for (size_t j = 0; j < indices_wrong.size(); ++j) { + indices_str += std::to_string(indices_wrong[j]); + indices_str += (j != indices_wrong.size() - 1) ? ", " : "}"; + } + + throw Exception{"Violated constraint " + constraints.constraints[i]->name + + " in table " + table + " at indices " + indices_str + ", constraint expression: " + + serializeAST(*(constraints.constraints[i]->expr), true), ErrorCodes::LOGICAL_ERROR}; + } } output->write(block); + rows_written += block.rows(); } void CheckConstraintsBlockOutputStream::flush() @@ -34,18 +46,34 @@ void CheckConstraintsBlockOutputStream::writeSuffix() output->writeSuffix(); } -bool CheckConstraintsBlockOutputStream::checkImplMemory(const Block & block, const ExpressionActionsPtr & constraint) +const ColumnUInt8 *CheckConstraintsBlockOutputStream::executeOnBlock( + const Block & block, + const ExpressionActionsPtr & constraint) { Block res = block; + constraint->execute(res); ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); - auto res_column_uint8 = checkAndGetColumn(res_column.column.get()); - return memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1); + return checkAndGetColumn(res_column.column.get()); } -bool CheckConstraintsBlockOutputStream::checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint) +std::vector CheckConstraintsBlockOutputStream::findAllWrong(const void *data, size_t size) { - return checkImplMemory(block, constraint); -} + std::vector res; + if (size == 0) + return res; + + auto ptr = reinterpret_cast(data); + + for (size_t i = 0; i < size; ++i) + { + if (*(ptr + i) == 0x0) + { + res.push_back(i); + } + } + + return res; +} } diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index 6ea42cf44af..ac2e7e974a1 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -17,14 +17,17 @@ class CheckConstraintsBlockOutputStream : public IBlockOutputStream { public: CheckConstraintsBlockOutputStream( + const String & table_, const BlockOutputStreamPtr & output_, const Block & header_, const ConstraintsDescription & constraints_, const Context & context_) - : output(output_), + : table(table_), + output(output_), header(header_), constraints(constraints_), - expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())) + expressions(constraints_.getExpressions(context_, header.getNamesAndTypesList())), + rows_written(0) { } Block getHeader() const override { return header; } @@ -35,14 +38,15 @@ public: void writePrefix() override; void writeSuffix() override; - bool checkImplMemory(const Block & block, const ExpressionActionsPtr & constraint); - bool checkImplBool(const Block & block, const ExpressionActionsPtr & constraint); - bool checkConstraintOnBlock(const Block & block, const ExpressionActionsPtr & constraint); - private: + const ColumnUInt8* executeOnBlock(const Block & block, const ExpressionActionsPtr & constraint); + std::vector findAllWrong(const void *data, size_t size); + + String table; BlockOutputStreamPtr output; Block header; const ConstraintsDescription constraints; const ConstraintsExpressions expressions; + size_t rows_written; }; } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 9c0cc31cb8e..8454df97f08 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -119,7 +119,7 @@ BlockIO InterpreterInsertQuery::execute() out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); out = std::make_shared( - out, query_sample_block, table->getConstraints(), context); + query.table, out, query_sample_block, table->getConstraints(), context); auto out_wrapper = std::make_shared(out); out_wrapper->setProcessListElement(context.getProcessListElement()); diff --git a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh index 1d2a46dae61..b6aa28c46bf 100755 --- a/dbms/tests/queries/0_stateless/00952_basic_constraints.sh +++ b/dbms/tests/queries/0_stateless/00952_basic_constraints.sh @@ -20,8 +20,8 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Constraint b_constraint is not satisfied" -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0);" 2>&1 \ +EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices {1, 3}" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (3, 4), (1, 0), (3, 4), (6, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" @@ -32,20 +32,20 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE test_constraints ( a UInt32, b UInt32, - CONSTRAINT b_constraint CHECK b > 10, - CONSTRAINT a_constraint CHECK a < 10 + CONSTRAINT a_constraint CHECK a < 10, + CONSTRAINT b_constraint CHECK b > 10 ) ENGINE = MergeTree ORDER BY (a);" # This one must throw an exception -EXCEPTION_TEXT="Constraint b_constraint is not satisfied" +EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices {0}" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw an exception -EXCEPTION_TEXT="Constraint a_constraint is not satisfied" -$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11);" 2>&1 \ +EXCEPTION_TEXT="Violated constraint a_constraint in table test_constraints at indices {1}" +$CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (5, 16), (10, 11), (9, 11), (8, 12);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" diff --git a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh index f0fc5b71fbf..8a563a21e02 100755 --- a/dbms/tests/queries/0_stateless/00953_constraints_operations.sh +++ b/dbms/tests/queries/0_stateless/00953_constraints_operations.sh @@ -20,7 +20,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 2);" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" # This one must throw and exception -EXCEPTION_TEXT="Constraint b_constraint is not satisfied" +EXCEPTION_TEXT="Violated constraint b_constraint in table test_constraints at indices" $CLICKHOUSE_CLIENT --query="INSERT INTO test_constraints VALUES (1, 0);" 2>&1 \ | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not thrown an exception" $CLICKHOUSE_CLIENT --query="SELECT * FROM test_constraints;" From 0fc47fbbe4b70b5ffc254024798d8f9ed45b0418 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 26 Jul 2019 23:04:45 +0300 Subject: [PATCH 047/133] fixes --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 14 +++++++++- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 ++ .../Storages/MergeTree/MergeTreeDataPart.cpp | 8 ++++-- .../Storages/MergeTree/MergeTreePartInfo.cpp | 28 ++++++++----------- .../Storages/MergeTree/MergeTreePartInfo.h | 9 +++--- dbms/src/Storages/StorageMergeTree.cpp | 15 ++-------- dbms/src/Storages/StorageMergeTree.h | 1 - .../Storages/StorageReplicatedMergeTree.cpp | 2 ++ .../System/StorageSystemDetachedParts.cpp | 25 ++++++++--------- .../00974_attach_invalid_parts.reference | 4 +-- .../0_stateless/00974_attach_invalid_parts.sh | 11 +++++--- .../0_stateless/00975_drop_detached.sh | 13 +++++---- 12 files changed, 69 insertions(+), 63 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 94e42c34d0f..9d12a9ee6ea 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -87,6 +87,8 @@ namespace ErrorCodes extern const int CANNOT_MUNMAP; extern const int CANNOT_MREMAP; extern const int BAD_TTL_EXPRESSION; + extern const int INCORRECT_FILE_NAME; + extern const int BAD_DATA_PART_NAME; } @@ -2583,11 +2585,21 @@ MergeTreeData::getDetachedParts() const res.emplace_back(); auto & part = res.back(); - DetachedPartInfo::tryParseDetachedPartName(dir_name, &part, format_version); + DetachedPartInfo::tryParseDetachedPartName(dir_name, part, format_version); } return res; } +void MergeTreeData::validateDetachedPartName(const String & name) const +{ + if (name.find('/') != std::string::npos || name == "." || name == "..") + throw DB::Exception("Invalid part name", ErrorCodes::INCORRECT_FILE_NAME); + + Poco::File detached_part_dir(full_path + "detached/" + name); + if (!detached_part_dir.exists()) + throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); +} + MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const { DataParts res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index cec3651652b..2333135d53e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -389,6 +389,8 @@ public: /// Returns all detached parts std::vector getDetachedParts() const; + void validateDetachedPartName(const String & name) const; + /// Returns Committed parts DataParts getDataParts() const; DataPartsVector getDataPartsVector() const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 865aaf80ed1..24bc5cd2463 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -356,16 +356,18 @@ void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const * - rename directory to temporary name; * - remove it recursive. * - * For temporary name we use "delete_tmp_" prefix. + * For temporary name we use "detached/deleting_" prefix. * - * NOTE: We cannot use "tmp_delete_" prefix, because there is a second thread, + * NOTE: We cannot use "tmp_*" prefix, because there is a second thread, * that calls "clearOldTemporaryDirectories" and removes all directories, that begin with "tmp_" and are old enough. * But when we removing data part, it can be old enough. And rename doesn't change mtime. * And a race condition can happen that will lead to "File not found" error here. + * We move directory to detached/, because if an attempt to remove directory after renaming failed for some reason + * there would be no way to remove directory from storage.full_path (except manually). */ String from = storage.full_path + relative_path; - String to = storage.full_path + "delete_tmp_" + name; + String to = storage.full_path + getRelativePathForDetachedPart("deleting_"); Poco::File from_dir{from}; Poco::File to_dir{to}; diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index de7150e4cea..a9e31a988b3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -188,35 +188,29 @@ String MergeTreePartInfo::getPartNameV0(DayNum left_date, DayNum right_date) con return wb.str(); } -bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, +bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version) { + part_info.dir_name = dir_name; + /// First, try to parse as . - if (MergeTreePartInfo::tryParsePartName(dir_name, part_info, format_version)) - return part_info->valid_name = true; + if (MergeTreePartInfo::tryParsePartName(dir_name, &part_info, format_version)) + return part_info.valid_name = true; /// Next, as _. Use entire name as prefix if it fails. - part_info->prefix = dir_name; + part_info.prefix = dir_name; const auto first_separator = dir_name.find_first_of('_'); if (first_separator == String::npos) - return part_info->valid_name = false; + return part_info.valid_name = false; // TODO what if contains '_'? const auto part_name = dir_name.substr(first_separator + 1, dir_name.size() - first_separator - 1); - if (!MergeTreePartInfo::tryParsePartName(part_name, part_info, format_version)) - return part_info->valid_name = false; + if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) + return part_info.valid_name = false; - part_info->prefix = dir_name.substr(0, first_separator); - return part_info->valid_name = true; + part_info.prefix = dir_name.substr(0, first_separator); + return part_info.valid_name = true; } -String DetachedPartInfo::fullDirName() const -{ - if (!valid_name) - return prefix; - if (prefix.empty()) - return getPartName(); - return prefix + "_" + getPartName(); -} } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index 7d0fb446ee3..25cf46ad46d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -92,15 +92,14 @@ struct MergeTreePartInfo /// addition to the above fields. struct DetachedPartInfo : public MergeTreePartInfo { + /// Suddenly, name of detached part may contain suffix (such as _tryN), which is ignored by MergeTreePartInfo::tryParsePartName(...) + String dir_name; String prefix; - /// If false, prefix contains full directory name and MergeTreePartInfo may be in invalid state - /// (directory name was not successfully parsed). + /// If false, MergeTreePartInfo is in invalid state (directory name was not successfully parsed). bool valid_name; - String fullDirName() const; - - static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo * part_info, MergeTreeDataFormatVersion format_version); + static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version); }; } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 0c1503347c0..ad4d0cd933f 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -36,7 +36,6 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int CANNOT_ASSIGN_OPTIMIZE; extern const int INCOMPATIBLE_COLUMNS; - extern const int BAD_DATA_PART_NAME; } namespace ActionLocks @@ -1003,7 +1002,7 @@ void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const C validateDetachedPartName(part_id); DetachedPartInfo info; - DetachedPartInfo::tryParseDetachedPartName(part_id, &info, format_version); + DetachedPartInfo::tryParseDetachedPartName(part_id, info, format_version); MergeTreeDataPart detached_part(*this, part_id, info); detached_part.relative_path = "detached/" + part_id; @@ -1038,7 +1037,8 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par { const String & name = it.name(); MergeTreePartInfo part_info; - /// Parts with prefix in name (e.g. attaching_1_3_3_0, delete_tmp_1_3_3_0) will be ignored + /// Parts with prefix in name (e.g. attaching_1_3_3_0, deleting_1_3_3_0) will be ignored + // TODO what if name contains "_tryN" suffix? if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version) || part_info.partition_id != partition_id) { @@ -1163,15 +1163,6 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } } -void StorageMergeTree::validateDetachedPartName(const String & name) const -{ - if (name.find('/') != std::string::npos || name == "." || name == "..") - throw DB::Exception("Invalid part name", ErrorCodes::INCORRECT_FILE_NAME); - - Poco::File detached_part_dir(full_path + "detached/" + name); - if (!detached_part_dir.exists()) - throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); -} ActionLock StorageMergeTree::getActionLock(StorageActionBlockType action_type) { diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 42061894a8e..fa2561e4ab2 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -124,7 +124,6 @@ private: void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); - void validateDetachedPartName(const String & name) const; friend class MergeTreeBlockOutputStream; friend class MergeTreeData; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 5f91c304e98..192384602eb 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3554,6 +3554,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool Strings parts; if (attach_part) { + validateDetachedPartName(partition_id); parts.push_back(partition_id); } else @@ -3566,6 +3567,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool { String name = it.name(); MergeTreePartInfo part_info; + // TODO what if name contains "_tryN" suffix? if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version)) continue; if (part_info.partition_id != partition_id) diff --git a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp index 9b32f1fb29b..e27c7945670 100644 --- a/dbms/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/dbms/src/Storages/System/StorageSystemDetachedParts.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -31,13 +32,12 @@ protected: setColumns(ColumnsDescription{{ {"database", std::make_shared()}, {"table", std::make_shared()}, - {"partition_id", std::make_shared()}, + {"partition_id", std::make_shared(std::make_shared())}, {"name", std::make_shared()}, - {"reason", std::make_shared()}, - {"min_block_number", std::make_shared()}, - {"max_block_number", std::make_shared()}, - {"level", std::make_shared()}, - {"directory_name", std::make_shared()} + {"reason", std::make_shared(std::make_shared())}, + {"min_block_number", std::make_shared(std::make_shared())}, + {"max_block_number", std::make_shared(std::make_shared())}, + {"level", std::make_shared(std::make_shared())} }}); } @@ -63,13 +63,12 @@ protected: int i = 0; columns[i++]->insert(info.database); columns[i++]->insert(info.table); - columns[i++]->insert(p.valid_name ? p.partition_id : ""); - columns[i++]->insert(p.valid_name ? p.getPartName() : ""); - columns[i++]->insert(p.prefix); - columns[i++]->insert(p.min_block); - columns[i++]->insert(p.max_block); - columns[i++]->insert(p.level); - columns[i++]->insert(p.fullDirName()); + columns[i++]->insert(p.valid_name ? p.partition_id : Field()); + columns[i++]->insert(p.dir_name); + columns[i++]->insert(p.valid_name ? p.prefix : Field()); + columns[i++]->insert(p.valid_name ? p.min_block : Field()); + columns[i++]->insert(p.valid_name ? p.max_block : Field()); + columns[i++]->insert(p.valid_name ? p.level : Field()); } } diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference index d44f46779ca..42a04fe5666 100644 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference @@ -13,5 +13,5 @@ OK 16 120 === detached === 0_5_5_0 -delete_tmp_0_7_7 -attaching_0_6_6 +deleting_0_7_7_0 +attaching_0_6_6_0 diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh index 89a6be183d2..4e9efa64ad1 100755 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh @@ -27,15 +27,18 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM sys $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; -mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part -cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ -cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/delete_tmp_0_7_7_0/ +sudo -n mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ + mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part +sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ + cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ +sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ + cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; $CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_partitions FORMAT TSV"; echo '=== detached ==='; -$CLICKHOUSE_CLIENT --query="SELECT directory_name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE attach_partitions"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh index 9f831560bdc..3a5e920da75 100755 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.sh +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.sh @@ -15,16 +15,19 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system. $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; -mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ -mkdir $ch_dir/data/$cur_db/drop_detached/detached/delete_tmp_0_7_7_0/ -mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ +sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ + mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ +sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ + mkdir $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ +sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ + mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'delete_tmp_0_7_7_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" -$CLICKHOUSE_CLIENT --query="SElECT directory_name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From a3ebe3153537170e344dc8766d48a2630e63146c Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 29 Jul 2019 13:05:43 +0300 Subject: [PATCH 048/133] Brace style fix --- dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index cb9b8871a68..5adf344cf0b 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -17,7 +17,8 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { auto indices_wrong = findAllWrong(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize()); std::string indices_str = "{"; - for (size_t j = 0; j < indices_wrong.size(); ++j) { + for (size_t j = 0; j < indices_wrong.size(); ++j) + { indices_str += std::to_string(indices_wrong[j]); indices_str += (j != indices_wrong.size() - 1) ? ", " : "}"; } From 2f33df1b2ef01ba19db8d465400f08a4d532e060 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 30 Jul 2019 20:24:40 +0300 Subject: [PATCH 049/133] rename all parts before attaching --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 40 ++++++++++++ dbms/src/Storages/MergeTree/MergeTreeData.h | 15 +++++ .../Storages/MergeTree/MergeTreeDataPart.cpp | 24 ++----- .../Storages/MergeTree/MergeTreeDataPart.h | 2 +- .../Storages/MergeTree/MergeTreePartInfo.cpp | 1 + dbms/src/Storages/StorageMergeTree.cpp | 65 ++++++------------- dbms/src/Storages/StorageMergeTree.h | 1 - .../Storages/StorageReplicatedMergeTree.cpp | 6 +- .../00974_attach_invalid_parts.reference | 17 +++-- .../0_stateless/00974_attach_invalid_parts.sh | 21 ++++-- .../0_stateless/00975_drop_detached.reference | 6 +- .../0_stateless/00975_drop_detached.sh | 20 +++--- 12 files changed, 130 insertions(+), 88 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 9d12a9ee6ea..4b13ebaa99f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1718,6 +1718,29 @@ MergeTreeData::AlterDataPartTransaction::~AlterDataPartTransaction() } } +void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const String & new_name) +{ + Poco::File(base_dir + old_name).renameTo(base_dir + new_name); + old_and_new_names.push_back({old_name, new_name}); +} + +MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename() +{ + for (const auto & names : old_and_new_names) + { + if (names.first.empty()) + continue; + try + { + Poco::File(base_dir + names.second).renameTo(base_dir + names.first); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } +} + MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( const MergeTreePartInfo & new_part_info, @@ -2600,6 +2623,23 @@ void MergeTreeData::validateDetachedPartName(const String & name) const throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); } +void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Context &) +{ + if (!part) // TODO + throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); + + String part_id = partition->as().value.safeGet(); + validateDetachedPartName(part_id); + if (startsWith(part_id, "attaching_") || startsWith(part_id, "deleting_")) + throw DB::Exception("Cannot drop part " + part_id + ": " + "most likely it is used by another DROP or ATTACH query.", ErrorCodes::BAD_DATA_PART_NAME); + + PartsTemporaryRename renamed_parts(full_path + "detached/"); + renamed_parts.addPart(part_id, "deleting_" + part_id); + Poco::File(renamed_parts.base_dir + renamed_parts.old_and_new_names.front().second).remove(true); + renamed_parts.old_and_new_names.front().first.clear(); +} + MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const { DataParts res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 099591a97e4..62cebf32f76 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -249,6 +249,19 @@ public: using AlterDataPartTransactionPtr = std::unique_ptr; + struct PartsTemporaryRename : private boost::noncopyable + { + PartsTemporaryRename(const String & base_dir_) : base_dir(base_dir_) {} + + /// Renames part from old_name to new_name + void addPart(const String & old_name, const String & new_name); + + /// Renames all added parts from new_name to old_name if old name is not empty + ~PartsTemporaryRename(); + + String base_dir; + std::vector> old_and_new_names; + }; /// Parameters for various modes. struct MergingParams @@ -392,6 +405,8 @@ public: void validateDetachedPartName(const String & name) const; + void dropDetached(const ASTPtr & partition, bool part, const Context & context); + /// Returns Committed parts DataParts getDataParts() const; DataPartsVector getDataPartsVector() const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index a0888732495..fa2847aa301 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -142,10 +142,7 @@ MergeTreeDataPart::MergeTreeDataPart(MergeTreeData & storage_, const String & na { } -MergeTreeDataPart::MergeTreeDataPart( - const MergeTreeData & storage_, - const String & name_, - const MergeTreePartInfo & info_) +MergeTreeDataPart::MergeTreeDataPart(const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_) : storage(storage_) , name(name_) , info(info_) @@ -350,7 +347,7 @@ UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) return res; } -void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const +void MergeTreeDataPart::remove() const { if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); @@ -359,18 +356,18 @@ void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const * - rename directory to temporary name; * - remove it recursive. * - * For temporary name we use "detached/deleting_" prefix. + * For temporary name we use "delete_tmp_" prefix. * - * NOTE: We cannot use "tmp_*" prefix, because there is a second thread, + * NOTE: We cannot use "tmp_delete_" prefix, because there is a second thread, * that calls "clearOldTemporaryDirectories" and removes all directories, that begin with "tmp_" and are old enough. * But when we removing data part, it can be old enough. And rename doesn't change mtime. * And a race condition can happen that will lead to "File not found" error here. - * We move directory to detached/, because if an attempt to remove directory after renaming failed for some reason - * there would be no way to remove directory from storage.full_path (except manually). */ + // TODO directory delete_tmp_ is never removed if server crashes before returning from this function + String from = storage.full_path + relative_path; - String to = storage.full_path + getRelativePathForDetachedPart("deleting_"); + String to = storage.full_path + "delete_tmp_" + name; Poco::File from_dir{from}; Poco::File to_dir{to}; @@ -403,13 +400,6 @@ void MergeTreeDataPart::remove(bool force_recursive /*= false*/) const return; } - if (force_recursive) - { - /// Part is not loaded (we don't know which files are there), so remove dir recursively. - to_dir.remove(true); - return; - } - try { /// Remove each expected file in directory, then remove directory itself. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h index 98af00c071a..f41ea8af424 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h @@ -241,7 +241,7 @@ struct MergeTreeDataPart /// Calculate the total size of the entire directory with all the files static UInt64 calculateTotalSizeOnDisk(const String & from); - void remove(bool force_recursive = false) const; + void remove() const; /// Makes checks and move part to new directory /// Changes only relative_dir_name, you need to update other metadata (name, is_temp) explicitly diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index a9e31a988b3..449ea143e17 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -194,6 +194,7 @@ bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, Detache part_info.dir_name = dir_name; /// First, try to parse as . + // TODO what if tryParsePartName will parse prefix as partition_id? if (MergeTreePartInfo::tryParsePartName(dir_name, &part_info, format_version)) return part_info.valid_name = true; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 98b3ae6ad6c..db5632c3fe9 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -994,6 +994,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons /// TODO: should we include PreComitted parts like in Replicated case? auto parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); + // TODO should we throw an exception if parts_to_remove is empty? removePartsFromWorkingSet(parts_to_remove, true); if (detach) @@ -1013,22 +1014,6 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } -void StorageMergeTree::dropDetached(const ASTPtr & partition, bool part, const Context & /*context*/) -{ - if (!part) // TODO - throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); - - String part_id = partition->as().value.safeGet(); - validateDetachedPartName(part_id); - - DetachedPartInfo info; - DetachedPartInfo::tryParseDetachedPartName(part_id, info, format_version); - MergeTreeDataPart detached_part(*this, part_id, info); - detached_part.relative_path = "detached/" + part_id; - - detached_part.remove(true); -} - void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_part, const Context & context) { // TODO: should get some locks to prevent race with 'alter … modify column' @@ -1069,42 +1054,30 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par } LOG_DEBUG(log, active_parts.size() << " of them are active"); parts = active_parts.getParts(); + + // TODO should we rename inactive parts? (see StorageReplicatedMergeTree::attachPartition) } + PartsTemporaryRename renamed_parts(full_path + source_dir); for (const auto & source_part_name : parts) + renamed_parts.addPart(source_part_name, "attaching_" + source_part_name); + + std::vector loaded_parts; + for (const auto & part_names : renamed_parts.old_and_new_names) { - MutableDataPartPtr part; - try - { - part = std::make_shared(*this, source_part_name); - part->relative_path = "detached/" + source_part_name; - part->renameTo("detached/attaching_" + source_part_name, false); - - LOG_DEBUG(log, "Checking data in " << part->relative_path); + LOG_DEBUG(log, "Checking data in " << part_names.second); + MutableDataPartPtr part = std::make_shared(*this, part_names.first); + part->relative_path = source_dir + part_names.second; loadPartAndFixMetadata(part); + loaded_parts.push_back(part); + } - LOG_INFO(log, "Attaching part " << source_part_name << " from " << part->relative_path); - renameTempPartAndAdd(part, &increment); - - LOG_INFO(log, "Finished attaching part"); - } - catch (...) - { - LOG_INFO(log, "Cannot attach part " << source_part_name << " :" << getCurrentExceptionMessage(false)); - - if (part && part->relative_path == "detached/attaching_" + source_part_name) - { - try - { - part->renameTo("detached/" + source_part_name, false); - } - catch (...) - { - tryLogCurrentException(log, __PRETTY_FUNCTION__); - } - } - - } + for (size_t i = 0; i < loaded_parts.size(); ++i) + { + LOG_INFO(log, "Attaching part " << loaded_parts[i]->name << " from " << renamed_parts.old_and_new_names[i].second); + renameTempPartAndAdd(loaded_parts[i], &increment); + renamed_parts.old_and_new_names[i].first.clear(); + LOG_INFO(log, "Finished attaching part"); } /// New parts with other data may appear in place of deleted parts. diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index fa2561e4ab2..0de9618d915 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -120,7 +120,6 @@ private: // Partition helpers void dropPartition(const ASTPtr & partition, bool detach, const Context & context); - void dropDetached(const ASTPtr & partition, bool part, const Context & context); void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context); void attachPartition(const ASTPtr & partition, bool part, const Context & context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 531678decc3..67577dee2b6 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3353,8 +3353,8 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part break; case PartitionCommand::DROP_DETACHED_PARTITION: - // TODO - throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); + dropDetached(command.partition, command.part, query_context); + break; case PartitionCommand::ATTACH_PARTITION: attachPartition(command.partition, command.part, query_context); @@ -3601,6 +3601,8 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool loaded_parts.push_back(loadPartAndFixMetadata(source_dir + part)); } + // TODO fix race with DROP DETACHED + ReplicatedMergeTreeBlockOutputStream output(*this, 0, 0, 0, false); /// TODO Allow to use quorum here. for (auto & part : loaded_parts) { diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference index 42a04fe5666..f30fc160dfb 100644 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference @@ -1,17 +1,26 @@ === cannot attach active === -OK +OK1 0_1_1_0 1_2_2_0 2_3_3_0 3_4_4_0 16 120 -=== attach all valid parts === +=== check all parts before attaching === +OK2 +1_2_2_0 +1_4_4_0 +=== detached === +0_1_1_0 +0_3_3_0 +0_5_5_0 +attaching_0_6_6_0 +deleting_0_7_7_0 +=== attach === 0_5_5_0 0_6_6_0 1_2_2_0 1_4_4_0 16 120 === detached === -0_5_5_0 -deleting_0_7_7_0 attaching_0_6_6_0 +deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh index 4e9efa64ad1..a4afbe8f817 100755 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh @@ -12,14 +12,13 @@ echo '=== cannot attach active ==='; $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_active"; $CLICKHOUSE_CLIENT --query="CREATE TABLE attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; $CLICKHOUSE_CLIENT --query="INSERT INTO attach_active SELECT number FROM system.numbers LIMIT 16"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_active ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_active ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' $CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_active' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_active FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE attach_active"; -echo '=== attach all valid parts ==='; $CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_partitions"; $CLICKHOUSE_CLIENT --query="CREATE TABLE attach_partitions (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; @@ -27,18 +26,28 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM sys $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; -sudo -n mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ - mkdir $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; +echo '=== check all parts before attaching ==='; +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0" 2>&1 | grep "No columns in part 0_5_5_0" > /dev/null && echo 'OK2'; +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; +echo '=== detached ==='; +$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; + +echo '=== attach ==='; +sudo -n rm -r $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ + rm -r $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ +$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; $CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_partitions FORMAT TSV"; + echo '=== detached ==='; -$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE attach_partitions"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.reference b/dbms/tests/queries/0_stateless/00975_drop_detached.reference index 40732c908ab..2a355138980 100644 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.reference +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.reference @@ -1,2 +1,6 @@ -OK +OK1 +OK2 +OK3 0_3_3_0 +attaching_0_6_6_0 +deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh index 3a5e920da75..71c0b5681fd 100755 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.sh +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.sh @@ -15,19 +15,19 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system. $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; -sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ - mkdir $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ -sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ - mkdir $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ -sudo -n mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ - mkdir $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK' +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK2' +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK3' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' FORMAT TSV"; +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; $CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From c6717e0d3f977e23aa9b778d00bcf64456e893cb Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 30 Jul 2019 22:11:15 +0300 Subject: [PATCH 050/133] refactor attachPartition --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 71 +++++++++++++++++++ dbms/src/Storages/MergeTree/MergeTreeData.h | 3 + dbms/src/Storages/StorageMergeTree.cpp | 55 +------------- .../Storages/StorageReplicatedMergeTree.cpp | 68 ++---------------- 4 files changed, 83 insertions(+), 114 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 4b13ebaa99f..11ad7835b51 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2640,6 +2640,77 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Cont renamed_parts.old_and_new_names.front().first.clear(); } +MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part, + const Context & context, PartsTemporaryRename & renamed_parts) +{ + String partition_id; + + if (attach_part) + partition_id = partition->as().value.safeGet(); + else + partition_id = getPartitionIDFromQuery(partition, context); + + String source_dir = "detached/"; + + /// Let's compose a list of parts that should be added. + Strings parts; + if (attach_part) + { + validateDetachedPartName(partition_id); + parts.push_back(partition_id); + } + else + { + LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); + ActiveDataPartSet active_parts(format_version); + + std::set part_names; + for (Poco::DirectoryIterator it = Poco::DirectoryIterator(full_path + source_dir); it != Poco::DirectoryIterator(); ++it) + { + String name = it.name(); + MergeTreePartInfo part_info; + // TODO what if name contains "_tryN" suffix? + if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version)) + continue; + if (part_info.partition_id != partition_id) + continue; + LOG_DEBUG(log, "Found part " << name); + active_parts.add(name); + part_names.insert(name); + } + LOG_DEBUG(log, active_parts.size() << " of them are active"); + parts = active_parts.getParts(); + + /// Inactive parts rename so they can not be attached in case of repeated ATTACH. + for (const auto & name : part_names) + { + // TODO maybe use PartsTemporaryRename here? + String containing_part = active_parts.getContainingPart(name); + if (!containing_part.empty() && containing_part != name) + Poco::File(full_path + source_dir + name).renameTo(full_path + source_dir + "inactive_" + name); + } + } + + /// Try to rename all parts before attaching to prevent race with DROP DETACHED and another ATTACH. + for (const auto & source_part_name : parts) + renamed_parts.addPart(source_part_name, "attaching_" + source_part_name); + + /// Synchronously check that added parts exist and are not broken. We will write checksums.txt if it does not exist. + LOG_DEBUG(log, "Checking parts"); + MutableDataPartsVector loaded_parts; + loaded_parts.reserve(parts.size()); + for (const auto & part_names : renamed_parts.old_and_new_names) + { + LOG_DEBUG(log, "Checking part " << part_names.second); + MutableDataPartPtr part = std::make_shared(*this, part_names.first); + part->relative_path = source_dir + part_names.second; + loadPartAndFixMetadata(part); + loaded_parts.push_back(part); + } + + return loaded_parts; +} + MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const { DataParts res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 62cebf32f76..3592164fed5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -407,6 +407,9 @@ public: void dropDetached(const ASTPtr & partition, bool part, const Context & context); + MutableDataPartsVector tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part, + const Context & context, PartsTemporaryRename & renamed_parts); + /// Returns Committed parts DataParts getDataParts() const; DataPartsVector getDataPartsVector() const; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index db5632c3fe9..3464255e1b8 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -1018,59 +1018,8 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par { // TODO: should get some locks to prevent race with 'alter … modify column' - String partition_id; - - if (attach_part) - partition_id = partition->as().value.safeGet(); - else - partition_id = getPartitionIDFromQuery(partition, context); - - String source_dir = "detached/"; - - /// Let's make a list of parts to add. - Strings parts; - if (attach_part) - { - validateDetachedPartName(partition_id); - parts.push_back(partition_id); - } - else - { - LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); - ActiveDataPartSet active_parts(format_version); - for (Poco::DirectoryIterator it = Poco::DirectoryIterator(full_path + source_dir); it != Poco::DirectoryIterator(); ++it) - { - const String & name = it.name(); - MergeTreePartInfo part_info; - /// Parts with prefix in name (e.g. attaching_1_3_3_0, deleting_1_3_3_0) will be ignored - // TODO what if name contains "_tryN" suffix? - if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version) - || part_info.partition_id != partition_id) - { - continue; - } - LOG_DEBUG(log, "Found part " << name); - active_parts.add(name); - } - LOG_DEBUG(log, active_parts.size() << " of them are active"); - parts = active_parts.getParts(); - - // TODO should we rename inactive parts? (see StorageReplicatedMergeTree::attachPartition) - } - - PartsTemporaryRename renamed_parts(full_path + source_dir); - for (const auto & source_part_name : parts) - renamed_parts.addPart(source_part_name, "attaching_" + source_part_name); - - std::vector loaded_parts; - for (const auto & part_names : renamed_parts.old_and_new_names) - { - LOG_DEBUG(log, "Checking data in " << part_names.second); - MutableDataPartPtr part = std::make_shared(*this, part_names.first); - part->relative_path = source_dir + part_names.second; - loadPartAndFixMetadata(part); - loaded_parts.push_back(part); - } + PartsTemporaryRename renamed_parts(full_path + "detached/"); + MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, context, renamed_parts); for (size_t i = 0; i < loaded_parts.size(); ++i) { diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 67577dee2b6..7e192d77a33 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3545,70 +3545,16 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool assertNotReadonly(); - String partition_id; - - if (attach_part) - partition_id = partition->as().value.safeGet(); - else - partition_id = getPartitionIDFromQuery(partition, query_context); - - String source_dir = "detached/"; - - /// Let's compose a list of parts that should be added. - Strings parts; - if (attach_part) - { - validateDetachedPartName(partition_id); - parts.push_back(partition_id); - } - else - { - LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); - ActiveDataPartSet active_parts(format_version); - - std::set part_names; - for (Poco::DirectoryIterator it = Poco::DirectoryIterator(full_path + source_dir); it != Poco::DirectoryIterator(); ++it) - { - String name = it.name(); - MergeTreePartInfo part_info; - // TODO what if name contains "_tryN" suffix? - if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version)) - continue; - if (part_info.partition_id != partition_id) - continue; - LOG_DEBUG(log, "Found part " << name); - active_parts.add(name); - part_names.insert(name); - } - LOG_DEBUG(log, active_parts.size() << " of them are active"); - parts = active_parts.getParts(); - - /// Inactive parts rename so they can not be attached in case of repeated ATTACH. - for (const auto & name : part_names) - { - String containing_part = active_parts.getContainingPart(name); - if (!containing_part.empty() && containing_part != name) - Poco::File(full_path + source_dir + name).renameTo(full_path + source_dir + "inactive_" + name); - } - } - - /// Synchronously check that added parts exist and are not broken. We will write checksums.txt if it does not exist. - LOG_DEBUG(log, "Checking parts"); - std::vector loaded_parts; - for (const String & part : parts) - { - LOG_DEBUG(log, "Checking part " << part); - loaded_parts.push_back(loadPartAndFixMetadata(source_dir + part)); - } - - // TODO fix race with DROP DETACHED + PartsTemporaryRename renamed_parts(full_path + "detached/"); + MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, query_context, renamed_parts); ReplicatedMergeTreeBlockOutputStream output(*this, 0, 0, 0, false); /// TODO Allow to use quorum here. - for (auto & part : loaded_parts) + for (size_t i = 0; i < loaded_parts.size(); ++i) { - String old_name = part->name; - output.writeExistingPart(part); - LOG_DEBUG(log, "Attached part " << old_name << " as " << part->name); + String old_name = loaded_parts[i]->name; + output.writeExistingPart(loaded_parts[i]); + renamed_parts.old_and_new_names[i].first.clear(); + LOG_DEBUG(log, "Attached part " << old_name << " as " << loaded_parts[i]->name); } } From f0836553d449368cac474e9ed9f60d3120ea79c4 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 31 Jul 2019 17:44:55 +0300 Subject: [PATCH 051/133] drop detached partition --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 94 +++++++++++++------ dbms/src/Storages/MergeTree/MergeTreeData.h | 10 +- .../Storages/MergeTree/MergeTreePartInfo.cpp | 2 +- .../Storages/MergeTree/MergeTreePartInfo.h | 3 +- dbms/src/Storages/PartitionCommands.cpp | 3 - dbms/src/Storages/StorageMergeTree.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 2 +- .../0_stateless/00974_attach_invalid_parts.sh | 8 +- .../0_stateless/00975_drop_detached.reference | 9 ++ .../0_stateless/00975_drop_detached.sh | 12 +++ 10 files changed, 103 insertions(+), 42 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 11ad7835b51..32cd3ad508e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1720,12 +1720,35 @@ MergeTreeData::AlterDataPartTransaction::~AlterDataPartTransaction() void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const String & new_name) { - Poco::File(base_dir + old_name).renameTo(base_dir + new_name); old_and_new_names.push_back({old_name, new_name}); } +void MergeTreeData::PartsTemporaryRename::tryRenameAll() +{ + renamed = true; + for (size_t i = 0; i < old_and_new_names.size(); ++i) + { + try + { + const auto & names = old_and_new_names[i]; + if (names.first.empty() || names.second.empty()) + throw DB::Exception("Empty part name. Most likely it's a bug.", ErrorCodes::INCORRECT_FILE_NAME); + Poco::File(base_dir + names.first).renameTo(base_dir + names.second); + } + catch (...) + { + old_and_new_names.resize(i); + LOG_WARNING(storage.log, "Cannot rename parts to perform operation on them: " << getCurrentExceptionMessage(false)); + throw; + } + } +} + MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename() { + // TODO what if server had crashed before this destructor was called? + if (!renamed) + return; for (const auto & names : old_and_new_names) { if (names.first.empty()) @@ -2621,46 +2644,60 @@ void MergeTreeData::validateDetachedPartName(const String & name) const Poco::File detached_part_dir(full_path + "detached/" + name); if (!detached_part_dir.exists()) throw DB::Exception("Detached part \"" + name + "\" not found" , ErrorCodes::BAD_DATA_PART_NAME); + + if (startsWith(name, "attaching_") || startsWith(name, "deleting_")) + throw DB::Exception("Cannot drop part " + name + ": " + "most likely it is used by another DROP or ATTACH query.", + ErrorCodes::BAD_DATA_PART_NAME); } -void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Context &) +void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, const Context & context) { - if (!part) // TODO - throw DB::Exception("DROP DETACHED PARTITION is not implemented, use DROP DETACHED PART", ErrorCodes::NOT_IMPLEMENTED); + PartsTemporaryRename renamed_parts(*this, full_path + "detached/"); - String part_id = partition->as().value.safeGet(); - validateDetachedPartName(part_id); - if (startsWith(part_id, "attaching_") || startsWith(part_id, "deleting_")) - throw DB::Exception("Cannot drop part " + part_id + ": " - "most likely it is used by another DROP or ATTACH query.", ErrorCodes::BAD_DATA_PART_NAME); + if (part) + { + String part_name = partition->as().value.safeGet(); + validateDetachedPartName(part_name); + renamed_parts.addPart(part_name, "deleting_" + part_name); + } + else + { + String partition_id = getPartitionIDFromQuery(partition, context); + DetachedPartsInfo detached_parts = getDetachedParts(); + for (const auto & part_info : detached_parts) + if (part_info.valid_name && part_info.partition_id == partition_id + && part_info.prefix != "attaching" && part_info.prefix != "deleting") + renamed_parts.addPart(part_info.dir_name, "deleting_" + part_info.dir_name); + } - PartsTemporaryRename renamed_parts(full_path + "detached/"); - renamed_parts.addPart(part_id, "deleting_" + part_id); - Poco::File(renamed_parts.base_dir + renamed_parts.old_and_new_names.front().second).remove(true); - renamed_parts.old_and_new_names.front().first.clear(); + LOG_DEBUG(log, "Will drop " << renamed_parts.old_and_new_names.size() << " detached parts."); + + renamed_parts.tryRenameAll(); + + for (auto & names : renamed_parts.old_and_new_names) + { + Poco::File(renamed_parts.base_dir + names.second).remove(true); + LOG_DEBUG(log, "Dropped detached part " << names.first); + names.first.clear(); + } } MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part, const Context & context, PartsTemporaryRename & renamed_parts) { - String partition_id; - - if (attach_part) - partition_id = partition->as().value.safeGet(); - else - partition_id = getPartitionIDFromQuery(partition, context); - String source_dir = "detached/"; /// Let's compose a list of parts that should be added. - Strings parts; if (attach_part) { - validateDetachedPartName(partition_id); - parts.push_back(partition_id); + String part_id = partition->as().value.safeGet(); + validateDetachedPartName(part_id); + renamed_parts.addPart(part_id, "attaching_" + part_id); } else { + String partition_id = getPartitionIDFromQuery(partition, context); LOG_DEBUG(log, "Looking for parts for partition " << partition_id << " in " << source_dir); ActiveDataPartSet active_parts(format_version); @@ -2670,6 +2707,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const String name = it.name(); MergeTreePartInfo part_info; // TODO what if name contains "_tryN" suffix? + /// Parts with prefix in name (e.g. attaching_1_3_3_0, deleting_1_3_3_0) will be ignored if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version)) continue; if (part_info.partition_id != partition_id) @@ -2679,26 +2717,26 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const part_names.insert(name); } LOG_DEBUG(log, active_parts.size() << " of them are active"); - parts = active_parts.getParts(); /// Inactive parts rename so they can not be attached in case of repeated ATTACH. for (const auto & name : part_names) { - // TODO maybe use PartsTemporaryRename here? String containing_part = active_parts.getContainingPart(name); if (!containing_part.empty() && containing_part != name) + // TODO maybe use PartsTemporaryRename here? Poco::File(full_path + source_dir + name).renameTo(full_path + source_dir + "inactive_" + name); + else + renamed_parts.addPart(name, "attaching_" + name); } } /// Try to rename all parts before attaching to prevent race with DROP DETACHED and another ATTACH. - for (const auto & source_part_name : parts) - renamed_parts.addPart(source_part_name, "attaching_" + source_part_name); + renamed_parts.tryRenameAll(); /// Synchronously check that added parts exist and are not broken. We will write checksums.txt if it does not exist. LOG_DEBUG(log, "Checking parts"); MutableDataPartsVector loaded_parts; - loaded_parts.reserve(parts.size()); + loaded_parts.reserve(renamed_parts.old_and_new_names.size()); for (const auto & part_names : renamed_parts.old_and_new_names) { LOG_DEBUG(log, "Checking part " << part_names.second); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 3592164fed5..9f5d0961d27 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -251,16 +251,20 @@ public: struct PartsTemporaryRename : private boost::noncopyable { - PartsTemporaryRename(const String & base_dir_) : base_dir(base_dir_) {} + PartsTemporaryRename(const MergeTreeData & storage_, const String & base_dir_) : storage(storage_), base_dir(base_dir_) {} + + void addPart(const String & old_name, const String & new_name); /// Renames part from old_name to new_name - void addPart(const String & old_name, const String & new_name); + void tryRenameAll(); /// Renames all added parts from new_name to old_name if old name is not empty ~PartsTemporaryRename(); + const MergeTreeData & storage; String base_dir; std::vector> old_and_new_names; + bool renamed = false; }; /// Parameters for various modes. @@ -401,7 +405,7 @@ public: DataPartsVector getAllDataPartsVector(DataPartStateVector * out_states = nullptr) const; /// Returns all detached parts - std::vector getDetachedParts() const; + DetachedPartsInfo getDetachedParts() const; void validateDetachedPartName(const String & name) const; diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 449ea143e17..3ee330b6d1a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -194,7 +194,7 @@ bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, Detache part_info.dir_name = dir_name; /// First, try to parse as . - // TODO what if tryParsePartName will parse prefix as partition_id? + // TODO what if tryParsePartName will parse prefix as partition_id? It can happen if dir_name doesn't contain mutation number at the end if (MergeTreePartInfo::tryParsePartName(dir_name, &part_info, format_version)) return part_info.valid_name = true; diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index 25cf46ad46d..9fe0fbab533 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -92,7 +92,6 @@ struct MergeTreePartInfo /// addition to the above fields. struct DetachedPartInfo : public MergeTreePartInfo { - /// Suddenly, name of detached part may contain suffix (such as _tryN), which is ignored by MergeTreePartInfo::tryParsePartName(...) String dir_name; String prefix; @@ -102,4 +101,6 @@ struct DetachedPartInfo : public MergeTreePartInfo static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version); }; +using DetachedPartsInfo = std::vector; + } diff --git a/dbms/src/Storages/PartitionCommands.cpp b/dbms/src/Storages/PartitionCommands.cpp index bab3f6ced24..0537482dbc1 100644 --- a/dbms/src/Storages/PartitionCommands.cpp +++ b/dbms/src/Storages/PartitionCommands.cpp @@ -25,9 +25,6 @@ std::optional PartitionCommand::parse(const ASTAlterCommand * } else if (command_ast->type == ASTAlterCommand::DROP_DETACHED_PARTITION) { - if (!command_ast->part) // TODO - throw DB::Exception("Not implemented yet", ErrorCodes::NOT_IMPLEMENTED); - PartitionCommand res; res.type = DROP_DETACHED_PARTITION; res.partition = command_ast->partition; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 3464255e1b8..c2ee4854c39 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -1018,7 +1018,7 @@ void StorageMergeTree::attachPartition(const ASTPtr & partition, bool attach_par { // TODO: should get some locks to prevent race with 'alter … modify column' - PartsTemporaryRename renamed_parts(full_path + "detached/"); + PartsTemporaryRename renamed_parts(*this, full_path + "detached/"); MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, context, renamed_parts); for (size_t i = 0; i < loaded_parts.size(); ++i) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 7e192d77a33..5109d9f7e54 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3545,7 +3545,7 @@ void StorageReplicatedMergeTree::attachPartition(const ASTPtr & partition, bool assertNotReadonly(); - PartsTemporaryRename renamed_parts(full_path + "detached/"); + PartsTemporaryRename renamed_parts(*this, full_path + "detached/"); MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, query_context, renamed_parts); ReplicatedMergeTreeBlockOutputStream output(*this, 0, 0, 0, false); /// TODO Allow to use quorum here. diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh index a4afbe8f817..db45cfe7f21 100755 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh +++ b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh @@ -28,10 +28,10 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM sys $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part -sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ - cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ -sudo -n cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ - cp -r $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ +sudo -n cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ + cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ +sudo -n cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ + cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ echo '=== check all parts before attaching ==='; $CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0" 2>&1 | grep "No columns in part 0_5_5_0" > /dev/null && echo 'OK2'; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.reference b/dbms/tests/queries/0_stateless/00975_drop_detached.reference index 2a355138980..414ac4b1927 100644 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.reference +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.reference @@ -1,6 +1,15 @@ +=== validate part name === OK1 OK2 OK3 +=== drop detached part === +0_3_3_0 +1_2_2_0 +1_4_4_0 +attaching_0_6_6_0 +deleting_0_7_7_0 +prefix_1_2_2_0_0 +=== drop detached partition === 0_3_3_0 attaching_0_6_6_0 deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh index 71c0b5681fd..8da831b019a 100755 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.sh +++ b/dbms/tests/queries/0_stateless/00975_drop_detached.sh @@ -15,19 +15,31 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system. $CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; $CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; +$CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 1"; sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ +sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0_0/ 2>/dev/null || \ + mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0_0/ +#sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0/ 2>/dev/null || \ +# mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0/ +echo '=== validate part name ===' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK2' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK3' $CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" +echo '=== drop detached part ===' $CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; + +echo '=== drop detached partition ===' +$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PARTITION 1" +$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; + $CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; $CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From 8e535a9cb0fe66b44de1be0f5537827aad8d7767 Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Mon, 8 Jul 2019 12:41:28 +0300 Subject: [PATCH 052/133] Implement nextInBlock function --- dbms/src/Functions/nextInBlock.cpp | 159 ++++++++++++++++++ .../registerFunctionsMiscellaneous.cpp | 2 + .../0_stateless/00957_next_in_block.reference | 12 ++ .../0_stateless/00957_next_in_block.sql | 22 +++ 4 files changed, 195 insertions(+) create mode 100644 dbms/src/Functions/nextInBlock.cpp create mode 100644 dbms/tests/queries/0_stateless/00957_next_in_block.reference create mode 100644 dbms/tests/queries/0_stateless/00957_next_in_block.sql diff --git a/dbms/src/Functions/nextInBlock.cpp b/dbms/src/Functions/nextInBlock.cpp new file mode 100644 index 00000000000..e672e539f25 --- /dev/null +++ b/dbms/src/Functions/nextInBlock.cpp @@ -0,0 +1,159 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +// Implements function, giving value for column in next row +// Example: +// | c1 | +// | 10 | +// | 20 | +// SELECT c1, nextInBlock(c1, 1) as c2: +// | c1 | c2 | +// | 10 | 20 | +// | 20 | 0 | +class FunctionNextInBlock : public IFunction +{ +public: + static constexpr auto name = "nextInBlock"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + /// Get the name of the function. + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isVariadic() const override { return true; } + + bool isDeterministic() const override { return false; } + + bool isDeterministicInScopeOfQuery() const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + size_t number_of_arguments = arguments.size(); + + if (number_of_arguments < 1 || number_of_arguments > 3) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(number_of_arguments) + + ", should be from 1 to 3", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + // second argument must be a positive, constant column + if (number_of_arguments == 2 && !isUnsignedInteger(arguments[1])) + throw Exception( + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + + " - should be positive integer", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + // check that default value has supertype with first argument + if (number_of_arguments == 3) + { + DataTypes types = {arguments[0], arguments[2]}; + try + { + return getLeastSupertype(types); + } + catch (const Exception &) + { + throw Exception( + "Illegal types of arguments (" + types[0]->getName() + ", " + types[1]->getName() + + ")" + " of function " + + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + + return arguments[0]; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + size_t offset_value = 1; + + if (arguments.size() > 1) + { + auto offset_column = block.getByPosition(arguments[1]); + if (!isColumnConst(*offset_column.column)) + throw Exception("Second argument of function " + getName() + " should be constant", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + Field offset_field = (*block.getByPosition(arguments[1]).column)[0]; + auto raw_value = safeGet(offset_field); + + if (raw_value == 0) + throw Exception( + "Second argument of function " + getName() + " should be positive integer, " + toString(raw_value) + " given", + ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + offset_value = raw_value; + } + + auto has_column_for_missing = arguments.size() == 3; + + DataTypes types = {block.getByPosition(arguments[0]).type}; + if (has_column_for_missing) + { + types.push_back(block.getByPosition(arguments[2]).type); + } + const DataTypePtr & result_type = getLeastSupertype(types); + + auto column = result_type->createColumn(); + column->reserve(input_rows_count); + + auto source_column = block.getByPosition(arguments[0]).column; + + for (size_t i = offset_value; i < input_rows_count; i++) + { + column->insertFrom(*source_column, i); + } + + if (has_column_for_missing) + { + auto default_values_column = block.getByPosition(arguments[2]).column; + size_t starting_pos = offset_value > input_rows_count ? 0 : input_rows_count - offset_value; + if (isColumnConst(*default_values_column)) + { + Field constant_value = (*default_values_column)[0]; + for (size_t i = starting_pos; i < input_rows_count; i++) + { + column->insert(constant_value); + } + } + else + { + for (size_t i = starting_pos; i < input_rows_count; i++) + { + column->insertFrom(*default_values_column, i); + } + } + } + else + { + for (size_t i = 0; i < std::min(offset_value, input_rows_count); i++) + { + column->insertDefault(); + } + } + + block.getByPosition(result).column = std::move(column); + } +}; + +void registerFunctionNextInBlock(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} \ No newline at end of file diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp index 6d201d65bd3..57ccfcd11c9 100644 --- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp @@ -17,6 +17,7 @@ void registerFunctionBlockSize(FunctionFactory &); void registerFunctionBlockNumber(FunctionFactory &); void registerFunctionRowNumberInBlock(FunctionFactory &); void registerFunctionRowNumberInAllBlocks(FunctionFactory &); +void registerFunctionNextInBlock(FunctionFactory &); void registerFunctionSleep(FunctionFactory &); void registerFunctionSleepEachRow(FunctionFactory &); void registerFunctionMaterialize(FunctionFactory &); @@ -67,6 +68,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionBlockNumber(factory); registerFunctionRowNumberInBlock(factory); registerFunctionRowNumberInAllBlocks(factory); + registerFunctionNextInBlock(factory); registerFunctionSleep(factory); registerFunctionSleepEachRow(factory); registerFunctionMaterialize(factory); diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.reference b/dbms/tests/queries/0_stateless/00957_next_in_block.reference new file mode 100644 index 00000000000..860ce6dc1ba --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_next_in_block.reference @@ -0,0 +1,12 @@ +0 1 +1 0 +0 2 +1 0 +2 0 +0 0 +1 0 +0 2 +1 3 +2 4 +3 1000 +4 1000 diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.sql b/dbms/tests/queries/0_stateless/00957_next_in_block.sql new file mode 100644 index 00000000000..7cbd932cf1a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_next_in_block.sql @@ -0,0 +1,22 @@ +-- no arguments +select nextInBlock(); -- { serverError 42 } +-- greater than 3 arguments +select nextInBlock(1,2,3,4); -- { serverError 42 } +-- zero offset value +select nextInBlock(dummy, 0); -- { serverError 69 } +-- negative offset value +select nextInBlock(dummy, -1); -- { serverError 43 } +-- non-constant offset value +select nextInBlock(dummy, dummy); -- { serverError 43 } +-- bad default value +select nextInBlock(dummy, 1, 'hello'); -- { serverError 43 } +-- single argument test +select number, nextInBlock(number) from numbers(2); +-- filling by column's default value +select number, nextInBlock(number, 2) from numbers(3); +-- offset is greater that block - should fill everything with defaults +select number, nextInBlock(number, 5) from numbers(2); +-- substitution by constant for missing values +select number, nextInBlock(number, 2, 1000) from numbers(5); +-- substitution by expression +-- select number, nextInBlock(number, 2, number % 2) from numbers(5); \ No newline at end of file From cfec857f2c5685aafc1aeaa061497baf9ab39c53 Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Mon, 8 Jul 2019 17:53:02 +0300 Subject: [PATCH 053/133] Add trailing newline --- dbms/src/Functions/nextInBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Functions/nextInBlock.cpp b/dbms/src/Functions/nextInBlock.cpp index e672e539f25..eeb33e28146 100644 --- a/dbms/src/Functions/nextInBlock.cpp +++ b/dbms/src/Functions/nextInBlock.cpp @@ -156,4 +156,4 @@ void registerFunctionNextInBlock(FunctionFactory & factory) factory.registerFunction(); } -} \ No newline at end of file +} From 2f36d80705d5a62c8efc15c623de20250cee1537 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 8 Aug 2019 18:51:17 +0300 Subject: [PATCH 054/133] move tests with sudo to integration tests --- .../integration/test_partition/__init__.py | 0 dbms/tests/integration/test_partition/test.py | 244 ++++++++++++++++++ .../0_stateless/00428_partition.reference | 54 ---- .../queries/0_stateless/00428_partition.sh | 60 ----- .../00974_attach_invalid_parts.reference | 26 -- .../0_stateless/00974_attach_invalid_parts.sh | 53 ---- .../0_stateless/00975_drop_detached.reference | 15 -- .../0_stateless/00975_drop_detached.sh | 45 ---- 8 files changed, 244 insertions(+), 253 deletions(-) create mode 100644 dbms/tests/integration/test_partition/__init__.py create mode 100644 dbms/tests/integration/test_partition/test.py delete mode 100644 dbms/tests/queries/0_stateless/00428_partition.reference delete mode 100755 dbms/tests/queries/0_stateless/00428_partition.sh delete mode 100644 dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference delete mode 100755 dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh delete mode 100644 dbms/tests/queries/0_stateless/00975_drop_detached.reference delete mode 100755 dbms/tests/queries/0_stateless/00975_drop_detached.sh diff --git a/dbms/tests/integration/test_partition/__init__.py b/dbms/tests/integration/test_partition/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_partition/test.py b/dbms/tests/integration/test_partition/test.py new file mode 100644 index 00000000000..59c48e5d9e9 --- /dev/null +++ b/dbms/tests/integration/test_partition/test.py @@ -0,0 +1,244 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') +q = instance.query +path_to_data = '/var/lib/clickhouse/' + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + q('CREATE DATABASE test') + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture +def partition_table_simple(started_cluster): + q("DROP TABLE IF EXISTS test.partition") + q("CREATE TABLE test.partition (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) " + "ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) " + "SETTINGS index_granularity=8192, index_granularity_bytes=0") + q("INSERT INTO test.partition ( x ) VALUES ( now() )") + q("INSERT INTO test.partition ( x ) VALUES ( now()+1 )") + + yield + + q('DROP TABLE test.partition') + + +def test_partition_simple(partition_table_simple): + q("ALTER TABLE test.partition DETACH PARTITION 197001") + q("ALTER TABLE test.partition ATTACH PARTITION 197001") + q("OPTIMIZE TABLE test.partition") + + +def exec_bash(cmd): + cmd = '/bin/bash -c "{}"'.format(cmd.replace('"', '\\"')) + return instance.exec_in_container(cmd) + + +def partition_complex_assert_columns_txt(): + path_to_parts = path_to_data + 'data/test/partition/' + parts = TSV(q("SELECT name FROM system.parts WHERE database='test' AND table='partition'")) + for part_name in parts.lines: + path_to_columns = path_to_parts + part_name + '/columns.txt' + # 2 header lines + 3 columns + assert exec_bash('cat {} | wc -l'.format(path_to_columns)) == u'5\n' + + +def partition_complex_assert_checksums(): + # Do `cd` for consistent output for reference + # Do not check increment.txt - it can be changed by other tests with FREEZE + cmd = 'cd ' + path_to_data + " && find shadow -type f -exec md5sum {} \\;" \ + " | grep partition" \ + " | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g'" \ + " | sort" \ + " | uniq" + + checksums = "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700102_2_2_0/k.bin\n" \ + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700201_1_1_0/v1.bin\n" \ + "13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition/19700102_2_2_0/minmax_p.idx\n" \ + "25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition/19700102_2_2_0/partition.dat\n" \ + "3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition/19700201_1_1_0/partition.dat\n" \ + "37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition/19700102_2_2_0/checksums.txt\n" \ + "38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition/19700102_2_2_0/v1.bin\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/k.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/p.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/v1.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/k.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/p.mrk\n" \ + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/v1.mrk\n" \ + "55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition/19700201_1_1_0/primary.idx\n" \ + "5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition/19700201_1_1_0/checksums.txt\n" \ + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700102_2_2_0/columns.txt\n" \ + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700201_1_1_0/columns.txt\n" \ + "88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition/19700201_1_1_0/p.bin\n" \ + "9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition/19700102_2_2_0/primary.idx\n" \ + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700102_2_2_0/count.txt\n" \ + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700201_1_1_0/count.txt\n" \ + "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition/19700102_2_2_0/p.bin\n" \ + "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition/19700201_1_1_0/k.bin\n" \ + "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition/19700201_1_1_0/minmax_p.idx\n" + + assert TSV(exec_bash(cmd).replace(' ', '\t')) == TSV(checksums) + + +@pytest.fixture +def partition_table_complex(started_cluster): + q("DROP TABLE IF EXISTS test.partition") + q("CREATE TABLE test.partition (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) " + "ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0") + q("INSERT INTO test.partition (p, k) VALUES(toDate(31), 1)") + q("INSERT INTO test.partition (p, k) VALUES(toDate(1), 2)") + + yield + + q("DROP TABLE test.partition") + + +def test_partition_complex(partition_table_complex): + + partition_complex_assert_columns_txt() + + q("ALTER TABLE test.partition FREEZE") + + partition_complex_assert_checksums() + + q("ALTER TABLE test.partition DETACH PARTITION 197001") + q("ALTER TABLE test.partition ATTACH PARTITION 197001") + + partition_complex_assert_columns_txt() + + q("ALTER TABLE test.partition MODIFY COLUMN v1 Int8") + + # Check the backup hasn't changed + partition_complex_assert_checksums() + + q("OPTIMIZE TABLE test.partition") + + expected = TSV('31\t1\t2\n' + '1\t2\t3') + res = q("SELECT toUInt16(p), k, v1 FROM test.partition ORDER BY k") + assert(TSV(res) == expected) + + +@pytest.fixture +def cannot_attach_active_part_table(started_cluster): + q("DROP TABLE IF EXISTS test.attach_active") + q("CREATE TABLE test.attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n") + q("INSERT INTO test.attach_active SELECT number FROM system.numbers LIMIT 16") + + yield + + q("DROP TABLE test.attach_active") + + +def test_cannot_attach_active_part(cannot_attach_active_part_table): + error = instance.client.query_and_get_error("ALTER TABLE test.attach_active ATTACH PART '../1_2_2_0'") + print error + assert 0 <= error.find('Invalid part name') + + res = q("SElECT name FROM system.parts WHERE table='attach_active' AND database='test' ORDER BY name") + assert TSV(res) == TSV('0_1_1_0\n1_2_2_0\n2_3_3_0\n3_4_4_0') + assert TSV(q("SElECT count(), sum(n) FROM test.attach_active")) == TSV('16\t120') + + +@pytest.fixture +def attach_check_all_parts_table(started_cluster): + q("SYSTEM STOP MERGES") + q("DROP TABLE IF EXISTS test.attach_partition") + q("CREATE TABLE test.attach_partition (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n") + q("INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8") + q("INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8") + + yield + + q("DROP TABLE test.attach_partition") + q("SYSTEM START MERGES") + + +def test_attach_check_all_parts(attach_check_all_parts_table): + q("ALTER TABLE test.attach_partition DETACH PARTITION 0") + + path_to_detached = path_to_data + 'data/test/attach_partition/detached/' + exec_bash('mkdir {}'.format(path_to_detached + '0_5_5_0')) + exec_bash('cp -pr {} {}'.format(path_to_detached + '0_1_1_0', path_to_detached + 'attaching_0_6_6_0')) + exec_bash('cp -pr {} {}'.format(path_to_detached + '0_3_3_0', path_to_detached + 'deleting_0_7_7_0')) + + error = instance.client.query_and_get_error("ALTER TABLE test.attach_partition ATTACH PARTITION 0") + assert 0 <= error.find('No columns in part 0_5_5_0') + + parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(parts) == TSV('1_2_2_0\n1_4_4_0') + detached = q("SELECT name FROM system.detached_parts " + "WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_1_1_0\n0_3_3_0\n0_5_5_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + + exec_bash('rm -r {}'.format(path_to_detached + '0_5_5_0')) + + q("ALTER TABLE test.attach_partition ATTACH PARTITION 0") + parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") + expected = '0_5_5_0\n0_6_6_0\n1_2_2_0\n1_4_4_0' + assert TSV(parts) == TSV(expected) + assert TSV(q("SElECT count(), sum(n) FROM test.attach_partition")) == TSV('16\t120') + + detached = q("SELECT name FROM system.detached_parts " + "WHERE table='attach_partition' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('attaching_0_6_6_0\ndeleting_0_7_7_0') + + +@pytest.fixture +def drop_detached_parts_table(started_cluster): + q("SYSTEM STOP MERGES") + q("DROP TABLE IF EXISTS test.drop_detached") + q("CREATE TABLE test.drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n") + q("INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8") + q("INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8") + + yield + + q("DROP TABLE test.drop_detached") + q("SYSTEM START MERGES") + + +def test_drop_detached_parts(drop_detached_parts_table): + s = {"allow_drop_detached_part": 1} + q("ALTER TABLE test.drop_detached DETACH PARTITION 0") + q("ALTER TABLE test.drop_detached DETACH PARTITION 1") + + path_to_detached = path_to_data + 'data/test/drop_detached/detached/' + exec_bash('mkdir {}'.format(path_to_detached + 'attaching_0_6_6_0')) + exec_bash('mkdir {}'.format(path_to_detached + 'deleting_0_7_7_0')) + exec_bash('mkdir {}'.format(path_to_detached + 'any_other_name')) + exec_bash('mkdir {}'.format(path_to_detached + 'prefix_1_2_2_0_0')) + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0'", settings=s) + assert 0 <= error.find('Invalid part name') + + q("ALTER TABLE test.drop_detached DROP DETACHED PART '0_1_1_0'", settings=s) + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0'", settings=s) + assert 0 <= error.find('Cannot drop part') + + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0'", settings=s) + assert 0 <= error.find('Cannot drop part') + + q("ALTER TABLE test.drop_detached DROP DETACHED PART 'any_other_name'", settings=s) + + detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_3_3_0\n1_2_2_0\n1_4_4_0\nattaching_0_6_6_0\ndeleting_0_7_7_0\nprefix_1_2_2_0_0') + + q("ALTER TABLE test.drop_detached DROP DETACHED PARTITION 1", settings=s) + detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") + assert TSV(detached) == TSV('0_3_3_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + diff --git a/dbms/tests/queries/0_stateless/00428_partition.reference b/dbms/tests/queries/0_stateless/00428_partition.reference deleted file mode 100644 index c777fd7a5c3..00000000000 --- a/dbms/tests/queries/0_stateless/00428_partition.reference +++ /dev/null @@ -1,54 +0,0 @@ -5 -5 -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700102_2_2_0/k.bin -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700201_1_1_0/v1.bin -13cae8e658e0ca4f75c56b1fc424e150 shadow/1/data/test/partition_428/19700102_2_2_0/minmax_p.idx -25daad3d9e60b45043a70c4ab7d3b1c6 shadow/1/data/test/partition_428/19700102_2_2_0/partition.dat -3726312af62aec86b64a7708d5751787 shadow/1/data/test/partition_428/19700201_1_1_0/partition.dat -37855b06a39b79a67ea4e86e4a3299aa shadow/1/data/test/partition_428/19700102_2_2_0/checksums.txt -38e62ff37e1e5064e9a3f605dfe09d13 shadow/1/data/test/partition_428/19700102_2_2_0/v1.bin -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/v1.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/v1.mrk -55a54008ad1ba589aa210d2629c1df41 shadow/1/data/test/partition_428/19700201_1_1_0/primary.idx -5f087cb3e7071bf9407e095821e2af8f shadow/1/data/test/partition_428/19700201_1_1_0/checksums.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700102_2_2_0/columns.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700201_1_1_0/columns.txt -88cdc31ded355e7572d68d8cde525d3a shadow/1/data/test/partition_428/19700201_1_1_0/p.bin -9e688c58a5487b8eaf69c9e1005ad0bf shadow/1/data/test/partition_428/19700102_2_2_0/primary.idx -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700102_2_2_0/count.txt -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700201_1_1_0/count.txt -cfcb770c3ecd0990dcceb1bde129e6c6 shadow/1/data/test/partition_428/19700102_2_2_0/p.bin -e2af3bef1fd129aea73a890ede1e7a30 shadow/1/data/test/partition_428/19700201_1_1_0/k.bin -f2312862cc01adf34a93151377be2ddf shadow/1/data/test/partition_428/19700201_1_1_0/minmax_p.idx -5 -5 -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700102_2_2_0/k.bin -082814b5aa5109160d5c0c5aff10d4df shadow/1/data/test/partition_428/19700201_1_1_0/v1.bin -13cae8e658e0ca4f75c56b1fc424e150 shadow/1/data/test/partition_428/19700102_2_2_0/minmax_p.idx -25daad3d9e60b45043a70c4ab7d3b1c6 shadow/1/data/test/partition_428/19700102_2_2_0/partition.dat -3726312af62aec86b64a7708d5751787 shadow/1/data/test/partition_428/19700201_1_1_0/partition.dat -37855b06a39b79a67ea4e86e4a3299aa shadow/1/data/test/partition_428/19700102_2_2_0/checksums.txt -38e62ff37e1e5064e9a3f605dfe09d13 shadow/1/data/test/partition_428/19700102_2_2_0/v1.bin -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700102_2_2_0/v1.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/k.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/p.mrk -4ae71336e44bf9bf79d2752e234818a5 shadow/1/data/test/partition_428/19700201_1_1_0/v1.mrk -55a54008ad1ba589aa210d2629c1df41 shadow/1/data/test/partition_428/19700201_1_1_0/primary.idx -5f087cb3e7071bf9407e095821e2af8f shadow/1/data/test/partition_428/19700201_1_1_0/checksums.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700102_2_2_0/columns.txt -77d5af402ada101574f4da114f242e02 shadow/1/data/test/partition_428/19700201_1_1_0/columns.txt -88cdc31ded355e7572d68d8cde525d3a shadow/1/data/test/partition_428/19700201_1_1_0/p.bin -9e688c58a5487b8eaf69c9e1005ad0bf shadow/1/data/test/partition_428/19700102_2_2_0/primary.idx -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700102_2_2_0/count.txt -c4ca4238a0b923820dcc509a6f75849b shadow/1/data/test/partition_428/19700201_1_1_0/count.txt -cfcb770c3ecd0990dcceb1bde129e6c6 shadow/1/data/test/partition_428/19700102_2_2_0/p.bin -e2af3bef1fd129aea73a890ede1e7a30 shadow/1/data/test/partition_428/19700201_1_1_0/k.bin -f2312862cc01adf34a93151377be2ddf shadow/1/data/test/partition_428/19700201_1_1_0/minmax_p.idx -31,1,2 -1,2,3 diff --git a/dbms/tests/queries/0_stateless/00428_partition.sh b/dbms/tests/queries/0_stateless/00428_partition.sh deleted file mode 100755 index 033d5e24c13..00000000000 --- a/dbms/tests/queries/0_stateless/00428_partition.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -# Not found column date in block. There are only columns: x. - -# Test 1. Complex test checking columns.txt - -chl="$CLICKHOUSE_CLIENT -q" -ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` - -$chl "DROP TABLE IF EXISTS test.partition_428" -$chl "CREATE TABLE test.partition_428 (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0" -$chl "INSERT INTO test.partition_428 (p, k) VALUES(toDate(31), 1)" -$chl "INSERT INTO test.partition_428 (p, k) VALUES(toDate(1), 2)" - -for part in `$chl "SELECT name FROM system.parts WHERE database='test' AND table='partition_428'"`; do - # 2 header lines + 3 columns - (sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt 2>/dev/null || \ - cat $ch_dir/data/test/partition_428/$part/columns.txt) | wc -l -done - -$chl "ALTER TABLE test.partition_428 FREEZE" - -# Do `cd` for consistent output for reference -# Do not check increment.txt - it can be changed by other tests with FREEZE -cd $ch_dir && find shadow -type f -exec md5sum {} \; | grep "partition_428" | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g' | sort | uniq - -$chl "ALTER TABLE test.partition_428 DETACH PARTITION 197001" -$chl "ALTER TABLE test.partition_428 ATTACH PARTITION 197001" - -for part in `$chl "SELECT name FROM system.parts WHERE database='test' AND table='partition_428'"`; do - # 2 header lines + 3 columns - (sudo -n cat $ch_dir/data/test/partition_428/$part/columns.txt 2>/dev/null || \ - cat $ch_dir/data/test/partition_428/$part/columns.txt) | wc -l -done - -$chl "ALTER TABLE test.partition_428 MODIFY COLUMN v1 Int8" - -# Check the backup hasn't changed -cd $ch_dir && find shadow -type f -exec md5sum {} \; | grep "partition_428" | sed 's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g' | sort | uniq - -$chl "OPTIMIZE TABLE test.partition_428" - -$chl "SELECT toUInt16(p), k, v1 FROM test.partition_428 ORDER BY k FORMAT CSV" -$chl "DROP TABLE test.partition_428" - -# Test 2. Simple test - -$chl "drop table if exists test.partition_428" -$chl "create table test.partition_428 (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) SETTINGS index_granularity=8192, index_granularity_bytes=0" -$chl "insert into test.partition_428 ( x ) VALUES ( now() )" -$chl "insert into test.partition_428 ( x ) VALUES ( now()+1 )" -$chl "alter table test.partition_428 detach partition 197001" -$chl "alter table test.partition_428 attach partition 197001" -$chl "optimize table test.partition_428" -$chl "drop table test.partition_428" diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference deleted file mode 100644 index f30fc160dfb..00000000000 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.reference +++ /dev/null @@ -1,26 +0,0 @@ -=== cannot attach active === -OK1 -0_1_1_0 -1_2_2_0 -2_3_3_0 -3_4_4_0 -16 120 -=== check all parts before attaching === -OK2 -1_2_2_0 -1_4_4_0 -=== detached === -0_1_1_0 -0_3_3_0 -0_5_5_0 -attaching_0_6_6_0 -deleting_0_7_7_0 -=== attach === -0_5_5_0 -0_6_6_0 -1_2_2_0 -1_4_4_0 -16 120 -=== detached === -attaching_0_6_6_0 -deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh b/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh deleted file mode 100755 index db45cfe7f21..00000000000 --- a/dbms/tests/queries/0_stateless/00974_attach_invalid_parts.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` -cur_db=`${CLICKHOUSE_CLIENT} --query "SELECT currentDatabase()"` - -echo '=== cannot attach active ==='; -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_active"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n"; -$CLICKHOUSE_CLIENT --query="INSERT INTO attach_active SELECT number FROM system.numbers LIMIT 16"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_active ATTACH PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_active' AND database='${cur_db}' ORDER BY name FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_active FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="DROP TABLE attach_active"; - - - -$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS attach_partitions"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE attach_partitions (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; -$CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8"; -$CLICKHOUSE_CLIENT --query="INSERT INTO attach_partitions SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; - -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions DETACH PARTITION 0"; -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ # broken part -sudo -n cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ 2>/dev/null || \ - cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_1_1_0/ $ch_dir/data/$cur_db/attach_partitions/detached/attaching_0_6_6_0/ -sudo -n cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ 2>/dev/null || \ - cp -pr $ch_dir/data/$cur_db/attach_partitions/detached/0_3_3_0/ $ch_dir/data/$cur_db/attach_partitions/detached/deleting_0_7_7_0/ - -echo '=== check all parts before attaching ==='; -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0" 2>&1 | grep "No columns in part 0_5_5_0" > /dev/null && echo 'OK2'; -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; -echo '=== detached ==='; -$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; - -echo '=== attach ==='; -sudo -n rm -r $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ 2>/dev/null || \ - rm -r $ch_dir/data/$cur_db/attach_partitions/detached/0_5_5_0/ -$CLICKHOUSE_CLIENT --query="ALTER TABLE attach_partitions ATTACH PARTITION 0"; -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; -$CLICKHOUSE_CLIENT --query="SElECT count(), sum(n) FROM attach_partitions FORMAT TSV"; - -echo '=== detached ==='; -$CLICKHOUSE_CLIENT --query="SELECT name FROM system.detached_parts WHERE table='attach_partitions' AND database='${cur_db}' ORDER BY name FORMAT TSV"; - -$CLICKHOUSE_CLIENT --query="DROP TABLE attach_partitions"; -$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.reference b/dbms/tests/queries/0_stateless/00975_drop_detached.reference deleted file mode 100644 index 414ac4b1927..00000000000 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.reference +++ /dev/null @@ -1,15 +0,0 @@ -=== validate part name === -OK1 -OK2 -OK3 -=== drop detached part === -0_3_3_0 -1_2_2_0 -1_4_4_0 -attaching_0_6_6_0 -deleting_0_7_7_0 -prefix_1_2_2_0_0 -=== drop detached partition === -0_3_3_0 -attaching_0_6_6_0 -deleting_0_7_7_0 diff --git a/dbms/tests/queries/0_stateless/00975_drop_detached.sh b/dbms/tests/queries/0_stateless/00975_drop_detached.sh deleted file mode 100755 index 8da831b019a..00000000000 --- a/dbms/tests/queries/0_stateless/00975_drop_detached.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -ch_dir=`${CLICKHOUSE_EXTRACT_CONFIG} -k path` -cur_db=`${CLICKHOUSE_CLIENT} --query "SELECT currentDatabase()"` - -$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES"; -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS drop_detached"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n"; -$CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8"; -$CLICKHOUSE_CLIENT --query="INSERT INTO drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8"; - -$CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 0"; -$CLICKHOUSE_CLIENT --query="ALTER TABLE drop_detached DETACH PARTITION 1"; -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/attaching_0_6_6_0/ -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/deleting_0_7_7_0/ -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/any_other_name/ -sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0_0/ 2>/dev/null || \ - mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0_0/ -#sudo -n mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0/ 2>/dev/null || \ -# mkdir --mode=777 $ch_dir/data/$cur_db/drop_detached/detached/prefix_1_2_2_0/ - -echo '=== validate part name ===' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '../1_2_2_0'" 2>&1 | grep "Invalid part name" > /dev/null && echo 'OK1' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART '0_1_1_0'" -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'attaching_0_6_6_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK2' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'deleting_0_7_7_0'" 2>&1 | grep "Cannot drop part" > /dev/null && echo 'OK3' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PART 'any_other_name'" - -echo '=== drop detached part ===' -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; - -echo '=== drop detached partition ===' -$CLICKHOUSE_CLIENT --allow_drop_detached=1 --query="ALTER TABLE drop_detached DROP DETACHED PARTITION 1" -$CLICKHOUSE_CLIENT --query="SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='${cur_db}' ORDER BY name FORMAT TSV"; - -$CLICKHOUSE_CLIENT --query="DROP TABLE drop_detached"; -$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES"; From b5eee531a9cbd9b3df2c5c373d99680e50d6b8cb Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 8 Aug 2019 19:08:43 +0300 Subject: [PATCH 055/133] fix setting name --- dbms/src/Core/Settings.h | 2 +- dbms/src/Interpreters/InterpreterAlterQuery.cpp | 2 +- dbms/tests/integration/test_partition/test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index fd11d645bd5..ffc11cef4a6 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -341,7 +341,7 @@ struct Settings : public SettingsCollection /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") \ - M(SettingBool, allow_drop_detached_part, false, "Allow ALTER TABLE ... DROP DETACHED PART ... queries") + M(SettingBool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries") DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 074fbb7d4c2..bc419f1ff84 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -56,7 +56,7 @@ BlockIO InterpreterAlterQuery::execute() else if (auto partition_command = PartitionCommand::parse(command_ast)) { if (partition_command->type == PartitionCommand::DROP_DETACHED_PARTITION - && !context.getSettingsRef().allow_drop_detached_part) + && !context.getSettingsRef().allow_drop_detached) throw DB::Exception("Cannot execute query: DROP DETACHED PART is disabled " "(see allow_drop_detached setting)", ErrorCodes::SUPPORT_IS_DISABLED); partition_commands.emplace_back(std::move(*partition_command)); diff --git a/dbms/tests/integration/test_partition/test.py b/dbms/tests/integration/test_partition/test.py index 59c48e5d9e9..3365343b6fb 100644 --- a/dbms/tests/integration/test_partition/test.py +++ b/dbms/tests/integration/test_partition/test.py @@ -212,7 +212,7 @@ def drop_detached_parts_table(started_cluster): def test_drop_detached_parts(drop_detached_parts_table): - s = {"allow_drop_detached_part": 1} + s = {"allow_drop_detached": 1} q("ALTER TABLE test.drop_detached DETACH PARTITION 0") q("ALTER TABLE test.drop_detached DETACH PARTITION 1") From d1ebfaacd6df35b2a6b55f25f6f9319e00c1f5ac Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 8 Aug 2019 22:28:25 +0300 Subject: [PATCH 056/133] update docs --- docs/en/operations/system_tables.md | 3 ++- docs/en/query_language/alter.md | 12 +++++++++++- docs/ru/operations/system_tables.md | 6 ++++++ docs/ru/query_language/alter.md | 11 ++++++++++- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index e63a9115270..e5eac2f1f58 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -87,13 +87,14 @@ This table contains a single String column called 'name' – the name of a datab Each database that the server knows about has a corresponding entry in the table. This system table is used for implementing the `SHOW DATABASES` query. -## system.detached_parts +## system.detached_parts {#system_tables-detached_parts} Contains information about detached parts of [MergeTree](table_engines/mergetree.md) tables. The `reason` column specifies why the part was detached. For user-detached parts, the reason is empty. Such parts can be attached with [ALTER TABLE ATTACH PARTITION|PART](../query_language/query_language/alter/#alter_attach-partition) command. For the description of other columns, see [system.parts](#system_tables-parts). +If part name is invalid, values of some columns may be `NULL`. Such parts can be deleted with [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). ## system.dictionaries diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md index 6e8e712ff30..2d42c4cc354 100644 --- a/docs/en/query_language/alter.md +++ b/docs/en/query_language/alter.md @@ -210,6 +210,16 @@ Read about setting the partition expression in a section [How to specify the par The query is replicated – it deletes data on all replicas. +#### DROP DETACHED PARTITION|PART {#alter_drop-detached} + +```sql +ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr +``` + +Removes the specified part or all parts of the specified partition from `detached`. +Read more about setting the partition expression in a section [How to specify the partition expression](#alter-how-to-specify-part-expr). + + #### ATTACH PARTITION|PART {#alter_attach-partition} ``` sql @@ -327,7 +337,7 @@ You can specify the partition expression in `ALTER ... PARTITION` queries in dif - As a value from the `partition` column of the `system.parts` table. For example, `ALTER TABLE visits DETACH PARTITION 201901`. - As the expression from the table column. Constants and constant expressions are supported. For example, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. - Using the partition ID. Partition ID is a string identifier of the partition (human-readable, if possible) that is used as the names of partitions in the file system and in ZooKeeper. The partition ID must be specified in the `PARTITION ID` clause, in a single quotes. For example, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- In the [ALTER ATTACH PART](#alter_attach-partition) query, to specify the name of a part, use a value from the `name` column of the `system.parts` table. For example, `ALTER TABLE visits ATTACH PART 201901_1_1_0`. +- In the [ALTER ATTACH PART](#alter_attach-partition) and [DROP DETACHED PART](#alter_drop-detached) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached_parts](../operations/system_tables.md#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. Usage of quotes when specifying the partition depends on the type of partition expression. For example, for the `String` type, you have to specify its name in quotes (`'`). For the `Date` and `Int*` types no quotes are needed. diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index eb452c8de4e..4345f83718b 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -47,6 +47,12 @@ default_expression String - выражение для значения по ум Для каждой базы данных, о которой знает сервер, будет присутствовать соответствующая запись в таблице. Эта системная таблица используется для реализации запроса `SHOW DATABASES`. +## system.detached_parts {#system_tables-detached_parts} + +Сожелржит информацию об отсоединённых кусках таблиц семейства [MergeTree](table_engines/mergetree.md). Столбец `reason` содержит причину, по которой кусок был отсоединён. Для кусов, отсоединённых пользователем, `reason` содержит пустую строку. +Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION|PART](../query_language/query_language/alter/#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). +Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../query_language/query_language/alter/#alter_drop-detached). + ## system.dictionaries Содержит информацию о внешних словарях. diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md index 2367386172a..3e0030e948e 100644 --- a/docs/ru/query_language/alter.md +++ b/docs/ru/query_language/alter.md @@ -209,6 +209,15 @@ ALTER TABLE table_name DROP PARTITION partition_expr Запрос реплицируется — данные будут удалены на всех репликах. +#### DROP DETACHED PARTITION|PART {#alter_drop-detached} + +```sql +ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr +``` + +Удаляет из `detached` кусок или все куски, принадлежащие партиции. +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + #### ATTACH PARTITION|PART {#alter_attach-partition} ```sql @@ -328,7 +337,7 @@ ALTER TABLE users ATTACH PARTITION 201902; - Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../operations/system_tables.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. - Произвольное выражение из столбцов исходной таблицы. Также поддерживаются константы и константные выражения. Например, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. - Строковый идентификатор партиции. Идентификатор партиции используется для именования кусков партиции на файловой системе и в ZooKeeper. В запросах `ALTER` идентификатор партиции нужно указывать в секции `PARTITION ID`, в одинарных кавычках. Например, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- Для запросов [ATTACH PART](#alter_attach-partition): чтобы задать имя куска партиции, используйте значение из столбца `name` системной таблицы `system.parts`. Например, `ALTER TABLE visits ATTACH PART 201901_1_1_0`. +- Для запросов [ATTACH PART](#alter_attach-partition) и [DROP DETACHED PART](#alter_drop-detached): чтобы задать имя куска партиции, используйте строковой литерал со значением из столбца `name` системной таблицы [system.detached_parts](../operations/system_tables.md#system_tables-detached_parts). Например, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. Использование кавычек в имени партиций зависит от типа данных столбца, по которому задано партиционирование. Например, для столбца с типом `String` имя партиции необходимо указывать в кавычках (одинарных). Для типов `Date` и `Int*` кавычки указывать не нужно. From 67331881356ee5dd1fb158ce728e8ab42016c9f2 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Mon, 5 Aug 2019 07:09:09 +0000 Subject: [PATCH 057/133] Added gcc-9 to docker/builder container --- docker/builder/Dockerfile | 6 ++++-- docker/builder/build.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index 03b4e242d6d..41a558f9eb8 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -1,6 +1,8 @@ FROM ubuntu:18.04 RUN apt-get update -y \ + && apt-get install -y software-properties-common \ + && add-apt-repository ppa:ubuntu-toolchain-r/test \ && env DEBIAN_FRONTEND=noninteractive \ apt-get install --yes --no-install-recommends \ bash \ @@ -8,8 +10,8 @@ RUN apt-get update -y \ cmake \ curl \ expect \ - g++ \ - gcc \ + g++-9 \ + gcc-9 \ libclang-6.0-dev \ libicu-dev \ liblld-6.0-dev \ diff --git a/docker/builder/build.sh b/docker/builder/build.sh index 6a5f1359bda..57999a4b483 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -3,7 +3,7 @@ #ccache -s mkdir -p /server/build_docker cd /server/build_docker -cmake -G Ninja /server -DENABLE_TESTS=1 +cmake -G Ninja /server -DENABLE_TESTS=1 -DCMAKE_C_COMPILER=`which gcc-9` -DCMAKE_CXX_COMPILER=`which g++-9` # Set the number of build jobs to the half of number of virtual CPU cores (rounded up). # By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time. From 0233f32f9b30e78d8f54c28ccdea16736a6293b1 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 11 Aug 2019 12:28:15 +0300 Subject: [PATCH 058/133] Fixed AddresSanitizer error --- .../DataStreams/CheckConstraintsBlockOutputStream.cpp | 11 +++++------ .../DataStreams/CheckConstraintsBlockOutputStream.h | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index 5adf344cf0b..4b4865f004f 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -11,8 +11,9 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) { for (size_t i = 0; i < expressions.size(); ++i) { + Block res = block; auto constraint_expr = expressions[i]; - auto res_column_uint8 = executeOnBlock(block, constraint_expr); + auto res_column_uint8 = executeOnBlock(res, constraint_expr); if (!memoryIsByte(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize(), 0x1)) { auto indices_wrong = findAllWrong(res_column_uint8->getRawDataBegin<1>(), res_column_uint8->byteSize()); @@ -48,13 +49,11 @@ void CheckConstraintsBlockOutputStream::writeSuffix() } const ColumnUInt8 *CheckConstraintsBlockOutputStream::executeOnBlock( - const Block & block, + Block & block, const ExpressionActionsPtr & constraint) { - Block res = block; - - constraint->execute(res); - ColumnWithTypeAndName res_column = res.safeGetByPosition(res.columns() - 1); + constraint->execute(block); + ColumnWithTypeAndName res_column = block.safeGetByPosition(block.columns() - 1); return checkAndGetColumn(res_column.column.get()); } diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index ac2e7e974a1..7ab6832fd28 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -39,7 +39,7 @@ public: void writeSuffix() override; private: - const ColumnUInt8* executeOnBlock(const Block & block, const ExpressionActionsPtr & constraint); + const ColumnUInt8* executeOnBlock(Block & block, const ExpressionActionsPtr & constraint); std::vector findAllWrong(const void *data, size_t size); String table; From 93a635d18a2119cc018cdbb5d73cfd12da6fbdc6 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 11 Aug 2019 12:30:01 +0300 Subject: [PATCH 059/133] Added clang-8 to docker builder --- docker/builder/Dockerfile | 4 ++++ docker/builder/Makefile | 2 +- docker/builder/build.sh | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index 41a558f9eb8..5978dcd08d0 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -28,6 +28,10 @@ RUN apt-get update -y \ tzdata \ gperf +RUN apt install -y wget +RUN printf "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main\ndeb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list \ + && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && apt update && apt-get install -y clang-8 lldb-8 lld-8 + COPY build.sh / CMD ["/bin/bash", "/build.sh"] diff --git a/docker/builder/Makefile b/docker/builder/Makefile index 779e944b723..a9a7cddf3f2 100644 --- a/docker/builder/Makefile +++ b/docker/builder/Makefile @@ -1,6 +1,6 @@ build: image mkdir -p $(HOME)/.ccache - docker run --network=host --rm --workdir /server --volume $(realpath ../..):/server --mount=type=bind,source=$(HOME)/.ccache,destination=/ccache -e CCACHE_DIR=/ccache -it yandex/clickhouse-builder + docker run --network=host --rm --workdir /server --volume $(realpath ../..):/server --cap-add=SYS_PTRACE --mount=type=bind,source=$(HOME)/.ccache,destination=/ccache -e CCACHE_DIR=/ccache -it yandex/clickhouse-builder pull: docker pull yandex/clickhouse-builder diff --git a/docker/builder/build.sh b/docker/builder/build.sh index 57999a4b483..96468d8d820 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -3,7 +3,8 @@ #ccache -s mkdir -p /server/build_docker cd /server/build_docker -cmake -G Ninja /server -DENABLE_TESTS=1 -DCMAKE_C_COMPILER=`which gcc-9` -DCMAKE_CXX_COMPILER=`which g++-9` + +cmake -G Ninja /server -DCMAKE_C_COMPILER=`which clang-8` -DCMAKE_CXX_COMPILER=`which clang++-8` -DCMAKE_BUILD_TYPE=Debug # Set the number of build jobs to the half of number of virtual CPU cores (rounded up). # By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time. From 3b9e1f9bf727764e1a4d9787b6e58313f3d381bf Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 11 Aug 2019 13:39:17 +0300 Subject: [PATCH 060/133] Fixed getIdentifierName call in AlterCommand::parse --- dbms/src/Storages/AlterCommands.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 48690df071a..13141683be8 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -190,7 +190,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::DROP_COLUMN) { command.type = AlterCommand::DROP_COLUMN; - command.column_name = *getIdentifierName(command_ast->column); + command.column_name = getIdentifierName(command_ast->column); } return command; From 1b54a52488441b753c99fe6e17d19d458e90313c Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Mon, 12 Aug 2019 17:10:29 +0300 Subject: [PATCH 061/133] Temp --- dbms/src/Functions/neighbour.cpp | 217 +++++++++++++++++++++++++++++ dbms/src/Functions/nextInBlock.cpp | 159 --------------------- 2 files changed, 217 insertions(+), 159 deletions(-) create mode 100644 dbms/src/Functions/neighbour.cpp delete mode 100644 dbms/src/Functions/nextInBlock.cpp diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp new file mode 100644 index 00000000000..cf96282725a --- /dev/null +++ b/dbms/src/Functions/neighbour.cpp @@ -0,0 +1,217 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +// Implements function, giving value for column within range of given +// Example: +// | c1 | +// | 10 | +// | 20 | +// SELECT c1, neighbour(c1, 1) as c2: +// | c1 | c2 | +// | 10 | 20 | +// | 20 | 0 | +class FunctionNeighbour : public IFunction +{ +public: + static constexpr auto name = "neighbour"; + static FunctionPtr create(const Context & context) { return std::make_shared(context); } + + FunctionNeighbour(const Context & context_) : context(context_) {} + + /// Get the name of the function. + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isVariadic() const override { return true; } + + bool isDeterministic() const override { return false; } + + bool isDeterministicInScopeOfQuery() const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + size_t number_of_arguments = arguments.size(); + + if (number_of_arguments < 2 || number_of_arguments > 3) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(number_of_arguments) + + ", should be from 2 to 3", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + // second argument must be a positive integer + if (!isInteger(arguments[1])) + throw Exception( + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + + " - should be positive integer", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + // check that default value column has supertype with first argument + if (number_of_arguments == 3) + { + DataTypes types = {arguments[0], arguments[2]}; + try + { + return getLeastSupertype(types); + } + catch (const Exception &) + { + throw Exception( + "Illegal types of arguments (" + types[0]->getName() + ", " + types[1]->getName() + + ")" + " of function " + + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + + return arguments[0]; + } + + static void insertDefaults(const MutableColumnPtr & target, size_t row_count, ColumnPtr & default_values_column, size_t offset) + { + if (row_count == 0) { + return; + } + if (default_values_column) + { + if (isColumnConst(*default_values_column)) + { + Field constant_value = (*default_values_column)[0]; + for(size_t row = 0; row < row_count;row++) + { + target->insert(constant_value); + } + } else { + target->insertRangeFrom(*default_values_column, offset, row_count); + } + } else { + for(size_t row = 0; row <= row_count;row++) { + target->insertDefault(); + } + } + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto offset_structure = block.getByPosition(arguments[1]); + ColumnPtr & offset_column = offset_structure.column; + + auto is_constant_offset = isColumnConst(*offset_structure.column); + ColumnPtr default_values_column = nullptr; + if (arguments.size() == 3) + { + default_values_column = block.getByPosition(arguments[2]).column; + } + +// Field offset_field = (*block.getByPosition(arguments[1]).column)[0]; +// auto raw_value = safeGet(offset_field); + + ColumnWithTypeAndName &source_column_name_and_type = block.getByPosition(arguments[0]); + DataTypes types = {source_column_name_and_type.type}; + if (default_values_column) + { + types.push_back(block.getByPosition(arguments[2]).type); + } + const DataTypePtr & result_type = getLeastSupertype(types); + auto source_column = source_column_name_and_type.column; + + // adjust source and default values columns to resulting datatype + if (!source_column_name_and_type.type->equals(*result_type)) { + source_column = castColumn(source_column_name_and_type, result_type, context); + } + + if (default_values_column && !block.getByPosition(arguments[2]).type->equals(*result_type)) { + default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); + } + + auto column = result_type->createColumn(); + column->reserve(input_rows_count); + + const DataTypePtr desired_type = std::make_shared(); + if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) { + offset_column = castColumn(offset_structure, desired_type, context); + } + + // with constant offset - insertRangeFrom + if (is_constant_offset) + { + Int64 offset_value = offset_column->getInt(0); + + if (offset_value > 0) + { + // insert shifted value + column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); + // insert defaults into the end + insertDefaults(column, input_rows_count - offset_value, default_values_column, offset_value); + } else if(offset_value < 0) { + // insert defaults up to offset_value + insertDefaults(column, input_rows_count - std::abs(offset_value), default_values_column, std::abs(offset_value)); + // insert range, where possible + column->insertRangeFrom(*source_column, 0, input_rows_count - std::abs(offset_value)); + } else { + // populate column with source values + column->insertRangeFrom(*source_column, 0, input_rows_count); + } + } else { + // with dynamic offset - handle row by row + for (size_t row = 0; row < input_rows_count; row++) + { + Int64 offset_value = offset_column->getInt(row); + if (offset_value == 0) { + column->insertFrom(*source_column, row); + } else if (offset_value > 0) { + size_t real_offset = row + offset_value; + if (real_offset > input_rows_count) { + if (default_values_column) { + column->insertFrom(*default_values_column, row); + } else { + column->insertDefault(); + } + } else { + column->insertFrom(*column, real_offset); + } + } else { + // out of range + if ((size_t)std::abs(offset_value) > row) + { + if (default_values_column) { + column->insertFrom(*default_values_column, row); + } else { + column->insertDefault(); + } + } else { + column->insertFrom(*column, row - std::abs(offset_value)); + } + } + } + } + + + block.getByPosition(result).column = std::move(column); + } +private: + const Context & context; +}; + +void registerFunctionNextInBlock(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/nextInBlock.cpp b/dbms/src/Functions/nextInBlock.cpp deleted file mode 100644 index eeb33e28146..00000000000 --- a/dbms/src/Functions/nextInBlock.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ARGUMENT_OUT_OF_BOUND; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - -// Implements function, giving value for column in next row -// Example: -// | c1 | -// | 10 | -// | 20 | -// SELECT c1, nextInBlock(c1, 1) as c2: -// | c1 | c2 | -// | 10 | 20 | -// | 20 | 0 | -class FunctionNextInBlock : public IFunction -{ -public: - static constexpr auto name = "nextInBlock"; - static FunctionPtr create(const Context &) { return std::make_shared(); } - - /// Get the name of the function. - String getName() const override { return name; } - - size_t getNumberOfArguments() const override { return 0; } - - bool isVariadic() const override { return true; } - - bool isDeterministic() const override { return false; } - - bool isDeterministicInScopeOfQuery() const override { return false; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - size_t number_of_arguments = arguments.size(); - - if (number_of_arguments < 1 || number_of_arguments > 3) - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(number_of_arguments) - + ", should be from 1 to 3", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - // second argument must be a positive, constant column - if (number_of_arguments == 2 && !isUnsignedInteger(arguments[1])) - throw Exception( - "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + " - should be positive integer", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - // check that default value has supertype with first argument - if (number_of_arguments == 3) - { - DataTypes types = {arguments[0], arguments[2]}; - try - { - return getLeastSupertype(types); - } - catch (const Exception &) - { - throw Exception( - "Illegal types of arguments (" + types[0]->getName() + ", " + types[1]->getName() - + ")" - " of function " - + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - } - - return arguments[0]; - } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override - { - size_t offset_value = 1; - - if (arguments.size() > 1) - { - auto offset_column = block.getByPosition(arguments[1]); - if (!isColumnConst(*offset_column.column)) - throw Exception("Second argument of function " + getName() + " should be constant", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - Field offset_field = (*block.getByPosition(arguments[1]).column)[0]; - auto raw_value = safeGet(offset_field); - - if (raw_value == 0) - throw Exception( - "Second argument of function " + getName() + " should be positive integer, " + toString(raw_value) + " given", - ErrorCodes::ARGUMENT_OUT_OF_BOUND); - - offset_value = raw_value; - } - - auto has_column_for_missing = arguments.size() == 3; - - DataTypes types = {block.getByPosition(arguments[0]).type}; - if (has_column_for_missing) - { - types.push_back(block.getByPosition(arguments[2]).type); - } - const DataTypePtr & result_type = getLeastSupertype(types); - - auto column = result_type->createColumn(); - column->reserve(input_rows_count); - - auto source_column = block.getByPosition(arguments[0]).column; - - for (size_t i = offset_value; i < input_rows_count; i++) - { - column->insertFrom(*source_column, i); - } - - if (has_column_for_missing) - { - auto default_values_column = block.getByPosition(arguments[2]).column; - size_t starting_pos = offset_value > input_rows_count ? 0 : input_rows_count - offset_value; - if (isColumnConst(*default_values_column)) - { - Field constant_value = (*default_values_column)[0]; - for (size_t i = starting_pos; i < input_rows_count; i++) - { - column->insert(constant_value); - } - } - else - { - for (size_t i = starting_pos; i < input_rows_count; i++) - { - column->insertFrom(*default_values_column, i); - } - } - } - else - { - for (size_t i = 0; i < std::min(offset_value, input_rows_count); i++) - { - column->insertDefault(); - } - } - - block.getByPosition(result).column = std::move(column); - } -}; - -void registerFunctionNextInBlock(FunctionFactory & factory) -{ - factory.registerFunction(); -} - -} From 31fdc99efc3b668b18e8ea830dfc669f743d275d Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Mon, 12 Aug 2019 18:44:28 +0300 Subject: [PATCH 062/133] In progress --- dbms/src/Functions/neighbour.cpp | 10 ++++---- .../0_stateless/00957_next_in_block.sql | 24 ++++++++----------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index cf96282725a..3eff660c996 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -101,7 +101,7 @@ public: target->insertRangeFrom(*default_values_column, offset, row_count); } } else { - for(size_t row = 0; row <= row_count;row++) { + for(size_t row = 0; row < row_count;row++) { target->insertDefault(); } } @@ -156,9 +156,11 @@ public: if (offset_value > 0) { // insert shifted value - column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); - // insert defaults into the end - insertDefaults(column, input_rows_count - offset_value, default_values_column, offset_value); + if ((size_t)std::abs(offset_value) <= input_rows_count) { + column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); + // insert defaults into the end + insertDefaults(column, input_rows_count - offset_value, default_values_column, offset_value); + } } else if(offset_value < 0) { // insert defaults up to offset_value insertDefaults(column, input_rows_count - std::abs(offset_value), default_values_column, std::abs(offset_value)); diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.sql b/dbms/tests/queries/0_stateless/00957_next_in_block.sql index 7cbd932cf1a..1efda43339e 100644 --- a/dbms/tests/queries/0_stateless/00957_next_in_block.sql +++ b/dbms/tests/queries/0_stateless/00957_next_in_block.sql @@ -1,22 +1,18 @@ -- no arguments -select nextInBlock(); -- { serverError 42 } +select neighbour(); -- { serverError 42 } +-- single argument +select neighbour(1); -- { serverError 42 } -- greater than 3 arguments -select nextInBlock(1,2,3,4); -- { serverError 42 } --- zero offset value -select nextInBlock(dummy, 0); -- { serverError 69 } --- negative offset value -select nextInBlock(dummy, -1); -- { serverError 43 } --- non-constant offset value -select nextInBlock(dummy, dummy); -- { serverError 43 } +select neighbour(1,2,3,4); -- { serverError 42 } -- bad default value -select nextInBlock(dummy, 1, 'hello'); -- { serverError 43 } +select neighbour(dummy, 1, 'hello'); -- { serverError 43 } -- single argument test -select number, nextInBlock(number) from numbers(2); +select number, neighbour(number,1) from numbers(2); -- filling by column's default value -select number, nextInBlock(number, 2) from numbers(3); +select number, neighbour(number, 2) from numbers(3); -- offset is greater that block - should fill everything with defaults -select number, nextInBlock(number, 5) from numbers(2); +select number, neighbour(number, 5) from numbers(2); -- substitution by constant for missing values -select number, nextInBlock(number, 2, 1000) from numbers(5); +select number, neighbour(number, 2, 1000) from numbers(5); -- substitution by expression --- select number, nextInBlock(number, 2, number % 2) from numbers(5); \ No newline at end of file +-- select number, neighbour(number, 2, number % 2) from numbers(5); \ No newline at end of file From ea9cf3a62f42f0ed48efe7baac2be46bfaa7ae5e Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Tue, 13 Aug 2019 16:11:24 +0300 Subject: [PATCH 063/133] Done --- dbms/src/Functions/neighbour.cpp | 105 +++++++++--------- .../0_stateless/00957_neighbour.reference | 42 +++++++ .../queries/0_stateless/00957_neighbour.sql | 30 +++++ .../0_stateless/00957_next_in_block.reference | 12 -- .../0_stateless/00957_next_in_block.sql | 18 --- .../functions/other_functions.md | 43 +++++++ .../functions/other_functions.md | 42 +++++++ 7 files changed, 209 insertions(+), 83 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00957_neighbour.reference create mode 100644 dbms/tests/queries/0_stateless/00957_neighbour.sql delete mode 100644 dbms/tests/queries/0_stateless/00957_next_in_block.reference delete mode 100644 dbms/tests/queries/0_stateless/00957_next_in_block.sql diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index 3eff660c996..79a1feec002 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -119,9 +119,6 @@ public: default_values_column = block.getByPosition(arguments[2]).column; } -// Field offset_field = (*block.getByPosition(arguments[1]).column)[0]; -// auto raw_value = safeGet(offset_field); - ColumnWithTypeAndName &source_column_name_and_type = block.getByPosition(arguments[0]); DataTypes types = {source_column_name_and_type.type}; if (default_values_column) @@ -140,72 +137,74 @@ public: default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); } - auto column = result_type->createColumn(); - column->reserve(input_rows_count); - const DataTypePtr desired_type = std::make_shared(); if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) { offset_column = castColumn(offset_structure, desired_type, context); } - - // with constant offset - insertRangeFrom - if (is_constant_offset) - { - Int64 offset_value = offset_column->getInt(0); - if (offset_value > 0) - { - // insert shifted value - if ((size_t)std::abs(offset_value) <= input_rows_count) { - column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); - // insert defaults into the end - insertDefaults(column, input_rows_count - offset_value, default_values_column, offset_value); - } - } else if(offset_value < 0) { - // insert defaults up to offset_value - insertDefaults(column, input_rows_count - std::abs(offset_value), default_values_column, std::abs(offset_value)); - // insert range, where possible - column->insertRangeFrom(*source_column, 0, input_rows_count - std::abs(offset_value)); - } else { - // populate column with source values - column->insertRangeFrom(*source_column, 0, input_rows_count); - } + if (isColumnConst(*source_column)) { + auto column = result_type->createColumnConst(input_rows_count, (*source_column)[0]); + block.getByPosition(result).column = std::move(column); } else { - // with dynamic offset - handle row by row - for (size_t row = 0; row < input_rows_count; row++) + auto column = result_type->createColumn(); + column->reserve(input_rows_count); + // with constant offset - insertRangeFrom + if (is_constant_offset) { - Int64 offset_value = offset_column->getInt(row); - if (offset_value == 0) { - column->insertFrom(*source_column, row); - } else if (offset_value > 0) { - size_t real_offset = row + offset_value; - if (real_offset > input_rows_count) { - if (default_values_column) { - column->insertFrom(*default_values_column, row); - } else { - column->insertDefault(); - } - } else { - column->insertFrom(*column, real_offset); + Int64 offset_value = offset_column->getInt(0); + + if (offset_value > 0) + { + // insert shifted value + if ((size_t)offset_value <= input_rows_count) { + column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); } + size_t row_count = (size_t)offset_value > input_rows_count ? input_rows_count : offset_value; + insertDefaults(column, row_count, default_values_column, input_rows_count - row_count); + } else if (offset_value < 0) { + size_t row_count = (size_t)std::abs(offset_value) > input_rows_count ? input_rows_count : std::abs(offset_value); + // insert defaults up to offset_value + insertDefaults(column, row_count, default_values_column, 0); + column->insertRangeFrom(*source_column, 0, input_rows_count - row_count); } else { - // out of range - if ((size_t)std::abs(offset_value) > row) - { - if (default_values_column) { - column->insertFrom(*default_values_column, row); + // populate column with source values + column->insertRangeFrom(*source_column, 0, input_rows_count); + } + } else { + // with dynamic offset - handle row by row + for (size_t row = 0; row < input_rows_count; row++) + { + Int64 offset_value = offset_column->getInt(row); + if (offset_value == 0) { + column->insertFrom(*source_column, row); + } else if (offset_value > 0) { + size_t real_offset = row + offset_value; + if (real_offset > input_rows_count) { + if (default_values_column) { + column->insertFrom(*default_values_column, row); + } else { + column->insertDefault(); + } } else { - column->insertDefault(); + column->insertFrom(*column, real_offset); } } else { - column->insertFrom(*column, row - std::abs(offset_value)); + // out of range + if ((size_t)std::abs(offset_value) > row) + { + if (default_values_column) { + column->insertFrom(*default_values_column, row); + } else { + column->insertDefault(); + } + } else { + column->insertFrom(*column, row - std::abs(offset_value)); + } } } } + block.getByPosition(result).column = std::move(column); } - - - block.getByPosition(result).column = std::move(column); } private: const Context & context; diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.reference b/dbms/tests/queries/0_stateless/00957_neighbour.reference new file mode 100644 index 00000000000..cd8c6310f22 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_neighbour.reference @@ -0,0 +1,42 @@ +Result with different type +0 1 +1 2 +2 -10 +Offset > block +0 0 +1 0 +2 0 +Abs(Offset) > block +0 0 +1 0 +2 0 +Positive offset +0 1 +1 2 +2 0 +Negative offset +0 1 +1 2 +2 0 +Positive offset with defaults +0 2 +1 3 +2 12 +3 13 +Negative offset with defaults +0 10 +1 11 +2 0 +3 1 +Positive offset with const defaults +0 1 +1 2 +2 1000 +Negative offset with const defaults +0 1000 +1 0 +2 1 +Constant column +0 1000 +1 1000 +2 1000 diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.sql b/dbms/tests/queries/0_stateless/00957_neighbour.sql new file mode 100644 index 00000000000..156672155a8 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00957_neighbour.sql @@ -0,0 +1,30 @@ +-- no arguments +select neighbour(); -- { serverError 42 } +-- single argument +select neighbour(1); -- { serverError 42 } +-- greater than 3 arguments +select neighbour(1,2,3,4); -- { serverError 42 } +-- bad default value +select neighbour(dummy, 1, 'hello'); -- { serverError 43 } +-- types without common supertype (UInt64 and Int8) +select number, neighbour(number, 1, -10) from numbers(3); -- { serverError 43 } +select 'Result with different type'; +select toInt32(number) as n, neighbour(n, 1, -10) from numbers(3); +select 'Offset > block'; +select number, neighbour(number, 10) from numbers(3); +select 'Abs(Offset) > block'; +select number, neighbour(number, -10) from numbers(3); +select 'Positive offset'; +select number, neighbour(number, 1) from numbers(3); +select 'Negative offset'; +select number, neighbour(number, 1) from numbers(3); +select 'Positive offset with defaults'; +select number, neighbour(number, 2, number + 10) from numbers(4); +select 'Negative offset with defaults'; +select number, neighbour(number, -2, number + 10) from numbers(4); +select 'Positive offset with const defaults'; +select number, neighbour(number, 1, 1000) from numbers(3); +select 'Negative offset with const defaults'; +select number, neighbour(number, -1, 1000) from numbers(3); +select 'Constant column'; +select number, neighbour(1000, 10) from numbers(3); \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.reference b/dbms/tests/queries/0_stateless/00957_next_in_block.reference deleted file mode 100644 index 860ce6dc1ba..00000000000 --- a/dbms/tests/queries/0_stateless/00957_next_in_block.reference +++ /dev/null @@ -1,12 +0,0 @@ -0 1 -1 0 -0 2 -1 0 -2 0 -0 0 -1 0 -0 2 -1 3 -2 4 -3 1000 -4 1000 diff --git a/dbms/tests/queries/0_stateless/00957_next_in_block.sql b/dbms/tests/queries/0_stateless/00957_next_in_block.sql deleted file mode 100644 index 1efda43339e..00000000000 --- a/dbms/tests/queries/0_stateless/00957_next_in_block.sql +++ /dev/null @@ -1,18 +0,0 @@ --- no arguments -select neighbour(); -- { serverError 42 } --- single argument -select neighbour(1); -- { serverError 42 } --- greater than 3 arguments -select neighbour(1,2,3,4); -- { serverError 42 } --- bad default value -select neighbour(dummy, 1, 'hello'); -- { serverError 43 } --- single argument test -select number, neighbour(number,1) from numbers(2); --- filling by column's default value -select number, neighbour(number, 2) from numbers(3); --- offset is greater that block - should fill everything with defaults -select number, neighbour(number, 5) from numbers(2); --- substitution by constant for missing values -select number, neighbour(number, 2, 1000) from numbers(5); --- substitution by expression --- select number, neighbour(number, 2, number % 2) from numbers(5); \ No newline at end of file diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md index 57fa8acfee3..05efe0fceb4 100644 --- a/docs/en/query_language/functions/other_functions.md +++ b/docs/en/query_language/functions/other_functions.md @@ -311,6 +311,49 @@ Returns the ordinal number of the row in the data block. Different data blocks a Returns the ordinal number of the row in the data block. This function only considers the affected data blocks. +## neighbour(column, offset\[, default_value\]) + +Returns value for `column`, in `offset` distance from current row. +This function is a partial implementation of [window functions](https://en.wikipedia.org/wiki/SQL_window_function) LEAD() and LAG(). + +The result of the function depends on the affected data blocks and the order of data in the block. +If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +If `offset` value is outside block bounds, a default value for `column` returned. If `default_value` is given, then it will be used. +This function can be used to compute year-over-year metric value: + +``` sql +WITH toDate('2018-01-01') AS start_date +SELECT + toStartOfMonth(start_date + (number * 32)) AS month, + toInt32(month) % 100 AS money, + neighbour(money, -12) AS prev_year, + round(prev_year / money, 2) AS year_over_year +FROM numbers(16) +``` + +``` +┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ +│ 2018-01-01 │ 32 │ 0 │ 0 │ +│ 2018-02-01 │ 63 │ 0 │ 0 │ +│ 2018-03-01 │ 91 │ 0 │ 0 │ +│ 2018-04-01 │ 22 │ 0 │ 0 │ +│ 2018-05-01 │ 52 │ 0 │ 0 │ +│ 2018-06-01 │ 83 │ 0 │ 0 │ +│ 2018-07-01 │ 13 │ 0 │ 0 │ +│ 2018-08-01 │ 44 │ 0 │ 0 │ +│ 2018-09-01 │ 75 │ 0 │ 0 │ +│ 2018-10-01 │ 5 │ 0 │ 0 │ +│ 2018-11-01 │ 36 │ 0 │ 0 │ +│ 2018-12-01 │ 66 │ 0 │ 0 │ +│ 2019-01-01 │ 97 │ 32 │ 0.33 │ +│ 2019-02-01 │ 28 │ 63 │ 2.25 │ +│ 2019-03-01 │ 56 │ 91 │ 1.62 │ +│ 2019-04-01 │ 87 │ 22 │ 0.25 │ +└────────────┴───────┴───────────┴────────────────┘ +``` + + ## runningDifference(x) {#other_functions-runningdifference} Calculates the difference between successive row values ​​in the data block. diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md index 1637c7bda93..6f5b6f1ff0d 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/query_language/functions/other_functions.md @@ -288,6 +288,48 @@ SELECT ## rowNumberInAllBlocks() Возвращает порядковый номер строки в блоке данных. Функция учитывает только задействованные блоки данных. +## neighbour(column, offset\[, default_value\]) + +Функция позволяет получить доступ к значению в колонке `column`, находящемуся на смещении `offset` относительно текущей строки. +Является частичной реализацией [оконных функций](https://en.wikipedia.org/wiki/SQL_window_function) LEAD() и LAG(). + +Результат функции зависит от затронутых блоков данных и порядка данных в блоке. +Если сделать подзапрос с ORDER BY и вызывать функцию извне подзапроса, можно будет получить ожидаемый результат. + +Если значение `offset` выходит за пределы блока данных, то берётся значение по-умолчанию для колонки `column`. Если передан параметр `default_value`, то значение берётся из него. +Например, эта функция может использоваться чтобы оценить year-over-year значение показателя: + +``` sql +WITH toDate('2018-01-01') AS start_date +SELECT + toStartOfMonth(start_date + (number * 32)) AS month, + toInt32(month) % 100 AS money, + neighbour(money, -12) AS prev_year, + round(prev_year / money, 2) AS year_over_year +FROM numbers(16) +``` + +``` +┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ +│ 2018-01-01 │ 32 │ 0 │ 0 │ +│ 2018-02-01 │ 63 │ 0 │ 0 │ +│ 2018-03-01 │ 91 │ 0 │ 0 │ +│ 2018-04-01 │ 22 │ 0 │ 0 │ +│ 2018-05-01 │ 52 │ 0 │ 0 │ +│ 2018-06-01 │ 83 │ 0 │ 0 │ +│ 2018-07-01 │ 13 │ 0 │ 0 │ +│ 2018-08-01 │ 44 │ 0 │ 0 │ +│ 2018-09-01 │ 75 │ 0 │ 0 │ +│ 2018-10-01 │ 5 │ 0 │ 0 │ +│ 2018-11-01 │ 36 │ 0 │ 0 │ +│ 2018-12-01 │ 66 │ 0 │ 0 │ +│ 2019-01-01 │ 97 │ 32 │ 0.33 │ +│ 2019-02-01 │ 28 │ 63 │ 2.25 │ +│ 2019-03-01 │ 56 │ 91 │ 1.62 │ +│ 2019-04-01 │ 87 │ 22 │ 0.25 │ +└────────────┴───────┴───────────┴────────────────┘ +``` + ## runningDifference(x) Считает разницу между последовательными значениями строк в блоке данных. Возвращает 0 для первой строки и разницу с предыдущей строкой для каждой последующей строки. From 6bf3902ce5d814758381bf15689c4585346478f0 Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Tue, 13 Aug 2019 16:20:32 +0300 Subject: [PATCH 064/133] Format file --- dbms/src/Functions/neighbour.cpp | 88 ++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index 79a1feec002..2efca01a66f 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -1,10 +1,10 @@ #include +#include #include #include #include #include #include -#include namespace DB { @@ -85,7 +85,8 @@ public: static void insertDefaults(const MutableColumnPtr & target, size_t row_count, ColumnPtr & default_values_column, size_t offset) { - if (row_count == 0) { + if (row_count == 0) + { return; } if (default_values_column) @@ -93,15 +94,20 @@ public: if (isColumnConst(*default_values_column)) { Field constant_value = (*default_values_column)[0]; - for(size_t row = 0; row < row_count;row++) + for (size_t row = 0; row < row_count; row++) { target->insert(constant_value); } - } else { + } + else + { target->insertRangeFrom(*default_values_column, offset, row_count); } - } else { - for(size_t row = 0; row < row_count;row++) { + } + else + { + for (size_t row = 0; row < row_count; row++) + { target->insertDefault(); } } @@ -119,7 +125,7 @@ public: default_values_column = block.getByPosition(arguments[2]).column; } - ColumnWithTypeAndName &source_column_name_and_type = block.getByPosition(arguments[0]); + ColumnWithTypeAndName & source_column_name_and_type = block.getByPosition(arguments[0]); DataTypes types = {source_column_name_and_type.type}; if (default_values_column) { @@ -129,23 +135,29 @@ public: auto source_column = source_column_name_and_type.column; // adjust source and default values columns to resulting datatype - if (!source_column_name_and_type.type->equals(*result_type)) { + if (!source_column_name_and_type.type->equals(*result_type)) + { source_column = castColumn(source_column_name_and_type, result_type, context); } - if (default_values_column && !block.getByPosition(arguments[2]).type->equals(*result_type)) { + if (default_values_column && !block.getByPosition(arguments[2]).type->equals(*result_type)) + { default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); } const DataTypePtr desired_type = std::make_shared(); - if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) { + if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) + { offset_column = castColumn(offset_structure, desired_type, context); } - if (isColumnConst(*source_column)) { + if (isColumnConst(*source_column)) + { auto column = result_type->createColumnConst(input_rows_count, (*source_column)[0]); block.getByPosition(result).column = std::move(column); - } else { + } + else + { auto column = result_type->createColumn(); column->reserve(input_rows_count); // with constant offset - insertRangeFrom @@ -156,48 +168,71 @@ public: if (offset_value > 0) { // insert shifted value - if ((size_t)offset_value <= input_rows_count) { + if ((size_t)offset_value <= input_rows_count) + { column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); } size_t row_count = (size_t)offset_value > input_rows_count ? input_rows_count : offset_value; insertDefaults(column, row_count, default_values_column, input_rows_count - row_count); - } else if (offset_value < 0) { + } + else if (offset_value < 0) + { size_t row_count = (size_t)std::abs(offset_value) > input_rows_count ? input_rows_count : std::abs(offset_value); // insert defaults up to offset_value insertDefaults(column, row_count, default_values_column, 0); column->insertRangeFrom(*source_column, 0, input_rows_count - row_count); - } else { + } + else + { // populate column with source values column->insertRangeFrom(*source_column, 0, input_rows_count); } - } else { + } + else + { // with dynamic offset - handle row by row for (size_t row = 0; row < input_rows_count; row++) { Int64 offset_value = offset_column->getInt(row); - if (offset_value == 0) { + if (offset_value == 0) + { column->insertFrom(*source_column, row); - } else if (offset_value > 0) { + } + else if (offset_value > 0) + { size_t real_offset = row + offset_value; - if (real_offset > input_rows_count) { - if (default_values_column) { + if (real_offset > input_rows_count) + { + if (default_values_column) + { column->insertFrom(*default_values_column, row); - } else { + } + else + { column->insertDefault(); } - } else { + } + else + { column->insertFrom(*column, real_offset); } - } else { + } + else + { // out of range if ((size_t)std::abs(offset_value) > row) { - if (default_values_column) { + if (default_values_column) + { column->insertFrom(*default_values_column, row); - } else { + } + else + { column->insertDefault(); } - } else { + } + else + { column->insertFrom(*column, row - std::abs(offset_value)); } } @@ -206,6 +241,7 @@ public: block.getByPosition(result).column = std::move(column); } } + private: const Context & context; }; From 986d56ba0c1d80b0fc879f22750e42a1f8b213f1 Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Wed, 14 Aug 2019 14:32:03 +0300 Subject: [PATCH 065/133] Fix casting style, work with Nullable --- dbms/src/Functions/neighbour.cpp | 30 ++++++++++++------- .../registerFunctionsMiscellaneous.cpp | 4 +-- .../queries/0_stateless/00957_neighbour.sql | 4 +++ 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index 2efca01a66f..153307abd4b 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -54,13 +54,14 @@ public: + ", should be from 2 to 3", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - // second argument must be a positive integer + // second argument must be an integer if (!isInteger(arguments[1])) throw Exception( "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + " - should be positive integer", + + " - should be an integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + // check that default value column has supertype with first argument if (number_of_arguments == 3) { @@ -116,7 +117,13 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { auto offset_structure = block.getByPosition(arguments[1]); + ColumnPtr & offset_column = offset_structure.column; + if (isColumnNullable(*offset_column)) + throw Exception( + "Illegal type " + offset_structure.type->getName() + " of second argument of function " + getName() + + " - can not be Nullable", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto is_constant_offset = isColumnConst(*offset_structure.column); ColumnPtr default_values_column = nullptr; @@ -145,6 +152,7 @@ public: default_values_column = castColumn(block.getByPosition(arguments[2]), result_type, context); } + // since we are working with both signed and unsigned - we'll try to use Int64 for handling all of them const DataTypePtr desired_type = std::make_shared(); if (!block.getByPosition(arguments[1]).type->equals(*desired_type)) { @@ -165,26 +173,26 @@ public: { Int64 offset_value = offset_column->getInt(0); + auto offset_value_casted = static_cast(std::abs(offset_value)); + size_t default_value_count = std::min(offset_value_casted, input_rows_count); if (offset_value > 0) { // insert shifted value - if ((size_t)offset_value <= input_rows_count) + if (offset_value_casted <= input_rows_count) { - column->insertRangeFrom(*source_column, offset_value, input_rows_count - offset_value); + column->insertRangeFrom(*source_column, offset_value_casted, input_rows_count - offset_value_casted); } - size_t row_count = (size_t)offset_value > input_rows_count ? input_rows_count : offset_value; - insertDefaults(column, row_count, default_values_column, input_rows_count - row_count); + insertDefaults(column, default_value_count, default_values_column, input_rows_count - default_value_count); } else if (offset_value < 0) { - size_t row_count = (size_t)std::abs(offset_value) > input_rows_count ? input_rows_count : std::abs(offset_value); // insert defaults up to offset_value - insertDefaults(column, row_count, default_values_column, 0); - column->insertRangeFrom(*source_column, 0, input_rows_count - row_count); + insertDefaults(column, default_value_count, default_values_column, 0); + column->insertRangeFrom(*source_column, 0, input_rows_count - default_value_count); } else { - // populate column with source values + // populate column with source values, when offset is equal to zero column->insertRangeFrom(*source_column, 0, input_rows_count); } } @@ -246,7 +254,7 @@ private: const Context & context; }; -void registerFunctionNextInBlock(FunctionFactory & factory) +void registerFunctionNeighbour(FunctionFactory & factory) { factory.registerFunction(); } diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp index 57ccfcd11c9..c96f5f05c7b 100644 --- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp @@ -17,7 +17,7 @@ void registerFunctionBlockSize(FunctionFactory &); void registerFunctionBlockNumber(FunctionFactory &); void registerFunctionRowNumberInBlock(FunctionFactory &); void registerFunctionRowNumberInAllBlocks(FunctionFactory &); -void registerFunctionNextInBlock(FunctionFactory &); +void registerFunctionNeighbour(FunctionFactory &); void registerFunctionSleep(FunctionFactory &); void registerFunctionSleepEachRow(FunctionFactory &); void registerFunctionMaterialize(FunctionFactory &); @@ -68,7 +68,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionBlockNumber(factory); registerFunctionRowNumberInBlock(factory); registerFunctionRowNumberInAllBlocks(factory); - registerFunctionNextInBlock(factory); + registerFunctionNeighbour(factory); registerFunctionSleep(factory); registerFunctionSleepEachRow(factory); registerFunctionMaterialize(factory); diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.sql b/dbms/tests/queries/0_stateless/00957_neighbour.sql index 156672155a8..665936fd70f 100644 --- a/dbms/tests/queries/0_stateless/00957_neighbour.sql +++ b/dbms/tests/queries/0_stateless/00957_neighbour.sql @@ -8,6 +8,10 @@ select neighbour(1,2,3,4); -- { serverError 42 } select neighbour(dummy, 1, 'hello'); -- { serverError 43 } -- types without common supertype (UInt64 and Int8) select number, neighbour(number, 1, -10) from numbers(3); -- { serverError 43 } +-- nullable offset is not allowed +select number, if(number > 1, number, null) as offset, neighbour(number, offset) from numbers(3); -- { serverError 43 } +select 'Zero offset'; +select number, neighbour(number, 0) from numbers(3); select 'Result with different type'; select toInt32(number) as n, neighbour(n, 1, -10) from numbers(3); select 'Offset > block'; From 2126196c8946126cd32322f259571c241b7f857d Mon Sep 17 00:00:00 2001 From: Alexandr Krasheninnikov Date: Wed, 14 Aug 2019 15:09:51 +0300 Subject: [PATCH 066/133] Nullable correct handling --- dbms/src/Functions/neighbour.cpp | 21 ++++++++++--------- .../0_stateless/00957_neighbour.reference | 8 +++++++ .../queries/0_stateless/00957_neighbour.sql | 2 ++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/dbms/src/Functions/neighbour.cpp b/dbms/src/Functions/neighbour.cpp index 153307abd4b..06307884f22 100644 --- a/dbms/src/Functions/neighbour.cpp +++ b/dbms/src/Functions/neighbour.cpp @@ -44,6 +44,8 @@ public: bool isDeterministicInScopeOfQuery() const override { return false; } + bool useDefaultImplementationForNulls() const override { return false; } + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { size_t number_of_arguments = arguments.size(); @@ -57,8 +59,11 @@ public: // second argument must be an integer if (!isInteger(arguments[1])) throw Exception( - "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + " - should be an integer", + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + " - should be an integer", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else if (arguments[1]->isNullable()) + throw Exception( + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + " - can not be Nullable", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -119,11 +124,6 @@ public: auto offset_structure = block.getByPosition(arguments[1]); ColumnPtr & offset_column = offset_structure.column; - if (isColumnNullable(*offset_column)) - throw Exception( - "Illegal type " + offset_structure.type->getName() + " of second argument of function " + getName() - + " - can not be Nullable", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto is_constant_offset = isColumnConst(*offset_structure.column); ColumnPtr default_values_column = nullptr; @@ -222,13 +222,14 @@ public: } else { - column->insertFrom(*column, real_offset); + column->insertFrom(*source_column, real_offset); } } else { // out of range - if ((size_t)std::abs(offset_value) > row) + auto offset_value_casted = static_cast(std::abs(offset_value)); + if (offset_value_casted > row) { if (default_values_column) { @@ -241,7 +242,7 @@ public: } else { - column->insertFrom(*column, row - std::abs(offset_value)); + column->insertFrom(*column, row - offset_value_casted); } } } diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.reference b/dbms/tests/queries/0_stateless/00957_neighbour.reference index cd8c6310f22..1983488cfc2 100644 --- a/dbms/tests/queries/0_stateless/00957_neighbour.reference +++ b/dbms/tests/queries/0_stateless/00957_neighbour.reference @@ -1,3 +1,11 @@ +Zero offset +0 0 +1 1 +2 2 +Nullable values +\N 0 \N +\N 1 2 +2 2 \N Result with different type 0 1 1 2 diff --git a/dbms/tests/queries/0_stateless/00957_neighbour.sql b/dbms/tests/queries/0_stateless/00957_neighbour.sql index 665936fd70f..753ae8493a3 100644 --- a/dbms/tests/queries/0_stateless/00957_neighbour.sql +++ b/dbms/tests/queries/0_stateless/00957_neighbour.sql @@ -12,6 +12,8 @@ select number, neighbour(number, 1, -10) from numbers(3); -- { serverError 43 } select number, if(number > 1, number, null) as offset, neighbour(number, offset) from numbers(3); -- { serverError 43 } select 'Zero offset'; select number, neighbour(number, 0) from numbers(3); +select 'Nullable values'; +select if(number > 1, number, null) as value, number as offset, neighbour(value, offset) as neighbour from numbers(3); select 'Result with different type'; select toInt32(number) as n, neighbour(n, 1, -10) from numbers(3); select 'Offset > block'; From ab1c4139deaad29f00104580df09f2334bb4efaa Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Wed, 14 Aug 2019 22:51:03 +0300 Subject: [PATCH 067/133] Added ReplicatedMergeTree support and test for constraints, also added VIOLATED_CONSTRAINT error --- dbms/programs/server/config.xml | 5 ++- dbms/src/Common/ErrorCodes.cpp | 1 + .../CheckConstraintsBlockOutputStream.cpp | 2 +- .../CheckConstraintsBlockOutputStream.h | 2 +- .../ReplicatedMergeTreeTableMetadata.cpp | 6 +++ .../Storages/StorageReplicatedMergeTree.cpp | 4 ++ ...onstraints_replication_zookeeper.reference | 0 ...0988_constraints_replication_zookeeper.sql | 43 +++++++++++++++++++ 8 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.reference create mode 100644 dbms/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index c09913cbd87..188a98779e9 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -217,7 +217,10 @@ See https://clickhouse.yandex/docs/en/table_engines/replication/ --> - + + + testkeeper + - - testkeeper - + diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md index 51ac4126e2d..38dfcb5ef64 120000 --- a/docs/zh/database_engines/mysql.md +++ b/docs/zh/database_engines/mysql.md @@ -1 +1,124 @@ -../../en/database_engines/mysql.md \ No newline at end of file +# MySQL + +MySQL引擎用于将远程的MySQL服务器中的表映射到ClickHouse中，并允许您对表进行`INSERT`和`SELECT`查询，以方便您在ClickHouse与MySQL之间进行数据交换。 + +`MySQL`数据库引擎会将对其的查询转换为MySQL语法并发送到MySQL服务器中，因此您可以执行诸如`SHOW TABLES`或`SHOW CREATE TABLE`之类的操作。 + +但您无法对其执行以下操作： + +- `ATTACH`/`DETACH` +- `DROP` +- `RENAME` +- `CREATE TABLE` +- `ALTER` + + +## CREATE DATABASE + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +ENGINE = MySQL('host:port', 'database', 'user', 'password') +``` + +**MySQL数据库引擎参数** + +- `host:port` — 链接的MySQL地址。 +- `database` — 链接的MySQL数据库。 +- `user` — 链接的MySQL用户。 +- `password` — 链接的MySQL用户密码。 + + +## 支持的类型对应 + +MySQL | ClickHouse +------|------------ +UNSIGNED TINYINT | [UInt8](../data_types/int_uint.md) +TINYINT | [Int8](../data_types/int_uint.md) +UNSIGNED SMALLINT | [UInt16](../data_types/int_uint.md) +SMALLINT | [Int16](../data_types/int_uint.md) +UNSIGNED INT, UNSIGNED MEDIUMINT | [UInt32](../data_types/int_uint.md) +INT, MEDIUMINT | [Int32](../data_types/int_uint.md) +UNSIGNED BIGINT | [UInt64](../data_types/int_uint.md) +BIGINT | [Int64](../data_types/int_uint.md) +FLOAT | [Float32](../data_types/float.md) +DOUBLE | [Float64](../data_types/float.md) +DATE | [Date](../data_types/date.md) +DATETIME, TIMESTAMP | [DateTime](../data_types/datetime.md) +BINARY | [FixedString](../data_types/fixedstring.md) + +其他的MySQL数据类型将全部都转换为[String](../data_types/string.md)。 + +同时以上的所有类型都支持[Nullable](../data_types/nullable.md)。 + + +## 使用示例 + +在MySQL中创建表: + +``` +mysql> USE test; +Database changed + +mysql> CREATE TABLE `mysql_table` ( + -> `int_id` INT NOT NULL AUTO_INCREMENT, + -> `float` FLOAT NOT NULL, + -> PRIMARY KEY (`int_id`)); +Query OK, 0 rows affected (0,09 sec) + +mysql> insert into mysql_table (`int_id`, `float`) VALUES (1,2); +Query OK, 1 row affected (0,00 sec) + +mysql> select * from mysql_table; ++--------+-------+ +| int_id | value | ++--------+-------+ +| 1 | 2 | ++--------+-------+ +1 row in set (0,00 sec) +``` + +在ClickHouse中创建MySQL类型的数据库，同时与MySQL服务器交换数据： + +```sql +CREATE DATABASE mysql_db ENGINE = MySQL('localhost:3306', 'test', 'my_user', 'user_password') +``` +```sql +SHOW DATABASES +``` +```text +┌─name─────┐ +│ default │ +│ mysql_db │ +│ system │ +└──────────┘ +``` +```sql +SHOW TABLES FROM mysql_db +``` +```text +┌─name─────────┐ +│ mysql_table │ +└──────────────┘ +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +└────────┴───────┘ +``` +```sql +INSERT INTO mysql_db.mysql_table VALUES (3,4) +``` +```sql +SELECT * FROM mysql_db.mysql_table +``` +```text +┌─int_id─┬─value─┐ +│ 1 │ 2 │ +│ 3 │ 4 │ +└────────┴───────┘ +``` + +[来源文章](https://clickhouse.yandex/docs/en/database_engines/mysql/) diff --git a/docs/zh/operations/table_engines/mergetree.md b/docs/zh/operations/table_engines/mergetree.md index 5ddf837708a..5e330164c5a 100644 --- a/docs/zh/operations/table_engines/mergetree.md +++ b/docs/zh/operations/table_engines/mergetree.md @@ -48,7 +48,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **子句** -- `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`. `MergeTree` 引擎没有参数。 +- `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`。 `MergeTree` 引擎不需要其他参数。 - `PARTITION BY` — [分区键](custom_partitioning_key.md) 。