From 4d5fb56c29ad6be3c0c7bd4b838c1a61c624be49 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Mon, 7 Feb 2022 23:21:10 +0000 Subject: [PATCH] added EPHEMERAL default for column (preliminary) --- .../processColumnTransformers.cpp | 2 +- src/Parsers/ParserCreateQuery.h | 5 ++++ src/Parsers/obfuscateQueries.cpp | 28 +++++++++---------- src/Storages/ColumnDefault.cpp | 7 +++-- src/Storages/ColumnDefault.h | 3 +- src/Storages/ColumnsDescription.cpp | 20 +++++++++++++ src/Storages/ColumnsDescription.h | 7 +++-- src/Storages/StorageInMemoryMetadata.cpp | 2 +- 8 files changed, 53 insertions(+), 21 deletions(-) diff --git a/src/Interpreters/processColumnTransformers.cpp b/src/Interpreters/processColumnTransformers.cpp index 2a704d4a937..6179d2c9f2f 100644 --- a/src/Interpreters/processColumnTransformers.cpp +++ b/src/Interpreters/processColumnTransformers.cpp @@ -19,7 +19,7 @@ ASTPtr processColumnTransformers( ASTPtr query_columns) { const auto & columns = metadata_snapshot->getColumns(); - auto names_and_types = columns.getOrdinary(); + auto names_and_types = columns.getInsertable(); removeDuplicateColumns(names_and_types); TablesWithColumns tables_with_columns; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 615121eae58..4adf343db15 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -124,6 +124,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserKeyword s_null{"NULL"}; ParserKeyword s_not{"NOT"}; ParserKeyword s_materialized{"MATERIALIZED"}; + ParserKeyword s_ephemeral{"EPHEMERAL"}; ParserKeyword s_alias{"ALIAS"}; ParserKeyword s_comment{"COMMENT"}; ParserKeyword s_codec{"CODEC"}; @@ -171,6 +172,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (!s_default.checkWithoutMoving(pos, expected) && !s_materialized.checkWithoutMoving(pos, expected) + && !s_ephemeral.checkWithoutMoving(pos, expected) && !s_alias.checkWithoutMoving(pos, expected) && (require_type || (!s_comment.checkWithoutMoving(pos, expected) @@ -190,6 +192,9 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E /// should be followed by an expression if (!expr_parser.parse(pos, default_expression, expected)) return false; + } else if (s_ephemeral.ignore(pos, expected)) { + default_specifier = Poco::toUpper(std::string{pos_before_specifier->begin, pos_before_specifier->end}); + expr_parser.parse(pos, default_expression, expected); } if (require_type && !type && !default_expression) diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index c0b57d9b1f5..a4eb5404552 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -26,20 +26,20 @@ namespace const std::unordered_set keywords { - "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", - "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", - "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", - "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", - "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", - "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", - "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", - "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", - "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", - "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE", - "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", - "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", - "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "DICTIONARY", "OFFSET", - "TRIM", "LTRIM", "RTRIM", "BOTH", "LEADING", "TRAILING" + "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", + "MATERIALIZED", "EPHEMERAL", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", + "DROP", "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", + "PROJECT", "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", + "INTO", "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", + "ELSE", "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", + "VALUES", "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", + "INNER", "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", + "BY", "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", + "ASC", "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", + "ROLE", "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", + "REPLACE", "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", + "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "DICTIONARY", "OFFSET", "TRIM", + "LTRIM", "RTRIM", "BOTH", "LEADING", "TRAILING" }; const std::unordered_set keep_words diff --git a/src/Storages/ColumnDefault.cpp b/src/Storages/ColumnDefault.cpp index d35d3c31470..3cf49ea69fc 100644 --- a/src/Storages/ColumnDefault.cpp +++ b/src/Storages/ColumnDefault.cpp @@ -9,6 +9,7 @@ struct AliasNames static constexpr const char * DEFAULT = "DEFAULT"; static constexpr const char * MATERIALIZED = "MATERIALIZED"; static constexpr const char * ALIAS = "ALIAS"; + static constexpr const char * EPHEMERAL = "EPHEMERAL"; }; } @@ -27,7 +28,8 @@ ColumnDefaultKind columnDefaultKindFromString(const std::string & str) static const std::unordered_map map{ { AliasNames::DEFAULT, ColumnDefaultKind::Default }, { AliasNames::MATERIALIZED, ColumnDefaultKind::Materialized }, - { AliasNames::ALIAS, ColumnDefaultKind::Alias } + { AliasNames::ALIAS, ColumnDefaultKind::Alias }, + { AliasNames::EPHEMERAL, ColumnDefaultKind::Ephemeral } }; const auto it = map.find(str); @@ -43,7 +45,8 @@ std::string toString(const ColumnDefaultKind kind) static const std::unordered_map map{ { ColumnDefaultKind::Default, AliasNames::DEFAULT }, { ColumnDefaultKind::Materialized, AliasNames::MATERIALIZED }, - { ColumnDefaultKind::Alias, AliasNames::ALIAS } + { ColumnDefaultKind::Alias, AliasNames::ALIAS }, + { ColumnDefaultKind::Ephemeral, AliasNames::EPHEMERAL } }; const auto it = map.find(kind); diff --git a/src/Storages/ColumnDefault.h b/src/Storages/ColumnDefault.h index 38b61415a9a..096a1f177ab 100644 --- a/src/Storages/ColumnDefault.h +++ b/src/Storages/ColumnDefault.h @@ -13,7 +13,8 @@ enum class ColumnDefaultKind { Default, Materialized, - Alias + Alias, + Ephemeral }; diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 568257b4fd7..a7315597456 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -340,6 +340,15 @@ NamesAndTypesList ColumnsDescription::getOrdinary() const return ret; } +NamesAndTypesList ColumnsDescription::getInsertable() const +{ + NamesAndTypesList ret; + for (const auto & col : columns) + if (col.default_desc.kind == ColumnDefaultKind::Default || col.default_desc.kind == ColumnDefaultKind::Ephemeral) + ret.emplace_back(col.name, col.type); + return ret; +} + NamesAndTypesList ColumnsDescription::getMaterialized() const { NamesAndTypesList ret; @@ -358,6 +367,15 @@ NamesAndTypesList ColumnsDescription::getAliases() const return ret; } +NamesAndTypesList ColumnsDescription::getEphemeral() const +{ + NamesAndTypesList ret; + for (const auto & col : columns) + if (col.default_desc.kind == ColumnDefaultKind::Ephemeral) + ret.emplace_back(col.name, col.type); + return ret; +} + NamesAndTypesList ColumnsDescription::getAll() const { NamesAndTypesList ret; @@ -402,6 +420,8 @@ static ColumnsDescription::GetFlags defaultKindToGetFlag(ColumnDefaultKind kind) return ColumnsDescription::Materialized; case ColumnDefaultKind::Alias: return ColumnsDescription::Aliases; + case ColumnDefaultKind::Ephemeral: + return ColumnsDescription::Ephemeral; } __builtin_unreachable(); } diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h index 44f895c89ce..1a80e09c193 100644 --- a/src/Storages/ColumnsDescription.h +++ b/src/Storages/ColumnsDescription.h @@ -84,18 +84,21 @@ public: Ordinary = 1, Materialized = 2, Aliases = 4, + Ephemeral = 8, AllPhysical = Ordinary | Materialized, - All = AllPhysical | Aliases, + All = AllPhysical | Aliases | Ephemeral, }; NamesAndTypesList getByNames(GetFlags flags, const Names & names, bool with_subcolumns) const; NamesAndTypesList getOrdinary() const; NamesAndTypesList getMaterialized() const; + NamesAndTypesList getInsertable() const; NamesAndTypesList getAliases() const; + NamesAndTypesList getEphemeral() const; NamesAndTypesList getAllPhysical() const; /// ordinary + materialized. - NamesAndTypesList getAll() const; /// ordinary + materialized + aliases + NamesAndTypesList getAll() const; /// ordinary + materialized + aliases + ephemeral NamesAndTypesList getAllWithSubcolumns() const; NamesAndTypesList getAllPhysicalWithSubcolumns() const; diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 061319bc1c0..bb1c0011ef3 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -304,7 +304,7 @@ Block StorageInMemoryMetadata::getSampleBlockNonMaterialized() const { Block res; - for (const auto & column : getColumns().getOrdinary()) + for (const auto & column : getColumns().getInsertable()) res.insert({column.type->createColumn(), column.type, column.name}); return res;