From 4e9d894e24041f7c462a486394f19d602403ae15 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 Oct 2022 01:32:35 +0200 Subject: [PATCH 001/308] Compress marks and primary key by default --- src/Storages/MergeTree/MergeTreeSettings.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index a0db39a97f1..210846b9bb2 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -147,8 +147,8 @@ struct Settings; M(Bool, remote_fs_zero_copy_path_compatible_mode, false, "Run zero-copy in compatible mode during conversion process.", 0) \ \ /** Compress marks and primary key. */ \ - M(Bool, compress_marks, false, "Marks support compression, reduce mark file size and speed up network transmission.", 0) \ - M(Bool, compress_primary_key, false, "Primary key support compression, reduce primary key file size and speed up network transmission.", 0) \ + M(Bool, compress_marks, true, "Marks support compression, reduce mark file size and speed up network transmission.", 0) \ + M(Bool, compress_primary_key, true, "Primary key support compression, reduce primary key file size and speed up network transmission.", 0) \ M(String, marks_compression_codec, "ZSTD(3)", "Compression encoding used by marks, marks are small enough and cached, so the default compression is ZSTD(3).", 0) \ M(String, primary_key_compression_codec, "ZSTD(3)", "Compression encoding used by primary, primary key is small enough and cached, so the default compression is ZSTD(3).", 0) \ M(UInt64, marks_compress_block_size, 65536, "Mark compress block size, the actual size of the block to compress.", 0) \ From 462e7f76bf20b2872e3605a17080a97d1d6e8e33 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Dec 2022 13:28:25 +0100 Subject: [PATCH 002/308] Attempt to fix the "system.stack_trace" test --- src/Core/Settings.h | 1 + src/Storages/System/StorageSystemStackTrace.cpp | 6 +++++- tests/queries/0_stateless/01051_system_stack_trace.sql | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b10760c8277..f26d6a2e12a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -655,6 +655,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(String, additional_result_filter, "", "Additional filter expression which would be applied to query result", 0) \ \ M(String, workload, "default", "Name of workload to be used to access resources", 0) \ + M(Milliseconds, storage_system_stack_trace_pipe_read_timeout_ms, 100, "Maximum time to read from a pipe for receiving information from the threads when querying the `system.stack_trace` table. This setting is used for testing purposes and not meant to be changed by users.", 0) \ \ /** Experimental functions */ \ M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index df3d8b74e6e..8f3f9744725 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -262,6 +263,9 @@ Pipe StorageSystemStackTrace::read( { storage_snapshot->check(column_names); + int pipe_read_timeout_ms = static_cast( + context->getSettingsRef().storage_system_stack_trace_pipe_read_timeout_ms.totalMilliseconds()); + /// It shouldn't be possible to do concurrent reads from this table. std::lock_guard lock(mutex); @@ -334,7 +338,7 @@ Pipe StorageSystemStackTrace::read( } /// Just in case we will wait for pipe with timeout. In case signal didn't get processed. - if (send_signal && wait(100) && sig_value.sival_int == data_ready_num.load(std::memory_order_acquire)) + if (send_signal && wait(pipe_read_timeout_ms) && sig_value.sival_int == data_ready_num.load(std::memory_order_acquire)) { size_t stack_trace_size = stack_trace.getSize(); size_t stack_trace_offset = stack_trace.getOffset(); diff --git a/tests/queries/0_stateless/01051_system_stack_trace.sql b/tests/queries/0_stateless/01051_system_stack_trace.sql index e322462a46a..a7e6660d06e 100644 --- a/tests/queries/0_stateless/01051_system_stack_trace.sql +++ b/tests/queries/0_stateless/01051_system_stack_trace.sql @@ -1,4 +1,4 @@ --- Tags: race +SET storage_system_stack_trace_pipe_read_timeout_ms = 10000; -- { echo } SELECT count() > 0 FROM system.stack_trace WHERE query_id != ''; From 62acc7a6cfa9b0dbb7613961684a46299eceac1e Mon Sep 17 00:00:00 2001 From: Mike Kot Date: Fri, 17 Feb 2023 10:36:58 +0000 Subject: [PATCH 003/308] setting and some docs --- .../operations/settings/settings-formats.md | 17 ++ docs/en/sql-reference/statements/grant.md | 8 + docs/en/sql-reference/statements/show.md | 9 +- src/Access/Common/AccessType.h | 1 + src/Core/Settings.h | 1 + .../Access/InterpreterShowAccessQuery.cpp | 3 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 3 +- .../InterpreterShowCreateQuery.cpp | 5 +- .../formatWithPossiblyHidingSecrets.h | 24 +++ src/Parsers/IAST.cpp | 8 +- src/Parsers/IAST.h | 28 +++- src/Parsers/ParserCreateQuery.h | 4 +- .../System/StorageSystemDatabases.cpp | 7 +- src/Storages/System/StorageSystemTables.cpp | 17 +- .../test_mask_sensitive_info/test.py | 155 +++++++++++------- .../01271_show_privileges.reference | 1 + .../02117_show_create_table_system.reference | 6 +- .../0_stateless/25337_show_secrets.reference | 17 ++ .../queries/0_stateless/25337_show_secrets.sh | 48 ++++++ 19 files changed, 264 insertions(+), 98 deletions(-) create mode 100644 src/Interpreters/formatWithPossiblyHidingSecrets.h create mode 100644 tests/queries/0_stateless/25337_show_secrets.reference create mode 100755 tests/queries/0_stateless/25337_show_secrets.sh diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 3f81dc528f5..dbd8f42412f 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -7,6 +7,23 @@ toc_max_heading_level: 2 # Format settings {#format-settings} +## display_secrets_in_show_and_select_query {#display_secrets_in_show_and_select_query} + +Enables or disables showing secrets in `SHOW` and `SELECT` queries. +User wishing to see secrets must also have +[`displaySecretsInShowSelect`](../../sql-reference/statements/grant#grant-display-secrets) privilege. + +:::info Warning Turning this setting on allows users with all privileges to view secrets immediately. +Please grant/revoke [`displaySecretsInShowSelect`](../../sql-reference/statements/grant#grant-display-secrets) +privilege first and turn this setting on after that ::: + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: 0. + ## input_format_skip_unknown_fields {#input_format_skip_unknown_fields} Enables or disables skipping insertion of extra data. diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 3383ea70a2b..0335e398659 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -187,6 +187,7 @@ Hierarchy of privileges: - `HDFS` - `S3` - [dictGet](#grant-dictget) +- [displaySecretsInShowSelect](#grant-display-secrets) Examples of how this hierarchy is treated: @@ -471,6 +472,13 @@ Privilege level: `DICTIONARY`. - `GRANT dictGet ON mydb.mydictionary TO john` - `GRANT dictGet ON mydictionary TO john` + +### displaySecretsInShowSelect {#grant-display-secrets} + +Allows a user to view secrets in `SHOW` and `SELECT` queries if +[`display_secrets_in_show_select_query`](../../operations/settings/formats#display_secrets_in_show_select_query) +setting is turned on. Otherwise this privilege does nothing. + ### ALL Grants all the privileges on regulated entity to a user account or a role. diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 18b019dd017..307814b8ff1 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -6,6 +6,11 @@ sidebar_label: SHOW # SHOW Statements +N.B. `SHOW CREATE (TABLE|DATABASE|USER)` hides secrets unless +[`display_secrets_in_show_select_query`](../../operations/settings/formats#display_secrets_in_show_select_query) +is turned on and user has +[`displaySecretsInShowSelect`](grant.md#grant-display-secrets) privilege. + ## SHOW CREATE TABLE ``` sql @@ -243,8 +248,6 @@ If user is not specified, the query returns privileges for the current user. Shows parameters that were used at a [user creation](../../sql-reference/statements/create/user.md). -`SHOW CREATE USER` does not output user passwords. - ### Syntax ``` sql @@ -521,4 +524,4 @@ Outputs the content of the [system.table_engines](../../operations/system-tables **See Also** -- [system.table_engines](../../operations/system-tables/table_engines.md) table \ No newline at end of file +- [system.table_engines](../../operations/system-tables/table_engines.md) table diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 497327c1bad..8150a536f57 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -181,6 +181,7 @@ enum class AccessType M(SYSTEM, "", GROUP, ALL) /* allows to execute SYSTEM {SHUTDOWN|RELOAD CONFIG|...} */ \ \ M(dictGet, "dictHas, dictGetHierarchy, dictIsIn", DICTIONARY, ALL) /* allows to execute functions dictGet(), dictHas(), dictGetHierarchy(), dictIsIn() */\ + M(displaySecretsInShowSelect, "", GLOBAL, ALL) /* allows to show plaintext secrets in SELECT and SHOW queries. display_secrets_in_show_and_select_query setting must be turned on, otherwise this privilege does nothing */\ \ M(addressToLine, "", GLOBAL, INTROSPECTION) /* allows to execute function addressToLine() */\ M(addressToLineWithInlines, "", GLOBAL, INTROSPECTION) /* allows to execute function addressToLineWithInlines() */\ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 1cd5e93c499..ed7d91dcf6a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -907,6 +907,7 @@ class IColumn; \ M(Bool, regexp_dict_allow_other_sources, false, "Allow regexp_tree dictionary to use sources other than yaml source.", 0) \ M(Bool, regexp_dict_allow_hyperscan, false, "Allow regexp_tree dictionary using Hyperscan library.", 0) \ + M(Bool, display_secrets_in_show_and_select_query, false, "Do not hide secrets in SHOW and SELECT queries. User must also have 'displaySecretsInShowSelect' privilege", IMPORTANT) \ // End of FORMAT_FACTORY_SETTINGS // Please add settings non-related to formats into the COMMON_SETTINGS above. diff --git a/src/Interpreters/Access/InterpreterShowAccessQuery.cpp b/src/Interpreters/Access/InterpreterShowAccessQuery.cpp index b5478f434b4..e9862e99393 100644 --- a/src/Interpreters/Access/InterpreterShowAccessQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowAccessQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -33,7 +34,7 @@ QueryPipeline InterpreterShowAccessQuery::executeImpl() const /// Build the result column. MutableColumnPtr column = ColumnString::create(); for (const auto & query : queries) - column->insert(query->formatWithSecretsHidden()); + column->insert(format({getContext(), *query})); String desc = "ACCESS"; return QueryPipeline(std::make_shared(Block{{std::move(column), std::make_shared(), desc}})); diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 7b9a8f98c8f..e02088acce4 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -254,7 +255,7 @@ QueryPipeline InterpreterShowCreateAccessEntityQuery::executeImpl() /// Build the result column. MutableColumnPtr column = ColumnString::create(); for (const auto & create_query : create_queries) - column->insert(create_query->formatWithSecretsHidden()); + column->insert(format({getContext(), *create_query})); /// Prepare description of the result column. const auto & show_query = query_ptr->as(); diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 5e1b74681fe..0d60f13af66 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -94,10 +95,8 @@ QueryPipeline InterpreterShowCreateQuery::executeImpl() create.to_inner_uuid = UUIDHelpers::Nil; } - String res = create_query->formatWithSecretsHidden(/* max_length= */ 0, /* one_line= */ false); - MutableColumnPtr column = ColumnString::create(); - column->insert(res); + column->insert(format({.ctx = getContext(), .query = *create_query, .one_line = false})); return QueryPipeline(std::make_shared(Block{{ std::move(column), diff --git a/src/Interpreters/formatWithPossiblyHidingSecrets.h b/src/Interpreters/formatWithPossiblyHidingSecrets.h new file mode 100644 index 00000000000..65cb019cf9b --- /dev/null +++ b/src/Interpreters/formatWithPossiblyHidingSecrets.h @@ -0,0 +1,24 @@ +#pragma once +#include "Access/ContextAccess.h" +#include "Interpreters/Context.h" + +namespace DB +{ +struct SecretHidingFormatSettings +{ + // We can't store const Context& as there's a dangerous usage {.ctx = *getContext()} + // which is UB in case getContext()'s return ptr is the only one holding the object + const ContextPtr & ctx; + const IAST & query; + size_t max_length = 0; + bool one_line = true; +}; + +inline String format(const SecretHidingFormatSettings & settings) +{ + const bool show_secrets = settings.ctx->getSettingsRef().display_secrets_in_show_and_select_query + && settings.ctx->getAccess()->isGranted(AccessType::displaySecretsInShowSelect); + + return settings.query.formatWithPossiblyHidingSensitiveData(settings.max_length, settings.one_line, show_secrets); +} +} diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index 869c0969dd6..0138372ce89 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -167,14 +167,10 @@ size_t IAST::checkDepthImpl(size_t max_depth) const return res; } -String IAST::formatWithSecretsHidden(size_t max_length, bool one_line) const +String IAST::formatWithPossiblyHidingSensitiveData(size_t max_length, bool one_line, bool show_secrets) const { WriteBufferFromOwnString buf; - - FormatSettings settings{buf, one_line}; - settings.show_secrets = false; - format(settings); - + format({buf, one_line, show_secrets}); return wipeSensitiveDataAndCutToLength(buf.str(), max_length); } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 627b1174b33..91f57cb8ed9 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -190,8 +190,8 @@ public: // Newline or whitespace. char nl_or_ws; - FormatSettings(WriteBuffer & ostr_, bool one_line_) - : ostr(ostr_), one_line(one_line_) + FormatSettings(WriteBuffer & ostr_, bool one_line_, bool show_secrets_ = true) + : ostr(ostr_), one_line(one_line_), show_secrets(show_secrets_) { nl_or_ws = one_line ? ' ' : '\n'; } @@ -240,12 +240,26 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown element in AST: {}", getID()); } - // A simple way to add some user-readable context to an error message. - String formatWithSecretsHidden(size_t max_length = 0, bool one_line = true) const; - String formatForLogging(size_t max_length = 0) const { return formatWithSecretsHidden(max_length, true); } - String formatForErrorMessage() const { return formatWithSecretsHidden(0, true); } + // Secrets are displayed regarding show_secrets, then SensitiveDataMasker is applied. + // You can use Interpreters/formatWithPossiblyHidingSecrets.h for convenience. + String formatWithPossiblyHidingSensitiveData(size_t max_length, bool one_line, bool show_secrets) const; + + /* + * formatForLogging and formatForErrorMessage always hide secrets. This inconsistent + * behaviour is due to the fact such functions are called from Client which knows nothing about + * access rights and settings. Moreover, the only use case for displaying secrets are backups, + * and backup tools use only direct input and ignore logs and error messages. + */ + String formatForLogging(size_t max_length = 0) const + { + return formatWithPossiblyHidingSensitiveData(max_length, true, false); + } + + String formatForErrorMessage() const + { + return formatWithPossiblyHidingSensitiveData(0, true, false); + } - /// If an AST has secret parts then formatForLogging() will replace them with the placeholder '[HIDDEN]'. virtual bool hasSecretParts() const { return childrenHaveSecretParts(); } void cloneChildren(); diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index ef87988aab2..838308752fe 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -244,7 +244,9 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E auto default_function = std::make_shared(); default_function->name = "defaultValueOfTypeName"; default_function->arguments = std::make_shared(); - default_function->arguments->children.emplace_back(std::make_shared(type->as()->formatWithSecretsHidden())); + // Ephemeral columns don't really have secrets but we need to format + // into a String, hence the strange call + default_function->arguments->children.emplace_back(std::make_shared(type->as()->formatForLogging())); default_expression = default_function; } diff --git a/src/Storages/System/StorageSystemDatabases.cpp b/src/Storages/System/StorageSystemDatabases.cpp index 4d1f6c171db..52243e383e4 100644 --- a/src/Storages/System/StorageSystemDatabases.cpp +++ b/src/Storages/System/StorageSystemDatabases.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ NamesAndAliases StorageSystemDatabases::getNamesAndAliases() }; } -static String getEngineFull(const DatabasePtr & database) +static String getEngineFull(const ContextPtr& ctx, const DatabasePtr & database) { DDLGuardPtr guard; while (true) @@ -59,7 +60,7 @@ static String getEngineFull(const DatabasePtr & database) if (!ast_create || !ast_create->storage) return {}; - String engine_full = ast_create->storage->formatWithSecretsHidden(); + String engine_full = format({ctx, *ast_create->storage}); static const char * const extra_head = " ENGINE = "; if (startsWith(engine_full, extra_head)) @@ -87,7 +88,7 @@ void StorageSystemDatabases::fillData(MutableColumns & res_columns, ContextPtr c res_columns[2]->insert(context->getPath() + database->getDataPath()); res_columns[3]->insert(database->getMetadataPath()); res_columns[4]->insert(database->getUUID()); - res_columns[5]->insert(getEngineFull(database)); + res_columns[5]->insert(getEngineFull(context, database)); res_columns[6]->insert(database->getDatabaseComment()); } } diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index e94d3710a64..e00d2d95568 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -220,7 +221,7 @@ protected: { auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables(); ASTPtr ast = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr; - res_columns[res_index++]->insert(ast ? ast->formatWithSecretsHidden() : ""); + res_columns[res_index++]->insert(ast ? format({context, *ast}) : ""); } // engine_full @@ -366,7 +367,7 @@ protected: } if (columns_mask[src_index++]) - res_columns[res_index++]->insert(ast ? ast->formatWithSecretsHidden() : ""); + res_columns[res_index++]->insert(ast ? format({context, *ast}) : ""); if (columns_mask[src_index++]) { @@ -374,7 +375,7 @@ protected: if (ast_create && ast_create->storage) { - engine_full = ast_create->storage->formatWithSecretsHidden(); + engine_full = format({context, *ast_create->storage}); static const char * const extra_head = " ENGINE = "; if (startsWith(engine_full, extra_head)) @@ -388,7 +389,7 @@ protected: { String as_select; if (ast_create && ast_create->select) - as_select = ast_create->select->formatWithSecretsHidden(); + as_select = format({context, *ast_create->select}); res_columns[res_index++]->insert(as_select); } } @@ -401,7 +402,7 @@ protected: if (columns_mask[src_index++]) { if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST())) - res_columns[res_index++]->insert(expression_ptr->formatWithSecretsHidden()); + res_columns[res_index++]->insert(format({context, *expression_ptr})); else res_columns[res_index++]->insertDefault(); } @@ -409,7 +410,7 @@ protected: if (columns_mask[src_index++]) { if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast)) - res_columns[res_index++]->insert(expression_ptr->formatWithSecretsHidden()); + res_columns[res_index++]->insert(format({context, *expression_ptr})); else res_columns[res_index++]->insertDefault(); } @@ -417,7 +418,7 @@ protected: if (columns_mask[src_index++]) { if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast)) - res_columns[res_index++]->insert(expression_ptr->formatWithSecretsHidden()); + res_columns[res_index++]->insert(format({context, *expression_ptr})); else res_columns[res_index++]->insertDefault(); } @@ -425,7 +426,7 @@ protected: if (columns_mask[src_index++]) { if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST())) - res_columns[res_index++]->insert(expression_ptr->formatWithSecretsHidden()); + res_columns[res_index++]->insert(format({context, *expression_ptr})); else res_columns[res_index++]->insertDefault(); } diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 3f71b047213..fce2d539320 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -72,7 +72,6 @@ def new_password(len=16): ) -# Passwords in CREATE/ALTER queries must be hidden in logs. def test_create_alter_user(): password = new_password() @@ -84,28 +83,42 @@ def test_create_alter_user(): f"CREATE USER u2 IDENTIFIED WITH plaintext_password BY '{password}' SETTINGS custom_c = 'c'" ) - assert ( - node.query("SHOW CREATE USER u1") - == "CREATE USER u1 IDENTIFIED WITH sha256_password SETTINGS custom_b = \\'b\\'\n" - ) - assert ( - node.query("SHOW CREATE USER u2") - == "CREATE USER u2 IDENTIFIED WITH plaintext_password SETTINGS custom_c = \\'c\\'\n" - ) + def check_no_secrets(): + assert ( + node.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH sha256_password SETTINGS custom_b = \\'b\\'\n" + ) + assert ( + node.query("SHOW CREATE USER u2") + == "CREATE USER u2 IDENTIFIED WITH plaintext_password SETTINGS custom_c = \\'c\\'\n" + ) - check_logs( - must_contain=[ - "CREATE USER u1 IDENTIFIED WITH sha256_password", - "ALTER USER u1 IDENTIFIED WITH sha256_password", - "CREATE USER u2 IDENTIFIED WITH plaintext_password", - ], - must_not_contain=[ - password, - "IDENTIFIED WITH sha256_password BY", - "IDENTIFIED WITH sha256_hash BY", - "IDENTIFIED WITH plaintext_password BY", - ], - ) + non_secrets = [ + "CREATE USER u1 IDENTIFIED WITH sha256_password", + "ALTER USER u1 IDENTIFIED WITH sha256_password", + "CREATE USER u2 IDENTIFIED WITH plaintext_password", + ] + + secrets = [ + password, + "IDENTIFIED WITH sha256_password BY", + "IDENTIFIED WITH sha256_hash BY", + "IDENTIFIED WITH plaintext_password BY", + ] + + check_no_secrets() + check_logs(must_contain=non_secrets, must_not_contain=secrets) + + node.query("SET show_secrets_in_show_query=1") + + assert node.query("SHOW CREATE USER u1").contains("BY") + assert node.query("SHOW CREATE USER u2").contains("BY") + check_logs(must_contain=secrets + non_secrets) + + node.query("SET show_secrets_in_show_query=0") + + check_no_secrets() + check_logs(must_contain=non_secrets, must_not_contain=secrets) node.query("DROP USER u1, u2") @@ -133,41 +146,50 @@ def test_create_table(): for i, table_engine in enumerate(table_engines): node.query(f"CREATE TABLE table{i} (x int) ENGINE = {table_engine}") - assert ( - node.query("SHOW CREATE TABLE table0") - == "CREATE TABLE default.table0\\n(\\n `x` Int32\\n)\\nENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')\n" - ) + def check_no_secrets(): + assert ( + node.query("SHOW CREATE TABLE table0") + == "CREATE TABLE default.table0\\n(\\n `x` Int32\\n)\\nENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')\n" + ) - assert node.query( - "SELECT create_table_query, engine_full FROM system.tables WHERE name = 'table0'" - ) == TSV( - [ + assert node.query( + "SELECT create_table_query, engine_full FROM system.tables WHERE name = 'table0'" + ) == TSV( [ - "CREATE TABLE default.table0 (`x` Int32) ENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", - "MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", - ], - ] - ) + [ + "CREATE TABLE default.table0 (`x` Int32) ENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", + "MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", + ], + ] + ) - check_logs( - must_contain=[ - "CREATE TABLE table0 (`x` int) ENGINE = MySQL('mysql57:3306', 'mysql_db', 'mysql_table', 'mysql_user', '[HIDDEN]')", - "CREATE TABLE table1 (`x` int) ENGINE = PostgreSQL('postgres1:5432', 'postgres_db', 'postgres_table', 'postgres_user', '[HIDDEN]')", - "CREATE TABLE table2 (`x` int) ENGINE = MongoDB('mongo1:27017', 'mongo_db', 'mongo_col', 'mongo_user', '[HIDDEN]')", - "CREATE TABLE table3 (x int) ENGINE = S3('http://minio1:9001/root/data/test1.csv')", - "CREATE TABLE table4 (x int) ENGINE = S3('http://minio1:9001/root/data/test2.csv', 'CSV')", - "CREATE TABLE table5 (x int) ENGINE = S3('http://minio1:9001/root/data/test3.csv.gz', 'CSV', 'gzip')", - "CREATE TABLE table6 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test4.csv', 'minio', '[HIDDEN]', 'CSV')", - "CREATE TABLE table7 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'minio', '[HIDDEN]', 'CSV', 'gzip')", - "CREATE TABLE table8 (`x` int) ENGINE = MySQL(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')", - "CREATE TABLE table9 (`x` int) ENGINE = MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '[HIDDEN]', table = 'mysql_table', user = 'mysql_user')", - "CREATE TABLE table10 (x int) ENGINE = MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')", - "CREATE TABLE table11 (`x` int) ENGINE = PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '[HIDDEN]')", - "CREATE TABLE table12 (`x` int) ENGINE = MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '[HIDDEN]'", - "CREATE TABLE table13 (`x` int) ENGINE = S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]', format = 'CSV')", - ], - must_not_contain=[password], - ) + check_logs( + must_contain=[ + "CREATE TABLE table0 (`x` int) ENGINE = MySQL('mysql57:3306', 'mysql_db', 'mysql_table', 'mysql_user', '[HIDDEN]')", + "CREATE TABLE table1 (`x` int) ENGINE = PostgreSQL('postgres1:5432', 'postgres_db', 'postgres_table', 'postgres_user', '[HIDDEN]')", + "CREATE TABLE table2 (`x` int) ENGINE = MongoDB('mongo1:27017', 'mongo_db', 'mongo_col', 'mongo_user', '[HIDDEN]')", + "CREATE TABLE table3 (x int) ENGINE = S3('http://minio1:9001/root/data/test1.csv')", + "CREATE TABLE table4 (x int) ENGINE = S3('http://minio1:9001/root/data/test2.csv', 'CSV')", + "CREATE TABLE table5 (x int) ENGINE = S3('http://minio1:9001/root/data/test3.csv.gz', 'CSV', 'gzip')", + "CREATE TABLE table6 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test4.csv', 'minio', '[HIDDEN]', 'CSV')", + "CREATE TABLE table7 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'minio', '[HIDDEN]', 'CSV', 'gzip')", + "CREATE TABLE table8 (`x` int) ENGINE = MySQL(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')", + "CREATE TABLE table9 (`x` int) ENGINE = MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '[HIDDEN]', table = 'mysql_table', user = 'mysql_user')", + "CREATE TABLE table10 (x int) ENGINE = MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')", + "CREATE TABLE table11 (`x` int) ENGINE = PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '[HIDDEN]')", + "CREATE TABLE table12 (`x` int) ENGINE = MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '[HIDDEN]'", + "CREATE TABLE table13 (`x` int) ENGINE = S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]', format = 'CSV')", + ], + must_not_contain=[password], + ) + + check_no_secrets() + + node.query("SET show_secrets_in_show_query=1") + check_logs(must_contain=[password]) + + node.query("SET show_secrets_in_show_query=0") + check_no_secrets() for i in range(0, len(table_engines)): node.query(f"DROP TABLE table{i}") @@ -189,14 +211,23 @@ def test_create_database(): f"CREATE DATABASE database{i} ENGINE = {database_engine}" ) - check_logs( - must_contain=[ - "CREATE DATABASE database0 ENGINE = MySQL('localhost:3306', 'mysql_db', 'mysql_user', '[HIDDEN]')", - "CREATE DATABASE database1 ENGINE = MySQL(named_collection_1, host = 'localhost', port = 3306, database = 'mysql_db', user = 'mysql_user', password = '[HIDDEN]')", - # "CREATE DATABASE database2 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')", - ], - must_not_contain=[password], - ) + def check_no_secrets(): + check_logs( + must_contain=[ + "CREATE DATABASE database0 ENGINE = MySQL('localhost:3306', 'mysql_db', 'mysql_user', '[HIDDEN]')", + "CREATE DATABASE database1 ENGINE = MySQL(named_collection_1, host = 'localhost', port = 3306, database = 'mysql_db', user = 'mysql_user', password = '[HIDDEN]')", + # "CREATE DATABASE database2 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')", + ], + must_not_contain=[password], + ) + + check_no_secrets() + + node.query("SET show_secrets_in_show_query=1") + check_logs(must_contain=[password]) + + node.query("SET show_secrets_in_show_query=0") + check_no_secrets() for i in range(0, len(database_engines)): node.query(f"DROP DATABASE IF EXISTS database{i}") diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 58b1cab6e20..2b9ed531091 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -134,6 +134,7 @@ SYSTEM THREAD FUZZER ['SYSTEM START THREAD FUZZER','SYSTEM STOP THREAD FUZZER',' SYSTEM UNFREEZE ['SYSTEM UNFREEZE'] GLOBAL SYSTEM SYSTEM [] \N ALL dictGet ['dictHas','dictGetHierarchy','dictIsIn'] DICTIONARY ALL +displaySecretsInShowSelect [] GLOBAL ALL addressToLine [] GLOBAL INTROSPECTION addressToLineWithInlines [] GLOBAL INTROSPECTION addressToSymbol [] GLOBAL INTROSPECTION diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index aabe05ea5e2..c4e0fcd8946 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -288,7 +288,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'addressToLine' = 136, 'addressToLineWithInlines' = 137, 'addressToSymbol' = 138, 'demangle' = 139, 'INTROSPECTION' = 140, 'FILE' = 141, 'URL' = 142, 'REMOTE' = 143, 'MONGO' = 144, 'MEILISEARCH' = 145, 'MYSQL' = 146, 'POSTGRES' = 147, 'SQLITE' = 148, 'ODBC' = 149, 'JDBC' = 150, 'HDFS' = 151, 'S3' = 152, 'HIVE' = 153, 'SOURCES' = 154, 'CLUSTER' = 155, 'ALL' = 156, 'NONE' = 157), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'displaySecretsInShowSelect' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -569,10 +569,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'addressToLine' = 136, 'addressToLineWithInlines' = 137, 'addressToSymbol' = 138, 'demangle' = 139, 'INTROSPECTION' = 140, 'FILE' = 141, 'URL' = 142, 'REMOTE' = 143, 'MONGO' = 144, 'MEILISEARCH' = 145, 'MYSQL' = 146, 'POSTGRES' = 147, 'SQLITE' = 148, 'ODBC' = 149, 'JDBC' = 150, 'HDFS' = 151, 'S3' = 152, 'HIVE' = 153, 'SOURCES' = 154, 'CLUSTER' = 155, 'ALL' = 156, 'NONE' = 157), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'displaySecretsInShowSelect' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'addressToLine' = 136, 'addressToLineWithInlines' = 137, 'addressToSymbol' = 138, 'demangle' = 139, 'INTROSPECTION' = 140, 'FILE' = 141, 'URL' = 142, 'REMOTE' = 143, 'MONGO' = 144, 'MEILISEARCH' = 145, 'MYSQL' = 146, 'POSTGRES' = 147, 'SQLITE' = 148, 'ODBC' = 149, 'JDBC' = 150, 'HDFS' = 151, 'S3' = 152, 'HIVE' = 153, 'SOURCES' = 154, 'CLUSTER' = 155, 'ALL' = 156, 'NONE' = 157)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'displaySecretsInShowSelect' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' diff --git a/tests/queries/0_stateless/25337_show_secrets.reference b/tests/queries/0_stateless/25337_show_secrets.reference new file mode 100644 index 00000000000..cc3567d0b0a --- /dev/null +++ b/tests/queries/0_stateless/25337_show_secrets.reference @@ -0,0 +1,17 @@ +CREATE USER u_25337 IDENTIFIED WITH sha256_hash BY \'A7EEC567280387F6DF7E13B0DB58D4F23AC3B9C2B93A8F2ECD71EB24E349F793\' SALT \'F3764AB20F28FE947C30D6CEE4C4D911A84BA6A20EF4D86E4AAA324950E543E9\' +CREATE USER u_25337 IDENTIFIED WITH sha256_password +CREATE USER u_25337 IDENTIFIED WITH sha256_password +CREATE USER u_25337 IDENTIFIED WITH sha256_password +CREATE USER u_25337 IDENTIFIED WITH sha256_hash BY \'A7EEC567280387F6DF7E13B0DB58D4F23AC3B9C2B93A8F2ECD71EB24E349F793\' SALT \'F3764AB20F28FE947C30D6CEE4C4D911A84BA6A20EF4D86E4AAA324950E543E9\' +CREATE USER u_25337 IDENTIFIED WITH sha256_password +CREATE USER u_25337 IDENTIFIED WITH sha256_password +CREATE USER u_25337 IDENTIFIED WITH sha256_password +CREATE USER u_25337 IDENTIFIED WITH sha256_hash BY \'AC2842359DAC91AD4330876D2FF9326BA4A241B07EADDCF113D7455CF34EBFD9\' SALT \'64D3F610C43CCCB1609FD27304BD94CB316B1DB1ACCFCB022D519A4074E59A07\' +CREATE USER u_25337 IDENTIFIED WITH sha256_hash BY \'A7EEC567280387F6DF7E13B0DB58D4F23AC3B9C2B93A8F2ECD71EB24E349F793\' SALT \'F3764AB20F28FE947C30D6CEE4C4D911A84BA6A20EF4D86E4AAA324950E543E9\' +CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'pass\') +CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'[HIDDEN]\') +CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'[HIDDEN]\') +CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'[HIDDEN]\') +CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'pass\') +CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'[HIDDEN]\') +CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'[HIDDEN]\') diff --git a/tests/queries/0_stateless/25337_show_secrets.sh b/tests/queries/0_stateless/25337_show_secrets.sh new file mode 100755 index 00000000000..cbc270674a5 --- /dev/null +++ b/tests/queries/0_stateless/25337_show_secrets.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2009 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +function query { $CLICKHOUSE_CLIENT --query "$1" $2; } +function user_query { $CLICKHOUSE_CLIENT --user u_25337 --pass pass --query "$1" $2; } + +pass_hash='A7EEC567280387F6DF7E13B0DB58D4F23AC3B9C2B93A8F2ECD71EB24E349F793' +pass_salt='F3764AB20F28FE947C30D6CEE4C4D911A84BA6A20EF4D86E4AAA324950E543E9' + +query "DROP USER IF EXISTS u_25337" +query "CREATE USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_hash' SALT '$pass_salt'" +query "GRANT SHOW USERS ON *.* TO u_25337" +query "GRANT SHOW COLUMNS ON *.* TO u_25337" + +function run_cases { + query "SHOW CREATE $1" --display_secrets_in_show_and_select_query=1 # setting, rights + query "SHOW CREATE $1" # no setting, rights + + user_query "SHOW CREATE $1" # no setting, no rights + user_query "SHOW CREATE $1" --display_secrets_in_show_and_select_query=1 # setting, no rights + query "GRANT displaySecretsInShowSelect ON *.* TO u_25337" + user_query "SHOW CREATE $1" --display_secrets_in_show_and_select_query=1 # setting, rights + user_query "SHOW CREATE $1" # no setting, rights + query "REVOKE displaySecretsInShowSelect ON *.* FROM u_25337" + user_query "SHOW CREATE $1" # no setting, no rights +} + +run_cases "USER u_25337" + +pass_2_hash='AC2842359DAC91AD4330876D2FF9326BA4A241B07EADDCF113D7455CF34EBFD9' +pass_2_salt='64D3F610C43CCCB1609FD27304BD94CB316B1DB1ACCFCB022D519A4074E59A07' + +query "ALTER USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_2_hash' SALT '$pass_2_salt'" +query "SHOW CREATE USER u_25337" +query "SHOW CREATE USER u_25337" --display_secrets_in_show_and_select_query=1 +query "ALTER USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_hash' SALT '$pass_salt'" +query "SHOW CREATE USER u_25337" --display_secrets_in_show_and_select_query=1 + +query "DROP TABLE IF EXISTS t_25337" +query "CREATE TABLE t_25337 (n Int32) ENGINE MySQL('mysql53:1234', 'db', 'table', 'user', 'pass')" +run_cases "TABLE t_25337" + +query "DROP TABLE t_25337" +query "DROP USER u_25337" From a6a9eaee5f5c9b278446cd86c7b559d6b53b6f03 Mon Sep 17 00:00:00 2001 From: Mike Kot Date: Mon, 27 Feb 2023 14:07:51 +0000 Subject: [PATCH 004/308] 02117 from master --- .../0_stateless/02117_show_create_table_system.reference | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index c4e0fcd8946..bfda5e7afec 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -229,7 +229,8 @@ CREATE TABLE system.distribution_queue `data_compressed_bytes` UInt64, `broken_data_files` UInt64, `broken_data_compressed_bytes` UInt64, - `last_exception` String + `last_exception` String, + `last_exception_time` DateTime ) ENGINE = SystemDistributionQueue COMMENT 'SYSTEM TABLE is built on the fly.' @@ -288,7 +289,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'displaySecretsInShowSelect' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -569,10 +570,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'displaySecretsInShowSelect' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'displaySecretsInShowSelect' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' From 00dfb27eb968638afc3b2654052b4cd570a2d27c Mon Sep 17 00:00:00 2001 From: Mike Kot Date: Tue, 28 Feb 2023 14:09:24 +0300 Subject: [PATCH 005/308] rm secrets test from fasttest, correct 02117 --- .../operations/settings/settings-formats.md | 3 +- .../test_mask_sensitive_info/test.py | 220 +++++++++--------- .../02117_show_create_table_system.reference | 6 +- .../0_stateless/25337_show_secrets.reference | 1 + .../queries/0_stateless/25337_show_secrets.sh | 48 ++-- 5 files changed, 143 insertions(+), 135 deletions(-) diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 380d7032558..e641d92bb69 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -9,7 +9,8 @@ toc_max_heading_level: 2 ## display_secrets_in_show_and_select_query {#display_secrets_in_show_and_select_query} -Enables or disables showing secrets in `SHOW` and `SELECT` queries. +Enables or disables showing secrets in `SHOW` and `SELECT` queries for tables, databases, +table functions, and dictionaries. User wishing to see secrets must also have [`displaySecretsInShowSelect`](../../sql-reference/statements/grant#grant-display-secrets) privilege. diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index fce2d539320..6b17cd92a54 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -65,13 +65,15 @@ def system_query_log_contains_search_pattern(search_pattern): ) -# Generates a random string. def new_password(len=16): return "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(len) ) +show_secrets = "SETTINGS display_secrets_in_show_and_select_query" + + def test_create_alter_user(): password = new_password() @@ -83,46 +85,48 @@ def test_create_alter_user(): f"CREATE USER u2 IDENTIFIED WITH plaintext_password BY '{password}' SETTINGS custom_c = 'c'" ) - def check_no_secrets(): - assert ( - node.query("SHOW CREATE USER u1") - == "CREATE USER u1 IDENTIFIED WITH sha256_password SETTINGS custom_b = \\'b\\'\n" - ) - assert ( - node.query("SHOW CREATE USER u2") - == "CREATE USER u2 IDENTIFIED WITH plaintext_password SETTINGS custom_c = \\'c\\'\n" - ) + assert ( + node.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH sha256_password SETTINGS custom_b = \\'b\\'\n" + ) + assert ( + node.query("SHOW CREATE USER u2") + == "CREATE USER u2 IDENTIFIED WITH plaintext_password SETTINGS custom_c = \\'c\\'\n" + ) - non_secrets = [ - "CREATE USER u1 IDENTIFIED WITH sha256_password", - "ALTER USER u1 IDENTIFIED WITH sha256_password", - "CREATE USER u2 IDENTIFIED WITH plaintext_password", - ] + check_logs( + must_contain=[ + "CREATE USER u1 IDENTIFIED WITH sha256_password", + "ALTER USER u1 IDENTIFIED WITH sha256_password", + "CREATE USER u2 IDENTIFIED WITH plaintext_password", + ], + must_not_contain=[ + password, + "IDENTIFIED WITH sha256_password BY", + "IDENTIFIED WITH sha256_hash BY", + "IDENTIFIED WITH plaintext_password BY", + ], + ) - secrets = [ - password, - "IDENTIFIED WITH sha256_password BY", - "IDENTIFIED WITH sha256_hash BY", - "IDENTIFIED WITH plaintext_password BY", - ] - - check_no_secrets() - check_logs(must_contain=non_secrets, must_not_contain=secrets) - - node.query("SET show_secrets_in_show_query=1") - - assert node.query("SHOW CREATE USER u1").contains("BY") - assert node.query("SHOW CREATE USER u2").contains("BY") - check_logs(must_contain=secrets + non_secrets) - - node.query("SET show_secrets_in_show_query=0") - - check_no_secrets() - check_logs(must_contain=non_secrets, must_not_contain=secrets) + assert "BY" in node.query(f"SHOW CREATE USER u1 {show_secrets}=1") + assert "BY" in node.query(f"SHOW CREATE USER u2 {show_secrets}=1") node.query("DROP USER u1, u2") +def check_secrets_for_tables(tables, table_name_prefix, password): + for i, table in enumerate(tables): + table_name = table_name_prefix + str(i) + if password in table: + assert password in node.query( + f"SHOW CREATE TABLE {table_name} {show_secrets}=1" + ) + assert password in node.query( + f"SELECT create_table_query, engine_full FROM system.tables WHERE name = '{table_name}' " + f"{show_secrets}=1" + ) + + def test_create_table(): password = new_password() @@ -146,52 +150,49 @@ def test_create_table(): for i, table_engine in enumerate(table_engines): node.query(f"CREATE TABLE table{i} (x int) ENGINE = {table_engine}") - def check_no_secrets(): + for toggle, secret in enumerate(["[HIDDEN]", password]): assert ( - node.query("SHOW CREATE TABLE table0") - == "CREATE TABLE default.table0\\n(\\n `x` Int32\\n)\\nENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')\n" + node.query(f"SHOW CREATE TABLE table0 {show_secrets}={toggle}") + == "CREATE TABLE default.table0\\n(\\n `x` Int32\\n)\\n" + "ENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', " + f"\\'mysql_table\\', \\'mysql_user\\', \\'{secret}\\')\n" ) assert node.query( - "SELECT create_table_query, engine_full FROM system.tables WHERE name = 'table0'" + f"SELECT create_table_query, engine_full FROM system.tables WHERE name = 'table0' {show_secrets}={toggle}" ) == TSV( [ [ - "CREATE TABLE default.table0 (`x` Int32) ENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", - "MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", + "CREATE TABLE default.table0 (`x` Int32) ENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', " + f"\\'mysql_table\\', \\'mysql_user\\', \\'{secret}\\')", + f"MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'{secret}\\')", ], ] ) - check_logs( - must_contain=[ - "CREATE TABLE table0 (`x` int) ENGINE = MySQL('mysql57:3306', 'mysql_db', 'mysql_table', 'mysql_user', '[HIDDEN]')", - "CREATE TABLE table1 (`x` int) ENGINE = PostgreSQL('postgres1:5432', 'postgres_db', 'postgres_table', 'postgres_user', '[HIDDEN]')", - "CREATE TABLE table2 (`x` int) ENGINE = MongoDB('mongo1:27017', 'mongo_db', 'mongo_col', 'mongo_user', '[HIDDEN]')", - "CREATE TABLE table3 (x int) ENGINE = S3('http://minio1:9001/root/data/test1.csv')", - "CREATE TABLE table4 (x int) ENGINE = S3('http://minio1:9001/root/data/test2.csv', 'CSV')", - "CREATE TABLE table5 (x int) ENGINE = S3('http://minio1:9001/root/data/test3.csv.gz', 'CSV', 'gzip')", - "CREATE TABLE table6 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test4.csv', 'minio', '[HIDDEN]', 'CSV')", - "CREATE TABLE table7 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'minio', '[HIDDEN]', 'CSV', 'gzip')", - "CREATE TABLE table8 (`x` int) ENGINE = MySQL(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')", - "CREATE TABLE table9 (`x` int) ENGINE = MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '[HIDDEN]', table = 'mysql_table', user = 'mysql_user')", - "CREATE TABLE table10 (x int) ENGINE = MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')", - "CREATE TABLE table11 (`x` int) ENGINE = PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '[HIDDEN]')", - "CREATE TABLE table12 (`x` int) ENGINE = MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '[HIDDEN]'", - "CREATE TABLE table13 (`x` int) ENGINE = S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]', format = 'CSV')", - ], - must_not_contain=[password], - ) + check_logs( + must_contain=[ + "CREATE TABLE table0 (`x` int) ENGINE = MySQL('mysql57:3306', 'mysql_db', 'mysql_table', 'mysql_user', '[HIDDEN]')", + "CREATE TABLE table1 (`x` int) ENGINE = PostgreSQL('postgres1:5432', 'postgres_db', 'postgres_table', 'postgres_user', '[HIDDEN]')", + "CREATE TABLE table2 (`x` int) ENGINE = MongoDB('mongo1:27017', 'mongo_db', 'mongo_col', 'mongo_user', '[HIDDEN]')", + "CREATE TABLE table3 (x int) ENGINE = S3('http://minio1:9001/root/data/test1.csv')", + "CREATE TABLE table4 (x int) ENGINE = S3('http://minio1:9001/root/data/test2.csv', 'CSV')", + "CREATE TABLE table5 (x int) ENGINE = S3('http://minio1:9001/root/data/test3.csv.gz', 'CSV', 'gzip')", + "CREATE TABLE table6 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test4.csv', 'minio', '[HIDDEN]', 'CSV')", + "CREATE TABLE table7 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'minio', '[HIDDEN]', 'CSV', 'gzip')", + "CREATE TABLE table8 (`x` int) ENGINE = MySQL(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')", + "CREATE TABLE table9 (`x` int) ENGINE = MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '[HIDDEN]', table = 'mysql_table', user = 'mysql_user')", + "CREATE TABLE table10 (x int) ENGINE = MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')", + "CREATE TABLE table11 (`x` int) ENGINE = PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '[HIDDEN]')", + "CREATE TABLE table12 (`x` int) ENGINE = MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '[HIDDEN]'", + "CREATE TABLE table13 (`x` int) ENGINE = S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]', format = 'CSV')", + ], + must_not_contain=[password], + ) - check_no_secrets() + check_secrets_for_tables(table_engines, "table", password) - node.query("SET show_secrets_in_show_query=1") - check_logs(must_contain=[password]) - - node.query("SET show_secrets_in_show_query=0") - check_no_secrets() - - for i in range(0, len(table_engines)): + for i in range(len(table_engines)): node.query(f"DROP TABLE table{i}") @@ -211,25 +212,16 @@ def test_create_database(): f"CREATE DATABASE database{i} ENGINE = {database_engine}" ) - def check_no_secrets(): - check_logs( - must_contain=[ - "CREATE DATABASE database0 ENGINE = MySQL('localhost:3306', 'mysql_db', 'mysql_user', '[HIDDEN]')", - "CREATE DATABASE database1 ENGINE = MySQL(named_collection_1, host = 'localhost', port = 3306, database = 'mysql_db', user = 'mysql_user', password = '[HIDDEN]')", - # "CREATE DATABASE database2 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')", - ], - must_not_contain=[password], - ) + check_logs( + must_contain=[ + "CREATE DATABASE database0 ENGINE = MySQL('localhost:3306', 'mysql_db', 'mysql_user', '[HIDDEN]')", + "CREATE DATABASE database1 ENGINE = MySQL(named_collection_1, host = 'localhost', port = 3306, database = 'mysql_db', user = 'mysql_user', password = '[HIDDEN]')", + # "CREATE DATABASE database2 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')", + ], + must_not_contain=[password], + ) - check_no_secrets() - - node.query("SET show_secrets_in_show_query=1") - check_logs(must_contain=[password]) - - node.query("SET show_secrets_in_show_query=0") - check_no_secrets() - - for i in range(0, len(database_engines)): + for i in range(len(database_engines)): node.query(f"DROP DATABASE IF EXISTS database{i}") @@ -272,21 +264,26 @@ def test_table_functions(): for i, table_function in enumerate(table_functions): node.query(f"CREATE TABLE tablefunc{i} (x int) AS {table_function}") - assert ( - node.query("SHOW CREATE TABLE tablefunc0") - == "CREATE TABLE default.tablefunc0\\n(\\n `x` Int32\\n) AS mysql(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')\n" - ) + for toggle, secret in enumerate(["[HIDDEN]", password]): + assert ( + node.query(f"SHOW CREATE TABLE tablefunc0 {show_secrets}={toggle}") + == "CREATE TABLE default.tablefunc0\\n(\\n `x` Int32\\n) AS " + "mysql(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', " + f"\\'mysql_user\\', \\'{secret}\\')\n" + ) - assert node.query( - "SELECT create_table_query, engine_full FROM system.tables WHERE name = 'tablefunc0'" - ) == TSV( - [ + assert node.query( + "SELECT create_table_query, engine_full FROM system.tables WHERE name = 'tablefunc0' " + f"{show_secrets}={toggle}" + ) == TSV( [ - "CREATE TABLE default.tablefunc0 (`x` Int32) AS mysql(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", - "", - ], - ] - ) + [ + "CREATE TABLE default.tablefunc0 (`x` Int32) AS mysql(\\'mysql57:3306\\', " + f"\\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'{secret}\\')", + "", + ], + ] + ) check_logs( must_contain=[ @@ -324,7 +321,9 @@ def test_table_functions(): must_not_contain=[password], ) - for i in range(0, len(table_functions)): + check_secrets_for_tables(table_functions, "tablefunc", password) + + for i in range(len(table_functions)): node.query(f"DROP TABLE tablefunc{i}") @@ -400,15 +399,18 @@ def test_create_dictionary(): f"LIFETIME(MIN 0 MAX 10) LAYOUT(FLAT())" ) - assert ( - node.query("SHOW CREATE TABLE dict1") - == "CREATE DICTIONARY default.dict1\\n(\\n `n` int DEFAULT 0,\\n `m` int DEFAULT 1\\n)\\nPRIMARY KEY n\\nSOURCE(CLICKHOUSE(HOST \\'localhost\\' PORT 9000 USER \\'user1\\' TABLE \\'test\\' PASSWORD \\'[HIDDEN]\\' DB \\'default\\'))\\nLIFETIME(MIN 0 MAX 10)\\nLAYOUT(FLAT())\n" - ) + for toggle, secret in enumerate(["[HIDDEN]", password]): + assert ( + node.query(f"SHOW CREATE TABLE dict1 {show_secrets}={toggle}") + == f"CREATE DICTIONARY default.dict1\\n(\\n `n` int DEFAULT 0,\\n `m` int DEFAULT 1\\n)\\nPRIMARY KEY n\\nSOURCE(CLICKHOUSE(HOST \\'localhost\\' PORT 9000 USER \\'user1\\' TABLE \\'test\\' PASSWORD \\'{secret}\\' DB \\'default\\'))\\nLIFETIME(MIN 0 MAX 10)\\nLAYOUT(FLAT())\n" + ) - assert ( - node.query("SELECT create_table_query FROM system.tables WHERE name = 'dict1'") - == "CREATE DICTIONARY default.dict1 (`n` int DEFAULT 0, `m` int DEFAULT 1) PRIMARY KEY n SOURCE(CLICKHOUSE(HOST \\'localhost\\' PORT 9000 USER \\'user1\\' TABLE \\'test\\' PASSWORD \\'[HIDDEN]\\' DB \\'default\\')) LIFETIME(MIN 0 MAX 10) LAYOUT(FLAT())\n" - ) + assert ( + node.query( + f"SELECT create_table_query FROM system.tables WHERE name = 'dict1' {show_secrets}={toggle}" + ) + == f"CREATE DICTIONARY default.dict1 (`n` int DEFAULT 0, `m` int DEFAULT 1) PRIMARY KEY n SOURCE(CLICKHOUSE(HOST \\'localhost\\' PORT 9000 USER \\'user1\\' TABLE \\'test\\' PASSWORD \\'{secret}\\' DB \\'default\\')) LIFETIME(MIN 0 MAX 10) LAYOUT(FLAT())\n" + ) check_logs( must_contain=[ @@ -479,4 +481,4 @@ def test_on_cluster(): "%CREATE TABLE default.table_oncl UUID \\'%\\' (`x` Int32) ENGINE = MySQL(\\'mysql57:3307\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')" ) - node.query(f"DROP TABLE table_oncl") + node.query("DROP TABLE table_oncl") diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index bfda5e7afec..6e548c7425a 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -289,7 +289,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'displaySecretsInShowSelect' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -570,10 +570,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'displaySecretsInShowSelect' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'displaySecretsInShowSelect' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' diff --git a/tests/queries/0_stateless/25337_show_secrets.reference b/tests/queries/0_stateless/25337_show_secrets.reference index cc3567d0b0a..ead0cb48901 100644 --- a/tests/queries/0_stateless/25337_show_secrets.reference +++ b/tests/queries/0_stateless/25337_show_secrets.reference @@ -15,3 +15,4 @@ CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234 CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'pass\') CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'[HIDDEN]\') CREATE TABLE default.t_25337\n(\n `n` Int32\n)\nENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'[HIDDEN]\') +CREATE TABLE default.t_25337 (`n` Int32) ENGINE = MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'pass\') MySQL(\'mysql53:1234\', \'db\', \'table\', \'user\', \'pass\') diff --git a/tests/queries/0_stateless/25337_show_secrets.sh b/tests/queries/0_stateless/25337_show_secrets.sh index cbc270674a5..09f6aebe24e 100755 --- a/tests/queries/0_stateless/25337_show_secrets.sh +++ b/tests/queries/0_stateless/25337_show_secrets.sh @@ -1,30 +1,30 @@ #!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, use-mysql +# Tag no-parallel: default/u_25337 queries may interfere, this is a purely sequential test # shellcheck disable=SC2009 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -function query { $CLICKHOUSE_CLIENT --query "$1" $2; } -function user_query { $CLICKHOUSE_CLIENT --user u_25337 --pass pass --query "$1" $2; } +function query { $CLICKHOUSE_CLIENT -n --query="$1"; } +function user_query { $CLICKHOUSE_CLIENT -n --user u_25337 --pass pass --query="$1"; } pass_hash='A7EEC567280387F6DF7E13B0DB58D4F23AC3B9C2B93A8F2ECD71EB24E349F793' pass_salt='F3764AB20F28FE947C30D6CEE4C4D911A84BA6A20EF4D86E4AAA324950E543E9' +show_secrets="SETTINGS display_secrets_in_show_and_select_query=1" -query "DROP USER IF EXISTS u_25337" -query "CREATE USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_hash' SALT '$pass_salt'" -query "GRANT SHOW USERS ON *.* TO u_25337" -query "GRANT SHOW COLUMNS ON *.* TO u_25337" +query " + DROP USER IF EXISTS u_25337; + CREATE USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_hash' SALT '$pass_salt'; + GRANT SHOW USERS ON *.* TO u_25337; + GRANT SHOW COLUMNS ON *.* TO u_25337" function run_cases { - query "SHOW CREATE $1" --display_secrets_in_show_and_select_query=1 # setting, rights - query "SHOW CREATE $1" # no setting, rights - - user_query "SHOW CREATE $1" # no setting, no rights - user_query "SHOW CREATE $1" --display_secrets_in_show_and_select_query=1 # setting, no rights + query "SHOW CREATE $1 $show_secrets; SHOW CREATE $1" + user_query "SHOW CREATE $1; SHOW CREATE $1 $show_secrets" query "GRANT displaySecretsInShowSelect ON *.* TO u_25337" - user_query "SHOW CREATE $1" --display_secrets_in_show_and_select_query=1 # setting, rights - user_query "SHOW CREATE $1" # no setting, rights + user_query "SHOW CREATE $1 $show_secrets; SHOW CREATE $1" query "REVOKE displaySecretsInShowSelect ON *.* FROM u_25337" user_query "SHOW CREATE $1" # no setting, no rights } @@ -34,15 +34,19 @@ run_cases "USER u_25337" pass_2_hash='AC2842359DAC91AD4330876D2FF9326BA4A241B07EADDCF113D7455CF34EBFD9' pass_2_salt='64D3F610C43CCCB1609FD27304BD94CB316B1DB1ACCFCB022D519A4074E59A07' -query "ALTER USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_2_hash' SALT '$pass_2_salt'" -query "SHOW CREATE USER u_25337" -query "SHOW CREATE USER u_25337" --display_secrets_in_show_and_select_query=1 -query "ALTER USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_hash' SALT '$pass_salt'" -query "SHOW CREATE USER u_25337" --display_secrets_in_show_and_select_query=1 +query " + ALTER USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_2_hash' SALT '$pass_2_salt'; + SHOW CREATE USER u_25337; + SHOW CREATE USER u_25337 $show_secrets; + ALTER USER u_25337 IDENTIFIED WITH sha256_hash BY '$pass_hash' SALT '$pass_salt'; + SHOW CREATE USER u_25337 $show_secrets; + + DROP TABLE IF EXISTS t_25337; + CREATE TABLE t_25337 (n Int32) ENGINE MySQL('mysql53:1234', 'db', 'table', 'user', 'pass')" -query "DROP TABLE IF EXISTS t_25337" -query "CREATE TABLE t_25337 (n Int32) ENGINE MySQL('mysql53:1234', 'db', 'table', 'user', 'pass')" run_cases "TABLE t_25337" -query "DROP TABLE t_25337" -query "DROP USER u_25337" +query " + SELECT create_table_query, engine_full FROM system.tables WHERE name='t_25337' $show_secrets; + DROP TABLE t_25337; + DROP USER u_25337" From 511ed93512b4ef376c7d341f5871ed5913a6881d Mon Sep 17 00:00:00 2001 From: Mike Kot Date: Wed, 15 Mar 2023 16:38:59 +0300 Subject: [PATCH 006/308] Update src/Storages/System/StorageSystemDatabases.cpp Co-authored-by: alesapin --- src/Storages/System/StorageSystemDatabases.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/System/StorageSystemDatabases.cpp b/src/Storages/System/StorageSystemDatabases.cpp index 52243e383e4..a3d05281b28 100644 --- a/src/Storages/System/StorageSystemDatabases.cpp +++ b/src/Storages/System/StorageSystemDatabases.cpp @@ -32,7 +32,7 @@ NamesAndAliases StorageSystemDatabases::getNamesAndAliases() }; } -static String getEngineFull(const ContextPtr& ctx, const DatabasePtr & database) +static String getEngineFull(const ContextPtr & ctx, const DatabasePtr & database) { DDLGuardPtr guard; while (true) From be40218a9a74e02e2890f17def66d32e39efd58f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 15 Mar 2023 22:14:54 +0100 Subject: [PATCH 007/308] Revert "Revert #46622 (test_async_insert_memory)" --- src/Common/CurrentThread.cpp | 19 +++++++++ src/Common/CurrentThread.h | 6 +++ src/Interpreters/AsynchronousInsertQueue.cpp | 6 ++- src/Interpreters/AsynchronousInsertQueue.h | 42 ++++++++++++++++++- .../test_async_insert_memory/__init__.py | 0 .../test_async_insert_memory/test.py | 40 ++++++++++++++++++ 6 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_async_insert_memory/__init__.py create mode 100644 tests/integration/test_async_insert_memory/test.py diff --git a/src/Common/CurrentThread.cpp b/src/Common/CurrentThread.cpp index 188e78fe69b..b54cf3b9371 100644 --- a/src/Common/CurrentThread.cpp +++ b/src/Common/CurrentThread.cpp @@ -110,4 +110,23 @@ ThreadGroupStatusPtr CurrentThread::getGroup() return current_thread->getThreadGroup(); } +MemoryTracker * CurrentThread::getUserMemoryTracker() +{ + if (unlikely(!current_thread)) + return nullptr; + + auto * tracker = current_thread->memory_tracker.getParent(); + while (tracker && tracker->level != VariableContext::User) + tracker = tracker->getParent(); + + return tracker; +} + +void CurrentThread::flushUntrackedMemory() +{ + if (unlikely(!current_thread)) + return; + current_thread->flushUntrackedMemory(); +} + } diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index f4975e800ca..ffc00c77504 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -40,6 +40,12 @@ public: /// Group to which belongs current thread static ThreadGroupStatusPtr getGroup(); + /// MemoryTracker for user that owns current thread if any + static MemoryTracker * getUserMemoryTracker(); + + /// Adjust counters in MemoryTracker hierarchy if untracked_memory is not 0. + static void flushUntrackedMemory(); + /// A logs queue used by TCPHandler to pass logs to a client static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue, LogsLevel client_logs_level); diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 590cbc9ba83..78b173de6dc 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -103,9 +104,10 @@ bool AsynchronousInsertQueue::InsertQuery::operator==(const InsertQuery & other) return query_str == other.query_str && settings == other.settings; } -AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && query_id_) +AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && query_id_, MemoryTracker * user_memory_tracker_) : bytes(std::move(bytes_)) , query_id(std::move(query_id_)) + , user_memory_tracker(user_memory_tracker_) , create_time(std::chrono::system_clock::now()) { } @@ -234,7 +236,7 @@ AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) if (auto quota = query_context->getQuota()) quota->used(QuotaType::WRITTEN_BYTES, bytes.size()); - auto entry = std::make_shared(std::move(bytes), query_context->getCurrentQueryId()); + auto entry = std::make_shared(std::move(bytes), query_context->getCurrentQueryId(), CurrentThread::getUserMemoryTracker()); InsertQuery key{query, settings}; InsertDataPtr data_to_process; diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 23a2860364d..e6b7bff8d26 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -59,6 +60,31 @@ private: UInt128 calculateHash() const; }; + struct UserMemoryTrackerSwitcher + { + explicit UserMemoryTrackerSwitcher(MemoryTracker * new_tracker) + { + auto * thread_tracker = CurrentThread::getMemoryTracker(); + prev_untracked_memory = current_thread->untracked_memory; + prev_memory_tracker_parent = thread_tracker->getParent(); + + current_thread->untracked_memory = 0; + thread_tracker->setParent(new_tracker); + } + + ~UserMemoryTrackerSwitcher() + { + CurrentThread::flushUntrackedMemory(); + auto * thread_tracker = CurrentThread::getMemoryTracker(); + + current_thread->untracked_memory = prev_untracked_memory; + thread_tracker->setParent(prev_memory_tracker_parent); + } + + MemoryTracker * prev_memory_tracker_parent; + Int64 prev_untracked_memory; + }; + struct InsertData { struct Entry @@ -66,9 +92,10 @@ private: public: const String bytes; const String query_id; + MemoryTracker * const user_memory_tracker; const std::chrono::time_point create_time; - Entry(String && bytes_, String && query_id_); + Entry(String && bytes_, String && query_id_, MemoryTracker * user_memory_tracker_); void finish(std::exception_ptr exception_ = nullptr); std::future getFuture() { return promise.get_future(); } @@ -79,6 +106,19 @@ private: std::atomic_bool finished = false; }; + ~InsertData() + { + auto it = entries.begin(); + // Entries must be destroyed in context of user who runs async insert. + // Each entry in the list may correspond to a different user, + // so we need to switch current thread's MemoryTracker parent on each iteration. + while (it != entries.end()) + { + UserMemoryTrackerSwitcher switcher((*it)->user_memory_tracker); + it = entries.erase(it); + } + } + using EntryPtr = std::shared_ptr; std::list entries; diff --git a/tests/integration/test_async_insert_memory/__init__.py b/tests/integration/test_async_insert_memory/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_async_insert_memory/test.py b/tests/integration/test_async_insert_memory/test.py new file mode 100644 index 00000000000..279542f087c --- /dev/null +++ b/tests/integration/test_async_insert_memory/test.py @@ -0,0 +1,40 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance("node") + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_memory_usage(): + node.query( + "CREATE TABLE async_table(data Array(UInt64)) ENGINE=MergeTree() ORDER BY data" + ) + + node.get_query_request("SELECT count() FROM system.numbers") + + INSERT_QUERY = "INSERT INTO async_table SETTINGS async_insert=1, wait_for_async_insert=1 VALUES ({})" + for iter in range(10): + values = list(range(iter * 5000000, (iter + 1) * 5000000)) + node.query(INSERT_QUERY.format(values)) + + response = node.get_query_request( + "SELECT groupArray(number) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format( + 30 * (2**23) + ) + ) + + _, err = response.get_answer_and_error() + assert err == "", "Query failed with error {}".format(err) + + node.query("DROP TABLE async_table") From bf2670ddaf854b0e36e346b904d89d505a048393 Mon Sep 17 00:00:00 2001 From: Mike Kot Date: Thu, 16 Mar 2023 10:39:13 +0000 Subject: [PATCH 008/308] recalc rights after changing setting --- src/Interpreters/Context.cpp | 31 ++++++++++++++++++------------- src/Interpreters/Context.h | 1 + 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index d1b09707bca..d77c06c7544 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1479,20 +1479,31 @@ Settings Context::getSettings() const return settings; } - void Context::setSettings(const Settings & settings_) { auto lock = getLock(); - auto old_readonly = settings.readonly; - auto old_allow_ddl = settings.allow_ddl; - auto old_allow_introspection_functions = settings.allow_introspection_functions; + const auto old_readonly = settings.readonly; + const auto old_allow_ddl = settings.allow_ddl; + const auto old_allow_introspection_functions = settings.allow_introspection_functions; + const auto old_display_secrets = settings.display_secrets_in_show_and_select_query; settings = settings_; - if ((settings.readonly != old_readonly) || (settings.allow_ddl != old_allow_ddl) || (settings.allow_introspection_functions != old_allow_introspection_functions)) + if ((settings.readonly != old_readonly) + || (settings.allow_ddl != old_allow_ddl) + || (settings.allow_introspection_functions != old_allow_introspection_functions) + || (settings.display_secrets_in_show_and_select_query != old_display_secrets)) calculateAccessRights(); } +void Context::recalcRightsIfNeeded(std::string_view name) +{ + if (name == "readonly" + || name == "allow_ddl" + || name == "allow_introspection_functions" + || name == "display_secrets_in_show_and_select_query") + calculateAccessRights(); +} void Context::setSetting(std::string_view name, const String & value) { @@ -1503,12 +1514,9 @@ void Context::setSetting(std::string_view name, const String & value) return; } settings.set(name, value); - - if (name == "readonly" || name == "allow_ddl" || name == "allow_introspection_functions") - calculateAccessRights(); + recalcRightsIfNeeded(name); } - void Context::setSetting(std::string_view name, const Field & value) { auto lock = getLock(); @@ -1518,12 +1526,9 @@ void Context::setSetting(std::string_view name, const Field & value) return; } settings.set(name, value); - - if (name == "readonly" || name == "allow_ddl" || name == "allow_introspection_functions") - calculateAccessRights(); + recalcRightsIfNeeded(name); } - void Context::applySettingChange(const SettingChange & change) { try diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 19bb6868331..f3b9fe88e7e 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1130,6 +1130,7 @@ private: /// Compute and set actual user settings, client_info.current_user should be set void calculateAccessRights(); + void recalcRightsIfNeeded(std::string_view setting_name); template void checkAccessImpl(const Args &... args) const; From 0f039f83590339ab636e9b2e59c25ed22227d246 Mon Sep 17 00:00:00 2001 From: Kuba Kaflik Date: Tue, 21 Mar 2023 09:09:57 +0100 Subject: [PATCH 009/308] Add Google Cloud Storage S3 compatible table function --- src/TableFunctions/TableFunctionS3.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 859da9e9201..a2e93476448 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -93,4 +93,18 @@ private: } +class TableFunctionGCS : public TableFunctionS3 +{ +public: + static constexpr auto name = "gcs"; + std::string getName() const override + { + return name; + } +private: + const char * getStorageTypeName() const override { return "GCS"; } +}; + +} + #endif From e6ddfc3486985393040222aec24ea70a4c60e7b8 Mon Sep 17 00:00:00 2001 From: Kuba Kaflik Date: Tue, 21 Mar 2023 09:51:37 +0100 Subject: [PATCH 010/308] Update GCS table function docs --- docs/en/sql-reference/table-functions/gcs.md | 184 +++++++++++++++++++ src/TableFunctions/TableFunctionS3.h | 8 + 2 files changed, 192 insertions(+) create mode 100644 docs/en/sql-reference/table-functions/gcs.md diff --git a/docs/en/sql-reference/table-functions/gcs.md b/docs/en/sql-reference/table-functions/gcs.md new file mode 100644 index 00000000000..8427a2db224 --- /dev/null +++ b/docs/en/sql-reference/table-functions/gcs.md @@ -0,0 +1,184 @@ +--- +slug: /en/sql-reference/table-functions/gcs +sidebar_position: 45 +sidebar_label: s3 +keywords: [gcs, bucket] +--- + +# gcs Table Function + +Provides a table-like interface to select/insert files in [Google Cloud Storage](https://cloud.google.com/storage/). + +**Syntax** + +``` sql +gcs(path [,hmac_key, hmac_secret] [,format] [,structure] [,compression]) +``` + +:::tip GCS +The GCS Table Function integrates with Google Cloud Storage by using the GCS XML API and HMAC keys. See the [Google interoperability docs]( https://cloud.google.com/storage/docs/interoperability) for more details about the endpoint and HMAC. + +::: + +**Arguments** + +- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [here](../../engines/table-engines/integrations/gcs.md#wildcards-in-path). + + :::note GCS + The GCS path is in this format as the endpoint for the Google XML API is different than the JSON API: + ``` + https://storage.googleapis.com/// + ``` + and not ~~https://storage.cloud.google.com~~. + ::: + +- `format` — The [format](../../interfaces/formats.md#formats) of the file. +- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. +- `compression` — Parameter is optional. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression by file extension. + +**Returned value** + +A table with the specified structure for reading or writing data in the specified file. + +**Examples** + +Selecting the first two rows from the table from S3 file `https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv`: + +``` sql +SELECT * +FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') +LIMIT 2; +``` + +``` text +┌─column1─┬─column2─┬─column3─┐ +│ 1 │ 2 │ 3 │ +│ 3 │ 2 │ 1 │ +└─────────┴─────────┴─────────┘ +``` + +The similar but from file with `gzip` compression: + +``` sql +SELECT * +FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip') +LIMIT 2; +``` + +``` text +┌─column1─┬─column2─┬─column3─┐ +│ 1 │ 2 │ 3 │ +│ 3 │ 2 │ 1 │ +└─────────┴─────────┴─────────┘ +``` + +## Usage + +Suppose that we have several files with following URIs on S3: + +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_1.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_2.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_3.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_4.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_1.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_2.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_3.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_4.csv' + +Count the amount of rows in files ending with numbers from 1 to 3: + +``` sql +SELECT count(*) +FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32') +``` + +``` text +┌─count()─┐ +│ 18 │ +└─────────┘ +``` + +Count the total amount of rows in all files in these two directories: + +``` sql +SELECT count(*) +FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32') +``` + +``` text +┌─count()─┐ +│ 24 │ +└─────────┘ +``` + +:::warning +If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. +::: + +Count the total amount of rows in files named `file-000.csv`, `file-001.csv`, … , `file-999.csv`: + +``` sql +SELECT count(*) +FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32'); +``` + +``` text +┌─count()─┐ +│ 12 │ +└─────────┘ +``` + +Insert data into file `test-data.csv.gz`: + +``` sql +INSERT INTO FUNCTION gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +VALUES ('test-data', 1), ('test-data-2', 2); +``` + +Insert data into file `test-data.csv.gz` from existing table: + +``` sql +INSERT INTO FUNCTION gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +SELECT name, value FROM existing_table; +``` + +Glob ** can be used for recursive directory traversal. Consider the below example, it will fetch all files from `my-test-bucket-768` directory recursively: + +``` sql +SELECT * FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**', 'CSV', 'name String, value UInt32', 'gzip'); +``` + +The below get data from all `test-data.csv.gz` files from any folder inside `my-test-bucket` directory recursively: + +``` sql +SELECT * FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip'); +``` + +## Partitioned Write + +If you specify `PARTITION BY` expression when inserting data into `S3` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency. + +**Examples** + +1. Using partition ID in a key creates separate files: + +```sql +INSERT INTO TABLE FUNCTION + gcs('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a String, b UInt32, c UInt32') + PARTITION BY a VALUES ('x', 2, 3), ('x', 4, 5), ('y', 11, 12), ('y', 13, 14), ('z', 21, 22), ('z', 23, 24); +``` +As a result, the data is written into three files: `file_x.csv`, `file_y.csv`, and `file_z.csv`. + +2. Using partition ID in a bucket name creates files in different buckets: + +```sql +INSERT INTO TABLE FUNCTION + gcs('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + PARTITION BY a VALUES (1, 2, 3), (1, 4, 5), (10, 11, 12), (10, 13, 14), (20, 21, 22), (20, 23, 24); +``` +As a result, the data is written into three files in different buckets: `my_bucket_1/file.csv`, `my_bucket_10/file.csv`, and `my_bucket_20/file.csv`. + +**See Also** + +- [S3 table function](s3.md) +- [S3 engine](../../engines/table-engines/integrations/s3.md) diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index a2e93476448..ed8cd3bd41a 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -97,6 +97,14 @@ class TableFunctionGCS : public TableFunctionS3 { public: static constexpr auto name = "gcs"; + static constexpr auto signature = " - url\n" + " - url, format\n" + " - url, format, structure\n" + " - url, hmac_key, hmac_secret\n" + " - url, format, structure, compression_method\n" + " - url, hmac_key, hmac_secret, format\n" + " - url, hmac_key, hmac_secret, format, structure\n" + " - url, hmac_key, hmac_secret, format, structure, compression_method"; std::string getName() const override { return name; From e2c32c3bc072e2290620d975ada42c37bcabcc52 Mon Sep 17 00:00:00 2001 From: Kuba Kaflik Date: Tue, 21 Mar 2023 13:46:37 +0100 Subject: [PATCH 011/308] Update GCS table function docs --- docs/en/sql-reference/table-functions/gcs.md | 40 ++++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/en/sql-reference/table-functions/gcs.md b/docs/en/sql-reference/table-functions/gcs.md index 8427a2db224..dcf49a5108b 100644 --- a/docs/en/sql-reference/table-functions/gcs.md +++ b/docs/en/sql-reference/table-functions/gcs.md @@ -42,11 +42,11 @@ A table with the specified structure for reading or writing data in the specifie **Examples** -Selecting the first two rows from the table from S3 file `https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv`: +Selecting the first two rows from the table from GCS file `https://storage.googleapis.com/my-test-bucket-768/data.csv`: ``` sql SELECT * -FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') +FROM gcs('https://storage.googleapis.com/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') LIMIT 2; ``` @@ -61,7 +61,7 @@ The similar but from file with `gzip` compression: ``` sql SELECT * -FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip') +FROM gcs('https://storage.googleapis.com/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip') LIMIT 2; ``` @@ -74,22 +74,22 @@ LIMIT 2; ## Usage -Suppose that we have several files with following URIs on S3: +Suppose that we have several files with following URIs on GCS: -- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_1.csv' -- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_2.csv' -- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_3.csv' -- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_4.csv' -- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_1.csv' -- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_2.csv' -- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_3.csv' -- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_4.csv' +- 'https://storage.googleapis.com/my-test-bucket-768/some_prefix/some_file_1.csv' +- 'https://storage.googleapis.com/my-test-bucket-768/some_prefix/some_file_2.csv' +- 'https://storage.googleapis.com/my-test-bucket-768/some_prefix/some_file_3.csv' +- 'https://storage.googleapis.com/my-test-bucket-768/some_prefix/some_file_4.csv' +- 'https://storage.googleapis.com/my-test-bucket-768/another_prefix/some_file_1.csv' +- 'https://storage.googleapis.com/my-test-bucket-768/another_prefix/some_file_2.csv' +- 'https://storage.googleapis.com/my-test-bucket-768/another_prefix/some_file_3.csv' +- 'https://storage.googleapis.com/my-test-bucket-768/another_prefix/some_file_4.csv' Count the amount of rows in files ending with numbers from 1 to 3: ``` sql SELECT count(*) -FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32') +FROM gcs('https://storage.googleapis.com/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32') ``` ``` text @@ -102,7 +102,7 @@ Count the total amount of rows in all files in these two directories: ``` sql SELECT count(*) -FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32') +FROM gcs('https://storage.googleapis.com/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32') ``` ``` text @@ -119,7 +119,7 @@ Count the total amount of rows in files named `file-000.csv`, `file-001.csv`, ``` sql SELECT count(*) -FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32'); +FROM gcs('https://storage.googleapis.com/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32'); ``` ``` text @@ -131,32 +131,32 @@ FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768 Insert data into file `test-data.csv.gz`: ``` sql -INSERT INTO FUNCTION gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +INSERT INTO FUNCTION gcs('https://storage.googleapis.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') VALUES ('test-data', 1), ('test-data-2', 2); ``` Insert data into file `test-data.csv.gz` from existing table: ``` sql -INSERT INTO FUNCTION gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +INSERT INTO FUNCTION gcs('https://storage.googleapis.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') SELECT name, value FROM existing_table; ``` Glob ** can be used for recursive directory traversal. Consider the below example, it will fetch all files from `my-test-bucket-768` directory recursively: ``` sql -SELECT * FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**', 'CSV', 'name String, value UInt32', 'gzip'); +SELECT * FROM gcs('https://storage.googleapis.com/my-test-bucket-768/**', 'CSV', 'name String, value UInt32', 'gzip'); ``` The below get data from all `test-data.csv.gz` files from any folder inside `my-test-bucket` directory recursively: ``` sql -SELECT * FROM gcs('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip'); +SELECT * FROM gcs('https://storage.googleapis.com/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip'); ``` ## Partitioned Write -If you specify `PARTITION BY` expression when inserting data into `S3` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency. +If you specify `PARTITION BY` expression when inserting data into `GCS` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency. **Examples** From d0a54ab21b2107ee6893a7480533c79c2919fd75 Mon Sep 17 00:00:00 2001 From: Kuba Kaflik Date: Tue, 21 Mar 2023 14:45:58 +0100 Subject: [PATCH 012/308] Update GCS table function docs --- docs/en/sql-reference/table-functions/gcs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/gcs.md b/docs/en/sql-reference/table-functions/gcs.md index dcf49a5108b..bfa7f36fa48 100644 --- a/docs/en/sql-reference/table-functions/gcs.md +++ b/docs/en/sql-reference/table-functions/gcs.md @@ -22,7 +22,7 @@ The GCS Table Function integrates with Google Cloud Storage by using the GCS XML **Arguments** -- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [here](../../engines/table-engines/integrations/gcs.md#wildcards-in-path). +- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. :::note GCS The GCS path is in this format as the endpoint for the Google XML API is different than the JSON API: From 576efc1da3384148664202acef1cdbd27ddc8a08 Mon Sep 17 00:00:00 2001 From: Kuba Kaflik Date: Wed, 22 Mar 2023 06:58:09 +0100 Subject: [PATCH 013/308] register GCP function in factory --- src/TableFunctions/TableFunctionS3.cpp | 5 +++++ src/TableFunctions/registerTableFunctions.cpp | 1 + src/TableFunctions/registerTableFunctions.h | 1 + 3 files changed, 7 insertions(+) diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index f082b192ee0..6f4e6acec8a 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -183,6 +183,11 @@ void registerTableFunctionOSS(TableFunctionFactory & factory) factory.registerFunction(); } +void registerTableFunctionGCS(TableFunctionFactory & factory) +{ + factory.registerFunction(); +} + } #endif diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index 7b2b989e724..c692173e689 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -28,6 +28,7 @@ void registerTableFunctions() registerTableFunctionS3Cluster(factory); registerTableFunctionCOS(factory); registerTableFunctionOSS(factory); + registerTableFunctionGCS(factory); registerTableFunctionHudi(factory); registerTableFunctionDeltaLake(factory); #if USE_AVRO diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index 911aae199e2..af1b7129ec4 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -25,6 +25,7 @@ void registerTableFunctionS3(TableFunctionFactory & factory); void registerTableFunctionS3Cluster(TableFunctionFactory & factory); void registerTableFunctionCOS(TableFunctionFactory & factory); void registerTableFunctionOSS(TableFunctionFactory & factory); +void registerTableFunctionGCS(TableFunctionFactory & factory); void registerTableFunctionHudi(TableFunctionFactory & factory); void registerTableFunctionDeltaLake(TableFunctionFactory & factory); #if USE_AVRO From d8e8dc39e8b929ce0299f96a1be4065a74b45b99 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 3 Apr 2023 01:29:55 +0200 Subject: [PATCH 014/308] Fix test --- .../test_vertical_merges_from_compact_parts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index 1781ed7c976..6fdf0942ee4 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -41,7 +41,9 @@ def test_vertical_merges_from_comapact_parts(start_cluster): vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1, min_bytes_for_wide_part = 0, - min_rows_for_wide_part = 100; + min_rows_for_wide_part = 100, + compress_marks = 0, + compress_primary_key = 0; """.format( i ) From 01728d6d391c07e40c42cbd50151be5de472a62e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 4 Apr 2023 01:05:06 +0200 Subject: [PATCH 015/308] Fix tests --- .../configs/no_compress_marks.xml | 6 ++++++ .../test_backward_compatibility/configs/wide_parts_only.xml | 4 +--- tests/integration/test_backward_compatibility/test.py | 2 +- .../test_vertical_merges_from_compact_parts.py | 5 ++--- 4 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 tests/integration/test_backward_compatibility/configs/no_compress_marks.xml diff --git a/tests/integration/test_backward_compatibility/configs/no_compress_marks.xml b/tests/integration/test_backward_compatibility/configs/no_compress_marks.xml new file mode 100644 index 00000000000..cc968525bbb --- /dev/null +++ b/tests/integration/test_backward_compatibility/configs/no_compress_marks.xml @@ -0,0 +1,6 @@ + + + 0 + 0 + + diff --git a/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml b/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml index c823dd02d5a..e9cf053f1c5 100644 --- a/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml +++ b/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml @@ -1,7 +1,5 @@ 0 - 0 - 0 - + diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 01ed02720f8..e4ee6aa2de9 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -12,7 +12,7 @@ node1 = cluster.add_instance( with_installed_binary=True, ) node2 = cluster.add_instance( - "node2", main_configs=["configs/wide_parts_only.xml"], with_zookeeper=True + "node2", main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"], with_zookeeper=True ) diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index 6fdf0942ee4..4b144a37ce9 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -14,6 +14,7 @@ node_old = cluster.add_instance( ) node_new = cluster.add_instance( "node2", + main_configs=["configs/no_compress_marks.xml"], with_zookeeper=True, stay_alive=True, ) @@ -41,9 +42,7 @@ def test_vertical_merges_from_comapact_parts(start_cluster): vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1, min_bytes_for_wide_part = 0, - min_rows_for_wide_part = 100, - compress_marks = 0, - compress_primary_key = 0; + min_rows_for_wide_part = 100; """.format( i ) From 806c7a5f9d1a6869cc464f79b6bfd83028874540 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 3 Apr 2023 23:13:50 +0000 Subject: [PATCH 016/308] Automatic style fix --- tests/integration/test_backward_compatibility/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index e4ee6aa2de9..ea1d3ab9c07 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -12,7 +12,9 @@ node1 = cluster.add_instance( with_installed_binary=True, ) node2 = cluster.add_instance( - "node2", main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"], with_zookeeper=True + "node2", + main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"], + with_zookeeper=True, ) From 88f77c686d6ab3f8a4fb6a7c0f3b922019c69053 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 6 Apr 2023 10:41:44 +0000 Subject: [PATCH 017/308] Correctly handle concurrent snapshots --- src/Coordination/KeeperStateMachine.cpp | 42 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 632aaec6b54..efd1c10e568 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -288,15 +288,20 @@ bool KeeperStateMachine::apply_snapshot(nuraft::snapshot & s) nuraft::ptr latest_snapshot_ptr; { /// save snapshot into memory std::lock_guard lock(snapshots_lock); - if (s.get_last_log_idx() != latest_snapshot_meta->get_last_log_idx()) + if (s.get_last_log_idx() > latest_snapshot_meta->get_last_log_idx()) { ProfileEvents::increment(ProfileEvents::KeeperSnapshotApplysFailed); throw Exception( ErrorCodes::LOGICAL_ERROR, - "Required to apply snapshot with last log index {}, but our last log index is {}", + "Required to apply snapshot with last log index {}, but last created snapshot was for smaller log index {}", s.get_last_log_idx(), latest_snapshot_meta->get_last_log_idx()); } + else if (s.get_last_log_idx() < latest_snapshot_meta->get_last_log_idx()) + { + LOG_INFO(log, "A snapshot with a larger last log index ({}) was created, skipping applying this snapshot", latest_snapshot_meta->get_last_log_idx()); + } + latest_snapshot_ptr = latest_snapshot_buf; } @@ -371,19 +376,32 @@ void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_res { { /// Read storage data without locks and create snapshot std::lock_guard lock(snapshots_lock); - auto [path, error_code] = snapshot_manager.serializeSnapshotToDisk(*snapshot); - if (error_code) + + if (latest_snapshot_meta && snapshot->snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx()) { - throw Exception( - ErrorCodes::SYSTEM_ERROR, - "Snapshot {} was created failed, error: {}", + LOG_INFO( + log, + "Will not create a snapshot with last log idx {} because a snapshot with bigger last log idx ({}) is already " + "created", snapshot->snapshot_meta->get_last_log_idx(), - error_code.message()); + latest_snapshot_meta->get_last_log_idx()); + } + else + { + auto [path, error_code] = snapshot_manager.serializeSnapshotToDisk(*snapshot); + if (error_code) + { + throw Exception( + ErrorCodes::SYSTEM_ERROR, + "Snapshot {} was created failed, error: {}", + snapshot->snapshot_meta->get_last_log_idx(), + error_code.message()); + } + latest_snapshot_path = path; + latest_snapshot_meta = snapshot->snapshot_meta; + ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreations); + LOG_DEBUG(log, "Created persistent snapshot {} with path {}", latest_snapshot_meta->get_last_log_idx(), path); } - latest_snapshot_path = path; - latest_snapshot_meta = snapshot->snapshot_meta; - ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreations); - LOG_DEBUG(log, "Created persistent snapshot {} with path {}", latest_snapshot_meta->get_last_log_idx(), path); } { From cf5d9a175ae7c61ad0a7293fd49a86a4f7e09f01 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 14 Apr 2023 16:34:19 +0200 Subject: [PATCH 018/308] Revert "Merge pull request #48760 from ClickHouse/revert-46089-background-memory-tracker" This reverts commit a61ed332239e4a35af1b9cd31479560da6f08cca, reversing changes made to 5f01b8a2b59f6f340c65bfbc4ed0dd655d997ff9. --- .../settings.md | 33 ++++++++++++++-- programs/server/Server.cpp | 20 ++++++++++ src/Common/CurrentMetrics.cpp | 1 + src/Common/MemoryTracker.cpp | 8 ++++ src/Common/MemoryTracker.h | 19 ++++++++++ src/Core/ServerSettings.h | 2 + src/Interpreters/ThreadStatusExt.cpp | 1 + src/Storages/MergeTree/MergeList.cpp | 5 +++ src/Storages/MergeTree/MergeList.h | 2 + src/Storages/StorageMergeTree.cpp | 11 +++++- src/Storages/StorageReplicatedMergeTree.cpp | 10 ++++- .../test_merges_memory_limit/__init__.py | 0 .../test_merges_memory_limit/test.py | 38 +++++++++++++++++++ 13 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_merges_memory_limit/__init__.py create mode 100644 tests/integration/test_merges_memory_limit/test.py diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 7c97d0ab640..a9f0cc276ff 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1045,7 +1045,7 @@ Default value: `0`. ## background_pool_size {#background_pool_size} -Sets the number of threads performing background merges and mutations for tables with MergeTree engines. This setting is also could be applied at server startup from the `default` profile configuration for backward compatibility at the ClickHouse server start. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. +Sets the number of threads performing background merges and mutations for tables with MergeTree engines. This setting is also could be applied at server startup from the `default` profile configuration for backward compatibility at the ClickHouse server start. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. Before changing it, please also take a look at related MergeTree settings, such as [number_of_free_entries_in_pool_to_lower_max_size_of_merge](../../operations/settings/merge-tree-settings.md#number-of-free-entries-in-pool-to-lower-max-size-of-merge) and [number_of_free_entries_in_pool_to_execute_mutation](../../operations/settings/merge-tree-settings.md#number-of-free-entries-in-pool-to-execute-mutation). @@ -1063,8 +1063,8 @@ Default value: 16. ## background_merges_mutations_concurrency_ratio {#background_merges_mutations_concurrency_ratio} -Sets a ratio between the number of threads and the number of background merges and mutations that can be executed concurrently. For example if the ratio equals to 2 and -`background_pool_size` is set to 16 then ClickHouse can execute 32 background merges concurrently. This is possible, because background operation could be suspended and postponed. This is needed to give small merges more execution priority. You can only increase this ratio at runtime. To lower it you have to restart the server. +Sets a ratio between the number of threads and the number of background merges and mutations that can be executed concurrently. For example, if the ratio equals to 2 and +`background_pool_size` is set to 16 then ClickHouse can execute 32 background merges concurrently. This is possible, because background operations could be suspended and postponed. This is needed to give small merges more execution priority. You can only increase this ratio at runtime. To lower it you have to restart the server. The same as for `background_pool_size` setting `background_merges_mutations_concurrency_ratio` could be applied from the `default` profile for backward compatibility. Possible values: @@ -1079,6 +1079,33 @@ Default value: 2. 3 ``` +## merges_mutations_memory_usage_soft_limit {#merges_mutations_memory_usage_soft_limit} + +Sets the limit on how much RAM is allowed to use for performing merge and mutation operations. +Zero means unlimited. +If ClickHouse reaches this limit, it won't schedule any new background merge or mutation operations but will continue to execute already scheduled tasks. + +Possible values: + +- Any positive integer. + +**Example** + +```xml +0 +``` + +## merges_mutations_memory_usage_to_ram_ratio {#merges_mutations_memory_usage_to_ram_ratio} + +The default `merges_mutations_memory_usage_soft_limit` value is calculated as `memory_amount * merges_mutations_memory_usage_to_ram_ratio`. + +Default value: `0.5`. + +**See also** + +- [max_memory_usage](../../operations/settings/query-complexity.md#settings_max_memory_usage) +- [merges_mutations_memory_usage_soft_limit](#merges_mutations_memory_usage_soft_limit) + ## background_merges_mutations_scheduling_policy {#background_merges_mutations_scheduling_policy} Algorithm used to select next merge or mutation to be executed by background thread pool. Policy may be changed at runtime without server restart. diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8c0d50bae55..cba7a4c4778 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -135,6 +135,7 @@ namespace CurrentMetrics extern const Metric Revision; extern const Metric VersionInteger; extern const Metric MemoryTracking; + extern const Metric MergesMutationsMemoryTracking; extern const Metric MaxDDLEntryID; extern const Metric MaxPushedDDLEntryID; } @@ -1225,6 +1226,25 @@ try total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); + size_t merges_mutations_memory_usage_soft_limit = server_settings_.merges_mutations_memory_usage_soft_limit; + + size_t default_merges_mutations_server_memory_usage = static_cast(memory_amount * server_settings_.merges_mutations_memory_usage_to_ram_ratio); + if (merges_mutations_memory_usage_soft_limit == 0 || merges_mutations_memory_usage_soft_limit > default_merges_mutations_server_memory_usage) + { + merges_mutations_memory_usage_soft_limit = default_merges_mutations_server_memory_usage; + LOG_WARNING(log, "Setting merges_mutations_memory_usage_soft_limit was set to {}" + " ({} available * {:.2f} merges_mutations_memory_usage_to_ram_ratio)", + formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit), + formatReadableSizeWithBinarySuffix(memory_amount), + server_settings_.merges_mutations_memory_usage_to_ram_ratio); + } + + LOG_INFO(log, "Merges and mutations memory limit is set to {}", + formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit)); + background_memory_tracker.setSoftLimit(merges_mutations_memory_usage_soft_limit); + background_memory_tracker.setDescription("(background)"); + background_memory_tracker.setMetric(CurrentMetrics::MergesMutationsMemoryTracking); + total_memory_tracker.setAllowUseJemallocMemory(server_settings_.allow_use_jemalloc_memory); auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker(); diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 81c1481e2de..ea248d996a7 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -53,6 +53,7 @@ M(QueryThread, "Number of query processing threads") \ M(ReadonlyReplica, "Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.") \ M(MemoryTracking, "Total amount of memory (bytes) allocated by the server.") \ + M(MergesMutationsMemoryTracking, "Total amount of memory (bytes) allocated by background tasks (merges and mutations).") \ M(EphemeralNode, "Number of ephemeral nodes hold in ZooKeeper.") \ M(ZooKeeperSession, "Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows.") \ M(ZooKeeperWatch, "Number of watches (event subscriptions) in ZooKeeper.") \ diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 674d8d469af..9bff365483d 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -96,6 +96,7 @@ using namespace std::chrono_literals; static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); +MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User); std::atomic MemoryTracker::free_memory_in_allocator_arenas; @@ -528,3 +529,10 @@ void MemoryTracker::setOrRaiseProfilerLimit(Int64 value) while ((value == 0 || old_value < value) && !profiler_limit.compare_exchange_weak(old_value, value)) ; } + +bool canEnqueueBackgroundTask() +{ + auto limit = background_memory_tracker.getSoftLimit(); + auto amount = background_memory_tracker.get(); + return limit == 0 || amount < limit; +} diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 0d7748856bd..260005fd536 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -110,6 +110,22 @@ public: return amount.load(std::memory_order_relaxed); } + // Merges and mutations may pass memory ownership to other threads thus in the end of execution + // MemoryTracker for background task may have a non-zero counter. + // This method is intended to fix the counter inside of background_memory_tracker. + // NOTE: We can't use alloc/free methods to do it, because they also will change the value inside + // of total_memory_tracker. + void adjustOnBackgroundTaskEnd(const MemoryTracker * child) + { + auto background_memory_consumption = child->amount.load(std::memory_order_relaxed); + amount.fetch_sub(background_memory_consumption, std::memory_order_relaxed); + + // Also fix CurrentMetrics::MergesMutationsMemoryTracking + auto metric_loaded = metric.load(std::memory_order_relaxed); + if (metric_loaded != CurrentMetrics::end()) + CurrentMetrics::sub(metric_loaded, background_memory_consumption); + } + Int64 getPeak() const { return peak.load(std::memory_order_relaxed); @@ -220,3 +236,6 @@ public: }; extern MemoryTracker total_memory_tracker; +extern MemoryTracker background_memory_tracker; + +bool canEnqueueBackgroundTask(); diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index aabc89cc6d7..2d8c37783db 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -40,6 +40,8 @@ namespace DB M(String, temporary_data_in_cache, "", "Cache disk name for temporary data.", 0) \ M(UInt64, max_server_memory_usage, 0, "Limit on total memory usage. Zero means Unlimited.", 0) \ M(Double, max_server_memory_usage_to_ram_ratio, 0.9, "Same as max_server_memory_usage but in to ram ratio. Allows to lower max memory on low-memory systems.", 0) \ + M(UInt64, merges_mutations_memory_usage_soft_limit, 0, "Limit on total memory usage for merges and mutations. Zero means Unlimited.", 0) \ + M(Double, merges_mutations_memory_usage_to_ram_ratio, 0.5, "Same as merges_mutations_memory_usage_soft_limit but in to ram ratio. Allows to lower memory limit on low-memory systems.", 0) \ M(Bool, allow_use_jemalloc_memory, true, "Allows to use jemalloc memory.", 0) \ \ M(UInt64, max_concurrent_queries, 0, "Limit on total number of concurrently executed queries. Zero means Unlimited.", 0) \ diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index fd4a6b5e996..559652fe56c 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -84,6 +84,7 @@ ThreadGroupPtr ThreadGroup::createForBackgroundProcess(ContextPtr storage_contex group->memory_tracker.setProfilerStep(settings.memory_profiler_step); group->memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability); group->memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator); + group->memory_tracker.setParent(&background_memory_tracker); if (settings.memory_tracker_fault_probability > 0.0) group->memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability); diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index 0bf662921ad..d54079bc7a5 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -78,4 +78,9 @@ MergeInfo MergeListElement::getInfo() const return res; } +MergeListElement::~MergeListElement() +{ + background_memory_tracker.adjustOnBackgroundTaskEnd(&getMemoryTracker()); +} + } diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index 9c8c2ebd1e4..d8271a66b45 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -113,6 +113,8 @@ struct MergeListElement : boost::noncopyable MergeListElement * ptr() { return this; } MergeListElement & ref() { return *this; } + + ~MergeListElement(); }; /** Maintains a list of currently running merges. diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 34bf5d55270..6cb3ce35e5b 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include namespace DB { @@ -918,7 +920,14 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge( SelectPartsDecision select_decision = SelectPartsDecision::CANNOT_SELECT; - if (partition_id.empty()) + if (!canEnqueueBackgroundTask()) + { + if (out_disable_reason) + *out_disable_reason = fmt::format("Current background tasks memory usage ({}) is more than the limit ({})", + formatReadableSizeWithBinarySuffix(background_memory_tracker.get()), + formatReadableSizeWithBinarySuffix(background_memory_tracker.getSoftLimit())); + } + else if (partition_id.empty()) { UInt64 max_source_parts_size = merger_mutator.getMaxSourcePartsSizeForMerge(); bool merge_with_ttl_allowed = getTotalMergesWithTTLInMergeList() < data_settings->max_number_of_merges_with_ttl_in_pool; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 5cd02c33d55..4a36cf03c2a 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -3223,7 +3224,14 @@ void StorageReplicatedMergeTree::mergeSelectingTask() auto merges_and_mutations_queued = queue.countMergesAndPartMutations(); size_t merges_and_mutations_sum = merges_and_mutations_queued.merges + merges_and_mutations_queued.mutations; - if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue) + if (!canEnqueueBackgroundTask()) + { + LOG_TRACE(log, "Reached memory limit for the background tasks ({}), so won't select new parts to merge or mutate." + "Current background tasks memory usage: {}.", + formatReadableSizeWithBinarySuffix(background_memory_tracker.getSoftLimit()), + formatReadableSizeWithBinarySuffix(background_memory_tracker.get())); + } + else if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue) { LOG_TRACE(log, "Number of queued merges ({}) and part mutations ({})" " is greater than max_replicated_merges_in_queue ({}), so won't select new parts to merge or mutate.", diff --git a/tests/integration/test_merges_memory_limit/__init__.py b/tests/integration/test_merges_memory_limit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_merges_memory_limit/test.py b/tests/integration/test_merges_memory_limit/test.py new file mode 100644 index 00000000000..04729f3a01c --- /dev/null +++ b/tests/integration/test_merges_memory_limit/test.py @@ -0,0 +1,38 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance("node") + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_memory_limit_success(): + node.query( + "CREATE TABLE test_merge_oom ENGINE=AggregatingMergeTree ORDER BY id EMPTY AS SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(20000)" + ) + node.query("SYSTEM STOP MERGES test_merge_oom") + node.query( + "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)" + ) + node.query( + "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)" + ) + node.query( + "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)" + ) + _, error = node.query_and_get_answer_with_error( + "SYSTEM START MERGES test_merge_oom;SET optimize_throw_if_noop=1;OPTIMIZE TABLE test_merge_oom FINAL" + ) + + assert not error + node.query("DROP TABLE test_merge_oom") From a1f21a5fc427e860199aa9e1db3c1c4177a340f7 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 14 Apr 2023 16:00:32 +0200 Subject: [PATCH 019/308] Do not log peak for background memory tracker --- src/Common/MemoryTracker.cpp | 8 ++++++-- src/Common/MemoryTracker.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 9bff365483d..81cac2617c5 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -96,13 +96,17 @@ using namespace std::chrono_literals; static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); -MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User); +MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User, false); std::atomic MemoryTracker::free_memory_in_allocator_arenas; MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} - +MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_, bool log_peak_memory_usage_in_destructor_) + : parent(parent_) + , log_peak_memory_usage_in_destructor(log_peak_memory_usage_in_destructor_) + , level(level_) +{} MemoryTracker::~MemoryTracker() { diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 260005fd536..4e29d40c953 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -98,6 +98,7 @@ public: explicit MemoryTracker(VariableContext level_ = VariableContext::Thread); explicit MemoryTracker(MemoryTracker * parent_, VariableContext level_ = VariableContext::Thread); + MemoryTracker(MemoryTracker * parent_, VariableContext level_, bool log_peak_memory_usage_in_destructor_); ~MemoryTracker(); From c49ff08a75fe1b4765196db90646ba1d003fe74f Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 14 Apr 2023 19:28:04 +0200 Subject: [PATCH 020/308] Better local object storage --- src/CMakeLists.txt | 1 + src/Disks/DiskLocal.cpp | 51 +----------- src/Disks/DiskType.h | 3 + src/Disks/IDisk.cpp | 2 +- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 3 + .../Cached/CachedObjectStorage.cpp | 2 +- src/Disks/ObjectStorages/IObjectStorage.h | 8 +- .../{ => Local}/LocalObjectStorage.cpp | 79 ++++++++++++++----- .../{ => Local}/LocalObjectStorage.h | 6 +- .../Local/registerLocalObjectStorage.cpp | 44 +++++++++++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 3 +- src/Disks/loadLocalDiskConfig.cpp | 63 +++++++++++++++ src/Disks/loadLocalDiskConfig.h | 16 ++++ src/Disks/registerDisks.cpp | 5 ++ .../02714_local_object_storage.reference | 2 + .../02714_local_object_storage.sql | 23 ++++++ 16 files changed, 235 insertions(+), 76 deletions(-) rename src/Disks/ObjectStorages/{ => Local}/LocalObjectStorage.cpp (66%) rename src/Disks/ObjectStorages/{ => Local}/LocalObjectStorage.h (96%) create mode 100644 src/Disks/ObjectStorages/Local/registerLocalObjectStorage.cpp create mode 100644 src/Disks/loadLocalDiskConfig.cpp create mode 100644 src/Disks/loadLocalDiskConfig.h create mode 100644 tests/queries/0_stateless/02714_local_object_storage.reference create mode 100644 tests/queries/0_stateless/02714_local_object_storage.sql diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 073c60c0392..58964b13dea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -135,6 +135,7 @@ if (TARGET ch_contrib::hdfs) endif() add_headers_and_sources(dbms Disks/ObjectStorages/Cached) +add_headers_and_sources(dbms Disks/ObjectStorages/Local) add_headers_and_sources(dbms Disks/ObjectStorages/Web) add_headers_and_sources(dbms Storages/Cache) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 160fcb5732c..473420cc0a8 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -9,9 +9,11 @@ #include #include #include -#include +#include #include #include +#include +#include #include #include @@ -51,53 +53,6 @@ std::mutex DiskLocal::reservation_mutex; using DiskLocalPtr = std::shared_ptr; -static void loadDiskLocalConfig(const String & name, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - ContextPtr context, - String & path, - UInt64 & keep_free_space_bytes) -{ - path = config.getString(config_prefix + ".path", ""); - if (name == "default") - { - if (!path.empty()) - throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, - "\"default\" disk path should be provided in not it "); - path = context->getPath(); - } - else - { - if (path.empty()) - throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Disk path can not be empty. Disk {}", name); - if (path.back() != '/') - throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Disk path must end with /. Disk {}", name); - if (path == context->getPath()) - throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Disk path ('{}') cannot be equal to . Use disk instead.", path); - } - - bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio"); - - if (config.has(config_prefix + ".keep_free_space_bytes") && has_space_ratio) - throw Exception(ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG, - "Only one of 'keep_free_space_bytes' and 'keep_free_space_ratio' can be specified"); - - keep_free_space_bytes = config.getUInt64(config_prefix + ".keep_free_space_bytes", 0); - - if (has_space_ratio) - { - auto ratio = config.getDouble(config_prefix + ".keep_free_space_ratio"); - if (ratio < 0 || ratio > 1) - throw Exception(ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG, "'keep_free_space_ratio' have to be between 0 and 1"); - String tmp_path = path; - if (tmp_path.empty()) - tmp_path = context->getPath(); - - // Create tmp disk for getting total disk space. - keep_free_space_bytes = static_cast(DiskLocal("tmp", tmp_path, 0).getTotalSpace() * ratio); - } -} - std::optional fileSizeSafe(const fs::path & path) { std::error_code ec; diff --git a/src/Disks/DiskType.h b/src/Disks/DiskType.h index 4d099e33a7a..840ed5549e6 100644 --- a/src/Disks/DiskType.h +++ b/src/Disks/DiskType.h @@ -15,6 +15,7 @@ enum class DataSourceType HDFS, WebServer, AzureBlobStorage, + LocalBlobStorage, }; inline String toString(DataSourceType data_source_type) @@ -35,6 +36,8 @@ inline String toString(DataSourceType data_source_type) return "web"; case DataSourceType::AzureBlobStorage: return "azure_blob_storage"; + case DataSourceType::LocalBlobStorage: + return "local_blob_storage"; } UNREACHABLE(); } diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index 4969cc7c700..c18e99bd58e 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include namespace DB diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 8450e740ab5..22f5576f320 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -122,6 +122,8 @@ void ReadBufferFromRemoteFSGather::initialize() if (object.bytes_size > current_buf_offset) { + LOG_TEST(log, "Reading from file: {} ({})", object.absolute_path, object.getMappedPath()); + /// Do not create a new buffer if we already have what we need. if (!current_buf || current_buf_idx != i) { @@ -173,6 +175,7 @@ bool ReadBufferFromRemoteFSGather::moveToNextBuffer() ++current_buf_idx; const auto & object = blobs_to_read[current_buf_idx]; + LOG_TEST(log, "Reading from next file: {} ({})", object.absolute_path, object.getMappedPath()); current_buf = createImplementationBuffer(object); return true; diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp index 505b26ebb3a..108246c7dd6 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp @@ -65,7 +65,7 @@ ReadSettings CachedObjectStorage::patchSettings(const ReadSettings & read_settin if (FileCache::isReadOnly()) modified_settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true; - return IObjectStorage::patchSettings(modified_settings); + return object_storage->patchSettings(modified_settings); } void CachedObjectStorage::startup() diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index 2f27dc18e4b..434940b3c9c 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -207,14 +207,14 @@ public: virtual WriteSettings getAdjustedSettingsFromMetadataFile(const WriteSettings & settings, const std::string & /* path */) const { return settings; } + virtual ReadSettings patchSettings(const ReadSettings & read_settings) const; + + virtual WriteSettings patchSettings(const WriteSettings & write_settings) const; + protected: /// Should be called from implementation of applyNewSettings() void applyRemoteThrottlingSettings(ContextPtr context); - /// Should be used by implementation of read* and write* methods - virtual ReadSettings patchSettings(const ReadSettings & read_settings) const; - virtual WriteSettings patchSettings(const WriteSettings & write_settings) const; - private: mutable std::mutex throttlers_mutex; ThrottlerPtr remote_read_throttler; diff --git a/src/Disks/ObjectStorages/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp similarity index 66% rename from src/Disks/ObjectStorages/LocalObjectStorage.cpp rename to src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index 67e2cc2d74b..0da1207cbca 100644 --- a/src/Disks/ObjectStorages/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -1,10 +1,16 @@ -#include +#include #include #include #include +#include +#include #include +#include +#include #include +#include +#include #include namespace fs = std::filesystem; @@ -47,10 +53,32 @@ std::unique_ptr LocalObjectStorage::readObjects( /// NOL std::optional read_hint, std::optional file_size) const { - if (objects.size() != 1) - throw Exception(ErrorCodes::LOGICAL_ERROR, "LocalObjectStorage support read only from single object"); + auto modified_settings = patchSettings(read_settings); + auto read_buffer_creator = + [=] (const std::string & file_path, size_t /* read_until_position */) + -> std::shared_ptr + { + auto impl = createReadBufferFromFileBase(file_path, modified_settings, read_hint, file_size); + if (modified_settings.enable_filesystem_cache) + { + return std::make_unique(std::move(impl)); + } + return impl; + }; - return readObject(objects[0], read_settings, read_hint, file_size); + auto impl = std::make_unique( + std::move(read_buffer_creator), objects, modified_settings); + + if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) + { + auto & reader = getThreadPoolReader(); + return std::make_unique(reader, modified_settings, std::move(impl)); + } + else + { + auto buf = std::make_unique(std::move(impl), modified_settings); + return std::make_unique(std::move(buf), modified_settings.remote_read_min_bytes_for_seek); + } } std::string LocalObjectStorage::getUniqueId(const std::string & path) const @@ -58,19 +86,13 @@ std::string LocalObjectStorage::getUniqueId(const std::string & path) const return toString(getINodeNumberFromPath(path)); } -std::unique_ptr LocalObjectStorage::readObject( /// NOLINT - const StoredObject & object, - const ReadSettings & read_settings, - std::optional read_hint, - std::optional file_size) const +ReadSettings LocalObjectStorage::patchSettings(const ReadSettings & read_settings) const { - const auto & path = object.absolute_path; - - if (!file_size) - file_size = tryGetSizeFromFilePath(path); + if (!read_settings.enable_filesystem_cache) + return IObjectStorage::patchSettings(read_settings); + auto modified_settings{read_settings}; /// For now we cannot allow asynchronous reader from local filesystem when CachedObjectStorage is used. - ReadSettings modified_settings{read_settings}; switch (modified_settings.local_fs_method) { case LocalFSReadMethod::pread_threadpool: @@ -85,23 +107,36 @@ std::unique_ptr LocalObjectStorage::readObject( /// NOLI break; } } + return IObjectStorage::patchSettings(modified_settings); +} + +std::unique_ptr LocalObjectStorage::readObject( /// NOLINT + const StoredObject & object, + const ReadSettings & read_settings, + std::optional read_hint, + std::optional file_size) const +{ + const auto & path = object.absolute_path; + + if (!file_size) + file_size = tryGetSizeFromFilePath(path); LOG_TEST(log, "Read object: {}", path); - return createReadBufferFromFileBase(path, modified_settings, read_hint, file_size); + return createReadBufferFromFileBase(path, patchSettings(read_settings), read_hint, file_size); } std::unique_ptr LocalObjectStorage::writeObject( /// NOLINT const StoredObject & object, WriteMode mode, std::optional /* attributes */, - FinalizeCallback && /* finalize_callback */, + FinalizeCallback && finalize_callback, size_t buf_size, const WriteSettings & /* write_settings */) { - const auto & path = object.absolute_path; int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1; - LOG_TEST(log, "Write object: {}", path); - return std::make_unique(path, buf_size, flags); + LOG_TEST(log, "Write object: {}", object.absolute_path); + auto impl = std::make_unique(object.absolute_path, buf_size, flags); + return std::make_unique(std::move(impl), std::move(finalize_callback), object.absolute_path); } void LocalObjectStorage::removeObject(const StoredObject & object) @@ -173,4 +208,10 @@ void LocalObjectStorage::applyNewSettings( { } +std::string LocalObjectStorage::generateBlobNameForPath(const std::string & /* path */) +{ + constexpr size_t key_name_total_size = 32; + return getRandomASCIIString(key_name_total_size); +} + } diff --git a/src/Disks/ObjectStorages/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h similarity index 96% rename from src/Disks/ObjectStorages/LocalObjectStorage.h rename to src/Disks/ObjectStorages/Local/LocalObjectStorage.h index b04e3fa6285..a55ed17c416 100644 --- a/src/Disks/ObjectStorages/LocalObjectStorage.h +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h @@ -79,12 +79,14 @@ public: bool supportsAppend() const override { return true; } - std::string generateBlobNameForPath(const std::string & path) override { return path; } - std::string getUniqueId(const std::string & path) const override; + std::string generateBlobNameForPath(const std::string & path) override; + bool isRemote() const override { return false; } + ReadSettings patchSettings(const ReadSettings & read_settings) const override; + private: Poco::Logger * log; DataSourceDescription data_source_description; diff --git a/src/Disks/ObjectStorages/Local/registerLocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/registerLocalObjectStorage.cpp new file mode 100644 index 00000000000..251fc77d1f8 --- /dev/null +++ b/src/Disks/ObjectStorages/Local/registerLocalObjectStorage.cpp @@ -0,0 +1,44 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace DB +{ +void registerDiskLocalObjectStorage(DiskFactory & factory, bool global_skip_access_check) +{ + auto creator = [global_skip_access_check]( + const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + ContextPtr context, + const DisksMap & /*map*/) -> DiskPtr + { + String path; + UInt64 keep_free_space_bytes; + loadDiskLocalConfig(name, config, config_prefix, context, path, keep_free_space_bytes); + fs::create_directories(path); + + String type = config.getString(config_prefix + ".type"); + chassert(type == "local_blob_storage"); + + std::shared_ptr local_storage = std::make_shared(); + MetadataStoragePtr metadata_storage; + auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context); + metadata_storage = std::make_shared(metadata_disk, path); + + auto disk = std::make_shared( + name, path, "Local", metadata_storage, local_storage, false, /* threadpool_size */16); + disk->startup(context, global_skip_access_check); + return disk; + }; + factory.registerDiskType("local_blob_storage", creator); +} + +} diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 83c0c7446a8..de7b88cef45 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -146,7 +146,8 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { auto & reader = getThreadPoolReader(); - return std::make_unique(reader, disk_read_settings, std::move(s3_impl)); + return std::make_unique( + reader, disk_read_settings, std::move(s3_impl), disk_read_settings.remote_read_min_bytes_for_seek); } else { diff --git a/src/Disks/loadLocalDiskConfig.cpp b/src/Disks/loadLocalDiskConfig.cpp new file mode 100644 index 00000000000..0e5eca17ca7 --- /dev/null +++ b/src/Disks/loadLocalDiskConfig.cpp @@ -0,0 +1,63 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_ELEMENT_IN_CONFIG; + extern const int EXCESSIVE_ELEMENT_IN_CONFIG; +} + +void loadDiskLocalConfig(const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + ContextPtr context, + String & path, + UInt64 & keep_free_space_bytes) +{ + path = config.getString(config_prefix + ".path", ""); + if (name == "default") + { + if (!path.empty()) + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, + "\"default\" disk path should be provided in not it "); + path = context->getPath(); + } + else + { + if (path.empty()) + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Disk path can not be empty. Disk {}", name); + if (path.back() != '/') + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Disk path must end with /. Disk {}", name); + if (path == context->getPath()) + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Disk path ('{}') cannot be equal to . Use disk instead.", path); + } + + bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio"); + + if (config.has(config_prefix + ".keep_free_space_bytes") && has_space_ratio) + throw Exception(ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG, + "Only one of 'keep_free_space_bytes' and 'keep_free_space_ratio' can be specified"); + + keep_free_space_bytes = config.getUInt64(config_prefix + ".keep_free_space_bytes", 0); + + if (has_space_ratio) + { + auto ratio = config.getDouble(config_prefix + ".keep_free_space_ratio"); + if (ratio < 0 || ratio > 1) + throw Exception(ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG, "'keep_free_space_ratio' have to be between 0 and 1"); + String tmp_path = path; + if (tmp_path.empty()) + tmp_path = context->getPath(); + + // Create tmp disk for getting total disk space. + keep_free_space_bytes = static_cast(DiskLocal("tmp", tmp_path, 0).getTotalSpace() * ratio); + } +} + +} diff --git a/src/Disks/loadLocalDiskConfig.h b/src/Disks/loadLocalDiskConfig.h new file mode 100644 index 00000000000..ba1ac5378a4 --- /dev/null +++ b/src/Disks/loadLocalDiskConfig.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +namespace Poco::Util { class AbstractConfiguration; } + +namespace DB +{ +void loadDiskLocalConfig( + const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + ContextPtr context, + String & path, + UInt64 & keep_free_space_bytes); +} diff --git a/src/Disks/registerDisks.cpp b/src/Disks/registerDisks.cpp index cae83203553..48d5a19fb61 100644 --- a/src/Disks/registerDisks.cpp +++ b/src/Disks/registerDisks.cpp @@ -29,6 +29,9 @@ void registerDiskWebServer(DiskFactory & factory, bool global_skip_access_check) void registerDiskCache(DiskFactory & factory, bool global_skip_access_check); +void registerDiskLocalObjectStorage(DiskFactory & factory, bool global_skip_access_check); + + void registerDisks(bool global_skip_access_check) { auto & factory = DiskFactory::instance(); @@ -54,6 +57,8 @@ void registerDisks(bool global_skip_access_check) registerDiskWebServer(factory, global_skip_access_check); registerDiskCache(factory, global_skip_access_check); + + registerDiskLocalObjectStorage(factory, global_skip_access_check); } } diff --git a/tests/queries/0_stateless/02714_local_object_storage.reference b/tests/queries/0_stateless/02714_local_object_storage.reference new file mode 100644 index 00000000000..b3f28057554 --- /dev/null +++ b/tests/queries/0_stateless/02714_local_object_storage.reference @@ -0,0 +1,2 @@ +1 test +1 test diff --git a/tests/queries/0_stateless/02714_local_object_storage.sql b/tests/queries/0_stateless/02714_local_object_storage.sql new file mode 100644 index 00000000000..6cc7909d237 --- /dev/null +++ b/tests/queries/0_stateless/02714_local_object_storage.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test (a Int32, b String) +ENGINE = MergeTree() ORDER BY tuple() +SETTINGS disk = disk( + type = 'local_blob_storage', + path = '/var/lib/clickhouse/disks/${CLICKHOUSE_TEST_UNIQUE_NAME}/'); + +INSERT INTO test SELECT 1, 'test'; +SELECT * FROM test; + +DROP TABLE test SYNC; + +CREATE TABLE test (a Int32, b String) +ENGINE = MergeTree() ORDER BY tuple() +SETTINGS disk = disk( + type = 'cache', + max_size = '10Mi', + path = '/var/lib/clickhouse/caches/${CLICKHOUSE_TEST_UNIQUE_NAME}/', + disk = disk(type='local_blob_storage', path='/var/lib/clickhouse/disks/${CLICKHOUSE_TEST_UNIQUE_NAME}/')); + +INSERT INTO test SELECT 1, 'test'; +SELECT * FROM test; From 9374666c0c238356c12e324086528e37cf1a2e62 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 15 Apr 2023 12:20:35 +0200 Subject: [PATCH 021/308] Deprecetae local object storage on fake metadata storage --- src/Disks/DiskLocal.cpp | 31 ------------------- src/Disks/DiskLocal.h | 4 --- src/Disks/IDisk.cpp | 3 -- src/Disks/IDisk.h | 8 ++++- .../Cached/registerDiskCache.cpp | 3 ++ .../Local/LocalObjectStorage.cpp | 1 - tests/config/config.d/storage_conf.xml | 6 ++-- 7 files changed, 13 insertions(+), 43 deletions(-) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 473420cc0a8..5305057a606 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -9,10 +9,6 @@ #include #include #include -#include -#include -#include -#include #include #include @@ -38,7 +34,6 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_ELEMENT_IN_CONFIG; - extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int PATH_ACCESS_DENIED; extern const int LOGICAL_ERROR; extern const int CANNOT_TRUNCATE_FILE; @@ -555,25 +550,6 @@ catch (...) return false; } -DiskObjectStoragePtr DiskLocal::createDiskObjectStorage() -{ - auto object_storage = std::make_shared(); - auto metadata_storage = std::make_shared( - /* metadata_storage */std::static_pointer_cast(shared_from_this()), - object_storage, - /* object_storage_root_path */getPath()); - - return std::make_shared( - getName(), - disk_path, - "Local", - metadata_storage, - object_storage, - false, - /* threadpool_size */16 - ); -} - void DiskLocal::checkAccessImpl(const String & path) { try @@ -701,13 +677,6 @@ void DiskLocal::chmod(const String & path, mode_t mode) DB::throwFromErrnoWithPath("Cannot chmod file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); } -MetadataStoragePtr DiskLocal::getMetadataStorage() -{ - auto object_storage = std::make_shared(); - return std::make_shared( - std::static_pointer_cast(shared_from_this()), object_storage, getPath()); -} - void registerDiskLocal(DiskFactory & factory, bool global_skip_access_check) { auto creator = [global_skip_access_check]( diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index d6182463ebf..c1f77e5359d 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -121,16 +121,12 @@ public: bool canRead() const noexcept; bool canWrite() const noexcept; - DiskObjectStoragePtr createDiskObjectStorage() override; - bool supportsStat() const override { return true; } struct stat stat(const String & path) const override; bool supportsChmod() const override { return true; } void chmod(const String & path, mode_t mode) override; - MetadataStoragePtr getMetadataStorage() override; - protected: void checkAccessImpl(const String & path) override; diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index c18e99bd58e..22197760d88 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -7,9 +7,6 @@ #include #include #include -#include -#include -#include #include namespace DB diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 797235b5fb8..bc56a5dc615 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -367,7 +367,13 @@ public: /// Actually it's a part of IDiskRemote implementation but we have so /// complex hierarchy of disks (with decorators), so we cannot even /// dynamic_cast some pointer to IDisk to pointer to IDiskRemote. - virtual MetadataStoragePtr getMetadataStorage() = 0; + virtual MetadataStoragePtr getMetadataStorage() + { + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Method getMetadataStorage() is not implemented for disk type: {}", + toString(getDataSourceDescription().type)); + } /// Very similar case as for getMetadataDiskIfExistsOrSelf(). If disk has "metadata" /// it will return mapping for each required path: path -> metadata as string. diff --git a/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp b/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp index d8c4a9d42fd..ff987f1d5ee 100644 --- a/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp +++ b/src/Disks/ObjectStorages/Cached/registerDiskCache.cpp @@ -47,6 +47,9 @@ void registerDiskCache(DiskFactory & factory, bool /* global_skip_access_check * auto disk = disk_it->second; auto cache = FileCacheFactory::instance().getOrCreate(cache_base_path, file_cache_settings, name); + if (!dynamic_cast(disk.get())) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cached disk is allowed only on top of object storage"); + auto disk_object_storage = disk->createDiskObjectStorage(); disk_object_storage->wrapWithCache(cache, file_cache_settings, name); diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index 0da1207cbca..7f705264721 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -20,7 +20,6 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; } diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml index bc9269e6ec1..39345bdf253 100644 --- a/tests/config/config.d/storage_conf.xml +++ b/tests/config/config.d/storage_conf.xml @@ -120,15 +120,15 @@ - local + local_blob_storage local_disk/ - local + local_blob_storage local_disk_2/ - local + local_blob_storage local_disk_3/ From d2c73a5522f778decbc4794bf0a17b6957cde47d Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 17 Apr 2023 16:41:21 +0200 Subject: [PATCH 022/308] Better --- .../IO/CachedOnDiskReadBufferFromFile.cpp | 12 ++--- src/Disks/IO/CachedOnDiskReadBufferFromFile.h | 7 +-- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 2 +- .../AzureBlobStorage/AzureObjectStorage.cpp | 2 +- .../Cached/CachedObjectStorage.cpp | 34 +------------- .../ObjectStorages/HDFS/HDFSObjectStorage.cpp | 2 +- .../Local/LocalObjectStorage.cpp | 45 ++++++++++--------- .../ObjectStorages/S3/S3ObjectStorage.cpp | 4 +- .../ObjectStorages/Web/WebObjectStorage.cpp | 4 +- src/Storages/StorageS3.cpp | 4 +- 10 files changed, 43 insertions(+), 73 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 31f9373de18..e7a2c3e1008 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -186,12 +187,11 @@ CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegment & file_segment if (!remote_fs_segment_reader) { - remote_fs_segment_reader = implementation_buffer_creator(); - - if (!remote_fs_segment_reader->supportsRightBoundedReads()) - throw Exception( - ErrorCodes::CANNOT_USE_CACHE, - "Cache cannot be used with a ReadBuffer which does not support right bounded reads"); + auto impl = implementation_buffer_creator(); + if (!impl->supportsRightBoundedReads()) + remote_fs_segment_reader = std::make_unique(std::move(impl)); + else + remote_fs_segment_reader = std::move(impl); file_segment.setRemoteFileReader(remote_fs_segment_reader); } diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index d3c265a522b..7e48d9b3109 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -20,8 +20,7 @@ namespace DB class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase { public: - using ImplementationBufferPtr = std::shared_ptr; - using ImplementationBufferCreator = std::function; + using ImplementationBufferCreator = std::function()>; CachedOnDiskReadBufferFromFile( const String & source_file_path_, @@ -61,7 +60,7 @@ public: }; private: - void initialize(size_t offset, size_t size); + using ImplementationBufferPtr = std::shared_ptr; ImplementationBufferPtr getImplementationBuffer(FileSegmentPtr & file_segment); @@ -77,6 +76,8 @@ private: bool nextImplStep(); + void initialize(size_t offset, size_t size); + void assertCorrectness() const; std::shared_ptr getRemoteFSReadBuffer(FileSegment & file_segment, ReadType read_type_); diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index abe57647a48..d8e55b648fa 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -20,7 +20,7 @@ class ReadBufferFromRemoteFSGather final : public ReadBuffer friend class ReadIndirectBufferFromRemoteFS; public: - using ReadBufferCreator = std::function(const std::string & path, size_t read_until_position)>; + using ReadBufferCreator = std::function(const std::string & path, size_t read_until_position)>; ReadBufferFromRemoteFSGather( ReadBufferCreator && read_buffer_creator_, diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index ad938e345bb..db0bbe034fd 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -89,7 +89,7 @@ std::unique_ptr AzureObjectStorage::readObjects( /// NOL auto read_buffer_creator = [this, settings_ptr, disk_read_settings] - (const std::string & path, size_t read_until_position) -> std::shared_ptr + (const std::string & path, size_t read_until_position) -> std::unique_ptr { return std::make_unique( client.get(), diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp index 108246c7dd6..58edbbef253 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp @@ -133,39 +133,7 @@ std::unique_ptr CachedObjectStorage::readObject( /// NOL std::optional read_hint, std::optional file_size) const { - /// Add cache relating settings to ReadSettings. - auto modified_read_settings = patchSettings(read_settings); - auto implementation_buffer = object_storage->readObject(object, read_settings, read_hint, file_size); - - /// If underlying read buffer does caching on its own, do not wrap it in caching buffer. - if (implementation_buffer->isIntegratedWithFilesystemCache() - && modified_read_settings.enable_filesystem_cache_on_lower_level) - { - return implementation_buffer; - } - else - { - if (!file_size) - file_size = implementation_buffer->getFileSize(); - - auto implementation_buffer_creator = [object, read_settings, read_hint, file_size, this]() - { - return std::make_unique(object_storage->readObject(object, read_settings, read_hint, file_size)); - }; - - FileCache::Key key = getCacheKey(object.getPathKeyForCache()); - LOG_TEST(log, "Reading from file `{}` with cache key `{}`", object.absolute_path, key.toString()); - return std::make_unique( - object.absolute_path, - key, - cache, - implementation_buffer_creator, - read_settings, - CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() ? std::string(CurrentThread::getQueryId()) : "", - file_size.value(), - /* allow_seeks */true, - /* use_external_buffer */false); - } + return object_storage->readObject(object, patchSettings(read_settings), read_hint, file_size); } diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp index 1d6c6f92280..7e3dfe738da 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp @@ -64,7 +64,7 @@ std::unique_ptr HDFSObjectStorage::readObjects( /// NOLI auto disk_read_settings = patchSettings(read_settings); auto read_buffer_creator = [this, disk_read_settings] - (const std::string & path, size_t /* read_until_position */) -> std::shared_ptr + (const std::string & path, size_t /* read_until_position */) -> std::unique_ptr { size_t begin_of_path = path.find('/', path.find("//") + 2); auto hdfs_path = path.substr(begin_of_path); diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index 7f705264721..160e2d77886 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -9,8 +10,8 @@ #include #include #include +#include #include -#include #include namespace fs = std::filesystem; @@ -55,14 +56,9 @@ std::unique_ptr LocalObjectStorage::readObjects( /// NOL auto modified_settings = patchSettings(read_settings); auto read_buffer_creator = [=] (const std::string & file_path, size_t /* read_until_position */) - -> std::shared_ptr + -> std::unique_ptr { - auto impl = createReadBufferFromFileBase(file_path, modified_settings, read_hint, file_size); - if (modified_settings.enable_filesystem_cache) - { - return std::make_unique(std::move(impl)); - } - return impl; + return createReadBufferFromFileBase(file_path, modified_settings, read_hint, file_size); }; auto impl = std::make_unique( @@ -76,7 +72,8 @@ std::unique_ptr LocalObjectStorage::readObjects( /// NOL else { auto buf = std::make_unique(std::move(impl), modified_settings); - return std::make_unique(std::move(buf), modified_settings.remote_read_min_bytes_for_seek); + return std::make_unique( + std::move(buf), modified_settings.remote_read_min_bytes_for_seek); } } @@ -132,10 +129,13 @@ std::unique_ptr LocalObjectStorage::writeObject( /// NO size_t buf_size, const WriteSettings & /* write_settings */) { - int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1; + if (mode != WriteMode::Rewrite) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "LocalObjectStorage doesn't support append to files"); + LOG_TEST(log, "Write object: {}", object.absolute_path); - auto impl = std::make_unique(object.absolute_path, buf_size, flags); - return std::make_unique(std::move(impl), std::move(finalize_callback), object.absolute_path); + auto impl = std::make_unique(object.absolute_path, buf_size); + return std::make_unique( + std::move(impl), std::move(finalize_callback), object.absolute_path); } void LocalObjectStorage::removeObject(const StoredObject & object) @@ -174,16 +174,17 @@ ObjectMetadata LocalObjectStorage::getObjectMetadata(const std::string & /* path void LocalObjectStorage::copyObject( // NOLINT const StoredObject & object_from, const StoredObject & object_to, std::optional /* object_to_attributes */) { - fs::path to = object_to.absolute_path; - fs::path from = object_from.absolute_path; - - /// Same logic as in DiskLocal. - if (object_from.absolute_path.ends_with('/')) - from = from.parent_path(); - if (fs::is_directory(from)) - to /= from.filename(); - - fs::copy(from, to, fs::copy_options::recursive | fs::copy_options::overwrite_existing); + if (Context::getGlobalContextInstance()->getSettingsRef().local_object_storage_has_copy) + { + fs::copy(object_from.absolute_path, object_to.absolute_path); + } + else + { + auto in = readObject(object_from); + auto out = writeObject(object_to, WriteMode::Rewrite); + copyData(*in, *out); + out->finalize(); + } } void LocalObjectStorage::shutdown() diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index de7b88cef45..4dee5feecb4 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -123,9 +123,9 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT auto read_buffer_creator = [this, settings_ptr, disk_read_settings] - (const std::string & path, size_t read_until_position) -> std::shared_ptr + (const std::string & path, size_t read_until_position) -> std::unique_ptr { - return std::make_shared( + return std::make_unique( client.get(), bucket, path, diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp index 4365156d93d..f80caf080e7 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp @@ -169,9 +169,9 @@ std::unique_ptr WebObjectStorage::readObject( /// NOLINT { auto read_buffer_creator = [this, read_settings] - (const std::string & path_, size_t read_until_position) -> std::shared_ptr + (const std::string & path_, size_t read_until_position) -> std::unique_ptr { - return std::make_shared( + return std::make_unique( fs::path(url) / path_, getContext(), read_settings, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index c698af698c6..d7f160b4793 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -645,9 +645,9 @@ std::unique_ptr StorageS3Source::createAsyncS3ReadBuffer( { auto read_buffer_creator = [this, read_settings] - (const std::string & path, size_t read_until_position) -> std::shared_ptr + (const std::string & path, size_t read_until_position) -> std::unique_ptr { - return std::make_shared( + return std::make_unique( client, bucket, path, From 86ca6c4f4465eb1a3cf011912132b03813d6e943 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 17 Apr 2023 16:44:49 +0200 Subject: [PATCH 023/308] Commit forgotten file --- src/Core/Settings.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index c47432ae14a..2edbe93bb07 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -664,6 +664,8 @@ class IColumn; M(Bool, allow_prefetched_read_pool_for_remote_filesystem, false, "Prefer prefethed threadpool if all parts are on remote filesystem", 0) \ M(Bool, allow_prefetched_read_pool_for_local_filesystem, false, "Prefer prefethed threadpool if all parts are on remote filesystem", 0) \ \ + M(Bool, local_object_storage_has_copy, true, "Allow to use copy command for LocalObjectStorage", 0) \ + \ M(UInt64, filesystem_prefetch_step_bytes, 0, "Prefetch step in bytes. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \ M(UInt64, filesystem_prefetch_step_marks, 0, "Prefetch step in marks. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \ M(UInt64, filesystem_prefetch_min_bytes_for_single_read_task, "8Mi", "Do not parallelize within one file read less than this amount of bytes. E.g. one reader will not receive a read task of size less than this amount. This setting is recommended to avoid spikes of time for aws getObject requests to aws", 0) \ From 6f501f983ed0f9d63df4abdbbe25afe64283776b Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 17 Apr 2023 17:47:03 +0200 Subject: [PATCH 024/308] Update test.py --- tests/integration/test_merges_memory_limit/test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/test_merges_memory_limit/test.py b/tests/integration/test_merges_memory_limit/test.py index 04729f3a01c..22bf143f4f7 100644 --- a/tests/integration/test_merges_memory_limit/test.py +++ b/tests/integration/test_merges_memory_limit/test.py @@ -27,9 +27,6 @@ def test_memory_limit_success(): node.query( "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)" ) - node.query( - "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(10000)" - ) _, error = node.query_and_get_answer_with_error( "SYSTEM START MERGES test_merge_oom;SET optimize_throw_if_noop=1;OPTIMIZE TABLE test_merge_oom FINAL" ) From a470deafbc02ec3fa7303f294114a663420ef14c Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 17 Apr 2023 19:11:42 +0200 Subject: [PATCH 025/308] Fix style check --- src/Common/ErrorCodes.cpp | 1 - src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 1 - src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp | 5 +---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 9abf3bba8ff..bbd37c3fefc 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -634,7 +634,6 @@ M(663, INCONSISTENT_METADATA_FOR_BACKUP) \ M(664, ACCESS_STORAGE_DOESNT_ALLOW_BACKUP) \ M(665, CANNOT_CONNECT_NATS) \ - M(666, CANNOT_USE_CACHE) \ M(667, NOT_INITIALIZED) \ M(668, INVALID_STATE) \ M(669, NAMED_COLLECTION_DOESNT_EXIST) \ diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index e7a2c3e1008..81385fd5f8a 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -33,7 +33,6 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_SEEK_THROUGH_FILE; - extern const int CANNOT_USE_CACHE; extern const int LOGICAL_ERROR; extern const int ARGUMENT_OUT_OF_BOUND; } diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index 160e2d77886..09630b6bc7d 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -22,10 +22,7 @@ namespace DB namespace ErrorCodes { extern const int NOT_IMPLEMENTED; -} - -namespace ErrorCodes -{ + extern const int BAD_ARGUMENTS; extern const int CANNOT_UNLINK; } From 0669306d15d4f9c5fd91c5b60da9c9f30181a0c7 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 18 Apr 2023 17:50:54 +0200 Subject: [PATCH 026/308] Fix --- src/Disks/ObjectStorages/IObjectStorage.h | 2 -- .../Local/LocalObjectStorage.cpp | 9 +++---- .../ObjectStorages/Local/LocalObjectStorage.h | 4 --- .../ObjectStorages/Web/WebObjectStorage.h | 2 -- .../02226_filesystem_cache_profile_events.sh | 26 +++++++++---------- 5 files changed, 16 insertions(+), 27 deletions(-) diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index cb9eebe34fb..94899eea3d5 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -189,8 +189,6 @@ public: /// Get unique id for passed absolute path in object storage. virtual std::string getUniqueId(const std::string & path) const { return path; } - virtual bool supportsAppend() const { return false; } - /// Remove filesystem cache. `path` is a result of object.getPathKeyForCache() method, /// which is used to define a cache key for the source object path. virtual void removeCacheIfExists(const std::string & /* path */) {} diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index 09630b6bc7d..70039419ad1 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -61,7 +61,9 @@ std::unique_ptr LocalObjectStorage::readObjects( /// NOL auto impl = std::make_unique( std::move(read_buffer_creator), objects, modified_settings); - if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) + /// We use `remove_fs_method` (not `local_fs_method`) because we are about to use + /// AsynchronousReadIndirectBufferFromRemoteFS which works by the remote_fs_* settings. + if (modified_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { auto & reader = getThreadPoolReader(); return std::make_unique(reader, modified_settings, std::move(impl)); @@ -74,11 +76,6 @@ std::unique_ptr LocalObjectStorage::readObjects( /// NOL } } -std::string LocalObjectStorage::getUniqueId(const std::string & path) const -{ - return toString(getINodeNumberFromPath(path)); -} - ReadSettings LocalObjectStorage::patchSettings(const ReadSettings & read_settings) const { if (!read_settings.enable_filesystem_cache) diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h index a55ed17c416..2d31ccff398 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h @@ -77,10 +77,6 @@ public: const std::string & config_prefix, ContextPtr context) override; - bool supportsAppend() const override { return true; } - - std::string getUniqueId(const std::string & path) const override; - std::string generateBlobNameForPath(const std::string & path) override; bool isRemote() const override { return false; } diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.h b/src/Disks/ObjectStorages/Web/WebObjectStorage.h index 2dab8fdb62d..f55e853b4fa 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.h +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.h @@ -87,8 +87,6 @@ public: const std::string & config_prefix, ContextPtr context) override; - bool supportsAppend() const override { return false; } - std::string generateBlobNameForPath(const std::string & path) override { return path; } bool isRemote() const override { return true; } diff --git a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh index 96e51a58cc4..545de1ac7cb 100755 --- a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh +++ b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh @@ -64,19 +64,6 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do set remote_filesystem_read_method='threadpool'; """ - clickhouse client --multiquery --multiline --query """ - SELECT * FROM test_02226 WHERE value LIKE '%abc%' ORDER BY value LIMIT 10 FORMAT Null; - - SET enable_filesystem_cache_on_write_operations = 1; - - TRUNCATE TABLE test_02226; - SELECT count() FROM test_02226; - - SYSTEM DROP FILESYSTEM CACHE; - - INSERT INTO test_02226 SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000; - """ - query_id=$(clickhouse client --query "select queryID() from ($query) limit 1") clickhouse client --multiquery --multiline --query """ @@ -93,4 +80,17 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do DROP TABLE test_02226; """ + + clickhouse client --multiquery --multiline --query """ + SELECT * FROM test_02226 WHERE value LIKE '%abc%' ORDER BY value LIMIT 10 FORMAT Null; + + SET enable_filesystem_cache_on_write_operations = 1; + + TRUNCATE TABLE test_02226; + SELECT count() FROM test_02226; + + SYSTEM DROP FILESYSTEM CACHE; + + INSERT INTO test_02226 SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000; + """ done From 277c616f787777cb6adab57f4aed6e8b7341e7a0 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 19 Apr 2023 16:57:49 +0200 Subject: [PATCH 027/308] Fix tests --- .../0_stateless/02226_filesystem_cache_profile_events.sh | 4 ++-- tests/queries/0_stateless/02714_local_object_storage.sql | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh index 545de1ac7cb..f071a570243 100755 --- a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh +++ b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh @@ -77,8 +77,6 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do AND current_database = currentDatabase() ORDER BY query_start_time DESC LIMIT 1; - - DROP TABLE test_02226; """ clickhouse client --multiquery --multiline --query """ @@ -93,4 +91,6 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do INSERT INTO test_02226 SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000; """ + + clickhouse client --query "DROP TABLE test_02226" done diff --git a/tests/queries/0_stateless/02714_local_object_storage.sql b/tests/queries/0_stateless/02714_local_object_storage.sql index 6cc7909d237..0375d07899f 100644 --- a/tests/queries/0_stateless/02714_local_object_storage.sql +++ b/tests/queries/0_stateless/02714_local_object_storage.sql @@ -1,3 +1,5 @@ +SET min_bytes_to_use_direct_io='1Gi'; -- It does not work (fixme) + DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) From fef179a9eaef9b4f72856d2a64990aa2e7555d01 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 19 Apr 2023 19:23:50 +0200 Subject: [PATCH 028/308] Remove redundant code --- src/Core/Settings.h | 1 - .../Cached/CachedObjectStorage.cpp | 40 +----------------- .../ObjectStorages/DiskObjectStorage.cpp | 2 +- .../FakeMetadataStorageFromDisk.cpp | 2 +- .../MetadataStorageFromDisk.cpp | 2 +- .../MetadataStorageFromPlainObjectStorage.cpp | 2 +- src/Disks/ObjectStorages/StoredObject.cpp | 41 +++---------------- src/Disks/ObjectStorages/StoredObject.h | 12 ++---- ...etadataStorageFromStaticFilesWebServer.cpp | 2 +- src/IO/ReadSettings.h | 4 -- src/Interpreters/Context.cpp | 1 - 11 files changed, 15 insertions(+), 94 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 9cf2e14ab6f..bb960f13899 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -656,7 +656,6 @@ class IColumn; M(Bool, enable_filesystem_cache_on_write_operations, false, "Write into cache on write operations. To actually work this setting requires be added to disk config too", 0) \ M(Bool, enable_filesystem_cache_log, false, "Allows to record the filesystem caching log for each query", 0) \ M(Bool, read_from_filesystem_cache_if_exists_otherwise_bypass_cache, false, "Allow to use the filesystem cache in passive mode - benefit from the existing cache entries, but don't put more entries into the cache. If you set this setting for heavy ad-hoc queries and leave it disabled for short real-time queries, this will allows to avoid cache threshing by too heavy queries and to improve the overall system efficiency.", 0) \ - M(Bool, enable_filesystem_cache_on_lower_level, true, "If read buffer supports caching inside threadpool, allow it to do it, otherwise cache outside ot threadpool. Do not use this setting, it is needed for testing", 0) \ M(Bool, skip_download_if_exceeds_query_cache, true, "Skip download from remote filesystem if exceeds query cache size", 0) \ M(UInt64, filesystem_cache_max_download_size, (128UL * 1024 * 1024 * 1024), "Max remote filesystem cache size that can be downloaded by a single query", 0) \ M(Bool, throw_on_error_from_cache_on_write_operations, false, "Ignore error from cache when caching on write operations (INSERT, merges)", 0) \ diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp index 3e625f6ccd7..dbcf31931e5 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp @@ -86,45 +86,8 @@ std::unique_ptr CachedObjectStorage::readObjects( /// NO { if (objects.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Received empty list of objects to read"); - assert(!objects[0].getPathKeyForCache().empty()); - - /// Add cache relating settings to ReadSettings. - auto modified_read_settings = patchSettings(read_settings); - auto implementation_buffer = object_storage->readObjects(objects, modified_read_settings, read_hint, file_size); - - /// If underlying read buffer does caching on its own, do not wrap it in caching buffer. - if (implementation_buffer->isIntegratedWithFilesystemCache() - && modified_read_settings.enable_filesystem_cache_on_lower_level) - { - return implementation_buffer; - } - else - { - if (!file_size) - file_size = implementation_buffer->getFileSize(); - - auto implementation_buffer_creator = [objects, modified_read_settings, read_hint, file_size, this]() - { - return std::make_unique( - object_storage->readObjects(objects, modified_read_settings, read_hint, file_size)); - }; - - /// TODO: A test is needed for the case of non-s3 storage and *Log family engines. - std::string path = objects[0].absolute_path; - FileCache::Key key = getCacheKey(objects[0].getPathKeyForCache()); - - return std::make_unique( - path, - key, - cache, - implementation_buffer_creator, - modified_read_settings, - CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() ? std::string(CurrentThread::getQueryId()) : "", - file_size.value(), - /* allow_seeks */true, - /* use_external_buffer */false); - } + return object_storage->readObjects(objects, patchSettings(read_settings), read_hint, file_size); } std::unique_ptr CachedObjectStorage::readObject( /// NOLINT @@ -136,7 +99,6 @@ std::unique_ptr CachedObjectStorage::readObject( /// NOL return object_storage->readObject(object, patchSettings(read_settings), read_hint, file_size); } - std::unique_ptr CachedObjectStorage::writeObject( /// NOLINT const StoredObject & object, WriteMode mode, // Cached doesn't support append, only rewrite diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 0b57b14eb1c..97a563ac562 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -308,7 +308,7 @@ bool DiskObjectStorage::checkUniqueId(const String & id) const return false; } - auto object = StoredObject::create(*object_storage, id, {}, {}, true); + auto object = StoredObject::create(*object_storage, id); return object_storage->exists(object); } diff --git a/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp index b9658821893..66e8f948042 100644 --- a/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp +++ b/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp @@ -105,7 +105,7 @@ StoredObjects FakeMetadataStorageFromDisk::getStorageObjects(const std::string & std::string object_path = fs::path(object_storage_root_path) / blob_name; size_t object_size = getFileSize(path); - auto object = StoredObject::create(*object_storage, object_path, object_size, path, /* exists */true); + auto object = StoredObject::create(*object_storage, object_path, object_size, path); return {std::move(object)}; } diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp index 96c8b3daf04..e0eb79fd819 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp @@ -145,7 +145,7 @@ StoredObjects MetadataStorageFromDisk::getStorageObjects(const std::string & pat for (auto & [object_relative_path, size] : object_storage_relative_paths) { auto object_path = fs::path(metadata->getBlobsCommonPrefix()) / object_relative_path; - StoredObject object{ object_path, size, path, [](const String & path_){ return path_; }}; + StoredObject object{ object_path, size, path, object_path }; object_storage_paths.push_back(object); } diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp index 214252530a5..a70f581afa7 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp @@ -108,7 +108,7 @@ StoredObjects MetadataStorageFromPlainObjectStorage::getStorageObjects(const std { std::string blob_name = object_storage->generateBlobNameForPath(path); size_t object_size = getFileSize(blob_name); - auto object = StoredObject::create(*object_storage, getAbsolutePath(blob_name), object_size, path, /* exists */true); + auto object = StoredObject::create(*object_storage, getAbsolutePath(blob_name), object_size, path); return {std::move(object)}; } diff --git a/src/Disks/ObjectStorages/StoredObject.cpp b/src/Disks/ObjectStorages/StoredObject.cpp index 20f50d7676d..263a5476426 100644 --- a/src/Disks/ObjectStorages/StoredObject.cpp +++ b/src/Disks/ObjectStorages/StoredObject.cpp @@ -12,20 +12,17 @@ StoredObject::StoredObject( const std::string & absolute_path_, uint64_t bytes_size_, const std::string & mapped_path_, - PathKeyForCacheCreator && path_key_for_cache_creator_) + const std::string & cache_path_) : absolute_path(absolute_path_) , mapped_path(mapped_path_) + , cache_path(cache_path_) , bytes_size(bytes_size_) - , path_key_for_cache_creator(std::move(path_key_for_cache_creator_)) { } std::string StoredObject::getPathKeyForCache() const { - if (!path_key_for_cache_creator) - return ""; /// This empty result need to be used with care. - - return path_key_for_cache_creator(absolute_path); + return cache_path; } const std::string & StoredObject::getMappedPath() const @@ -37,36 +34,10 @@ StoredObject StoredObject::create( const IObjectStorage & object_storage, const std::string & object_path, size_t object_size, - const std::string & mapped_path, - bool exists, - bool object_bypasses_cache) + const std::string & mapped_path) { - if (object_bypasses_cache) - return StoredObject(object_path, object_size, mapped_path, {}); - - PathKeyForCacheCreator path_key_for_cache_creator = [&object_storage](const std::string & path) -> std::string - { - try - { - return object_storage.getUniqueId(path); - } - catch (...) - { - LOG_DEBUG( - &Poco::Logger::get("StoredObject"), - "Object does not exist while getting cache path hint (object path: {})", - path); - - return ""; - } - }; - - if (exists) - { - path_key_for_cache_creator = [path = path_key_for_cache_creator(object_path)](const std::string &) { return path; }; - } - - return StoredObject(object_path, object_size, mapped_path, std::move(path_key_for_cache_creator)); + const auto cache_path = object_storage.getUniqueId(object_path); + return StoredObject(object_path, object_size, mapped_path, cache_path); } } diff --git a/src/Disks/ObjectStorages/StoredObject.h b/src/Disks/ObjectStorages/StoredObject.h index 2b6e76eec01..c64093a2729 100644 --- a/src/Disks/ObjectStorages/StoredObject.h +++ b/src/Disks/ObjectStorages/StoredObject.h @@ -15,6 +15,7 @@ struct StoredObject std::string absolute_path; /// A map which is mapped to current blob (for example, a corresponding local path as clickhouse sees it). std::string mapped_path; + std::string cache_path; uint64_t bytes_size = 0; @@ -27,14 +28,7 @@ struct StoredObject const IObjectStorage & object_storage, const std::string & object_path, size_t object_size = 0, - const std::string & mapped_path = "", - bool exists = false, - bool object_bypasses_cache = false); - - /// Optional hint for cache. Use delayed initialization - /// because somecache hint implementation requires it. - using PathKeyForCacheCreator = std::function; - PathKeyForCacheCreator path_key_for_cache_creator; + const std::string & mapped_path = ""); StoredObject() = default; @@ -42,7 +36,7 @@ struct StoredObject const std::string & absolute_path_, uint64_t bytes_size_ = 0, const std::string & mapped_path_ = "", - PathKeyForCacheCreator && path_key_for_cache_creator_ = {}); + const std::string & cache_path_ = ""); }; using StoredObjects = std::vector; diff --git a/src/Disks/ObjectStorages/Web/MetadataStorageFromStaticFilesWebServer.cpp b/src/Disks/ObjectStorages/Web/MetadataStorageFromStaticFilesWebServer.cpp index 69536354aa9..5d2f734c9c9 100644 --- a/src/Disks/ObjectStorages/Web/MetadataStorageFromStaticFilesWebServer.cpp +++ b/src/Disks/ObjectStorages/Web/MetadataStorageFromStaticFilesWebServer.cpp @@ -108,7 +108,7 @@ StoredObjects MetadataStorageFromStaticFilesWebServer::getStorageObjects(const s auto fs_path = fs::path(object_storage.url) / path; std::string remote_path = fs_path.parent_path() / (escapeForFileName(fs_path.stem()) + fs_path.extension().string()); remote_path = remote_path.substr(object_storage.url.size()); - return {StoredObject::create(object_storage, remote_path, object_storage.files.at(path).size, path, true)}; + return {StoredObject::create(object_storage, remote_path, object_storage.files.at(path).size, path)}; } std::vector MetadataStorageFromStaticFilesWebServer::listDirectory(const std::string & path) const diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 7d44b0b7ad1..daddeaf5e07 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -95,10 +95,6 @@ struct ReadSettings bool read_from_filesystem_cache_if_exists_otherwise_bypass_cache = false; bool enable_filesystem_cache_log = false; bool is_file_cache_persistent = false; /// Some files can be made non-evictable. - /// Some buffers which read via thread pool can also do caching in threadpool - /// (instead of caching the result outside of threadpool). By default, if they support it, - /// they will do it. But this behaviour can be changed with this setting. - bool enable_filesystem_cache_on_lower_level = true; size_t filesystem_cache_max_download_size = (128UL * 1024 * 1024 * 1024); bool skip_download_if_exceeds_query_cache = true; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 73d78e84198..86de9459a3c 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4247,7 +4247,6 @@ ReadSettings Context::getReadSettings() const res.enable_filesystem_cache = settings.enable_filesystem_cache; res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache; res.enable_filesystem_cache_log = settings.enable_filesystem_cache_log; - res.enable_filesystem_cache_on_lower_level = settings.enable_filesystem_cache_on_lower_level; res.filesystem_cache_max_download_size = settings.filesystem_cache_max_download_size; res.skip_download_if_exceeds_query_cache = settings.skip_download_if_exceeds_query_cache; From 0b2f693b8c5364be1019960fa162b1b1e9c66439 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 20 Apr 2023 12:29:12 +0200 Subject: [PATCH 029/308] Fix tests --- ...02226_filesystem_cache_profile_events.reference | 6 +++--- .../0_stateless/02344_show_caches.reference | 14 -------------- tests/queries/0_stateless/02344_show_caches.sql | 2 -- .../0_stateless/02714_local_object_storage.sql | 1 + 4 files changed, 4 insertions(+), 19 deletions(-) delete mode 100644 tests/queries/0_stateless/02344_show_caches.reference delete mode 100644 tests/queries/0_stateless/02344_show_caches.sql diff --git a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.reference b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.reference index d895040ef59..2ee0f256949 100644 --- a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.reference +++ b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.reference @@ -1,15 +1,15 @@ Using storage policy: s3_cache 1 0 1 0 1 0 -0 0 1 0 +0 Using storage policy: local_cache 1 0 1 0 1 0 -0 0 1 0 +0 Using storage policy: azure_cache 1 0 1 0 1 0 -0 0 1 0 +0 diff --git a/tests/queries/0_stateless/02344_show_caches.reference b/tests/queries/0_stateless/02344_show_caches.reference deleted file mode 100644 index 2ee4f902ba1..00000000000 --- a/tests/queries/0_stateless/02344_show_caches.reference +++ /dev/null @@ -1,14 +0,0 @@ -cached_azure -s3_cache_2 -s3_cache -s3_cache_3 -s3_cache_multi -s3_cache_4 -s3_cache_5 -s3_cache_small_segment_size -local_cache -s3_cache_6 -s3_cache_small -local_cache_2 -local_cache_3 -s3_cache_multi_2 diff --git a/tests/queries/0_stateless/02344_show_caches.sql b/tests/queries/0_stateless/02344_show_caches.sql deleted file mode 100644 index 56f00b89051..00000000000 --- a/tests/queries/0_stateless/02344_show_caches.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Tags: no-fasttest, no-replicated-database, no-cpu-aarch64 -SHOW FILESYSTEM CACHES; diff --git a/tests/queries/0_stateless/02714_local_object_storage.sql b/tests/queries/0_stateless/02714_local_object_storage.sql index 0375d07899f..986d97ba872 100644 --- a/tests/queries/0_stateless/02714_local_object_storage.sql +++ b/tests/queries/0_stateless/02714_local_object_storage.sql @@ -1,4 +1,5 @@ SET min_bytes_to_use_direct_io='1Gi'; -- It does not work (fixme) +SET local_filesystem_read_method='pread'; -- ui_uring local_fs_method does not work here (fixme) DROP TABLE IF EXISTS test; From ae100defa279c8e3343482551041758f6a7c925c Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 20 Apr 2023 15:51:10 +0000 Subject: [PATCH 030/308] Add Array data type to MongoDB --- src/Processors/Sources/MongoDBSource.cpp | 204 +++++++++++++++++- src/Processors/Sources/MongoDBSource.h | 9 + .../integration/test_storage_mongodb/test.py | 75 +++++++ 3 files changed, 284 insertions(+), 4 deletions(-) diff --git a/src/Processors/Sources/MongoDBSource.cpp b/src/Processors/Sources/MongoDBSource.cpp index a8bfefdf8a6..8ebedc3e877 100644 --- a/src/Processors/Sources/MongoDBSource.cpp +++ b/src/Processors/Sources/MongoDBSource.cpp @@ -6,7 +6,9 @@ #include #include #include +#include +#include #include #include #include @@ -17,6 +19,9 @@ #include #include +#include +#include + // only after poco // naming conflict: // Poco/MongoDB/BSONWriter.h:54: void writeCString(const std::string & value); @@ -33,6 +38,11 @@ namespace ErrorCodes extern const int MONGODB_ERROR; } +namespace +{ + void prepareMongoDBArrayInfo( + std::unordered_map & array_info, size_t column_idx, const DataTypePtr data_type); +} std::unique_ptr createCursor(const std::string & database, const std::string & collection, const Block & sample_block_to_select) { @@ -58,6 +68,10 @@ MongoDBSource::MongoDBSource( , max_block_size{max_block_size_} { description.init(sample_block); + + for (const auto idx : collections::range(0, description.sample_block.columns())) + if (description.types[idx].first == ExternalResultDescription::ValueType::vtArray) + prepareMongoDBArrayInfo(array_info, idx, description.sample_block.getByPosition(idx).type); } @@ -68,6 +82,7 @@ namespace { using ValueType = ExternalResultDescription::ValueType; using ObjectId = Poco::MongoDB::ObjectId; + using MongoArray = Poco::MongoDB::Array; template void insertNumber(IColumn & column, const Poco::MongoDB::Element & value, const std::string & name) @@ -103,7 +118,129 @@ namespace } } - void insertValue(IColumn & column, const ValueType type, const Poco::MongoDB::Element & value, const std::string & name) + template + Field getNumber(const Poco::MongoDB::Element & value, const std::string & name) + { + switch (value.type()) + { + case Poco::MongoDB::ElementTraits::TypeId: + return static_cast(static_cast &>(value).value()); + case Poco::MongoDB::ElementTraits::TypeId: + return static_cast(static_cast &>(value).value()); + case Poco::MongoDB::ElementTraits::TypeId: + return static_cast(static_cast &>(value).value()); + case Poco::MongoDB::ElementTraits::TypeId: + return static_cast(static_cast &>(value).value()); + case Poco::MongoDB::ElementTraits::TypeId: + return Field(); + case Poco::MongoDB::ElementTraits::TypeId: + return parse(static_cast &>(value).value()); + default: + throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected a number, got type id = {} for column {}", + toString(value.type()), name); + } + } + + void prepareMongoDBArrayInfo( + std::unordered_map & array_info, size_t column_idx, const DataTypePtr data_type) + { + const auto * array_type = typeid_cast(data_type.get()); + auto nested = array_type->getNestedType(); + + size_t count_dimensions = 1; + while (isArray(nested)) + { + ++count_dimensions; + nested = typeid_cast(nested.get())->getNestedType(); + } + + Field default_value = nested->getDefault(); + if (nested->isNullable()) + nested = static_cast(nested.get())->getNestedType(); + + WhichDataType which(nested); + std::function parser; + + if (which.isUInt8()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isUInt16()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isUInt32()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isUInt64()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isInt8()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isInt16()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isInt32()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isInt64()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isFloat32()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isFloat64()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field { return getNumber(value, name); }; + else if (which.isString() || which.isFixedString()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field + { + if (value.type() == Poco::MongoDB::ElementTraits::TypeId) + { + String string_id = value.toString(); + return Field(string_id.data(), string_id.size()); + } + else if (value.type() == Poco::MongoDB::ElementTraits::TypeId) + { + String string = static_cast &>(value).value(); + return Field(string.data(), string.size()); + } + + throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String, got type id = {} for column {}", + toString(value.type()), name); + }; + else if (which.isDate()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field + { + if (value.type() != Poco::MongoDB::ElementTraits::TypeId) + throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected Timestamp, got type id = {} for column {}", + toString(value.type()), name); + + return static_cast(DateLUT::instance().toDayNum( + static_cast &>(value).value().epochTime())); + }; + else if (which.isDateTime()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field + { + if (value.type() != Poco::MongoDB::ElementTraits::TypeId) + throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected Timestamp, got type id = {} for column {}", + toString(value.type()), name); + + return static_cast(static_cast &>(value).value().epochTime()); + }; + else if (which.isUUID()) + parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field + { + if (value.type() != Poco::MongoDB::ElementTraits::TypeId) + throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String (UUID), got type id = {} for column {}", + toString(value.type()), name); + + String string = static_cast &>(value).value(); + return parse(string); + }; + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Type conversion to {} is not supported", nested->getName()); + + array_info[column_idx] = {count_dimensions, default_value, parser}; + + } + + void insertValue( + IColumn & column, + const ValueType type, + const Poco::MongoDB::Element & value, + const std::string & name, + std::unordered_map & array_info, + size_t idx) { switch (type) { @@ -192,8 +329,67 @@ namespace toString(value.type()), name); break; } + case ValueType::vtArray: + { + if (value.type() != Poco::MongoDB::ElementTraits::TypeId) + throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected Array, got type id = {} for column {}", + toString(value.type()), name); + + size_t max_dimension = 0, expected_dimensions = array_info[idx].num_dimensions; + const auto parse_value = array_info[idx].parser; + std::vector dimensions(expected_dimensions + 1); + + auto array = static_cast &>(value).value(); + + std::vector> arrays; + arrays.emplace_back(&value, 0); + + while (!arrays.empty()) + { + size_t dimension = arrays.size(); + max_dimension = std::max(max_dimension, dimension); + + auto [element, i] = arrays.back(); + + auto parent = static_cast &>(*element).value(); + + if (i >= parent->size()) + { + dimensions[dimension].emplace_back(Array(dimensions[dimension + 1].begin(), dimensions[dimension + 1].end())); + dimensions[dimension + 1].clear(); + + arrays.pop_back(); + continue; + } + + Poco::MongoDB::Element::Ptr child = parent->get(static_cast(i)); + + if (child->type() == Poco::MongoDB::ElementTraits::TypeId) + { + if (dimension + 1 > expected_dimensions) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Got more dimensions than expected"); + + arrays.back().second += 1; + arrays.emplace_back(child.get(), 0); + } + else + { + dimensions[dimension].emplace_back(parse_value(*child, name)); + } + } + + if (max_dimension < expected_dimensions) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Got less dimensions than expected. ({} instead of {})", max_dimension, expected_dimensions); + + // TODO: default value + + assert_cast(column).insert(Array(dimensions[1].begin(), dimensions[1].end())); + break; + + } default: - throw Exception(ErrorCodes::UNKNOWN_TYPE, "Value of unsupported type:{}", column.getName()); + throw Exception(ErrorCodes::UNKNOWN_TYPE, "Value of unsupported type: {}", column.getName()); } } @@ -252,11 +448,11 @@ Chunk MongoDBSource::generate() if (is_nullable) { ColumnNullable & column_nullable = assert_cast(*columns[idx]); - insertValue(column_nullable.getNestedColumn(), description.types[idx].first, *value, name); + insertValue(column_nullable.getNestedColumn(), description.types[idx].first, *value, name, array_info, idx); column_nullable.getNullMapData().emplace_back(0); } else - insertValue(*columns[idx], description.types[idx].first, *value, name); + insertValue(*columns[idx], description.types[idx].first, *value, name, array_info, idx); } } } diff --git a/src/Processors/Sources/MongoDBSource.h b/src/Processors/Sources/MongoDBSource.h index d03a7a45477..ec73f00f378 100644 --- a/src/Processors/Sources/MongoDBSource.h +++ b/src/Processors/Sources/MongoDBSource.h @@ -19,6 +19,13 @@ namespace MongoDB namespace DB { +struct MongoDBArrayInfo +{ + size_t num_dimensions; + Field default_value; + std::function parser; +}; + void authenticate(Poco::MongoDB::Connection & connection, const std::string & database, const std::string & user, const std::string & password); std::unique_ptr createCursor(const std::string & database, const std::string & collection, const Block & sample_block_to_select); @@ -45,6 +52,8 @@ private: const UInt64 max_block_size; ExternalResultDescription description; bool all_read = false; + + std::unordered_map array_info; }; } diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py index 74b2b15fda0..cf843ddd489 100644 --- a/tests/integration/test_storage_mongodb/test.py +++ b/tests/integration/test_storage_mongodb/test.py @@ -70,6 +70,81 @@ def test_simple_select(started_cluster): simple_mongo_table.drop() +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_arrays(started_cluster): + mongo_connection = get_mongo_connection(started_cluster) + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] + data = [] + for i in range(0, 100): + data.append({ + "key": i, + "arr_int64": [- (i + 1), - (i + 2), - (i + 3)], + "arr_int32": [- (i + 1), - (i + 2), - (i + 3)], + "arr_int16": [- (i + 1), - (i + 2), - (i + 3)], + "arr_int8": [- (i + 1), - (i + 2), - (i + 3)], + "arr_uint64": [i + 1, i + 2, i + 3], + "arr_uint32": [i + 1, i + 2, i + 3], + "arr_uint16": [i + 1, i + 2, i + 3], + "arr_uint8": [i + 1, i + 2, i + 3], + "arr_float32": [i + 1.125, i + 2.5, i + 3.750], + "arr_float64": [i + 1.125, i + 2.5, i + 3.750], + "arr_date": ['2023-11-01', '2023-06-19'], + "arr_datetime": ['2023-03-31 06:03:12', '2023-02-01 12:46:34'], + "arr_string": [str(i + 1), str(i + 2), str(i + 3)], + "arr_uuid": ['f0e77736-91d1-48ce-8f01-15123ca1c7ed', '93376a07-c044-4281-a76e-ad27cf6973c5'], + "arr_arr_bool": [[True, False, True]] + }) + + simple_mongo_table.insert_many(data) + + node = started_cluster.instances["node"] + node.query( + "CREATE TABLE simple_mongo_table(" + "key UInt64," + "arr_int64 Array(Int64)," + "arr_int32 Array(Int32)," + "arr_int16 Array(Int16)," + "arr_int8 Array(Int8)," + "arr_uint64 Array(UInt64)," + "arr_uint32 Array(UInt32)," + "arr_uint16 Array(UInt16)," + "arr_uint8 Array(UInt8)," + "arr_float32 Array(Float32)," + "arr_float64 Array(Float64)," + "arr_date Array(Date)," + "arr_datetime Array(DateTime)," + "arr_string Array(String)," + "arr_uuid Array(UUID)," + "arr_arr_bool Array(Array(Bool))" + ") ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')" + ) + + assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n" + + for column_name in ["arr_int64", "arr_int32", "arr_int16", "arr_int8"]: + assert ( + node.query(f"SELECT {column_name} from simple_mongo_table where key = 42") + == "[-43,-44,-45]\n" + ) + + for column_name in ["arr_uint64", "arr_uint32", "arr_uint16", "arr_uint8"]: + assert ( + node.query(f"SELECT {column_name} from simple_mongo_table where key = 42") + == "[43,44,45]\n" + ) + + for column_name in ["arr_float32", "arr_float64"]: + assert ( + node.query(f"SELECT {column_name} from simple_mongo_table where key = 42") + == "[43,44,45]\n" + ) + + node.query("DROP TABLE simple_mongo_table") + simple_mongo_table.drop() + + @pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) def test_complex_data_type(started_cluster): mongo_connection = get_mongo_connection(started_cluster) From 88f412ad23b87aa39bd200df2113320e92d9a63e Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 21 Apr 2023 12:44:08 +0200 Subject: [PATCH 031/308] Fix upgrade check, fix integration test --- docker/test/upgrade/run.sh | 6 ++++++ .../configs/config.d/storage_configuration.xml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index fce90ca2537..abb4a87a3fa 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -69,6 +69,12 @@ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ > /etc/clickhouse-server/config.d/keeper_port.xml.tmp sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml +# local_blob_storage disk type does not exist in older versions +sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ + | sed "s|local_blob_storage|local|" \ + > /etc/clickhouse-server/config.d/storage_conf.xml.tmp +sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml + # But we still need default disk because some tables loaded only into it sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \ | sed "s|
s3
|
s3
default|" \ diff --git a/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml b/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml index acf0f765c6c..b527c74e8de 100644 --- a/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml @@ -2,7 +2,7 @@ - local + local_blob_storage /local_disk/ From 071e9a6bc062b9e1a23f71f93dc41132ae094c91 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 21 Apr 2023 12:50:17 +0200 Subject: [PATCH 032/308] Minor change --- src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 4c5d62e36ab..7989d7889d1 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -192,10 +192,10 @@ CachedOnDiskReadBufferFromFile::getRemoteFSReadBuffer(FileSegment & file_segment if (!remote_fs_segment_reader) { auto impl = implementation_buffer_creator(); - if (!impl->supportsRightBoundedReads()) - remote_fs_segment_reader = std::make_unique(std::move(impl)); - else + if (impl->supportsRightBoundedReads()) remote_fs_segment_reader = std::move(impl); + else + remote_fs_segment_reader = std::make_unique(std::move(impl)); file_segment.setRemoteFileReader(remote_fs_segment_reader); } From cbc4ba67370a852153dae6283fd09f71ba11177b Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 21 Apr 2023 16:50:41 +0000 Subject: [PATCH 033/308] Add sleep --- tests/integration/test_async_insert_memory/test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_async_insert_memory/test.py b/tests/integration/test_async_insert_memory/test.py index 279542f087c..97534e09fa6 100644 --- a/tests/integration/test_async_insert_memory/test.py +++ b/tests/integration/test_async_insert_memory/test.py @@ -1,3 +1,5 @@ +import time + import pytest from helpers.cluster import ClickHouseCluster @@ -28,6 +30,9 @@ def test_memory_usage(): values = list(range(iter * 5000000, (iter + 1) * 5000000)) node.query(INSERT_QUERY.format(values)) + # Wait until buffers are freed + time.sleep(5) + response = node.get_query_request( "SELECT groupArray(number) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format( 30 * (2**23) From 370aa60159d323c9775ee4df29984ec48a077576 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 21 Apr 2023 22:10:06 +0200 Subject: [PATCH 034/308] Update run.sh --- docker/test/upgrade/run.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index abb4a87a3fa..493ddddaf2e 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -70,10 +70,7 @@ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml # local_blob_storage disk type does not exist in older versions -sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ - | sed "s|local_blob_storage|local|" \ - > /etc/clickhouse-server/config.d/storage_conf.xml.tmp -sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml +sudo sed -i "s|local_blob_storage|local|" /etc/clickhouse-server/config.d/storage_conf.xml # But we still need default disk because some tables loaded only into it sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \ From 93563ad6be3eb0d06ae4fab74e5dee4eeaeea9ab Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Sat, 22 Apr 2023 17:08:28 +0200 Subject: [PATCH 035/308] Update 02714_local_object_storage.sql --- tests/queries/0_stateless/02714_local_object_storage.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02714_local_object_storage.sql b/tests/queries/0_stateless/02714_local_object_storage.sql index 986d97ba872..fa9025b8b6e 100644 --- a/tests/queries/0_stateless/02714_local_object_storage.sql +++ b/tests/queries/0_stateless/02714_local_object_storage.sql @@ -24,3 +24,5 @@ SETTINGS disk = disk( INSERT INTO test SELECT 1, 'test'; SELECT * FROM test; + +DROP TABLE test SYNC; From f38dfc920696dfa62024a51661e6086444ef43b9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 23 Apr 2023 12:49:22 +0200 Subject: [PATCH 036/308] Update run.sh --- docker/test/upgrade/run.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index 493ddddaf2e..313feb2fc12 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -59,6 +59,12 @@ install_packages previous_release_package_folder # available for dump via clickhouse-local configure +# local_blob_storage disk type does not exist in older versions +sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ + | sed "s|local_blob_storage|local|" \ + > /etc/clickhouse-server/config.d/storage_conf.xml.tmp +sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml + start stop mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.initial.log @@ -69,9 +75,6 @@ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ > /etc/clickhouse-server/config.d/keeper_port.xml.tmp sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml -# local_blob_storage disk type does not exist in older versions -sudo sed -i "s|local_blob_storage|local|" /etc/clickhouse-server/config.d/storage_conf.xml - # But we still need default disk because some tables loaded only into it sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \ | sed "s|
s3
|
s3
default|" \ From 9e1e94648e4849d15785e00b34fdbf388b639686 Mon Sep 17 00:00:00 2001 From: Larry Luo Date: Sun, 23 Apr 2023 21:59:51 -0700 Subject: [PATCH 037/308] support saving interval types to tables --- .../data-types/special-data-types/interval.md | 7 +++---- src/DataTypes/DataTypeInterval.h | 4 ---- .../0_stateless/02724_persist_interval_type.reference | 4 ++++ .../queries/0_stateless/02724_persist_interval_type.sql | 9 +++++++++ 4 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02724_persist_interval_type.reference create mode 100644 tests/queries/0_stateless/02724_persist_interval_type.sql diff --git a/docs/en/sql-reference/data-types/special-data-types/interval.md b/docs/en/sql-reference/data-types/special-data-types/interval.md index c89c2e78752..bedbcf0bd28 100644 --- a/docs/en/sql-reference/data-types/special-data-types/interval.md +++ b/docs/en/sql-reference/data-types/special-data-types/interval.md @@ -8,10 +8,6 @@ sidebar_label: Interval The family of data types representing time and date intervals. The resulting types of the [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator. -:::note -`Interval` data type values can’t be stored in tables. -::: - Structure: - Time interval as an unsigned integer value. @@ -19,6 +15,9 @@ Structure: Supported interval types: +- `NANOSECOND` +- `MICROSECOND` +- `MILLISECOND` - `SECOND` - `MINUTE` - `HOUR` diff --git a/src/DataTypes/DataTypeInterval.h b/src/DataTypes/DataTypeInterval.h index 83d89a73460..05abe1d9b24 100644 --- a/src/DataTypes/DataTypeInterval.h +++ b/src/DataTypes/DataTypeInterval.h @@ -11,9 +11,6 @@ namespace DB * * Mostly the same as Int64. * But also tagged with interval kind. - * - * Intended usage is for temporary elements in expressions, - * not for storing values in tables. */ class DataTypeInterval final : public DataTypeNumberBase { @@ -34,7 +31,6 @@ public: bool equals(const IDataType & rhs) const override; bool isParametric() const override { return true; } - bool cannotBeStoredInTables() const override { return true; } bool isCategorial() const override { return false; } bool canBeInsideNullable() const override { return true; } }; diff --git a/tests/queries/0_stateless/02724_persist_interval_type.reference b/tests/queries/0_stateless/02724_persist_interval_type.reference new file mode 100644 index 00000000000..9139216d644 --- /dev/null +++ b/tests/queries/0_stateless/02724_persist_interval_type.reference @@ -0,0 +1,4 @@ +2023-01-01 00:00:01.000000001 2023-01-01 02:00:00.000000001 2023-01-01 00:00:00.000000004 1 2 0 +2023-01-01 00:00:02.000000001 2023-01-01 03:00:00.000000001 2023-01-01 00:00:00.000000005 2 3 0 +2023-01-01 00:00:01.000000001 2023-01-01 02:00:00.000000001 2023-01-01 00:00:00.000000004 1 2 0 +2023-01-01 00:00:02.000000001 2023-01-01 03:00:00.000000001 2023-01-01 00:00:00.000000005 2 3 0 diff --git a/tests/queries/0_stateless/02724_persist_interval_type.sql b/tests/queries/0_stateless/02724_persist_interval_type.sql new file mode 100644 index 00000000000..72133aceb00 --- /dev/null +++ b/tests/queries/0_stateless/02724_persist_interval_type.sql @@ -0,0 +1,9 @@ + +DROP TABLE IF EXISTS saved_intervals_tmp; +DROP TABLE IF EXISTS saved_intervals_mgt; +create table saved_intervals_tmp Engine=Memory as SELECT number as EventID, toIntervalSecond(number+1) as v1, toIntervalHour(number+2) as v2, toIntervalNanosecond(number+3) as v3 from numbers(2); +create table saved_intervals_mgt Engine=MergeTree() ORDER BY EventID as SELECT number as EventID, toIntervalSecond(number+1) as v1, toIntervalHour(number+2) as v2, toIntervalNanosecond(number+3) as v3 from numbers(2); +with toDateTime64('2023-01-01 00:00:00.000000001', 9, 'US/Eastern') as c select c+v1 as c_v1, c+v2 as c_v2, c+v3 as c_v3, date_diff(second, c, c_v1), date_diff(hour, c, c_v2), date_diff(second, c, c_v3) from saved_intervals_tmp; +with toDateTime64('2023-01-01 00:00:00.000000001', 9, 'US/Eastern') as c select c+v1 as c_v1, c+v2 as c_v2, c+v3 as c_v3, date_diff(second, c, c_v1), date_diff(hour, c, c_v2), date_diff(second, c, c_v3) from saved_intervals_mgt; +DROP TABLE IF EXISTS saved_intervals_tmp; +DROP TABLE IF EXISTS saved_intervals_mgt; \ No newline at end of file From 65e63235ec953df55698abec692eeb7560b0e92d Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 24 Apr 2023 10:41:43 +0000 Subject: [PATCH 038/308] Custom default window frame for ntile function --- src/Processors/Transforms/WindowTransform.cpp | 54 +++++++++++-------- .../0_stateless/02560_window_ntile.reference | 39 ++++++++++---- .../0_stateless/02560_window_ntile.sql | 21 ++++---- 3 files changed, 74 insertions(+), 40 deletions(-) diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 04a1f12f30a..d2cce2cb890 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -44,6 +44,8 @@ public: // Must insert the result for current_row. virtual void windowInsertResultInto(const WindowTransform * transform, size_t function_index) = 0; + + virtual std::optional getDefaultFrame() const { return {}; } }; // Compares ORDER BY column values at given rows to find the boundaries of frame: @@ -222,6 +224,15 @@ WindowTransform::WindowTransform(const Block & input_header_, /// Currently we have slightly wrong mixup of the interfaces of Window and Aggregate functions. workspace.window_function_impl = dynamic_cast(const_cast(aggregate_function.get())); + /// Some functions may have non-standard default frame. + /// Use it if it's the only function over the current window. + if (window_description.frame.is_default && functions.size() == 1 && workspace.window_function_impl) + { + auto custom_default_frame = workspace.window_function_impl->getDefaultFrame(); + if (custom_default_frame) + window_description.frame = *custom_default_frame; + } + workspace.aggregate_function_state.reset( aggregate_function->sizeOfData(), aggregate_function->alignOfData()); @@ -1977,18 +1988,23 @@ struct WindowFunctionNtile final : public WindowFunction : WindowFunction(name_, argument_types_, parameters_, std::make_shared()) { if (argument_types.size() != 1) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} takes exactly one parameter", name_); - } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} takes exactly one argument", name_); + auto type_id = argument_types[0]->getTypeId(); if (type_id != TypeIndex::UInt8 && type_id != TypeIndex::UInt16 && type_id != TypeIndex::UInt32 && type_id != TypeIndex::UInt64) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "ntile's argument type must be an unsigned integer (not larger then 64-bit), but got {}", argument_types[0]->getName()); - } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "'{}' argument type must be an unsigned integer (not larger than 64-bit), got {}", name_, argument_types[0]->getName()); } bool allocatesMemoryInArena() const override { return false; } + std::optional getDefaultFrame() const override + { + WindowFrame frame; + frame.type = WindowFrame::FrameType::ROWS; + frame.end_type = WindowFrame::BoundaryType::Unbounded; + return frame; + } + void windowInsertResultInto(const WindowTransform * transform, size_t function_index) override { @@ -1999,7 +2015,7 @@ struct WindowFunctionNtile final : public WindowFunction const auto & workspace = transform->workspaces[function_index]; const auto & arg_col = *current_block.original_input_columns[workspace.argument_column_indices[0]]; if (!isColumnConst(arg_col)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "ntile's argument must be a constant"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument of 'ntile' function must be a constant"); auto type_id = argument_types[0]->getTypeId(); if (type_id == TypeIndex::UInt8) buckets = arg_col[transform->current_row.row].get(); @@ -2012,7 +2028,7 @@ struct WindowFunctionNtile final : public WindowFunction if (!buckets) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "ntile's argument must > 0"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument of 'ntile' funtcion must be greater than zero"); } } // new partition @@ -2090,22 +2106,16 @@ private: static void checkWindowFrameType(const WindowTransform * transform) { if (transform->order_by_indices.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "ntile's window frame must have order by clause"); - if (transform->window_description.frame.type != WindowFrame::FrameType::ROWS) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "ntile's frame type must be ROWS"); - } - if (transform->window_description.frame.begin_type != WindowFrame::BoundaryType::Unbounded) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "ntile's frame start type must be UNBOUNDED PRECEDING"); - } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Window frame for 'ntile' function must have ORDER BY clause"); - if (transform->window_description.frame.end_type != WindowFrame::BoundaryType::Unbounded) + // We must wait all for the partition end and get the total rows number in this + // partition. So before the end of this partition, there is no any block could be + // dropped out. + bool is_frame_supported = transform->window_description.frame.begin_type == WindowFrame::BoundaryType::Unbounded + && transform->window_description.frame.end_type == WindowFrame::BoundaryType::Unbounded; + if (!is_frame_supported) { - // We must wait all for the partition end and get the total rows number in this - // partition. So before the end of this partition, there is no any block could be - // dropped out. - throw Exception(ErrorCodes::BAD_ARGUMENTS, "ntile's frame end type must be UNBOUNDED FOLLOWING"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Window frame for function 'ntile' should be 'ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"); } } }; diff --git a/tests/queries/0_stateless/02560_window_ntile.reference b/tests/queries/0_stateless/02560_window_ntile.reference index cae0586fa8c..1045fc1011a 100644 --- a/tests/queries/0_stateless/02560_window_ntile.reference +++ b/tests/queries/0_stateless/02560_window_ntile.reference @@ -22,7 +22,28 @@ select a, b, ntile(3) over (partition by a order by b rows between unbounded pre 1 7 3 1 8 3 1 9 3 -select a, b, ntile(2) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(3) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +0 0 1 +0 1 1 +0 2 1 +0 3 1 +0 4 2 +0 5 2 +0 6 2 +0 7 3 +0 8 3 +0 9 3 +1 0 1 +1 1 1 +1 2 1 +1 3 1 +1 4 2 +1 5 2 +1 6 2 +1 7 3 +1 8 3 +1 9 3 +select a, b, ntile(2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); 0 0 1 0 1 1 0 2 1 @@ -43,7 +64,7 @@ select a, b, ntile(2) over (partition by a order by b rows between unbounded pre 1 7 2 1 8 2 1 9 2 -select a, b, ntile(1) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); 0 0 1 0 1 1 0 2 1 @@ -64,7 +85,7 @@ select a, b, ntile(1) over (partition by a order by b rows between unbounded pre 1 7 1 1 8 1 1 9 1 -select a, b, ntile(100) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(100) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); 0 0 1 0 1 2 0 2 3 @@ -85,7 +106,7 @@ select a, b, ntile(100) over (partition by a order by b rows between unbounded p 1 7 8 1 8 9 1 9 10 -select a, b, ntile(65535) over (partition by a order by b rows between unbounded preceding and unbounded following) from (select 1 as a, number as b from numbers(65535)) limit 100; +select a, b, ntile(65535) over (partition by a order by b) from (select 1 as a, number as b from numbers(65535)) limit 100; 1 0 1 1 1 2 1 2 3 @@ -187,11 +208,11 @@ select a, b, ntile(65535) over (partition by a order by b rows between unbounded 1 98 99 1 99 100 -- Bad arguments -select a, b, ntile(3.0) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile('2') over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(0) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(-2) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(b + 1) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(3.0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile('2') over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(-2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(b + 1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -- Bad window type select a, b, ntile(2) over (partition by a) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } select a, b, ntile(2) over (partition by a order by b rows between 4 preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } diff --git a/tests/queries/0_stateless/02560_window_ntile.sql b/tests/queries/0_stateless/02560_window_ntile.sql index 4c25ecf4dd2..f2acf8fc94e 100644 --- a/tests/queries/0_stateless/02560_window_ntile.sql +++ b/tests/queries/0_stateless/02560_window_ntile.sql @@ -2,17 +2,20 @@ -- Normal cases select a, b, ntile(3) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -select a, b, ntile(2) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -select a, b, ntile(1) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -select a, b, ntile(100) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -select a, b, ntile(65535) over (partition by a order by b rows between unbounded preceding and unbounded following) from (select 1 as a, number as b from numbers(65535)) limit 100; +select a, b, ntile(3) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(100) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(65535) over (partition by a order by b) from (select 1 as a, number as b from numbers(65535)) limit 100; + + -- Bad arguments -select a, b, ntile(3.0) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile('2') over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(0) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(-2) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(b + 1) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(3.0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile('2') over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(-2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(b + 1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -- Bad window type select a, b, ntile(2) over (partition by a) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } From 2d0812e3c745c1c589a489c824db2f21785733fb Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 24 Apr 2023 14:55:31 +0000 Subject: [PATCH 039/308] Refactor ColumnLowCardinality::cutAndCompact to avoid calling IColumn::assumeMutable. --- src/Columns/ColumnLowCardinality.cpp | 27 ++++++++++++--------------- src/Columns/ColumnLowCardinality.h | 8 +++++--- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 11d02b023d6..4f9ab8215be 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -485,13 +485,8 @@ void ColumnLowCardinality::setSharedDictionary(const ColumnPtr & column_unique) ColumnLowCardinality::MutablePtr ColumnLowCardinality::cutAndCompact(size_t start, size_t length) const { auto sub_positions = IColumn::mutate(idx.getPositions()->cut(start, length)); - /// Create column with new indexes and old dictionary. - /// Dictionary is shared, but will be recreated after compactInplace call. - auto column = ColumnLowCardinality::create(getDictionary().assumeMutable(), std::move(sub_positions)); - /// Will create new dictionary. - column->compactInplace(); - - return column; + auto new_column_unique = Dictionary::compact(dictionary.getColumnUnique(), sub_positions); + return ColumnLowCardinality::create(std::move(new_column_unique), std::move(sub_positions)); } void ColumnLowCardinality::compactInplace() @@ -589,7 +584,7 @@ size_t ColumnLowCardinality::Index::getSizeOfIndexType(const IColumn & column, s column.getName()); } -void ColumnLowCardinality::Index::attachPositions(ColumnPtr positions_) +void ColumnLowCardinality::Index::attachPositions(MutableColumnPtr positions_) { positions = std::move(positions_); updateSizeOfType(); @@ -820,21 +815,23 @@ void ColumnLowCardinality::Dictionary::setShared(const ColumnPtr & column_unique shared = true; } -void ColumnLowCardinality::Dictionary::compact(ColumnPtr & positions) +void ColumnLowCardinality::Dictionary::compact(MutableColumnPtr & positions) { - auto new_column_unique = column_unique->cloneEmpty(); + column_unique = compact(getColumnUnique(), positions); + shared = false; +} - auto & unique = getColumnUnique(); +MutableColumnPtr ColumnLowCardinality::Dictionary::compact(const IColumnUnique & unique, MutableColumnPtr & positions) +{ + auto new_column_unique = unique.cloneEmpty(); auto & new_unique = static_cast(*new_column_unique); - auto indexes = mapUniqueIndex(positions->assumeMutableRef()); + auto indexes = mapUniqueIndex(*positions); auto sub_keys = unique.getNestedColumn()->index(*indexes, 0); auto new_indexes = new_unique.uniqueInsertRangeFrom(*sub_keys, 0, sub_keys->size()); positions = IColumn::mutate(new_indexes->index(*positions, 0)); - column_unique = std::move(new_column_unique); - - shared = false; + return new_column_unique; } ColumnPtr ColumnLowCardinality::cloneWithDefaultOnNull() const diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index e7f4b92d733..df707039b03 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -301,8 +301,8 @@ public: void checkSizeOfType(); - ColumnPtr detachPositions() { return std::move(positions); } - void attachPositions(ColumnPtr positions_); + MutableColumnPtr detachPositions() { return IColumn::mutate(std::move(positions)); } + void attachPositions(MutableColumnPtr positions_); void countKeys(ColumnUInt64::Container & counts) const; @@ -350,7 +350,9 @@ private: bool isShared() const { return shared; } /// Create new dictionary with only keys that are mentioned in positions. - void compact(ColumnPtr & positions); + void compact(MutableColumnPtr & positions); + + static MutableColumnPtr compact(const IColumnUnique & column_unique, MutableColumnPtr & positions); private: WrappedPtr column_unique; From 38622d07703804570643cc3ab0b5c764efbf675c Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 24 Apr 2023 18:21:49 +0000 Subject: [PATCH 040/308] add settings to delay or throw in case of too many mutations --- src/Common/ErrorCodes.cpp | 1 + src/Common/ProfileEvents.cpp | 3 + src/Core/Settings.h | 2 + src/Storages/MergeTree/MergeTreeData.cpp | 49 +++++++++++++ src/Storages/MergeTree/MergeTreeData.h | 9 ++- src/Storages/MergeTree/MergeTreeSettings.h | 4 ++ .../MergeTree/ReplicatedMergeTreeQueue.cpp | 20 ++++-- .../MergeTree/ReplicatedMergeTreeQueue.h | 2 + src/Storages/StorageMergeTree.cpp | 27 +++++++- src/Storages/StorageMergeTree.h | 2 + src/Storages/StorageReplicatedMergeTree.cpp | 12 +++- src/Storages/StorageReplicatedMergeTree.h | 2 + .../02724_delay_mutations.reference | 8 +++ .../0_stateless/02724_delay_mutations.sh | 59 ++++++++++++++++ .../02724_limit_num_mutations.reference | 9 +++ .../0_stateless/02724_limit_num_mutations.sh | 69 +++++++++++++++++++ 16 files changed, 269 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/02724_delay_mutations.reference create mode 100755 tests/queries/0_stateless/02724_delay_mutations.sh create mode 100644 tests/queries/0_stateless/02724_limit_num_mutations.reference create mode 100755 tests/queries/0_stateless/02724_limit_num_mutations.sh diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 9abf3bba8ff..d570eab8f18 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -650,6 +650,7 @@ M(679, IO_URING_SUBMIT_ERROR) \ M(690, MIXED_ACCESS_PARAMETER_TYPES) \ M(691, UNKNOWN_ELEMENT_OF_ENUM) \ + M(692, TOO_MANY_MUTATIONS) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index da096085d5b..387eafdc145 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -102,6 +102,9 @@ M(DelayedInserts, "Number of times the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.") \ M(RejectedInserts, "Number of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.") \ M(DelayedInsertsMilliseconds, "Total number of milliseconds spent while the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.") \ + M(DelayedMutations, "Number of times the mutation of a MergeTree table was throttled due to high number of unfinished mutations for table.") \ + M(RejectedMutations, "Number of times the mutation of a MergeTree table was rejected with 'Too many mutations' exception due to high number of unfinished mutations for table.") \ + M(DelayedMutationsMilliseconds, "Total number of milliseconds spent while the mutation of a MergeTree table was throttled due to high number of unfinished mutations for table.") \ M(DistributedDelayedInserts, "Number of times the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.") \ M(DistributedRejectedInserts, "Number of times the INSERT of a block to a Distributed table was rejected with 'Too many bytes' exception due to high number of pending bytes.") \ M(DistributedDelayedInsertsMilliseconds, "Total number of milliseconds spent while the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.") \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 26409e98763..8fd2af5fa23 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -275,6 +275,8 @@ class IColumn; \ M(UInt64, parts_to_delay_insert, 0, "If the destination table contains at least that many active parts in a single partition, artificially slow down insert into table.", 0) \ M(UInt64, parts_to_throw_insert, 0, "If more than this number active parts in a single partition of the destination table, throw 'Too many parts ...' exception.", 0) \ + M(UInt64, number_of_mutations_to_delay, 0, "If the mutated table contains at least that many unfinished mutations, artificially slow down mutations of table. 0 - disabled", 0) \ + M(UInt64, number_of_mutations_to_throw, 0, "If the mutated table contains at least that many unfinished mutations, throw 'Too many mutations ...' exception. 0 - disabled", 0) \ M(Bool, insert_distributed_sync, false, "If setting is enabled, insert query into distributed waits until data will be sent to all nodes in cluster.", 0) \ M(UInt64, insert_distributed_timeout, 0, "Timeout for insert query into distributed. Setting is used only with insert_distributed_sync enabled. Zero value means no timeout.", 0) \ M(Int64, distributed_ddl_task_timeout, 180, "Timeout for DDL query responses from all hosts in cluster. If a ddl request has not been performed on all hosts, a response will contain a timeout error and a request will be executed in an async mode. Negative value means infinite. Zero means async mode.", 0) \ diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index f5f12660223..e9e3548f66f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -114,6 +114,9 @@ namespace ProfileEvents extern const Event MergedIntoWideParts; extern const Event MergedIntoCompactParts; extern const Event MergedIntoInMemoryParts; + extern const Event RejectedMutations; + extern const Event DelayedMutations; + extern const Event DelayedMutationsMilliseconds; } namespace CurrentMetrics @@ -171,6 +174,7 @@ namespace ErrorCodes extern const int SERIALIZATION_ERROR; extern const int NETWORK_ERROR; extern const int SOCKET_TIMEOUT; + extern const int TOO_MANY_MUTATIONS; } @@ -4296,6 +4300,51 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, const Contex std::this_thread::sleep_for(std::chrono::milliseconds(static_cast(delay_milliseconds))); } +void MergeTreeData::delayMutationOrThrowIfNeeded(Poco::Event * until, const ContextPtr & query_context) const +{ + const auto settings = getSettings(); + const auto & query_settings = query_context->getSettingsRef(); + + size_t num_mutations_to_delay = query_settings.number_of_mutations_to_delay + ? query_settings.number_of_mutations_to_delay + : settings->number_of_mutations_to_delay; + + size_t num_mutations_to_throw = query_settings.number_of_mutations_to_throw + ? query_settings.number_of_mutations_to_throw + : settings->number_of_mutations_to_throw; + + if (!num_mutations_to_delay && !num_mutations_to_throw) + return; + + size_t num_unfinished_mutations = getNumberOfUnfinishedMutations(); + if (num_mutations_to_throw && num_unfinished_mutations >= num_mutations_to_throw) + { + ProfileEvents::increment(ProfileEvents::RejectedMutations); + throw Exception(ErrorCodes::TOO_MANY_MUTATIONS, + "Too many unfinished mutations ({}) in table {}", + num_unfinished_mutations, getLogName()); + } + + if (num_mutations_to_delay && num_unfinished_mutations >= num_mutations_to_delay) + { + if (!num_mutations_to_throw) + num_mutations_to_throw = num_mutations_to_delay * 2; + + size_t mutations_over_threshold = num_unfinished_mutations - num_mutations_to_delay; + size_t allowed_mutations_over_threshold = num_mutations_to_throw - num_mutations_to_delay; + + double delay_factor = std::min(static_cast(mutations_over_threshold) / allowed_mutations_over_threshold, 1.0); + size_t delay_milliseconds = static_cast(std::lerp(settings->min_delay_to_mutate_ms, settings->max_delay_to_mutate_ms, delay_factor)); + + ProfileEvents::increment(ProfileEvents::DelayedMutations); + ProfileEvents::increment(ProfileEvents::DelayedMutationsMilliseconds, delay_milliseconds); + + if (until) + until->tryWait(delay_milliseconds); + else + std::this_thread::sleep_for(std::chrono::milliseconds(delay_milliseconds)); + } +} MergeTreeData::DataPartPtr MergeTreeData::getActiveContainingPart( const MergeTreePartInfo & part_info, MergeTreeData::DataPartState state, DataPartsLock & /*lock*/) const diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index b03b7d4a71e..cc5deb7c786 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -540,7 +540,6 @@ public: /// Makes sense only for ordinary MergeTree engines because for them block numbering doesn't depend on partition. std::optional getMinPartDataVersion() const; - /// Returns all detached parts DetachedPartsInfo getDetachedParts() const; @@ -551,11 +550,17 @@ public: MutableDataPartsVector tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part, ContextPtr context, PartsTemporaryRename & renamed_parts); - /// If the table contains too many active parts, sleep for a while to give them time to merge. /// If until is non-null, wake up from the sleep earlier if the event happened. void delayInsertOrThrowIfNeeded(Poco::Event * until, const ContextPtr & query_context) const; + /// If the table contains too many unfinished mutations, sleep for a while to give them time to execute. + /// If until is non-null, wake up from the sleep earlier if the event happened. + void delayMutationOrThrowIfNeeded(Poco::Event * until, const ContextPtr & query_context) const; + + /// Returns number of unfinished mutations (is_done = 0). + virtual size_t getNumberOfUnfinishedMutations() const = 0; + /// Renames temporary part to a permanent part and adds it to the parts set. /// It is assumed that the part does not intersect with existing parts. /// Adds the part in the PreActive state (the part will be added to the active set later with out_transaction->commit()). diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index ad55c9d47f3..b7b94359ccf 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -83,6 +83,10 @@ struct Settings; M(UInt64, max_delay_to_insert, 1, "Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, min_delay_to_insert_ms, 10, "Min delay of inserting data into MergeTree table in milliseconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, max_parts_in_total, 100000, "If more than this number active parts in all partitions in total, throw 'Too many parts ...' exception.", 0) \ + M(UInt64, number_of_mutations_to_delay, 0, "If table has at least that many unfinished mutations, artificially slow down mutations of table. Disabled if set to 0", 0) \ + M(UInt64, number_of_mutations_to_throw, 0, "If table has at least that many unfinished mutations, throw 'Too many mutations' exception. Disabled if set to 0", 0) \ + M(UInt64, min_delay_to_mutate_ms, 10, "Min delay of mutating MergeTree table in milliseconds, if there are a lot of unfinished mutations", 0) \ + M(UInt64, max_delay_to_mutate_ms, 1000, "Max delay of mutating MergeTree table in milliseconds, if there are a lot of unfinished mutations", 0) \ \ /* Part removal settings. */ \ M(UInt64, simultaneous_parts_removal_limit, 0, "Maximum number of parts to remove during one CleanupThread iteration (0 means unlimited).", 0) \ diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 1006bd5ab49..1762c7aabe9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1727,18 +1727,30 @@ size_t ReplicatedMergeTreeQueue::countMutations() const return mutations_by_znode.size(); } - size_t ReplicatedMergeTreeQueue::countFinishedMutations() const { std::lock_guard lock(state_mutex); size_t count = 0; - for (const auto & pair : mutations_by_znode) + for (const auto & [_, status] : mutations_by_znode) { - const auto & mutation = pair.second; - if (!mutation.is_done) + if (!status.is_done) break; + ++count; + } + return count; +} + +size_t ReplicatedMergeTreeQueue::countUnfinishedMutations() const +{ + std::lock_guard lock(state_mutex); + + size_t count = 0; + for (const auto & [_, status] : mutations_by_znode | std::views::reverse) + { + if (status.is_done) + break; ++count; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 72796ddd4eb..368f2d4bc1f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -386,6 +386,8 @@ public: /// Count the total number of active mutations that are finished (is_done = true). size_t countFinishedMutations() const; + /// Count the total number of active mutations that are not finished (is_done = false). + size_t countUnfinishedMutations() const; /// Returns functor which used by MergeTreeMergerMutator to select parts for merge ReplicatedMergeTreeMergePredicate getMergePredicate(zkutil::ZooKeeperPtr & zookeeper, PartitionIdsHint && partition_ids_hint); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 5513603bca6..5592004d599 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -3,6 +3,7 @@ #include "Storages/MergeTree/IMergeTreeDataPart.h" #include +#include #include #include @@ -313,7 +314,11 @@ void StorageMergeTree::alter( StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); + auto maybe_mutation_commands = commands.getMutationCommands(new_metadata, local_context->getSettingsRef().materialize_ttl_after_modify, local_context); + if (!maybe_mutation_commands.empty()) + delayMutationOrThrowIfNeeded(nullptr, local_context); + Int64 mutation_version = -1; commands.apply(new_metadata, local_context); @@ -321,7 +326,6 @@ void StorageMergeTree::alter( if (commands.isSettingsAlter()) { changeSettings(new_metadata.settings_changes, table_lock_holder); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(local_context, table_id, new_metadata); } else @@ -587,11 +591,12 @@ void StorageMergeTree::setMutationCSN(const String & mutation_id, CSN csn) void StorageMergeTree::mutate(const MutationCommands & commands, ContextPtr query_context) { + delayMutationOrThrowIfNeeded(nullptr, query_context); + /// Validate partition IDs (if any) before starting mutation getPartitionIdsAffectedByCommands(commands, query_context); Int64 version = startMutation(commands, query_context); - if (query_context->getSettingsRef().mutations_sync > 0 || query_context->getCurrentTransaction()) waitForMutation(version); } @@ -1332,6 +1337,24 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign return scheduled; } +size_t StorageMergeTree::getNumberOfUnfinishedMutations() const +{ + size_t count = 0; + for (const auto & [version, _] : current_mutations_by_version | std::views::reverse) + { + auto status = getIncompleteMutationsStatus(version); + if (!status) + continue; + + if (status->is_done) + break; + + ++count; + } + + return count; +} + UInt64 StorageMergeTree::getCurrentMutationVersion( const DataPartPtr & part, std::unique_lock & /*currently_processing_in_background_mutex_lock*/) const diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 6f8acf9965a..78bd6e3f374 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -113,6 +113,8 @@ public: bool scheduleDataProcessingJob(BackgroundJobsAssignee & assignee) override; + size_t getNumberOfUnfinishedMutations() const override; + MergeTreeDeduplicationLog * getDeduplicationLog() { return deduplication_log.get(); } private: diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9b4972ade59..7bca3cbf581 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5215,7 +5215,10 @@ void StorageReplicatedMergeTree::alter( alter_entry->create_time = time(nullptr); auto maybe_mutation_commands = commands.getMutationCommands( - *current_metadata, query_context->getSettingsRef().materialize_ttl_after_modify, query_context); + *current_metadata, + query_context->getSettingsRef().materialize_ttl_after_modify, + query_context); + bool have_mutation = !maybe_mutation_commands.empty(); alter_entry->have_mutation = have_mutation; @@ -5226,6 +5229,7 @@ void StorageReplicatedMergeTree::alter( PartitionBlockNumbersHolder partition_block_numbers_holder; if (have_mutation) { + delayMutationOrThrowIfNeeded(&partial_shutdown_event, query_context); const String mutations_path(fs::path(zookeeper_path) / "mutations"); ReplicatedMergeTreeMutationEntry mutation_entry; @@ -6406,6 +6410,8 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte /// After all needed parts are mutated (i.e. all active parts have the mutation version greater than /// the version of this mutation), the mutation is considered done and can be deleted. + delayMutationOrThrowIfNeeded(&partial_shutdown_event, query_context); + ReplicatedMergeTreeMutationEntry mutation_entry; mutation_entry.source_replica = replica_name; mutation_entry.commands = commands; @@ -8036,6 +8042,10 @@ String StorageReplicatedMergeTree::getTableSharedID() const return toString(table_shared_id); } +size_t StorageReplicatedMergeTree::getNumberOfUnfinishedMutations() const +{ + return queue.countUnfinishedMutations(); +} void StorageReplicatedMergeTree::createTableSharedID() const { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index ade4e4f0b4b..e81be299144 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -311,6 +311,8 @@ public: // Return table id, common for different replicas String getTableSharedID() const override; + size_t getNumberOfUnfinishedMutations() const override; + /// Returns the same as getTableSharedID(), but extracts it from a create query. static std::optional tryGetTableSharedIDFromCreateQuery(const IAST & create_query, const ContextPtr & global_context); diff --git a/tests/queries/0_stateless/02724_delay_mutations.reference b/tests/queries/0_stateless/02724_delay_mutations.reference new file mode 100644 index 00000000000..16bd972a06d --- /dev/null +++ b/tests/queries/0_stateless/02724_delay_mutations.reference @@ -0,0 +1,8 @@ +1 2 +4 +1 6 +0 +ALTER TABLE t_delay_mutations UPDATE v = 3 WHERE 1; 0 0 +ALTER TABLE t_delay_mutations UPDATE v = 4 WHERE 1; 0 0 +ALTER TABLE t_delay_mutations UPDATE v = 5 WHERE 1; 1 1 +ALTER TABLE t_delay_mutations UPDATE v = 6 WHERE 1; 1 1 diff --git a/tests/queries/0_stateless/02724_delay_mutations.sh b/tests/queries/0_stateless/02724_delay_mutations.sh new file mode 100755 index 00000000000..f349e29253a --- /dev/null +++ b/tests/queries/0_stateless/02724_delay_mutations.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# shellcheck source=./mergetree_mutations.lib +. "$CURDIR"/mergetree_mutations.lib + +${CLICKHOUSE_CLIENT} -n --query " +DROP TABLE IF EXISTS t_delay_mutations SYNC; + +CREATE TABLE t_delay_mutations (id UInt64, v UInt64) +ENGINE = MergeTree ORDER BY id +SETTINGS + number_of_mutations_to_delay = 2, + number_of_mutations_to_throw = 10, + min_delay_to_mutate_ms = 10, + min_delay_to_mutate_ms = 1000; + +SET mutations_sync = 0; +SYSTEM STOP MERGES t_delay_mutations; + +INSERT INTO t_delay_mutations VALUES (1, 2); + +ALTER TABLE t_delay_mutations UPDATE v = 3 WHERE 1; +ALTER TABLE t_delay_mutations UPDATE v = 4 WHERE 1; + +ALTER TABLE t_delay_mutations UPDATE v = 5 WHERE 1; +ALTER TABLE t_delay_mutations UPDATE v = 6 WHERE 1; + +SELECT * FROM t_delay_mutations ORDER BY id; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_delay_mutations' AND NOT is_done; +" + +${CLICKHOUSE_CLIENT} --query "SYSTEM START MERGES t_delay_mutations" +wait_for_mutation "t_delay_mutations" "mutation_5.txt" + +${CLICKHOUSE_CLIENT} -n --query " +SELECT * FROM t_delay_mutations ORDER BY id; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_delay_mutations' AND NOT is_done; + +DROP TABLE IF EXISTS t_delay_mutations SYNC; +" + +${CLICKHOUSE_CLIENT} -n --query " +SYSTEM FLUSH LOGS; + +SELECT + query, + ProfileEvents['DelayedMutations'], + ProfileEvents['DelayedMutationsMilliseconds'] BETWEEN 10 AND 1000 +FROM system.query_log +WHERE + type = 'QueryFinish' AND + current_database = '$CLICKHOUSE_DATABASE' AND + query ILIKE 'ALTER TABLE t_delay_mutations UPDATE%' +ORDER BY query; +" diff --git a/tests/queries/0_stateless/02724_limit_num_mutations.reference b/tests/queries/0_stateless/02724_limit_num_mutations.reference new file mode 100644 index 00000000000..ecd1ce23ca2 --- /dev/null +++ b/tests/queries/0_stateless/02724_limit_num_mutations.reference @@ -0,0 +1,9 @@ +1 2 +2 +CREATE TABLE default.t_limit_mutations\n(\n `id` UInt64,\n `v` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/t_limit_mutations/\', \'1\')\nORDER BY id\nSETTINGS number_of_mutations_to_throw = 2, index_granularity = 8192 +1 2 +4 +CREATE TABLE default.t_limit_mutations\n(\n `id` UInt64,\n `v` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/t_limit_mutations/\', \'1\')\nORDER BY id\nSETTINGS number_of_mutations_to_throw = 2, index_granularity = 8192 +1 6 +0 +CREATE TABLE default.t_limit_mutations\n(\n `id` UInt64,\n `v` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/t_limit_mutations/\', \'1\')\nORDER BY id\nSETTINGS number_of_mutations_to_throw = 2, index_granularity = 8192 diff --git a/tests/queries/0_stateless/02724_limit_num_mutations.sh b/tests/queries/0_stateless/02724_limit_num_mutations.sh new file mode 100755 index 00000000000..98bfdbbb551 --- /dev/null +++ b/tests/queries/0_stateless/02724_limit_num_mutations.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# shellcheck source=./mergetree_mutations.lib +. "$CURDIR"/mergetree_mutations.lib + +function wait_for_alter() +{ + type=$1 + for i in {1..100}; do + sleep 0.1 + ${CLICKHOUSE_CLIENT} --query "SHOW CREATE TABLE t_limit_mutations" | grep -q "\`v\` $type" && break; + + if [[ $i -eq 100 ]]; then + echo "Timed out while waiting for alter to execute" + fi + done +} + +${CLICKHOUSE_CLIENT} -n --query " +DROP TABLE IF EXISTS t_limit_mutations SYNC; + +CREATE TABLE t_limit_mutations (id UInt64, v UInt64) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/t_limit_mutations/', '1') ORDER BY id +SETTINGS number_of_mutations_to_throw = 2; + +SET mutations_sync = 0; +SYSTEM STOP MERGES t_limit_mutations; + +INSERT INTO t_limit_mutations VALUES (1, 2); + +ALTER TABLE t_limit_mutations UPDATE v = 3 WHERE 1; +ALTER TABLE t_limit_mutations UPDATE v = 4 WHERE 1; + +ALTER TABLE t_limit_mutations UPDATE v = 5 WHERE 1; -- { serverError TOO_MANY_MUTATIONS } +ALTER TABLE t_limit_mutations MODIFY COLUMN v String; -- { serverError TOO_MANY_MUTATIONS } + +SELECT * FROM t_limit_mutations ORDER BY id; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_limit_mutations' AND NOT is_done; +SHOW CREATE TABLE t_limit_mutations; +" + +${CLICKHOUSE_CLIENT} -n --query " +ALTER TABLE t_limit_mutations UPDATE v = 6 WHERE 1 SETTINGS number_of_mutations_to_throw = 100; +ALTER TABLE t_limit_mutations MODIFY COLUMN v String SETTINGS number_of_mutations_to_throw = 100, alter_sync = 0; +" + +wait_for_alter "String" + +${CLICKHOUSE_CLIENT} -n --query " +SELECT * FROM t_limit_mutations ORDER BY id; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_limit_mutations' AND NOT is_done; +SHOW CREATE TABLE t_limit_mutations; +" + +${CLICKHOUSE_CLIENT} --query "SYSTEM START MERGES t_limit_mutations" + +wait_for_mutation "t_limit_mutations" "0000000003" + +${CLICKHOUSE_CLIENT} -n --query " +SELECT * FROM t_limit_mutations ORDER BY id; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_limit_mutations' AND NOT is_done; +SHOW CREATE TABLE t_limit_mutations; + +DROP TABLE IF EXISTS t_limit_mutations SYNC; +" From b565257a39355996ff271ace9c0f45462c427e37 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 20 Apr 2023 13:23:08 +0000 Subject: [PATCH 041/308] Support totals and extremes in query cache --- src/Interpreters/Cache/QueryCache.cpp | 147 +++++++++++++----- src/Interpreters/Cache/QueryCache.h | 28 +++- src/Interpreters/executeQuery.cpp | 17 +- src/Processors/ISource.cpp | 2 +- src/Processors/Sources/SourceFromChunks.cpp | 86 ++++++---- src/Processors/Sources/SourceFromChunks.h | 25 ++- .../StreamInQueryCacheTransform.cpp | 15 +- .../Transforms/StreamInQueryCacheTransform.h | 11 +- src/QueryPipeline/QueryPipeline.cpp | 33 +++- src/QueryPipeline/QueryPipeline.h | 4 +- .../System/StorageSystemQueryCache.cpp | 2 +- ...2494_query_cache_totals_extremes.reference | 43 +++++ .../02494_query_cache_totals_extremes.sql | 48 ++++++ 13 files changed, 348 insertions(+), 113 deletions(-) create mode 100644 tests/queries/0_stateless/02494_query_cache_totals_extremes.reference create mode 100644 tests/queries/0_stateless/02494_query_cache_totals_extremes.sql diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 6337fd9b04c..512abc82529 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -157,11 +157,13 @@ size_t QueryCache::KeyHasher::operator()(const Key & key) const return res; } -size_t QueryCache::QueryResultWeight::operator()(const Chunks & chunks) const +size_t QueryCache::QueryCacheEntryWeight::operator()(const Entry & entry) const { size_t res = 0; - for (const auto & chunk : chunks) + for (const auto & chunk : entry.chunks) res += chunk.allocatedBytes(); + res += entry.totals.has_value() ? entry.totals->allocatedBytes() : 0; + res += entry.extremes.has_value() ? entry.extremes->allocatedBytes() : 0; return res; } @@ -191,25 +193,48 @@ QueryCache::Writer::Writer( } } -void QueryCache::Writer::buffer(Chunk && partial_query_result) +QueryCache::Writer::Writer(const Writer & other) + : cache(other.cache) + , key(other.key) + , max_entry_size_in_bytes(other.max_entry_size_in_bytes) + , max_entry_size_in_rows(other.max_entry_size_in_rows) + , min_query_runtime(other.min_query_runtime) + , squash_partial_results(other.squash_partial_results) + , max_block_size(other.max_block_size) +{ +} + +void QueryCache::Writer::buffer(Chunk && chunk, Type type) { if (skip_insert) return; std::lock_guard lock(mutex); - auto & chunks = *query_result; - - chunks.emplace_back(std::move(partial_query_result)); - - new_entry_size_in_bytes += chunks.back().allocatedBytes(); - new_entry_size_in_rows += chunks.back().getNumRows(); - - if ((new_entry_size_in_bytes > max_entry_size_in_bytes) || (new_entry_size_in_rows > max_entry_size_in_rows)) + switch (type) { - chunks.clear(); /// eagerly free some space - skip_insert = true; - LOG_TRACE(&Poco::Logger::get("QueryCache"), "Skipped insert (query result too big), new_entry_size_in_bytes: {} ({}), new_entry_size_in_rows: {} ({}), query: {}", new_entry_size_in_bytes, max_entry_size_in_bytes, new_entry_size_in_rows, max_entry_size_in_rows, key.queryStringFromAst()); + case Type::Result: + { + /// Normal query result chunks are simply buffered. They are squashed and compressed later in finalizeWrite(). + query_result->chunks.emplace_back(std::move(chunk)); + break; + } + case Type::Totals: + case Type::Extremes: + { + /// For simplicity, totals and extremes chunks are immediately squashed (totals/extremes are obscure and even if enabled, few + /// such chunks are expected). + auto & buffered_chunk = (type == Type::Totals) ? query_result->totals : query_result->extremes; + + convertToFullIfSparse(chunk); + + if (!buffered_chunk.has_value()) + buffered_chunk = std::move(chunk); + else + buffered_chunk->append(chunk); + + break; + } } } @@ -222,6 +247,8 @@ void QueryCache::Writer::finalizeWrite() chassert(!was_finalized); + /// Check some reasons why the entry must not be cached: + if (std::chrono::duration_cast(std::chrono::system_clock::now() - query_start_time) < min_query_runtime) { LOG_TRACE(&Poco::Logger::get("QueryCache"), "Skipped insert (query not expensive enough), query: {}", key.queryStringFromAst()); @@ -244,7 +271,7 @@ void QueryCache::Writer::finalizeWrite() Chunks squashed_chunks; size_t rows_remaining_in_squashed = 0; /// how many further rows can the last squashed chunk consume until it reaches max_block_size - for (auto & chunk : *query_result) + for (auto & chunk : query_result->chunks) { convertToFullIfSparse(chunk); @@ -272,26 +299,49 @@ void QueryCache::Writer::finalizeWrite() } } - *query_result = std::move(squashed_chunks); + query_result->chunks = std::move(squashed_chunks); } if (key.is_compressed) { + /// Compress result chunks. Reduces the space consumption of the cache but means reading from it will be slower due to decompression. + Chunks compressed_chunks; - const Chunks & decompressed_chunks = *query_result; - for (const auto & decompressed_chunk : decompressed_chunks) + + for (const auto & chunk : query_result->chunks) { - const Columns & decompressed_columns = decompressed_chunk.getColumns(); + const Columns & columns = chunk.getColumns(); Columns compressed_columns; - for (const auto & decompressed_column : decompressed_columns) + for (const auto & column : columns) { - auto compressed_column = decompressed_column->compress(); + auto compressed_column = column->compress(); compressed_columns.push_back(compressed_column); } - Chunk compressed_chunk(compressed_columns, decompressed_chunk.getNumRows()); + Chunk compressed_chunk(compressed_columns, chunk.getNumRows()); compressed_chunks.push_back(std::move(compressed_chunk)); } - *query_result = std::move(compressed_chunks); + query_result->chunks = std::move(compressed_chunks); + } + + /// Check more reasons why the entry must not be cached. + + auto count_rows_in_chunks = [](const Entry & entry) + { + size_t res = 0; + for (const auto & chunk : entry.chunks) + res += chunk.getNumRows(); + res += entry.totals.has_value() ? entry.totals->getNumRows() : 0; + res += entry.extremes.has_value() ? entry.extremes->getNumRows() : 0; + return res; + }; + + size_t new_entry_size_in_bytes = QueryCacheEntryWeight()(*query_result); + size_t new_entry_size_in_rows = count_rows_in_chunks(*query_result); + + if ((new_entry_size_in_bytes > max_entry_size_in_bytes) || (new_entry_size_in_rows > max_entry_size_in_rows)) + { + LOG_TRACE(&Poco::Logger::get("QueryCache"), "Skipped insert (query result too big), new_entry_size_in_bytes: {} ({}), new_entry_size_in_rows: {} ({}), query: {}", new_entry_size_in_bytes, max_entry_size_in_bytes, new_entry_size_in_rows, max_entry_size_in_rows, key.queryStringFromAst()); + return; } cache.set(key, query_result); @@ -299,6 +349,21 @@ void QueryCache::Writer::finalizeWrite() was_finalized = true; } +/// Constructs a pipe with just one processor: SourceFromChunks which serves data from the query cache +void QueryCache::Reader::buildPipe(Block header, Chunks && chunks, const std::optional & totals, const std::optional & extremes) +{ + auto source_from_chunks = std::make_shared( + header, std::move(chunks), + totals.has_value() ? std::optional(totals->clone()) : std::nullopt, + extremes.has_value() ? std::optional(extremes->clone()) : std::nullopt); + + OutputPort * out = &source_from_chunks->getPort(); + OutputPort * out_totals = source_from_chunks->getTotalsPort(); + OutputPort * out_extremes = source_from_chunks->getExtremesPort(); + + pipe = Pipe(source_from_chunks, out, out_totals, out_extremes); +} + QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guard &) { auto entry = cache_.getWithKey(key); @@ -322,25 +387,37 @@ QueryCache::Reader::Reader(Cache & cache_, const Key & key, const std::lock_guar } if (!entry->key.is_compressed) - pipe = Pipe(std::make_shared(entry->key.header, entry->mapped)); + { + // Cloning chunks isn't exactly great. It could be avoided by another indirection, i.e. wrapping Entry's members chunks, totals and + // extremes into shared_ptrs and assuming that the lifecycle of these shared_ptrs coincides with the lifecycle of the Entry + // shared_ptr. This is not done 1. to keep things simple 2. this case (uncompressed chunks) is the exceptional case, in the other + // case (the default case aka. compressed chunks) we need to decompress the entry anyways and couldn't apply the potential + // optimization. + + Chunks cloned_chunks; + for (const auto & chunk : entry->mapped->chunks) + cloned_chunks.push_back(chunk.clone()); + + buildPipe(entry->key.header, std::move(cloned_chunks), entry->mapped->totals, entry->mapped->extremes); + } else { - auto decompressed_chunks = std::make_shared(); - const Chunks & compressed_chunks = *entry->mapped; - for (const auto & compressed_chunk : compressed_chunks) + Chunks decompressed_chunks; + const Chunks & chunks = entry->mapped->chunks; + for (const auto & chunk : chunks) { - const Columns & compressed_chunk_columns = compressed_chunk.getColumns(); + const Columns & columns = chunk.getColumns(); Columns decompressed_columns; - for (const auto & compressed_column : compressed_chunk_columns) + for (const auto & column : columns) { - auto column = compressed_column->decompress(); - decompressed_columns.push_back(column); + auto decompressed_column = column->decompress(); + decompressed_columns.push_back(decompressed_column); } - Chunk decompressed_chunk(decompressed_columns, compressed_chunk.getNumRows()); - decompressed_chunks->push_back(std::move(decompressed_chunk)); + Chunk decompressed_chunk(decompressed_columns, chunk.getNumRows()); + decompressed_chunks.push_back(std::move(decompressed_chunk)); } - pipe = Pipe(std::make_shared(entry->key.header, decompressed_chunks)); + buildPipe(entry->key.header, std::move(decompressed_chunks), entry->mapped->totals, entry->mapped->extremes); } LOG_TRACE(&Poco::Logger::get("QueryCache"), "Entry found for query {}", key.queryStringFromAst()); @@ -406,7 +483,7 @@ std::vector QueryCache::dump() const } QueryCache::QueryCache() - : cache(std::make_unique>(std::make_unique())) + : cache(std::make_unique>(std::make_unique())) { } diff --git a/src/Interpreters/Cache/QueryCache.h b/src/Interpreters/Cache/QueryCache.h index 64753974f1c..f614ca248d4 100644 --- a/src/Interpreters/Cache/QueryCache.h +++ b/src/Interpreters/Cache/QueryCache.h @@ -53,7 +53,8 @@ public: /// When does the entry expire? const std::chrono::time_point expires_at; - /// Is the entry compressed? + /// Are the chunks in the entry compressed? + /// (we could theoretically apply compression also to the totals and extremes but it's an obscure use case) const bool is_compressed; Key(ASTPtr ast_, @@ -66,15 +67,22 @@ public: String queryStringFromAst() const; }; + struct Entry + { + Chunks chunks; + std::optional totals = std::nullopt; + std::optional extremes = std::nullopt; + }; + private: struct KeyHasher { size_t operator()(const Key & key) const; }; - struct QueryResultWeight + struct QueryCacheEntryWeight { - size_t operator()(const Chunks & chunks) const; + size_t operator()(const Entry & entry) const; }; struct IsStale @@ -83,7 +91,7 @@ private: }; /// query --> query result - using Cache = CacheBase; + using Cache = CacheBase; /// query --> query execution count using TimesExecuted = std::unordered_map; @@ -103,21 +111,24 @@ public: class Writer { public: - void buffer(Chunk && partial_query_result); + + Writer(const Writer & other); + + enum class Type {Result, Totals, Extremes}; + void buffer(Chunk && chunk, Type type); + void finalizeWrite(); private: std::mutex mutex; Cache & cache; const Key key; - size_t new_entry_size_in_bytes TSA_GUARDED_BY(mutex) = 0; const size_t max_entry_size_in_bytes; - size_t new_entry_size_in_rows TSA_GUARDED_BY(mutex) = 0; const size_t max_entry_size_in_rows; const std::chrono::time_point query_start_time = std::chrono::system_clock::now(); /// Writer construction and finalizeWrite() coincide with query start/end const std::chrono::milliseconds min_query_runtime; const bool squash_partial_results; const size_t max_block_size; - std::shared_ptr query_result TSA_GUARDED_BY(mutex) = std::make_shared(); + Cache::MappedPtr query_result TSA_GUARDED_BY(mutex) = std::make_shared(); std::atomic skip_insert = false; bool was_finalized = false; @@ -138,6 +149,7 @@ public: Pipe && getPipe(); /// must be called only if hasCacheEntryForKey() returns true private: Reader(Cache & cache_, const Key & key, const std::lock_guard &); + void buildPipe(Block header, Chunks && chunks, const std::optional & totals, const std::optional & extremes); Pipe pipe; friend class QueryCache; /// for createReader() }; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 00a5d0ed1d8..b94287d1d69 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -755,15 +755,14 @@ static std::tuple executeQueryImpl( const size_t num_query_runs = query_cache->recordQueryRun(key); if (num_query_runs > settings.query_cache_min_query_runs) { - auto stream_in_query_cache_transform = - std::make_shared( - res.pipeline.getHeader(), query_cache, key, - std::chrono::milliseconds(context->getSettings().query_cache_min_query_duration.totalMilliseconds()), - context->getSettings().query_cache_squash_partial_results, - context->getSettings().max_block_size, - context->getSettings().query_cache_max_size_in_bytes, - context->getSettings().query_cache_max_entries); - res.pipeline.streamIntoQueryCache(stream_in_query_cache_transform); + auto query_cache_writer = std::make_shared(query_cache->createWriter( + key, + std::chrono::milliseconds(settings.query_cache_min_query_duration.totalMilliseconds()), + settings.query_cache_squash_partial_results, + settings.max_block_size, + settings.query_cache_max_size_in_bytes, + settings.query_cache_max_entries)); + res.pipeline.streamIntoQueryCache(query_cache_writer); } } diff --git a/src/Processors/ISource.cpp b/src/Processors/ISource.cpp index e6aeffa4807..6a88d3973a1 100644 --- a/src/Processors/ISource.cpp +++ b/src/Processors/ISource.cpp @@ -124,7 +124,7 @@ std::optional ISource::tryGenerate() { auto chunk = generate(); if (!chunk) - return {}; + return std::nullopt; return chunk; } diff --git a/src/Processors/Sources/SourceFromChunks.cpp b/src/Processors/Sources/SourceFromChunks.cpp index 7b73b877d2e..54a6e4b54e2 100644 --- a/src/Processors/Sources/SourceFromChunks.cpp +++ b/src/Processors/Sources/SourceFromChunks.cpp @@ -3,20 +3,61 @@ namespace DB { -SourceFromChunks::SourceFromChunks(Block header, Chunks chunks_) - : SourceFromChunks(header, std::make_shared(std::move(chunks_)), true) -{} - -SourceFromChunks::SourceFromChunks(Block header, std::shared_ptr chunks_) - : SourceFromChunks(header, chunks_, false) -{} - -SourceFromChunks::SourceFromChunks(Block header, std::shared_ptr chunks_, bool move_from_chunks_) - : ISource(std::move(header)) - , chunks(chunks_) - , it(chunks->begin()) - , move_from_chunks(move_from_chunks_) +SourceFromChunks::SourceFromChunks(Block header, Chunks && chunks_, std::optional && chunk_totals_, std::optional && chunk_extremes_) + : ISource(header) + , chunks(std::move(chunks_)) + , it(chunks.begin()) { + if (chunk_totals_ != std::nullopt) + { + outputs.emplace_back(header, this); + chunk_totals = std::move(chunk_totals_); + output_totals = &outputs.back(); + } + + if (chunk_extremes_ != std::nullopt) + { + outputs.emplace_back(header, this); + chunk_extremes = std::move(chunk_extremes_); + output_extremes = &outputs.back(); + } +} + +SourceFromChunks::Status SourceFromChunks::prepare() +{ + if (!finished_chunks) + { + Status status = ISource::prepare(); + + if (status != Status::Finished) + return status; + + finished_chunks = true; + } + + if (getTotalsPort()) + { + /// This logic force-pushes the data into the port instead of checking the port status first (isFinished(), canPush()). Seems to + /// work but should be improved (TODO). + chassert(chunk_totals.has_value()); + output_totals->push(std::move(*chunk_totals)); + output_totals->finish(); + } + + if (getExtremesPort()) + { + chassert(chunk_extremes.has_value()); + output_extremes->push(std::move(*chunk_extremes)); + output_extremes->finish(); + } + + return Status::Finished; +} + +void SourceFromChunks::work() +{ + if (!finished_chunks) + ISource::work(); } String SourceFromChunks::getName() const @@ -26,19 +67,12 @@ String SourceFromChunks::getName() const Chunk SourceFromChunks::generate() { - if (it != chunks->end()) - if (move_from_chunks) - { - Chunk && chunk = std::move(*it); - it++; - return chunk; - } - else - { - Chunk chunk = it->clone(); - it++; - return chunk; - } + if (it != chunks.end()) + { + Chunk && chunk = std::move(*it); + it++; + return chunk; + } else return {}; } diff --git a/src/Processors/Sources/SourceFromChunks.h b/src/Processors/Sources/SourceFromChunks.h index d41999208a0..58616a8058c 100644 --- a/src/Processors/Sources/SourceFromChunks.h +++ b/src/Processors/Sources/SourceFromChunks.h @@ -7,24 +7,33 @@ namespace DB { +/// The big brother of SourceFromSingleChunk. Supports multiple chunks and totals/extremes. class SourceFromChunks : public ISource { public: - SourceFromChunks(Block header, Chunks chunks_); - SourceFromChunks(Block header, std::shared_ptr chunks_); + SourceFromChunks(Block header, Chunks && chunks_, std::optional && chunk_totals_, std::optional && chunk_extremes_); String getName() const override; -protected: + Status prepare() override; + void work() override; + + OutputPort * getTotalsPort() const { return output_totals; } + OutputPort * getExtremesPort() const { return output_extremes; } + Chunk generate() override; private: - SourceFromChunks(Block header, std::shared_ptr chunks_, bool move_from_chunks_); - - const std::shared_ptr chunks; + Chunks chunks; Chunks::iterator it; - /// Optimization: if the chunks are exclusively owned by SourceFromChunks, then generate() can move from them - const bool move_from_chunks; + + std::optional chunk_totals = std::nullopt; + std::optional chunk_extremes = std::nullopt; + + OutputPort * output_totals = nullptr; + OutputPort * output_extremes = nullptr; + + bool finished_chunks = false; }; } diff --git a/src/Processors/Transforms/StreamInQueryCacheTransform.cpp b/src/Processors/Transforms/StreamInQueryCacheTransform.cpp index 34dec20ac8e..c4210d7afa5 100644 --- a/src/Processors/Transforms/StreamInQueryCacheTransform.cpp +++ b/src/Processors/Transforms/StreamInQueryCacheTransform.cpp @@ -5,26 +5,23 @@ namespace DB StreamInQueryCacheTransform::StreamInQueryCacheTransform( const Block & header_, - QueryCachePtr cache, - const QueryCache::Key & cache_key, - std::chrono::milliseconds min_query_duration, - bool squash_partial_results, - size_t max_block_size, - size_t max_query_cache_size_in_bytes_quota, size_t max_query_cache_entries_quota) + std::shared_ptr query_cache_writer_, + QueryCache::Writer::Type type_) : ISimpleTransform(header_, header_, false) - , cache_writer(cache->createWriter(cache_key, min_query_duration, squash_partial_results, max_block_size, max_query_cache_size_in_bytes_quota, max_query_cache_entries_quota)) + , query_cache_writer(query_cache_writer_) + , type(type_) { } void StreamInQueryCacheTransform::transform(Chunk & chunk) { - cache_writer.buffer(chunk.clone()); + query_cache_writer->buffer(chunk.clone(), type); } void StreamInQueryCacheTransform::finalizeWriteInQueryCache() { if (!isCancelled()) - cache_writer.finalizeWrite(); + query_cache_writer->finalizeWrite(); } }; diff --git a/src/Processors/Transforms/StreamInQueryCacheTransform.h b/src/Processors/Transforms/StreamInQueryCacheTransform.h index 2989ff5d1a5..e26f343290d 100644 --- a/src/Processors/Transforms/StreamInQueryCacheTransform.h +++ b/src/Processors/Transforms/StreamInQueryCacheTransform.h @@ -11,12 +11,8 @@ class StreamInQueryCacheTransform : public ISimpleTransform public: StreamInQueryCacheTransform( const Block & header_, - QueryCachePtr cache, - const QueryCache::Key & cache_key, - std::chrono::milliseconds min_query_duration, - bool squash_partial_results, - size_t max_block_size, - size_t max_query_cache_size_in_bytes_quota, size_t max_query_cache_entries_quota); + std::shared_ptr query_cache_writer, + QueryCache::Writer::Type type); protected: void transform(Chunk & chunk) override; @@ -26,7 +22,8 @@ public: String getName() const override { return "StreamInQueryCacheTransform"; } private: - QueryCache::Writer cache_writer; + const std::shared_ptr query_cache_writer; + const QueryCache::Writer::Type type; }; } diff --git a/src/QueryPipeline/QueryPipeline.cpp b/src/QueryPipeline/QueryPipeline.cpp index f060f2f508f..1149cef0094 100644 --- a/src/QueryPipeline/QueryPipeline.cpp +++ b/src/QueryPipeline/QueryPipeline.cpp @@ -1,3 +1,5 @@ +#include + #include #include #include @@ -7,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -576,13 +577,31 @@ bool QueryPipeline::tryGetResultRowsAndBytes(UInt64 & result_rows, UInt64 & resu return true; } -void QueryPipeline::streamIntoQueryCache(std::shared_ptr transform) +void QueryPipeline::streamIntoQueryCache(std::shared_ptr query_cache_writer) { assert(pulling()); - connect(*output, transform->getInputPort()); - output = &transform->getOutputPort(); - processors->emplace_back(transform); + /// Attach a special transform to all output ports (result + possibly totals/extremes). The only purpose of the transform is + /// to write each chunk into the query cache. All transforms hold a refcounted reference to the same query cache writer object. + /// This ensures that all transforms write to the single same cache entry. The writer object synchronizes internally, the + /// expensive stuff like cloning chunks happens outside lock scopes). + + auto add_stream_in_query_cache_transform = [&](OutputPort *& out_port, QueryCache::Writer::Type type) + { + if (!out_port) + return; + + auto transform = std::make_shared(out_port->getHeader(), query_cache_writer, type); + connect(*out_port, transform->getInputPort()); + out_port = &transform->getOutputPort(); + processors->emplace_back(std::move(transform)); + }; + + using enum QueryCache::Writer::Type; + + add_stream_in_query_cache_transform(output, Result); + add_stream_in_query_cache_transform(totals, Totals); + add_stream_in_query_cache_transform(extremes, Extremes); } void QueryPipeline::finalizeWriteInQueryCache() @@ -591,8 +610,8 @@ void QueryPipeline::finalizeWriteInQueryCache() processors->begin(), processors->end(), [](ProcessorPtr processor){ return dynamic_cast(&*processor); }); - /// the pipeline should theoretically contain just one StreamInQueryCacheTransform - + /// The pipeline can contain up to three StreamInQueryCacheTransforms which all point to the same query cache writer object. + /// We can call finalize() on any of them. if (it != processors->end()) dynamic_cast(**it).finalizeWriteInQueryCache(); } diff --git a/src/QueryPipeline/QueryPipeline.h b/src/QueryPipeline/QueryPipeline.h index 55c78ca78ed..297178715fd 100644 --- a/src/QueryPipeline/QueryPipeline.h +++ b/src/QueryPipeline/QueryPipeline.h @@ -2,6 +2,7 @@ #include #include #include +#include /// nested classes such as QC::Writer can't be fwd declared #include namespace DB @@ -31,7 +32,6 @@ class SinkToStorage; class ISource; class ISink; class ReadProgressCallback; -class StreamInQueryCacheTransform; struct ColumnWithTypeAndName; using ColumnsWithTypeAndName = std::vector; @@ -105,7 +105,7 @@ public: void setLimitsAndQuota(const StreamLocalLimits & limits, std::shared_ptr quota_); bool tryGetResultRowsAndBytes(UInt64 & result_rows, UInt64 & result_bytes) const; - void streamIntoQueryCache(std::shared_ptr transform); + void streamIntoQueryCache(std::shared_ptr query_cache_writer); void finalizeWriteInQueryCache(); void setQuota(std::shared_ptr quota_); diff --git a/src/Storages/System/StorageSystemQueryCache.cpp b/src/Storages/System/StorageSystemQueryCache.cpp index 0e589b1b183..245f4b7fd26 100644 --- a/src/Storages/System/StorageSystemQueryCache.cpp +++ b/src/Storages/System/StorageSystemQueryCache.cpp @@ -45,7 +45,7 @@ void StorageSystemQueryCache::fillData(MutableColumns & res_columns, ContextPtr continue; res_columns[0]->insert(key.queryStringFromAst()); /// approximates the original query string - res_columns[1]->insert(QueryCache::QueryResultWeight()(*query_result)); + res_columns[1]->insert(QueryCache::QueryCacheEntryWeight()(*query_result)); res_columns[2]->insert(key.expires_at < std::chrono::system_clock::now()); res_columns[3]->insert(!key.is_shared); res_columns[4]->insert(key.is_compressed); diff --git a/tests/queries/0_stateless/02494_query_cache_totals_extremes.reference b/tests/queries/0_stateless/02494_query_cache_totals_extremes.reference new file mode 100644 index 00000000000..2f6e5e7bd87 --- /dev/null +++ b/tests/queries/0_stateless/02494_query_cache_totals_extremes.reference @@ -0,0 +1,43 @@ +1st run: +1 8 +2 2 + +0 10 +2nd run: +1 8 +2 2 + +0 10 +1 +--- +1st run: +1 8 +2 2 + +1 2 +2 8 +2nd run: +1 8 +2 2 + +1 2 +2 8 +1 +--- +1st run: +1 8 +2 2 + +0 10 + +1 2 +2 8 +2nd run: +1 8 +2 2 + +0 10 + +1 2 +2 8 +1 diff --git a/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql new file mode 100644 index 00000000000..d49609885b1 --- /dev/null +++ b/tests/queries/0_stateless/02494_query_cache_totals_extremes.sql @@ -0,0 +1,48 @@ +-- Tags: no-parallel +-- Tag no-parallel: Messes with internal cache + +SET allow_experimental_query_cache = true; + +SYSTEM DROP QUERY CACHE; +DROP TABLE IF EXISTS tbl; + +CREATE TABLE tbl (key UInt64, agg UInt64) ENGINE = MergeTree ORDER BY key; +INSERT INTO tbl VALUES (1, 3), (2, 2), (1, 4), (1, 1); + +-- A query with totals calculation. The result should written into / read from the query cache. +-- Check that both queries produce the same result and that a query cache entry exists. +SELECT '1st run:'; +SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1; +SELECT '2nd run:'; +SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1; + +SELECT count(*) FROM system.query_cache; + +SELECT '---'; + +SYSTEM DROP QUERY CACHE; + +-- A query with extremes calculation. The result should written into / read from the query cache. +-- Check that both queries produce the same result. +SELECT '1st run:'; +SELECT key, sum(agg) FROM tbl GROUP BY key ORDER BY key SETTINGS use_query_cache = 1, extremes = 1; +SELECT '2nd run:'; +SELECT key, sum(agg) FROM tbl GROUP BY key ORDER BY key SETTINGS use_query_cache = 1, extremes = 1; + +SELECT count(*) FROM system.query_cache; + +SELECT '---'; + +SYSTEM DROP QUERY CACHE; + +-- A query with totals and extremes calculation. The result should written into / read from the query cache. +-- Check that both queries produce the same result. +SELECT '1st run:'; +SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1, extremes = 1; +SELECT '2nd run:'; +SELECT key, sum(agg) FROM tbl GROUP BY key WITH totals ORDER BY key SETTINGS use_query_cache = 1, extremes = 1; + +SELECT count(*) FROM system.query_cache; +DROP TABLE IF EXISTS tbl; + +SYSTEM DROP QUERY CACHE; From c4abe66d7705f1d816de600d75eb1621f37600c0 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 25 Apr 2023 12:25:49 +0200 Subject: [PATCH 042/308] Update run.sh --- docker/test/upgrade/run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index 313feb2fc12..10ba597a33a 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -89,6 +89,11 @@ export USE_S3_STORAGE_FOR_MERGE_TREE=1 export ZOOKEEPER_FAULT_INJECTION=0 configure +sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ + | sed "s|local_blob_storage|local|" \ + > /etc/clickhouse-server/config.d/storage_conf.xml.tmp +sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml + start clickhouse-client --query="SELECT 'Server version: ', version()" From 1da6ea7988785db0b800b68d0b6dcc86adc4ea7a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 25 Apr 2023 11:41:33 +0000 Subject: [PATCH 043/308] Move SHOW PROCESSLIST to a better position --- docs/en/sql-reference/statements/show.md | 32 ++++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 428a04ae030..6179d053b0f 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -98,22 +98,6 @@ Result: - [CREATE DATABASE](https://clickhouse.com/docs/en/sql-reference/statements/create/database/#query-language-create-database) -## SHOW PROCESSLIST - -``` sql -SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] -``` - -Outputs the content of the [system.processes](../../operations/system-tables/processes.md#system_tables-processes) table, that contains a list of queries that is being processed at the moment, excepting `SHOW PROCESSLIST` queries. - -The `SELECT * FROM system.processes` query returns data about all the current queries. - -Tip (execute in the console): - -``` bash -$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" -``` - ## SHOW TABLES Displays a list of tables. @@ -277,6 +261,22 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 └──────────────┘ ``` +## SHOW PROCESSLIST + +``` sql +SHOW PROCESSLIST [INTO OUTFILE filename] [FORMAT format] +``` + +Outputs the content of the [system.processes](../../operations/system-tables/processes.md#system_tables-processes) table, that contains a list of queries that is being processed at the moment, excepting `SHOW PROCESSLIST` queries. + +The `SELECT * FROM system.processes` query returns data about all the current queries. + +Tip (execute in the console): + +``` bash +$ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" +``` + ## SHOW GRANTS Shows privileges for a user. From c4ab1e12f1deb1044230c278cbc8d55df425da09 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 25 Apr 2023 18:04:19 +0000 Subject: [PATCH 044/308] Initial version of SHOW INDEXES --- docs/en/sql-reference/statements/show.md | 51 +++++++ src/Interpreters/InterpreterFactory.cpp | 6 + .../InterpreterShowIndexesQuery.cpp | 124 ++++++++++++++++++ .../InterpreterShowIndexesQuery.h | 33 +++++ src/Parsers/ASTShowIndexesQuery.cpp | 38 ++++++ src/Parsers/ASTShowIndexesQuery.h | 29 ++++ src/Parsers/ParserQueryWithOutput.cpp | 3 + src/Parsers/ParserShowColumnsQuery.cpp | 2 +- src/Parsers/ParserShowIndexesQuery.cpp | 58 ++++++++ src/Parsers/ParserShowIndexesQuery.h | 20 +++ .../0_stateless/02724_show_indexes.reference | 37 ++++++ .../0_stateless/02724_show_indexes.sql | 59 +++++++++ 12 files changed, 459 insertions(+), 1 deletion(-) create mode 100644 src/Interpreters/InterpreterShowIndexesQuery.cpp create mode 100644 src/Interpreters/InterpreterShowIndexesQuery.h create mode 100644 src/Parsers/ASTShowIndexesQuery.cpp create mode 100644 src/Parsers/ASTShowIndexesQuery.h create mode 100644 src/Parsers/ParserShowIndexesQuery.cpp create mode 100644 src/Parsers/ParserShowIndexesQuery.h create mode 100644 tests/queries/0_stateless/02724_show_indexes.reference create mode 100644 tests/queries/0_stateless/02724_show_indexes.sql diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 6179d053b0f..db4d55305ce 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -261,6 +261,57 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 └──────────────┘ ``` +## SHOW INDEX + +Displays a list of primary and data skipping indexes of a table. + +```sql +SHOW [EXTENDED] {INDEX | INDEXES |KEYS } {FROM | IN} [{FROM | IN} ] [WHERE }] [INTO OUTFILE ] [FORMAT ] +``` + +The database and table name can be specified in abbreviated form as `.
`, i.e. `FROM tab FROM db` and `FROM db.tab` are +equivalent. If no database is specified, the query returns the list of columns from the current database. + +The optional keyword `EXTENDED` currently has no effect, it only exists for MySQL compatibility. + +`SHOW INDEX` produces a result table with the following structure: +- table - The name of the table (String) +- non_unique - 0 if the index can contain duplicates, 1 otherwise (UInt8) +- key_name - The name of the index. 'PRIMARY' if the index is a primary key index. (String) +- seq_in_index - currently unused +- column_name - currently unused +- collation - currently unused +- cardinality - currently unused +- sub_part - currently unused +- packed - currently unused +- null - currently unused +- index_type - the index type, e.g. `primary`, `minmax`, `bloom_filter` etc. (String) +- comment - currently unused +- index_comment - currently unused +- visible - if the index is visible to the optimizer, always `YES` (String) +- expression - the index expression (String) + +**Examples** + +Getting information about all indexes in table 'tbl' + +```sql +SHOW INDEX FROM 'orders' +``` + +Result: + +``` text +┌─table─┬─non_unique─┬─key_name─┬─seq_in_index─┬─column_name─┬─collation─┬─cardinality─┬─sub_part─┬─packed─┬─null─┬─index_type─┬─comment─┬─index_comment─┬─visible─┬─expression─┐ +│ tbl │ 0 │ mm1_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ a, c, d │ +│ tbl │ 0 │ mm2_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ c, d, e │ +└───────┴────────────┴──────────┴──────────────┴─────────────┴───────────┴─────────────┴──────────┴────────┴──────┴────────────┴─────────┴───────────────┴─────────┴────────────┘ +``` + +**See also** + +- [system.columns](https://clickhouse.com/docs/en/operations/system-tables/columns) + ## SHOW PROCESSLIST ``` sql diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 68b22b550f6..eec73fa56e2 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -81,6 +82,7 @@ #include #include #include +#include #include #include #include @@ -181,6 +183,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp new file mode 100644 index 00000000000..cb282b95451 --- /dev/null +++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp @@ -0,0 +1,124 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + + +InterpreterShowIndexesQuery::InterpreterShowIndexesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_) + , query_ptr(query_ptr_) +{ +} + + +String InterpreterShowIndexesQuery::getRewrittenQuery() +{ + const auto & query = query_ptr->as(); + + String database; + String table; + if (query.from_table.contains(".")) + { + /// FROM .
(abbreviated form) + chassert(query.from_database.empty()); + std::vector split; + boost::split(split, query.from_table, boost::is_any_of(".")); + chassert(split.size() == 2); + database = split[0]; + table = split[1]; + } + else if (query.from_database.empty()) + { + /// FROM
+ chassert(!query.from_table.empty()); + database = getContext()->getCurrentDatabase(); + table = query.from_table; + } + else + { + /// FROM FROM
+ chassert(!query.from_database.empty()); + chassert(!query.from_table.empty()); + database = query.from_database; + table = query.from_table; + } + + WriteBufferFromOwnString where_expression_buf; + if (query.where_expression) + where_expression_buf << "WHERE (" << query.where_expression << ")"; + String where_expression = where_expression_buf.str(); + + WriteBufferFromOwnString rewritten_query; + rewritten_query << "SELECT * FROM (" + << "(SELECT " + << "name AS table, " + << "0 AS non_unique, " + << "'PRIMARY' AS key_name, " + << "NULL AS seq_in_index, " + << "NULL AS column_name, " + << "'A' AS collation, " + << "NULL AS cardinality, " + << "NULL AS sub_part, " + << "NULL AS packed, " + << "NULL AS null, " + << "'primary' AS index_type, " + << "NULL AS comment, " + << "NULL AS index_comment, " + << "'YES' AS visible, " + << "primary_key AS expression " + << "FROM system.tables " + << "WHERE " + << "database = '" << database << "' " + << "AND name = '" << table << "'" + << ") UNION ALL (" + << "SELECT " + << "table AS table, " + << "0 AS non_unique, " + << "name AS key_name, " + << "NULL AS seq_in_index, " + << "NULL AS column_name, " + << "NULL AS collation, " + << "NULL AS cardinality, " + << "NULL AS sub_part, " + << "NULL AS packed, " + << "NULL AS null, " + << "type AS index_type, " + << "NULL AS comment, " + << "NULL AS index_comment, " + << "'YES' AS visible, " + << "expr AS expression " + << "FROM system.data_skipping_indices " + << "WHERE " + << "database = '" << database << "' " + << "AND table = '" << table << "'" + << ")) " + << where_expression; + + /// Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to + /// sort the output of SHOW COLUMNS otherwise (SELECT * FROM (SHOW COLUMNS ...) ORDER BY ...) is rejected) and 3. some + /// SQL tests can take advantage of this. + rewritten_query << " ORDER BY index_type, expression"; + + std::cout << rewritten_query.str() << std::endl; + + return rewritten_query.str(); +} + + +BlockIO InterpreterShowIndexesQuery::execute() +{ + return executeQuery(getRewrittenQuery(), getContext(), true); +} + + +} + diff --git a/src/Interpreters/InterpreterShowIndexesQuery.h b/src/Interpreters/InterpreterShowIndexesQuery.h new file mode 100644 index 00000000000..0b4fa591e35 --- /dev/null +++ b/src/Interpreters/InterpreterShowIndexesQuery.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class Context; + + +/// Returns a list of indexes which meet some conditions. +class InterpreterShowIndexesQuery : public IInterpreter, WithMutableContext +{ +public: + InterpreterShowIndexesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); + + BlockIO execute() override; + + /// Ignore quota and limits here because execute() produces a SELECT query which checks quotas/limits by itself. + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } + +private: + ASTPtr query_ptr; + + String getRewrittenQuery(); +}; + + +} + diff --git a/src/Parsers/ASTShowIndexesQuery.cpp b/src/Parsers/ASTShowIndexesQuery.cpp new file mode 100644 index 00000000000..6930043e5ed --- /dev/null +++ b/src/Parsers/ASTShowIndexesQuery.cpp @@ -0,0 +1,38 @@ +#include + +#include +#include +#include + +namespace DB +{ + +ASTPtr ASTShowIndexesQuery::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + cloneOutputOptions(*res); + return res; +} + +void ASTShowIndexesQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : "") + << "SHOW " + << (extended ? "EXTENDED " : "") + << "INDEXES" + << (settings.hilite ? hilite_none : ""); + + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from_table); + if (!from_database.empty()) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from_database); + + if (where_expression) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); + where_expression->formatImpl(settings, state, frame); + } +} + +} + diff --git a/src/Parsers/ASTShowIndexesQuery.h b/src/Parsers/ASTShowIndexesQuery.h new file mode 100644 index 00000000000..a0d5e0c6b6f --- /dev/null +++ b/src/Parsers/ASTShowIndexesQuery.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Query SHOW INDEXES +class ASTShowIndexesQuery : public ASTQueryWithOutput +{ +public: + bool extended = false; + + ASTPtr where_expression; + + String from_database; + String from_table; + + String getID(char) const override { return "ShowColumns"; } + ASTPtr clone() const override; + QueryKind getQueryKind() const override { return QueryKind::Show; } + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; + +} + diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index 2fb7c406d74..22e32fa5c45 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserShowTablesQuery show_tables_p; ParserShowColumnsQuery show_columns_p; ParserShowEnginesQuery show_engine_p; + ParserShowIndexesQuery show_indexes_p; ParserSelectWithUnionQuery select_p; ParserTablePropertiesQuery table_p; ParserDescribeTableQuery describe_table_p; @@ -69,6 +71,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec || show_tables_p.parse(pos, query, expected) || show_columns_p.parse(pos, query, expected) || show_engine_p.parse(pos, query, expected) + || show_indexes_p.parse(pos, query, expected) || table_p.parse(pos, query, expected) || describe_cache_p.parse(pos, query, expected) || describe_table_p.parse(pos, query, expected) diff --git a/src/Parsers/ParserShowColumnsQuery.cpp b/src/Parsers/ParserShowColumnsQuery.cpp index 03e66e4ae0f..57328bb0d8b 100644 --- a/src/Parsers/ParserShowColumnsQuery.cpp +++ b/src/Parsers/ParserShowColumnsQuery.cpp @@ -27,7 +27,7 @@ bool ParserShowColumnsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (ParserKeyword("FULL").ignore(pos, expected)) query->full = true; - if (!ParserKeyword("COLUMNS").ignore(pos, expected) || ParserKeyword("FIELDS").ignore(pos, expected)) + if (!(ParserKeyword("COLUMNS").ignore(pos, expected) || ParserKeyword("FIELDS").ignore(pos, expected))) return false; if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected)) diff --git a/src/Parsers/ParserShowIndexesQuery.cpp b/src/Parsers/ParserShowIndexesQuery.cpp new file mode 100644 index 00000000000..4b7eba8636e --- /dev/null +++ b/src/Parsers/ParserShowIndexesQuery.cpp @@ -0,0 +1,58 @@ +#include + +// TODO +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ParserShowIndexesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ASTPtr from_database; + ASTPtr from_table; + + auto query = std::make_shared(); + + if (!ParserKeyword("SHOW").ignore(pos, expected)) + return false; + + if (ParserKeyword("EXTENDED").ignore(pos, expected)) + query->extended = true; + + if (!(ParserKeyword("INDEX").ignore(pos, expected) || ParserKeyword("INDEXES").ignore(pos, expected) || ParserKeyword("KEYS").ignore(pos, expected))) + return false; + + if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected)) + { + if (!ParserCompoundIdentifier().parse(pos, from_table, expected)) + return false; + } + else + return false; + + tryGetIdentifierNameInto(from_table, query->from_table); + bool abbreviated_form = query->from_table.contains("."); /// FROM .
+ + if (!abbreviated_form) + if (ParserKeyword("FROM").ignore(pos, expected) || ParserKeyword("IN").ignore(pos, expected)) + if (!ParserIdentifier().parse(pos, from_database, expected)) + return false; + + tryGetIdentifierNameInto(from_database, query->from_database); + + if (ParserKeyword("WHERE").ignore(pos, expected)) + if (!ParserExpressionWithOptionalAlias(false).parse(pos, query->where_expression, expected)) + return false; + + node = query; + + return true; +} + +} + diff --git a/src/Parsers/ParserShowIndexesQuery.h b/src/Parsers/ParserShowIndexesQuery.h new file mode 100644 index 00000000000..aaa9e8789ff --- /dev/null +++ b/src/Parsers/ParserShowIndexesQuery.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace DB +{ + +/** Parses queries of the form + * SHOW [EXTENDED] INDEX|INDEXES|KEYS (FROM|IN) tbl [(FROM|IN) db] (WHERE expr) + */ +class ParserShowIndexesQuery : public IParserBase +{ +protected: + const char * getName() const override { return "SHOW INDEXES query"; } + + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} + diff --git a/tests/queries/0_stateless/02724_show_indexes.reference b/tests/queries/0_stateless/02724_show_indexes.reference new file mode 100644 index 00000000000..2ca55c77015 --- /dev/null +++ b/tests/queries/0_stateless/02724_show_indexes.reference @@ -0,0 +1,37 @@ +--- Aliases of SHOW INDEX +tbl 0 blf_idx \N \N \N \N \N \N \N bloom_filter \N \N YES d, b +tbl 0 mm1_idx \N \N \N \N \N \N \N minmax \N \N YES a, c, d +tbl 0 mm2_idx \N \N \N \N \N \N \N minmax \N \N YES c, d, e +tbl 0 PRIMARY \N \N A \N \N \N \N primary \N \N YES c, a +tbl 0 set_idx \N \N \N \N \N \N \N set \N \N YES e +tbl 0 blf_idx \N \N \N \N \N \N \N bloom_filter \N \N YES d, b +tbl 0 mm1_idx \N \N \N \N \N \N \N minmax \N \N YES a, c, d +tbl 0 mm2_idx \N \N \N \N \N \N \N minmax \N \N YES c, d, e +tbl 0 PRIMARY \N \N A \N \N \N \N primary \N \N YES c, a +tbl 0 set_idx \N \N \N \N \N \N \N set \N \N YES e +tbl 0 blf_idx \N \N \N \N \N \N \N bloom_filter \N \N YES d, b +tbl 0 mm1_idx \N \N \N \N \N \N \N minmax \N \N YES a, c, d +tbl 0 mm2_idx \N \N \N \N \N \N \N minmax \N \N YES c, d, e +tbl 0 PRIMARY \N \N A \N \N \N \N primary \N \N YES c, a +tbl 0 set_idx \N \N \N \N \N \N \N set \N \N YES e +--- EXTENDED +tbl 0 blf_idx \N \N \N \N \N \N \N bloom_filter \N \N YES d, b +tbl 0 mm1_idx \N \N \N \N \N \N \N minmax \N \N YES a, c, d +tbl 0 mm2_idx \N \N \N \N \N \N \N minmax \N \N YES c, d, e +tbl 0 PRIMARY \N \N A \N \N \N \N primary \N \N YES c, a +tbl 0 set_idx \N \N \N \N \N \N \N set \N \N YES e +--- WHERE +tbl 0 mm1_idx \N \N \N \N \N \N \N minmax \N \N YES a, c, d +tbl 0 mm2_idx \N \N \N \N \N \N \N minmax \N \N YES c, d, e +--- Original table +tbl 0 blf_idx \N \N \N \N \N \N \N bloom_filter \N \N YES d, b +tbl 0 mm1_idx \N \N \N \N \N \N \N minmax \N \N YES a, c, d +tbl 0 mm2_idx \N \N \N \N \N \N \N minmax \N \N YES c, d, e +tbl 0 PRIMARY \N \N A \N \N \N \N primary \N \N YES c, a +tbl 0 set_idx \N \N \N \N \N \N \N set \N \N YES e +--- Equally named table in other database +tbl 0 mmi_idx \N \N \N \N \N \N \N minmax \N \N YES b +tbl 0 PRIMARY \N \N A \N \N \N \N primary \N \N YES a +--- Short form +tbl 0 mmi_idx \N \N \N \N \N \N \N minmax \N \N YES b +tbl 0 PRIMARY \N \N A \N \N \N \N primary \N \N YES a diff --git a/tests/queries/0_stateless/02724_show_indexes.sql b/tests/queries/0_stateless/02724_show_indexes.sql new file mode 100644 index 00000000000..477e2f049c2 --- /dev/null +++ b/tests/queries/0_stateless/02724_show_indexes.sql @@ -0,0 +1,59 @@ +-- Tags: no-parallel +-- no-parallel: creates a custom database schema and expects to use it exclusively + +-- Create a test table and verify that the output of SHOW INDEXES is sane. +-- The matching of actual/expected results relies on the fact that the output of SHOW INDEX is sorted. +DROP TABLE IF EXISTS tbl; +CREATE TABLE tbl +( + a UInt64, + b UInt64, + c UInt64, + d UInt64, + e UInt64, + INDEX mm1_idx (a, c, d) TYPE minmax, + INDEX mm2_idx (c, d, e) TYPE minmax, + INDEX set_idx (e) TYPE set(100), + INDEX blf_idx (d, b) TYPE bloom_filter(0.8) +) +ENGINE = MergeTree +PRIMARY KEY (c, a); + +SELECT '--- Aliases of SHOW INDEX'; +SHOW INDEX FROM tbl; +SHOW INDEXES FROM tbl; +SHOW KEYS FROM tbl; + +SELECT '--- EXTENDED'; +SHOW EXTENDED INDEX FROM tbl; +-- +SELECT '--- WHERE'; +SHOW INDEX FROM tbl WHERE index_type LIKE '%minmax%'; + +-- Create a table in a different database. Intentionally using the same table/column names as above so +-- we notice if something is buggy in the implementation of SHOW INDEX. +DROP DATABASE IF EXISTS database_123456789abcde; +CREATE DATABASE database_123456789abcde; -- pseudo-random database name + +DROP TABLE IF EXISTS database_123456789abcde.tbl; +CREATE TABLE database_123456789abcde.tbl +( + a UInt64, + b UInt64, + INDEX mmi_idx b TYPE minmax +) +ENGINE = MergeTree +PRIMARY KEY a; + +SELECT '--- Original table'; +SHOW INDEX FROM tbl; + +SELECT '--- Equally named table in other database'; +SHOW INDEX FROM tbl FROM database_123456789abcde; + +SELECT '--- Short form'; +SHOW INDEX FROM database_123456789abcde.tbl; + +DROP DATABASE database_123456789abcde; + +DROP TABLE tbl; From 403396d0bdec5a75ac736f483eb7b3e779d2a18c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 25 Apr 2023 21:04:55 +0000 Subject: [PATCH 045/308] Fix style --- src/Interpreters/InterpreterShowIndexesQuery.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp index cb282b95451..d47a20e1ba6 100644 --- a/src/Interpreters/InterpreterShowIndexesQuery.cpp +++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp @@ -108,8 +108,6 @@ String InterpreterShowIndexesQuery::getRewrittenQuery() /// SQL tests can take advantage of this. rewritten_query << " ORDER BY index_type, expression"; - std::cout << rewritten_query.str() << std::endl; - return rewritten_query.str(); } From c58e0e347cc987dc67573b0b980b56f3d72727fd Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 26 Apr 2023 09:46:35 +0000 Subject: [PATCH 046/308] Various cleanups --- docs/en/sql-reference/statements/show.md | 46 ++++++++++--------- .../InterpreterShowColumnsQuery.cpp | 31 ++----------- .../InterpreterShowIndexesQuery.cpp | 30 +----------- src/Interpreters/parseDatabaseAndTableName.h | 44 ++++++++++++++++++ src/Parsers/ParserShowColumnsQuery.h | 2 +- src/Parsers/ParserShowIndexesQuery.cpp | 1 - src/Parsers/ParserShowIndexesQuery.h | 2 +- .../0_stateless/02706_show_columns.reference | 4 ++ .../0_stateless/02706_show_columns.sql | 2 + 9 files changed, 82 insertions(+), 80 deletions(-) create mode 100644 src/Interpreters/parseDatabaseAndTableName.h diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index db4d55305ce..45d09c2dec7 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -266,51 +266,55 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 Displays a list of primary and data skipping indexes of a table. ```sql -SHOW [EXTENDED] {INDEX | INDEXES |KEYS } {FROM | IN}
[{FROM | IN} ] [WHERE }] [INTO OUTFILE ] [FORMAT ] +SHOW [EXTENDED] {INDEX | INDEXES | KEYS } {FROM | IN}
[{FROM | IN} ] [WHERE ] [INTO OUTFILE ] [FORMAT ] ``` The database and table name can be specified in abbreviated form as `.
`, i.e. `FROM tab FROM db` and `FROM db.tab` are -equivalent. If no database is specified, the query returns the list of columns from the current database. +equivalent. If no database is specified, the query assumes the current database as database. The optional keyword `EXTENDED` currently has no effect, it only exists for MySQL compatibility. `SHOW INDEX` produces a result table with the following structure: - table - The name of the table (String) - non_unique - 0 if the index can contain duplicates, 1 otherwise (UInt8) -- key_name - The name of the index. 'PRIMARY' if the index is a primary key index. (String) -- seq_in_index - currently unused -- column_name - currently unused -- collation - currently unused -- cardinality - currently unused -- sub_part - currently unused -- packed - currently unused -- null - currently unused -- index_type - the index type, e.g. `primary`, `minmax`, `bloom_filter` etc. (String) -- comment - currently unused -- index_comment - currently unused -- visible - if the index is visible to the optimizer, always `YES` (String) -- expression - the index expression (String) +- key_name - The name of the index, `PRIMARY` if the index is a primary key index (String) +- seq_in_index - Currently unused +- column_name - Currently unused +- collation - The sorting of the column in the index, `A` if ascending, `D` if descending, `NULL` if unsorted (Nullable(String)) +- cardinality - Currently unused +- sub_part - Currently unused +- packed - Currently unused +- null - Currently unused +- index_type - The index type, e.g. `primary`, `minmax`, `bloom_filter` etc. (String) +- comment - Currently unused +- index_comment - Currently unused +- visible - If the index is visible to the optimizer, always `YES` (String) +- expression - The index expression (String) **Examples** Getting information about all indexes in table 'tbl' ```sql -SHOW INDEX FROM 'orders' +SHOW INDEX FROM 'tbl' ``` Result: ``` text -┌─table─┬─non_unique─┬─key_name─┬─seq_in_index─┬─column_name─┬─collation─┬─cardinality─┬─sub_part─┬─packed─┬─null─┬─index_type─┬─comment─┬─index_comment─┬─visible─┬─expression─┐ -│ tbl │ 0 │ mm1_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ a, c, d │ -│ tbl │ 0 │ mm2_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ c, d, e │ -└───────┴────────────┴──────────┴──────────────┴─────────────┴───────────┴─────────────┴──────────┴────────┴──────┴────────────┴─────────┴───────────────┴─────────┴────────────┘ +┌─table─┬─non_unique─┬─key_name─┬─seq_in_index─┬─column_name─┬─collation─┬─cardinality─┬─sub_part─┬─packed─┬─null─┬─index_type───┬─comment─┬─index_comment─┬─visible─┬─expression─┐ +│ tbl │ 0 │ blf_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ bloom_filter │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ d, b │ +│ tbl │ 0 │ mm1_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ a, c, d │ +│ tbl │ 0 │ mm2_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ minmax │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ c, d, e │ +│ tbl │ 0 │ PRIMARY │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ A │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ primary │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ c, a │ +│ tbl │ 0 │ set_idx │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ set │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ YES │ e │ +└───────┴────────────┴──────────┴──────────────┴─────────────┴───────────┴─────────────┴──────────┴────────┴──────┴──────────────┴─────────┴───────────────┴─────────┴────────────┘ ``` **See also** -- [system.columns](https://clickhouse.com/docs/en/operations/system-tables/columns) +- [system.tables](../../operations/system-tables/tables.md) +- [system.data_skipping_indices](../../operations/system-tables/data_skipping_indices.md) ## SHOW PROCESSLIST diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp index 4474be21d8b..86788007e3f 100644 --- a/src/Interpreters/InterpreterShowColumnsQuery.cpp +++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp @@ -5,8 +5,8 @@ #include #include #include +#include #include -#include namespace DB @@ -42,33 +42,8 @@ String InterpreterShowColumnsQuery::getRewrittenQuery() rewritten_query << "FROM system.columns WHERE "; - String database; - String table; - if (query.from_table.contains(".")) - { - /// FROM .
(abbreviated form) - chassert(query.from_database.empty()); - std::vector split; - boost::split(split, query.from_table, boost::is_any_of(".")); - chassert(split.size() == 2); - database = split[0]; - table = split[1]; - } - else if (query.from_database.empty()) - { - /// FROM
- chassert(!query.from_table.empty()); - database = getContext()->getCurrentDatabase(); - table = query.from_table; - } - else - { - /// FROM FROM
- chassert(!query.from_database.empty()); - chassert(!query.from_table.empty()); - database = query.from_database; - table = query.from_table; - } + auto [database, table] = parseDatabaseAndTableName(query, getContext()->getCurrentDatabase()); + rewritten_query << "database = " << DB::quote << database; rewritten_query << " AND table = " << DB::quote << table; diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp index d47a20e1ba6..c5a63cde51e 100644 --- a/src/Interpreters/InterpreterShowIndexesQuery.cpp +++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp @@ -5,8 +5,8 @@ #include #include #include +#include #include -#include namespace DB @@ -24,33 +24,7 @@ String InterpreterShowIndexesQuery::getRewrittenQuery() { const auto & query = query_ptr->as(); - String database; - String table; - if (query.from_table.contains(".")) - { - /// FROM .
(abbreviated form) - chassert(query.from_database.empty()); - std::vector split; - boost::split(split, query.from_table, boost::is_any_of(".")); - chassert(split.size() == 2); - database = split[0]; - table = split[1]; - } - else if (query.from_database.empty()) - { - /// FROM
- chassert(!query.from_table.empty()); - database = getContext()->getCurrentDatabase(); - table = query.from_table; - } - else - { - /// FROM FROM
- chassert(!query.from_database.empty()); - chassert(!query.from_table.empty()); - database = query.from_database; - table = query.from_table; - } + auto [database, table] = parseDatabaseAndTableName(query, getContext()->getCurrentDatabase()); WriteBufferFromOwnString where_expression_buf; if (query.where_expression) diff --git a/src/Interpreters/parseDatabaseAndTableName.h b/src/Interpreters/parseDatabaseAndTableName.h new file mode 100644 index 00000000000..b336a82028b --- /dev/null +++ b/src/Interpreters/parseDatabaseAndTableName.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + template + inline std::pair parseDatabaseAndTableName(const Query & query, const String & current_database) + { + String database; + String table; + + if (query.from_table.contains(".")) + { + /// FROM .
(abbreviated form) + chassert(query.from_database.empty()); + std::vector split; + boost::split(split, query.from_table, boost::is_any_of(".")); + chassert(split.size() == 2); + database = split[0]; + table = split[1]; + } + else if (query.from_database.empty()) + { + /// FROM
+ chassert(!query.from_table.empty()); + database = current_database; + table = query.from_table; + } + else + { + /// FROM FROM
+ chassert(!query.from_database.empty()); + chassert(!query.from_table.empty()); + database = query.from_database; + table = query.from_table; + } + + return {database, table}; + } +} diff --git a/src/Parsers/ParserShowColumnsQuery.h b/src/Parsers/ParserShowColumnsQuery.h index 999acf722af..a2941ca71d1 100644 --- a/src/Parsers/ParserShowColumnsQuery.h +++ b/src/Parsers/ParserShowColumnsQuery.h @@ -6,7 +6,7 @@ namespace DB { /** Parses queries of the form - * SHOW [EXTENDED] [FULL] COLUMNS (FROM|IN) tbl [(FROM|IN) db] [(([NOT] (LIKE|ILIKE) expr) | (WHERE expr))] [LIMIT n] + * SHOW [EXTENDED] [FULL] COLUMNS FROM|IN tbl [FROM|IN db] [[NOT] LIKE|ILIKE expr | WHERE expr] [LIMIT n] */ class ParserShowColumnsQuery : public IParserBase { diff --git a/src/Parsers/ParserShowIndexesQuery.cpp b/src/Parsers/ParserShowIndexesQuery.cpp index 4b7eba8636e..b4651ed21f9 100644 --- a/src/Parsers/ParserShowIndexesQuery.cpp +++ b/src/Parsers/ParserShowIndexesQuery.cpp @@ -1,6 +1,5 @@ #include -// TODO #include #include #include diff --git a/src/Parsers/ParserShowIndexesQuery.h b/src/Parsers/ParserShowIndexesQuery.h index aaa9e8789ff..0ece5e89ef7 100644 --- a/src/Parsers/ParserShowIndexesQuery.h +++ b/src/Parsers/ParserShowIndexesQuery.h @@ -6,7 +6,7 @@ namespace DB { /** Parses queries of the form - * SHOW [EXTENDED] INDEX|INDEXES|KEYS (FROM|IN) tbl [(FROM|IN) db] (WHERE expr) + * SHOW [EXTENDED] INDEX|INDEXES|KEYS FROM|IN tbl [FROM|IN db] [WHERE expr] */ class ParserShowIndexesQuery : public IParserBase { diff --git a/tests/queries/0_stateless/02706_show_columns.reference b/tests/queries/0_stateless/02706_show_columns.reference index da967d59cda..fff8d38ede7 100644 --- a/tests/queries/0_stateless/02706_show_columns.reference +++ b/tests/queries/0_stateless/02706_show_columns.reference @@ -1,3 +1,7 @@ +--- Aliases of SHOW COLUMNS +int32 Nullable(Int32) 1 \N +str String 0 SOR \N +uint64 UInt64 0 PRI SOR \N int32 Nullable(Int32) 1 \N str String 0 SOR \N uint64 UInt64 0 PRI SOR \N diff --git a/tests/queries/0_stateless/02706_show_columns.sql b/tests/queries/0_stateless/02706_show_columns.sql index ce206fedee4..522f513b685 100644 --- a/tests/queries/0_stateless/02706_show_columns.sql +++ b/tests/queries/0_stateless/02706_show_columns.sql @@ -15,7 +15,9 @@ ENGINE = MergeTree PRIMARY KEY (uint64) ORDER BY (uint64, str); +SELECT '--- Aliases of SHOW COLUMNS'; SHOW COLUMNS FROM tab; +SHOW FIELDS FROM tab; SELECT '--- EXTENDED'; SHOW EXTENDED COLUMNS FROM tab; From 30f1bef6e8d1a2086d8ef240601cf99e0e47887e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 27 Apr 2023 16:57:42 +0300 Subject: [PATCH 047/308] Update TableFunctionS3.h --- src/TableFunctions/TableFunctionS3.h | 44 ---------------------------- 1 file changed, 44 deletions(-) diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index ed8cd3bd41a..70c4a020669 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -67,52 +67,8 @@ protected: ColumnsDescription structure_hint; }; -class TableFunctionCOS : public TableFunctionS3 -{ -public: - static constexpr auto name = "cosn"; - std::string getName() const override - { - return name; - } -private: - const char * getStorageTypeName() const override { return "COSN"; } -}; - -class TableFunctionOSS : public TableFunctionS3 -{ -public: - static constexpr auto name = "oss"; - std::string getName() const override - { - return name; - } -private: - const char * getStorageTypeName() const override { return "OSS"; } -}; - } -class TableFunctionGCS : public TableFunctionS3 -{ -public: - static constexpr auto name = "gcs"; - static constexpr auto signature = " - url\n" - " - url, format\n" - " - url, format, structure\n" - " - url, hmac_key, hmac_secret\n" - " - url, format, structure, compression_method\n" - " - url, hmac_key, hmac_secret, format\n" - " - url, hmac_key, hmac_secret, format, structure\n" - " - url, hmac_key, hmac_secret, format, structure, compression_method"; - std::string getName() const override - { - return name; - } -private: - const char * getStorageTypeName() const override { return "GCS"; } -}; - } #endif From aec8f17614371f8dc4ea161098736adafeddc70b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 27 Apr 2023 16:58:18 +0300 Subject: [PATCH 048/308] Update TableFunctionS3.cpp --- src/TableFunctions/TableFunctionS3.cpp | 47 +++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index 4153c6a81c9..841f5a91bc8 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -215,6 +215,48 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, Context } +class TableFunctionGCS : public TableFunctionS3 +{ +public: + static constexpr auto name = "gcs"; + std::string getName() const override + { + return name; + } +private: + const char * getStorageTypeName() const override { return "GCS"; } +}; + +class TableFunctionCOS : public TableFunctionS3 +{ +public: + static constexpr auto name = "cosn"; + std::string getName() const override + { + return name; + } +private: + const char * getStorageTypeName() const override { return "COSN"; } +}; + +class TableFunctionOSS : public TableFunctionS3 +{ +public: + static constexpr auto name = "oss"; + std::string getName() const override + { + return name; + } +private: + const char * getStorageTypeName() const override { return "OSS"; } +}; + + +void registerTableFunctionGCS(TableFunctionFactory & factory) +{ + factory.registerFunction(); +} + void registerTableFunctionS3(TableFunctionFactory & factory) { factory.registerFunction(); @@ -230,11 +272,6 @@ void registerTableFunctionOSS(TableFunctionFactory & factory) factory.registerFunction(); } -void registerTableFunctionGCS(TableFunctionFactory & factory) -{ - factory.registerFunction(); -} - } #endif From 536720605797f37a2870c79855a633cca3cd5faa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 27 Apr 2023 23:24:36 +0300 Subject: [PATCH 049/308] Update TableFunctionS3.h --- src/TableFunctions/TableFunctionS3.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 70c4a020669..4724684712b 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -69,6 +69,4 @@ protected: } -} - #endif From b24de29c705021ee3cdea39b91cdf46abea042d4 Mon Sep 17 00:00:00 2001 From: Mike Kot Date: Thu, 27 Apr 2023 20:37:12 +0000 Subject: [PATCH 050/308] reverting mr changes --- src/Interpreters/Context.cpp | 6 +-- src/Interpreters/Context.h | 2 +- tests/integration/ch_runner.sh | 1 - .../_gen/cluster_for_concurrency_test.xml | 48 ------------------- 4 files changed, 4 insertions(+), 53 deletions(-) delete mode 100755 tests/integration/ch_runner.sh delete mode 100644 tests/integration/test_backup_restore_on_cluster/_gen/cluster_for_concurrency_test.xml diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e8c333ae9b0..0cb3f8884ef 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1630,7 +1630,7 @@ void Context::setSettings(const Settings & settings_) calculateAccessRights(); } -void Context::recalcRightsIfNeeded(std::string_view name) +void Context::recalculateAccessRightsIfNeeded(std::string_view name) { if (name == "readonly" || name == "allow_ddl" @@ -1648,7 +1648,7 @@ void Context::setSetting(std::string_view name, const String & value) return; } settings.set(name, value); - recalcRightsIfNeeded(name); + recalculateAccessRightsIfNeeded(name); } void Context::setSetting(std::string_view name, const Field & value) @@ -1660,7 +1660,7 @@ void Context::setSetting(std::string_view name, const Field & value) return; } settings.set(name, value); - recalcRightsIfNeeded(name); + recalculateAccessRightsIfNeeded(name); } void Context::applySettingChange(const SettingChange & change) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 3283b9260e0..e9dd6d899b6 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1145,7 +1145,7 @@ private: /// Compute and set actual user settings, client_info.current_user should be set void calculateAccessRights(); - void recalcRightsIfNeeded(std::string_view setting_name); + void recalculateAccessRightsIfNeeded(std::string_view setting_name); template void checkAccessImpl(const Args &... args) const; diff --git a/tests/integration/ch_runner.sh b/tests/integration/ch_runner.sh deleted file mode 100755 index 73cd8cbb1c5..00000000000 --- a/tests/integration/ch_runner.sh +++ /dev/null @@ -1 +0,0 @@ -./runner --binary $HOME/ch/build/programs/clickhouse --odbc-bridge-binary $HOME/ch/build/programs/clickhouse-odbc-bridge --base-configs-dir $HOME/ch/programs/server/ "$1 -ss" diff --git a/tests/integration/test_backup_restore_on_cluster/_gen/cluster_for_concurrency_test.xml b/tests/integration/test_backup_restore_on_cluster/_gen/cluster_for_concurrency_test.xml deleted file mode 100644 index 08684e34e45..00000000000 --- a/tests/integration/test_backup_restore_on_cluster/_gen/cluster_for_concurrency_test.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - node0 - 9000 - - - node1 - 9000 - - - node2 - 9000 - - - node3 - 9000 - - - node4 - 9000 - - - node5 - 9000 - - - node6 - 9000 - - - node7 - 9000 - - - node8 - 9000 - - - node9 - 9000 - - - - - \ No newline at end of file From e428af0b6387a1e669e2ecaa457907b4589279e2 Mon Sep 17 00:00:00 2001 From: Mike Kot Date: Thu, 27 Apr 2023 21:27:18 +0000 Subject: [PATCH 051/308] replace format settings with server setting --- .../settings.md | 18 ++++++++++++++++++ .../en/operations/settings/settings-formats.md | 18 ------------------ docs/en/sql-reference/statements/grant.md | 4 ++-- docs/en/sql-reference/statements/show.md | 2 +- src/Core/ServerSettings.h | 1 + src/Core/Settings.h | 1 - src/Interpreters/Context.cpp | 12 +++++++----- src/Interpreters/Context.h | 1 + .../formatWithPossiblyHidingSecrets.h | 2 +- 9 files changed, 31 insertions(+), 28 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index af0d2415f1d..bd35b08ed88 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -2056,3 +2056,21 @@ Possible values: - Positive integer. Default value: `10000`. + +## display_secrets_in_show_and_select_query {#display_secrets_in_show_and_select_query} + +Enables or disables showing secrets in `SHOW` and `SELECT` queries for tables, databases, +table functions, and dictionaries. +User wishing to see secrets must also have +[`displaySecretsInShowSelect`](../../sql-reference/statements/grant#grant-display-secrets) privilege. + +:::info Warning Turning this setting on allows users with all privileges to view secrets immediately. +Please grant/revoke [`displaySecretsInShowSelect`](../../sql-reference/statements/grant#grant-display-secrets) +privilege first and turn this setting on after that ::: + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: 0. diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 683a173375f..ef4bbeeba89 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -7,24 +7,6 @@ toc_max_heading_level: 2 # Format settings {#format-settings} -## display_secrets_in_show_and_select_query {#display_secrets_in_show_and_select_query} - -Enables or disables showing secrets in `SHOW` and `SELECT` queries for tables, databases, -table functions, and dictionaries. -User wishing to see secrets must also have -[`displaySecretsInShowSelect`](../../sql-reference/statements/grant#grant-display-secrets) privilege. - -:::info Warning Turning this setting on allows users with all privileges to view secrets immediately. -Please grant/revoke [`displaySecretsInShowSelect`](../../sql-reference/statements/grant#grant-display-secrets) -privilege first and turn this setting on after that ::: - -Possible values: - -- 0 — Disabled. -- 1 — Enabled. - -Default value: 0. - ## input_format_skip_unknown_fields {#input_format_skip_unknown_fields} Enables or disables skipping insertion of extra data. diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 6ec1999fc3f..7185d86b5bc 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -188,7 +188,7 @@ Hierarchy of privileges: - `HDFS` - `S3` - [dictGet](#grant-dictget) -- [displaySecretsInShowSelect](#grant-display-secrets) +- [displaySecretsInShowSelect](#grant-display-secrets) Examples of how this hierarchy is treated: @@ -478,7 +478,7 @@ Privilege level: `DICTIONARY`. ### displaySecretsInShowSelect {#grant-display-secrets} Allows a user to view secrets in `SHOW` and `SELECT` queries if -[`display_secrets_in_show_select_query`](../../operations/settings/formats#display_secrets_in_show_select_query) +[`display_secrets_in_show_and_select_query`](../../operations/settings/server-configuration-parameters/settings#display_secrets_in_show_and_select_query) setting is turned on. Otherwise this privilege does nothing. ### ALL diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 95bf0dfffdf..84fca82eb9c 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -7,7 +7,7 @@ sidebar_label: SHOW # SHOW Statements N.B. `SHOW CREATE (TABLE|DATABASE|USER)` hides secrets unless -[`display_secrets_in_show_select_query`](../../operations/settings/formats#display_secrets_in_show_select_query) +[`display_secrets_in_show_and_select_query`](../../operations/server-configuration-parameters/settings#display_secrets_in_show_and_select_query) is turned on and user has [`displaySecretsInShowSelect`](grant.md#grant-display-secrets) privilege. diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index aabc89cc6d7..5819583fc12 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -74,6 +74,7 @@ namespace DB M(UInt64, background_schedule_pool_size, 128, "The maximum number of threads that will be used for constantly executing some lightweight periodic operations.", 0) \ M(UInt64, background_message_broker_schedule_pool_size, 16, "The maximum number of threads that will be used for executing background operations for message streaming.", 0) \ M(UInt64, background_distributed_schedule_pool_size, 16, "The maximum number of threads that will be used for executing distributed sends.", 0) \ + M(Bool, display_secrets_in_show_and_select_query, false, "Do not hide secrets in SHOW and SELECT queries. User must also have 'displaySecretsInShowSelect' privilege", IMPORTANT) /* Although formally this is a format setting, it may introduce security issues, so it's better to add it at a server level */ \ DECLARE_SETTINGS_TRAITS(ServerSettingsTraits, SERVER_SETTINGS) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 320a3b7455b..81c5267c4ba 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -960,7 +960,6 @@ class IColumn; M(Bool, input_format_bson_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format BSON.", 0) \ \ M(Bool, regexp_dict_allow_other_sources, false, "Allow regexp_tree dictionary to use sources other than yaml source.", 0) \ - M(Bool, display_secrets_in_show_and_select_query, false, "Do not hide secrets in SHOW and SELECT queries. User must also have 'displaySecretsInShowSelect' privilege", IMPORTANT) \ M(Bool, dictionary_use_async_executor, false, "Execute a pipeline for reading from a dictionary with several threads. It's supported only by DIRECT dictionary with CLICKHOUSE source.", 0) \ // End of FORMAT_FACTORY_SETTINGS diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 0cb3f8884ef..c61dfededa3 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1607,6 +1607,11 @@ StoragePtr Context::getViewSource() const return view_source; } +bool Context::displaySecretsInShowSelect() const +{ + return shared->server_settings.display_secrets_in_show_and_select_query; +} + Settings Context::getSettings() const { auto lock = getLock(); @@ -1619,14 +1624,12 @@ void Context::setSettings(const Settings & settings_) const auto old_readonly = settings.readonly; const auto old_allow_ddl = settings.allow_ddl; const auto old_allow_introspection_functions = settings.allow_introspection_functions; - const auto old_display_secrets = settings.display_secrets_in_show_and_select_query; settings = settings_; if ((settings.readonly != old_readonly) || (settings.allow_ddl != old_allow_ddl) - || (settings.allow_introspection_functions != old_allow_introspection_functions) - || (settings.display_secrets_in_show_and_select_query != old_display_secrets)) + || (settings.allow_introspection_functions != old_allow_introspection_functions)) calculateAccessRights(); } @@ -1634,8 +1637,7 @@ void Context::recalculateAccessRightsIfNeeded(std::string_view name) { if (name == "readonly" || name == "allow_ddl" - || name == "allow_introspection_functions" - || name == "display_secrets_in_show_and_select_query") + || name == "allow_introspection_functions") calculateAccessRights(); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index e9dd6d899b6..b80597e5bfb 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -683,6 +683,7 @@ public: MultiVersion::Version getMacros() const; void setMacros(std::unique_ptr && macros); + bool displaySecretsInShowSelect() const; Settings getSettings() const; void setSettings(const Settings & settings_); diff --git a/src/Interpreters/formatWithPossiblyHidingSecrets.h b/src/Interpreters/formatWithPossiblyHidingSecrets.h index 65cb019cf9b..fb6b90da025 100644 --- a/src/Interpreters/formatWithPossiblyHidingSecrets.h +++ b/src/Interpreters/formatWithPossiblyHidingSecrets.h @@ -16,7 +16,7 @@ struct SecretHidingFormatSettings inline String format(const SecretHidingFormatSettings & settings) { - const bool show_secrets = settings.ctx->getSettingsRef().display_secrets_in_show_and_select_query + const bool show_secrets = settings.ctx->displaySecretsInShowSelect() && settings.ctx->getAccess()->isGranted(AccessType::displaySecretsInShowSelect); return settings.query.formatWithPossiblyHidingSensitiveData(settings.max_length, settings.one_line, show_secrets); From 2da33b96eba3698dd347d7a0c5d487cfa031f85d Mon Sep 17 00:00:00 2001 From: xmy Date: Fri, 28 Apr 2023 10:31:49 +0800 Subject: [PATCH 052/308] Allow Int* type argument for groupBitAnd/GroupBitOr/groupBitXor --- .../aggregate-functions/reference/groupbitand.md | 4 ++-- .../aggregate-functions/reference/groupbitor.md | 4 ++-- .../aggregate-functions/reference/groupbitxor.md | 4 ++-- .../AggregateFunctionBitwise.cpp | 2 +- .../AggregateFunctionSequenceMatch.cpp | 2 +- .../AggregateFunctionWindowFunnel.cpp | 2 +- src/AggregateFunctions/Helpers.h | 14 ++++++++++++-- 7 files changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md index f89e3796aaa..5fd5029751a 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitand.md @@ -13,11 +13,11 @@ groupBitAnd(expr) **Arguments** -`expr` – An expression that results in `UInt*` type. +`expr` – An expression that results in `UInt* or Int*` type. **Return value** -Value of the `UInt*` type. +Value of the `UInt* or Int*` type. **Example** diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md index 75b34d9c5a3..08a5c15da46 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md @@ -13,11 +13,11 @@ groupBitOr(expr) **Arguments** -`expr` – An expression that results in `UInt*` type. +`expr` – An expression that results in `UInt* or Int*` type. **Returned value** -Value of the `UInt*` type. +Value of the `UInt* or Int*` type. **Example** diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md index ca6fb9f8352..f33e375953c 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitxor.md @@ -13,11 +13,11 @@ groupBitXor(expr) **Arguments** -`expr` – An expression that results in `UInt*` type. +`expr` – An expression that results in `UInt* or Int*` type. **Return value** -Value of the `UInt*` type. +Value of the `UInt* or Int*` type. **Example** diff --git a/src/AggregateFunctions/AggregateFunctionBitwise.cpp b/src/AggregateFunctions/AggregateFunctionBitwise.cpp index b87e899a685..82cb3b327f0 100644 --- a/src/AggregateFunctions/AggregateFunctionBitwise.cpp +++ b/src/AggregateFunctions/AggregateFunctionBitwise.cpp @@ -27,7 +27,7 @@ AggregateFunctionPtr createAggregateFunctionBitwise(const std::string & name, co "is illegal, because it cannot be used in bitwise operations", argument_types[0]->getName(), name); - AggregateFunctionPtr res(createWithUnsignedIntegerType(*argument_types[0], argument_types[0])); + AggregateFunctionPtr res(createWithOptionSignedIntegerType(*argument_types[0], argument_types[0])); if (!res) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp index 3dd9a8b658d..f2fe9014ceb 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp +++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp @@ -53,7 +53,7 @@ AggregateFunctionPtr createAggregateFunctionSequenceBase( String pattern = params.front().safeGet(); - AggregateFunctionPtr res(createWithUnsignedIntegerType(*argument_types[0], argument_types, params, pattern)); + AggregateFunctionPtr res(createWithOptionSignedIntegerType(*argument_types[0], argument_types, params, pattern)); if (res) return res; diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index d80d683fd04..71c675f7a3b 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -48,7 +48,7 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes & cond_arg->getName(), toString(i + 1), name); } - AggregateFunctionPtr res(createWithUnsignedIntegerType(*arguments[0], arguments, params)); + AggregateFunctionPtr res(createWithOptionSignedIntegerType(*arguments[0], arguments, params)); WhichDataType which(arguments.front().get()); if (res) return res; diff --git a/src/AggregateFunctions/Helpers.h b/src/AggregateFunctions/Helpers.h index 19904dd9215..4131279a897 100644 --- a/src/AggregateFunctions/Helpers.h +++ b/src/AggregateFunctions/Helpers.h @@ -87,8 +87,8 @@ static IAggregateFunction * createWithNumericType(const IDataType & argument_typ return nullptr; } -template