Merge branch 'master' into iaadeflate_upgrade_qpl_v1.0.0

src/Coordination/Changelog.cpp

@@ -157,11 +157,13 @@ public:
         assert(file_buffer && current_file_description);
 
         assert(record.header.index - getStartIndex() <= current_file_description->expectedEntriesCountInLog());
-        const bool log_is_complete = record.header.index - getStartIndex() == current_file_description->expectedEntriesCountInLog();
+        // check if log file reached the limit for amount of records it can contain
-
+        if (record.header.index - getStartIndex() == current_file_description->expectedEntriesCountInLog())
-        if (log_is_complete)
+        {
             rotate(record.header.index);
-
+        }
+        else
+        {
             // writing at least 1 log is requirement - we don't want empty log files
             // we use count() that can be unreliable for more complex WriteBuffers, so we should be careful if we change the type of it in the future
             const bool log_too_big = record.header.index != getStartIndex() && log_file_settings.max_size != 0
@@ -172,6 +174,7 @@ public:
                 LOG_TRACE(log, "Log file reached maximum allowed size ({} bytes), creating new log file", log_file_settings.max_size);
                 rotate(record.header.index);
             }
+        }
 
         if (!prealloc_done) [[unlikely]]
         {

src/Parsers/ASTFunction.cpp

@@ -37,81 +37,118 @@ namespace
 {
     /// Finds arguments of a specified function which should not be displayed for most users for security reasons.
     /// That involves passwords and secret keys.
-    /// The member function getRange() returns a pair of numbers [first, last) specifying arguments
-    /// which must be hidden. If the function returns {-1, -1} that means no arguments must be hidden.
     class FunctionSecretArgumentsFinder
     {
     public:
         explicit FunctionSecretArgumentsFinder(const ASTFunction & function_) : function(function_)
         {
-            if (function.arguments)
+            if (!function.arguments)
-            {
+                return;
-                if (const auto * expr_list = function.arguments->as<ASTExpressionList>())
+
+            const auto * expr_list = function.arguments->as<ASTExpressionList>();
+            if (!expr_list)
+                return;
+
             arguments = &expr_list->children;
-            }
-        }
-
-        std::pair<size_t, size_t> getRange() const
-        {
-            if (!arguments)
-                return npos;
-
             switch (function.kind)
             {
-                case ASTFunction::Kind::ORDINARY_FUNCTION: return findOrdinaryFunctionSecretArguments();
+                case ASTFunction::Kind::ORDINARY_FUNCTION: findOrdinaryFunctionSecretArguments(); break;
-                case ASTFunction::Kind::WINDOW_FUNCTION: return npos;
+                case ASTFunction::Kind::WINDOW_FUNCTION: break;
-                case ASTFunction::Kind::LAMBDA_FUNCTION: return npos;
+                case ASTFunction::Kind::LAMBDA_FUNCTION:  break;
-                case ASTFunction::Kind::TABLE_ENGINE: return findTableEngineSecretArguments();
+                case ASTFunction::Kind::TABLE_ENGINE: findTableEngineSecretArguments(); break;
-                case ASTFunction::Kind::DATABASE_ENGINE: return findDatabaseEngineSecretArguments();
+                case ASTFunction::Kind::DATABASE_ENGINE: findDatabaseEngineSecretArguments(); break;
-                case ASTFunction::Kind::BACKUP_NAME: return findBackupNameSecretArguments();
+                case ASTFunction::Kind::BACKUP_NAME: findBackupNameSecretArguments(); break;
             }
         }
 
-        static const constexpr std::pair<size_t, size_t> npos{static_cast<size_t>(-1), static_cast<size_t>(-1)};
+        struct Result
+        {
+            /// Result constructed by default means no arguments will be hidden.
+            size_t start = static_cast<size_t>(-1);
+            size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`).
+                              /// In all known cases secret arguments are consecutive
+            bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments.
+        };
+
+        Result getResult() const { return result; }
 
     private:
-        std::pair<size_t, size_t> findOrdinaryFunctionSecretArguments() const
+        const ASTFunction & function;
+        const ASTs * arguments = nullptr;
+        Result result;
+
+        void markSecretArgument(size_t index, bool argument_is_named = false)
+        {
+            if (!result.count)
+            {
+                result.start = index;
+                result.are_named = argument_is_named;
+            }
+            chassert(index >= result.start); /// We always check arguments consecutively
+            result.count = index + 1 - result.start;
+            if (!argument_is_named)
+                result.are_named = false;
+        }
+
+        void findOrdinaryFunctionSecretArguments()
         {
             if ((function.name == "mysql") || (function.name == "postgresql") || (function.name == "mongodb"))
             {
                 /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
                 /// postgresql('host:port', 'database', 'table', 'user', 'password', ...)
                 /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...)
-                return {4, 5};
+                findMySQLFunctionSecretArguments();
             }
             else if ((function.name == "s3") || (function.name == "cosn") || (function.name == "oss"))
             {
                 /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
-                return findS3FunctionSecretArguments(/* is_cluster_function= */ false);
+                findS3FunctionSecretArguments(/* is_cluster_function= */ false);
             }
             else if (function.name == "s3Cluster")
             {
                 /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...)
-                return findS3FunctionSecretArguments(/* is_cluster_function= */ true);
+                findS3FunctionSecretArguments(/* is_cluster_function= */ true);
             }
             else if ((function.name == "remote") || (function.name == "remoteSecure"))
             {
                 /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...)
-                return findRemoteFunctionSecretArguments();
+                findRemoteFunctionSecretArguments();
             }
             else if ((function.name == "encrypt") || (function.name == "decrypt") ||
                      (function.name == "aes_encrypt_mysql") || (function.name == "aes_decrypt_mysql") ||
                      (function.name == "tryDecrypt"))
             {
                 /// encrypt('mode', 'plaintext', 'key' [, iv, aad])
-                return findEncryptionFunctionSecretArguments();
+                findEncryptionFunctionSecretArguments();
-            }
-            else
-            {
-                return npos;
             }
         }
 
-        std::pair<size_t, size_t> findS3FunctionSecretArguments(bool is_cluster_function) const
+        void findMySQLFunctionSecretArguments()
+        {
+            if (isNamedCollectionName(0))
+            {
+                /// mysql(named_collection, ..., password = 'password', ...)
+                findSecretNamedArgument("password", 1);
+            }
+            else
+            {
+                /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
+                markSecretArgument(4);
+            }
+        }
+
+        void findS3FunctionSecretArguments(bool is_cluster_function)
         {
             /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument.
             size_t url_arg_idx = is_cluster_function ? 1 : 0;
 
+            if (!is_cluster_function && isNamedCollectionName(0))
+            {
+                /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...)
+                findSecretNamedArgument("secret_access_key", 1);
+                return;
+            }
+
             /// We're going to replace 'aws_secret_access_key' with '[HIDDEN'] for the following signatures:
             /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
             /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
@@ -119,12 +156,12 @@ namespace
             /// But we should check the number of arguments first because we don't need to do any replacements in case of
             /// s3('url' [, 'format']) or s3Cluster('cluster_name', 'url' [, 'format'])
             if (arguments->size() < url_arg_idx + 3)
-                return npos;
+                return;
 
             if (arguments->size() >= url_arg_idx + 5)
             {
                 /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'structure', ...)
-                return {url_arg_idx + 2, url_arg_idx + 3};
+                markSecretArgument(url_arg_idx + 2);
             }
             else
             {
@@ -136,15 +173,16 @@ namespace
                 {
                     /// We couldn't evaluate the argument after 'url' so we don't know whether it is a format or `aws_access_key_id`.
                     /// So it's safer to wipe the next argument just in case.
-                    return {url_arg_idx + 2, url_arg_idx + 3}; /// Wipe either `aws_secret_access_key` or `structure`.
+                    markSecretArgument(url_arg_idx + 2); /// Wipe either `aws_secret_access_key` or `structure`.
+                    return;
                 }
 
                 if (KnownFormatNames::instance().exists(format))
-                    return npos; /// The argument after 'url' is a format: s3('url', 'format', ...)
+                    return; /// The argument after 'url' is a format: s3('url', 'format', ...)
 
                 /// The argument after 'url' is not a format so we do our replacement:
                 /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) -> s3('url', 'aws_access_key_id', '[HIDDEN]', ...)
-                return {url_arg_idx + 2, url_arg_idx + 3};
+                markSecretArgument(url_arg_idx + 2);
             }
         }
 
@@ -153,8 +191,12 @@ namespace
             if (arg_idx >= arguments->size())
                 return false;
 
-            ASTPtr argument = (*arguments)[arg_idx];
+            return tryGetStringFromArgument(*(*arguments)[arg_idx], res, allow_identifier);
-            if (const auto * literal = argument->as<ASTLiteral>())
+        }
+
+        static bool tryGetStringFromArgument(const IAST & argument, String * res, bool allow_identifier = true)
+        {
+            if (const auto * literal = argument.as<ASTLiteral>())
             {
                 if (literal->value.getType() != Field::Types::String)
                     return false;
@@ -165,7 +207,7 @@ namespace
 
             if (allow_identifier)
             {
-                if (const auto * id = argument->as<ASTIdentifier>())
+                if (const auto * id = argument.as<ASTIdentifier>())
                 {
                     if (res)
                         *res = id->name();
@@ -176,8 +218,15 @@ namespace
             return false;
         }
 
-        std::pair<size_t, size_t> findRemoteFunctionSecretArguments() const
+        void findRemoteFunctionSecretArguments()
         {
+            if (isNamedCollectionName(0))
+            {
+                /// remote(named_collection, ..., password = 'password', ...)
+                findSecretNamedArgument("password", 1);
+                return;
+            }
+
             /// We're going to replace 'password' with '[HIDDEN'] for the following signatures:
             /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key])
             /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key])
@@ -186,7 +235,7 @@ namespace
             /// But we should check the number of arguments first because we don't need to do any replacements in case of
             /// remote('addresses_expr', db.table)
             if (arguments->size() < 3)
-                return npos;
+                return;
 
             size_t arg_num = 1;
 
@@ -207,20 +256,17 @@ namespace
                     /// before the argument 'password'. So it's safer to wipe two arguments just in case.
                     /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
                     /// before wiping it (because the `password` argument is always a literal string).
-                    auto res = npos;
                     if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false))
                     {
                         /// Wipe either `password` or `user`.
-                        res = {arg_num + 2, arg_num + 3};
+                        markSecretArgument(arg_num + 2);
                     }
                     if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false))
                     {
                         /// Wipe either `password` or `sharding_key`.
-                        if (res == npos)
+                        markSecretArgument(arg_num + 3);
-                            res.first = arg_num + 3;
-                        res.second = arg_num + 4;
                     }
-                    return res;
+                    return;
                 }
 
                 /// Skip the current argument (which is either a database name or a qualified table name).
@@ -241,9 +287,7 @@ namespace
             /// before wiping it (because the `password` argument is always a literal string).
             bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false);
             if (can_be_password)
-                return {arg_num, arg_num + 1};
+                markSecretArgument(arg_num);
-
-            return npos;
         }
 
         /// Tries to get either a database name or a qualified table name from an argument.
@@ -278,20 +322,24 @@ namespace
             return true;
         }
 
-        std::pair<size_t, size_t> findEncryptionFunctionSecretArguments() const
+        void findEncryptionFunctionSecretArguments()
         {
+            if (arguments->empty())
+                return;
+
             /// We replace all arguments after 'mode' with '[HIDDEN]':
             /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]')
-            return {1, arguments->size()};
+            result.start = 1;
+            result.count = arguments->size() - 1;
         }
 
-        std::pair<size_t, size_t> findTableEngineSecretArguments() const
+        void findTableEngineSecretArguments()
         {
             const String & engine_name = function.name;
             if (engine_name == "ExternalDistributed")
             {
                 /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password')
-                return {5, 6};
+                findExternalDistributedTableEngineSecretArguments();
             }
             else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") ||
                      (engine_name == "MaterializedPostgreSQL") || (engine_name == "MongoDB"))
@@ -300,21 +348,38 @@ namespace
                 /// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...)
                 /// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...)
                 /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...)
-                return {4, 5};
+                findMySQLFunctionSecretArguments();
             }
             else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS"))
             {
                 /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...)
-                return findS3TableEngineSecretArguments();
+                findS3TableEngineSecretArguments();
-            }
-            else
-            {
-                return npos;
             }
         }
 
-        std::pair<size_t, size_t> findS3TableEngineSecretArguments() const
+        void findExternalDistributedTableEngineSecretArguments()
         {
+            if (isNamedCollectionName(1))
+            {
+                /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...)
+                findSecretNamedArgument("password", 2);
+            }
+            else
+            {
+                /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password')
+                markSecretArgument(5);
+            }
+        }
+
+        void findS3TableEngineSecretArguments()
+        {
+            if (isNamedCollectionName(0))
+            {
+                /// S3(named_collection, ..., secret_access_key = 'secret_access_key')
+                findSecretNamedArgument("secret_access_key", 1);
+                return;
+            }
+
             /// We replace 'aws_secret_access_key' with '[HIDDEN'] for the following signatures:
             /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format')
             /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
@@ -322,12 +387,12 @@ namespace
             /// But we should check the number of arguments first because we don't need to do that replacements in case of
             /// S3('url' [, 'format' [, 'compression']])
             if (arguments->size() < 4)
-                return npos;
+                return;
 
-            return {2, 3};
+            markSecretArgument(2);
         }
 
-        std::pair<size_t, size_t> findDatabaseEngineSecretArguments() const
+        void findDatabaseEngineSecretArguments()
         {
             const String & engine_name = function.name;
             if ((engine_name == "MySQL") || (engine_name == "MaterializeMySQL") ||
@@ -335,31 +400,71 @@ namespace
                 (engine_name == "MaterializedPostgreSQL"))
             {
                 /// MySQL('host:port', 'database', 'user', 'password')
-                /// PostgreSQL('host:port', 'database', 'user', 'password', ...)
+                /// PostgreSQL('host:port', 'database', 'user', 'password')
-                return {3, 4};
+                findMySQLDatabaseSecretArguments();
-            }
-            else
-            {
-                return npos;
             }
         }
 
-        std::pair<size_t, size_t> findBackupNameSecretArguments() const
+        void findMySQLDatabaseSecretArguments()
+        {
+            if (isNamedCollectionName(0))
+            {
+                /// MySQL(named_collection, ..., password = 'password', ...)
+                findSecretNamedArgument("password", 1);
+            }
+            else
+            {
+                /// MySQL('host:port', 'database', 'user', 'password')
+                markSecretArgument(3);
+            }
+        }
+
+        void findBackupNameSecretArguments()
         {
             const String & engine_name = function.name;
             if (engine_name == "S3")
             {
                 /// BACKUP ... TO S3(url, [aws_access_key_id, aws_secret_access_key])
-                return {2, 3};
+                markSecretArgument(2);
-            }
-            else
-            {
-                return npos;
             }
         }
 
-        const ASTFunction & function;
+        /// Whether a specified argument can be the name of a named collection?
-        const ASTs * arguments = nullptr;
+        bool isNamedCollectionName(size_t arg_idx) const
+        {
+            if (arguments->size() <= arg_idx)
+                return false;
+
+            const auto * identifier = (*arguments)[arg_idx]->as<ASTIdentifier>();
+            return identifier != nullptr;
+        }
+
+        /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified.
+        void findSecretNamedArgument(const std::string_view & key, size_t start = 0)
+        {
+            for (size_t i = start; i < arguments->size(); ++i)
+            {
+                const auto & argument = (*arguments)[i];
+                const auto * equals_func = argument->as<ASTFunction>();
+                if (!equals_func || (equals_func->name != "equals"))
+                    continue;
+
+                const auto * expr_list = equals_func->arguments->as<ASTExpressionList>();
+                if (!expr_list)
+                    continue;
+
+                const auto & equal_args = expr_list->children;
+                if (equal_args.size() != 2)
+                    continue;
+
+                String found_key;
+                if (!tryGetStringFromArgument(*equal_args[0], &found_key))
+                    continue;
+
+                if (found_key == key)
+                    markSecretArgument(i, /* argument_is_named= */ true);
+            }
+        }
     };
 }
 
@@ -966,32 +1071,39 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
             && (name == "match" || name == "extract" || name == "extractAll" || name == "replaceRegexpOne"
                 || name == "replaceRegexpAll");
 
-        auto secret_arguments = std::make_pair(static_cast<size_t>(-1), static_cast<size_t>(-1));
+        FunctionSecretArgumentsFinder::Result secret_arguments;
         if (!settings.show_secrets)
-            secret_arguments = FunctionSecretArgumentsFinder(*this).getRange();
+            secret_arguments = FunctionSecretArgumentsFinder{*this}.getResult();
 
         for (size_t i = 0, size = arguments->children.size(); i < size; ++i)
         {
             if (i != 0)
                 settings.ostr << ", ";
-            if (arguments->children[i]->as<ASTSetQuery>())
+
+            const auto & argument = arguments->children[i];
+            if (argument->as<ASTSetQuery>())
                 settings.ostr << "SETTINGS ";
 
-            if (!settings.show_secrets && (secret_arguments.first <= i) && (i < secret_arguments.second))
+            if (!settings.show_secrets && (secret_arguments.start <= i) && (i < secret_arguments.start + secret_arguments.count))
             {
+                if (secret_arguments.are_named)
+                {
+                   assert_cast<const ASTFunction *>(argument.get())->arguments->children[0]->formatImpl(settings, state, nested_dont_need_parens);
+                   settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : "");
+                }
                 settings.ostr << "'[HIDDEN]'";
-                if (size - 1 < secret_arguments.second)
+                if (size <= secret_arguments.start + secret_arguments.count && !secret_arguments.are_named)
                     break; /// All other arguments should also be hidden.
                 continue;
             }
 
             if ((i == 1) && special_hilite_regexp
-                && highlightStringLiteralWithMetacharacters(arguments->children[i], settings, "|()^$.[]?*+{:-"))
+                && highlightStringLiteralWithMetacharacters(argument, settings, "|()^$.[]?*+{:-"))
             {
                 continue;
             }
 
-            arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens);
+            argument->formatImpl(settings, state, nested_dont_need_parens);
         }
     }
 
@@ -1005,14 +1117,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
 
 bool ASTFunction::hasSecretParts() const
 {
-    if (arguments)
+    return (FunctionSecretArgumentsFinder{*this}.getResult().count > 0) || childrenHaveSecretParts();
-    {
-        size_t num_arguments = arguments->children.size();
-        auto secret_arguments = FunctionSecretArgumentsFinder(*this).getRange();
-        if ((secret_arguments.first < num_arguments) && (secret_arguments.first < secret_arguments.second))
-            return true;
-    }
-    return childrenHaveSecretParts();
 }
 
 String getFunctionName(const IAST * ast)

tests/integration/test_mask_sensitive_info/configs/named_collections.xml

@@ -0,0 +1,10 @@
+<clickhouse>
+    <named_collections>
+        <named_collection_1/>
+        <named_collection_2/>
+        <named_collection_3/>
+        <named_collection_4/>
+        <named_collection_5/>
+        <named_collection_6/>
+    </named_collections>
+</clickhouse>

tests/integration/test_mask_sensitive_info/test.py

@@ -4,7 +4,13 @@ from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
 
 cluster = ClickHouseCluster(__file__)
-node = cluster.add_instance("node", with_zookeeper=True)
+node = cluster.add_instance(
+    "node",
+    main_configs=[
+        "configs/named_collections.xml",
+    ],
+    with_zookeeper=True,
+)
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -116,6 +122,12 @@ def test_create_table():
         f"S3('http://minio1:9001/root/data/test3.csv.gz', 'CSV', 'gzip')",
         f"S3('http://minio1:9001/root/data/test4.csv', 'minio', '{password}', 'CSV')",
         f"S3('http://minio1:9001/root/data/test5.csv.gz', 'minio', '{password}', 'CSV', 'gzip')",
+        f"MySQL(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '{password}')",
+        f"MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '{password}', table = 'mysql_table', user = 'mysql_user')",
+        f"MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')",
+        f"PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '{password}')",
+        f"MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '{password}')",
+        f"S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '{password}', format = 'CSV')",
     ]
 
     for i, table_engine in enumerate(table_engines):
@@ -147,6 +159,12 @@ def test_create_table():
             "CREATE TABLE table5 (x int) ENGINE = S3('http://minio1:9001/root/data/test3.csv.gz', 'CSV', 'gzip')",
             "CREATE TABLE table6 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test4.csv', 'minio', '[HIDDEN]', 'CSV')",
             "CREATE TABLE table7 (`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'minio', '[HIDDEN]', 'CSV', 'gzip')",
+            "CREATE TABLE table8 (`x` int) ENGINE = MySQL(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')",
+            "CREATE TABLE table9 (`x` int) ENGINE = MySQL(named_collection_2, database = 'mysql_db', host = 'mysql57', port = 3306, password = '[HIDDEN]', table = 'mysql_table', user = 'mysql_user')",
+            "CREATE TABLE table10 (x int) ENGINE = MySQL(named_collection_3, database = 'mysql_db', host = 'mysql57', port = 3306, table = 'mysql_table')",
+            "CREATE TABLE table11 (`x` int) ENGINE = PostgreSQL(named_collection_4, host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user', password = '[HIDDEN]')",
+            "CREATE TABLE table12 (`x` int) ENGINE = MongoDB(named_collection_5, host = 'mongo1', port = 5432, database = 'mongo_db', collection = 'mongo_col', user = 'mongo_user', password = '[HIDDEN]'",
+            "CREATE TABLE table13 (`x` int) ENGINE = S3(named_collection_6, url = 'http://minio1:9001/root/data/test8.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]', format = 'CSV')",
         ],
         must_not_contain=[password],
     )
@@ -160,6 +178,7 @@ def test_create_database():
 
     database_engines = [
         f"MySQL('localhost:3306', 'mysql_db', 'mysql_user', '{password}') SETTINGS connect_timeout=1, connection_max_tries=1",
+        f"MySQL(named_collection_1, host = 'localhost', port = 3306, database = 'mysql_db', user = 'mysql_user', password = '{password}') SETTINGS connect_timeout=1, connection_max_tries=1",
         # f"PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '{password}')",
     ]
 
@@ -173,7 +192,8 @@ def test_create_database():
     check_logs(
         must_contain=[
             "CREATE DATABASE database0 ENGINE = MySQL('localhost:3306', 'mysql_db', 'mysql_user', '[HIDDEN]')",
-            # "CREATE DATABASE database1 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')",
+            "CREATE DATABASE database1 ENGINE = MySQL(named_collection_1, host = 'localhost', port = 3306, database = 'mysql_db', user = 'mysql_user', password = '[HIDDEN]')",
+            # "CREATE DATABASE database2 ENGINE = PostgreSQL('localhost:5432', 'postgres_db', 'postgres_user', '[HIDDEN]')",
         ],
         must_not_contain=[password],
     )
@@ -211,6 +231,11 @@ def test_table_functions():
         f"remote('127.{{2..11}}', numbers(10), 'remote_user', '{password}', rand())",
         f"remoteSecure('127.{{2..11}}', 'default', 'remote_table', 'remote_user', '{password}')",
         f"remoteSecure('127.{{2..11}}', 'default', 'remote_table', 'remote_user', rand())",
+        f"mysql(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '{password}')",
+        f"postgresql(named_collection_2, password = '{password}', host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user')",
+        f"s3(named_collection_3, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '{password}')",
+        f"remote(named_collection_4, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}', sharding_key = rand())",
+        f"remoteSecure(named_collection_5, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}')",
     ]
 
     for i, table_function in enumerate(table_functions):
@@ -259,6 +284,11 @@ def test_table_functions():
             "CREATE TABLE tablefunc22 (`x` int) AS remote('127.{2..11}', numbers(10), 'remote_user', '[HIDDEN]', rand())",
             "CREATE TABLE tablefunc23 (`x` int) AS remoteSecure('127.{2..11}', 'default', 'remote_table', 'remote_user', '[HIDDEN]')",
             "CREATE TABLE tablefunc24 (x int) AS remoteSecure('127.{2..11}', 'default', 'remote_table', 'remote_user', rand())",
+            "CREATE TABLE tablefunc25 (`x` int) AS mysql(named_collection_1, host = 'mysql57', port = 3306, database = 'mysql_db', table = 'mysql_table', user = 'mysql_user', password = '[HIDDEN]')",
+            "CREATE TABLE tablefunc26 (`x` int) AS postgresql(named_collection_2, password = '[HIDDEN]', host = 'postgres1', port = 5432, database = 'postgres_db', table = 'postgres_table', user = 'postgres_user')",
+            "CREATE TABLE tablefunc27 (`x` int) AS s3(named_collection_3, url = 'http://minio1:9001/root/data/test4.csv', access_key_id = 'minio', secret_access_key = '[HIDDEN]')",
+            "CREATE TABLE tablefunc28 (`x` int) AS remote(named_collection_4, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]', sharding_key = rand())",
+            "CREATE TABLE tablefunc29 (`x` int) AS remoteSecure(named_collection_5, addresses_expr = '127.{2..11}', database = 'default', table = 'remote_table', user = 'remote_user', password = '[HIDDEN]')",
         ],
         must_not_contain=[password],
     )

utils/keeper-data-dumper/main.cpp

@@ -69,7 +69,8 @@ int main(int argc, char *argv[])
 
     LOG_INFO(logger, "Last committed index: {}", last_commited_index);
 
-    DB::KeeperLogStore changelog(argv[2], 10000000, true, settings->compress_logs);
+    DB::KeeperLogStore changelog(
+        argv[2], LogFileSettings{.force_sync = true, .compress_logs = settings->compress_logs, .rotate_interval = 10000000});
     changelog.init(last_commited_index, 10000000000UL); /// collect all logs
     if (changelog.size() == 0)
         LOG_INFO(logger, "Changelog empty");