Merge pull request #69201 from NikBarykin/allow_arguments_in_custom_database_engines

Allow custom settings in database engine

src/Databases/DatabaseFactory.cpp

@@ -59,35 +59,27 @@ void cckMetadataPathForOrdinary(const ASTCreateQuery & create, const String & me
 
 }
 
-/// validate validates the database engine that's specified in the create query for
+void DatabaseFactory::validate(const ASTCreateQuery & create_query) const
-/// engine arguments, settings and table overrides.
-void validate(const ASTCreateQuery & create_query)
-
 {
     auto * storage = create_query.storage;
 
-    /// Check engine may have arguments
-    static const std::unordered_set<std::string_view> engines_with_arguments{"MySQL", "MaterializeMySQL", "MaterializedMySQL",
-        "Lazy", "Replicated", "PostgreSQL", "MaterializedPostgreSQL", "SQLite", "Filesystem", "S3", "HDFS"};
-
     const String & engine_name = storage->engine->name;
-    bool engine_may_have_arguments = engines_with_arguments.contains(engine_name);
+    const EngineFeatures & engine_features = database_engines.at(engine_name).features;
 
-    if (storage->engine->arguments && !engine_may_have_arguments)
+    /// Check engine may have arguments
+    if (storage->engine->arguments && !engine_features.supports_arguments)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine `{}` cannot have arguments", engine_name);
 
     /// Check engine may have settings
-    bool may_have_settings = endsWith(engine_name, "MySQL") || engine_name == "Replicated" || engine_name == "MaterializedPostgreSQL";
     bool has_unexpected_element = storage->engine->parameters || storage->partition_by ||
         storage->primary_key || storage->order_by ||
         storage->sample_by;
-    if (has_unexpected_element || (!may_have_settings && storage->settings))
+    if (has_unexpected_element || (!engine_features.supports_settings && storage->settings))
         throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_AST,
                         "Database engine `{}` cannot have parameters, primary_key, order_by, sample_by, settings", engine_name);
 
     /// Check engine with table overrides
-    static const std::unordered_set<std::string_view> engines_with_table_overrides{"MaterializeMySQL", "MaterializedMySQL", "MaterializedPostgreSQL"};
+    if (create_query.table_overrides && !engine_features.supports_table_overrides)
-    if (create_query.table_overrides && !engines_with_table_overrides.contains(engine_name))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine `{}` cannot have table overrides", engine_name);
 }
 
@@ -121,9 +113,9 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m
     return impl;
 }
 
-void DatabaseFactory::registerDatabase(const std::string & name, CreatorFn creator_fn)
+void DatabaseFactory::registerDatabase(const std::string & name, CreatorFn creator_fn, EngineFeatures features)
 {
-    if (!database_engines.emplace(name, std::move(creator_fn)).second)
+    if (!database_engines.emplace(name, Creator{std::move(creator_fn), features}).second)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "DatabaseFactory: the database engine name '{}' is not unique", name);
 }
 
@@ -154,7 +146,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String
         .context = context};
 
     // creator_fn creates and returns a DatabasePtr with the supplied arguments
-    auto creator_fn = database_engines.at(engine_name);
+    auto creator_fn = database_engines.at(engine_name).creator_fn;
 
     return creator_fn(arguments);
 }

src/Databases/DatabaseFactory.h

@@ -43,13 +43,30 @@ public:
         ContextPtr & context;
     };
 
-    DatabasePtr get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context);
+    struct EngineFeatures
+    {
+        bool supports_arguments = false;
+        bool supports_settings = false;
+        bool supports_table_overrides = false;
+    };
 
     using CreatorFn = std::function<DatabasePtr(const Arguments & arguments)>;
 
-    using DatabaseEngines = std::unordered_map<std::string, CreatorFn>;
+    struct Creator
+    {
+        CreatorFn creator_fn;
+        EngineFeatures features;
+    };
 
-    void registerDatabase(const std::string & name, CreatorFn creator_fn);
+    DatabasePtr get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context);
+
+    using DatabaseEngines = std::unordered_map<std::string, Creator>;
+
+    void registerDatabase(const std::string & name, CreatorFn creator_fn, EngineFeatures features = EngineFeatures{
+        .supports_arguments = false,
+        .supports_settings = false,
+        .supports_table_overrides = false,
+    });
 
     const DatabaseEngines & getDatabaseEngines() const { return database_engines; }
 
@@ -65,6 +82,10 @@ private:
     DatabaseEngines database_engines;
 
     DatabasePtr getImpl(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context);
+
+    /// validate validates the database engine that's specified in the create query for
+    /// engine arguments, settings and table overrides.
+    void validate(const ASTCreateQuery & create_query) const;
 };
 
 }

src/Databases/DatabaseFilesystem.cpp

@@ -257,6 +257,6 @@ void registerDatabaseFilesystem(DatabaseFactory & factory)
 
         return std::make_shared<DatabaseFilesystem>(args.database_name, init_path, args.context);
     };
-    factory.registerDatabase("Filesystem", create_fn);
+    factory.registerDatabase("Filesystem", create_fn, {.supports_arguments = true});
 }
 }

src/Databases/DatabaseHDFS.cpp

@@ -253,7 +253,7 @@ void registerDatabaseHDFS(DatabaseFactory & factory)
 
         return std::make_shared<DatabaseHDFS>(args.database_name, source_url, args.context);
     };
-    factory.registerDatabase("HDFS", create_fn);
+    factory.registerDatabase("HDFS", create_fn, {.supports_arguments = true});
 }
 } // DB
 

src/Databases/DatabaseLazy.cpp

@@ -398,6 +398,6 @@ void registerDatabaseLazy(DatabaseFactory & factory)
             cache_expiration_time_seconds,
             args.context);
     };
-    factory.registerDatabase("Lazy", create_fn);
+    factory.registerDatabase("Lazy", create_fn, {.supports_arguments = true});
 }
 }

src/Databases/DatabaseReplicated.cpp

@@ -2001,6 +2001,6 @@ void registerDatabaseReplicated(DatabaseFactory & factory)
             replica_name,
             std::move(database_replicated_settings), args.context);
     };
-    factory.registerDatabase("Replicated", create_fn);
+    factory.registerDatabase("Replicated", create_fn, {.supports_arguments = true, .supports_settings = true});
 }
 }

src/Databases/DatabaseS3.cpp

@@ -326,7 +326,7 @@ void registerDatabaseS3(DatabaseFactory & factory)
 
         return std::make_shared<DatabaseS3>(args.database_name, config, args.context);
     };
-    factory.registerDatabase("S3", create_fn);
+    factory.registerDatabase("S3", create_fn, {.supports_arguments = true});
 }
 }
 #endif

src/Databases/MySQL/DatabaseMaterializedMySQL.cpp

@@ -290,8 +290,14 @@ void registerDatabaseMaterializedMySQL(DatabaseFactory & factory)
             binlog_client,
             std::move(materialize_mode_settings));
     };
-    factory.registerDatabase("MaterializeMySQL", create_fn);
+
-    factory.registerDatabase("MaterializedMySQL", create_fn);
+    DatabaseFactory::EngineFeatures features{
+        .supports_arguments = true,
+        .supports_settings = true,
+        .supports_table_overrides = true,
+    };
+    factory.registerDatabase("MaterializeMySQL", create_fn, features);
+    factory.registerDatabase("MaterializedMySQL", create_fn, features);
 }
 
 }

src/Databases/MySQL/DatabaseMySQL.cpp

@@ -584,7 +584,7 @@ void registerDatabaseMySQL(DatabaseFactory & factory)
             throw Exception(ErrorCodes::CANNOT_CREATE_DATABASE, "Cannot create MySQL database, because {}", exception_message);
         }
     };
-    factory.registerDatabase("MySQL", create_fn);
+    factory.registerDatabase("MySQL", create_fn, {.supports_arguments = true, .supports_settings = true});
 }
 }
 

src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp

@@ -546,7 +546,11 @@ void registerDatabaseMaterializedPostgreSQL(DatabaseFactory & factory)
             args.database_name, configuration.database, connection_info,
             std::move(postgresql_replica_settings));
     };
-    factory.registerDatabase("MaterializedPostgreSQL", create_fn);
+    factory.registerDatabase("MaterializedPostgreSQL", create_fn, {
+        .supports_arguments = true,
+        .supports_settings = true,
+        .supports_table_overrides = true,
+    });
 }
 }
 

src/Databases/PostgreSQL/DatabasePostgreSQL.cpp

@@ -558,7 +558,7 @@ void registerDatabasePostgreSQL(DatabaseFactory & factory)
             pool,
             use_table_cache);
     };
-    factory.registerDatabase("PostgreSQL", create_fn);
+    factory.registerDatabase("PostgreSQL", create_fn, {.supports_arguments = true});
 }
 }
 

src/Databases/SQLite/DatabaseSQLite.cpp

@@ -220,7 +220,7 @@ void registerDatabaseSQLite(DatabaseFactory & factory)
 
         return std::make_shared<DatabaseSQLite>(args.context, engine_define, args.create_query.attach, database_path);
     };
-    factory.registerDatabase("SQLite", create_fn);
+    factory.registerDatabase("SQLite", create_fn, {.supports_arguments = true});
 }
 }