thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 17:41:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use only atomic database

Browse Source
This commit is contained in:
kssenii 2021-04-08 22:38:17 +00:00
parent 093e53e65a
commit bd207daa74
6 changed files with 29 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/Databases/DatabaseFactory.cpp
View File

@ -294,20 +294,10 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String
if (engine_define->settings) if (engine_define->settings)
postgresql_replica_settings->loadFromQuery(*engine_define); postgresql_replica_settings->loadFromQuery(*engine_define);
if (create.uuid == UUIDHelpers::Nil) return std::make_shared<DatabaseMaterializePostgreSQL>(
{ context, metadata_path, uuid, engine_define,
return std::make_shared<DatabaseMaterializePostgreSQL<DatabaseOrdinary>>( database_name, postgres_database_name, connection_string,
context, metadata_path, uuid, engine_define, std::move(postgresql_replica_settings));
database_name, postgres_database_name, connection_string,
std::move(postgresql_replica_settings));
}
else
{
return std::make_shared<DatabaseMaterializePostgreSQL<DatabaseAtomic>>(
context, metadata_path, uuid, engine_define,
database_name, postgres_database_name, connection_string,
std::move(postgresql_replica_settings));
}
} }

74
src/Databases/PostgreSQL/DatabaseMaterializePostgreSQL.cpp
View File

@ -34,33 +34,7 @@ namespace ErrorCodes
static const auto METADATA_SUFFIX = ".postgresql_replica_metadata"; static const auto METADATA_SUFFIX = ".postgresql_replica_metadata";
template<> DatabaseMaterializePostgreSQL::DatabaseMaterializePostgreSQL(
DatabaseMaterializePostgreSQL<DatabaseOrdinary>::DatabaseMaterializePostgreSQL(
const Context & context,
const String & metadata_path_,
UUID /* uuid */,
const ASTStorage * database_engine_define_,
const String & database_name_,
const String & postgres_database_name,
const String & connection_string,
std::unique_ptr<MaterializePostgreSQLSettings> settings_)
: DatabaseOrdinary(
database_name_, metadata_path_, "data/" + escapeForFileName(database_name_) + "/",
"DatabaseMaterializePostgreSQL<Ordinary> (" + database_name_ + ")", context)
, log(&Poco::Logger::get("MaterializePostgreSQLDatabaseEngine"))
, global_context(context.getGlobalContext())
, metadata_path(metadata_path_)
, database_engine_define(database_engine_define_->clone())
, database_name(database_name_)
, remote_database_name(postgres_database_name)
, connection(std::make_shared<postgres::Connection>(connection_string, ""))
, settings(std::move(settings_))
{
}
template<>
DatabaseMaterializePostgreSQL<DatabaseAtomic>::DatabaseMaterializePostgreSQL(
const Context & context, const Context & context,
const String & metadata_path_, const String & metadata_path_,
UUID uuid, UUID uuid,
@ -70,8 +44,6 @@ DatabaseMaterializePostgreSQL<DatabaseAtomic>::DatabaseMaterializePostgreSQL(
const String & connection_string, const String & connection_string,
std::unique_ptr<MaterializePostgreSQLSettings> settings_) std::unique_ptr<MaterializePostgreSQLSettings> settings_)
: DatabaseAtomic(database_name_, metadata_path_, uuid, "DatabaseMaterializePostgreSQL<Atomic> (" + database_name_ + ")", context) : DatabaseAtomic(database_name_, metadata_path_, uuid, "DatabaseMaterializePostgreSQL<Atomic> (" + database_name_ + ")", context)
, global_context(context.getGlobalContext())
, metadata_path(metadata_path_)
, database_engine_define(database_engine_define_->clone()) , database_engine_define(database_engine_define_->clone())
, remote_database_name(postgres_database_name) , remote_database_name(postgres_database_name)
, connection(std::make_shared<postgres::Connection>(connection_string, "")) , connection(std::make_shared<postgres::Connection>(connection_string, ""))
@ -80,8 +52,7 @@ DatabaseMaterializePostgreSQL<DatabaseAtomic>::DatabaseMaterializePostgreSQL(
} }
template<typename Base> void DatabaseMaterializePostgreSQL::startSynchronization()
void DatabaseMaterializePostgreSQL<Base>::startSynchronization()
{ {
replication_handler = std::make_unique<PostgreSQLReplicationHandler>( replication_handler = std::make_unique<PostgreSQLReplicationHandler>(
remote_database_name, remote_database_name,
@ -104,27 +75,24 @@ void DatabaseMaterializePostgreSQL<Base>::startSynchronization()
} }
replication_handler->addStorage(table_name, storage->template as<StorageMaterializePostgreSQL>()); replication_handler->addStorage(table_name, storage->template as<StorageMaterializePostgreSQL>());
tables[table_name] = storage; materialized_tables[table_name] = storage;
} }
LOG_TRACE(log, "Loaded {} tables. Starting synchronization", tables.size()); LOG_TRACE(log, "Loaded {} tables. Starting synchronization", materialized_tables.size());
replication_handler->startup(); replication_handler->startup();
} }
template<typename Base> void DatabaseMaterializePostgreSQL::shutdown()
void DatabaseMaterializePostgreSQL<Base>::shutdown()
{ {
if (replication_handler) if (replication_handler)
replication_handler->shutdown(); replication_handler->shutdown();
} }
template<typename Base> void DatabaseMaterializePostgreSQL::loadStoredObjects(Context & context, bool has_force_restore_data_flag, bool force_attach)
void DatabaseMaterializePostgreSQL<Base>::loadStoredObjects(
Context & context, bool has_force_restore_data_flag, bool force_attach)
{ {
Base::loadStoredObjects(context, has_force_restore_data_flag, force_attach); DatabaseAtomic::loadStoredObjects(context, has_force_restore_data_flag, force_attach);
try try
{ {
@ -132,7 +100,7 @@ void DatabaseMaterializePostgreSQL<Base>::loadStoredObjects(
} }
catch (...) catch (...)
{ {
tryLogCurrentException(Base::log, "Cannot load nested database objects for PostgreSQL database engine."); tryLogCurrentException(log, "Cannot load nested database objects for PostgreSQL database engine.");
if (!force_attach) if (!force_attach)
throw; throw;
@ -141,8 +109,7 @@ void DatabaseMaterializePostgreSQL<Base>::loadStoredObjects(
} }
template<typename Base> StoragePtr DatabaseMaterializePostgreSQL::tryGetTable(const String & name, const Context & context) const
StoragePtr DatabaseMaterializePostgreSQL<Base>::tryGetTable(const String & name, const Context & context) const
{ {
/// When a nested ReplacingMergeTree table is managed from PostgreSQLReplicationHandler, its context is modified /// When a nested ReplacingMergeTree table is managed from PostgreSQLReplicationHandler, its context is modified
/// to show the type of managed table. /// to show the type of managed table.
@ -151,29 +118,28 @@ StoragePtr DatabaseMaterializePostgreSQL<Base>::tryGetTable(const String & name,
auto storage_set = context.getQueryContext().getQueryFactoriesInfo().storages; auto storage_set = context.getQueryContext().getQueryFactoriesInfo().storages;
if (storage_set.find("ReplacingMergeTree") != storage_set.end()) if (storage_set.find("ReplacingMergeTree") != storage_set.end())
{ {
return Base::tryGetTable(name, context); return DatabaseAtomic::tryGetTable(name, context);
} }
} }
auto table = tables.find(name); auto table = materialized_tables.find(name);
/// Here it is possible that nested table is temporarily out of reach, but return storage anyway, /// Here it is possible that nested table is temporarily out of reach, but return storage anyway,
/// it will not allow to read if nested is unavailable at the moment /// it will not allow to read if nested is unavailable at the moment
if (table != tables.end()) if (table != materialized_tables.end())
return table->second; return table->second;
return StoragePtr{}; return StoragePtr{};
} }
template<typename Base> void DatabaseMaterializePostgreSQL::createTable(const Context & context, const String & name, const StoragePtr & table, const ASTPtr & query)
void DatabaseMaterializePostgreSQL<Base>::createTable(const Context & context, const String & name, const StoragePtr & table, const ASTPtr & query)
{ {
if (context.hasQueryContext()) if (context.hasQueryContext())
{ {
auto storage_set = context.getQueryContext().getQueryFactoriesInfo().storages; auto storage_set = context.getQueryContext().getQueryFactoriesInfo().storages;
if (storage_set.find("ReplacingMergeTree") != storage_set.end()) if (storage_set.find("ReplacingMergeTree") != storage_set.end())
{ {
Base::createTable(context, name, table, query); DatabaseAtomic::createTable(context, name, table, query);
return; return;
} }
} }
@ -183,8 +149,7 @@ void DatabaseMaterializePostgreSQL<Base>::createTable(const Context & context, c
} }
template<typename Base> void DatabaseMaterializePostgreSQL::drop(const Context & context)
void DatabaseMaterializePostgreSQL<Base>::drop(const Context & context)
{ {
if (replication_handler) if (replication_handler)
{ {
@ -193,21 +158,20 @@ void DatabaseMaterializePostgreSQL<Base>::drop(const Context & context)
} }
/// Remove metadata /// Remove metadata
Poco::File metadata(Base::getMetadataPath() + METADATA_SUFFIX); Poco::File metadata(getMetadataPath() + METADATA_SUFFIX);
if (metadata.exists()) if (metadata.exists())
metadata.remove(false); metadata.remove(false);
Base::drop(context); DatabaseAtomic::drop(context);
} }
template<typename Base> DatabaseTablesIteratorPtr DatabaseMaterializePostgreSQL::getTablesIterator(
DatabaseTablesIteratorPtr DatabaseMaterializePostgreSQL<Base>::getTablesIterator(
const Context & /* context */, const DatabaseOnDisk::FilterByNameFunction & /* filter_by_table_name */) const Context & /* context */, const DatabaseOnDisk::FilterByNameFunction & /* filter_by_table_name */)
{ {
Tables nested_tables; Tables nested_tables;
for (const auto & [table_name, storage] : tables) for (const auto & [table_name, storage] : materialized_tables)
{ {
auto nested_storage = storage->template as<StorageMaterializePostgreSQL>()->tryGetNested(); auto nested_storage = storage->template as<StorageMaterializePostgreSQL>()->tryGetNested();

12
src/Databases/PostgreSQL/DatabaseMaterializePostgreSQL.h
View File

@ -14,6 +14,7 @@
#include <Parsers/ASTCreateQuery.h> #include <Parsers/ASTCreateQuery.h>
#include <Databases/IDatabase.h> #include <Databases/IDatabase.h>
#include <Databases/DatabaseOnDisk.h> #include <Databases/DatabaseOnDisk.h>
#include <Databases/DatabaseAtomic.h>
namespace DB namespace DB
@ -24,8 +25,7 @@ class PostgreSQLConnection;
using PostgreSQLConnectionPtr = std::shared_ptr<PostgreSQLConnection>; using PostgreSQLConnectionPtr = std::shared_ptr<PostgreSQLConnection>;
template<typename Base> class DatabaseMaterializePostgreSQL : public DatabaseAtomic
class DatabaseMaterializePostgreSQL : public Base
{ {
public: public:
@ -59,17 +59,13 @@ public:
private: private:
void startSynchronization(); void startSynchronization();
Poco::Logger * log;
const Context & global_context;
String metadata_path;
ASTPtr database_engine_define; ASTPtr database_engine_define;
String remote_database_name;
String database_name, remote_database_name;
postgres::ConnectionPtr connection; postgres::ConnectionPtr connection;
std::unique_ptr<MaterializePostgreSQLSettings> settings; std::unique_ptr<MaterializePostgreSQLSettings> settings;
std::shared_ptr<PostgreSQLReplicationHandler> replication_handler; std::shared_ptr<PostgreSQLReplicationHandler> replication_handler;
std::map<std::string, StoragePtr> tables; std::map<std::string, StoragePtr> materialized_tables;
}; };
} }

2
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -150,7 +150,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", serializeAST(*create.storage)); throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", serializeAST(*create.storage));
} }
if (create.storage->engine->name == "Atomic" || create.storage->engine->name == "Replicated") if (create.storage->engine->name == "Atomic" || create.storage->engine->name == "Replicated" || create.storage->engine->name == "MaterializePostgreSQL")
{ {
if (create.attach && create.uuid == UUIDHelpers::Nil) if (create.attach && create.uuid == UUIDHelpers::Nil)
throw Exception(ErrorCodes::INCORRECT_QUERY, "UUID must be specified for ATTACH. " throw Exception(ErrorCodes::INCORRECT_QUERY, "UUID must be specified for ATTACH. "

3
src/Storages/PostgreSQL/StorageMaterializePostgreSQL.cpp
View File

@ -75,8 +75,7 @@ StorageMaterializePostgreSQL::StorageMaterializePostgreSQL(
: IStorage(table_id_) : IStorage(table_id_)
, global_context(context_) , global_context(context_)
, nested_storage(nested_storage_) , nested_storage(nested_storage_)
, is_postgresql_replica_database( , is_postgresql_replica_database(true)
DatabaseCatalog::instance().getDatabase(getStorageID().database_name)->getEngineName() == "MaterializePostgreSQL")
{ {
} }

2
tests/integration/test_postgresql_replica_database_engine/test.py
View File

@ -58,7 +58,6 @@ def assert_nested_table_is_created(table_name):
@pytest.mark.timeout(30) @pytest.mark.timeout(30)
def check_tables_are_synchronized(table_name, order_by='key'): def check_tables_are_synchronized(table_name, order_by='key'):
assert_nested_table_is_created(table_name) assert_nested_table_is_created(table_name)
print("nested ok")
expected = instance.query('select * from postgres_database.{} order by {};'.format(table_name, order_by)) expected = instance.query('select * from postgres_database.{} order by {};'.format(table_name, order_by))
result = instance.query('select * from test_database.{} order by {};'.format(table_name, order_by)) result = instance.query('select * from test_database.{} order by {};'.format(table_name, order_by))
@ -81,7 +80,6 @@ def started_cluster():
instance.query(''' instance.query('''
CREATE DATABASE postgres_database CREATE DATABASE postgres_database
ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword')''') ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword')''')
yield cluster yield cluster
finally: finally: