2020-02-03 12:54:36 +00:00
|
|
|
#pragma once
|
2021-04-10 23:33:54 +00:00
|
|
|
|
|
|
|
#include <Core/UUID.h>
|
|
|
|
#include <Interpreters/Context_fwd.h>
|
2020-03-13 10:30:55 +00:00
|
|
|
#include <Interpreters/StorageID.h>
|
2021-11-01 18:53:07 +00:00
|
|
|
#include <Databases/TablesLoader.h>
|
2020-03-10 19:36:17 +00:00
|
|
|
#include <Parsers/IAST_fwd.h>
|
2021-04-10 23:33:54 +00:00
|
|
|
#include <Storages/IStorage_fwd.h>
|
|
|
|
|
2020-02-03 12:54:36 +00:00
|
|
|
#include <boost/noncopyable.hpp>
|
2021-04-10 23:33:54 +00:00
|
|
|
#include <Poco/Logger.h>
|
|
|
|
|
|
|
|
#include <array>
|
2021-06-13 19:14:05 +00:00
|
|
|
#include <condition_variable>
|
2021-04-10 23:33:54 +00:00
|
|
|
#include <list>
|
2020-02-03 12:54:36 +00:00
|
|
|
#include <map>
|
2021-04-10 23:33:54 +00:00
|
|
|
#include <memory>
|
|
|
|
#include <mutex>
|
2020-02-12 16:54:26 +00:00
|
|
|
#include <set>
|
2021-04-10 23:33:54 +00:00
|
|
|
#include <shared_mutex>
|
2020-02-03 12:54:36 +00:00
|
|
|
#include <unordered_map>
|
2020-09-29 13:42:58 +00:00
|
|
|
#include <unordered_set>
|
2020-02-03 12:54:36 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
class IDatabase;
|
|
|
|
class Exception;
|
2020-03-13 15:41:36 +00:00
|
|
|
class ColumnsDescription;
|
2020-05-29 02:08:48 +00:00
|
|
|
struct ConstraintsDescription;
|
2020-02-12 16:54:26 +00:00
|
|
|
|
2020-02-03 12:54:36 +00:00
|
|
|
using DatabasePtr = std::shared_ptr<IDatabase>;
|
|
|
|
using DatabaseAndTable = std::pair<DatabasePtr, StoragePtr>;
|
2020-02-10 18:19:35 +00:00
|
|
|
using Databases = std::map<String, std::shared_ptr<IDatabase>>;
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2020-02-12 16:54:26 +00:00
|
|
|
/// Table -> set of table-views that make SELECT from it.
|
|
|
|
using ViewDependencies = std::map<StorageID, std::set<StorageID>>;
|
|
|
|
using Dependencies = std::vector<StorageID>;
|
2020-02-10 18:31:52 +00:00
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
|
2020-02-10 18:31:52 +00:00
|
|
|
/// Allows executing DDL query only in one thread.
|
|
|
|
/// Puts an element into the map, locks tables's mutex, counts how much threads run parallel query on the table,
|
|
|
|
/// when counter is 0 erases element in the destructor.
|
2020-08-15 11:16:43 +00:00
|
|
|
/// If the element already exists in the map, waits when ddl query will be finished in other thread.
|
2020-02-10 18:31:52 +00:00
|
|
|
class DDLGuard
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
struct Entry
|
|
|
|
{
|
|
|
|
std::unique_ptr<std::mutex> mutex;
|
|
|
|
UInt32 counter;
|
|
|
|
};
|
|
|
|
|
|
|
|
/// Element name -> (mutex, counter).
|
|
|
|
/// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion.
|
|
|
|
using Map = std::map<String, Entry>;
|
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
DDLGuard(
|
|
|
|
Map & map_,
|
|
|
|
std::shared_mutex & db_mutex_,
|
|
|
|
std::unique_lock<std::mutex> guards_lock_,
|
|
|
|
const String & elem,
|
|
|
|
const String & database_name);
|
2020-02-10 18:31:52 +00:00
|
|
|
~DDLGuard();
|
|
|
|
|
2021-02-02 19:39:04 +00:00
|
|
|
/// Unlocks table name, keeps holding read lock for database name
|
|
|
|
void releaseTableLock() noexcept;
|
|
|
|
|
2020-02-10 18:31:52 +00:00
|
|
|
private:
|
|
|
|
Map & map;
|
2020-08-18 15:15:27 +00:00
|
|
|
std::shared_mutex & db_mutex;
|
2020-02-10 18:31:52 +00:00
|
|
|
Map::iterator it;
|
|
|
|
std::unique_lock<std::mutex> guards_lock;
|
|
|
|
std::unique_lock<std::mutex> table_lock;
|
2021-02-02 19:39:04 +00:00
|
|
|
bool table_lock_removed = false;
|
|
|
|
bool is_database_guard = false;
|
2020-02-10 18:31:52 +00:00
|
|
|
};
|
|
|
|
|
2021-02-04 19:41:44 +00:00
|
|
|
using DDLGuardPtr = std::unique_ptr<DDLGuard>;
|
|
|
|
|
2020-03-10 19:36:17 +00:00
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
/// Creates temporary table in `_temporary_and_external_tables` with randomly generated unique StorageID.
|
|
|
|
/// Such table can be accessed from everywhere by its ID.
|
|
|
|
/// Removes the table from database on destruction.
|
|
|
|
/// TemporaryTableHolder object can be attached to a query or session Context, so table will be accessible through the context.
|
2021-04-10 23:33:54 +00:00
|
|
|
struct TemporaryTableHolder : boost::noncopyable, WithContext
|
2020-03-10 19:36:17 +00:00
|
|
|
{
|
2021-04-10 23:33:54 +00:00
|
|
|
using Creator = std::function<StoragePtr (const StorageID &)>;
|
2020-03-10 19:36:17 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
TemporaryTableHolder(ContextPtr context, const Creator & creator, const ASTPtr & query = {});
|
2020-03-10 19:36:17 +00:00
|
|
|
|
|
|
|
/// Creates temporary table with Engine=Memory
|
2020-05-29 02:08:48 +00:00
|
|
|
TemporaryTableHolder(
|
2021-04-10 23:33:54 +00:00
|
|
|
ContextPtr context,
|
2020-05-29 02:08:48 +00:00
|
|
|
const ColumnsDescription & columns,
|
|
|
|
const ConstraintsDescription & constraints,
|
2020-09-04 08:36:47 +00:00
|
|
|
const ASTPtr & query = {},
|
|
|
|
bool create_for_global_subquery = false);
|
2020-03-10 19:36:17 +00:00
|
|
|
|
|
|
|
TemporaryTableHolder(TemporaryTableHolder && rhs);
|
|
|
|
TemporaryTableHolder & operator = (TemporaryTableHolder && rhs);
|
|
|
|
|
|
|
|
~TemporaryTableHolder();
|
|
|
|
|
|
|
|
StorageID getGlobalTableID() const;
|
|
|
|
|
|
|
|
StoragePtr getTable() const;
|
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
operator bool () const { return id != UUIDHelpers::Nil; }
|
|
|
|
|
|
|
|
IDatabase * temporary_tables = nullptr;
|
|
|
|
UUID id = UUIDHelpers::Nil;
|
2020-03-10 19:36:17 +00:00
|
|
|
};
|
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
///TODO maybe remove shared_ptr from here?
|
2020-03-10 19:36:17 +00:00
|
|
|
using TemporaryTablesMapping = std::map<String, std::shared_ptr<TemporaryTableHolder>>;
|
2020-02-10 18:31:52 +00:00
|
|
|
|
2020-03-19 21:14:52 +00:00
|
|
|
class BackgroundSchedulePoolTaskHolder;
|
2020-03-13 15:41:36 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
/// For some reason Context is required to get Storage from Database object
|
2021-05-31 14:49:02 +00:00
|
|
|
class DatabaseCatalog : boost::noncopyable, WithMutableContext
|
2020-02-03 12:54:36 +00:00
|
|
|
{
|
|
|
|
public:
|
|
|
|
static constexpr const char * TEMPORARY_DATABASE = "_temporary_and_external_tables";
|
|
|
|
static constexpr const char * SYSTEM_DATABASE = "system";
|
2021-09-09 09:37:51 +00:00
|
|
|
static constexpr const char * INFORMATION_SCHEMA = "information_schema";
|
|
|
|
static constexpr const char * INFORMATION_SCHEMA_UPPERCASE = "INFORMATION_SCHEMA";
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2021-05-31 14:49:02 +00:00
|
|
|
static DatabaseCatalog & init(ContextMutablePtr global_context_);
|
2020-02-10 13:10:17 +00:00
|
|
|
static DatabaseCatalog & instance();
|
2020-04-08 23:59:39 +00:00
|
|
|
static void shutdown();
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2021-06-03 14:05:37 +00:00
|
|
|
void initializeAndLoadTemporaryDatabase();
|
2020-02-03 12:54:36 +00:00
|
|
|
void loadDatabases();
|
2021-09-02 18:05:13 +00:00
|
|
|
void loadMarkedAsDroppedTables();
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2020-02-10 18:31:52 +00:00
|
|
|
/// Get an object that protects the table from concurrently executing multiple DDL operations.
|
2021-02-04 19:41:44 +00:00
|
|
|
DDLGuardPtr getDDLGuard(const String & database, const String & table);
|
2020-08-18 15:15:27 +00:00
|
|
|
/// Get an object that protects the database from concurrent DDL queries all tables in the database
|
|
|
|
std::unique_lock<std::shared_mutex> getExclusiveDDLGuardForDatabase(const String & database);
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
|
2020-02-03 12:54:36 +00:00
|
|
|
void assertDatabaseExists(const String & database_name) const;
|
|
|
|
void assertDatabaseDoesntExist(const String & database_name) const;
|
|
|
|
|
|
|
|
DatabasePtr getDatabaseForTemporaryTables() const;
|
2020-02-10 13:10:17 +00:00
|
|
|
DatabasePtr getSystemDatabase() const;
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2020-07-08 14:28:07 +00:00
|
|
|
void attachDatabase(const String & database_name, const DatabasePtr & database);
|
2021-10-13 10:34:18 +00:00
|
|
|
DatabasePtr detachDatabase(ContextPtr local_context, const String & database_name, bool drop = false, bool check_empty = true);
|
2021-11-02 12:58:45 +00:00
|
|
|
void updateDatabaseName(const String & old_name, const String & new_name, const Strings & tables_in_database);
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
/// database_name must be not empty
|
2020-02-17 13:52:59 +00:00
|
|
|
DatabasePtr getDatabase(const String & database_name) const;
|
|
|
|
DatabasePtr tryGetDatabase(const String & database_name) const;
|
2020-07-06 08:30:11 +00:00
|
|
|
DatabasePtr getDatabase(const UUID & uuid) const;
|
|
|
|
DatabasePtr tryGetDatabase(const UUID & uuid) const;
|
2020-02-17 13:52:59 +00:00
|
|
|
bool isDatabaseExist(const String & database_name) const;
|
2020-02-03 12:54:36 +00:00
|
|
|
Databases getDatabases() const;
|
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
/// Same as getDatabase(const String & database_name), but if database_name is empty, current database of local_context is used
|
2021-04-10 23:33:54 +00:00
|
|
|
DatabasePtr getDatabase(const String & database_name, ContextPtr local_context) const;
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
/// For all of the following methods database_name in table_id must be not empty (even for temporary tables).
|
2021-04-10 23:33:54 +00:00
|
|
|
void assertTableDoesntExist(const StorageID & table_id, ContextPtr context) const;
|
|
|
|
bool isTableExist(const StorageID & table_id, ContextPtr context) const;
|
2020-03-13 15:41:36 +00:00
|
|
|
bool isDictionaryExist(const StorageID & table_id) const;
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
StoragePtr getTable(const StorageID & table_id, ContextPtr context) const;
|
|
|
|
StoragePtr tryGetTable(const StorageID & table_id, ContextPtr context) const;
|
|
|
|
DatabaseAndTable getDatabaseAndTable(const StorageID & table_id, ContextPtr context) const;
|
|
|
|
DatabaseAndTable tryGetDatabaseAndTable(const StorageID & table_id, ContextPtr context) const;
|
2020-05-28 23:01:18 +00:00
|
|
|
DatabaseAndTable getTableImpl(const StorageID & table_id,
|
2021-04-10 23:33:54 +00:00
|
|
|
ContextPtr context,
|
2020-05-28 23:01:18 +00:00
|
|
|
std::optional<Exception> * exception = nullptr) const;
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2020-02-12 16:54:26 +00:00
|
|
|
void addDependency(const StorageID & from, const StorageID & where);
|
|
|
|
void removeDependency(const StorageID & from, const StorageID & where);
|
|
|
|
Dependencies getDependencies(const StorageID & from) const;
|
|
|
|
|
|
|
|
/// For Materialized and Live View
|
2020-03-11 19:10:55 +00:00
|
|
|
void updateDependency(const StorageID & old_from, const StorageID & old_where,const StorageID & new_from, const StorageID & new_where);
|
2020-02-12 16:54:26 +00:00
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
/// If table has UUID, addUUIDMapping(...) must be called when table attached to some database
|
2020-10-25 17:27:21 +00:00
|
|
|
/// removeUUIDMapping(...) must be called when it detached,
|
|
|
|
/// and removeUUIDMappingFinally(...) must be called when table is dropped and its data removed from disk.
|
2020-03-13 15:41:36 +00:00
|
|
|
/// Such tables can be accessed by persistent UUID instead of database and table name.
|
2020-10-25 17:27:21 +00:00
|
|
|
void addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table);
|
2020-03-13 15:41:36 +00:00
|
|
|
void removeUUIDMapping(const UUID & uuid);
|
2020-10-25 17:27:21 +00:00
|
|
|
void removeUUIDMappingFinally(const UUID & uuid);
|
2020-03-23 00:12:13 +00:00
|
|
|
/// For moving table between databases
|
|
|
|
void updateUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table);
|
2020-10-27 20:52:49 +00:00
|
|
|
/// This method adds empty mapping (with database and storage equal to nullptr).
|
|
|
|
/// It's required to "lock" some UUIDs and protect us from collision.
|
|
|
|
/// Collisions of random 122-bit integers are very unlikely to happen,
|
|
|
|
/// but we allow to explicitly specify UUID in CREATE query (in particular for testing).
|
|
|
|
/// If some UUID was already added and we are trying to add it again,
|
|
|
|
/// this method will throw an exception.
|
|
|
|
void addUUIDMapping(const UUID & uuid);
|
2020-03-23 00:12:13 +00:00
|
|
|
|
2020-04-06 23:22:44 +00:00
|
|
|
static String getPathForUUID(const UUID & uuid);
|
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
DatabaseAndTable tryGetByUUID(const UUID & uuid) const;
|
|
|
|
|
2020-03-19 21:14:52 +00:00
|
|
|
String getPathForDroppedMetadata(const StorageID & table_id) const;
|
|
|
|
void enqueueDroppedTableCleanup(StorageID table_id, StoragePtr table, String dropped_metadata_path, bool ignore_delay = false);
|
|
|
|
|
2020-09-29 13:42:58 +00:00
|
|
|
void waitTableFinallyDropped(const UUID & uuid);
|
|
|
|
|
2021-11-02 12:58:45 +00:00
|
|
|
void addLoadingDependencies(const QualifiedTableName & table, TableNamesSet && dependencies);
|
2021-11-01 18:53:07 +00:00
|
|
|
void addLoadingDependencies(const DependenciesInfos & new_infos);
|
|
|
|
DependenciesInfo getLoadingDependenciesInfo(const StorageID & table_id) const;
|
|
|
|
|
2021-11-02 12:58:45 +00:00
|
|
|
TableNamesSet tryRemoveLoadingDependencies(const StorageID & table_id, bool check_dependencies, bool is_drop_database = false);
|
|
|
|
TableNamesSet tryRemoveLoadingDependenciesUnlocked(const QualifiedTableName & table_name, bool check_dependencies, bool is_drop_database = false);
|
|
|
|
void checkTableCanBeRemovedOrRenamed(const StorageID & table_id) const;
|
2021-11-01 18:53:07 +00:00
|
|
|
|
2021-11-02 14:30:22 +00:00
|
|
|
void updateLoadingDependencies(const StorageID & table_id, TableNamesSet && new_dependencies);
|
|
|
|
|
2020-02-03 12:54:36 +00:00
|
|
|
private:
|
2020-06-03 12:59:11 +00:00
|
|
|
// The global instance of database catalog. unique_ptr is to allow
|
|
|
|
// deferred initialization. Thought I'd use std::optional, but I can't
|
|
|
|
// make emplace(global_context_) compile with private constructor ¯\_(ツ)_/¯.
|
|
|
|
static std::unique_ptr<DatabaseCatalog> database_catalog;
|
|
|
|
|
2021-05-31 14:49:02 +00:00
|
|
|
explicit DatabaseCatalog(ContextMutablePtr global_context_);
|
2020-02-03 12:54:36 +00:00
|
|
|
void assertDatabaseExistsUnlocked(const String & database_name) const;
|
|
|
|
void assertDatabaseDoesntExistUnlocked(const String & database_name) const;
|
|
|
|
|
2020-04-08 23:59:39 +00:00
|
|
|
void shutdownImpl();
|
|
|
|
|
2020-03-19 21:14:52 +00:00
|
|
|
|
2020-02-03 12:54:36 +00:00
|
|
|
struct UUIDToStorageMapPart
|
|
|
|
{
|
|
|
|
std::unordered_map<UUID, DatabaseAndTable> map;
|
|
|
|
mutable std::mutex mutex;
|
|
|
|
};
|
|
|
|
|
2020-03-12 18:04:29 +00:00
|
|
|
static constexpr UInt64 bits_for_first_level = 4;
|
2020-02-03 12:54:36 +00:00
|
|
|
using UUIDToStorageMap = std::array<UUIDToStorageMapPart, 1ull << bits_for_first_level>;
|
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
static inline size_t getFirstLevelIdx(const UUID & uuid)
|
2020-02-03 12:54:36 +00:00
|
|
|
{
|
2021-01-27 00:54:57 +00:00
|
|
|
return uuid.toUnderType().items[0] >> (64 - bits_for_first_level);
|
2020-02-03 12:54:36 +00:00
|
|
|
}
|
|
|
|
|
2020-03-19 21:14:52 +00:00
|
|
|
struct TableMarkedAsDropped
|
|
|
|
{
|
|
|
|
StorageID table_id = StorageID::createEmpty();
|
|
|
|
StoragePtr table;
|
|
|
|
String metadata_path;
|
2021-05-08 15:20:40 +00:00
|
|
|
time_t drop_time{};
|
2020-03-19 21:14:52 +00:00
|
|
|
};
|
|
|
|
using TablesMarkedAsDropped = std::list<TableMarkedAsDropped>;
|
|
|
|
|
|
|
|
void dropTableDataTask();
|
2020-10-25 17:27:21 +00:00
|
|
|
void dropTableFinally(const TableMarkedAsDropped & table);
|
2020-03-19 21:14:52 +00:00
|
|
|
|
|
|
|
static constexpr size_t reschedule_time_ms = 100;
|
2020-11-01 17:37:54 +00:00
|
|
|
static constexpr time_t drop_error_cooldown_sec = 5;
|
2020-03-19 21:14:52 +00:00
|
|
|
|
2020-07-06 08:30:11 +00:00
|
|
|
using UUIDToDatabaseMap = std::unordered_map<UUID, DatabasePtr>;
|
|
|
|
|
2020-02-17 13:52:59 +00:00
|
|
|
mutable std::mutex databases_mutex;
|
2020-02-12 16:54:26 +00:00
|
|
|
|
2020-03-13 15:41:36 +00:00
|
|
|
ViewDependencies view_dependencies;
|
2020-02-12 16:54:26 +00:00
|
|
|
|
2020-02-03 12:54:36 +00:00
|
|
|
Databases databases;
|
2020-07-06 08:30:11 +00:00
|
|
|
UUIDToDatabaseMap db_uuid_map;
|
2020-02-03 12:54:36 +00:00
|
|
|
UUIDToStorageMap uuid_map;
|
|
|
|
|
2021-11-01 18:53:07 +00:00
|
|
|
DependenciesInfos loading_dependencies;
|
|
|
|
|
2020-02-12 16:54:26 +00:00
|
|
|
Poco::Logger * log;
|
2020-02-03 12:54:36 +00:00
|
|
|
|
2020-02-10 18:31:52 +00:00
|
|
|
/// Do not allow simultaneous execution of DDL requests on the same table.
|
2020-08-18 15:15:27 +00:00
|
|
|
/// database name -> database guard -> (table name mutex, counter),
|
|
|
|
/// counter: how many threads are running a query on the table at the same time
|
2020-02-10 18:31:52 +00:00
|
|
|
/// For the duration of the operation, an element is placed here, and an object is returned,
|
|
|
|
/// which deletes the element in the destructor when counter becomes zero.
|
2020-08-15 11:16:43 +00:00
|
|
|
/// In case the element already exists, waits when query will be executed in other thread. See class DDLGuard below.
|
2020-08-18 15:15:27 +00:00
|
|
|
using DatabaseGuard = std::pair<DDLGuard::Map, std::shared_mutex>;
|
|
|
|
using DDLGuards = std::map<String, DatabaseGuard>;
|
2020-02-10 18:31:52 +00:00
|
|
|
DDLGuards ddl_guards;
|
|
|
|
/// If you capture mutex and ddl_guards_mutex, then you need to grab them strictly in this order.
|
|
|
|
mutable std::mutex ddl_guards_mutex;
|
2020-03-19 21:14:52 +00:00
|
|
|
|
2020-04-08 01:02:00 +00:00
|
|
|
TablesMarkedAsDropped tables_marked_dropped;
|
2020-09-29 13:42:58 +00:00
|
|
|
std::unordered_set<UUID> tables_marked_dropped_ids;
|
2020-04-08 01:02:00 +00:00
|
|
|
mutable std::mutex tables_marked_dropped_mutex;
|
2020-03-19 21:14:52 +00:00
|
|
|
|
|
|
|
std::unique_ptr<BackgroundSchedulePoolTaskHolder> drop_task;
|
2020-05-07 11:29:58 +00:00
|
|
|
static constexpr time_t default_drop_delay_sec = 8 * 60;
|
|
|
|
time_t drop_delay_sec = default_drop_delay_sec;
|
2020-09-29 13:42:58 +00:00
|
|
|
std::condition_variable wait_table_finally_dropped;
|
2020-02-03 12:54:36 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|