ClickHouse/src/Databases/TablesLoader.h

113 lines
3.1 KiB
C++
Raw Normal View History

#pragma once
2021-08-31 08:53:48 +00:00
#include <Core/Types.h>
#include <Core/QualifiedTableName.h>
2021-08-31 08:53:48 +00:00
#include <Parsers/IAST_fwd.h>
#include <Interpreters/Context_fwd.h>
#include <Common/ThreadPool.h>
2021-09-01 19:42:49 +00:00
#include <Common/Stopwatch.h>
2021-08-31 08:53:48 +00:00
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <mutex>
2021-08-31 08:53:48 +00:00
namespace Poco
{
2021-08-31 08:53:48 +00:00
class Logger;
}
class AtomicStopwatch;
2021-08-31 08:53:48 +00:00
namespace DB
{
2021-08-31 08:53:48 +00:00
2021-09-01 19:42:49 +00:00
void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch);
2021-08-31 08:53:48 +00:00
class IDatabase;
using DatabasePtr = std::shared_ptr<IDatabase>;
2021-09-13 19:11:16 +00:00
struct ParsedTableMetadata
{
String path;
ASTPtr ast;
};
using ParsedMetadata = std::map<QualifiedTableName, ParsedTableMetadata>;
2021-09-01 19:42:49 +00:00
using TableNames = std::vector<QualifiedTableName>;
struct DependenciesInfo
{
/// How many dependencies this table have
size_t dependencies_count = 0;
2021-09-13 19:11:16 +00:00
/// List of tables/dictionaries which depend on this table/dictionary
TableNames dependent_database_objects;
2021-09-01 19:42:49 +00:00
};
using DependenciesInfos = std::unordered_map<QualifiedTableName, DependenciesInfo>;
using DependenciesInfosIter = std::unordered_map<QualifiedTableName, DependenciesInfo>::iterator;
struct ParsedTablesMetadata
{
String default_database;
std::mutex mutex;
2021-09-13 19:11:16 +00:00
ParsedMetadata parsed_tables;
2021-09-01 19:42:49 +00:00
/// For logging
size_t total_dictionaries = 0;
2021-09-01 19:42:49 +00:00
2021-09-13 19:11:16 +00:00
/// List of tables/dictionaries that do not have any dependencies and can be loaded
TableNames independent_database_objects;
2021-09-01 19:42:49 +00:00
/// Actually it contains two different maps (with, probably, intersecting keys):
2021-09-13 19:11:16 +00:00
/// 1. table/dictionary name -> number of dependencies
/// 2. table/dictionary name -> dependent tables/dictionaries list (adjacency list of dependencies graph).
/// If table A depends on table B, then there is an edge B --> A, i.e. dependencies_info[B].dependent_database_objects contains A.
2021-09-01 19:42:49 +00:00
/// And dependencies_info[C].dependencies_count is a number of incoming edges for vertex C (how many tables we have to load before C).
DependenciesInfos dependencies_info;
};
2021-09-01 19:42:49 +00:00
/// Loads tables (and dictionaries) from specified databases
/// taking into account dependencies between them.
class TablesLoader
{
public:
2021-09-01 19:42:49 +00:00
using Databases = std::map<String, DatabasePtr>;
TablesLoader(ContextMutablePtr global_context_, Databases databases_, bool force_restore_ = false, bool force_attach_ = false);
2021-09-01 19:42:49 +00:00
TablesLoader() = delete;
void loadTables();
2021-09-01 19:42:49 +00:00
void startupTables();
private:
ContextMutablePtr global_context;
Databases databases;
bool force_restore;
bool force_attach;
2021-09-01 19:42:49 +00:00
Strings databases_to_load;
2021-09-13 19:11:16 +00:00
ParsedTablesMetadata metadata;
Poco::Logger * log;
std::atomic<size_t> tables_processed{0};
2021-09-01 19:42:49 +00:00
AtomicStopwatch stopwatch;
2021-09-01 19:42:49 +00:00
ThreadPool pool;
2021-09-01 19:42:49 +00:00
void removeUnresolvableDependencies();
2021-09-01 19:42:49 +00:00
void loadTablesInTopologicalOrder(ThreadPool & pool);
2021-09-13 19:11:16 +00:00
DependenciesInfosIter removeResolvedDependency(const DependenciesInfosIter & info_it, TableNames & independent_database_objects);
2021-09-01 19:42:49 +00:00
void startLoadingIndependentTables(ThreadPool & pool, size_t level);
void checkCyclicDependencies() const;
2021-09-01 19:42:49 +00:00
size_t getNumberOfTablesWithDependencies() const;
void logDependencyGraph() const;
};
}