2021-08-26 13:19:52 +00:00
|
|
|
#pragma once
|
2021-08-31 08:53:48 +00:00
|
|
|
#include <Core/Types.h>
|
2021-08-26 13:19:52 +00:00
|
|
|
#include <Core/QualifiedTableName.h>
|
2021-08-31 08:53:48 +00:00
|
|
|
#include <Parsers/IAST_fwd.h>
|
|
|
|
#include <Interpreters/Context_fwd.h>
|
2021-08-26 13:19:52 +00:00
|
|
|
#include <Common/ThreadPool.h>
|
2021-09-01 19:42:49 +00:00
|
|
|
#include <Common/Stopwatch.h>
|
2021-08-31 08:53:48 +00:00
|
|
|
#include <map>
|
|
|
|
#include <unordered_map>
|
|
|
|
#include <unordered_set>
|
|
|
|
#include <mutex>
|
2021-08-26 13:19:52 +00:00
|
|
|
|
2021-08-31 08:53:48 +00:00
|
|
|
namespace Poco
|
2021-08-26 13:19:52 +00:00
|
|
|
{
|
2021-08-31 08:53:48 +00:00
|
|
|
class Logger;
|
|
|
|
}
|
|
|
|
|
|
|
|
class AtomicStopwatch;
|
2021-08-26 13:19:52 +00:00
|
|
|
|
2021-08-31 08:53:48 +00:00
|
|
|
namespace DB
|
2021-08-26 13:19:52 +00:00
|
|
|
{
|
2021-08-31 08:53:48 +00:00
|
|
|
|
2021-09-01 19:42:49 +00:00
|
|
|
void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch);
|
|
|
|
|
|
|
|
|
2021-08-31 08:53:48 +00:00
|
|
|
class IDatabase;
|
|
|
|
using DatabasePtr = std::shared_ptr<IDatabase>;
|
2021-08-26 13:19:52 +00:00
|
|
|
|
2021-09-13 19:11:16 +00:00
|
|
|
struct ParsedTableMetadata
|
|
|
|
{
|
|
|
|
String path;
|
|
|
|
ASTPtr ast;
|
|
|
|
};
|
|
|
|
|
|
|
|
using ParsedMetadata = std::map<QualifiedTableName, ParsedTableMetadata>;
|
2021-09-01 19:42:49 +00:00
|
|
|
using TableNames = std::vector<QualifiedTableName>;
|
|
|
|
|
|
|
|
struct DependenciesInfo
|
|
|
|
{
|
|
|
|
/// How many dependencies this table have
|
|
|
|
size_t dependencies_count = 0;
|
2021-09-13 19:11:16 +00:00
|
|
|
/// List of tables/dictionaries which depend on this table/dictionary
|
|
|
|
TableNames dependent_database_objects;
|
2021-09-01 19:42:49 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
using DependenciesInfos = std::unordered_map<QualifiedTableName, DependenciesInfo>;
|
|
|
|
using DependenciesInfosIter = std::unordered_map<QualifiedTableName, DependenciesInfo>::iterator;
|
2021-08-26 13:19:52 +00:00
|
|
|
|
|
|
|
struct ParsedTablesMetadata
|
|
|
|
{
|
|
|
|
String default_database;
|
|
|
|
|
|
|
|
std::mutex mutex;
|
2021-09-13 19:11:16 +00:00
|
|
|
ParsedMetadata parsed_tables;
|
2021-09-01 19:42:49 +00:00
|
|
|
|
|
|
|
/// For logging
|
2021-08-26 13:19:52 +00:00
|
|
|
size_t total_dictionaries = 0;
|
2021-09-01 19:42:49 +00:00
|
|
|
|
2021-09-13 19:11:16 +00:00
|
|
|
/// List of tables/dictionaries that do not have any dependencies and can be loaded
|
|
|
|
TableNames independent_database_objects;
|
2021-09-01 19:42:49 +00:00
|
|
|
|
|
|
|
/// Actually it contains two different maps (with, probably, intersecting keys):
|
2021-09-13 19:11:16 +00:00
|
|
|
/// 1. table/dictionary name -> number of dependencies
|
|
|
|
/// 2. table/dictionary name -> dependent tables/dictionaries list (adjacency list of dependencies graph).
|
|
|
|
/// If table A depends on table B, then there is an edge B --> A, i.e. dependencies_info[B].dependent_database_objects contains A.
|
2021-09-01 19:42:49 +00:00
|
|
|
/// And dependencies_info[C].dependencies_count is a number of incoming edges for vertex C (how many tables we have to load before C).
|
|
|
|
DependenciesInfos dependencies_info;
|
2021-08-26 13:19:52 +00:00
|
|
|
};
|
|
|
|
|
2021-09-01 19:42:49 +00:00
|
|
|
/// Loads tables (and dictionaries) from specified databases
|
|
|
|
/// taking into account dependencies between them.
|
2021-08-26 13:19:52 +00:00
|
|
|
class TablesLoader
|
|
|
|
{
|
|
|
|
public:
|
2021-09-01 19:42:49 +00:00
|
|
|
using Databases = std::map<String, DatabasePtr>;
|
2021-08-26 13:19:52 +00:00
|
|
|
|
|
|
|
TablesLoader(ContextMutablePtr global_context_, Databases databases_, bool force_restore_ = false, bool force_attach_ = false);
|
2021-09-01 19:42:49 +00:00
|
|
|
TablesLoader() = delete;
|
2021-08-26 13:19:52 +00:00
|
|
|
|
|
|
|
void loadTables();
|
2021-09-01 19:42:49 +00:00
|
|
|
void startupTables();
|
2021-08-26 13:19:52 +00:00
|
|
|
|
|
|
|
private:
|
|
|
|
ContextMutablePtr global_context;
|
|
|
|
Databases databases;
|
|
|
|
bool force_restore;
|
|
|
|
bool force_attach;
|
|
|
|
|
2021-09-01 19:42:49 +00:00
|
|
|
Strings databases_to_load;
|
2021-09-13 19:11:16 +00:00
|
|
|
ParsedTablesMetadata metadata;
|
2021-08-26 13:19:52 +00:00
|
|
|
Poco::Logger * log;
|
|
|
|
std::atomic<size_t> tables_processed{0};
|
2021-09-01 19:42:49 +00:00
|
|
|
AtomicStopwatch stopwatch;
|
2021-08-26 13:19:52 +00:00
|
|
|
|
2021-09-01 19:42:49 +00:00
|
|
|
ThreadPool pool;
|
2021-08-26 13:19:52 +00:00
|
|
|
|
2021-09-01 19:42:49 +00:00
|
|
|
void removeUnresolvableDependencies();
|
2021-08-26 13:19:52 +00:00
|
|
|
|
2021-09-01 19:42:49 +00:00
|
|
|
void loadTablesInTopologicalOrder(ThreadPool & pool);
|
|
|
|
|
2021-09-13 19:11:16 +00:00
|
|
|
DependenciesInfosIter removeResolvedDependency(const DependenciesInfosIter & info_it, TableNames & independent_database_objects);
|
2021-09-01 19:42:49 +00:00
|
|
|
|
|
|
|
void startLoadingIndependentTables(ThreadPool & pool, size_t level);
|
2021-08-26 13:19:52 +00:00
|
|
|
|
|
|
|
void checkCyclicDependencies() const;
|
|
|
|
|
2021-09-01 19:42:49 +00:00
|
|
|
size_t getNumberOfTablesWithDependencies() const;
|
|
|
|
|
|
|
|
void logDependencyGraph() const;
|
2021-08-26 13:19:52 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|