mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 13:10:48 +00:00
244 lines
11 KiB
C++
244 lines
11 KiB
C++
#pragma once
|
|
|
|
#include <chrono>
|
|
#include <functional>
|
|
#include <unordered_map>
|
|
#include <base/types.h>
|
|
#include <Interpreters/IExternalLoadable.h>
|
|
#include <Interpreters/IExternalLoaderConfigRepository.h>
|
|
#include <base/logger_useful.h>
|
|
#include <base/scope_guard.h>
|
|
#include <Common/ExternalLoaderStatus.h>
|
|
|
|
|
|
namespace DB
|
|
{
|
|
/* External configuration structure.
|
|
*
|
|
* <external_group>
|
|
* <external_config>
|
|
* <external_name>name</external_name>
|
|
* ....
|
|
* </external_config>
|
|
* </external_group>
|
|
*/
|
|
struct ExternalLoaderConfigSettings
|
|
{
|
|
std::string external_config;
|
|
std::string external_name;
|
|
std::string external_database;
|
|
std::string external_uuid;
|
|
};
|
|
|
|
/** Interface for manage user-defined objects.
|
|
* Monitors configuration file and automatically reloads objects in separate threads.
|
|
* The monitoring thread wakes up every 'check_period_sec' seconds and checks
|
|
* modification time of objects' configuration file. If said time is greater than
|
|
* 'config_last_modified', the objects are created from scratch using configuration file,
|
|
* possibly overriding currently existing objects with the same name (previous versions of
|
|
* overridden objects will live as long as there are any users retaining them).
|
|
*
|
|
* Apart from checking configuration file for modifications, each object
|
|
* has a lifetime of its own and may be updated if it supportUpdates.
|
|
* The time of next update is calculated by choosing uniformly a random number
|
|
* distributed between lifetime.min_sec and lifetime.max_sec.
|
|
* If either of lifetime.min_sec and lifetime.max_sec is zero, such object is never updated.
|
|
*/
|
|
class ExternalLoader
|
|
{
|
|
public:
|
|
using LoadablePtr = std::shared_ptr<const IExternalLoadable>;
|
|
using Loadables = std::vector<LoadablePtr>;
|
|
using Status = ExternalLoaderStatus;
|
|
|
|
using Duration = std::chrono::milliseconds;
|
|
using TimePoint = std::chrono::system_clock::time_point;
|
|
|
|
struct ObjectConfig
|
|
{
|
|
Poco::AutoPtr<Poco::Util::AbstractConfiguration> config;
|
|
String key_in_config;
|
|
String repository_name;
|
|
bool from_temp_repository = false;
|
|
String path;
|
|
};
|
|
|
|
struct LoadResult
|
|
{
|
|
Status status = Status::NOT_EXIST;
|
|
String name;
|
|
LoadablePtr object;
|
|
TimePoint loading_start_time;
|
|
TimePoint last_successful_update_time;
|
|
Duration loading_duration;
|
|
std::exception_ptr exception;
|
|
std::shared_ptr<const ObjectConfig> config;
|
|
};
|
|
|
|
using LoadResults = std::vector<LoadResult>;
|
|
|
|
template <typename T>
|
|
static constexpr bool is_scalar_load_result_type = std::is_same_v<T, LoadResult> || std::is_same_v<T, LoadablePtr>;
|
|
|
|
template <typename T>
|
|
static constexpr bool is_vector_load_result_type = std::is_same_v<T, LoadResults> || std::is_same_v<T, Loadables>;
|
|
|
|
ExternalLoader(const String & type_name_, Poco::Logger * log);
|
|
virtual ~ExternalLoader();
|
|
|
|
/// Adds a repository which will be used to read configurations from.
|
|
scope_guard addConfigRepository(std::unique_ptr<IExternalLoaderConfigRepository> config_repository) const;
|
|
|
|
void setConfigSettings(const ExternalLoaderConfigSettings & settings_);
|
|
|
|
/// Sets whether all the objects from the configuration should be always loaded (even those which are never used).
|
|
void enableAlwaysLoadEverything(bool enable);
|
|
|
|
/// Sets whether the objects should be loaded asynchronously, each loading in a new thread (from the thread pool).
|
|
void enableAsyncLoading(bool enable);
|
|
|
|
/// Sets settings for periodic updates.
|
|
void enablePeriodicUpdates(bool enable);
|
|
|
|
/// Returns the status of the object.
|
|
/// If the object has not been loaded yet then the function returns Status::NOT_LOADED.
|
|
/// If the specified name isn't found in the configuration then the function returns Status::NOT_EXIST.
|
|
Status getCurrentStatus(const String & name) const;
|
|
|
|
/// Returns the result of loading the object.
|
|
/// The function doesn't load anything, it just returns the current load result as is.
|
|
template <typename ReturnType = LoadResult, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>>
|
|
ReturnType getLoadResult(const String & name) const;
|
|
|
|
using FilterByNameFunction = std::function<bool(const String &)>;
|
|
|
|
/// Returns all the load results as a map.
|
|
/// The function doesn't load anything, it just returns the current load results as is.
|
|
template <typename ReturnType = LoadResults, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType getLoadResults() const { return getLoadResults<ReturnType>(FilterByNameFunction{}); }
|
|
|
|
template <typename ReturnType = LoadResults, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType getLoadResults(const FilterByNameFunction & filter) const;
|
|
|
|
/// Returns all loaded objects as a map.
|
|
/// The function doesn't load anything, it just returns the current load results as is.
|
|
Loadables getLoadedObjects() const;
|
|
Loadables getLoadedObjects(const FilterByNameFunction & filter) const;
|
|
|
|
/// Returns true if any object was loaded.
|
|
bool hasLoadedObjects() const;
|
|
size_t getNumberOfLoadedObjects() const;
|
|
|
|
/// Returns true if there is no object.
|
|
bool hasObjects() const { return getNumberOfObjects() == 0; }
|
|
|
|
/// Returns number of objects.
|
|
size_t getNumberOfObjects() const;
|
|
|
|
static constexpr Duration NO_WAIT = Duration::zero();
|
|
static constexpr Duration WAIT = Duration::max();
|
|
|
|
/// Loads a specified object.
|
|
/// The function does nothing if it's already loaded.
|
|
/// The function doesn't throw an exception if it's failed to load.
|
|
template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>>
|
|
ReturnType tryLoad(const String & name, Duration timeout = WAIT) const;
|
|
|
|
/// Loads objects by filter.
|
|
/// The function does nothing for already loaded objects, it just returns them.
|
|
/// The function doesn't throw an exception if it's failed to load something.
|
|
template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType tryLoad(const FilterByNameFunction & filter, Duration timeout = WAIT) const;
|
|
|
|
/// Loads all objects.
|
|
/// The function does nothing for already loaded objects, it just returns them.
|
|
/// The function doesn't throw an exception if it's failed to load something.
|
|
template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType tryLoadAll(Duration timeout = WAIT) const { return tryLoad<ReturnType>(FilterByNameFunction{}, timeout); }
|
|
|
|
/// Loads a specified object.
|
|
/// The function does nothing if it's already loaded.
|
|
/// The function throws an exception if it's failed to load.
|
|
template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>>
|
|
ReturnType load(const String & name) const;
|
|
|
|
/// Loads objects by filter.
|
|
/// The function does nothing for already loaded objects, it just returns them.
|
|
/// The function throws an exception if it's failed to load something.
|
|
template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType load(const FilterByNameFunction & filter) const;
|
|
|
|
/// Loads all objects. Not recommended to use.
|
|
/// The function does nothing for already loaded objects, it just returns them.
|
|
/// The function throws an exception if it's failed to load something.
|
|
template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType loadAll() const { return load<ReturnType>(FilterByNameFunction{}); }
|
|
|
|
/// Loads or reloads a specified object.
|
|
/// The function reloads the object if it's already loaded.
|
|
/// The function throws an exception if it's failed to load or reload.
|
|
template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>>
|
|
ReturnType loadOrReload(const String & name) const;
|
|
|
|
/// Loads or reloads objects by filter.
|
|
/// The function reloads the objects which are already loaded.
|
|
/// The function throws an exception if it's failed to load or reload something.
|
|
template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType loadOrReload(const FilterByNameFunction & filter) const;
|
|
|
|
/// Load or reloads all objects. Not recommended to use.
|
|
/// The function throws an exception if it's failed to load or reload something.
|
|
template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType loadOrReloadAll() const { return loadOrReload<ReturnType>(FilterByNameFunction{}); }
|
|
|
|
/// Reloads objects by filter which were tried to load before (successfully or not).
|
|
/// The function throws an exception if it's failed to load or reload something.
|
|
template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
|
|
ReturnType reloadAllTriedToLoad() const;
|
|
|
|
/// Check if object with name exists in configuration
|
|
bool has(const String & name) const;
|
|
|
|
/// Reloads all config repositories.
|
|
void reloadConfig() const;
|
|
|
|
/// Reloads only a specified config repository.
|
|
void reloadConfig(const String & repository_name) const;
|
|
|
|
/// Reload only a specified path in a specified config repository.
|
|
void reloadConfig(const String & repository_name, const String & path) const;
|
|
|
|
protected:
|
|
virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0;
|
|
|
|
private:
|
|
void checkLoaded(const LoadResult & result, bool check_no_errors) const;
|
|
void checkLoaded(const LoadResults & results, bool check_no_errors) const;
|
|
|
|
Strings getAllTriedToLoadNames() const;
|
|
|
|
LoadablePtr createObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const;
|
|
|
|
/// We have to read configuration from LoadablesConfigReader and load configuration using LoadingDispatcher atomically.
|
|
/// Otherwise we can read configuration in one thread, then read and load newer configuration in another thread,
|
|
/// and then load outdated configuration from the first thread.
|
|
/// Remarkably, each class (LoadablesConfigReader, LoadingDispatcher, PeriodicUpdater) has own mutex for own purposes,
|
|
/// but it does not save from complex logical race conditions.
|
|
/// TODO Refactor dictionaries loading and get rid of this.
|
|
mutable std::recursive_mutex config_mutex;
|
|
|
|
class LoadablesConfigReader;
|
|
std::unique_ptr<LoadablesConfigReader> config_files_reader;
|
|
|
|
class LoadingDispatcher;
|
|
std::unique_ptr<LoadingDispatcher> loading_dispatcher;
|
|
|
|
class PeriodicUpdater;
|
|
std::unique_ptr<PeriodicUpdater> periodic_updater;
|
|
|
|
const String type_name;
|
|
Poco::Logger * log;
|
|
};
|
|
|
|
}
|