mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-22 09:40:49 +00:00
fix build [#CLICKHOUSE-3305]
fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305] fix build [#CLICKHOUSE-3305]
This commit is contained in:
parent
61bef2adf1
commit
99e9c0c486
@ -8,16 +8,14 @@
|
||||
#include <Common/PODArray.h>
|
||||
#include <memory>
|
||||
#include <chrono>
|
||||
#include <Dictionaries/IDictionarySource.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class IDictionarySource;
|
||||
|
||||
struct IDictionaryBase;
|
||||
using DictionaryPtr = std::unique_ptr<IDictionaryBase>;
|
||||
|
||||
struct DictionaryLifetime;
|
||||
struct DictionaryStructure;
|
||||
class ColumnString;
|
||||
|
||||
@ -57,9 +55,14 @@ struct IDictionaryBase : public IExternalLoadable
|
||||
|
||||
bool supportUpdates() const override { return !isCached(); }
|
||||
|
||||
virtual std::shared_ptr<IExternalLoadable> cloneObject() const override
|
||||
bool isModified() const override
|
||||
{ auto source = getSource();
|
||||
return source && source->isModified();
|
||||
}
|
||||
|
||||
std::unique_ptr<IExternalLoadable> cloneObject() const override
|
||||
{
|
||||
return std::static_pointer_cast<IDictionaryBase>(clone());
|
||||
return clone();
|
||||
};
|
||||
|
||||
std::shared_ptr<IDictionaryBase> shared_from_this()
|
||||
|
@ -44,13 +44,13 @@ ExternalDictionaries::ExternalDictionaries(Context & context, bool throw_on_erro
|
||||
getExternalDictionariesUpdateSettings(),
|
||||
getExternalDictionariesConfigSettings(),
|
||||
&Logger::get("ExternalDictionaries"),
|
||||
"external dictionary", throw_on_error),
|
||||
"external dictionary"),
|
||||
context(context)
|
||||
{
|
||||
|
||||
init(throw_on_error);
|
||||
}
|
||||
|
||||
ExternalDictionaries::LoadablePtr ExternalDictionaries::create(
|
||||
std::unique_ptr<IExternalLoadable> ExternalDictionaries::create(
|
||||
const std::string & name, const Configuration & config, const std::string & config_prefix)
|
||||
{
|
||||
return DictionaryFactory::instance().create(name, config, config_prefix, context);
|
||||
|
@ -30,8 +30,8 @@ public:
|
||||
|
||||
protected:
|
||||
|
||||
LoadablePtr create(const std::string & name, const Configuration & config,
|
||||
const std::string & config_prefix) override;
|
||||
std::unique_ptr<IExternalLoadable> create(const std::string & name, const Configuration & config,
|
||||
const std::string & config_prefix) override;
|
||||
|
||||
using ExternalLoader::getObjectsMap;
|
||||
|
||||
|
@ -1,7 +1,9 @@
|
||||
#include <Interpreters/ExternalLoader.h>
|
||||
#include <Common/StringUtils.h>
|
||||
#include <Common/MemoryTracker.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/getMultipleKeysFromConfig.h>
|
||||
#include <Common/setThreadName.h>
|
||||
#include <ext/scope_guard.h>
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <Poco/Glob.h>
|
||||
@ -46,10 +48,19 @@ void ExternalLoader::reloadPeriodically()
|
||||
ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config,
|
||||
const ExternalLoaderUpdateSettings & update_settings,
|
||||
const ExternalLoaderConfigSettings & config_settings,
|
||||
Logger * log, const std::string & loadable_object_name, bool throw_on_error)
|
||||
Logger * log, const std::string & loadable_object_name)
|
||||
: config(config), update_settings(update_settings), config_settings(config_settings),
|
||||
log(log), object_name(loadable_object_name)
|
||||
{
|
||||
}
|
||||
|
||||
void ExternalLoader::init(bool throw_on_error)
|
||||
{
|
||||
if (is_initialized)
|
||||
return;
|
||||
|
||||
is_initialized = true;
|
||||
|
||||
{
|
||||
/// During synchronous loading of external dictionaries at moment of query execution,
|
||||
/// we should not use per query memory limit.
|
||||
@ -116,7 +127,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
|
||||
|
||||
try
|
||||
{
|
||||
auto loadable_ptr = failed_loadable_object.second.loadable->clone();
|
||||
auto loadable_ptr = failed_loadable_object.second.loadable->cloneObject();
|
||||
if (const auto exception_ptr = loadable_ptr->getCreationException())
|
||||
{
|
||||
/// recalculate next attempt time
|
||||
@ -143,7 +154,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
|
||||
const auto dict_it = loadable_objects.find(name);
|
||||
|
||||
dict_it->second.loadable.reset();
|
||||
dict_it->second.loadable = loadable_ptr;
|
||||
dict_it->second.loadable = std::move(loadable_ptr);
|
||||
|
||||
/// clear stored exception on success
|
||||
dict_it->second.exception = std::exception_ptr{};
|
||||
@ -200,13 +211,13 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
|
||||
if (current->isModified())
|
||||
{
|
||||
/// create new version of loadable object
|
||||
auto new_version = current->clone();
|
||||
auto new_version = current->cloneObject();
|
||||
|
||||
if (const auto exception_ptr = new_version->getCreationException())
|
||||
std::rethrow_exception(exception_ptr);
|
||||
|
||||
loadable_object.second.loadable.reset();
|
||||
loadable_object.second.loadable = new_version;
|
||||
loadable_object.second.loadable = std::move(new_version);
|
||||
}
|
||||
}
|
||||
|
||||
@ -321,9 +332,9 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const
|
||||
const auto failed_dict_it = failed_loadable_objects.find(name);
|
||||
FailedLoadableInfo info{std::move(object_ptr), std::chrono::system_clock::now() + delay, 0};
|
||||
if (failed_dict_it != std::end(failed_loadable_objects))
|
||||
failed_dict_it->second = info;
|
||||
(*failed_dict_it).second = std::move(info);
|
||||
else
|
||||
failed_loadable_objects.emplace(name, info);
|
||||
failed_loadable_objects.emplace(name, std::move(info));
|
||||
|
||||
std::rethrow_exception(exception_ptr);
|
||||
}
|
||||
@ -343,12 +354,12 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const
|
||||
|
||||
/// add new loadable object or update an existing version
|
||||
if (object_it == std::end(loadable_objects))
|
||||
loadable_objects.emplace(name, LoadableInfo{object_ptr, config_path});
|
||||
loadable_objects.emplace(name, LoadableInfo{std::move(object_ptr), config_path});
|
||||
else
|
||||
{
|
||||
if (object_it->second.loadable)
|
||||
object_it->second.loadable.reset();
|
||||
object_it->second.loadable = object_ptr;
|
||||
object_it->second.loadable = std::move(object_ptr);
|
||||
|
||||
/// erase stored exception on success
|
||||
object_it->second.exception = std::exception_ptr{};
|
||||
@ -414,4 +425,9 @@ ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name
|
||||
return it->second.loadable;
|
||||
}
|
||||
|
||||
std::tuple<std::unique_lock<std::mutex>, const ExternalLoader::ObjectsMap &> ExternalLoader::getObjectsMap() const
|
||||
{
|
||||
return std::make_tuple(std::unique_lock<std::mutex>(map_mutex), std::cref(loadable_objects));
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -1,20 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/setThreadName.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <common/MultiVersion.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Poco/Event.h>
|
||||
#include <unistd.h>
|
||||
#include <ctime>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <chrono>
|
||||
#include <pcg_random.hpp>
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
#include <tuple>
|
||||
#include <Interpreters/IExternalLoadable.h>
|
||||
#include <Core/Types.h>
|
||||
#include <pcg_random.hpp>
|
||||
#include <Common/randomSeed.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -65,43 +61,9 @@ struct ExternalLoaderConfigSettings
|
||||
class ExternalLoader
|
||||
{
|
||||
public:
|
||||
using Configuration = Poco::Util::AbstractConfiguration;
|
||||
using ObjectsMap = std::unordered_map<std::string, LoadableInfo>;
|
||||
|
||||
/// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
|
||||
ExternalLoader(const Configuration & config,
|
||||
const ExternalLoaderUpdateSettings & update_settings,
|
||||
const ExternalLoaderConfigSettings & config_settings,
|
||||
Logger * log, const std::string & loadable_object_name, bool throw_on_error);
|
||||
~ExternalLoader();
|
||||
|
||||
/// Forcibly reloads all loadable objects.
|
||||
void reload();
|
||||
|
||||
/// Forcibly reloads specified loadable object.
|
||||
void reload(const std::string & name);
|
||||
|
||||
LoadablePtr getLoadable(const std::string & name) const;
|
||||
|
||||
protected:
|
||||
virtual LoadablePtr create(const std::string & name, const Configuration & config,
|
||||
const std::string & config_prefix) = 0;
|
||||
|
||||
/// Direct access to objects.
|
||||
std::tuple<std::lock_guard<std::mutex>, const ObjectsMap &> getObjectsMap()
|
||||
{
|
||||
return std::make_tuple(std::lock_guard<std::mutex>(map_mutex), std::cref(loadable_objects));
|
||||
}
|
||||
using LoadablePtr = std::shared_ptr<IExternalLoadable>;
|
||||
|
||||
private:
|
||||
|
||||
/// Protects only dictionaries map.
|
||||
mutable std::mutex map_mutex;
|
||||
|
||||
/// Protects all data, currently used to avoid races between updating thread and SYSTEM queries
|
||||
mutable std::mutex all_mutex;
|
||||
|
||||
using LoadablePtr = std::shared_ptr<IExternalLoadable>;
|
||||
struct LoadableInfo final
|
||||
{
|
||||
LoadablePtr loadable;
|
||||
@ -116,16 +78,53 @@ private:
|
||||
UInt64 error_count;
|
||||
};
|
||||
|
||||
public:
|
||||
using Configuration = Poco::Util::AbstractConfiguration;
|
||||
using ObjectsMap = std::unordered_map<std::string, LoadableInfo>;
|
||||
|
||||
/// Objects will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
|
||||
ExternalLoader(const Configuration & config,
|
||||
const ExternalLoaderUpdateSettings & update_settings,
|
||||
const ExternalLoaderConfigSettings & config_settings,
|
||||
Logger * log, const std::string & loadable_object_name);
|
||||
virtual ~ExternalLoader();
|
||||
|
||||
/// Forcibly reloads all loadable objects.
|
||||
void reload();
|
||||
|
||||
/// Forcibly reloads specified loadable object.
|
||||
void reload(const std::string & name);
|
||||
|
||||
LoadablePtr getLoadable(const std::string & name) const;
|
||||
|
||||
protected:
|
||||
virtual std::unique_ptr<IExternalLoadable> create(const std::string & name, const Configuration & config,
|
||||
const std::string & config_prefix) = 0;
|
||||
|
||||
/// Direct access to objects.
|
||||
std::tuple<std::unique_lock<std::mutex>, const ObjectsMap &> getObjectsMap() const;
|
||||
|
||||
/// Should be called in derived constructor (to avoid pure virtual call).
|
||||
void init(bool throw_on_error);
|
||||
|
||||
private:
|
||||
|
||||
bool is_initialized = false;
|
||||
|
||||
/// Protects only objects map.
|
||||
mutable std::mutex map_mutex;
|
||||
|
||||
/// Protects all data, currently used to avoid races between updating thread and SYSTEM queries
|
||||
mutable std::mutex all_mutex;
|
||||
|
||||
/// name -> loadable.
|
||||
ObjectsMap loadable_objects;
|
||||
|
||||
/** Here are loadable objects, that has been never loaded successfully.
|
||||
* They are also in 'loadable_objects', but with nullptr as 'loadable'.
|
||||
*/
|
||||
/// Here are loadable objects, that has been never loaded successfully.
|
||||
/// They are also in 'loadable_objects', but with nullptr as 'loadable'.
|
||||
std::unordered_map<std::string, FailedLoadableInfo> failed_loadable_objects;
|
||||
|
||||
/** Both for dictionaries and failed_dictionaries.
|
||||
*/
|
||||
/// Both for loadable_objects and failed_loadable_objects.
|
||||
std::unordered_map<std::string, std::chrono::system_clock::time_point> update_times;
|
||||
|
||||
pcg64 rnd_engine{randomSeed()};
|
||||
@ -143,7 +142,7 @@ private:
|
||||
|
||||
std::unordered_map<std::string, Poco::Timestamp> last_modification_times;
|
||||
|
||||
/// Check dictionaries definitions in config files and reload or/and add new ones if the definition is changed
|
||||
/// Check objects definitions in config files and reload or/and add new ones if the definition is changed
|
||||
/// If loadable_name is not empty, load only loadable object with name loadable_name
|
||||
void reloadFromConfigFiles(bool throw_on_error, bool force_reload = false, const std::string & loadable_name = "");
|
||||
void reloadFromConfigFile(const std::string & config_path, bool throw_on_error, bool force_reload,
|
||||
|
@ -1,14 +1,18 @@
|
||||
#pragma once
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <Core/Types.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace Poco::Util
|
||||
{
|
||||
class AbstractConfiguration;
|
||||
class AbstractConfiguration;
|
||||
}
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Min and max lifetimes for a loadable object or it's entry
|
||||
struct ExternalLoadableLifetime final
|
||||
{
|
||||
@ -31,7 +35,7 @@ public:
|
||||
|
||||
virtual bool isModified() const = 0;
|
||||
|
||||
virtual std::shared_ptr<IExternalLoadable> cloneObject() const = 0;
|
||||
virtual std::unique_ptr<IExternalLoadable> cloneObject() const = 0;
|
||||
|
||||
virtual std::exception_ptr getCreationException() const = 0;
|
||||
};
|
||||
|
Loading…
Reference in New Issue
Block a user