2018-11-27 16:11:46 +00:00
# include "ExternalLoader.h"
2019-06-02 12:11:01 +00:00
# include <mutex>
# include <pcg_random.hpp>
# include <Common/Config/AbstractConfigurationComparison.h>
2017-10-06 11:10:01 +00:00
# include <Common/Exception.h>
2019-06-02 12:11:01 +00:00
# include <Common/StringUtils/StringUtils.h>
# include <Common/ThreadPool.h>
# include <Common/randomSeed.h>
2017-10-06 11:10:01 +00:00
# include <Common/setThreadName.h>
2020-03-16 10:32:23 +00:00
# include <Common/StatusInfo.h>
2021-10-02 07:13:14 +00:00
# include <base/chrono_io.h>
2022-04-27 15:05:45 +00:00
# include <Common/scope_guard_safe.h>
2019-12-12 16:41:41 +00:00
# include <boost/range/adaptor/map.hpp>
# include <boost/range/algorithm/copy.hpp>
2019-12-12 18:33:43 +00:00
# include <unordered_set>
2019-06-02 12:11:01 +00:00
2017-10-06 10:31:06 +00:00
2020-03-11 15:30:02 +00:00
namespace CurrentStatusInfo
{
2020-03-12 12:29:28 +00:00
extern const Status DictionaryStatus ;
2020-03-11 15:30:02 +00:00
}
2017-10-06 10:31:06 +00:00
namespace DB
{
namespace ErrorCodes
{
2019-06-02 12:11:01 +00:00
extern const int LOGICAL_ERROR ;
extern const int BAD_ARGUMENTS ;
2020-10-27 14:21:51 +00:00
extern const int DICTIONARIES_WAS_NOT_LOADED ;
2017-10-06 10:31:06 +00:00
}
2019-12-12 18:33:43 +00:00
2019-10-22 12:57:58 +00:00
namespace
{
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
ReturnType convertTo ( ExternalLoader : : LoadResult result )
{
if constexpr ( std : : is_same_v < ReturnType , ExternalLoader : : LoadResult > )
return result ;
else
{
static_assert ( std : : is_same_v < ReturnType , ExternalLoader : : LoadablePtr > ) ;
return std : : move ( result . object ) ;
}
}
2019-10-22 12:57:58 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
ReturnType convertTo ( ExternalLoader : : LoadResults results )
2019-10-22 12:57:58 +00:00
{
2019-12-12 18:33:43 +00:00
if constexpr ( std : : is_same_v < ReturnType , ExternalLoader : : LoadResults > )
return results ;
else
{
static_assert ( std : : is_same_v < ReturnType , ExternalLoader : : Loadables > ) ;
ExternalLoader : : Loadables objects ;
objects . reserve ( results . size ( ) ) ;
2022-03-02 17:22:12 +00:00
for ( auto & & result : results )
2019-12-12 18:33:43 +00:00
{
if ( auto object = std : : move ( result . object ) )
objects . push_back ( std : : move ( object ) ) ;
}
return objects ;
}
2019-10-22 12:57:58 +00:00
}
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
ReturnType notExists ( const String & name )
2019-10-22 12:57:58 +00:00
{
2019-12-12 18:33:43 +00:00
if constexpr ( std : : is_same_v < ReturnType , ExternalLoader : : LoadResult > )
{
ExternalLoader : : LoadResult res ;
res . name = name ;
return res ;
}
else
{
static_assert ( std : : is_same_v < ReturnType , ExternalLoader : : LoadablePtr > ) ;
return nullptr ;
}
2019-10-22 12:57:58 +00:00
}
2019-12-12 18:33:43 +00:00
/// Lock mutex only in async mode
/// In other case does nothing
struct LoadingGuardForAsyncLoad
{
std : : unique_lock < std : : mutex > lock ;
LoadingGuardForAsyncLoad ( bool async , std : : mutex & mutex )
{
if ( async )
lock = std : : unique_lock ( mutex ) ;
}
} ;
2019-10-22 12:57:58 +00:00
}
2017-10-06 10:31:06 +00:00
2019-09-30 16:12:08 +00:00
/** Reads configurations from configuration repository and parses it.
2019-06-02 12:11:01 +00:00
*/
2019-09-30 16:12:08 +00:00
class ExternalLoader : : LoadablesConfigReader : private boost : : noncopyable
2017-10-06 10:31:06 +00:00
{
2019-06-02 12:11:01 +00:00
public :
2020-05-30 21:57:37 +00:00
LoadablesConfigReader ( const String & type_name_ , Poco : : Logger * log_ )
2019-09-30 16:12:08 +00:00
: type_name ( type_name_ ) , log ( log_ )
2017-10-06 10:31:06 +00:00
{
2019-06-02 12:11:01 +00:00
}
2019-09-30 16:12:08 +00:00
~ LoadablesConfigReader ( ) = default ;
2017-10-06 10:31:06 +00:00
2019-12-30 23:30:06 +00:00
using Repository = IExternalLoaderConfigRepository ;
2019-12-12 16:41:41 +00:00
2019-12-30 23:30:06 +00:00
void addConfigRepository ( std : : unique_ptr < Repository > repository )
2019-06-02 12:11:01 +00:00
{
std : : lock_guard lock { mutex } ;
2019-12-30 23:30:06 +00:00
auto * ptr = repository . get ( ) ;
repositories . emplace ( ptr , RepositoryInfo { std : : move ( repository ) , { } } ) ;
2019-12-12 16:41:41 +00:00
need_collect_object_configs = true ;
2017-10-06 10:31:06 +00:00
}
2019-12-30 23:30:06 +00:00
void removeConfigRepository ( Repository * repository )
2019-10-15 18:04:17 +00:00
{
std : : lock_guard lock { mutex } ;
2019-12-30 23:30:06 +00:00
auto it = repositories . find ( repository ) ;
2019-12-12 16:41:41 +00:00
if ( it = = repositories . end ( ) )
2019-12-30 23:30:06 +00:00
return ;
2019-12-12 16:41:41 +00:00
repositories . erase ( it ) ;
need_collect_object_configs = true ;
2019-12-30 23:30:06 +00:00
}
void setConfigSettings ( const ExternalLoaderConfigSettings & settings_ )
{
std : : lock_guard lock { mutex } ;
settings = settings_ ;
2019-10-15 18:04:17 +00:00
}
2021-10-13 06:56:34 +00:00
struct ObjectConfigs
{
std : : unordered_map < String /* object's name */ , std : : shared_ptr < const ObjectConfig > > configs_by_name ;
size_t counter = 0 ;
} ;
using ObjectConfigsPtr = std : : shared_ptr < const ObjectConfigs > ;
2017-10-06 10:31:06 +00:00
2019-12-12 16:41:41 +00:00
/// Reads all repositories.
2019-09-30 11:18:01 +00:00
ObjectConfigsPtr read ( )
2019-06-02 12:11:01 +00:00
{
2019-10-18 15:44:32 +00:00
std : : lock_guard lock ( mutex ) ;
2019-12-12 16:41:41 +00:00
readRepositories ( ) ;
collectObjectConfigs ( ) ;
return object_configs ;
2019-10-18 15:44:32 +00:00
}
2019-12-12 16:41:41 +00:00
/// Reads only a specified repository.
/// This functions checks only a specified repository but returns configs from all repositories.
ObjectConfigsPtr read ( const String & repository_name )
2019-10-18 15:44:32 +00:00
{
std : : lock_guard lock ( mutex ) ;
2019-12-12 16:41:41 +00:00
readRepositories ( repository_name ) ;
collectObjectConfigs ( ) ;
return object_configs ;
}
2019-10-18 15:44:32 +00:00
2019-12-12 16:41:41 +00:00
/// Reads only a specified path from a specified repository.
/// This functions checks only a specified repository but returns configs from all repositories.
ObjectConfigsPtr read ( const String & repository_name , const String & path )
{
std : : lock_guard lock ( mutex ) ;
readRepositories ( repository_name , path ) ;
collectObjectConfigs ( ) ;
return object_configs ;
2019-10-18 15:44:32 +00:00
}
private :
2019-12-12 16:41:41 +00:00
struct FileInfo
2019-10-18 15:44:32 +00:00
{
Poco : : Timestamp last_update_time = 0 ;
2019-12-12 16:41:41 +00:00
bool in_use = true ; // Whether the `FileInfo` should be destroyed because the correspondent file is deleted.
2020-04-12 20:50:32 +00:00
Poco : : AutoPtr < Poco : : Util : : AbstractConfiguration > file_contents ; // Parsed contents of the file.
std : : unordered_map < String /* object name */ , String /* key in file_contents */ > objects ;
2019-10-18 15:44:32 +00:00
} ;
2019-12-12 16:41:41 +00:00
struct RepositoryInfo
2019-10-18 15:44:32 +00:00
{
2019-12-30 23:30:06 +00:00
std : : unique_ptr < Repository > repository ;
2019-12-12 16:41:41 +00:00
std : : unordered_map < String /* path */ , FileInfo > files ;
} ;
2019-04-10 18:36:52 +00:00
2019-12-12 16:41:41 +00:00
/// Reads the repositories.
/// Checks last modification times of files and read those files which are new or changed.
void readRepositories ( const std : : optional < String > & only_repository_name = { } , const std : : optional < String > & only_path = { } )
2017-10-06 10:31:06 +00:00
{
2019-12-30 23:30:06 +00:00
for ( auto & [ repository , repository_info ] : repositories )
2019-06-02 12:11:01 +00:00
{
2019-12-30 23:30:06 +00:00
if ( only_repository_name & & ( repository - > getName ( ) ! = * only_repository_name ) )
continue ;
2019-12-12 16:41:41 +00:00
for ( auto & file_info : repository_info . files | boost : : adaptors : : map_values )
file_info . in_use = false ;
Strings existing_paths ;
if ( only_path )
{
2019-12-30 23:30:06 +00:00
if ( repository - > exists ( * only_path ) )
2019-12-12 16:41:41 +00:00
existing_paths . push_back ( * only_path ) ;
}
else
2019-12-30 23:30:06 +00:00
boost : : copy ( repository - > getAllLoadablesDefinitionNames ( ) , std : : back_inserter ( existing_paths ) ) ;
2019-12-12 16:41:41 +00:00
for ( const auto & path : existing_paths )
2019-06-02 12:11:01 +00:00
{
2019-12-12 16:41:41 +00:00
auto it = repository_info . files . find ( path ) ;
if ( it ! = repository_info . files . end ( ) )
2019-06-02 12:11:01 +00:00
{
2019-12-12 16:41:41 +00:00
FileInfo & file_info = it - > second ;
2019-12-30 23:30:06 +00:00
if ( readFileInfo ( file_info , * repository , path ) )
2019-12-12 16:41:41 +00:00
need_collect_object_configs = true ;
2019-06-02 12:11:01 +00:00
}
else
{
2019-12-12 16:41:41 +00:00
FileInfo file_info ;
2019-12-30 23:30:06 +00:00
if ( readFileInfo ( file_info , * repository , path ) )
2019-06-02 12:11:01 +00:00
{
2019-12-12 16:41:41 +00:00
repository_info . files . emplace ( path , std : : move ( file_info ) ) ;
need_collect_object_configs = true ;
2019-06-02 12:11:01 +00:00
}
}
}
2017-10-06 10:31:06 +00:00
2019-12-12 16:41:41 +00:00
Strings deleted_paths ;
for ( auto & [ path , file_info ] : repository_info . files )
{
if ( file_info . in_use )
continue ;
if ( only_path & & ( * only_path ! = path ) )
continue ;
deleted_paths . emplace_back ( path ) ;
}
if ( ! deleted_paths . empty ( ) )
{
for ( const String & deleted_path : deleted_paths )
repository_info . files . erase ( deleted_path ) ;
need_collect_object_configs = true ;
}
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
}
2019-12-12 16:41:41 +00:00
/// Reads a file, returns true if the file is new or changed.
bool readFileInfo (
FileInfo & file_info ,
2019-09-30 16:12:08 +00:00
IExternalLoaderConfigRepository & repository ,
2019-12-30 23:30:06 +00:00
const String & path ) const
2019-06-02 12:11:01 +00:00
{
try
{
2019-12-12 16:41:41 +00:00
if ( path . empty ( ) | | ! repository . exists ( path ) )
2019-06-02 12:11:01 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_WARNING ( log , " Config file '{}' does not exist " , path ) ;
2019-06-02 12:11:01 +00:00
return false ;
}
2017-10-06 10:31:06 +00:00
2019-12-12 16:41:41 +00:00
auto update_time_from_repository = repository . getUpdateTime ( path ) ;
2019-10-01 08:58:47 +00:00
2020-03-19 02:17:30 +00:00
// We can't count on that the mtime increases or that it has
// a particular relation to system time, so just check for strict
// equality.
// Note that on 1.x versions on Poco, the granularity of update
// time is one second, so the window where we can miss the changes
// is that wide (i.e. when we read the file and after that it
// is updated, but in the same second).
// The solution to this is probably switching to std::filesystem
// -- the work is underway to do so.
if ( update_time_from_repository = = file_info . last_update_time )
2019-06-02 12:11:01 +00:00
{
2019-12-12 16:41:41 +00:00
file_info . in_use = true ;
2019-06-02 12:11:01 +00:00
return false ;
}
2017-10-06 10:31:06 +00:00
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Loading config file '{}'. " , path ) ;
2020-04-12 20:50:32 +00:00
file_info . file_contents = repository . load ( path ) ;
auto & file_contents = * file_info . file_contents ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
/// get all objects' definitions
Poco : : Util : : AbstractConfiguration : : Keys keys ;
2020-04-12 20:50:32 +00:00
file_contents . keys ( keys ) ;
2017-10-06 10:31:06 +00:00
2019-10-17 17:18:54 +00:00
/// for each object defined in repositories
2020-04-12 20:50:32 +00:00
std : : unordered_map < String , String > objects ;
2019-06-02 12:11:01 +00:00
for ( const auto & key : keys )
{
if ( ! startsWith ( key , settings . external_config ) )
{
if ( ! startsWith ( key , " comment " ) & & ! startsWith ( key , " include_from " ) )
2020-05-23 22:24:01 +00:00
LOG_WARNING ( log , " {}: file contains unknown node '{}', expected '{}' " , path , key , settings . external_config ) ;
2019-06-02 12:11:01 +00:00
continue ;
}
2020-07-15 19:25:31 +00:00
/// Use uuid as name if possible
String object_uuid = file_contents . getString ( key + " . " + settings . external_uuid , " " ) ;
String object_name ;
if ( object_uuid . empty ( ) )
object_name = file_contents . getString ( key + " . " + settings . external_name ) ;
else
object_name = object_uuid ;
2019-12-12 16:41:41 +00:00
if ( object_name . empty ( ) )
2019-06-02 12:11:01 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_WARNING ( log , " {}: node '{}' defines {} with an empty name. It's not allowed " , path , key , type_name ) ;
2019-06-02 12:11:01 +00:00
continue ;
}
2020-07-15 19:25:31 +00:00
if ( object_uuid . empty ( ) )
{
String database ;
if ( ! settings . external_database . empty ( ) )
database = file_contents . getString ( key + " . " + settings . external_database , " " ) ;
if ( ! database . empty ( ) )
object_name = database + " . " + object_name ;
}
2019-12-25 23:12:12 +00:00
2020-04-12 20:50:32 +00:00
objects . emplace ( object_name , key ) ;
2019-06-02 12:11:01 +00:00
}
2020-04-12 20:50:32 +00:00
file_info . objects = std : : move ( objects ) ;
2019-12-12 16:41:41 +00:00
file_info . last_update_time = update_time_from_repository ;
file_info . in_use = true ;
2019-06-02 12:11:01 +00:00
return true ;
}
catch ( . . . )
{
2019-12-12 16:41:41 +00:00
tryLogCurrentException ( log , " Failed to load config file ' " + path + " ' " ) ;
2019-06-02 12:11:01 +00:00
return false ;
}
}
2019-12-12 16:41:41 +00:00
/// Builds a map of current configurations of objects.
void collectObjectConfigs ( )
{
if ( ! need_collect_object_configs )
return ;
need_collect_object_configs = false ;
// Generate new result.
2021-10-13 06:56:34 +00:00
auto new_configs = std : : make_shared < ObjectConfigs > ( ) ;
2019-12-12 16:41:41 +00:00
2019-12-30 23:30:06 +00:00
for ( const auto & [ repository , repository_info ] : repositories )
2019-12-12 16:41:41 +00:00
{
for ( const auto & [ path , file_info ] : repository_info . files )
{
2020-04-12 20:50:32 +00:00
for ( const auto & [ object_name , key_in_config ] : file_info . objects )
2019-12-12 16:41:41 +00:00
{
2021-10-13 06:56:34 +00:00
auto already_added_it = new_configs - > configs_by_name . find ( object_name ) ;
if ( already_added_it = = new_configs - > configs_by_name . end ( ) )
2019-12-12 16:41:41 +00:00
{
2020-04-12 20:50:32 +00:00
auto new_config = std : : make_shared < ObjectConfig > ( ) ;
new_config - > config = file_info . file_contents ;
new_config - > key_in_config = key_in_config ;
new_config - > repository_name = repository - > getName ( ) ;
new_config - > from_temp_repository = repository - > isTemporary ( ) ;
new_config - > path = path ;
2021-10-13 06:56:34 +00:00
new_configs - > configs_by_name . emplace ( object_name , std : : move ( new_config ) ) ;
2019-12-12 16:41:41 +00:00
}
else
{
const auto & already_added = already_added_it - > second ;
2020-04-12 20:50:32 +00:00
if ( ! already_added - > from_temp_repository & & ! repository - > isTemporary ( ) )
2019-12-12 16:41:41 +00:00
{
2020-05-23 21:50:34 +00:00
if ( path = = already_added - > path & & repository - > getName ( ) = = already_added - > repository_name )
2020-05-23 22:24:01 +00:00
LOG_WARNING ( log , " {} '{}' is found twice in the same file '{}' " ,
2020-05-23 21:50:34 +00:00
type_name , object_name , path ) ;
else
2020-05-23 22:24:01 +00:00
LOG_WARNING ( log , " {} '{}' is found both in file '{}' and '{}' " ,
2020-05-23 21:50:34 +00:00
type_name , object_name , already_added - > path , path ) ;
2019-12-12 16:41:41 +00:00
}
}
}
}
}
2021-10-13 06:56:34 +00:00
new_configs - > counter = counter + + ;
2019-12-12 16:41:41 +00:00
object_configs = new_configs ;
}
2019-10-18 15:44:32 +00:00
2019-06-02 12:11:01 +00:00
const String type_name ;
2020-05-30 21:57:37 +00:00
Poco : : Logger * log ;
2019-04-10 18:36:52 +00:00
2019-06-02 12:11:01 +00:00
std : : mutex mutex ;
2019-12-30 23:30:06 +00:00
ExternalLoaderConfigSettings settings ;
std : : unordered_map < Repository * , RepositoryInfo > repositories ;
2019-12-12 16:41:41 +00:00
ObjectConfigsPtr object_configs ;
bool need_collect_object_configs = false ;
2021-10-13 06:56:34 +00:00
size_t counter = 0 ;
2019-06-02 12:11:01 +00:00
} ;
2017-10-06 10:31:06 +00:00
2019-09-30 16:12:08 +00:00
/** Manages loading and reloading objects. Uses configurations from the class LoadablesConfigReader.
2019-06-02 12:11:01 +00:00
* Supports parallel loading .
*/
class ExternalLoader : : LoadingDispatcher : private boost : : noncopyable
{
public :
/// Called to load or reload an object.
2019-07-17 08:39:36 +00:00
using CreateObjectFunction = std : : function < LoadablePtr (
2019-12-04 23:53:06 +00:00
const String & /* name */ , const ObjectConfig & /* config */ , const LoadablePtr & /* previous_version */ ) > ;
2019-06-02 12:11:01 +00:00
LoadingDispatcher (
const CreateObjectFunction & create_object_function_ ,
const String & type_name_ ,
2020-05-30 21:57:37 +00:00
Poco : : Logger * log_ )
2019-06-02 12:11:01 +00:00
: create_object ( create_object_function_ )
, type_name ( type_name_ )
, log ( log_ )
2017-10-06 10:31:06 +00:00
{
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
~ LoadingDispatcher ( )
{
std : : unique_lock lock { mutex } ;
infos . clear ( ) ; /// We clear this map to tell the threads that we don't want any load results anymore.
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
/// Wait for all the threads to finish.
2019-12-15 13:22:04 +00:00
while ( ! loading_threads . empty ( ) )
2017-10-06 10:31:06 +00:00
{
2019-12-15 13:22:04 +00:00
auto it = loading_threads . begin ( ) ;
2019-06-02 12:11:01 +00:00
auto thread = std : : move ( it - > second ) ;
2019-12-15 13:22:04 +00:00
loading_threads . erase ( it ) ;
2019-06-02 12:11:01 +00:00
lock . unlock ( ) ;
event . notify_all ( ) ;
thread . join ( ) ;
lock . lock ( ) ;
}
}
2019-09-30 16:12:08 +00:00
using ObjectConfigsPtr = LoadablesConfigReader : : ObjectConfigsPtr ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
/// Sets new configurations for all the objects.
2019-09-30 11:18:01 +00:00
void setConfiguration ( const ObjectConfigsPtr & new_configs )
2019-06-02 12:11:01 +00:00
{
std : : lock_guard lock { mutex } ;
if ( configs = = new_configs )
return ;
2017-10-06 10:31:06 +00:00
2021-10-13 06:56:34 +00:00
/// The following check prevents a race when two threads are trying to update configuration
/// at almost the same time:
/// 1) first thread reads a configuration (for example as a part of periodic updates)
/// 2) second thread sets a new configuration (for example after executing CREATE DICTIONARY)
/// 3) first thread sets the configuration it read in 1) and thus discards the changes made in 2).
/// So we use `counter` here to ensure we exchange the current configuration only for a newer one.
if ( configs & & ( configs - > counter > = new_configs - > counter ) )
return ;
2019-06-02 12:11:01 +00:00
configs = new_configs ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
std : : vector < String > removed_names ;
for ( auto & [ name , info ] : infos )
{
2021-10-13 06:56:34 +00:00
auto new_config_it = new_configs - > configs_by_name . find ( name ) ;
if ( new_config_it = = new_configs - > configs_by_name . end ( ) )
2020-03-19 02:17:30 +00:00
{
2019-06-02 12:11:01 +00:00
removed_names . emplace_back ( name ) ;
2020-03-19 02:17:30 +00:00
}
2017-10-06 10:31:06 +00:00
else
{
2019-06-02 12:11:01 +00:00
const auto & new_config = new_config_it - > second ;
2020-04-12 20:50:32 +00:00
bool config_is_same = isSameConfiguration ( * info . config - > config , info . config - > key_in_config , * new_config - > config , new_config - > key_in_config ) ;
info . config = new_config ;
2019-12-12 16:41:41 +00:00
if ( ! config_is_same )
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
if ( info . triedToLoad ( ) )
2019-06-02 12:11:01 +00:00
{
/// The object has been tried to load before, so it is currently in use or was in use
/// and we should try to reload it with the new config.
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Will reload '{}' because its configuration has been changed and there were attempts to load it before " , name ) ;
2019-12-15 13:22:04 +00:00
startLoading ( info , true ) ;
2019-06-02 12:11:01 +00:00
}
}
}
}
/// Insert to the map those objects which added to the new configuration.
2021-10-13 06:56:34 +00:00
for ( const auto & [ name , config ] : new_configs - > configs_by_name )
2019-06-02 12:11:01 +00:00
{
if ( infos . find ( name ) = = infos . end ( ) )
{
2019-12-12 18:33:43 +00:00
Info & info = infos . emplace ( name , Info { name , config } ) . first - > second ;
2019-06-02 12:11:01 +00:00
if ( always_load_everything )
2020-03-19 02:17:30 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Will load '{}' because always_load_everything flag is set. " , name ) ;
2019-12-15 13:22:04 +00:00
startLoading ( info ) ;
2020-03-19 02:17:30 +00:00
}
2019-06-02 12:11:01 +00:00
}
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
/// Remove from the map those objects which were removed from the configuration.
for ( const String & name : removed_names )
2020-04-07 18:23:25 +00:00
{
if ( auto it = infos . find ( name ) ; it ! = infos . end ( ) )
{
const auto & info = it - > second ;
if ( info . loaded ( ) | | info . isLoading ( ) )
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Unloading '{}' because its configuration has been removed or detached " , name ) ;
2020-04-07 18:23:25 +00:00
infos . erase ( it ) ;
}
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
/// Maybe we have just added new objects which require to be loaded
/// or maybe we have just removed object which were been loaded,
/// so we should notify `event` to recheck conditions in load() and loadAll() now.
event . notify_all ( ) ;
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
/// Sets whether all the objects from the configuration should be always loaded (even if they aren't used).
void enableAlwaysLoadEverything ( bool enable )
{
std : : lock_guard lock { mutex } ;
if ( always_load_everything = = enable )
return ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
always_load_everything = enable ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
if ( enable )
2017-10-06 10:31:06 +00:00
{
2019-06-02 12:11:01 +00:00
/// Start loading all the objects which were not loaded yet.
for ( auto & [ name , info ] : infos )
2019-12-12 18:33:43 +00:00
if ( ! info . triedToLoad ( ) )
2019-12-15 13:22:04 +00:00
startLoading ( info ) ;
2019-06-02 12:11:01 +00:00
}
}
/// Sets whether the objects should be loaded asynchronously, each loading in a new thread (from the thread pool).
void enableAsyncLoading ( bool enable )
{
enable_async_loading = enable ;
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
/// Returns the status of the object.
/// If the object has not been loaded yet then the function returns Status::NOT_LOADED.
/// If the specified name isn't found in the configuration then the function returns Status::NOT_EXIST.
Status getCurrentStatus ( const String & name ) const
{
std : : lock_guard lock { mutex } ;
const Info * info = getInfo ( name ) ;
if ( ! info )
return Status : : NOT_EXIST ;
return info - > status ( ) ;
}
2018-12-30 01:09:06 +00:00
2019-06-02 12:11:01 +00:00
/// Returns the load result of the object.
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
2020-04-12 20:50:32 +00:00
ReturnType getLoadResult ( const String & name ) const
2017-10-06 10:31:06 +00:00
{
2019-06-02 12:11:01 +00:00
std : : lock_guard lock { mutex } ;
const Info * info = getInfo ( name ) ;
if ( ! info )
2019-12-12 18:33:43 +00:00
return notExists < ReturnType > ( name ) ;
return info - > getLoadResult < ReturnType > ( ) ;
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
/// Returns all the load results as a map.
/// The function doesn't load anything, it just returns the current load results as is.
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
2020-04-12 20:50:32 +00:00
ReturnType getLoadResults ( const FilterByNameFunction & filter ) const
2019-06-02 12:11:01 +00:00
{
std : : lock_guard lock { mutex } ;
2019-12-12 18:33:43 +00:00
return collectLoadResults < ReturnType > ( filter ) ;
2019-06-02 12:11:01 +00:00
}
2020-04-12 20:50:32 +00:00
size_t getNumberOfLoadedObjects ( ) const
2019-06-02 12:11:01 +00:00
{
std : : lock_guard lock { mutex } ;
size_t count = 0 ;
for ( const auto & name_and_info : infos )
2017-10-06 10:31:06 +00:00
{
2019-06-02 12:11:01 +00:00
const auto & info = name_and_info . second ;
if ( info . loaded ( ) )
+ + count ;
}
return count ;
}
2019-03-04 18:19:48 +00:00
2020-04-12 20:50:32 +00:00
bool hasLoadedObjects ( ) const
2019-06-29 17:27:32 +00:00
{
std : : lock_guard lock { mutex } ;
2020-04-22 06:01:33 +00:00
for ( const auto & name_info : infos )
2019-07-02 17:24:22 +00:00
if ( name_info . second . loaded ( ) )
2019-06-29 17:27:32 +00:00
return true ;
return false ;
}
2017-10-06 10:31:06 +00:00
2019-12-12 18:33:43 +00:00
Strings getAllTriedToLoadNames ( ) const
2019-06-02 12:11:01 +00:00
{
2020-01-23 13:35:12 +00:00
std : : lock_guard lock { mutex } ;
2019-12-12 18:33:43 +00:00
Strings names ;
2020-04-22 06:01:33 +00:00
for ( const auto & [ name , info ] : infos )
2019-12-12 18:33:43 +00:00
if ( info . triedToLoad ( ) )
names . push_back ( name ) ;
return names ;
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2020-04-12 20:50:32 +00:00
size_t getNumberOfObjects ( ) const
{
std : : lock_guard lock { mutex } ;
return infos . size ( ) ;
}
2019-12-12 18:33:43 +00:00
/// Tries to load a specified object during the timeout.
template < typename ReturnType >
ReturnType tryLoad ( const String & name , Duration timeout )
2019-06-02 12:11:01 +00:00
{
std : : unique_lock lock { mutex } ;
2019-12-15 13:22:04 +00:00
Info * info = loadImpl ( name , timeout , false , lock ) ;
2019-06-02 12:11:01 +00:00
if ( ! info )
2019-12-12 18:33:43 +00:00
return notExists < ReturnType > ( name ) ;
return info - > getLoadResult < ReturnType > ( ) ;
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
ReturnType tryLoad ( const FilterByNameFunction & filter , Duration timeout )
2019-06-02 12:11:01 +00:00
{
std : : unique_lock lock { mutex } ;
2019-12-15 13:22:04 +00:00
loadImpl ( filter , timeout , false , lock ) ;
2019-12-12 18:33:43 +00:00
return collectLoadResults < ReturnType > ( filter ) ;
2019-06-02 12:11:01 +00:00
}
2019-12-12 18:33:43 +00:00
/// Tries to load or reload a specified object.
template < typename ReturnType >
ReturnType tryLoadOrReload ( const String & name , Duration timeout )
2019-10-17 13:05:12 +00:00
{
std : : unique_lock lock { mutex } ;
2019-12-15 13:22:04 +00:00
Info * info = loadImpl ( name , timeout , true , lock ) ;
2019-06-02 12:11:01 +00:00
if ( ! info )
2019-12-12 18:33:43 +00:00
return notExists < ReturnType > ( name ) ;
return info - > getLoadResult < ReturnType > ( ) ;
2019-06-02 12:11:01 +00:00
}
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
ReturnType tryLoadOrReload ( const FilterByNameFunction & filter , Duration timeout )
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
std : : unique_lock lock { mutex } ;
2019-12-15 13:22:04 +00:00
loadImpl ( filter , timeout , true , lock ) ;
2019-12-12 18:33:43 +00:00
return collectLoadResults < ReturnType > ( filter ) ;
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2021-03-18 14:03:22 +00:00
bool has ( const String & name ) const
{
std : : lock_guard lock { mutex } ;
return infos . contains ( name ) ;
}
2019-06-02 12:11:01 +00:00
/// Starts reloading all the object which update time is earlier than now.
/// The function doesn't touch the objects which were never tried to load.
void reloadOutdated ( )
{
2019-12-04 15:23:05 +00:00
/// Iterate through all the objects and find loaded ones which should be checked if they need update.
std : : unordered_map < LoadablePtr , bool > should_update_map ;
2019-07-16 17:47:18 +00:00
{
std : : lock_guard lock { mutex } ;
TimePoint now = std : : chrono : : system_clock : : now ( ) ;
for ( const auto & name_and_info : infos )
2019-06-02 12:11:01 +00:00
{
2019-07-16 17:47:18 +00:00
const auto & info = name_and_info . second ;
2020-03-23 02:12:31 +00:00
if ( ( now > = info . next_update_time ) & & ! info . isLoading ( ) & & info . loaded ( ) )
2019-12-04 16:20:24 +00:00
should_update_map . emplace ( info . object , info . failedToReload ( ) ) ;
2019-07-16 17:47:18 +00:00
}
}
2019-07-19 21:01:20 +00:00
/// Find out which of the loaded objects were modified.
2019-12-04 15:23:05 +00:00
/// We couldn't perform these checks while we were building `should_update_map` because
2019-09-27 12:36:54 +00:00
/// the `mutex` should be unlocked while we're calling the function object->isModified()
2019-12-04 15:23:05 +00:00
for ( auto & [ object , should_update_flag ] : should_update_map )
2019-07-16 17:47:18 +00:00
{
try
{
2020-01-11 09:50:41 +00:00
/// Maybe already true, if we have an exception
2019-12-04 15:23:05 +00:00
if ( ! should_update_flag )
should_update_flag = object - > isModified ( ) ;
2019-03-04 18:19:48 +00:00
}
2019-07-16 17:47:18 +00:00
catch ( . . . )
{
2019-12-25 23:12:12 +00:00
tryLogCurrentException ( log , " Could not check if " + type_name + " ' " + object - > getLoadableName ( ) + " ' was modified " ) ;
2019-12-04 15:23:05 +00:00
/// Cannot check isModified, so update
should_update_flag = true ;
2019-07-16 17:47:18 +00:00
}
}
2019-07-19 21:01:20 +00:00
/// Iterate through all the objects again and either start loading or just set `next_update_time`.
2019-07-16 17:47:18 +00:00
{
std : : lock_guard lock { mutex } ;
TimePoint now = std : : chrono : : system_clock : : now ( ) ;
for ( auto & [ name , info ] : infos )
2019-07-19 19:37:48 +00:00
{
2020-03-23 02:12:31 +00:00
if ( ( now > = info . next_update_time ) & & ! info . isLoading ( ) )
2019-07-16 17:47:18 +00:00
{
2019-07-19 21:03:25 +00:00
if ( info . loaded ( ) )
{
2019-12-04 15:23:05 +00:00
auto it = should_update_map . find ( info . object ) ;
if ( it = = should_update_map . end ( ) )
continue ; /// Object has been just loaded (it wasn't loaded while we were building the map `should_update_map`), so we don't have to reload it right now.
bool should_update_flag = it - > second ;
if ( ! should_update_flag )
2019-07-19 21:03:25 +00:00
{
2019-09-27 12:36:54 +00:00
info . next_update_time = calculateNextUpdateTime ( info . object , info . error_count ) ;
2021-06-15 19:55:21 +00:00
LOG_TRACE ( log , " Object '{}' not modified, will not reload. Next update at {} " , info . name , to_string ( info . next_update_time ) ) ;
2019-07-19 21:03:25 +00:00
continue ;
}
2019-07-19 21:01:20 +00:00
2019-12-04 16:20:24 +00:00
/// Object was modified or it was failed to reload last time, so it should be reloaded.
2019-12-15 13:22:04 +00:00
startLoading ( info ) ;
2019-07-19 21:01:20 +00:00
}
else if ( info . failed ( ) )
{
/// Object was never loaded successfully and should be reloaded.
2019-12-15 13:22:04 +00:00
startLoading ( info ) ;
2019-07-19 21:03:25 +00:00
}
2022-10-24 09:58:37 +00:00
else
{
LOG_TRACE ( log , " Object '{}' is neither loaded nor failed, so it will not be reloaded as outdated. " , info . name ) ;
}
2019-07-16 17:47:18 +00:00
}
2019-07-19 19:37:48 +00:00
}
2019-07-16 17:47:18 +00:00
}
2018-12-30 01:09:06 +00:00
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
private :
struct Info
2019-03-05 16:01:13 +00:00
{
2020-04-12 20:50:32 +00:00
Info ( const String & name_ , const std : : shared_ptr < const ObjectConfig > & config_ ) : name ( name_ ) , config ( config_ ) { }
2019-06-02 12:11:01 +00:00
bool loaded ( ) const { return object ! = nullptr ; }
bool failed ( ) const { return ! object & & exception ; }
2019-12-15 13:22:04 +00:00
bool loadedOrFailed ( ) const { return loaded ( ) | | failed ( ) ; }
2020-03-23 02:12:31 +00:00
bool triedToLoad ( ) const { return loaded ( ) | | failed ( ) | | isLoading ( ) ; }
2019-12-04 16:20:24 +00:00
bool failedToReload ( ) const { return loaded ( ) & & exception ! = nullptr ; }
2020-03-23 02:12:31 +00:00
bool isLoading ( ) const { return loading_id > state_id ; }
2019-06-02 12:11:01 +00:00
Status status ( ) const
{
if ( object )
2020-03-23 02:12:31 +00:00
return isLoading ( ) ? Status : : LOADED_AND_RELOADING : Status : : LOADED ;
2019-06-02 12:11:01 +00:00
else if ( exception )
2020-03-23 02:12:31 +00:00
return isLoading ( ) ? Status : : FAILED_AND_RELOADING : Status : : FAILED ;
2019-06-02 12:11:01 +00:00
else
2020-03-23 02:12:31 +00:00
return isLoading ( ) ? Status : : LOADING : Status : : NOT_LOADED ;
2019-06-02 12:11:01 +00:00
}
2019-09-27 12:36:54 +00:00
Duration loadingDuration ( ) const
2019-06-02 12:11:01 +00:00
{
2020-03-23 02:12:31 +00:00
if ( isLoading ( ) )
2019-06-02 12:11:01 +00:00
return std : : chrono : : duration_cast < Duration > ( std : : chrono : : system_clock : : now ( ) - loading_start_time ) ;
return std : : chrono : : duration_cast < Duration > ( loading_end_time - loading_start_time ) ;
}
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
ReturnType getLoadResult ( ) const
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
if constexpr ( std : : is_same_v < ReturnType , LoadResult > )
{
LoadResult result ;
result . name = name ;
result . status = status ( ) ;
result . object = object ;
result . exception = exception ;
result . loading_start_time = loading_start_time ;
2020-02-27 10:07:50 +00:00
result . last_successful_update_time = last_successful_update_time ;
2019-12-12 18:33:43 +00:00
result . loading_duration = loadingDuration ( ) ;
2020-04-12 20:50:32 +00:00
result . config = config ;
2019-12-12 18:33:43 +00:00
return result ;
}
else
{
static_assert ( std : : is_same_v < ReturnType , ExternalLoader : : LoadablePtr > ) ;
return object ;
}
2019-06-02 12:11:01 +00:00
}
2019-12-12 18:33:43 +00:00
String name ;
2019-06-02 12:11:01 +00:00
LoadablePtr object ;
2020-04-12 20:50:32 +00:00
std : : shared_ptr < const ObjectConfig > config ;
2019-06-02 12:11:01 +00:00
TimePoint loading_start_time ;
TimePoint loading_end_time ;
2020-02-27 10:07:50 +00:00
TimePoint last_successful_update_time ;
2019-12-15 13:22:04 +00:00
size_t state_id = 0 ; /// Index of the current state of this `info`, this index is incremented every loading.
size_t loading_id = 0 ; /// The value which will be stored in `state_id` after finishing the current loading.
2019-06-02 12:11:01 +00:00
size_t error_count = 0 ; /// Numbers of errors since last successful loading.
std : : exception_ptr exception ; /// Last error occurred.
TimePoint next_update_time = TimePoint : : max ( ) ; /// Time of the next update, `TimePoint::max()` means "never".
2019-03-05 16:01:13 +00:00
} ;
2019-06-02 12:11:01 +00:00
Info * getInfo ( const String & name )
2018-12-30 01:09:06 +00:00
{
2019-06-02 12:11:01 +00:00
auto it = infos . find ( name ) ;
if ( it = = infos . end ( ) )
return nullptr ;
return & it - > second ;
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
const Info * getInfo ( const String & name ) const
{
auto it = infos . find ( name ) ;
if ( it = = infos . end ( ) )
return nullptr ;
return & it - > second ;
}
2019-12-12 18:33:43 +00:00
template < typename ReturnType >
ReturnType collectLoadResults ( const FilterByNameFunction & filter ) const
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
ReturnType results ;
results . reserve ( infos . size ( ) ) ;
2019-06-02 12:11:01 +00:00
for ( const auto & [ name , info ] : infos )
2019-10-17 13:05:12 +00:00
{
2020-04-12 20:50:32 +00:00
if ( ! filter | | filter ( name ) )
2019-12-12 18:33:43 +00:00
{
auto result = info . template getLoadResult < typename ReturnType : : value_type > ( ) ;
if constexpr ( std : : is_same_v < typename ReturnType : : value_type , LoadablePtr > )
{
if ( ! result )
continue ;
}
results . emplace_back ( std : : move ( result ) ) ;
}
2019-10-17 13:05:12 +00:00
}
2019-12-12 18:33:43 +00:00
return results ;
2019-06-02 12:11:01 +00:00
}
2019-12-15 13:22:04 +00:00
Info * loadImpl ( const String & name , Duration timeout , bool forced_to_reload , std : : unique_lock < std : : mutex > & lock )
2019-06-02 12:11:01 +00:00
{
2019-12-15 13:22:04 +00:00
std : : optional < size_t > min_id ;
Info * info = nullptr ;
auto pred = [ & ]
2018-12-30 01:09:06 +00:00
{
2019-06-02 12:11:01 +00:00
info = getInfo ( name ) ;
2019-12-15 13:22:04 +00:00
if ( ! info )
return true ; /// stop
if ( ! min_id )
min_id = getMinIDToFinishLoading ( forced_to_reload ) ;
if ( info - > loading_id < min_id )
startLoading ( * info , forced_to_reload , * min_id ) ;
2021-03-17 18:57:40 +00:00
2021-03-23 12:15:44 +00:00
/// Wait for the next event if loading wasn't completed, or stop otherwise.
2021-03-17 18:57:40 +00:00
return ( info - > state_id > = min_id ) ;
2019-06-02 12:11:01 +00:00
} ;
2019-12-12 18:33:43 +00:00
if ( timeout = = WAIT )
2019-06-02 12:11:01 +00:00
event . wait ( lock , pred ) ;
else
event . wait_for ( lock , timeout , pred ) ;
return info ;
}
2019-12-15 13:22:04 +00:00
void loadImpl ( const FilterByNameFunction & filter , Duration timeout , bool forced_to_reload , std : : unique_lock < std : : mutex > & lock )
2019-06-02 12:11:01 +00:00
{
2019-12-15 13:22:04 +00:00
std : : optional < size_t > min_id ;
auto pred = [ & ]
2019-06-02 12:11:01 +00:00
{
2019-12-15 13:22:04 +00:00
if ( ! min_id )
min_id = getMinIDToFinishLoading ( forced_to_reload ) ;
2019-06-02 12:11:01 +00:00
bool all_ready = true ;
for ( auto & [ name , info ] : infos )
2019-03-05 16:01:13 +00:00
{
2020-04-12 20:50:32 +00:00
if ( filter & & ! filter ( name ) )
2019-06-02 12:11:01 +00:00
continue ;
2019-12-15 13:22:04 +00:00
if ( info . loading_id < min_id )
startLoading ( info , forced_to_reload , * min_id ) ;
2021-03-17 18:57:40 +00:00
all_ready & = ( info . state_id > = min_id ) ;
2019-03-05 16:01:13 +00:00
}
2019-06-02 12:11:01 +00:00
return all_ready ;
} ;
2019-12-12 18:33:43 +00:00
if ( timeout = = WAIT )
2019-06-02 12:11:01 +00:00
event . wait ( lock , pred ) ;
else
event . wait_for ( lock , timeout , pred ) ;
}
2019-12-15 13:22:04 +00:00
/// When state_id >= getMinIDToFinishLoading() the loading is considered as finished.
size_t getMinIDToFinishLoading ( bool forced_to_reload ) const
2019-06-02 12:11:01 +00:00
{
2019-12-15 13:22:04 +00:00
if ( forced_to_reload )
{
/// We need to force reloading, that's why we return next_id_counter here
/// (because info.state_id < next_id_counter for any info).
return next_id_counter ;
}
/// The loading of an object can cause the loading of another object.
/// We use the same "min_id" in this case to allows reloading multiple objects at once
/// taking into account their dependencies.
auto it = min_id_to_finish_loading_dependencies . find ( std : : this_thread : : get_id ( ) ) ;
if ( it ! = min_id_to_finish_loading_dependencies . end ( ) )
return it - > second ;
/// We just need the first loading to be finished, that's why we return 1 here
/// (because info.state_id >= 1 since the first loading is finished, successfully or not).
return 1 ;
}
void startLoading ( Info & info , bool forced_to_reload = false , size_t min_id_to_finish_loading_dependencies_ = 1 )
{
2020-03-23 02:12:31 +00:00
if ( info . isLoading ( ) )
2019-12-15 13:22:04 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " The object '{}' is already being loaded, force = {}. " , info . name , forced_to_reload ) ;
2020-03-19 02:17:30 +00:00
2019-12-15 13:22:04 +00:00
if ( ! forced_to_reload )
2020-03-19 02:17:30 +00:00
{
2019-12-15 13:22:04 +00:00
return ;
2020-03-19 02:17:30 +00:00
}
2019-12-15 13:22:04 +00:00
cancelLoading ( info ) ;
}
2019-06-02 12:11:01 +00:00
2020-09-07 23:09:03 +00:00
putBackFinishedThreadsToPool ( ) ;
2019-06-02 12:11:01 +00:00
/// All loadings have unique loading IDs.
2019-12-15 13:22:04 +00:00
size_t loading_id = next_id_counter + + ;
2019-06-02 12:11:01 +00:00
info . loading_id = loading_id ;
info . loading_start_time = std : : chrono : : system_clock : : now ( ) ;
info . loading_end_time = TimePoint { } ;
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Will load the object '{}' {}, force = {}, loading_id = {} " , info . name , ( enable_async_loading ? std : : string ( " in background " ) : " immediately " ) , forced_to_reload , info . loading_id ) ;
2020-03-19 02:17:30 +00:00
2019-10-21 13:54:23 +00:00
if ( enable_async_loading )
2019-06-02 12:11:01 +00:00
{
/// Put a job to the thread pool for the loading.
2021-05-09 11:28:31 +00:00
auto thread = ThreadFromGlobalPool { & LoadingDispatcher : : doLoading , this , info . name , loading_id , forced_to_reload , min_id_to_finish_loading_dependencies_ , true , CurrentThread : : getGroup ( ) } ;
2019-12-15 13:22:04 +00:00
loading_threads . try_emplace ( loading_id , std : : move ( thread ) ) ;
2017-10-06 10:31:06 +00:00
}
2019-06-02 12:11:01 +00:00
else
2017-10-06 10:31:06 +00:00
{
2019-06-02 12:11:01 +00:00
/// Perform the loading immediately.
2019-12-15 13:22:04 +00:00
doLoading ( info . name , loading_id , forced_to_reload , min_id_to_finish_loading_dependencies_ , false ) ;
2018-12-30 01:09:06 +00:00
}
2019-06-02 12:11:01 +00:00
}
2018-12-30 01:09:06 +00:00
2020-09-07 23:09:03 +00:00
void putBackFinishedThreadsToPool ( )
{
for ( auto loading_id : recently_finished_loadings )
{
auto it = loading_threads . find ( loading_id ) ;
if ( it ! = loading_threads . end ( ) )
{
auto thread = std : : move ( it - > second ) ;
loading_threads . erase ( it ) ;
thread . join ( ) ; /// It's very likely that `thread` has already finished.
}
}
recently_finished_loadings . clear ( ) ;
}
2020-03-18 00:57:00 +00:00
static void cancelLoading ( Info & info )
2019-12-15 13:22:04 +00:00
{
2020-03-23 02:12:31 +00:00
if ( ! info . isLoading ( ) )
2019-12-15 13:22:04 +00:00
return ;
2017-10-06 10:31:06 +00:00
2019-12-15 13:22:04 +00:00
/// In fact we cannot actually CANCEL the loading (because it's possibly already being performed in another thread).
/// But we can reset the `loading_id` and doLoading() will understand it as a signal to stop loading.
info . loading_id = info . state_id ;
info . loading_end_time = std : : chrono : : system_clock : : now ( ) ;
}
/// Does the loading, possibly in the separate thread.
2021-05-09 11:28:31 +00:00
void doLoading ( const String & name , size_t loading_id , bool forced_to_reload , size_t min_id_to_finish_loading_dependencies_ , bool async , ThreadGroupStatusPtr thread_group = { } )
2019-10-22 12:57:58 +00:00
{
2022-01-27 15:51:36 +00:00
SCOPE_EXIT_SAFE (
if ( thread_group )
CurrentThread : : detachQueryIfNotDetached ( ) ;
) ;
2021-05-09 11:28:31 +00:00
if ( thread_group )
CurrentThread : : attachTo ( thread_group ) ;
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Start loading object '{}' " , name ) ;
2017-10-06 10:31:06 +00:00
try
{
2019-12-15 13:22:04 +00:00
/// Prepare for loading.
std : : optional < Info > info ;
{
LoadingGuardForAsyncLoad lock ( async , mutex ) ;
info = prepareToLoadSingleObject ( name , loading_id , min_id_to_finish_loading_dependencies_ , lock ) ;
if ( ! info )
2020-03-19 02:17:30 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Could not lock object '{}' for loading " , name ) ;
2019-12-15 13:22:04 +00:00
return ;
2020-03-19 02:17:30 +00:00
}
2019-12-15 13:22:04 +00:00
}
/// Previous version can be used as the base for new loading, enabling loading only part of data.
auto previous_version_as_base_for_loading = info - > object ;
if ( forced_to_reload )
previous_version_as_base_for_loading = nullptr ; /// Need complete reloading, cannot use the previous version.
/// Loading.
2020-04-12 20:50:32 +00:00
auto [ new_object , new_exception ] = loadSingleObject ( name , * info - > config , previous_version_as_base_for_loading ) ;
2019-12-15 13:22:04 +00:00
if ( ! new_object & & ! new_exception )
2023-01-23 21:13:58 +00:00
throw Exception ( ErrorCodes : : LOGICAL_ERROR , " No object created and no exception raised for {} " , type_name ) ;
2019-12-15 13:22:04 +00:00
/// Saving the result of the loading.
{
LoadingGuardForAsyncLoad lock ( async , mutex ) ;
saveResultOfLoadingSingleObject ( name , loading_id , info - > object , new_object , new_exception , info - > error_count , lock ) ;
finishLoadingSingleObject ( name , loading_id , lock ) ;
}
event . notify_all ( ) ;
2017-10-06 10:31:06 +00:00
}
catch ( . . . )
{
2019-12-15 13:22:04 +00:00
LoadingGuardForAsyncLoad lock ( async , mutex ) ;
finishLoadingSingleObject ( name , loading_id , lock ) ;
throw ;
2019-06-02 12:11:01 +00:00
}
2019-10-22 12:57:58 +00:00
}
2017-10-06 10:31:06 +00:00
2019-12-15 13:22:04 +00:00
/// Returns single object info, checks loading_id and name.
std : : optional < Info > prepareToLoadSingleObject (
const String & name , size_t loading_id , size_t min_id_to_finish_loading_dependencies_ , const LoadingGuardForAsyncLoad & )
2019-10-22 12:57:58 +00:00
{
Info * info = getInfo ( name ) ;
2019-12-15 13:22:04 +00:00
/// We check here if this is exactly the same loading as we planned to perform.
/// This check is necessary because the object could be removed or load with another config before this thread even starts.
2020-03-23 02:12:31 +00:00
if ( ! info | | ! info - > isLoading ( ) | | ( info - > loading_id ! = loading_id ) )
2019-10-22 12:57:58 +00:00
return { } ;
2019-10-17 17:36:53 +00:00
2019-12-15 13:22:04 +00:00
min_id_to_finish_loading_dependencies [ std : : this_thread : : get_id ( ) ] = min_id_to_finish_loading_dependencies_ ;
2019-10-22 12:57:58 +00:00
return * info ;
}
2019-12-15 13:22:04 +00:00
/// Load one object, returns object ptr or exception.
std : : pair < LoadablePtr , std : : exception_ptr >
loadSingleObject ( const String & name , const ObjectConfig & config , LoadablePtr previous_version )
2019-10-22 13:41:17 +00:00
{
2019-12-15 13:22:04 +00:00
/// Use `create_function` to perform the actual loading.
/// It's much better to do it with `mutex` unlocked because the loading can take a lot of time
/// and require access to other objects.
LoadablePtr new_object ;
std : : exception_ptr new_exception ;
try
2019-10-22 13:41:17 +00:00
{
2019-12-15 13:22:04 +00:00
new_object = create_object ( name , config , previous_version ) ;
2019-10-22 13:41:17 +00:00
}
2019-12-15 13:22:04 +00:00
catch ( . . . )
{
new_exception = std : : current_exception ( ) ;
}
return std : : make_pair ( new_object , new_exception ) ;
2019-10-22 13:41:17 +00:00
}
2019-12-15 13:22:04 +00:00
/// Saves the result of the loading, calculates the time of the next update, and handles errors.
void saveResultOfLoadingSingleObject (
2019-10-22 12:57:58 +00:00
const String & name ,
size_t loading_id ,
LoadablePtr previous_version ,
LoadablePtr new_object ,
std : : exception_ptr new_exception ,
size_t error_count ,
2019-12-15 13:22:04 +00:00
const LoadingGuardForAsyncLoad & )
2019-10-22 12:57:58 +00:00
{
2019-06-02 12:11:01 +00:00
/// Calculate a new update time.
TimePoint next_update_time ;
try
{
if ( new_exception )
+ + error_count ;
else
error_count = 0 ;
2019-10-17 17:36:53 +00:00
2019-10-23 13:02:40 +00:00
LoadablePtr object = previous_version ;
if ( new_object )
object = new_object ;
next_update_time = calculateNextUpdateTime ( object , error_count ) ;
2019-06-02 12:11:01 +00:00
}
catch ( . . . )
{
tryLogCurrentException ( log , " Cannot find out when the " + type_name + " ' " + name + " ' should be updated " ) ;
next_update_time = TimePoint : : max ( ) ;
2017-10-06 10:31:06 +00:00
}
2018-08-10 01:41:54 +00:00
2019-10-23 09:27:34 +00:00
2019-10-22 12:57:58 +00:00
Info * info = getInfo ( name ) ;
2019-06-02 12:11:01 +00:00
2019-12-15 13:22:04 +00:00
/// We should check if this is still the same loading as we were doing.
2019-06-02 12:11:01 +00:00
/// This is necessary because the object could be removed or load with another config while the `mutex` was unlocked.
2020-03-19 02:17:30 +00:00
if ( ! info )
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Next update time for '{}' will not be set because this object was not found. " , name ) ;
2020-03-19 02:17:30 +00:00
return ;
}
2020-03-23 02:12:31 +00:00
if ( ! info - > isLoading ( ) )
2020-03-19 02:17:30 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Next update time for '{}' will not be set because this object is not currently loading. " , name ) ;
2019-06-02 12:11:01 +00:00
return ;
2020-03-19 02:17:30 +00:00
}
if ( info - > loading_id ! = loading_id )
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Next update time for '{}' will not be set because this object's current loading_id {} is different from the specified {}. " , name , info - > loading_id , loading_id ) ;
2020-03-19 02:17:30 +00:00
return ;
}
2019-06-02 12:11:01 +00:00
2019-10-23 09:40:09 +00:00
if ( new_exception )
2019-04-10 18:36:52 +00:00
{
2019-06-02 12:11:01 +00:00
auto next_update_time_description = [ next_update_time ]
{
if ( next_update_time = = TimePoint : : max ( ) )
return String ( ) ;
2021-06-15 19:55:21 +00:00
return " , next update is scheduled at " + to_string ( next_update_time ) ;
2019-06-02 12:11:01 +00:00
} ;
if ( previous_version )
tryLogException ( new_exception , log , " Could not update " + type_name + " ' " + name + " ' "
" , leaving the previous version " + next_update_time_description ( ) ) ;
else
tryLogException ( new_exception , log , " Could not load " + type_name + " ' " + name + " ' " + next_update_time_description ( ) ) ;
2019-04-10 18:36:52 +00:00
}
2019-06-02 12:11:01 +00:00
if ( new_object )
info - > object = new_object ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
info - > exception = new_exception ;
info - > error_count = error_count ;
2020-02-27 10:07:50 +00:00
const auto current_time = std : : chrono : : system_clock : : now ( ) ;
info - > loading_end_time = current_time ;
if ( ! info - > exception )
info - > last_successful_update_time = current_time ;
2019-12-15 13:22:04 +00:00
info - > state_id = info - > loading_id ;
2019-06-02 12:11:01 +00:00
info - > next_update_time = next_update_time ;
2021-06-15 19:55:21 +00:00
LOG_TRACE ( log , " Next update time for '{}' was set to {} " , info - > name , to_string ( next_update_time ) ) ;
2019-10-22 12:57:58 +00:00
}
2019-04-15 19:53:46 +00:00
2019-12-15 13:22:04 +00:00
/// Removes the references to the loading thread from the maps.
void finishLoadingSingleObject ( const String & name , size_t loading_id , const LoadingGuardForAsyncLoad & )
2019-10-22 12:57:58 +00:00
{
2019-12-15 13:22:04 +00:00
Info * info = getInfo ( name ) ;
if ( info & & ( info - > loading_id = = loading_id ) )
2020-03-16 09:21:38 +00:00
{
2019-12-15 13:22:04 +00:00
info - > loading_id = info - > state_id ;
2020-03-16 09:21:38 +00:00
CurrentStatusInfo : : set ( CurrentStatusInfo : : DictionaryStatus , name , static_cast < Int8 > ( info - > status ( ) ) ) ;
}
2019-12-15 13:22:04 +00:00
min_id_to_finish_loading_dependencies . erase ( std : : this_thread : : get_id ( ) ) ;
2019-10-22 12:57:58 +00:00
2020-09-07 23:09:03 +00:00
/// Add `loading_id` to the list of recently finished loadings.
/// This list is used to later put the threads which finished loading back to the thread pool.
/// (We can't put the loading thread back to the thread pool immediately here because at this point
/// the loading thread is about to finish but it's not finished yet right now.)
recently_finished_loadings . push_back ( loading_id ) ;
2019-06-02 12:11:01 +00:00
}
2019-09-27 12:36:54 +00:00
/// Calculate next update time for loaded_object. Can be called without mutex locking,
/// because single loadable can be loaded in single thread only.
TimePoint calculateNextUpdateTime ( const LoadablePtr & loaded_object , size_t error_count ) const
{
static constexpr auto never = TimePoint : : max ( ) ;
2019-10-23 13:02:40 +00:00
if ( loaded_object )
2019-09-27 12:36:54 +00:00
{
if ( ! loaded_object - > supportUpdates ( ) )
2020-03-19 02:17:30 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Supposed update time for '{}' is never (loaded, does not support updates) " , loaded_object - > getLoadableName ( ) ) ;
2020-03-19 02:17:30 +00:00
2019-09-27 12:36:54 +00:00
return never ;
2020-03-19 02:17:30 +00:00
}
2019-09-27 12:36:54 +00:00
/// do not update loadable objects with zero as lifetime
const auto & lifetime = loaded_object - > getLifetime ( ) ;
2019-12-02 13:05:43 +00:00
if ( lifetime . min_sec = = 0 & & lifetime . max_sec = = 0 )
2020-03-19 02:17:30 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Supposed update time for '{}' is never (loaded, lifetime 0) " , loaded_object - > getLoadableName ( ) ) ;
2019-09-27 12:36:54 +00:00
return never ;
2020-03-19 02:17:30 +00:00
}
2019-09-27 12:36:54 +00:00
2019-10-23 13:02:40 +00:00
if ( ! error_count )
{
std : : uniform_int_distribution < UInt64 > distribution { lifetime . min_sec , lifetime . max_sec } ;
2020-03-19 02:17:30 +00:00
auto result = std : : chrono : : system_clock : : now ( ) + std : : chrono : : seconds { distribution ( rnd_engine ) } ;
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Supposed update time for '{}' is {} (loaded, lifetime [{}, {}], no errors) " ,
2021-06-15 19:55:21 +00:00
loaded_object - > getLoadableName ( ) , to_string ( result ) , lifetime . min_sec , lifetime . max_sec ) ;
2020-03-19 02:17:30 +00:00
return result ;
2019-10-23 13:02:40 +00:00
}
2019-09-27 12:36:54 +00:00
2020-03-19 02:17:30 +00:00
auto result = std : : chrono : : system_clock : : now ( ) + std : : chrono : : seconds ( calculateDurationWithBackoff ( rnd_engine , error_count ) ) ;
2021-06-15 19:55:21 +00:00
LOG_TRACE ( log , " Supposed update time for '{}' is {} (backoff, {} errors) " , loaded_object - > getLoadableName ( ) , to_string ( result ) , error_count ) ;
2020-03-19 02:17:30 +00:00
return result ;
}
else
{
auto result = std : : chrono : : system_clock : : now ( ) + std : : chrono : : seconds ( calculateDurationWithBackoff ( rnd_engine , error_count ) ) ;
2021-06-15 19:55:21 +00:00
LOG_TRACE ( log , " Supposed update time for unspecified object is {} (backoff, {} errors. " , to_string ( result ) , error_count ) ;
2020-03-19 02:17:30 +00:00
return result ;
}
2019-09-27 12:36:54 +00:00
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
const CreateObjectFunction create_object ;
const String type_name ;
2020-05-30 21:57:37 +00:00
Poco : : Logger * log ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
mutable std : : mutex mutex ;
std : : condition_variable event ;
2019-09-30 11:18:01 +00:00
ObjectConfigsPtr configs ;
2019-06-02 12:11:01 +00:00
std : : unordered_map < String , Info > infos ;
bool always_load_everything = false ;
2019-10-18 15:44:32 +00:00
std : : atomic < bool > enable_async_loading = false ;
2019-12-15 13:22:04 +00:00
std : : unordered_map < size_t , ThreadFromGlobalPool > loading_threads ;
2020-09-07 23:09:03 +00:00
std : : vector < size_t > recently_finished_loadings ;
2019-12-15 13:22:04 +00:00
std : : unordered_map < std : : thread : : id , size_t > min_id_to_finish_loading_dependencies ;
size_t next_id_counter = 1 ; /// should always be > 0
2019-09-27 12:36:54 +00:00
mutable pcg64 rnd_engine { randomSeed ( ) } ;
2019-06-02 12:11:01 +00:00
} ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
class ExternalLoader : : PeriodicUpdater : private boost : : noncopyable
{
public :
2019-08-30 09:50:38 +00:00
static constexpr UInt64 check_period_sec = 5 ;
2021-10-13 06:56:34 +00:00
PeriodicUpdater ( LoadablesConfigReader & config_files_reader_ , LoadingDispatcher & loading_dispatcher_ )
: config_files_reader ( config_files_reader_ ) , loading_dispatcher ( loading_dispatcher_ )
2019-06-02 12:11:01 +00:00
{
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
~ PeriodicUpdater ( ) { enable ( false ) ; }
2017-10-06 10:31:06 +00:00
2019-08-30 09:50:38 +00:00
void enable ( bool enable_ )
2019-06-02 12:11:01 +00:00
{
std : : unique_lock lock { mutex } ;
enabled = enable_ ;
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
if ( enable_ )
{
if ( ! thread . joinable ( ) )
{
/// Starts the thread which will do periodic updates.
thread = ThreadFromGlobalPool { & PeriodicUpdater : : doPeriodicUpdates , this } ;
}
}
else
{
if ( thread . joinable ( ) )
{
/// Wait for the thread to finish.
auto temp_thread = std : : move ( thread ) ;
lock . unlock ( ) ;
event . notify_one ( ) ;
temp_thread . join ( ) ;
2019-04-10 18:36:52 +00:00
}
}
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
private :
void doPeriodicUpdates ( )
{
setThreadName ( " ExterLdrReload " ) ;
std : : unique_lock lock { mutex } ;
auto pred = [ this ] { return ! enabled ; } ;
2019-08-30 09:50:38 +00:00
while ( ! event . wait_for ( lock , std : : chrono : : seconds ( check_period_sec ) , pred ) )
2019-06-02 12:11:01 +00:00
{
lock . unlock ( ) ;
2021-10-13 06:56:34 +00:00
loading_dispatcher . setConfiguration ( config_files_reader . read ( ) ) ;
loading_dispatcher . reloadOutdated ( ) ;
2019-06-02 12:11:01 +00:00
lock . lock ( ) ;
}
}
2019-04-15 19:53:46 +00:00
2019-09-30 16:12:08 +00:00
LoadablesConfigReader & config_files_reader ;
2019-06-02 12:11:01 +00:00
LoadingDispatcher & loading_dispatcher ;
mutable std : : mutex mutex ;
bool enabled = false ;
ThreadFromGlobalPool thread ;
std : : condition_variable event ;
} ;
2020-05-30 21:57:37 +00:00
ExternalLoader : : ExternalLoader ( const String & type_name_ , Poco : : Logger * log_ )
2019-12-12 18:33:43 +00:00
: config_files_reader ( std : : make_unique < LoadablesConfigReader > ( type_name_ , log_ ) )
2019-06-02 12:11:01 +00:00
, loading_dispatcher ( std : : make_unique < LoadingDispatcher > (
2020-04-22 06:01:33 +00:00
[ this ] ( auto & & a , auto & & b , auto & & c ) { return createObject ( a , b , c ) ; } ,
2019-06-02 12:11:01 +00:00
type_name_ ,
2019-12-12 18:33:43 +00:00
log_ ) )
2021-10-13 06:56:34 +00:00
, periodic_updater ( std : : make_unique < PeriodicUpdater > ( * config_files_reader , * loading_dispatcher ) )
2019-06-02 12:11:01 +00:00
, type_name ( type_name_ )
2019-12-12 18:33:43 +00:00
, log ( log_ )
2019-06-02 12:11:01 +00:00
{
2019-04-15 19:53:46 +00:00
}
2019-06-02 12:11:01 +00:00
ExternalLoader : : ~ ExternalLoader ( ) = default ;
2019-04-15 19:53:46 +00:00
2021-06-15 19:55:21 +00:00
scope_guard ExternalLoader : : addConfigRepository ( std : : unique_ptr < IExternalLoaderConfigRepository > repository ) const
2019-10-15 18:04:17 +00:00
{
2019-12-30 23:30:06 +00:00
auto * ptr = repository . get ( ) ;
String name = ptr - > getName ( ) ;
2021-10-13 06:56:34 +00:00
2019-12-30 23:30:06 +00:00
config_files_reader - > addConfigRepository ( std : : move ( repository ) ) ;
reloadConfig ( name ) ;
return [ this , ptr , name ] ( )
{
config_files_reader - > removeConfigRepository ( ptr ) ;
2022-08-05 11:42:40 +00:00
CurrentStatusInfo : : unset ( CurrentStatusInfo : : DictionaryStatus , name ) ;
2019-12-30 23:30:06 +00:00
reloadConfig ( name ) ;
} ;
2019-10-15 18:04:17 +00:00
}
2019-12-30 23:30:06 +00:00
void ExternalLoader : : setConfigSettings ( const ExternalLoaderConfigSettings & settings )
2019-04-15 19:53:46 +00:00
{
2019-12-30 23:30:06 +00:00
config_files_reader - > setConfigSettings ( settings ) ;
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
void ExternalLoader : : enableAlwaysLoadEverything ( bool enable )
{
loading_dispatcher - > enableAlwaysLoadEverything ( enable ) ;
}
2017-10-06 10:31:06 +00:00
2019-06-02 12:11:01 +00:00
void ExternalLoader : : enableAsyncLoading ( bool enable )
{
loading_dispatcher - > enableAsyncLoading ( enable ) ;
2019-04-10 18:36:52 +00:00
}
2017-10-06 10:31:06 +00:00
2019-08-30 09:50:38 +00:00
void ExternalLoader : : enablePeriodicUpdates ( bool enable_ )
2019-06-02 12:11:01 +00:00
{
2019-08-30 09:50:38 +00:00
periodic_updater - > enable ( enable_ ) ;
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2020-04-12 20:50:32 +00:00
bool ExternalLoader : : hasLoadedObjects ( ) const
2019-06-02 12:11:01 +00:00
{
2020-04-12 20:50:32 +00:00
return loading_dispatcher - > hasLoadedObjects ( ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2019-06-02 12:11:01 +00:00
ExternalLoader : : Status ExternalLoader : : getCurrentStatus ( const String & name ) const
{
return loading_dispatcher - > getCurrentStatus ( name ) ;
2019-04-10 18:36:52 +00:00
}
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
2020-04-12 20:50:32 +00:00
ReturnType ExternalLoader : : getLoadResult ( const String & name ) const
2019-04-10 18:36:52 +00:00
{
2020-04-12 20:50:32 +00:00
return loading_dispatcher - > getLoadResult < ReturnType > ( name ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-15 19:53:46 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
2020-04-12 20:50:32 +00:00
ReturnType ExternalLoader : : getLoadResults ( const FilterByNameFunction & filter ) const
{
return loading_dispatcher - > getLoadResults < ReturnType > ( filter ) ;
}
ExternalLoader : : Loadables ExternalLoader : : getLoadedObjects ( ) const
2019-06-02 12:11:01 +00:00
{
2020-04-12 20:50:32 +00:00
return getLoadResults < Loadables > ( ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2020-04-12 20:50:32 +00:00
ExternalLoader : : Loadables ExternalLoader : : getLoadedObjects ( const FilterByNameFunction & filter ) const
2019-06-02 12:11:01 +00:00
{
2020-04-12 20:50:32 +00:00
return getLoadResults < Loadables > ( filter ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2020-04-12 20:50:32 +00:00
size_t ExternalLoader : : getNumberOfLoadedObjects ( ) const
2019-06-02 12:11:01 +00:00
{
2020-04-12 20:50:32 +00:00
return loading_dispatcher - > getNumberOfLoadedObjects ( ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2020-04-12 20:50:32 +00:00
size_t ExternalLoader : : getNumberOfObjects ( ) const
2019-06-02 12:11:01 +00:00
{
2020-04-12 20:50:32 +00:00
return loading_dispatcher - > getNumberOfObjects ( ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
ReturnType ExternalLoader : : tryLoad ( const String & name , Duration timeout ) const
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
return loading_dispatcher - > tryLoad < ReturnType > ( name , timeout ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
ReturnType ExternalLoader : : tryLoad ( const FilterByNameFunction & filter , Duration timeout ) const
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
return loading_dispatcher - > tryLoad < ReturnType > ( filter , timeout ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
ReturnType ExternalLoader : : load ( const String & name ) const
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
auto result = tryLoad < LoadResult > ( name ) ;
checkLoaded ( result , false ) ;
return convertTo < ReturnType > ( result ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
ReturnType ExternalLoader : : load ( const FilterByNameFunction & filter ) const
{
auto results = tryLoad < LoadResults > ( filter ) ;
checkLoaded ( results , false ) ;
return convertTo < ReturnType > ( results ) ;
}
2019-10-17 13:05:12 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
ReturnType ExternalLoader : : loadOrReload ( const String & name ) const
2019-10-17 13:05:12 +00:00
{
2021-10-13 06:56:34 +00:00
loading_dispatcher - > setConfiguration ( config_files_reader - > read ( ) ) ;
2019-12-12 18:33:43 +00:00
auto result = loading_dispatcher - > tryLoadOrReload < LoadResult > ( name , WAIT ) ;
checkLoaded ( result , true ) ;
return convertTo < ReturnType > ( result ) ;
2019-10-17 13:05:12 +00:00
}
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
ReturnType ExternalLoader : : loadOrReload ( const FilterByNameFunction & filter ) const
{
2021-10-13 06:56:34 +00:00
loading_dispatcher - > setConfiguration ( config_files_reader - > read ( ) ) ;
2019-12-12 18:33:43 +00:00
auto results = loading_dispatcher - > tryLoadOrReload < LoadResults > ( filter , WAIT ) ;
checkLoaded ( results , true ) ;
return convertTo < ReturnType > ( results ) ;
}
2019-10-17 13:05:12 +00:00
2019-12-12 18:33:43 +00:00
template < typename ReturnType , typename >
ReturnType ExternalLoader : : reloadAllTriedToLoad ( ) const
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
std : : unordered_set < String > names ;
boost : : range : : copy ( getAllTriedToLoadNames ( ) , std : : inserter ( names , names . end ( ) ) ) ;
return loadOrReload < ReturnType > ( [ & names ] ( const String & name ) { return names . count ( name ) ; } ) ;
2019-06-02 12:11:01 +00:00
}
2019-04-10 18:36:52 +00:00
2021-03-18 14:03:22 +00:00
bool ExternalLoader : : has ( const String & name ) const
{
return loading_dispatcher - > has ( name ) ;
}
2019-12-12 18:33:43 +00:00
Strings ExternalLoader : : getAllTriedToLoadNames ( ) const
2019-06-02 12:11:01 +00:00
{
2019-12-12 18:33:43 +00:00
return loading_dispatcher - > getAllTriedToLoadNames ( ) ;
2017-10-06 10:31:06 +00:00
}
2019-12-12 18:33:43 +00:00
void ExternalLoader : : checkLoaded ( const ExternalLoader : : LoadResult & result ,
bool check_no_errors ) const
2017-10-06 10:31:06 +00:00
{
2019-12-12 18:33:43 +00:00
if ( result . object & & ( ! check_no_errors | | ! result . exception ) )
return ;
if ( result . status = = ExternalLoader : : Status : : LOADING )
2023-01-23 21:13:58 +00:00
throw Exception ( ErrorCodes : : BAD_ARGUMENTS , " {} '{}' is still loading " , type_name , result . name ) ;
2019-12-12 18:33:43 +00:00
if ( result . exception )
2020-10-27 14:21:51 +00:00
{
// Exception is shared for multiple threads.
// Don't just rethrow it, because sharing the same exception object
// between multiple threads can lead to weird effects if they decide to
// modify it, for example, by adding some error context.
try
{
std : : rethrow_exception ( result . exception ) ;
}
2020-10-27 16:12:53 +00:00
catch ( const Poco : : Exception & e )
{
/// This will create a copy for Poco::Exception and DB::Exception
e . rethrow ( ) ;
}
2020-10-27 14:21:51 +00:00
catch ( . . . )
{
throw DB : : Exception ( ErrorCodes : : DICTIONARIES_WAS_NOT_LOADED ,
" Failed to load dictionary '{}': {} " ,
result . name ,
getCurrentExceptionMessage ( true /*with stack trace*/ ,
true /*check embedded stack trace*/ ) ) ;
}
}
2019-12-12 18:33:43 +00:00
if ( result . status = = ExternalLoader : : Status : : NOT_EXIST )
2023-01-23 21:13:58 +00:00
throw Exception ( ErrorCodes : : BAD_ARGUMENTS , " {} '{}' not found " , type_name , result . name ) ;
2019-12-12 18:33:43 +00:00
if ( result . status = = ExternalLoader : : Status : : NOT_LOADED )
2023-01-23 21:13:58 +00:00
throw Exception ( ErrorCodes : : BAD_ARGUMENTS , " {} '{}' not tried to load " , type_name , result . name ) ;
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2019-12-12 18:33:43 +00:00
void ExternalLoader : : checkLoaded ( const ExternalLoader : : LoadResults & results ,
bool check_no_errors ) const
2019-11-25 22:48:23 +00:00
{
2019-12-12 18:33:43 +00:00
std : : exception_ptr exception ;
for ( const auto & result : results )
{
try
{
checkLoaded ( result , check_no_errors ) ;
}
catch ( . . . )
{
if ( ! exception )
exception = std : : current_exception ( ) ;
else
tryLogCurrentException ( log ) ;
}
}
if ( exception )
std : : rethrow_exception ( exception ) ;
2019-11-25 22:48:23 +00:00
}
2019-12-12 18:33:43 +00:00
2019-12-12 16:41:41 +00:00
void ExternalLoader : : reloadConfig ( ) const
2019-10-18 15:44:32 +00:00
{
2019-12-12 16:41:41 +00:00
loading_dispatcher - > setConfiguration ( config_files_reader - > read ( ) ) ;
2019-10-18 15:44:32 +00:00
}
2019-12-12 16:41:41 +00:00
void ExternalLoader : : reloadConfig ( const String & repository_name ) const
{
loading_dispatcher - > setConfiguration ( config_files_reader - > read ( repository_name ) ) ;
}
void ExternalLoader : : reloadConfig ( const String & repository_name , const String & path ) const
{
loading_dispatcher - > setConfiguration ( config_files_reader - > read ( repository_name , path ) ) ;
}
2019-10-18 15:44:32 +00:00
2019-07-17 08:39:36 +00:00
ExternalLoader : : LoadablePtr ExternalLoader : : createObject (
2019-12-04 23:53:06 +00:00
const String & name , const ObjectConfig & config , const LoadablePtr & previous_version ) const
2019-06-02 12:11:01 +00:00
{
2019-12-04 23:53:06 +00:00
if ( previous_version )
2019-07-17 08:39:36 +00:00
return previous_version - > clone ( ) ;
2017-10-06 10:31:06 +00:00
2019-12-11 11:09:21 +00:00
return create ( name , * config . config , config . key_in_config , config . repository_name ) ;
2019-06-02 12:11:01 +00:00
}
2017-10-06 10:31:06 +00:00
2020-04-12 20:50:32 +00:00
template ExternalLoader : : LoadablePtr ExternalLoader : : getLoadResult < ExternalLoader : : LoadablePtr > ( const String & ) const ;
template ExternalLoader : : LoadResult ExternalLoader : : getLoadResult < ExternalLoader : : LoadResult > ( const String & ) const ;
template ExternalLoader : : Loadables ExternalLoader : : getLoadResults < ExternalLoader : : Loadables > ( const FilterByNameFunction & ) const ;
template ExternalLoader : : LoadResults ExternalLoader : : getLoadResults < ExternalLoader : : LoadResults > ( const FilterByNameFunction & ) const ;
2019-12-12 18:33:43 +00:00
template ExternalLoader : : LoadablePtr ExternalLoader : : tryLoad < ExternalLoader : : LoadablePtr > ( const String & , Duration ) const ;
template ExternalLoader : : LoadResult ExternalLoader : : tryLoad < ExternalLoader : : LoadResult > ( const String & , Duration ) const ;
template ExternalLoader : : Loadables ExternalLoader : : tryLoad < ExternalLoader : : Loadables > ( const FilterByNameFunction & , Duration ) const ;
template ExternalLoader : : LoadResults ExternalLoader : : tryLoad < ExternalLoader : : LoadResults > ( const FilterByNameFunction & , Duration ) const ;
template ExternalLoader : : LoadablePtr ExternalLoader : : load < ExternalLoader : : LoadablePtr > ( const String & ) const ;
template ExternalLoader : : LoadResult ExternalLoader : : load < ExternalLoader : : LoadResult > ( const String & ) const ;
template ExternalLoader : : Loadables ExternalLoader : : load < ExternalLoader : : Loadables > ( const FilterByNameFunction & ) const ;
template ExternalLoader : : LoadResults ExternalLoader : : load < ExternalLoader : : LoadResults > ( const FilterByNameFunction & ) const ;
template ExternalLoader : : LoadablePtr ExternalLoader : : loadOrReload < ExternalLoader : : LoadablePtr > ( const String & ) const ;
template ExternalLoader : : LoadResult ExternalLoader : : loadOrReload < ExternalLoader : : LoadResult > ( const String & ) const ;
template ExternalLoader : : Loadables ExternalLoader : : loadOrReload < ExternalLoader : : Loadables > ( const FilterByNameFunction & ) const ;
template ExternalLoader : : LoadResults ExternalLoader : : loadOrReload < ExternalLoader : : LoadResults > ( const FilterByNameFunction & ) const ;
template ExternalLoader : : Loadables ExternalLoader : : reloadAllTriedToLoad < ExternalLoader : : Loadables > ( ) const ;
template ExternalLoader : : LoadResults ExternalLoader : : reloadAllTriedToLoad < ExternalLoader : : LoadResults > ( ) const ;
2017-10-06 10:31:06 +00:00
}