#pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace DB { class Context; /** Manages user-defined dictionaries. * Monitors configuration file and automatically reloads dictionaries in a separate thread. * The monitoring thread wakes up every @check_period_sec seconds and checks * modification time of dictionaries' configuration file. If said time is greater than * @config_last_modified, the dictionaries are created from scratch using configuration file, * possibly overriding currently existing dictionaries with the same name (previous versions of * overridden dictionaries will live as long as there are any users retaining them). * * Apart from checking configuration file for modifications, each non-cached dictionary * has a lifetime of its own and may be updated if it's source reports that it has been * modified. The time of next update is calculated by choosing uniformly a random number * distributed between lifetime.min_sec and lifetime.max_sec. * If either of lifetime.min_sec and lifetime.max_sec is zero, such dictionary is never updated. */ class ExternalDictionaries { private: static const auto check_period_sec = 5; friend class StorageSystemDictionaries; mutable std::mutex dictionaries_mutex; using DictionaryPtr = std::shared_ptr>; struct DictionaryInfo final { DictionaryPtr dict; std::string origin; std::exception_ptr exception; }; struct FailedDictionaryInfo final { std::unique_ptr dict; std::chrono::system_clock::time_point next_attempt_time; std::uint64_t error_count; }; /** Имя словаря -> словарь. */ std::unordered_map dictionaries; /** Здесь находятся словари, которых ещё ни разу не удалось загрузить. * В dictionaries они тоже присутствуют, но с нулевым указателем dict. */ std::unordered_map failed_dictionaries; /** И для обычных и для failed_dictionaries. */ std::unordered_map update_times; std::mt19937_64 rnd_engine{randomSeed()}; Context & context; std::thread reloading_thread; Poco::Event destroy; Logger * log; std::unordered_map last_modification_times; void reloadImpl(bool throw_on_error = false); void reloadFromFile(const std::string & config_path, bool throw_on_error); void reloadPeriodically() { setThreadName("ExterDictReload"); while (true) { if (destroy.tryWait(check_period_sec * 1000)) return; reloadImpl(); } } public: /// Справочники будут обновляться в отдельном потоке, каждые reload_period секунд. ExternalDictionaries(Context & context, const bool throw_on_error) : context(context), log(&Logger::get("ExternalDictionaries")) { { /** При синхронной загрузки внешних словарей в момент выполнения запроса, * не нужно использовать ограничение на расход памяти запросом. */ struct TemporarilyDisableMemoryTracker { MemoryTracker * memory_tracker; TemporarilyDisableMemoryTracker() { memory_tracker = current_memory_tracker; current_memory_tracker = nullptr; } ~TemporarilyDisableMemoryTracker() { current_memory_tracker = memory_tracker; } } temporarily_disable_memory_tracker; reloadImpl(throw_on_error); } reloading_thread = std::thread{&ExternalDictionaries::reloadPeriodically, this}; } ~ExternalDictionaries() { destroy.set(); reloading_thread.join(); } MultiVersion::Version getDictionary(const std::string & name) const; }; }