ClickHouse/dbms/include/DB/Interpreters/ExternalDictionaries.h

#pragma once

#include <DB/Dictionaries/IDictionary.h>
#include <DB/Common/Exception.h>
#include <DB/Common/setThreadName.h>
#include <DB/Common/randomSeed.h>
#include <common/MultiVersion.h>
#include <common/logger_useful.h>
#include <Poco/Event.h>
#include <unistd.h>
#include <time.h>
#include <mutex>
#include <thread>
#include <unordered_map>
#include <chrono>
#include <random>

namespace DB
{

class Context;

/** Manages user-defined dictionaries.
*	Monitors configuration file and automatically reloads dictionaries in a separate thread.
*	The monitoring thread wakes up every @check_period_sec seconds and checks
*	modification time of dictionaries' configuration file. If said time is greater than
*	@config_last_modified, the dictionaries are created from scratch using configuration file,
*	possibly overriding currently existing dictionaries with the same name (previous versions of
*	overridden dictionaries will live as long as there are any users retaining them).
*
*	Apart from checking configuration file for modifications, each non-cached dictionary
*	has a lifetime of its own and may be updated if it's source reports that it has been
*	modified. The time of next update is calculated by choosing uniformly a random number
*	distributed between lifetime.min_sec and lifetime.max_sec.
*	If either of lifetime.min_sec and lifetime.max_sec is zero, such dictionary is never updated.
*/
class ExternalDictionaries
{
private:
	static const auto check_period_sec = 5;

	friend class StorageSystemDictionaries;

	mutable std::mutex dictionaries_mutex;

	using DictionaryPtr = std::shared_ptr<MultiVersion<IDictionaryBase>>;
	struct DictionaryInfo final
	{
		DictionaryPtr dict;
		std::string origin;
		std::exception_ptr exception;
	};

	struct FailedDictionaryInfo final
	{
		std::unique_ptr<IDictionaryBase> dict;
		std::chrono::system_clock::time_point next_attempt_time;
		UInt64 error_count;
	};

	/** name -> dictionary.
	  */
	std::unordered_map<std::string, DictionaryInfo> dictionaries;

	/** Here are dictionaries, that has been never loaded sussessfully.
	  * They are also in 'dictionaries', but with nullptr as 'dict'.
	  */
	std::unordered_map<std::string, FailedDictionaryInfo> failed_dictionaries;

	/** Both for dictionaries and failed_dictionaries.
	  */
	std::unordered_map<std::string, std::chrono::system_clock::time_point> update_times;

	std::mt19937_64 rnd_engine{randomSeed()};

	Context & context;

	std::thread reloading_thread;
	Poco::Event destroy;

	Logger * log;

	std::unordered_map<std::string, Poco::Timestamp> last_modification_times;

	void reloadImpl(bool throw_on_error = false);
	void reloadFromFile(const std::string & config_path, bool throw_on_error);

	void reloadPeriodically()
	{
		setThreadName("ExterDictReload");

		while (true)
		{
			if (destroy.tryWait(check_period_sec * 1000))
				return;

			reloadImpl();
		}
	}

public:
	/// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
	ExternalDictionaries(Context & context, const bool throw_on_error)
		: context(context), log(&Logger::get("ExternalDictionaries"))
	{
		{
			/** During synchronous loading of external dictionaries at moment of query execution,
			  *  we should not use per query memory limit.
			  */
			TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker;

			reloadImpl(throw_on_error);
		}

		reloading_thread = std::thread{&ExternalDictionaries::reloadPeriodically, this};
	}

	~ExternalDictionaries()
	{
		destroy.set();
		reloading_thread.join();
	}

	MultiVersion<IDictionaryBase>::Version getDictionary(const std::string & name) const;
};

}