2013-08-28 20:47:22 +00:00
|
|
|
|
#include <iomanip>
|
|
|
|
|
|
|
|
|
|
#include <Yandex/logger_useful.h>
|
|
|
|
|
|
2013-08-28 17:23:00 +00:00
|
|
|
|
#include <DB/Common/SipHash.h>
|
2013-08-28 21:36:16 +00:00
|
|
|
|
#include <DB/IO/ReadHelpers.h>
|
2013-08-12 00:36:18 +00:00
|
|
|
|
#include <DB/Interpreters/Quota.h>
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
#include <set>
|
|
|
|
|
|
2013-08-12 00:36:18 +00:00
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
|
{
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
void QuotaValues::initFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
2013-08-28 21:36:16 +00:00
|
|
|
|
queries = parse<UInt64>(config.getString(config_elem + ".queries", "0"));
|
|
|
|
|
errors = parse<UInt64>(config.getString(config_elem + ".errors", "0"));
|
|
|
|
|
result_rows = parse<UInt64>(config.getString(config_elem + ".result_rows", "0"));
|
|
|
|
|
result_bytes = parse<UInt64>(config.getString(config_elem + ".result_bytes", "0"));
|
|
|
|
|
read_rows = parse<UInt64>(config.getString(config_elem + ".read_rows", "0"));
|
|
|
|
|
read_bytes = parse<UInt64>(config.getString(config_elem + ".read_bytes", "0"));
|
2013-08-12 00:36:18 +00:00
|
|
|
|
execution_time = Poco::Timespan(config.getInt(config_elem + ".execution_time", 0), 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
void QuotaForInterval::initFromConfig(const String & config_elem, time_t duration_, Poco::Util::AbstractConfiguration & config)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
rounded_time = 0;
|
|
|
|
|
duration = duration_;
|
2014-02-13 07:17:22 +00:00
|
|
|
|
max.initFromConfig(config_elem, config);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForInterval::checkExceeded(time_t current_time, const String & quota_name)
|
|
|
|
|
{
|
|
|
|
|
updateTime(current_time);
|
|
|
|
|
check(max.queries, used.queries, current_time, quota_name, "Queries");
|
|
|
|
|
check(max.errors, used.errors, current_time, quota_name, "Errors");
|
|
|
|
|
check(max.result_rows, used.result_rows, current_time, quota_name, "Total result rows");
|
2013-08-28 17:23:00 +00:00
|
|
|
|
check(max.result_bytes, used.result_bytes, current_time, quota_name, "Total result bytes");
|
2013-08-12 00:36:18 +00:00
|
|
|
|
check(max.read_rows, used.read_rows, current_time, quota_name, "Total rows read");
|
2013-08-28 17:23:00 +00:00
|
|
|
|
check(max.read_bytes, used.read_bytes, current_time, quota_name, "Total bytes read");
|
2013-08-12 00:36:18 +00:00
|
|
|
|
check(max.execution_time.totalSeconds(), used.execution_time.totalSeconds(), current_time, quota_name, "Total execution time");
|
2013-08-28 20:47:22 +00:00
|
|
|
|
}
|
2013-08-12 00:36:18 +00:00
|
|
|
|
|
2013-08-28 20:47:22 +00:00
|
|
|
|
String QuotaForInterval::toString() const
|
|
|
|
|
{
|
|
|
|
|
std::stringstream res;
|
|
|
|
|
|
|
|
|
|
res << std::fixed << std::setprecision(3)
|
2013-08-28 21:36:16 +00:00
|
|
|
|
<< "Interval: " << mysqlxx::DateTime(rounded_time) << " - " << mysqlxx::DateTime(rounded_time + duration) << ".\n"
|
|
|
|
|
<< "Queries: " << used.queries << ".\n"
|
|
|
|
|
<< "Errors: " << used.errors << ".\n"
|
|
|
|
|
<< "Result rows: " << used.result_rows << ".\n"
|
|
|
|
|
<< "Result bytes: " << used.result_bytes << ".\n"
|
|
|
|
|
<< "Read rows: " << used.read_rows << ".\n"
|
|
|
|
|
<< "Read bytes: " << used.read_bytes << ".\n"
|
|
|
|
|
<< "Execution time: " << used.execution_time.totalMilliseconds() / 1000.0 << " sec.\n";
|
2013-08-28 20:47:22 +00:00
|
|
|
|
|
|
|
|
|
return res.str();
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForInterval::addQuery(time_t current_time, const String & quota_name)
|
|
|
|
|
{
|
2013-09-06 23:05:57 +00:00
|
|
|
|
__sync_fetch_and_add(&used.queries, 1);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForInterval::addError(time_t current_time, const String & quota_name)
|
|
|
|
|
{
|
2013-09-06 23:05:57 +00:00
|
|
|
|
__sync_fetch_and_add(&used.errors, 1);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-08-28 20:47:22 +00:00
|
|
|
|
void QuotaForInterval::checkAndAddResultRowsBytes(time_t current_time, const String & quota_name, size_t rows, size_t bytes)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
checkExceeded(current_time, quota_name);
|
2013-09-06 23:05:57 +00:00
|
|
|
|
__sync_fetch_and_add(&used.result_rows, rows);
|
|
|
|
|
__sync_fetch_and_add(&used.result_bytes, bytes);
|
2013-08-28 17:23:00 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-08-28 20:47:22 +00:00
|
|
|
|
void QuotaForInterval::checkAndAddReadRowsBytes(time_t current_time, const String & quota_name, size_t rows, size_t bytes)
|
2013-08-28 17:23:00 +00:00
|
|
|
|
{
|
|
|
|
|
checkExceeded(current_time, quota_name);
|
2013-09-06 23:05:57 +00:00
|
|
|
|
__sync_fetch_and_add(&used.read_rows, rows);
|
|
|
|
|
__sync_fetch_and_add(&used.read_bytes, bytes);
|
2013-08-28 17:23:00 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForInterval::checkAndAddExecutionTime(time_t current_time, const String & quota_name, Poco::Timespan amount)
|
|
|
|
|
{
|
|
|
|
|
checkExceeded(current_time, quota_name);
|
2013-09-06 23:05:57 +00:00
|
|
|
|
/// Используется информация о внутреннем представлении Poco::Timespan.
|
|
|
|
|
__sync_fetch_and_add(reinterpret_cast<Int64 *>(&used.execution_time), amount.totalMicroseconds());
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForInterval::updateTime(time_t current_time)
|
|
|
|
|
{
|
|
|
|
|
if (current_time >= rounded_time + static_cast<int>(duration))
|
|
|
|
|
{
|
|
|
|
|
rounded_time = current_time / duration * duration;
|
|
|
|
|
used.clear();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2013-08-28 17:23:00 +00:00
|
|
|
|
void QuotaForInterval::check(size_t max_amount, size_t used_amount, time_t current_time, const String & quota_name, const char * resource_name)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
2013-08-28 17:23:00 +00:00
|
|
|
|
if (max_amount && used_amount >= max_amount)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
std::stringstream message;
|
|
|
|
|
message << "Quota '" << quota_name << "' for ";
|
|
|
|
|
|
|
|
|
|
if (duration == 3600)
|
|
|
|
|
message << "1 hour";
|
|
|
|
|
else if (duration == 60)
|
|
|
|
|
message << "1 minute";
|
|
|
|
|
else if (duration % 3600 == 0)
|
|
|
|
|
message << (duration / 3600) << " hours";
|
|
|
|
|
else if (duration % 60 == 0)
|
|
|
|
|
message << (duration / 60) << " minutes";
|
|
|
|
|
else
|
|
|
|
|
message << duration << " seconds";
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
message << " has been exceeded. "
|
2013-08-28 17:23:00 +00:00
|
|
|
|
<< resource_name << ": " << used_amount << ", max: " << max_amount << ". "
|
2013-08-12 00:36:18 +00:00
|
|
|
|
<< "Interval will end at " << mysqlxx::DateTime(rounded_time + duration) << ".";
|
|
|
|
|
|
|
|
|
|
throw Exception(message.str(), ErrorCodes::QUOTA_EXPIRED);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
void QuotaForIntervals::initFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
Poco::Util::AbstractConfiguration::Keys config_keys;
|
|
|
|
|
config.keys(config_elem, config_keys);
|
|
|
|
|
|
|
|
|
|
for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it)
|
|
|
|
|
{
|
|
|
|
|
if (0 != it->compare(0, strlen("interval"), "interval"))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
String interval_config_elem = config_elem + "." + *it;
|
|
|
|
|
time_t duration = config.getInt(interval_config_elem + ".duration");
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
cont[duration].initFromConfig(interval_config_elem, duration, config);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForIntervals::setMax(const QuotaForIntervals & quota)
|
|
|
|
|
{
|
|
|
|
|
for (Container::iterator it = cont.begin(); it != cont.end();)
|
|
|
|
|
{
|
|
|
|
|
if (quota.cont.count(it->first))
|
|
|
|
|
++it;
|
|
|
|
|
else
|
|
|
|
|
cont.erase(it++);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (auto & x : quota.cont)
|
|
|
|
|
{
|
|
|
|
|
if (!cont.count(x.first))
|
|
|
|
|
cont[x.first] = x.second;
|
|
|
|
|
else
|
|
|
|
|
cont[x.first].max = x.second.max;
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForIntervals::checkExceeded(time_t current_time)
|
|
|
|
|
{
|
|
|
|
|
for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it)
|
2014-02-13 07:17:22 +00:00
|
|
|
|
it->second.checkExceeded(current_time, name);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForIntervals::addQuery(time_t current_time)
|
|
|
|
|
{
|
|
|
|
|
for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it)
|
2014-02-13 07:17:22 +00:00
|
|
|
|
it->second.addQuery(current_time, name);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void QuotaForIntervals::addError(time_t current_time)
|
|
|
|
|
{
|
|
|
|
|
for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it)
|
2014-02-13 07:17:22 +00:00
|
|
|
|
it->second.addError(current_time, name);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-08-28 20:47:22 +00:00
|
|
|
|
void QuotaForIntervals::checkAndAddResultRowsBytes(time_t current_time, size_t rows, size_t bytes)
|
2013-08-28 17:23:00 +00:00
|
|
|
|
{
|
|
|
|
|
for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it)
|
2014-02-13 07:17:22 +00:00
|
|
|
|
it->second.checkAndAddResultRowsBytes(current_time, name, rows, bytes);
|
2013-08-28 17:23:00 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-08-28 20:47:22 +00:00
|
|
|
|
void QuotaForIntervals::checkAndAddReadRowsBytes(time_t current_time, size_t rows, size_t bytes)
|
2013-08-28 17:23:00 +00:00
|
|
|
|
{
|
|
|
|
|
for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it)
|
2014-02-13 07:17:22 +00:00
|
|
|
|
it->second.checkAndAddReadRowsBytes(current_time, name, rows, bytes);
|
2013-08-28 17:23:00 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-08-28 20:47:22 +00:00
|
|
|
|
void QuotaForIntervals::checkAndAddExecutionTime(time_t current_time, Poco::Timespan amount)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it)
|
2014-02-13 07:17:22 +00:00
|
|
|
|
it->second.checkAndAddExecutionTime(current_time, name, amount);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-08-28 20:47:22 +00:00
|
|
|
|
String QuotaForIntervals::toString() const
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
2013-08-28 20:47:22 +00:00
|
|
|
|
std::stringstream res;
|
2013-08-12 00:36:18 +00:00
|
|
|
|
|
2013-09-06 23:05:57 +00:00
|
|
|
|
for (Container::const_reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it)
|
|
|
|
|
res << std::endl << it->second.toString();
|
2013-08-28 20:47:22 +00:00
|
|
|
|
|
|
|
|
|
return res.str();
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
void Quota::loadFromConfig(const String & config_elem, const String & name_, Poco::Util::AbstractConfiguration & config)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
name = name_;
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
bool new_keyed_by_ip = config.has(config_elem + ".keyed_by_ip");
|
|
|
|
|
bool new_is_keyed = new_keyed_by_ip || config.has(config_elem + ".keyed");
|
2013-08-12 00:36:18 +00:00
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
if (new_is_keyed != is_keyed || new_keyed_by_ip != keyed_by_ip)
|
|
|
|
|
{
|
|
|
|
|
keyed_by_ip = new_keyed_by_ip;
|
|
|
|
|
is_keyed = new_is_keyed;
|
|
|
|
|
/// Смысл ключей поменялся. Выбросим накопленные значения.
|
|
|
|
|
quota_for_keys.clear();
|
|
|
|
|
}
|
2013-08-12 00:36:18 +00:00
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
QuotaForIntervals new_max(name);
|
|
|
|
|
new_max.initFromConfig(config_elem, config);
|
|
|
|
|
if (!(new_max == max))
|
|
|
|
|
{
|
|
|
|
|
max = new_max;
|
|
|
|
|
for (auto & quota : quota_for_keys)
|
|
|
|
|
{
|
|
|
|
|
quota.second->setMax(max);
|
|
|
|
|
}
|
|
|
|
|
}
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
QuotaForIntervalsPtr Quota::get(const String & quota_key, const String & user_name, const Poco::Net::IPAddress & ip)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
if (!quota_key.empty() && (!is_keyed || keyed_by_ip))
|
|
|
|
|
throw Exception("Quota " + name + " doesn't allow client supplied keys.", ErrorCodes::QUOTA_DOESNT_ALLOW_KEYS);
|
|
|
|
|
|
2013-11-03 00:24:46 +00:00
|
|
|
|
/** Квота считается отдельно:
|
|
|
|
|
* - для каждого IP-адреса, если keyed_by_ip;
|
|
|
|
|
* - иначе для каждого quota_key, если он есть;
|
|
|
|
|
* - иначе для каждого пользователя.
|
|
|
|
|
*/
|
2013-08-12 00:36:18 +00:00
|
|
|
|
|
2013-11-03 00:24:46 +00:00
|
|
|
|
UInt64 quota_key_hashed = sipHash64(
|
|
|
|
|
keyed_by_ip
|
|
|
|
|
? ip.toString()
|
|
|
|
|
: (!quota_key.empty()
|
|
|
|
|
? quota_key
|
|
|
|
|
: user_name));
|
2013-08-12 00:36:18 +00:00
|
|
|
|
|
|
|
|
|
Poco::ScopedLock<Poco::FastMutex> lock(mutex);
|
|
|
|
|
|
|
|
|
|
Container::iterator it = quota_for_keys.find(quota_key_hashed);
|
|
|
|
|
if (quota_for_keys.end() == it)
|
|
|
|
|
{
|
2014-02-13 07:17:22 +00:00
|
|
|
|
it = quota_for_keys.insert(std::make_pair(quota_key_hashed, new QuotaForIntervals(max))).first;
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return it->second;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
void Quotas::loadFromConfig(Poco::Util::AbstractConfiguration & config)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
Poco::Util::AbstractConfiguration::Keys config_keys;
|
|
|
|
|
config.keys("quotas", config_keys);
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
/// Удалим ключи, которых больше нет в кофиге.
|
|
|
|
|
std::set<std::string> keys_set(config_keys.begin(), config_keys.end());
|
|
|
|
|
for (Container::iterator it = cont.begin(); it != cont.end();)
|
|
|
|
|
{
|
|
|
|
|
if (keys_set.count(it->first))
|
|
|
|
|
++it;
|
|
|
|
|
else
|
|
|
|
|
cont.erase(it++);
|
|
|
|
|
}
|
|
|
|
|
|
2013-08-12 00:36:18 +00:00
|
|
|
|
for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it)
|
|
|
|
|
{
|
2014-02-13 07:17:22 +00:00
|
|
|
|
if (!cont[*it])
|
|
|
|
|
cont[*it] = new Quota();
|
|
|
|
|
cont[*it]->loadFromConfig("quotas." + *it, *it, config);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-02-13 07:17:22 +00:00
|
|
|
|
QuotaForIntervalsPtr Quotas::get(const String & name, const String & quota_key, const String & user_name, const Poco::Net::IPAddress & ip)
|
2013-08-12 00:36:18 +00:00
|
|
|
|
{
|
|
|
|
|
Container::iterator it = cont.find(name);
|
|
|
|
|
if (cont.end() == it)
|
|
|
|
|
throw Exception("Unknown quota " + name, ErrorCodes::UNKNOWN_QUOTA);
|
|
|
|
|
|
2013-11-03 00:24:46 +00:00
|
|
|
|
return it->second->get(quota_key, user_name, ip);
|
2013-08-12 00:36:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|