diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index cefa3712997..29d186def2d 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 76ea69cc737..7103769d54e 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Access/AccessControlManager.cpp b/dbms/src/Access/AccessControlManager.cpp new file mode 100644 index 00000000000..1f1a57816a8 --- /dev/null +++ b/dbms/src/Access/AccessControlManager.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + std::vector> createStorages() + { + std::vector> list; + list.emplace_back(std::make_unique()); + list.emplace_back(std::make_unique()); + return list; + } +} + + +AccessControlManager::AccessControlManager() + : MultipleAccessStorage(createStorages()), + quota_context_factory(std::make_unique(*this)) +{ +} + + +AccessControlManager::~AccessControlManager() +{ +} + + +void AccessControlManager::loadFromConfig(const Poco::Util::AbstractConfiguration & users_config) +{ + auto & users_config_access_storage = dynamic_cast(getStorageByIndex(1)); + users_config_access_storage.loadFromConfig(users_config); +} + + +std::shared_ptr AccessControlManager::createQuotaContext( + const String & user_name, const Poco::Net::IPAddress & address, const String & custom_quota_key) +{ + return quota_context_factory->createContext(user_name, address, custom_quota_key); +} + + +std::vector AccessControlManager::getQuotaUsageInfo() const +{ + return quota_context_factory->getUsageInfo(); +} +} diff --git a/dbms/src/Access/AccessControlManager.h b/dbms/src/Access/AccessControlManager.h new file mode 100644 index 00000000000..2133717d676 --- /dev/null +++ b/dbms/src/Access/AccessControlManager.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + + +namespace Poco +{ + namespace Net + { + class IPAddress; + } + namespace Util + { + class AbstractConfiguration; + } +} + +namespace DB +{ +class QuotaContext; +class QuotaContextFactory; +struct QuotaUsageInfo; + + +/// Manages access control entities. +class AccessControlManager : public MultipleAccessStorage +{ +public: + AccessControlManager(); + ~AccessControlManager(); + + void loadFromConfig(const Poco::Util::AbstractConfiguration & users_config); + + std::shared_ptr + createQuotaContext(const String & user_name, const Poco::Net::IPAddress & address, const String & custom_quota_key); + + std::vector getQuotaUsageInfo() const; + +private: + std::unique_ptr quota_context_factory; +}; + +} diff --git a/dbms/src/Access/IAccessEntity.cpp b/dbms/src/Access/IAccessEntity.cpp new file mode 100644 index 00000000000..6a2f928ae9e --- /dev/null +++ b/dbms/src/Access/IAccessEntity.cpp @@ -0,0 +1,19 @@ +#include +#include +#include + + +namespace DB +{ +String IAccessEntity::getTypeName(std::type_index type) +{ + if (type == typeid(Quota)) + return "Quota"; + return demangle(type.name()); +} + +bool IAccessEntity::equal(const IAccessEntity & other) const +{ + return (full_name == other.full_name) && (getType() == other.getType()); +} +} diff --git a/dbms/src/Access/IAccessEntity.h b/dbms/src/Access/IAccessEntity.h new file mode 100644 index 00000000000..272fde006ac --- /dev/null +++ b/dbms/src/Access/IAccessEntity.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ +/// Access entity is a set of data which have a name and a type. Access entity control something related to the access control. +/// Entities can be stored to a file or another storage, see IAccessStorage. +struct IAccessEntity +{ + IAccessEntity() = default; + IAccessEntity(const IAccessEntity &) = default; + virtual ~IAccessEntity() = default; + virtual std::shared_ptr clone() const = 0; + + std::type_index getType() const { return typeid(*this); } + static String getTypeName(std::type_index type); + const String getTypeName() const { return getTypeName(getType()); } + + template + bool isTypeOf() const { return isTypeOf(typeid(EntityType)); } + bool isTypeOf(std::type_index type) const { return type == getType(); } + + virtual void setName(const String & name_) { full_name = name_; } + virtual String getName() const { return full_name; } + String getFullName() const { return full_name; } + + friend bool operator ==(const IAccessEntity & lhs, const IAccessEntity & rhs) { return lhs.equal(rhs); } + friend bool operator !=(const IAccessEntity & lhs, const IAccessEntity & rhs) { return !(lhs == rhs); } + +protected: + String full_name; + + virtual bool equal(const IAccessEntity & other) const; + + /// Helper function to define clone() in the derived classes. + template + std::shared_ptr cloneImpl() const + { + return std::make_shared(typeid_cast(*this)); + } +}; + +using AccessEntityPtr = std::shared_ptr; +} diff --git a/dbms/src/Access/IAccessStorage.cpp b/dbms/src/Access/IAccessStorage.cpp new file mode 100644 index 00000000000..4283ec9e6dc --- /dev/null +++ b/dbms/src/Access/IAccessStorage.cpp @@ -0,0 +1,450 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_CAST; + extern const int ACCESS_ENTITY_NOT_FOUND; + extern const int ACCESS_ENTITY_ALREADY_EXISTS; + extern const int ACCESS_ENTITY_FOUND_DUPLICATES; + extern const int ACCESS_ENTITY_STORAGE_READONLY; +} + + +std::vector IAccessStorage::findAll(std::type_index type) const +{ + return findAllImpl(type); +} + + +std::optional IAccessStorage::find(std::type_index type, const String & name) const +{ + return findImpl(type, name); +} + + +std::vector IAccessStorage::find(std::type_index type, const Strings & names) const +{ + std::vector ids; + ids.reserve(names.size()); + for (const String & name : names) + { + auto id = findImpl(type, name); + if (id) + ids.push_back(*id); + } + return ids; +} + + +UUID IAccessStorage::getID(std::type_index type, const String & name) const +{ + auto id = findImpl(type, name); + if (id) + return *id; + throwNotFound(type, name); +} + + +std::vector IAccessStorage::getIDs(std::type_index type, const Strings & names) const +{ + std::vector ids; + ids.reserve(names.size()); + for (const String & name : names) + ids.push_back(getID(type, name)); + return ids; +} + + +bool IAccessStorage::exists(const UUID & id) const +{ + return existsImpl(id); +} + + + +AccessEntityPtr IAccessStorage::tryReadBase(const UUID & id) const +{ + try + { + return readImpl(id); + } + catch (Exception &) + { + return nullptr; + } +} + + +String IAccessStorage::readName(const UUID & id) const +{ + return readNameImpl(id); +} + + +std::optional IAccessStorage::tryReadName(const UUID & id) const +{ + try + { + return readNameImpl(id); + } + catch (Exception &) + { + return {}; + } +} + + +UUID IAccessStorage::insert(const AccessEntityPtr & entity) +{ + return insertImpl(entity, false); +} + + +std::vector IAccessStorage::insert(const std::vector & multiple_entities) +{ + std::vector ids; + ids.reserve(multiple_entities.size()); + String error_message; + for (const auto & entity : multiple_entities) + { + try + { + ids.push_back(insertImpl(entity, false)); + } + catch (Exception & e) + { + if (e.code() != ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS) + throw; + error_message += (error_message.empty() ? "" : ". ") + e.message(); + } + } + if (!error_message.empty()) + throw Exception(error_message, ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); + return ids; +} + + +std::optional IAccessStorage::tryInsert(const AccessEntityPtr & entity) +{ + try + { + return insertImpl(entity, false); + } + catch (Exception &) + { + return {}; + } +} + + +std::vector IAccessStorage::tryInsert(const std::vector & multiple_entities) +{ + std::vector ids; + ids.reserve(multiple_entities.size()); + for (const auto & entity : multiple_entities) + { + try + { + ids.push_back(insertImpl(entity, false)); + } + catch (Exception &) + { + } + } + return ids; +} + + +UUID IAccessStorage::insertOrReplace(const AccessEntityPtr & entity) +{ + return insertImpl(entity, true); +} + + +std::vector IAccessStorage::insertOrReplace(const std::vector & multiple_entities) +{ + std::vector ids; + ids.reserve(multiple_entities.size()); + for (const auto & entity : multiple_entities) + ids.push_back(insertImpl(entity, true)); + return ids; +} + + +void IAccessStorage::remove(const UUID & id) +{ + removeImpl(id); +} + + +void IAccessStorage::remove(const std::vector & ids) +{ + String error_message; + for (const auto & id : ids) + { + try + { + removeImpl(id); + } + catch (Exception & e) + { + if (e.code() != ErrorCodes::ACCESS_ENTITY_NOT_FOUND) + throw; + error_message += (error_message.empty() ? "" : ". ") + e.message(); + } + } + if (!error_message.empty()) + throw Exception(error_message, ErrorCodes::ACCESS_ENTITY_NOT_FOUND); +} + + +bool IAccessStorage::tryRemove(const UUID & id) +{ + try + { + removeImpl(id); + return true; + } + catch (Exception &) + { + return false; + } +} + + +std::vector IAccessStorage::tryRemove(const std::vector & ids) +{ + std::vector removed; + removed.reserve(ids.size()); + for (const auto & id : ids) + { + try + { + removeImpl(id); + removed.push_back(id); + } + catch (Exception &) + { + } + } + return removed; +} + + +void IAccessStorage::update(const UUID & id, const UpdateFunc & update_func) +{ + updateImpl(id, update_func); +} + + +void IAccessStorage::update(const std::vector & ids, const UpdateFunc & update_func) +{ + String error_message; + for (const auto & id : ids) + { + try + { + updateImpl(id, update_func); + } + catch (Exception & e) + { + if (e.code() != ErrorCodes::ACCESS_ENTITY_NOT_FOUND) + throw; + error_message += (error_message.empty() ? "" : ". ") + e.message(); + } + } + if (!error_message.empty()) + throw Exception(error_message, ErrorCodes::ACCESS_ENTITY_NOT_FOUND); +} + + +bool IAccessStorage::tryUpdate(const UUID & id, const UpdateFunc & update_func) +{ + try + { + updateImpl(id, update_func); + return true; + } + catch (Exception &) + { + return false; + } +} + + +std::vector IAccessStorage::tryUpdate(const std::vector & ids, const UpdateFunc & update_func) +{ + std::vector updated; + updated.reserve(ids.size()); + for (const auto & id : ids) + { + try + { + updateImpl(id, update_func); + updated.push_back(id); + } + catch (Exception &) + { + } + } + return updated; +} + + +IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const +{ + return subscribeForChangesImpl(type, handler); +} + + +IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const +{ + return subscribeForChangesImpl(id, handler); +} + + +IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) const +{ + if (ids.empty()) + return nullptr; + if (ids.size() == 1) + return subscribeForChangesImpl(ids[0], handler); + + std::vector subscriptions; + subscriptions.reserve(ids.size()); + for (const auto & id : ids) + { + auto subscription = subscribeForChangesImpl(id, handler); + if (subscription) + subscriptions.push_back(std::move(subscription)); + } + + class SubscriptionImpl : public Subscription + { + public: + SubscriptionImpl(std::vector subscriptions_) + : subscriptions(std::move(subscriptions_)) {} + private: + std::vector subscriptions; + }; + + return std::make_unique(std::move(subscriptions)); +} + + +bool IAccessStorage::hasSubscription(std::type_index type) const +{ + return hasSubscriptionImpl(type); +} + + +bool IAccessStorage::hasSubscription(const UUID & id) const +{ + return hasSubscriptionImpl(id); +} + + +void IAccessStorage::notify(const Notifications & notifications) +{ + for (const auto & [fn, id, new_entity] : notifications) + fn(id, new_entity); +} + + +UUID IAccessStorage::generateRandomID() +{ + static Poco::UUIDGenerator generator; + UUID id; + generator.createRandom().copyTo(reinterpret_cast(&id)); + return id; +} + + +Poco::Logger * IAccessStorage::getLogger() const +{ + Poco::Logger * ptr = log.load(); + if (!ptr) + log.store(ptr = &Poco::Logger::get("Access(" + storage_name + ")"), std::memory_order_relaxed); + return ptr; +} + + +void IAccessStorage::throwNotFound(const UUID & id) const +{ + throw Exception("ID {" + toString(id) + "} not found in " + getStorageName(), ErrorCodes::ACCESS_ENTITY_NOT_FOUND); +} + + +void IAccessStorage::throwNotFound(std::type_index type, const String & name) const +{ + throw Exception( + getTypeName(type) + " " + backQuote(name) + " not found in " + getStorageName(), ErrorCodes::ACCESS_ENTITY_NOT_FOUND); +} + + +void IAccessStorage::throwBadCast(const UUID & id, std::type_index type, const String & name, std::type_index required_type) const +{ + throw Exception( + "ID {" + toString(id) + "}: " + getTypeName(type) + backQuote(name) + " expected to be of type " + getTypeName(required_type), + ErrorCodes::BAD_CAST); +} + + +void IAccessStorage::throwIDCollisionCannotInsert(const UUID & id, std::type_index type, const String & name, std::type_index existing_type, const String & existing_name) const +{ + throw Exception( + getTypeName(type) + " " + backQuote(name) + ": cannot insert because the ID {" + toString(id) + "} is already used by " + + getTypeName(existing_type) + " " + backQuote(existing_name) + " in " + getStorageName(), + ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); +} + + +void IAccessStorage::throwNameCollisionCannotInsert(std::type_index type, const String & name) const +{ + throw Exception( + getTypeName(type) + " " + backQuote(name) + ": cannot insert because " + getTypeName(type) + " " + backQuote(name) + + " already exists in " + getStorageName(), + ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); +} + + +void IAccessStorage::throwNameCollisionCannotRename(std::type_index type, const String & old_name, const String & new_name) const +{ + throw Exception( + getTypeName(type) + " " + backQuote(old_name) + ": cannot rename to " + backQuote(new_name) + " because " + getTypeName(type) + " " + + backQuote(new_name) + " already exists in " + getStorageName(), + ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); +} + + +void IAccessStorage::throwReadonlyCannotInsert(std::type_index type, const String & name) const +{ + throw Exception( + "Cannot insert " + getTypeName(type) + " " + backQuote(name) + " to " + getStorageName() + " because this storage is readonly", + ErrorCodes::ACCESS_ENTITY_STORAGE_READONLY); +} + + +void IAccessStorage::throwReadonlyCannotUpdate(std::type_index type, const String & name) const +{ + throw Exception( + "Cannot update " + getTypeName(type) + " " + backQuote(name) + " in " + getStorageName() + " because this storage is readonly", + ErrorCodes::ACCESS_ENTITY_STORAGE_READONLY); +} + + +void IAccessStorage::throwReadonlyCannotRemove(std::type_index type, const String & name) const +{ + throw Exception( + "Cannot remove " + getTypeName(type) + " " + backQuote(name) + " from " + getStorageName() + " because this storage is readonly", + ErrorCodes::ACCESS_ENTITY_STORAGE_READONLY); +} +} diff --git a/dbms/src/Access/IAccessStorage.h b/dbms/src/Access/IAccessStorage.h new file mode 100644 index 00000000000..b4153bce87d --- /dev/null +++ b/dbms/src/Access/IAccessStorage.h @@ -0,0 +1,209 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + + +namespace Poco { class Logger; } + +namespace DB +{ +/// Contains entities, i.e. instances of classes derived from IAccessEntity. +/// The implementations of this class MUST be thread-safe. +class IAccessStorage +{ +public: + IAccessStorage(const String & storage_name_) : storage_name(storage_name_) {} + virtual ~IAccessStorage() {} + + /// Returns the name of this storage. + const String & getStorageName() const { return storage_name; } + + /// Returns the identifiers of all the entities of a specified type contained in the storage. + std::vector findAll(std::type_index type) const; + + template + std::vector findAll() const { return findAll(typeid(EntityType)); } + + /// Searchs for an entity with specified type and name. Returns std::nullopt if not found. + std::optional find(std::type_index type, const String & name) const; + + template + std::optional find(const String & name) const { return find(typeid(EntityType), name); } + + std::vector find(std::type_index type, const Strings & names) const; + + template + std::vector find(const Strings & names) const { return find(typeid(EntityType), names); } + + /// Searchs for an entity with specified name and type. Throws an exception if not found. + UUID getID(std::type_index type, const String & name) const; + + template + UUID getID(const String & name) const { return getID(typeid(EntityType), name); } + + std::vector getIDs(std::type_index type, const Strings & names) const; + + template + std::vector getIDs(const Strings & names) const { return getIDs(typeid(EntityType), names); } + + /// Returns whether there is an entity with such identifier in the storage. + bool exists(const UUID & id) const; + + /// Reads an entity. Throws an exception if not found. + template + std::shared_ptr read(const UUID & id) const; + + template + std::shared_ptr read(const String & name) const; + + /// Reads an entity. Returns nullptr if not found. + template + std::shared_ptr tryRead(const UUID & id) const; + + template + std::shared_ptr tryRead(const String & name) const; + + /// Reads only name of an entity. + String readName(const UUID & id) const; + std::optional tryReadName(const UUID & id) const; + + /// Inserts an entity to the storage. Returns ID of a new entry in the storage. + /// Throws an exception if the specified name already exists. + UUID insert(const AccessEntityPtr & entity); + std::vector insert(const std::vector & multiple_entities); + + /// Inserts an entity to the storage. Returns ID of a new entry in the storage. + std::optional tryInsert(const AccessEntityPtr & entity); + std::vector tryInsert(const std::vector & multiple_entities); + + /// Inserts an entity to the storage. Return ID of a new entry in the storage. + /// Replaces an existing entry in the storage if the specified name already exists. + UUID insertOrReplace(const AccessEntityPtr & entity); + std::vector insertOrReplace(const std::vector & multiple_entities); + + /// Removes an entity from the storage. Throws an exception if couldn't remove. + void remove(const UUID & id); + void remove(const std::vector & ids); + + /// Removes an entity from the storage. Returns false if couldn't remove. + bool tryRemove(const UUID & id); + + /// Removes multiple entities from the storage. Returns the list of successfully dropped. + std::vector tryRemove(const std::vector & ids); + + using UpdateFunc = std::function; + + /// Updates an entity stored in the storage. Throws an exception if couldn't update. + void update(const UUID & id, const UpdateFunc & update_func); + void update(const std::vector & ids, const UpdateFunc & update_func); + + /// Updates an entity stored in the storage. Returns false if couldn't update. + bool tryUpdate(const UUID & id, const UpdateFunc & update_func); + + /// Updates multiple entities in the storage. Returns the list of successfully updated. + std::vector tryUpdate(const std::vector & ids, const UpdateFunc & update_func); + + class Subscription + { + public: + virtual ~Subscription() {} + }; + + using SubscriptionPtr = std::unique_ptr; + using OnChangedHandler = std::function; + + /// Subscribes for all changes. + /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). + SubscriptionPtr subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const; + + template + SubscriptionPtr subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(typeid(EntityType), handler); } + + /// Subscribes for changes of a specific entry. + /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). + SubscriptionPtr subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const; + SubscriptionPtr subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) const; + + bool hasSubscription(std::type_index type) const; + bool hasSubscription(const UUID & id) const; + +protected: + virtual std::optional findImpl(std::type_index type, const String & name) const = 0; + virtual std::vector findAllImpl(std::type_index type) const = 0; + virtual bool existsImpl(const UUID & id) const = 0; + virtual AccessEntityPtr readImpl(const UUID & id) const = 0; + virtual String readNameImpl(const UUID & id) const = 0; + virtual UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) = 0; + virtual void removeImpl(const UUID & id) = 0; + virtual void updateImpl(const UUID & id, const UpdateFunc & update_func) = 0; + virtual SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0; + virtual SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const = 0; + virtual bool hasSubscriptionImpl(const UUID & id) const = 0; + virtual bool hasSubscriptionImpl(std::type_index type) const = 0; + + static UUID generateRandomID(); + Poco::Logger * getLogger() const; + static String getTypeName(std::type_index type) { return IAccessEntity::getTypeName(type); } + [[noreturn]] void throwNotFound(const UUID & id) const; + [[noreturn]] void throwNotFound(std::type_index type, const String & name) const; + [[noreturn]] void throwBadCast(const UUID & id, std::type_index type, const String & name, std::type_index required_type) const; + [[noreturn]] void throwIDCollisionCannotInsert(const UUID & id, std::type_index type, const String & name, std::type_index existing_type, const String & existing_name) const; + [[noreturn]] void throwNameCollisionCannotInsert(std::type_index type, const String & name) const; + [[noreturn]] void throwNameCollisionCannotRename(std::type_index type, const String & old_name, const String & new_name) const; + [[noreturn]] void throwReadonlyCannotInsert(std::type_index type, const String & name) const; + [[noreturn]] void throwReadonlyCannotUpdate(std::type_index type, const String & name) const; + [[noreturn]] void throwReadonlyCannotRemove(std::type_index type, const String & name) const; + + using Notification = std::tuple; + using Notifications = std::vector; + static void notify(const Notifications & notifications); + +private: + AccessEntityPtr tryReadBase(const UUID & id) const; + + const String storage_name; + mutable std::atomic log = nullptr; +}; + + +template +std::shared_ptr IAccessStorage::read(const UUID & id) const +{ + auto entity = readImpl(id); + auto ptr = typeid_cast>(entity); + if (ptr) + return ptr; + throwBadCast(id, entity->getType(), entity->getFullName(), typeid(EntityType)); +} + + +template +std::shared_ptr IAccessStorage::read(const String & name) const +{ + return read(getID(name)); +} + + +template +std::shared_ptr IAccessStorage::tryRead(const UUID & id) const +{ + auto entity = tryReadBase(id); + if (!entity) + return nullptr; + return typeid_cast>(entity); +} + + +template +std::shared_ptr IAccessStorage::tryRead(const String & name) const +{ + auto id = find(name); + return id ? tryRead(*id) : nullptr; +} +} diff --git a/dbms/src/Access/MemoryAccessStorage.cpp b/dbms/src/Access/MemoryAccessStorage.cpp new file mode 100644 index 00000000000..ed42acca1a7 --- /dev/null +++ b/dbms/src/Access/MemoryAccessStorage.cpp @@ -0,0 +1,358 @@ +#include +#include +#include + + +namespace DB +{ +MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_) + : IAccessStorage(storage_name_), shared_ptr_to_this{std::make_shared(this)} +{ +} + + +MemoryAccessStorage::~MemoryAccessStorage() {} + + +std::optional MemoryAccessStorage::findImpl(std::type_index type, const String & name) const +{ + std::lock_guard lock{mutex}; + auto it = names.find({name, type}); + if (it == names.end()) + return {}; + + Entry & entry = *(it->second); + return entry.id; +} + + +std::vector MemoryAccessStorage::findAllImpl(std::type_index type) const +{ + std::lock_guard lock{mutex}; + std::vector result; + result.reserve(entries.size()); + for (const auto & [id, entry] : entries) + if (entry.entity->isTypeOf(type)) + result.emplace_back(id); + return result; +} + + +bool MemoryAccessStorage::existsImpl(const UUID & id) const +{ + std::lock_guard lock{mutex}; + return entries.count(id); +} + + +AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id) const +{ + std::lock_guard lock{mutex}; + auto it = entries.find(id); + if (it == entries.end()) + throwNotFound(id); + const Entry & entry = it->second; + return entry.entity; +} + + +String MemoryAccessStorage::readNameImpl(const UUID & id) const +{ + return readImpl(id)->getFullName(); +} + + +UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists) +{ + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + + UUID id = generateRandomID(); + std::lock_guard lock{mutex}; + insertNoLock(generateRandomID(), new_entity, replace_if_exists, notifications); + return id; +} + + +void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications) +{ + const String & name = new_entity->getFullName(); + std::type_index type = new_entity->getType(); + + /// Check that we can insert. + auto it = entries.find(id); + if (it != entries.end()) + { + const auto & existing_entry = it->second; + throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getFullName()); + } + + auto it2 = names.find({name, type}); + if (it2 != names.end()) + { + const auto & existing_entry = *(it2->second); + if (replace_if_exists) + removeNoLock(existing_entry.id, notifications); + else + throwNameCollisionCannotInsert(type, name); + } + + /// Do insertion. + auto & entry = entries[id]; + entry.id = id; + entry.entity = new_entity; + names[std::pair{name, type}] = &entry; + prepareNotifications(entry, false, notifications); +} + + +void MemoryAccessStorage::removeImpl(const UUID & id) +{ + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + + std::lock_guard lock{mutex}; + removeNoLock(id, notifications); +} + + +void MemoryAccessStorage::removeNoLock(const UUID & id, Notifications & notifications) +{ + auto it = entries.find(id); + if (it == entries.end()) + throwNotFound(id); + + Entry & entry = it->second; + const String & name = entry.entity->getFullName(); + std::type_index type = entry.entity->getType(); + + prepareNotifications(entry, true, notifications); + + /// Do removing. + names.erase({name, type}); + entries.erase(it); +} + + +void MemoryAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func) +{ + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + + std::lock_guard lock{mutex}; + updateNoLock(id, update_func, notifications); +} + + +void MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications) +{ + auto it = entries.find(id); + if (it == entries.end()) + throwNotFound(id); + + Entry & entry = it->second; + auto old_entity = entry.entity; + auto new_entity = update_func(old_entity); + + if (*new_entity == *old_entity) + return; + + entry.entity = new_entity; + + if (new_entity->getFullName() != old_entity->getFullName()) + { + auto it2 = names.find({new_entity->getFullName(), new_entity->getType()}); + if (it2 != names.end()) + throwNameCollisionCannotRename(old_entity->getType(), old_entity->getFullName(), new_entity->getFullName()); + + names.erase({old_entity->getFullName(), old_entity->getType()}); + names[std::pair{new_entity->getFullName(), new_entity->getType()}] = &entry; + } + + prepareNotifications(entry, false, notifications); +} + + +void MemoryAccessStorage::setAll(const std::vector & all_entities) +{ + std::vector> entities_with_ids; + entities_with_ids.reserve(all_entities.size()); + for (const auto & entity : all_entities) + entities_with_ids.emplace_back(generateRandomID(), entity); + setAll(entities_with_ids); +} + + +void MemoryAccessStorage::setAll(const std::vector> & all_entities) +{ + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + + std::lock_guard lock{mutex}; + setAllNoLock(all_entities, notifications); +} + + +void MemoryAccessStorage::setAllNoLock(const std::vector> & all_entities, Notifications & notifications) +{ + /// Get list of the currently used IDs. Later we will remove those of them which are not used anymore. + std::unordered_set not_used_ids; + for (const auto & id_and_entry : entries) + not_used_ids.emplace(id_and_entry.first); + + /// Remove conflicting entities. + for (const auto & [id, entity] : all_entities) + { + auto it = entries.find(id); + if (it != entries.end()) + { + not_used_ids.erase(id); /// ID is used. + Entry & entry = it->second; + if (entry.entity->getType() != entity->getType()) + { + removeNoLock(id, notifications); + continue; + } + } + auto it2 = names.find({entity->getFullName(), entity->getType()}); + if (it2 != names.end()) + { + Entry & entry = *(it2->second); + if (entry.id != id) + removeNoLock(id, notifications); + } + } + + /// Remove entities which are not used anymore. + for (const auto & id : not_used_ids) + removeNoLock(id, notifications); + + /// Insert or update entities. + for (const auto & [id, entity] : all_entities) + { + auto it = entries.find(id); + if (it != entries.end()) + { + if (*(it->second.entity) != *entity) + { + const AccessEntityPtr & changed_entity = entity; + updateNoLock(id, [&changed_entity](const AccessEntityPtr &) { return changed_entity; }, notifications); + } + } + else + insertNoLock(id, entity, false, notifications); + } +} + + +void MemoryAccessStorage::prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const +{ + for (const auto & handler : entry.handlers_by_id) + notifications.push_back({handler, entry.id, remove ? nullptr : entry.entity}); + + auto range = handlers_by_type.equal_range(entry.entity->getType()); + for (auto it = range.first; it != range.second; ++it) + notifications.push_back({it->second, entry.id, remove ? nullptr : entry.entity}); +} + + +IAccessStorage::SubscriptionPtr MemoryAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +{ + class SubscriptionImpl : public Subscription + { + public: + SubscriptionImpl( + const MemoryAccessStorage & storage_, + std::type_index type_, + const OnChangedHandler & handler_) + : storage_weak(storage_.shared_ptr_to_this) + { + std::lock_guard lock{storage_.mutex}; + handler_it = storage_.handlers_by_type.emplace(type_, handler_); + } + + ~SubscriptionImpl() override + { + auto storage = storage_weak.lock(); + if (storage) + { + std::lock_guard lock{(*storage)->mutex}; + (*storage)->handlers_by_type.erase(handler_it); + } + } + + private: + std::weak_ptr storage_weak; + std::unordered_multimap::iterator handler_it; + }; + + return std::make_unique(*this, type, handler); +} + + +IAccessStorage::SubscriptionPtr MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const +{ + class SubscriptionImpl : public Subscription + { + public: + SubscriptionImpl( + const MemoryAccessStorage & storage_, + const UUID & id_, + const OnChangedHandler & handler_) + : storage_weak(storage_.shared_ptr_to_this), + id(id_) + { + std::lock_guard lock{storage_.mutex}; + auto it = storage_.entries.find(id); + if (it == storage_.entries.end()) + { + storage_weak.reset(); + return; + } + const Entry & entry = it->second; + handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler_); + } + + ~SubscriptionImpl() override + { + auto storage = storage_weak.lock(); + if (storage) + { + std::lock_guard lock{(*storage)->mutex}; + auto it = (*storage)->entries.find(id); + if (it != (*storage)->entries.end()) + { + const Entry & entry = it->second; + entry.handlers_by_id.erase(handler_it); + } + } + } + + private: + std::weak_ptr storage_weak; + UUID id; + std::list::iterator handler_it; + }; + + return std::make_unique(*this, id, handler); +} + + +bool MemoryAccessStorage::hasSubscriptionImpl(const UUID & id) const +{ + auto it = entries.find(id); + if (it != entries.end()) + { + const Entry & entry = it->second; + return !entry.handlers_by_id.empty(); + } + return false; +} + + +bool MemoryAccessStorage::hasSubscriptionImpl(std::type_index type) const +{ + auto range = handlers_by_type.equal_range(type); + return range.first != range.second; +} +} diff --git a/dbms/src/Access/MemoryAccessStorage.h b/dbms/src/Access/MemoryAccessStorage.h new file mode 100644 index 00000000000..51c55487ca7 --- /dev/null +++ b/dbms/src/Access/MemoryAccessStorage.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace DB +{ +/// Implementation of IAccessStorage which keeps all data in memory. +class MemoryAccessStorage : public IAccessStorage +{ +public: + MemoryAccessStorage(const String & storage_name_ = "memory"); + ~MemoryAccessStorage() override; + + /// Sets all entities at once. + void setAll(const std::vector & all_entities); + void setAll(const std::vector> & all_entities); + +private: + std::optional findImpl(std::type_index type, const String & name) const override; + std::vector findAllImpl(std::type_index type) const override; + bool existsImpl(const UUID & id) const override; + AccessEntityPtr readImpl(const UUID & id) const override; + String readNameImpl(const UUID & id) const override; + UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override; + void removeImpl(const UUID & id) override; + void updateImpl(const UUID & id, const UpdateFunc & update_func) override; + SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; + SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + bool hasSubscriptionImpl(const UUID & id) const override; + bool hasSubscriptionImpl(std::type_index type) const override; + + struct Entry + { + UUID id; + AccessEntityPtr entity; + mutable std::list handlers_by_id; + }; + + void insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, Notifications & notifications); + void removeNoLock(const UUID & id, Notifications & notifications); + void updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications); + void setAllNoLock(const std::vector> & all_entities, Notifications & notifications); + void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const; + + using NameTypePair = std::pair; + struct Hash + { + size_t operator()(const NameTypePair & key) const + { + return std::hash{}(key.first) - std::hash{}(key.second); + } + }; + + mutable std::mutex mutex; + std::unordered_map entries; /// We want to search entries both by ID and by the pair of name and type. + std::unordered_map names; /// and by the pair of name and type. + mutable std::unordered_multimap handlers_by_type; + std::shared_ptr shared_ptr_to_this; /// We need weak pointers to `this` to implement subscriptions. +}; +} diff --git a/dbms/src/Access/MultipleAccessStorage.cpp b/dbms/src/Access/MultipleAccessStorage.cpp new file mode 100644 index 00000000000..f3db0b0fbbc --- /dev/null +++ b/dbms/src/Access/MultipleAccessStorage.cpp @@ -0,0 +1,246 @@ +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ACCESS_ENTITY_NOT_FOUND; + extern const int ACCESS_ENTITY_FOUND_DUPLICATES; +} + + +namespace +{ + template + String joinStorageNames(const std::vector & storages) + { + String result; + for (const auto & storage : storages) + { + if (!result.empty()) + result += ", "; + result += storage->getStorageName(); + } + return result; + } +} + + +MultipleAccessStorage::MultipleAccessStorage( + std::vector> nested_storages_, size_t index_of_nested_storage_for_insertion_) + : IAccessStorage(joinStorageNames(nested_storages_)) + , nested_storages(std::move(nested_storages_)) + , nested_storage_for_insertion(nested_storages[index_of_nested_storage_for_insertion_].get()) + , ids_cache(512 /* cache size */) +{ +} + + +MultipleAccessStorage::~MultipleAccessStorage() +{ +} + + +std::vector MultipleAccessStorage::findMultiple(std::type_index type, const String & name) const +{ + std::vector ids; + for (const auto & nested_storage : nested_storages) + { + auto id = nested_storage->find(type, name); + if (id) + { + std::lock_guard lock{ids_cache_mutex}; + ids_cache.set(*id, std::make_shared(nested_storage.get())); + ids.push_back(*id); + } + } + return ids; +} + + +std::optional MultipleAccessStorage::findImpl(std::type_index type, const String & name) const +{ + auto ids = findMultiple(type, name); + if (ids.empty()) + return {}; + if (ids.size() == 1) + return ids[0]; + + std::vector storages_with_duplicates; + for (const auto & id : ids) + { + auto * storage = findStorage(id); + if (storage) + storages_with_duplicates.push_back(storage); + } + + throw Exception( + "Found " + getTypeName(type) + " " + backQuote(name) + " in " + std::to_string(ids.size()) + + " storages: " + joinStorageNames(storages_with_duplicates), + ErrorCodes::ACCESS_ENTITY_FOUND_DUPLICATES); +} + + +std::vector MultipleAccessStorage::findAllImpl(std::type_index type) const +{ + std::vector all_ids; + for (const auto & nested_storage : nested_storages) + { + auto ids = nested_storage->findAll(type); + all_ids.insert(all_ids.end(), std::make_move_iterator(ids.begin()), std::make_move_iterator(ids.end())); + } + return all_ids; +} + + +bool MultipleAccessStorage::existsImpl(const UUID & id) const +{ + return findStorage(id) != nullptr; +} + + +IAccessStorage * MultipleAccessStorage::findStorage(const UUID & id) +{ + { + std::lock_guard lock{ids_cache_mutex}; + auto from_cache = ids_cache.get(id); + if (from_cache) + { + auto * storage = *from_cache; + if (storage->exists(id)) + return storage; + } + } + + for (const auto & nested_storage : nested_storages) + { + if (nested_storage->exists(id)) + { + std::lock_guard lock{ids_cache_mutex}; + ids_cache.set(id, std::make_shared(nested_storage.get())); + return nested_storage.get(); + } + } + + return nullptr; +} + + +const IAccessStorage * MultipleAccessStorage::findStorage(const UUID & id) const +{ + return const_cast(this)->findStorage(id); +} + + +IAccessStorage & MultipleAccessStorage::getStorage(const UUID & id) +{ + auto * storage = findStorage(id); + if (storage) + return *storage; + throwNotFound(id); +} + + +const IAccessStorage & MultipleAccessStorage::getStorage(const UUID & id) const +{ + return const_cast(this)->getStorage(id); +} + + +AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id) const +{ + return getStorage(id).read(id); +} + + +String MultipleAccessStorage::readNameImpl(const UUID & id) const +{ + return getStorage(id).readName(id); +} + + +UUID MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) +{ + auto id = replace_if_exists ? nested_storage_for_insertion->insertOrReplace(entity) : nested_storage_for_insertion->insert(entity); + + std::lock_guard lock{ids_cache_mutex}; + ids_cache.set(id, std::make_shared(nested_storage_for_insertion)); + + return id; +} + + +void MultipleAccessStorage::removeImpl(const UUID & id) +{ + getStorage(id).remove(id); +} + + +void MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func) +{ + getStorage(id).update(id, update_func); +} + + +IAccessStorage::SubscriptionPtr MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const +{ + auto storage = findStorage(id); + if (!storage) + return nullptr; + return storage->subscribeForChanges(id, handler); +} + + +IAccessStorage::SubscriptionPtr MultipleAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +{ + std::vector subscriptions; + for (const auto & nested_storage : nested_storages) + { + auto subscription = nested_storage->subscribeForChanges(type, handler); + if (subscription) + subscriptions.emplace_back(std::move(subscription)); + } + + if (subscriptions.empty()) + return nullptr; + + if (subscriptions.size() == 1) + return std::move(subscriptions[0]); + + class SubscriptionImpl : public Subscription + { + public: + SubscriptionImpl(std::vector subscriptions_) + : subscriptions(std::move(subscriptions_)) {} + private: + std::vector subscriptions; + }; + + return std::make_unique(std::move(subscriptions)); +} + + +bool MultipleAccessStorage::hasSubscriptionImpl(const UUID & id) const +{ + for (const auto & nested_storage : nested_storages) + { + if (nested_storage->hasSubscription(id)) + return true; + } + return false; +} + + +bool MultipleAccessStorage::hasSubscriptionImpl(std::type_index type) const +{ + for (const auto & nested_storage : nested_storages) + { + if (nested_storage->hasSubscription(type)) + return true; + } + return false; +} +} diff --git a/dbms/src/Access/MultipleAccessStorage.h b/dbms/src/Access/MultipleAccessStorage.h new file mode 100644 index 00000000000..42e500a1851 --- /dev/null +++ b/dbms/src/Access/MultipleAccessStorage.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +/// Implementation of IAccessStorage which contains multiple nested storages. +class MultipleAccessStorage : public IAccessStorage +{ +public: + using Storage = IAccessStorage; + + MultipleAccessStorage(std::vector> nested_storages_, size_t index_of_nested_storage_for_insertion_ = 0); + ~MultipleAccessStorage() override; + + std::vector findMultiple(std::type_index type, const String & name) const; + + template + std::vector findMultiple(const String & name) const { return findMultiple(EntityType::TYPE, name); } + + const Storage * findStorage(const UUID & id) const; + Storage * findStorage(const UUID & id); + const Storage & getStorage(const UUID & id) const; + Storage & getStorage(const UUID & id); + + Storage & getStorageByIndex(size_t i) { return *(nested_storages[i]); } + const Storage & getStorageByIndex(size_t i) const { return *(nested_storages[i]); } + +protected: + std::optional findImpl(std::type_index type, const String & name) const override; + std::vector findAllImpl(std::type_index type) const override; + bool existsImpl(const UUID & id) const override; + AccessEntityPtr readImpl(const UUID & id) const override; + String readNameImpl(const UUID &id) const override; + UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override; + void removeImpl(const UUID & id) override; + void updateImpl(const UUID & id, const UpdateFunc & update_func) override; + SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; + SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + bool hasSubscriptionImpl(const UUID & id) const override; + bool hasSubscriptionImpl(std::type_index type) const override; + +private: + std::vector> nested_storages; + IAccessStorage * nested_storage_for_insertion; + mutable LRUCache ids_cache; + mutable std::mutex ids_cache_mutex; +}; + +} diff --git a/dbms/src/Access/Quota.cpp b/dbms/src/Access/Quota.cpp new file mode 100644 index 00000000000..d178307ca51 --- /dev/null +++ b/dbms/src/Access/Quota.cpp @@ -0,0 +1,46 @@ +#include +#include +#include + + +namespace DB +{ +Quota::Limits::Limits() +{ + boost::range::fill(max, 0); +} + + +bool operator ==(const Quota::Limits & lhs, const Quota::Limits & rhs) +{ + return boost::range::equal(lhs.max, rhs.max) && (lhs.duration == rhs.duration) + && (lhs.randomize_interval == rhs.randomize_interval); +} + + +bool Quota::equal(const IAccessEntity & other) const +{ + if (!IAccessEntity::equal(other)) + return false; + const auto & other_quota = typeid_cast(other); + return (all_limits == other_quota.all_limits) && (key_type == other_quota.key_type) && (roles == other_quota.roles) + && (all_roles == other_quota.all_roles) && (except_roles == other_quota.except_roles); +} + + +const char * Quota::resourceTypeToColumnName(ResourceType resource_type) +{ + switch (resource_type) + { + case Quota::QUERIES: return "queries"; + case Quota::ERRORS: return "errors"; + case Quota::RESULT_ROWS: return "result_rows"; + case Quota::RESULT_BYTES: return "result_bytes"; + case Quota::READ_ROWS: return "read_rows"; + case Quota::READ_BYTES: return "read_bytes"; + case Quota::EXECUTION_TIME: return "execution_time"; + } + __builtin_unreachable(); +} +} + diff --git a/dbms/src/Access/Quota.h b/dbms/src/Access/Quota.h new file mode 100644 index 00000000000..716bccbe1ff --- /dev/null +++ b/dbms/src/Access/Quota.h @@ -0,0 +1,141 @@ +#pragma once + +#include +#include + + + +namespace DB +{ +/** Quota for resources consumption for specific interval. + * Used to limit resource usage by user. + * Quota is applied "softly" - could be slightly exceed, because it is checked usually only on each block of processed data. + * Accumulated values are not persisted and are lost on server restart. + * Quota is local to server, + * but for distributed queries, accumulated values for read rows and bytes + * are collected from all participating servers and accumulated locally. + */ +struct Quota : public IAccessEntity +{ + enum ResourceType + { + QUERIES, /// Number of queries. + ERRORS, /// Number of queries with exceptions. + RESULT_ROWS, /// Number of rows returned as result. + RESULT_BYTES, /// Number of bytes returned as result. + READ_ROWS, /// Number of rows read from tables. + READ_BYTES, /// Number of bytes read from tables. + EXECUTION_TIME, /// Total amount of query execution time in nanoseconds. + }; + static constexpr size_t MAX_RESOURCE_TYPE = 7; + + using ResourceAmount = UInt64; + static constexpr ResourceAmount UNLIMITED = 0; /// 0 means unlimited. + + /// Amount of resources available to consume for each duration. + struct Limits + { + ResourceAmount max[MAX_RESOURCE_TYPE]; + std::chrono::seconds duration = std::chrono::seconds::zero(); + + /// Intervals can be randomized (to avoid DoS if intervals for many users end at one time). + bool randomize_interval = false; + + Limits(); + friend bool operator ==(const Limits & lhs, const Limits & rhs); + friend bool operator !=(const Limits & lhs, const Limits & rhs) { return !(lhs == rhs); } + }; + + std::vector all_limits; + + /// Key to share quota consumption. + /// Users with the same key share the same amount of resource. + enum class KeyType + { + NONE, /// All users share the same quota. + USER_NAME, /// Connections with the same user name share the same quota. + IP_ADDRESS, /// Connections from the same IP share the same quota. + CLIENT_KEY, /// Client should explicitly supply a key to use. + CLIENT_KEY_OR_USER_NAME, /// Same as CLIENT_KEY, but use USER_NAME if the client doesn't supply a key. + CLIENT_KEY_OR_IP_ADDRESS, /// Same as CLIENT_KEY, but use IP_ADDRESS if the client doesn't supply a key. + }; + static constexpr size_t MAX_KEY_TYPE = 6; + KeyType key_type = KeyType::NONE; + + /// Which roles or users should use this quota. + Strings roles; + bool all_roles = false; + Strings except_roles; + + bool equal(const IAccessEntity & other) const override; + std::shared_ptr clone() const override { return cloneImpl(); } + + static const char * getNameOfResourceType(ResourceType resource_type); + static const char * resourceTypeToKeyword(ResourceType resource_type); + static const char * resourceTypeToColumnName(ResourceType resource_type); + static const char * getNameOfKeyType(KeyType key_type); + static double executionTimeToSeconds(ResourceAmount ns); + static ResourceAmount secondsToExecutionTime(double s); +}; + + +inline const char * Quota::getNameOfResourceType(ResourceType resource_type) +{ + switch (resource_type) + { + case Quota::QUERIES: return "queries"; + case Quota::ERRORS: return "errors"; + case Quota::RESULT_ROWS: return "result rows"; + case Quota::RESULT_BYTES: return "result bytes"; + case Quota::READ_ROWS: return "read rows"; + case Quota::READ_BYTES: return "read bytes"; + case Quota::EXECUTION_TIME: return "execution time"; + } + __builtin_unreachable(); +} + + +inline const char * Quota::resourceTypeToKeyword(ResourceType resource_type) +{ + switch (resource_type) + { + case Quota::QUERIES: return "QUERIES"; + case Quota::ERRORS: return "ERRORS"; + case Quota::RESULT_ROWS: return "RESULT ROWS"; + case Quota::RESULT_BYTES: return "RESULT BYTES"; + case Quota::READ_ROWS: return "READ ROWS"; + case Quota::READ_BYTES: return "READ BYTES"; + case Quota::EXECUTION_TIME: return "EXECUTION TIME"; + } + __builtin_unreachable(); +} + + +inline const char * Quota::getNameOfKeyType(KeyType key_type) +{ + switch (key_type) + { + case KeyType::NONE: return "none"; + case KeyType::USER_NAME: return "user name"; + case KeyType::IP_ADDRESS: return "ip address"; + case KeyType::CLIENT_KEY: return "client key"; + case KeyType::CLIENT_KEY_OR_USER_NAME: return "client key or user name"; + case KeyType::CLIENT_KEY_OR_IP_ADDRESS: return "client key or ip address"; + } + __builtin_unreachable(); +} + + +inline double Quota::executionTimeToSeconds(ResourceAmount ns) +{ + return std::chrono::duration_cast>(std::chrono::nanoseconds{ns}).count(); +} + +inline Quota::ResourceAmount Quota::secondsToExecutionTime(double s) +{ + return std::chrono::duration_cast(std::chrono::duration(s)).count(); +} + + +using QuotaPtr = std::shared_ptr; +} diff --git a/dbms/src/Access/QuotaContext.cpp b/dbms/src/Access/QuotaContext.cpp new file mode 100644 index 00000000000..11666e5d4b8 --- /dev/null +++ b/dbms/src/Access/QuotaContext.cpp @@ -0,0 +1,264 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int QUOTA_EXPIRED; +} + +struct QuotaContext::Impl +{ + [[noreturn]] static void throwQuotaExceed( + const String & user_name, + const String & quota_name, + ResourceType resource_type, + ResourceAmount used, + ResourceAmount max, + std::chrono::seconds duration, + std::chrono::system_clock::time_point end_of_interval) + { + std::function amount_to_string = [](UInt64 amount) { return std::to_string(amount); }; + if (resource_type == Quota::EXECUTION_TIME) + amount_to_string = [&](UInt64 amount) { return ext::to_string(std::chrono::nanoseconds(amount)); }; + + throw Exception( + "Quota for user " + backQuote(user_name) + " for " + ext::to_string(duration) + " has been exceeded: " + + Quota::getNameOfResourceType(resource_type) + " = " + amount_to_string(used) + "/" + amount_to_string(max) + ". " + + "Interval will end at " + ext::to_string(end_of_interval) + ". " + "Name of quota template: " + backQuote(quota_name), + ErrorCodes::QUOTA_EXPIRED); + } + + + static std::chrono::system_clock::time_point getEndOfInterval( + const Interval & interval, std::chrono::system_clock::time_point current_time, bool * counters_were_reset = nullptr) + { + auto & end_of_interval = interval.end_of_interval; + auto end_loaded = end_of_interval.load(); + auto end = std::chrono::system_clock::time_point{end_loaded}; + if (current_time < end) + { + if (counters_were_reset) + *counters_were_reset = false; + return end; + } + + const auto duration = interval.duration; + + do + { + end = end + (current_time - end + duration) / duration * duration; + if (end_of_interval.compare_exchange_strong(end_loaded, end.time_since_epoch())) + { + boost::range::fill(interval.used, 0); + break; + } + end = std::chrono::system_clock::time_point{end_loaded}; + } + while (current_time >= end); + + if (counters_were_reset) + *counters_were_reset = true; + return end; + } + + + static void used( + const String & user_name, + const Intervals & intervals, + ResourceType resource_type, + ResourceAmount amount, + std::chrono::system_clock::time_point current_time, + bool check_exceeded) + { + for (const auto & interval : intervals.intervals) + { + ResourceAmount used = (interval.used[resource_type] += amount); + ResourceAmount max = interval.max[resource_type]; + if (max == Quota::UNLIMITED) + continue; + if (used > max) + { + bool counters_were_reset = false; + auto end_of_interval = getEndOfInterval(interval, current_time, &counters_were_reset); + if (counters_were_reset) + { + used = (interval.used[resource_type] += amount); + if ((used > max) && check_exceeded) + throwQuotaExceed(user_name, intervals.quota_name, resource_type, used, max, interval.duration, end_of_interval); + } + else if (check_exceeded) + throwQuotaExceed(user_name, intervals.quota_name, resource_type, used, max, interval.duration, end_of_interval); + } + } + } + + static void checkExceeded( + const String & user_name, + const Intervals & intervals, + ResourceType resource_type, + std::chrono::system_clock::time_point current_time) + { + for (const auto & interval : intervals.intervals) + { + ResourceAmount used = interval.used[resource_type]; + ResourceAmount max = interval.max[resource_type]; + if (max == Quota::UNLIMITED) + continue; + if (used > max) + { + bool used_counters_reset = false; + std::chrono::system_clock::time_point end_of_interval = getEndOfInterval(interval, current_time, &used_counters_reset); + if (!used_counters_reset) + throwQuotaExceed(user_name, intervals.quota_name, resource_type, used, max, interval.duration, end_of_interval); + } + } + } + + static void checkExceeded( + const String & user_name, + const Intervals & intervals, + std::chrono::system_clock::time_point current_time) + { + for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + checkExceeded(user_name, intervals, resource_type, current_time); + } +}; + + +QuotaContext::Interval & QuotaContext::Interval::operator =(const Interval & src) +{ + randomize_interval = src.randomize_interval; + duration = src.duration; + end_of_interval.store(src.end_of_interval.load()); + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + max[resource_type] = src.max[resource_type]; + used[resource_type].store(src.used[resource_type].load()); + } + return *this; +} + + +QuotaUsageInfo QuotaContext::Intervals::getUsageInfo(std::chrono::system_clock::time_point current_time) const +{ + QuotaUsageInfo info; + info.quota_id = quota_id; + info.quota_name = quota_name; + info.quota_key = quota_key; + info.intervals.reserve(intervals.size()); + for (const auto & in : intervals) + { + info.intervals.push_back({}); + auto & out = info.intervals.back(); + out.duration = in.duration; + out.randomize_interval = in.randomize_interval; + out.end_of_interval = Impl::getEndOfInterval(in, current_time); + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + out.max[resource_type] = in.max[resource_type]; + out.used[resource_type] = in.used[resource_type]; + } + } + return info; +} + + +QuotaContext::QuotaContext() + : atomic_intervals(std::make_shared()) /// Unlimited quota. +{ +} + + +QuotaContext::QuotaContext( + const String & user_name_, + const Poco::Net::IPAddress & address_, + const String & client_key_) + : user_name(user_name_), address(address_), client_key(client_key_) +{ +} + + +QuotaContext::~QuotaContext() = default; + + +void QuotaContext::used(ResourceType resource_type, ResourceAmount amount, bool check_exceeded) +{ + used({resource_type, amount}, check_exceeded); +} + + +void QuotaContext::used(const std::pair & resource, bool check_exceeded) +{ + auto intervals_ptr = std::atomic_load(&atomic_intervals); + auto current_time = std::chrono::system_clock::now(); + Impl::used(user_name, *intervals_ptr, resource.first, resource.second, current_time, check_exceeded); +} + + +void QuotaContext::used(const std::pair & resource1, const std::pair & resource2, bool check_exceeded) +{ + auto intervals_ptr = std::atomic_load(&atomic_intervals); + auto current_time = std::chrono::system_clock::now(); + Impl::used(user_name, *intervals_ptr, resource1.first, resource1.second, current_time, check_exceeded); + Impl::used(user_name, *intervals_ptr, resource2.first, resource2.second, current_time, check_exceeded); +} + + +void QuotaContext::used(const std::pair & resource1, const std::pair & resource2, const std::pair & resource3, bool check_exceeded) +{ + auto intervals_ptr = std::atomic_load(&atomic_intervals); + auto current_time = std::chrono::system_clock::now(); + Impl::used(user_name, *intervals_ptr, resource1.first, resource1.second, current_time, check_exceeded); + Impl::used(user_name, *intervals_ptr, resource2.first, resource2.second, current_time, check_exceeded); + Impl::used(user_name, *intervals_ptr, resource3.first, resource3.second, current_time, check_exceeded); +} + + +void QuotaContext::used(const std::vector> & resources, bool check_exceeded) +{ + auto intervals_ptr = std::atomic_load(&atomic_intervals); + auto current_time = std::chrono::system_clock::now(); + for (const auto & resource : resources) + Impl::used(user_name, *intervals_ptr, resource.first, resource.second, current_time, check_exceeded); +} + + +void QuotaContext::checkExceeded() +{ + auto intervals_ptr = std::atomic_load(&atomic_intervals); + Impl::checkExceeded(user_name, *intervals_ptr, std::chrono::system_clock::now()); +} + + +void QuotaContext::checkExceeded(ResourceType resource_type) +{ + auto intervals_ptr = std::atomic_load(&atomic_intervals); + Impl::checkExceeded(user_name, *intervals_ptr, resource_type, std::chrono::system_clock::now()); +} + + +QuotaUsageInfo QuotaContext::getUsageInfo() const +{ + auto intervals_ptr = std::atomic_load(&atomic_intervals); + return intervals_ptr->getUsageInfo(std::chrono::system_clock::now()); +} + + +QuotaUsageInfo::QuotaUsageInfo() : quota_id(UUID(UInt128(0))) +{ +} + + +QuotaUsageInfo::Interval::Interval() +{ + boost::range::fill(used, 0); + boost::range::fill(max, 0); +} +} diff --git a/dbms/src/Access/QuotaContext.h b/dbms/src/Access/QuotaContext.h new file mode 100644 index 00000000000..122d0df6ee7 --- /dev/null +++ b/dbms/src/Access/QuotaContext.h @@ -0,0 +1,110 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +struct QuotaUsageInfo; + + +/// Instances of `QuotaContext` are used to track resource consumption. +class QuotaContext : public boost::noncopyable +{ +public: + using ResourceType = Quota::ResourceType; + using ResourceAmount = Quota::ResourceAmount; + + /// Default constructors makes an unlimited quota. + QuotaContext(); + + ~QuotaContext(); + + /// Tracks resource consumption. If the quota exceeded and `check_exceeded == true`, throws an exception. + void used(ResourceType resource_type, ResourceAmount amount, bool check_exceeded = true); + void used(const std::pair & resource, bool check_exceeded = true); + void used(const std::pair & resource1, const std::pair & resource2, bool check_exceeded = true); + void used(const std::pair & resource1, const std::pair & resource2, const std::pair & resource3, bool check_exceeded = true); + void used(const std::vector> & resources, bool check_exceeded = true); + + /// Checks if the quota exceeded. If so, throws an exception. + void checkExceeded(); + void checkExceeded(ResourceType resource_type); + + /// Returns the information about this quota context. + QuotaUsageInfo getUsageInfo() const; + +private: + friend class QuotaContextFactory; + friend struct ext::shared_ptr_helper; + + /// Instances of this class are created by QuotaContextFactory. + QuotaContext(const String & user_name_, const Poco::Net::IPAddress & address_, const String & client_key_); + + static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + + struct Interval + { + mutable std::atomic used[MAX_RESOURCE_TYPE]; + ResourceAmount max[MAX_RESOURCE_TYPE]; + std::chrono::seconds duration; + bool randomize_interval; + mutable std::atomic end_of_interval; + + Interval() {} + Interval(const Interval & src) { *this = src; } + Interval & operator =(const Interval & src); + }; + + struct Intervals + { + std::vector intervals; + UUID quota_id; + String quota_name; + String quota_key; + + QuotaUsageInfo getUsageInfo(std::chrono::system_clock::time_point current_time) const; + }; + + struct Impl; + + const String user_name; + const Poco::Net::IPAddress address; + const String client_key; + std::shared_ptr atomic_intervals; /// atomically changed by QuotaUsageManager +}; + +using QuotaContextPtr = std::shared_ptr; + + +/// The information about a quota context. +struct QuotaUsageInfo +{ + using ResourceType = Quota::ResourceType; + using ResourceAmount = Quota::ResourceAmount; + static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + + struct Interval + { + ResourceAmount used[MAX_RESOURCE_TYPE]; + ResourceAmount max[MAX_RESOURCE_TYPE]; + std::chrono::seconds duration = std::chrono::seconds::zero(); + bool randomize_interval = false; + std::chrono::system_clock::time_point end_of_interval; + Interval(); + }; + + std::vector intervals; + UUID quota_id; + String quota_name; + String quota_key; + QuotaUsageInfo(); +}; +} diff --git a/dbms/src/Access/QuotaContextFactory.cpp b/dbms/src/Access/QuotaContextFactory.cpp new file mode 100644 index 00000000000..c6ecb947102 --- /dev/null +++ b/dbms/src/Access/QuotaContextFactory.cpp @@ -0,0 +1,299 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int QUOTA_REQUIRES_CLIENT_KEY; +} + + +namespace +{ + std::chrono::system_clock::duration randomDuration(std::chrono::seconds max) + { + auto count = std::chrono::duration_cast(max).count(); + std::uniform_int_distribution distribution{0, count - 1}; + return std::chrono::system_clock::duration(distribution(thread_local_rng)); + } +} + + +void QuotaContextFactory::QuotaInfo::setQuota(const QuotaPtr & quota_, const UUID & quota_id_) +{ + quota = quota_; + quota_id = quota_id_; + + boost::range::copy(quota->roles, std::inserter(roles, roles.end())); + all_roles = quota->all_roles; + boost::range::copy(quota->except_roles, std::inserter(except_roles, except_roles.end())); + + rebuildAllIntervals(); +} + + +bool QuotaContextFactory::QuotaInfo::canUseWithContext(const QuotaContext & context) const +{ + if (roles.count(context.user_name)) + return true; + + if (all_roles && !except_roles.count(context.user_name)) + return true; + + return false; +} + + +String QuotaContextFactory::QuotaInfo::calculateKey(const QuotaContext & context) const +{ + using KeyType = Quota::KeyType; + switch (quota->key_type) + { + case KeyType::NONE: + return ""; + case KeyType::USER_NAME: + return context.user_name; + case KeyType::IP_ADDRESS: + return context.address.toString(); + case KeyType::CLIENT_KEY: + { + if (!context.client_key.empty()) + return context.client_key; + throw Exception( + "Quota " + quota->getName() + " (for user " + context.user_name + ") requires a client supplied key.", + ErrorCodes::QUOTA_REQUIRES_CLIENT_KEY); + } + case KeyType::CLIENT_KEY_OR_USER_NAME: + { + if (!context.client_key.empty()) + return context.client_key; + return context.user_name; + } + case KeyType::CLIENT_KEY_OR_IP_ADDRESS: + { + if (!context.client_key.empty()) + return context.client_key; + return context.address.toString(); + } + } + __builtin_unreachable(); +} + + +std::shared_ptr QuotaContextFactory::QuotaInfo::getOrBuildIntervals(const String & key) +{ + auto it = key_to_intervals.find(key); + if (it != key_to_intervals.end()) + return it->second; + return rebuildIntervals(key); +} + + +void QuotaContextFactory::QuotaInfo::rebuildAllIntervals() +{ + for (const String & key : key_to_intervals | boost::adaptors::map_keys) + rebuildIntervals(key); +} + + +std::shared_ptr QuotaContextFactory::QuotaInfo::rebuildIntervals(const String & key) +{ + auto new_intervals = std::make_shared(); + new_intervals->quota_name = quota->getName(); + new_intervals->quota_id = quota_id; + new_intervals->quota_key = key; + auto & intervals = new_intervals->intervals; + intervals.reserve(quota->all_limits.size()); + constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + for (const auto & limits : quota->all_limits) + { + intervals.emplace_back(); + auto & interval = intervals.back(); + interval.duration = limits.duration; + std::chrono::system_clock::time_point end_of_interval{}; + interval.randomize_interval = limits.randomize_interval; + if (limits.randomize_interval) + end_of_interval += randomDuration(limits.duration); + interval.end_of_interval = end_of_interval.time_since_epoch(); + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + interval.max[resource_type] = limits.max[resource_type]; + interval.used[resource_type] = 0; + } + } + + /// Order intervals by durations from largest to smallest. + /// To report first about largest interval on what quota was exceeded. + struct GreaterByDuration + { + bool operator()(const Interval & lhs, const Interval & rhs) const { return lhs.duration > rhs.duration; } + }; + boost::range::stable_sort(intervals, GreaterByDuration{}); + + auto it = key_to_intervals.find(key); + if (it == key_to_intervals.end()) + { + /// Just put new intervals into the map. + key_to_intervals.try_emplace(key, new_intervals); + } + else + { + /// We need to keep usage information from the old intervals. + const auto & old_intervals = it->second->intervals; + for (auto & new_interval : new_intervals->intervals) + { + /// Check if an interval with the same duration is already in use. + auto lower_bound = boost::range::lower_bound(old_intervals, new_interval, GreaterByDuration{}); + if ((lower_bound == old_intervals.end()) || (lower_bound->duration != new_interval.duration)) + continue; + + /// Found an interval with the same duration, we need to copy its usage information to `result`. + auto & current_interval = *lower_bound; + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + new_interval.used[resource_type].store(current_interval.used[resource_type].load()); + new_interval.end_of_interval.store(current_interval.end_of_interval.load()); + } + } + it->second = new_intervals; + } + + return new_intervals; +} + + +QuotaContextFactory::QuotaContextFactory(const AccessControlManager & access_control_manager_) + : access_control_manager(access_control_manager_) +{ +} + + +QuotaContextFactory::~QuotaContextFactory() +{ +} + + +std::shared_ptr QuotaContextFactory::createContext(const String & user_name, const Poco::Net::IPAddress & address, const String & client_key) +{ + std::lock_guard lock{mutex}; + ensureAllQuotasRead(); + auto context = ext::shared_ptr_helper::create(user_name, address, client_key); + contexts.push_back(context); + chooseQuotaForContext(context); + return context; +} + + +void QuotaContextFactory::ensureAllQuotasRead() +{ + /// `mutex` is already locked. + if (all_quotas_read) + return; + all_quotas_read = true; + + subscription = access_control_manager.subscribeForChanges( + [&](const UUID & id, const AccessEntityPtr & entity) + { + if (entity) + quotaAddedOrChanged(id, typeid_cast(entity)); + else + quotaRemoved(id); + }); + + for (const UUID & quota_id : access_control_manager.findAll()) + { + auto quota = access_control_manager.tryRead(quota_id); + if (quota) + all_quotas.emplace(quota_id, QuotaInfo(quota, quota_id)); + } +} + + +void QuotaContextFactory::quotaAddedOrChanged(const UUID & quota_id, const std::shared_ptr & new_quota) +{ + std::lock_guard lock{mutex}; + auto it = all_quotas.find(quota_id); + if (it == all_quotas.end()) + { + it = all_quotas.emplace(quota_id, QuotaInfo(new_quota, quota_id)).first; + } + else + { + if (it->second.quota == new_quota) + return; + } + + auto & info = it->second; + info.setQuota(new_quota, quota_id); + chooseQuotaForAllContexts(); +} + + +void QuotaContextFactory::quotaRemoved(const UUID & quota_id) +{ + std::lock_guard lock{mutex}; + all_quotas.erase(quota_id); + chooseQuotaForAllContexts(); +} + + +void QuotaContextFactory::chooseQuotaForAllContexts() +{ + /// `mutex` is already locked. + boost::range::remove_erase_if( + contexts, + [&](const std::weak_ptr & weak) + { + auto context = weak.lock(); + if (!context) + return true; // remove from the `contexts` list. + chooseQuotaForContext(context); + return false; // keep in the `contexts` list. + }); +} + +void QuotaContextFactory::chooseQuotaForContext(const std::shared_ptr & context) +{ + /// `mutex` is already locked. + std::shared_ptr intervals; + for (auto & info : all_quotas | boost::adaptors::map_values) + { + if (info.canUseWithContext(*context)) + { + String key = info.calculateKey(*context); + intervals = info.getOrBuildIntervals(key); + break; + } + } + + if (!intervals) + intervals = std::make_shared(); /// No quota == no limits. + + std::atomic_store(&context->atomic_intervals, intervals); +} + + +std::vector QuotaContextFactory::getUsageInfo() const +{ + std::lock_guard lock{mutex}; + std::vector all_infos; + auto current_time = std::chrono::system_clock::now(); + for (const auto & info : all_quotas | boost::adaptors::map_values) + { + for (const auto & intervals : info.key_to_intervals | boost::adaptors::map_values) + all_infos.push_back(intervals->getUsageInfo(current_time)); + } + return all_infos; +} +} diff --git a/dbms/src/Access/QuotaContextFactory.h b/dbms/src/Access/QuotaContextFactory.h new file mode 100644 index 00000000000..159ffe1fa09 --- /dev/null +++ b/dbms/src/Access/QuotaContextFactory.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +class AccessControlManager; + + +/// Stores information how much amount of resources have been consumed and how much are left. +class QuotaContextFactory +{ +public: + QuotaContextFactory(const AccessControlManager & access_control_manager_); + ~QuotaContextFactory(); + + QuotaContextPtr createContext(const String & user_name, const Poco::Net::IPAddress & address, const String & client_key); + std::vector getUsageInfo() const; + +private: + using Interval = QuotaContext::Interval; + using Intervals = QuotaContext::Intervals; + + struct QuotaInfo + { + QuotaInfo(const QuotaPtr & quota_, const UUID & quota_id_) { setQuota(quota_, quota_id_); } + void setQuota(const QuotaPtr & quota_, const UUID & quota_id_); + + bool canUseWithContext(const QuotaContext & context) const; + String calculateKey(const QuotaContext & context) const; + std::shared_ptr getOrBuildIntervals(const String & key); + std::shared_ptr rebuildIntervals(const String & key); + void rebuildAllIntervals(); + + QuotaPtr quota; + UUID quota_id; + std::unordered_set roles; + bool all_roles = false; + std::unordered_set except_roles; + std::unordered_map> key_to_intervals; + }; + + void ensureAllQuotasRead(); + void quotaAddedOrChanged(const UUID & quota_id, const std::shared_ptr & new_quota); + void quotaRemoved(const UUID & quota_id); + void chooseQuotaForAllContexts(); + void chooseQuotaForContext(const std::shared_ptr & context); + + const AccessControlManager & access_control_manager; + mutable std::mutex mutex; + std::unordered_map all_quotas; + bool all_quotas_read = false; + IAccessStorage::SubscriptionPtr subscription; + std::vector> contexts; +}; +} diff --git a/dbms/src/Access/UsersConfigAccessStorage.cpp b/dbms/src/Access/UsersConfigAccessStorage.cpp new file mode 100644 index 00000000000..d417968bb64 --- /dev/null +++ b/dbms/src/Access/UsersConfigAccessStorage.cpp @@ -0,0 +1,207 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + char getTypeChar(std::type_index type) + { + if (type == typeid(Quota)) + return 'Q'; + return 0; + } + + + UUID generateID(std::type_index type, const String & name) + { + Poco::MD5Engine md5; + md5.update(name); + char type_storage_chars[] = " USRSXML"; + type_storage_chars[0] = getTypeChar(type); + md5.update(type_storage_chars, strlen(type_storage_chars)); + UUID result; + memcpy(&result, md5.digest().data(), md5.digestLength()); + return result; + } + + + UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getFullName()); } + + QuotaPtr parseQuota(const Poco::Util::AbstractConfiguration & config, const String & quota_name, const Strings & user_names) + { + auto quota = std::make_shared(); + quota->setName(quota_name); + + using KeyType = Quota::KeyType; + String quota_config = "quotas." + quota_name; + if (config.has(quota_config + ".keyed_by_ip")) + quota->key_type = KeyType::IP_ADDRESS; + else if (config.has(quota_config + ".keyed")) + quota->key_type = KeyType::CLIENT_KEY_OR_USER_NAME; + else + quota->key_type = KeyType::USER_NAME; + + Poco::Util::AbstractConfiguration::Keys interval_keys; + config.keys(quota_config, interval_keys); + + for (const String & interval_key : interval_keys) + { + if (!startsWith(interval_key, "interval")) + continue; + + String interval_config = quota_config + "." + interval_key; + std::chrono::seconds duration{config.getInt(interval_config + ".duration", 0)}; + if (duration.count() <= 0) /// Skip quotas with non-positive duration. + continue; + + quota->all_limits.emplace_back(); + auto & limits = quota->all_limits.back(); + limits.duration = duration; + limits.randomize_interval = config.getBool(interval_config + ".randomize", false); + + using ResourceType = Quota::ResourceType; + limits.max[ResourceType::QUERIES] = config.getUInt64(interval_config + ".queries", Quota::UNLIMITED); + limits.max[ResourceType::ERRORS] = config.getUInt64(interval_config + ".errors", Quota::UNLIMITED); + limits.max[ResourceType::RESULT_ROWS] = config.getUInt64(interval_config + ".result_rows", Quota::UNLIMITED); + limits.max[ResourceType::RESULT_BYTES] = config.getUInt64(interval_config + ".result_bytes", Quota::UNLIMITED); + limits.max[ResourceType::READ_ROWS] = config.getUInt64(interval_config + ".read_rows", Quota::UNLIMITED); + limits.max[ResourceType::READ_BYTES] = config.getUInt64(interval_config + ".read_bytes", Quota::UNLIMITED); + limits.max[ResourceType::EXECUTION_TIME] = Quota::secondsToExecutionTime(config.getUInt64(interval_config + ".execution_time", Quota::UNLIMITED)); + } + + quota->roles = user_names; + + return quota; + } + + + std::vector parseQuotas(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) + { + Poco::Util::AbstractConfiguration::Keys user_names; + config.keys("users", user_names); + std::unordered_map quota_to_user_names; + for (const auto & user_name : user_names) + { + if (config.has("users." + user_name + ".quota")) + quota_to_user_names[config.getString("users." + user_name + ".quota")].push_back(user_name); + } + + Poco::Util::AbstractConfiguration::Keys quota_names; + config.keys("quotas", quota_names); + std::vector quotas; + quotas.reserve(quota_names.size()); + for (const auto & quota_name : quota_names) + { + try + { + auto it = quota_to_user_names.find(quota_name); + const Strings quota_users = (it != quota_to_user_names.end()) ? std::move(it->second) : Strings{}; + quotas.push_back(parseQuota(config, quota_name, quota_users)); + } + catch (...) + { + tryLogCurrentException(log, "Could not parse quota " + backQuote(quota_name)); + } + } + return quotas; + } +} + + +UsersConfigAccessStorage::UsersConfigAccessStorage() : IAccessStorage("users.xml") +{ +} + + +UsersConfigAccessStorage::~UsersConfigAccessStorage() {} + + +void UsersConfigAccessStorage::loadFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + std::vector> all_entities; + for (const auto & entity : parseQuotas(config, getLogger())) + all_entities.emplace_back(generateID(*entity), entity); + memory_storage.setAll(all_entities); +} + + +std::optional UsersConfigAccessStorage::findImpl(std::type_index type, const String & name) const +{ + return memory_storage.find(type, name); +} + + +std::vector UsersConfigAccessStorage::findAllImpl(std::type_index type) const +{ + return memory_storage.findAll(type); +} + + +bool UsersConfigAccessStorage::existsImpl(const UUID & id) const +{ + return memory_storage.exists(id); +} + + +AccessEntityPtr UsersConfigAccessStorage::readImpl(const UUID & id) const +{ + return memory_storage.read(id); +} + + +String UsersConfigAccessStorage::readNameImpl(const UUID & id) const +{ + return memory_storage.readName(id); +} + + +UUID UsersConfigAccessStorage::insertImpl(const AccessEntityPtr & entity, bool) +{ + throwReadonlyCannotInsert(entity->getType(), entity->getFullName()); +} + + +void UsersConfigAccessStorage::removeImpl(const UUID & id) +{ + auto entity = read(id); + throwReadonlyCannotRemove(entity->getType(), entity->getFullName()); +} + + +void UsersConfigAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) +{ + auto entity = read(id); + throwReadonlyCannotUpdate(entity->getType(), entity->getFullName()); +} + + +IAccessStorage::SubscriptionPtr UsersConfigAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const +{ + return memory_storage.subscribeForChanges(id, handler); +} + + +IAccessStorage::SubscriptionPtr UsersConfigAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +{ + return memory_storage.subscribeForChanges(type, handler); +} + + +bool UsersConfigAccessStorage::hasSubscriptionImpl(const UUID & id) const +{ + return memory_storage.hasSubscription(id); +} + + +bool UsersConfigAccessStorage::hasSubscriptionImpl(std::type_index type) const +{ + return memory_storage.hasSubscription(type); +} +} diff --git a/dbms/src/Access/UsersConfigAccessStorage.h b/dbms/src/Access/UsersConfigAccessStorage.h new file mode 100644 index 00000000000..9b0bf2ed17c --- /dev/null +++ b/dbms/src/Access/UsersConfigAccessStorage.h @@ -0,0 +1,42 @@ +#pragma once + +#include + + +namespace Poco +{ + namespace Util + { + class AbstractConfiguration; + } +} + + +namespace DB +{ +/// Implementation of IAccessStorage which loads all from users.xml periodically. +class UsersConfigAccessStorage : public IAccessStorage +{ +public: + UsersConfigAccessStorage(); + ~UsersConfigAccessStorage() override; + + void loadFromConfig(const Poco::Util::AbstractConfiguration & config); + +private: + std::optional findImpl(std::type_index type, const String & name) const override; + std::vector findAllImpl(std::type_index type) const override; + bool existsImpl(const UUID & id) const override; + AccessEntityPtr readImpl(const UUID & id) const override; + String readNameImpl(const UUID & id) const override; + UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override; + void removeImpl(const UUID & id) override; + void updateImpl(const UUID & id, const UpdateFunc & update_func) override; + SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; + SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + bool hasSubscriptionImpl(const UUID & id) const override; + bool hasSubscriptionImpl(std::type_index type) const override; + + MemoryAccessStorage memory_storage; +}; +} diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 296164239cc..a30daef9042 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -466,7 +466,13 @@ namespace ErrorCodes extern const int INCORRECT_DICTIONARY_DEFINITION = 489; extern const int CANNOT_FORMAT_DATETIME = 490; extern const int UNACCEPTABLE_URL = 491; - extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED = 492; + extern const int ACCESS_ENTITY_NOT_FOUND = 492; + extern const int ACCESS_ENTITY_ALREADY_EXISTS = 493; + extern const int ACCESS_ENTITY_FOUND_DUPLICATES = 494; + extern const int ACCESS_ENTITY_STORAGE_READONLY = 495; + extern const int QUOTA_REQUIRES_CLIENT_KEY = 496; + extern const int NOT_ENOUGH_PRIVILEGES = 497; + extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED = 498; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Common/IntervalKind.cpp b/dbms/src/Common/IntervalKind.cpp new file mode 100644 index 00000000000..9443844a54b --- /dev/null +++ b/dbms/src/Common/IntervalKind.cpp @@ -0,0 +1,162 @@ +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int SYNTAX_ERROR; +} + +const char * IntervalKind::toString() const +{ + switch (kind) + { + case IntervalKind::Second: return "Second"; + case IntervalKind::Minute: return "Minute"; + case IntervalKind::Hour: return "Hour"; + case IntervalKind::Day: return "Day"; + case IntervalKind::Week: return "Week"; + case IntervalKind::Month: return "Month"; + case IntervalKind::Quarter: return "Quarter"; + case IntervalKind::Year: return "Year"; + } + __builtin_unreachable(); +} + + +Int32 IntervalKind::toAvgSeconds() const +{ + switch (kind) + { + case IntervalKind::Second: return 1; + case IntervalKind::Minute: return 60; + case IntervalKind::Hour: return 3600; + case IntervalKind::Day: return 86400; + case IntervalKind::Week: return 604800; + case IntervalKind::Month: return 2629746; /// Exactly 1/12 of a year. + case IntervalKind::Quarter: return 7889238; /// Exactly 1/4 of a year. + case IntervalKind::Year: return 31556952; /// The average length of a Gregorian year is equal to 365.2425 days + } + __builtin_unreachable(); +} + + +IntervalKind IntervalKind::fromAvgSeconds(Int64 num_seconds) +{ + if (num_seconds) + { + if (!(num_seconds % 31556952)) + return IntervalKind::Year; + if (!(num_seconds % 7889238)) + return IntervalKind::Quarter; + if (!(num_seconds % 604800)) + return IntervalKind::Week; + if (!(num_seconds % 2629746)) + return IntervalKind::Month; + if (!(num_seconds % 86400)) + return IntervalKind::Day; + if (!(num_seconds % 3600)) + return IntervalKind::Hour; + if (!(num_seconds % 60)) + return IntervalKind::Minute; + } + return IntervalKind::Second; +} + + +const char * IntervalKind::toKeyword() const +{ + switch (kind) + { + case IntervalKind::Second: return "SECOND"; + case IntervalKind::Minute: return "MINUTE"; + case IntervalKind::Hour: return "HOUR"; + case IntervalKind::Day: return "DAY"; + case IntervalKind::Week: return "WEEK"; + case IntervalKind::Month: return "MONTH"; + case IntervalKind::Quarter: return "QUARTER"; + case IntervalKind::Year: return "YEAR"; + } + __builtin_unreachable(); +} + + +const char * IntervalKind::toDateDiffUnit() const +{ + switch (kind) + { + case IntervalKind::Second: + return "second"; + case IntervalKind::Minute: + return "minute"; + case IntervalKind::Hour: + return "hour"; + case IntervalKind::Day: + return "day"; + case IntervalKind::Week: + return "week"; + case IntervalKind::Month: + return "month"; + case IntervalKind::Quarter: + return "quarter"; + case IntervalKind::Year: + return "year"; + } + __builtin_unreachable(); +} + + +const char * IntervalKind::toNameOfFunctionToIntervalDataType() const +{ + switch (kind) + { + case IntervalKind::Second: + return "toIntervalSecond"; + case IntervalKind::Minute: + return "toIntervalMinute"; + case IntervalKind::Hour: + return "toIntervalHour"; + case IntervalKind::Day: + return "toIntervalDay"; + case IntervalKind::Week: + return "toIntervalWeek"; + case IntervalKind::Month: + return "toIntervalMonth"; + case IntervalKind::Quarter: + return "toIntervalQuarter"; + case IntervalKind::Year: + return "toIntervalYear"; + } + __builtin_unreachable(); +} + + +const char * IntervalKind::toNameOfFunctionExtractTimePart() const +{ + switch (kind) + { + case IntervalKind::Second: + return "toSecond"; + case IntervalKind::Minute: + return "toMinute"; + case IntervalKind::Hour: + return "toHour"; + case IntervalKind::Day: + return "toDayOfMonth"; + case IntervalKind::Week: + // TODO: SELECT toRelativeWeekNum(toDate('2017-06-15')) - toRelativeWeekNum(toStartOfYear(toDate('2017-06-15'))) + // else if (ParserKeyword("WEEK").ignore(pos, expected)) + // function_name = "toRelativeWeekNum"; + throw Exception("The syntax 'EXTRACT(WEEK FROM date)' is not supported, cannot extract the number of a week", ErrorCodes::SYNTAX_ERROR); + case IntervalKind::Month: + return "toMonth"; + case IntervalKind::Quarter: + return "toQuarter"; + case IntervalKind::Year: + return "toYear"; + } + __builtin_unreachable(); +} +} diff --git a/dbms/src/Common/IntervalKind.h b/dbms/src/Common/IntervalKind.h new file mode 100644 index 00000000000..9b7c4bd504e --- /dev/null +++ b/dbms/src/Common/IntervalKind.h @@ -0,0 +1,54 @@ +#pragma once + +#include + + +namespace DB +{ +/// Kind of a temporal interval. +struct IntervalKind +{ + enum Kind + { + Second, + Minute, + Hour, + Day, + Week, + Month, + Quarter, + Year, + }; + Kind kind = Second; + + IntervalKind(Kind kind_ = Second) : kind(kind_) {} + operator Kind() const { return kind; } + + const char * toString() const; + + /// Returns number of seconds in one interval. + /// For `Month`, `Quarter` and `Year` the function returns an average number of seconds. + Int32 toAvgSeconds() const; + + /// Chooses an interval kind based on number of seconds. + /// For example, `IntervalKind::fromAvgSeconds(3600)` returns `IntervalKind::Hour`. + static IntervalKind fromAvgSeconds(Int64 num_seconds); + + /// Returns an uppercased version of what `toString()` returns. + const char * toKeyword() const; + + /// Returns the string which can be passed to the `unit` parameter of the dateDiff() function. + /// For example, `IntervalKind{IntervalKind::Day}.getDateDiffParameter()` returns "day". + const char * toDateDiffUnit() const; + + /// Returns the name of the function converting a number to the interval data type. + /// For example, `IntervalKind{IntervalKind::Day}.getToIntervalDataTypeFunctionName()` + /// returns "toIntervalDay". + const char * toNameOfFunctionToIntervalDataType() const; + + /// Returns the name of the function extracting time part from a date or a time. + /// For example, `IntervalKind{IntervalKind::Day}.getExtractTimePartFunctionName()` + /// returns "toDayOfMonth". + const char * toNameOfFunctionExtractTimePart() const; +}; +} diff --git a/dbms/src/Common/intExp.h b/dbms/src/Common/intExp.h index 163d835819f..0212eb4c084 100644 --- a/dbms/src/Common/intExp.h +++ b/dbms/src/Common/intExp.h @@ -3,11 +3,18 @@ #include #include -#include + +// Also defined in Core/Defines.h +#if !defined(NO_SANITIZE_UNDEFINED) +#if defined(__clang__) + #define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined"))) +#else + #define NO_SANITIZE_UNDEFINED +#endif +#endif /// On overlow, the function returns unspecified value. - inline NO_SANITIZE_UNDEFINED uint64_t intExp2(int x) { return 1ULL << x; diff --git a/dbms/src/Common/typeid_cast.h b/dbms/src/Common/typeid_cast.h index 9285355e788..29ad2e520c0 100644 --- a/dbms/src/Common/typeid_cast.h +++ b/dbms/src/Common/typeid_cast.h @@ -3,8 +3,10 @@ #include #include #include +#include #include +#include #include #include @@ -27,7 +29,7 @@ std::enable_if_t, To> typeid_cast(From & from) { try { - if (typeid(from) == typeid(To)) + if ((typeid(From) == typeid(To)) || (typeid(from) == typeid(To))) return static_cast(from); } catch (const std::exception & e) @@ -39,12 +41,13 @@ std::enable_if_t, To> typeid_cast(From & from) DB::ErrorCodes::BAD_CAST); } + template -To typeid_cast(From * from) +std::enable_if_t, To> typeid_cast(From * from) { try { - if (typeid(*from) == typeid(std::remove_pointer_t)) + if ((typeid(From) == typeid(std::remove_pointer_t)) || (typeid(*from) == typeid(std::remove_pointer_t))) return static_cast(from); else return nullptr; @@ -54,3 +57,20 @@ To typeid_cast(From * from) throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST); } } + + +template +std::enable_if_t, To> typeid_cast(const std::shared_ptr & from) +{ + try + { + if ((typeid(From) == typeid(typename To::element_type)) || (typeid(*from) == typeid(typename To::element_type))) + return std::static_pointer_cast(from); + else + return nullptr; + } + catch (const std::exception & e) + { + throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST); + } +} diff --git a/dbms/src/Core/DecimalComparison.h b/dbms/src/Core/DecimalComparison.h index bc676ae86c8..cb332ad3779 100644 --- a/dbms/src/Core/DecimalComparison.h +++ b/dbms/src/Core/DecimalComparison.h @@ -88,9 +88,9 @@ public: Shift shift; if (scale_a < scale_b) - shift.a = DataTypeDecimal(maxDecimalPrecision(), scale_b).getScaleMultiplier(scale_b - scale_a); + shift.a = B::getScaleMultiplier(scale_b - scale_a); if (scale_a > scale_b) - shift.b = DataTypeDecimal(maxDecimalPrecision(), scale_a).getScaleMultiplier(scale_a - scale_b); + shift.b = A::getScaleMultiplier(scale_a - scale_b); return applyWithScale(a, b, shift); } diff --git a/dbms/src/Core/Field.cpp b/dbms/src/Core/Field.cpp index 9d27e33c414..505627aaedb 100644 --- a/dbms/src/Core/Field.cpp +++ b/dbms/src/Core/Field.cpp @@ -300,21 +300,6 @@ namespace DB } - template <> Decimal32 DecimalField::getScaleMultiplier() const - { - return DataTypeDecimal::getScaleMultiplier(scale); - } - - template <> Decimal64 DecimalField::getScaleMultiplier() const - { - return DataTypeDecimal::getScaleMultiplier(scale); - } - - template <> Decimal128 DecimalField::getScaleMultiplier() const - { - return DataTypeDecimal::getScaleMultiplier(scale); - } - template static bool decEqual(T x, T y, UInt32 x_scale, UInt32 y_scale) { diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h index 3d34502c339..885545844f4 100644 --- a/dbms/src/Core/Field.h +++ b/dbms/src/Core/Field.h @@ -102,7 +102,7 @@ public: operator T() const { return dec; } T getValue() const { return dec; } - T getScaleMultiplier() const; + T getScaleMultiplier() const { return T::getScaleMultiplier(scale); } UInt32 getScale() const { return scale; } template diff --git a/dbms/src/Core/Types.h b/dbms/src/Core/Types.h index bbc309aff94..511446b442f 100644 --- a/dbms/src/Core/Types.h +++ b/dbms/src/Core/Types.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -145,6 +146,8 @@ struct Decimal const Decimal & operator /= (const T & x) { value /= x; return *this; } const Decimal & operator %= (const T & x) { value %= x; return *this; } + static T getScaleMultiplier(UInt32 scale); + T value; }; @@ -170,6 +173,10 @@ template <> struct NativeType { using Type = Int32; }; template <> struct NativeType { using Type = Int64; }; template <> struct NativeType { using Type = Int128; }; +template <> inline Int32 Decimal32::getScaleMultiplier(UInt32 scale) { return common::exp10_i32(scale); } +template <> inline Int64 Decimal64::getScaleMultiplier(UInt32 scale) { return common::exp10_i64(scale); } +template <> inline Int128 Decimal128::getScaleMultiplier(UInt32 scale) { return common::exp10_i128(scale); } + inline const char * getTypeName(TypeIndex idx) { switch (idx) diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index 2e30749e89f..df81f26f665 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -70,7 +70,7 @@ Block IBlockInputStream::read() if (limits.mode == LIMITS_CURRENT && !limits.size_limits.check(info.rows, info.bytes, "result", ErrorCodes::TOO_MANY_ROWS_OR_BYTES)) limit_exceeded_need_break = true; - if (quota != nullptr) + if (quota) checkQuota(res); } else @@ -240,12 +240,8 @@ void IBlockInputStream::checkQuota(Block & block) case LIMITS_CURRENT: { - time_t current_time = time(nullptr); - double total_elapsed = info.total_stopwatch.elapsedSeconds(); - - quota->checkAndAddResultRowsBytes(current_time, block.rows(), block.bytes()); - quota->checkAndAddExecutionTime(current_time, Poco::Timespan((total_elapsed - prev_elapsed) * 1000000.0)); - + UInt64 total_elapsed = info.total_stopwatch.elapsedNanoseconds(); + quota->used({Quota::RESULT_ROWS, block.rows()}, {Quota::RESULT_BYTES, block.bytes()}, {Quota::EXECUTION_TIME, total_elapsed - prev_elapsed}); prev_elapsed = total_elapsed; break; } @@ -291,10 +287,8 @@ void IBlockInputStream::progressImpl(const Progress & value) limits.speed_limits.throttle(progress.read_rows, progress.read_bytes, total_rows, total_elapsed_microseconds); - if (quota != nullptr && limits.mode == LIMITS_TOTAL) - { - quota->checkAndAddReadRowsBytes(time(nullptr), value.read_rows, value.read_bytes); - } + if (quota && limits.mode == LIMITS_TOTAL) + quota->used({Quota::READ_ROWS, value.read_rows}, {Quota::READ_BYTES, value.read_bytes}); } } diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index dfa9194a6f9..69aadf44c09 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -23,7 +23,7 @@ namespace ErrorCodes } class ProcessListElement; -class QuotaForIntervals; +class QuotaContext; class QueryStatus; struct SortColumnDescription; using SortDescription = std::vector; @@ -220,9 +220,9 @@ public: /** Set the quota. If you set a quota on the amount of raw data, * then you should also set mode = LIMITS_TOTAL to LocalLimits with setLimits. */ - virtual void setQuota(QuotaForIntervals & quota_) + virtual void setQuota(const std::shared_ptr & quota_) { - quota = "a_; + quota = quota_; } /// Enable calculation of minimums and maximums by the result columns. @@ -273,8 +273,8 @@ private: LocalLimits limits; - QuotaForIntervals * quota = nullptr; /// If nullptr - the quota is not used. - double prev_elapsed = 0; + std::shared_ptr quota; /// If nullptr - the quota is not used. + UInt64 prev_elapsed = 0; /// The approximate total number of rows to read. For progress bar. size_t total_rows_approx = 0; diff --git a/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp b/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp index 21233da9327..c894af82580 100644 --- a/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp @@ -1,5 +1,4 @@ #include -#include "ParallelParsingBlockInputStream.h" namespace DB { @@ -15,7 +14,7 @@ void ParallelParsingBlockInputStream::segmentatorThreadFunction() auto & unit = processing_units[current_unit_number]; { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); segmentator_condvar.wait(lock, [&]{ return unit.status == READY_TO_INSERT || finished; }); } @@ -85,7 +84,7 @@ void ParallelParsingBlockInputStream::parserThreadFunction(size_t current_unit_n // except at the end of file. Also see a matching assert in readImpl(). assert(unit.is_last || unit.block_ext.block.size() > 0); - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); unit.status = READY_TO_READ; reader_condvar.notify_all(); } @@ -99,7 +98,7 @@ void ParallelParsingBlockInputStream::onBackgroundException() { tryLogCurrentException(__PRETTY_FUNCTION__); - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); if (!background_exception) { background_exception = std::current_exception(); @@ -116,7 +115,7 @@ Block ParallelParsingBlockInputStream::readImpl() /** * Check for background exception and rethrow it before we return. */ - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); if (background_exception) { lock.unlock(); @@ -134,7 +133,7 @@ Block ParallelParsingBlockInputStream::readImpl() { // We have read out all the Blocks from the previous Processing Unit, // wait for the current one to become ready. - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); reader_condvar.wait(lock, [&](){ return unit.status == READY_TO_READ || finished; }); if (finished) @@ -190,7 +189,7 @@ Block ParallelParsingBlockInputStream::readImpl() else { // Pass the unit back to the segmentator. - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); unit.status = READY_TO_INSERT; segmentator_condvar.notify_all(); } diff --git a/dbms/src/DataStreams/ParallelParsingBlockInputStream.h b/dbms/src/DataStreams/ParallelParsingBlockInputStream.h index 4b5e091cfc9..8c276f2f7dd 100644 --- a/dbms/src/DataStreams/ParallelParsingBlockInputStream.h +++ b/dbms/src/DataStreams/ParallelParsingBlockInputStream.h @@ -227,7 +227,7 @@ private: finished = true; { - std::unique_lock lock(mutex); + std::unique_lock lock(mutex); segmentator_condvar.notify_all(); reader_condvar.notify_all(); } @@ -255,4 +255,4 @@ private: void onBackgroundException(); }; -}; +} diff --git a/dbms/src/DataTypes/DataTypeInterval.cpp b/dbms/src/DataTypes/DataTypeInterval.cpp index c7ee3ede334..57d071a8666 100644 --- a/dbms/src/DataTypes/DataTypeInterval.cpp +++ b/dbms/src/DataTypes/DataTypeInterval.cpp @@ -13,14 +13,14 @@ bool DataTypeInterval::equals(const IDataType & rhs) const void registerDataTypeInterval(DataTypeFactory & factory) { - factory.registerSimpleDataType("IntervalSecond", [] { return DataTypePtr(std::make_shared(DataTypeInterval::Second)); }); - factory.registerSimpleDataType("IntervalMinute", [] { return DataTypePtr(std::make_shared(DataTypeInterval::Minute)); }); - factory.registerSimpleDataType("IntervalHour", [] { return DataTypePtr(std::make_shared(DataTypeInterval::Hour)); }); - factory.registerSimpleDataType("IntervalDay", [] { return DataTypePtr(std::make_shared(DataTypeInterval::Day)); }); - factory.registerSimpleDataType("IntervalWeek", [] { return DataTypePtr(std::make_shared(DataTypeInterval::Week)); }); - factory.registerSimpleDataType("IntervalMonth", [] { return DataTypePtr(std::make_shared(DataTypeInterval::Month)); }); - factory.registerSimpleDataType("IntervalQuarter", [] { return DataTypePtr(std::make_shared(DataTypeInterval::Quarter)); }); - factory.registerSimpleDataType("IntervalYear", [] { return DataTypePtr(std::make_shared(DataTypeInterval::Year)); }); + factory.registerSimpleDataType("IntervalSecond", [] { return DataTypePtr(std::make_shared(IntervalKind::Second)); }); + factory.registerSimpleDataType("IntervalMinute", [] { return DataTypePtr(std::make_shared(IntervalKind::Minute)); }); + factory.registerSimpleDataType("IntervalHour", [] { return DataTypePtr(std::make_shared(IntervalKind::Hour)); }); + factory.registerSimpleDataType("IntervalDay", [] { return DataTypePtr(std::make_shared(IntervalKind::Day)); }); + factory.registerSimpleDataType("IntervalWeek", [] { return DataTypePtr(std::make_shared(IntervalKind::Week)); }); + factory.registerSimpleDataType("IntervalMonth", [] { return DataTypePtr(std::make_shared(IntervalKind::Month)); }); + factory.registerSimpleDataType("IntervalQuarter", [] { return DataTypePtr(std::make_shared(IntervalKind::Quarter)); }); + factory.registerSimpleDataType("IntervalYear", [] { return DataTypePtr(std::make_shared(IntervalKind::Year)); }); } } diff --git a/dbms/src/DataTypes/DataTypeInterval.h b/dbms/src/DataTypes/DataTypeInterval.h index fa99ac430b6..111a2489d65 100644 --- a/dbms/src/DataTypes/DataTypeInterval.h +++ b/dbms/src/DataTypes/DataTypeInterval.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -16,47 +17,17 @@ namespace DB */ class DataTypeInterval final : public DataTypeNumberBase { -public: - enum Kind - { - Second, - Minute, - Hour, - Day, - Week, - Month, - Quarter, - Year - }; - private: - Kind kind; + IntervalKind kind; public: static constexpr bool is_parametric = true; - Kind getKind() const { return kind; } + IntervalKind getKind() const { return kind; } - const char * kindToString() const - { - switch (kind) - { - case Second: return "Second"; - case Minute: return "Minute"; - case Hour: return "Hour"; - case Day: return "Day"; - case Week: return "Week"; - case Month: return "Month"; - case Quarter: return "Quarter"; - case Year: return "Year"; - } + DataTypeInterval(IntervalKind kind_) : kind(kind_) {} - __builtin_unreachable(); - } - - DataTypeInterval(Kind kind_) : kind(kind_) {} - - std::string doGetName() const override { return std::string("Interval") + kindToString(); } + std::string doGetName() const override { return std::string("Interval") + kind.toString(); } const char * getFamilyName() const override { return "Interval"; } TypeIndex getTypeId() const override { return TypeIndex::Interval; } diff --git a/dbms/src/DataTypes/DataTypesDecimal.cpp b/dbms/src/DataTypes/DataTypesDecimal.cpp index e8caae63a09..84fc31a5ed7 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.cpp +++ b/dbms/src/DataTypes/DataTypesDecimal.cpp @@ -58,7 +58,7 @@ bool DataTypeDecimal::tryReadText(T & x, ReadBuffer & istr, UInt32 precision, { UInt32 unread_scale = scale; bool done = tryReadDecimalText(istr, x, precision, unread_scale); - x *= getScaleMultiplier(unread_scale); + x *= T::getScaleMultiplier(unread_scale); return done; } @@ -70,7 +70,7 @@ void DataTypeDecimal::readText(T & x, ReadBuffer & istr, UInt32 precision, UI readCSVDecimalText(istr, x, precision, unread_scale); else readDecimalText(istr, x, precision, unread_scale); - x *= getScaleMultiplier(unread_scale); + x *= T::getScaleMultiplier(unread_scale); } template @@ -96,7 +96,7 @@ T DataTypeDecimal::parseFromString(const String & str) const T x; UInt32 unread_scale = scale; readDecimalText(buf, x, precision, unread_scale, true); - x *= getScaleMultiplier(unread_scale); + x *= T::getScaleMultiplier(unread_scale); return x; } @@ -271,25 +271,6 @@ void registerDataTypeDecimal(DataTypeFactory & factory) } -template <> -Decimal32 DataTypeDecimal::getScaleMultiplier(UInt32 scale_) -{ - return decimalScaleMultiplier(scale_); -} - -template <> -Decimal64 DataTypeDecimal::getScaleMultiplier(UInt32 scale_) -{ - return decimalScaleMultiplier(scale_); -} - -template <> -Decimal128 DataTypeDecimal::getScaleMultiplier(UInt32 scale_) -{ - return decimalScaleMultiplier(scale_); -} - - /// Explicit template instantiations. template class DataTypeDecimal; template class DataTypeDecimal; diff --git a/dbms/src/DataTypes/DataTypesDecimal.h b/dbms/src/DataTypes/DataTypesDecimal.h index e59a2b6e3fd..8de80050bae 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.h +++ b/dbms/src/DataTypes/DataTypesDecimal.h @@ -130,7 +130,7 @@ public: UInt32 getPrecision() const { return precision; } UInt32 getScale() const { return scale; } - T getScaleMultiplier() const { return getScaleMultiplier(scale); } + T getScaleMultiplier() const { return T::getScaleMultiplier(scale); } T wholePart(T x) const { @@ -148,7 +148,7 @@ public: return x % getScaleMultiplier(); } - T maxWholeValue() const { return getScaleMultiplier(maxPrecision() - scale) - T(1); } + T maxWholeValue() const { return T::getScaleMultiplier(maxPrecision() - scale) - T(1); } bool canStoreWhole(T x) const { @@ -165,7 +165,7 @@ public: if (getScale() < x.getScale()) throw Exception("Decimal result's scale is less then argiment's one", ErrorCodes::ARGUMENT_OUT_OF_BOUND); UInt32 scale_delta = getScale() - x.getScale(); /// scale_delta >= 0 - return getScaleMultiplier(scale_delta); + return T::getScaleMultiplier(scale_delta); } template @@ -181,7 +181,6 @@ public: void readText(T & x, ReadBuffer & istr, bool csv = false) const { readText(x, istr, precision, scale, csv); } static void readText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale, bool csv = false); static bool tryReadText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale); - static T getScaleMultiplier(UInt32 scale); private: const UInt32 precision; @@ -264,12 +263,12 @@ convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_fro MaxNativeType converted_value; if (scale_to > scale_from) { - converted_value = DataTypeDecimal::getScaleMultiplier(scale_to - scale_from); + converted_value = MaxFieldType::getScaleMultiplier(scale_to - scale_from); if (common::mulOverflow(static_cast(value), converted_value, converted_value)) throw Exception("Decimal convert overflow", ErrorCodes::DECIMAL_OVERFLOW); } else - converted_value = value / DataTypeDecimal::getScaleMultiplier(scale_from - scale_to); + converted_value = value / MaxFieldType::getScaleMultiplier(scale_from - scale_to); if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType)) { @@ -289,7 +288,7 @@ convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale) using ToFieldType = typename ToDataType::FieldType; if constexpr (std::is_floating_point_v) - return static_cast(value) / FromDataType::getScaleMultiplier(scale); + return static_cast(value) / FromFieldType::getScaleMultiplier(scale); else { FromFieldType converted_value = convertDecimals(value, scale, 0); @@ -320,14 +319,15 @@ inline std::enable_if_t && IsDataTypeDecimal) { if (!std::isfinite(value)) throw Exception("Decimal convert overflow. Cannot convert infinity or NaN to decimal", ErrorCodes::DECIMAL_OVERFLOW); - auto out = value * ToDataType::getScaleMultiplier(scale); + auto out = value * ToFieldType::getScaleMultiplier(scale); if constexpr (std::is_same_v) { static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64; diff --git a/dbms/src/Dictionaries/CacheDictionary.inc.h b/dbms/src/Dictionaries/CacheDictionary.inc.h index c10cde8c4fd..87005ac821f 100644 --- a/dbms/src/Dictionaries/CacheDictionary.inc.h +++ b/dbms/src/Dictionaries/CacheDictionary.inc.h @@ -3,8 +3,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -334,7 +334,7 @@ void CacheDictionary::update( backoff_end_time = now + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count)); tryLogException(last_exception, log, "Could not update cache dictionary '" + getName() + - "', next update is scheduled at " + DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(backoff_end_time))); + "', next update is scheduled at " + ext::to_string(backoff_end_time)); } } diff --git a/dbms/src/Functions/FunctionBinaryArithmetic.h b/dbms/src/Functions/FunctionBinaryArithmetic.h index 407c096a9e5..cab13a405c7 100644 --- a/dbms/src/Functions/FunctionBinaryArithmetic.h +++ b/dbms/src/Functions/FunctionBinaryArithmetic.h @@ -508,7 +508,7 @@ class FunctionBinaryArithmetic : public IFunction } std::stringstream function_name; - function_name << (function_is_plus ? "add" : "subtract") << interval_data_type->kindToString() << 's'; + function_name << (function_is_plus ? "add" : "subtract") << interval_data_type->getKind().toString() << 's'; return FunctionFactory::instance().get(function_name.str(), context); } diff --git a/dbms/src/Functions/FunctionsConversion.h b/dbms/src/Functions/FunctionsConversion.h index 09a23f83414..e0f828c395a 100644 --- a/dbms/src/Functions/FunctionsConversion.h +++ b/dbms/src/Functions/FunctionsConversion.h @@ -735,7 +735,7 @@ struct NameToDecimal128 { static constexpr auto name = "toDecimal128"; }; struct NameToInterval ## INTERVAL_KIND \ { \ static constexpr auto name = "toInterval" #INTERVAL_KIND; \ - static constexpr int kind = DataTypeInterval::INTERVAL_KIND; \ + static constexpr auto kind = IntervalKind::INTERVAL_KIND; \ }; DEFINE_NAME_TO_INTERVAL(Second) @@ -786,7 +786,7 @@ public: if constexpr (std::is_same_v) { - return std::make_shared(DataTypeInterval::Kind(Name::kind)); + return std::make_shared(Name::kind); } else if constexpr (to_decimal) { diff --git a/dbms/src/Functions/currentQuota.cpp b/dbms/src/Functions/currentQuota.cpp new file mode 100644 index 00000000000..fef26f333fc --- /dev/null +++ b/dbms/src/Functions/currentQuota.cpp @@ -0,0 +1,134 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +class FunctionCurrentQuota : public IFunction +{ + const String quota_name; + +public: + static constexpr auto name = "currentQuota"; + static FunctionPtr create(const Context & context) + { + return std::make_shared(context.getQuota()->getUsageInfo().quota_name); + } + + explicit FunctionCurrentQuota(const String & quota_name_) : quota_name{quota_name_} + { + } + + String getName() const override + { + return name; + } + size_t getNumberOfArguments() const override + { + return 0; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + return std::make_shared(); + } + + bool isDeterministic() const override { return false; } + + void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override + { + block.getByPosition(result).column = DataTypeString().createColumnConst(input_rows_count, quota_name); + } +}; + + +class FunctionCurrentQuotaId : public IFunction +{ + const UUID quota_id; + +public: + static constexpr auto name = "currentQuotaID"; + static FunctionPtr create(const Context & context) + { + return std::make_shared(context.getQuota()->getUsageInfo().quota_id); + } + + explicit FunctionCurrentQuotaId(const UUID quota_id_) : quota_id{quota_id_} + { + } + + String getName() const override + { + return name; + } + size_t getNumberOfArguments() const override + { + return 0; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + return std::make_shared(); + } + + bool isDeterministic() const override { return false; } + + void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override + { + block.getByPosition(result).column = DataTypeUUID().createColumnConst(input_rows_count, quota_id); + } +}; + + +class FunctionCurrentQuotaKey : public IFunction +{ + const String quota_key; + +public: + static constexpr auto name = "currentQuotaKey"; + static FunctionPtr create(const Context & context) + { + return std::make_shared(context.getQuota()->getUsageInfo().quota_key); + } + + explicit FunctionCurrentQuotaKey(const String & quota_key_) : quota_key{quota_key_} + { + } + + String getName() const override + { + return name; + } + size_t getNumberOfArguments() const override + { + return 0; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + return std::make_shared(); + } + + bool isDeterministic() const override { return false; } + + void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override + { + block.getByPosition(result).column = DataTypeString().createColumnConst(input_rows_count, quota_key); + } +}; + + +void registerFunctionCurrentQuota(FunctionFactory & factory) +{ + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp index ae75b9c0962..9529cd3a56a 100644 --- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp @@ -7,6 +7,7 @@ class FunctionFactory; void registerFunctionCurrentDatabase(FunctionFactory &); void registerFunctionCurrentUser(FunctionFactory &); +void registerFunctionCurrentQuota(FunctionFactory &); void registerFunctionHostName(FunctionFactory &); void registerFunctionFQDN(FunctionFactory &); void registerFunctionVisibleWidth(FunctionFactory &); @@ -62,6 +63,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) { registerFunctionCurrentDatabase(factory); registerFunctionCurrentUser(factory); + registerFunctionCurrentQuota(factory); registerFunctionHostName(factory); registerFunctionFQDN(factory); registerFunctionVisibleWidth(factory); diff --git a/dbms/src/Functions/toStartOfInterval.cpp b/dbms/src/Functions/toStartOfInterval.cpp index 21e500602e1..29ec814a6ee 100644 --- a/dbms/src/Functions/toStartOfInterval.cpp +++ b/dbms/src/Functions/toStartOfInterval.cpp @@ -23,11 +23,11 @@ namespace { static constexpr auto function_name = "toStartOfInterval"; - template + template struct Transform; template <> - struct Transform + struct Transform { static UInt16 execute(UInt16 d, UInt64 years, const DateLUTImpl & time_zone) { @@ -41,7 +41,7 @@ namespace }; template <> - struct Transform + struct Transform { static UInt16 execute(UInt16 d, UInt64 quarters, const DateLUTImpl & time_zone) { @@ -55,7 +55,7 @@ namespace }; template <> - struct Transform + struct Transform { static UInt16 execute(UInt16 d, UInt64 months, const DateLUTImpl & time_zone) { @@ -69,7 +69,7 @@ namespace }; template <> - struct Transform + struct Transform { static UInt16 execute(UInt16 d, UInt64 weeks, const DateLUTImpl & time_zone) { @@ -83,7 +83,7 @@ namespace }; template <> - struct Transform + struct Transform { static UInt32 execute(UInt16 d, UInt64 days, const DateLUTImpl & time_zone) { @@ -97,7 +97,7 @@ namespace }; template <> - struct Transform + struct Transform { static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } @@ -105,7 +105,7 @@ namespace }; template <> - struct Transform + struct Transform { static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } @@ -116,7 +116,7 @@ namespace }; template <> - struct Transform + struct Transform { static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } @@ -163,9 +163,9 @@ public: "Illegal type " + arguments[1].type->getName() + " of argument of function " + getName() + ". Should be an interval of time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - result_type_is_date = (interval_type->getKind() == DataTypeInterval::Year) - || (interval_type->getKind() == DataTypeInterval::Quarter) || (interval_type->getKind() == DataTypeInterval::Month) - || (interval_type->getKind() == DataTypeInterval::Week); + result_type_is_date = (interval_type->getKind() == IntervalKind::Year) + || (interval_type->getKind() == IntervalKind::Quarter) || (interval_type->getKind() == IntervalKind::Month) + || (interval_type->getKind() == IntervalKind::Week); }; auto check_timezone_argument = [&] @@ -177,7 +177,7 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (first_argument_is_date && result_type_is_date) throw Exception( - "The timezone argument of function " + getName() + " with interval type " + interval_type->kindToString() + "The timezone argument of function " + getName() + " with interval type " + interval_type->getKind().toString() + " is allowed only when the 1st argument has the type DateTime", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); }; @@ -269,28 +269,28 @@ private: switch (interval_type->getKind()) { - case DataTypeInterval::Second: - return execute(time_column, num_units, time_zone); - case DataTypeInterval::Minute: - return execute(time_column, num_units, time_zone); - case DataTypeInterval::Hour: - return execute(time_column, num_units, time_zone); - case DataTypeInterval::Day: - return execute(time_column, num_units, time_zone); - case DataTypeInterval::Week: - return execute(time_column, num_units, time_zone); - case DataTypeInterval::Month: - return execute(time_column, num_units, time_zone); - case DataTypeInterval::Quarter: - return execute(time_column, num_units, time_zone); - case DataTypeInterval::Year: - return execute(time_column, num_units, time_zone); + case IntervalKind::Second: + return execute(time_column, num_units, time_zone); + case IntervalKind::Minute: + return execute(time_column, num_units, time_zone); + case IntervalKind::Hour: + return execute(time_column, num_units, time_zone); + case IntervalKind::Day: + return execute(time_column, num_units, time_zone); + case IntervalKind::Week: + return execute(time_column, num_units, time_zone); + case IntervalKind::Month: + return execute(time_column, num_units, time_zone); + case IntervalKind::Quarter: + return execute(time_column, num_units, time_zone); + case IntervalKind::Year: + return execute(time_column, num_units, time_zone); } __builtin_unreachable(); } - template + template ColumnPtr execute(const ColumnVector & time_column, UInt64 num_units, const DateLUTImpl & time_zone) { const auto & time_data = time_column.getData(); diff --git a/dbms/src/IO/WriteHelpers.h b/dbms/src/IO/WriteHelpers.h index 509c37257ad..1f5b6ca7f42 100644 --- a/dbms/src/IO/WriteHelpers.h +++ b/dbms/src/IO/WriteHelpers.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include @@ -764,12 +763,6 @@ inline void writeText(const LocalDateTime & x, WriteBuffer & buf) { writeDateTim inline void writeText(const UUID & x, WriteBuffer & buf) { writeUUIDText(x, buf); } inline void writeText(const UInt128 & x, WriteBuffer & buf) { writeText(UUID(x), buf); } -template inline T decimalScaleMultiplier(UInt32 scale); -template <> inline Int32 decimalScaleMultiplier(UInt32 scale) { return common::exp10_i32(scale); } -template <> inline Int64 decimalScaleMultiplier(UInt32 scale) { return common::exp10_i64(scale); } -template <> inline Int128 decimalScaleMultiplier(UInt32 scale) { return common::exp10_i128(scale); } - - template void writeText(Decimal value, UInt32 scale, WriteBuffer & ostr) { @@ -781,7 +774,7 @@ void writeText(Decimal value, UInt32 scale, WriteBuffer & ostr) T whole_part = value; if (scale) - whole_part = value / decimalScaleMultiplier(scale); + whole_part = value / Decimal::getScaleMultiplier(scale); writeIntText(whole_part, ostr); if (scale) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 8dc4e57739f..f303356be34 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -24,9 +25,11 @@ #include #include #include +#include +#include +#include #include #include -#include #include #include #include @@ -37,7 +40,6 @@ #include #include #include -#include #include #include #include @@ -91,6 +93,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int SCALAR_ALREADY_EXISTS; extern const int UNKNOWN_SCALAR; + extern const int NOT_ENOUGH_PRIVILEGES; } @@ -130,8 +133,8 @@ struct ContextShared mutable std::optional external_models_loader; String default_profile_name; /// Default profile name used for default values. String system_profile_name; /// Profile used by system processes + AccessControlManager access_control_manager; std::unique_ptr users_manager; /// Known users. - Quotas quotas; /// Known quotas for resource use. mutable UncompressedCachePtr uncompressed_cache; /// The cache of decompressed blocks. mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files. ProcessList process_list; /// Executing queries at the moment. @@ -326,7 +329,7 @@ Context & Context::operator=(const Context &) = default; Context Context::createGlobal() { Context res; - res.quota = std::make_shared(); + res.quota = std::make_shared(); res.shared = std::make_shared(); return res; } @@ -585,12 +588,31 @@ const Poco::Util::AbstractConfiguration & Context::getConfigRef() const return shared->config ? *shared->config : Poco::Util::Application::instance().config(); } +AccessControlManager & Context::getAccessControlManager() +{ + auto lock = getLock(); + return shared->access_control_manager; +} + +const AccessControlManager & Context::getAccessControlManager() const +{ + auto lock = getLock(); + return shared->access_control_manager; +} + +void Context::checkQuotaManagementIsAllowed() +{ + if (!is_quota_management_allowed) + throw Exception( + "User " + client_info.current_user + " doesn't have enough privileges to manage quotas", ErrorCodes::NOT_ENOUGH_PRIVILEGES); +} + void Context::setUsersConfig(const ConfigurationPtr & config) { auto lock = getLock(); shared->users_config = config; + shared->access_control_manager.loadFromConfig(*shared->users_config); shared->users_manager->loadFromConfig(*shared->users_config); - shared->quotas.loadFromConfig(*shared->users_config); } ConfigurationPtr Context::getUsersConfig() @@ -631,7 +653,8 @@ void Context::calculateUserSettings() { auto lock = getLock(); - String profile = shared->users_manager->getUser(client_info.current_user)->profile; + auto user = getUser(client_info.current_user); + String profile = user->profile; /// 1) Set default settings (hardcoded values) /// NOTE: we ignore global_context settings (from which it is usually copied) @@ -646,6 +669,10 @@ void Context::calculateUserSettings() /// 3) Apply settings from current user setProfile(profile); + + quota = getAccessControlManager().createQuotaContext( + client_info.current_user, client_info.current_address.host(), client_info.quota_key); + is_quota_management_allowed = user->is_quota_management_allowed; } @@ -678,24 +705,9 @@ void Context::setUser(const String & name, const String & password, const Poco:: client_info.quota_key = quota_key; calculateUserSettings(); - - setQuota(user_props->quota, quota_key, name, address.host()); } -void Context::setQuota(const String & name, const String & quota_key, const String & user_name, const Poco::Net::IPAddress & address) -{ - auto lock = getLock(); - quota = shared->quotas.get(name, quota_key, user_name, address); -} - - -QuotaForIntervals & Context::getQuota() -{ - auto lock = getLock(); - return *quota; -} - void Context::checkDatabaseAccessRights(const std::string & database_name) const { auto lock = getLock(); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index e94d8125064..c3671990dc3 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -44,7 +44,7 @@ namespace DB struct ContextShared; class Context; -class QuotaForIntervals; +class QuotaContext; class EmbeddedDictionaries; class ExternalDictionariesLoader; class ExternalModelsLoader; @@ -77,6 +77,7 @@ class ActionLocksManager; using ActionLocksManagerPtr = std::shared_ptr; class ShellCommand; class ICompressionCodec; +class AccessControlManager; class SettingsConstraints; class RemoteHostFilter; @@ -137,7 +138,8 @@ private: InputInitializer input_initializer_callback; InputBlocksReader input_blocks_reader; - std::shared_ptr quota; /// Current quota. By default - empty quota, that have no limits. + std::shared_ptr quota; /// Current quota. By default - empty quota, that have no limits. + bool is_quota_management_allowed = false; /// Whether the current user is allowed to manage quotas via SQL commands. String current_database; Settings settings; /// Setting for query execution. std::shared_ptr settings_constraints; @@ -201,6 +203,11 @@ public: void setConfig(const ConfigurationPtr & config); const Poco::Util::AbstractConfiguration & getConfigRef() const; + AccessControlManager & getAccessControlManager(); + const AccessControlManager & getAccessControlManager() const; + std::shared_ptr getQuota() const { return quota; } + void checkQuotaManagementIsAllowed(); + /** Take the list of users, quotas and configuration profiles from this config. * The list of users is completely replaced. * The accumulated quota values are not reset if the quota is not deleted. @@ -240,9 +247,6 @@ public: ClientInfo & getClientInfo() { return client_info; } const ClientInfo & getClientInfo() const { return client_info; } - void setQuota(const String & name, const String & quota_key, const String & user_name, const Poco::Net::IPAddress & address); - QuotaForIntervals & getQuota(); - void addDependency(const DatabaseAndTableName & from, const DatabaseAndTableName & where); void removeDependency(const DatabaseAndTableName & from, const DatabaseAndTableName & where); Dependencies getDependencies(const String & database_name, const String & table_name) const; @@ -410,7 +414,6 @@ public: const Settings & getSettingsRef() const { return settings; } Settings & getSettingsRef() { return settings; } - void setProgressCallback(ProgressCallback callback); /// Used in InterpreterSelectQuery to pass it to the IBlockInputStream. ProgressCallback getProgressCallback() const; diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 10b8a02d660..c7c883fde79 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -2,13 +2,13 @@ #include #include -#include #include #include #include #include #include #include +#include #include @@ -288,7 +288,7 @@ class ExternalLoader::LoadingDispatcher : private boost::noncopyable public: /// Called to load or reload an object. using CreateObjectFunction = std::function; + const String & /* name */, const ObjectConfig & /* config */, const LoadablePtr & /* previous_version */)>; LoadingDispatcher( const CreateObjectFunction & create_object_function_, @@ -791,14 +791,13 @@ private: std::pair loadOneObject( const String & name, const ObjectConfig & config, - bool config_changed, LoadablePtr previous_version) { LoadablePtr new_object; std::exception_ptr new_exception; try { - new_object = create_object(name, config, config_changed, previous_version); + new_object = create_object(name, config, previous_version); } catch (...) { @@ -878,8 +877,7 @@ private: { if (next_update_time == TimePoint::max()) return String(); - return ", next update is scheduled at " - + DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(next_update_time)); + return ", next update is scheduled at " + ext::to_string(next_update_time); }; if (previous_version) tryLogException(new_exception, log, "Could not update " + type_name + " '" + name + "'" @@ -919,7 +917,8 @@ private: /// Use `create_function` to perform the actual loading. /// It's much better to do it with `mutex` unlocked because the loading can take a lot of time /// and require access to other objects. - auto [new_object, new_exception] = loadOneObject(name, info->object_config, info->config_changed, info->object); + bool need_complete_loading = !info->object || info->config_changed || info->forced_to_reload; + auto [new_object, new_exception] = loadOneObject(name, info->object_config, need_complete_loading ? nullptr : info->object); if (!new_object && !new_exception) throw Exception("No object created and no exception raised for " + type_name, ErrorCodes::LOGICAL_ERROR); @@ -1076,7 +1075,7 @@ private: ExternalLoader::ExternalLoader(const String & type_name_, Logger * log) : config_files_reader(std::make_unique(type_name_, log)) , loading_dispatcher(std::make_unique( - std::bind(&ExternalLoader::createObject, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4), + std::bind(&ExternalLoader::createObject, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3), type_name_, log)) , periodic_updater(std::make_unique(*config_files_reader, *loading_dispatcher)) @@ -1226,9 +1225,9 @@ void ExternalLoader::addObjectAndLoad( ExternalLoader::LoadablePtr ExternalLoader::createObject( - const String & name, const ObjectConfig & config, bool config_changed, const LoadablePtr & previous_version) const + const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const { - if (previous_version && !config_changed) + if (previous_version) return previous_version->clone(); return create(name, *config.config, config.key_in_config); diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index 19570d897a5..16cdbd449f2 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -175,7 +175,7 @@ protected: private: struct ObjectConfig; - LoadablePtr createObject(const String & name, const ObjectConfig & config, bool config_changed, const LoadablePtr & previous_version) const; + LoadablePtr createObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const; class LoadablesConfigReader; std::unique_ptr config_files_reader; diff --git a/dbms/src/Interpreters/IInterpreter.h b/dbms/src/Interpreters/IInterpreter.h index e1090061cf3..e2248a7ec7a 100644 --- a/dbms/src/Interpreters/IInterpreter.h +++ b/dbms/src/Interpreters/IInterpreter.h @@ -22,6 +22,9 @@ public: virtual bool canExecuteWithProcessors() const { return false; } + virtual bool ignoreQuota() const { return false; } + virtual bool ignoreLimits() const { return false; } + virtual ~IInterpreter() {} }; diff --git a/dbms/src/Interpreters/InterpreterCreateQuotaQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuotaQuery.cpp new file mode 100644 index 00000000000..0dd81f5cb27 --- /dev/null +++ b/dbms/src/Interpreters/InterpreterCreateQuotaQuery.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +BlockIO InterpreterCreateQuotaQuery::execute() +{ + context.checkQuotaManagementIsAllowed(); + const auto & query = query_ptr->as(); + auto & access_control = context.getAccessControlManager(); + + if (query.alter) + { + auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + { + auto updated_quota = typeid_cast>(entity->clone()); + updateQuotaFromQuery(*updated_quota, query); + return updated_quota; + }; + if (query.if_exists) + { + if (auto id = access_control.find(query.name)) + access_control.tryUpdate(*id, update_func); + } + else + access_control.update(access_control.getID(query.name), update_func); + } + else + { + auto new_quota = std::make_shared(); + updateQuotaFromQuery(*new_quota, query); + + if (query.if_not_exists) + access_control.tryInsert(new_quota); + else if (query.or_replace) + access_control.insertOrReplace(new_quota); + else + access_control.insert(new_quota); + } + + return {}; +} + + +void InterpreterCreateQuotaQuery::updateQuotaFromQuery(Quota & quota, const ASTCreateQuotaQuery & query) +{ + if (query.alter) + { + if (!query.new_name.empty()) + quota.setName(query.new_name); + } + else + quota.setName(query.name); + + if (query.key_type) + quota.key_type = *query.key_type; + + auto & quota_all_limits = quota.all_limits; + for (const auto & query_limits : query.all_limits) + { + auto duration = query_limits.duration; + + auto it = boost::range::find_if(quota_all_limits, [&](const Quota::Limits & x) { return x.duration == duration; }); + if (query_limits.unset_tracking) + { + if (it != quota_all_limits.end()) + quota_all_limits.erase(it); + continue; + } + + if (it == quota_all_limits.end()) + { + /// We keep `all_limits` sorted by duration. + it = quota_all_limits.insert( + boost::range::upper_bound( + quota_all_limits, + duration, + [](const std::chrono::seconds & lhs, const Quota::Limits & rhs) { return lhs < rhs.duration; }), + Quota::Limits{}); + it->duration = duration; + } + + auto & quota_limits = *it; + quota_limits.randomize_interval = query_limits.randomize_interval; + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + { + if (query_limits.max[resource_type]) + quota_limits.max[resource_type] = *query_limits.max[resource_type]; + } + } + + if (query.roles) + { + const auto & query_roles = *query.roles; + + /// We keep `roles` sorted. + quota.roles = query_roles.roles; + if (query_roles.current_user) + quota.roles.push_back(context.getClientInfo().current_user); + boost::range::sort(quota.roles); + quota.roles.erase(std::unique(quota.roles.begin(), quota.roles.end()), quota.roles.end()); + + quota.all_roles = query_roles.all_roles; + + /// We keep `except_roles` sorted. + quota.except_roles = query_roles.except_roles; + if (query_roles.except_current_user) + quota.except_roles.push_back(context.getClientInfo().current_user); + boost::range::sort(quota.except_roles); + quota.except_roles.erase(std::unique(quota.except_roles.begin(), quota.except_roles.end()), quota.except_roles.end()); + } +} +} diff --git a/dbms/src/Interpreters/InterpreterCreateQuotaQuery.h b/dbms/src/Interpreters/InterpreterCreateQuotaQuery.h new file mode 100644 index 00000000000..bbf91bbe1d3 --- /dev/null +++ b/dbms/src/Interpreters/InterpreterCreateQuotaQuery.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class ASTCreateQuotaQuery; +struct Quota; + + +class InterpreterCreateQuotaQuery : public IInterpreter +{ +public: + InterpreterCreateQuotaQuery(const ASTPtr & query_ptr_, Context & context_) : query_ptr(query_ptr_), context(context_) {} + + BlockIO execute() override; + + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } + +private: + void updateQuotaFromQuery(Quota & quota, const ASTCreateQuotaQuery & query); + + ASTPtr query_ptr; + Context & context; +}; +} diff --git a/dbms/src/Interpreters/InterpreterDropAccessEntityQuery.cpp b/dbms/src/Interpreters/InterpreterDropAccessEntityQuery.cpp new file mode 100644 index 00000000000..7f18084038c --- /dev/null +++ b/dbms/src/Interpreters/InterpreterDropAccessEntityQuery.cpp @@ -0,0 +1,31 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +BlockIO InterpreterDropAccessEntityQuery::execute() +{ + const auto & query = query_ptr->as(); + auto & access_control = context.getAccessControlManager(); + using Kind = ASTDropAccessEntityQuery::Kind; + + switch (query.kind) + { + case Kind::QUOTA: + { + context.checkQuotaManagementIsAllowed(); + if (query.if_exists) + access_control.tryRemove(access_control.find(query.names)); + else + access_control.remove(access_control.getIDs(query.names)); + return {}; + } + } + + __builtin_unreachable(); +} +} diff --git a/dbms/src/Interpreters/InterpreterDropAccessEntityQuery.h b/dbms/src/Interpreters/InterpreterDropAccessEntityQuery.h new file mode 100644 index 00000000000..2a0e749b265 --- /dev/null +++ b/dbms/src/Interpreters/InterpreterDropAccessEntityQuery.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class InterpreterDropAccessEntityQuery : public IInterpreter +{ +public: + InterpreterDropAccessEntityQuery(const ASTPtr & query_ptr_, Context & context_) : query_ptr(query_ptr_), context(context_) {} + + BlockIO execute() override; + +private: + ASTPtr query_ptr; + Context & context; +}; +} diff --git a/dbms/src/Interpreters/InterpreterFactory.cpp b/dbms/src/Interpreters/InterpreterFactory.cpp index d27c9c8baeb..33e9da95dfc 100644 --- a/dbms/src/Interpreters/InterpreterFactory.cpp +++ b/dbms/src/Interpreters/InterpreterFactory.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -9,7 +11,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -19,8 +23,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -31,8 +37,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -187,6 +195,22 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } + else if (query->as()) + { + return std::make_unique(query, context); + } + else if (query->as()) + { + return std::make_unique(query, context); + } + else if (query->as()) + { + return std::make_unique(query, context); + } else throw Exception("Unknown type of query: " + query->getID(), ErrorCodes::UNKNOWN_TYPE_OF_QUERY); } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 0f3d5d82f96..d4565f6894d 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -419,6 +419,17 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// null non-const columns to avoid useless memory allocations. However, a valid block sample /// requires all columns to be of size 0, thus we need to sanitize the block here. sanitizeBlock(result_header); + + /// Remove limits for some tables in the `system` database. + if (storage && (storage->getDatabaseName() == "system")) + { + String table_name = storage->getTableName(); + if ((table_name == "quotas") || (table_name == "quota_usage") || (table_name == "one")) + { + options.ignore_quota = true; + options.ignore_limits = true; + } + } } @@ -1776,14 +1787,14 @@ void InterpreterSelectQuery::executeFetchColumns( limits.speed_limits.timeout_before_checking_execution_speed = settings.timeout_before_checking_execution_speed; } - QuotaForIntervals & quota = context->getQuota(); + auto quota = context->getQuota(); for (auto & stream : streams) { if (!options.ignore_limits) stream->setLimits(limits); - if (options.to_stage == QueryProcessingStage::Complete) + if (!options.ignore_quota && (options.to_stage == QueryProcessingStage::Complete)) stream->setQuota(quota); } @@ -1793,7 +1804,7 @@ void InterpreterSelectQuery::executeFetchColumns( if (!options.ignore_limits) pipe.setLimits(limits); - if (options.to_stage == QueryProcessingStage::Complete) + if (!options.ignore_quota && (options.to_stage == QueryProcessingStage::Complete)) pipe.setQuota(quota); } } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.h b/dbms/src/Interpreters/InterpreterSelectQuery.h index 083a4ebe680..6b95d7aeea7 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.h +++ b/dbms/src/Interpreters/InterpreterSelectQuery.h @@ -74,6 +74,9 @@ public: QueryPipeline executeWithProcessors() override; bool canExecuteWithProcessors() const override { return true; } + bool ignoreLimits() const override { return options.ignore_limits; } + bool ignoreQuota() const override { return options.ignore_quota; } + Block getSampleBlock(); void ignoreWithTotals(); @@ -260,7 +263,7 @@ private: */ void initSettings(); - const SelectQueryOptions options; + SelectQueryOptions options; ASTPtr query_ptr; std::shared_ptr context; SyntaxAnalyzerResultPtr syntax_analyzer_result; diff --git a/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 2d7fde72875..e76f2668d3d 100644 --- a/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -107,6 +107,19 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( result_header = getCommonHeaderForUnion(headers); } + + /// InterpreterSelectWithUnionQuery ignores limits if all nested interpreters ignore limits. + bool all_nested_ignore_limits = true; + bool all_nested_ignore_quota = true; + for (auto & interpreter : nested_interpreters) + { + if (!interpreter->ignoreLimits()) + all_nested_ignore_limits = false; + if (!interpreter->ignoreQuota()) + all_nested_ignore_quota = false; + } + options.ignore_limits |= all_nested_ignore_limits; + options.ignore_quota |= all_nested_ignore_quota; } diff --git a/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.h b/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.h index 4e425d260e6..e18627fec2a 100644 --- a/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/dbms/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -34,6 +34,9 @@ public: QueryPipeline executeWithProcessors() override; bool canExecuteWithProcessors() const override { return true; } + bool ignoreLimits() const override { return options.ignore_limits; } + bool ignoreQuota() const override { return options.ignore_quota; } + Block getSampleBlock(); static Block getSampleBlock( @@ -45,7 +48,7 @@ public: ASTPtr getQuery() const { return query_ptr; } private: - const SelectQueryOptions options; + SelectQueryOptions options; ASTPtr query_ptr; std::shared_ptr context; diff --git a/dbms/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp new file mode 100644 index 00000000000..d0ef8992691 --- /dev/null +++ b/dbms/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +BlockIO InterpreterShowCreateAccessEntityQuery::execute() +{ + BlockIO res; + res.in = executeImpl(); + return res; +} + + +BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() +{ + const auto & show_query = query_ptr->as(); + + /// Build a create query. + ASTPtr create_query = getCreateQuotaQuery(show_query); + + /// Build the result column. + std::stringstream create_query_ss; + formatAST(*create_query, create_query_ss, false, true); + String create_query_str = create_query_ss.str(); + MutableColumnPtr column = ColumnString::create(); + column->insert(create_query_str); + + /// Prepare description of the result column. + std::stringstream desc_ss; + formatAST(show_query, desc_ss, false, true); + String desc = desc_ss.str(); + String prefix = "SHOW "; + if (startsWith(desc, prefix)) + desc = desc.substr(prefix.length()); /// `desc` always starts with "SHOW ", so we can trim this prefix. + + return std::make_shared(Block{{std::move(column), std::make_shared(), desc}}); +} + + +ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuotaQuery(const ASTShowCreateAccessEntityQuery & show_query) const +{ + auto & access_control = context.getAccessControlManager(); + + QuotaPtr quota; + if (show_query.current_quota) + quota = access_control.read(context.getQuota()->getUsageInfo().quota_id); + else + quota = access_control.read(show_query.name); + + auto create_query = std::make_shared(); + create_query->name = quota->getName(); + create_query->key_type = quota->key_type; + create_query->all_limits.reserve(quota->all_limits.size()); + + for (const auto & limits : quota->all_limits) + { + ASTCreateQuotaQuery::Limits create_query_limits; + create_query_limits.duration = limits.duration; + create_query_limits.randomize_interval = limits.randomize_interval; + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + if (limits.max[resource_type]) + create_query_limits.max[resource_type] = limits.max[resource_type]; + create_query->all_limits.push_back(create_query_limits); + } + + if (!quota->roles.empty() || quota->all_roles) + { + auto create_query_roles = std::make_shared(); + create_query_roles->roles = quota->roles; + create_query_roles->all_roles = quota->all_roles; + create_query_roles->except_roles = quota->except_roles; + create_query->roles = std::move(create_query_roles); + } + + return create_query; +} +} diff --git a/dbms/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h b/dbms/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h new file mode 100644 index 00000000000..94b06dadb19 --- /dev/null +++ b/dbms/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; +class ASTShowCreateAccessEntityQuery; + + +/** Returns a single item containing a statement which could be used to create a specified role. + */ +class InterpreterShowCreateAccessEntityQuery : public IInterpreter +{ +public: + InterpreterShowCreateAccessEntityQuery(const ASTPtr & query_ptr_, const Context & context_) + : query_ptr(query_ptr_), context(context_) {} + + BlockIO execute() override; + + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } + +private: + ASTPtr query_ptr; + const Context & context; + + BlockInputStreamPtr executeImpl(); + ASTPtr getCreateQuotaQuery(const ASTShowCreateAccessEntityQuery & show_query) const; +}; + + +} diff --git a/dbms/src/Interpreters/InterpreterShowQuotasQuery.cpp b/dbms/src/Interpreters/InterpreterShowQuotasQuery.cpp new file mode 100644 index 00000000000..73653e26781 --- /dev/null +++ b/dbms/src/Interpreters/InterpreterShowQuotasQuery.cpp @@ -0,0 +1,73 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +InterpreterShowQuotasQuery::InterpreterShowQuotasQuery(const ASTPtr & query_ptr_, Context & context_) + : query_ptr(query_ptr_), context(context_) +{ +} + + +String InterpreterShowQuotasQuery::getRewrittenQuery() +{ + const auto & query = query_ptr->as(); + + /// Transform the query into some kind of "SELECT from system.quotas" query. + String expr; + String filter; + String table_name; + String order_by; + if (query.usage) + { + expr = "name || ' key=\\'' || key || '\\'' || if(isNull(end_of_interval), '', ' interval=[' || " + "toString(end_of_interval - duration) || ' .. ' || " + "toString(end_of_interval) || ']'"; + for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + { + String column_name = Quota::resourceTypeToColumnName(resource_type); + expr += String{" || ' "} + column_name + "=' || toString(" + column_name + ")"; + expr += String{" || if(max_"} + column_name + "=0, '', '/' || toString(max_" + column_name + "))"; + } + expr += ")"; + + if (query.current) + filter = "(id = currentQuotaID()) AND (key = currentQuotaKey())"; + + table_name = "system.quota_usage"; + order_by = "name, key, duration"; + } + else + { + expr = "name"; + table_name = "system.quotas"; + order_by = "name"; + } + + /// Prepare description of the result column. + std::stringstream ss; + formatAST(query, ss, false, true); + String desc = ss.str(); + String prefix = "SHOW "; + if (startsWith(desc, prefix)) + desc = desc.substr(prefix.length()); /// `desc` always starts with "SHOW ", so we can trim this prefix. + + /// Build a new query. + return "SELECT " + expr + " AS " + backQuote(desc) + " FROM " + table_name + (filter.empty() ? "" : (" WHERE " + filter)) + + (order_by.empty() ? "" : (" ORDER BY " + order_by)); +} + + +BlockIO InterpreterShowQuotasQuery::execute() +{ + return executeQuery(getRewrittenQuery(), context, true); +} + +} diff --git a/dbms/src/Interpreters/InterpreterShowQuotasQuery.h b/dbms/src/Interpreters/InterpreterShowQuotasQuery.h new file mode 100644 index 00000000000..ae608e81ce5 --- /dev/null +++ b/dbms/src/Interpreters/InterpreterShowQuotasQuery.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +class InterpreterShowQuotasQuery : public IInterpreter +{ +public: + InterpreterShowQuotasQuery(const ASTPtr & query_ptr_, Context & context_); + + BlockIO execute() override; + + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } + +private: + ASTPtr query_ptr; + Context & context; + + String getRewrittenQuery(); +}; + +} diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index 664efca90f9..c742ac37a5f 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -121,7 +121,9 @@ void startStopAction(Context & context, ASTSystemQuery & query, StorageActionBlo InterpreterSystemQuery::InterpreterSystemQuery(const ASTPtr & query_ptr_, Context & context_) - : query_ptr(query_ptr_->clone()), context(context_), log(&Poco::Logger::get("InterpreterSystemQuery")) {} + : query_ptr(query_ptr_->clone()), context(context_), log(&Poco::Logger::get("InterpreterSystemQuery")) +{ +} BlockIO InterpreterSystemQuery::execute() diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.h b/dbms/src/Interpreters/InterpreterSystemQuery.h index 31945745c1e..97ff9d348e6 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.h +++ b/dbms/src/Interpreters/InterpreterSystemQuery.h @@ -20,6 +20,9 @@ public: BlockIO execute() override; + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } + private: ASTPtr query_ptr; Context & context; diff --git a/dbms/src/Interpreters/Quota.cpp b/dbms/src/Interpreters/Quota.cpp deleted file mode 100644 index 5123f4fd3e8..00000000000 --- a/dbms/src/Interpreters/Quota.cpp +++ /dev/null @@ -1,345 +0,0 @@ -#include - -#include - -#include -#include -#include -#include - -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int QUOTA_EXPIRED; - extern const int QUOTA_DOESNT_ALLOW_KEYS; - extern const int UNKNOWN_QUOTA; -} - - -template -void QuotaValues::initFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config) -{ - queries = config.getUInt64(config_elem + ".queries", 0); - errors = config.getUInt64(config_elem + ".errors", 0); - result_rows = config.getUInt64(config_elem + ".result_rows", 0); - result_bytes = config.getUInt64(config_elem + ".result_bytes", 0); - read_rows = config.getUInt64(config_elem + ".read_rows", 0); - read_bytes = config.getUInt64(config_elem + ".read_bytes", 0); - execution_time_usec = config.getUInt64(config_elem + ".execution_time", 0) * 1000000ULL; -} - -template void QuotaValues::initFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config); -template void QuotaValues>::initFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config); - - -void QuotaForInterval::initFromConfig( - const String & config_elem, time_t duration_, bool randomize_, time_t offset_, const Poco::Util::AbstractConfiguration & config) -{ - rounded_time.store(0, std::memory_order_relaxed); - duration = duration_; - randomize = randomize_; - offset = offset_; - max.initFromConfig(config_elem, config); -} - -void QuotaForInterval::checkExceeded(time_t current_time, const String & quota_name, const String & user_name) -{ - updateTime(current_time); - check(max.queries, used.queries, quota_name, user_name, "Queries"); - check(max.errors, used.errors, quota_name, user_name, "Errors"); - check(max.result_rows, used.result_rows, quota_name, user_name, "Total result rows"); - check(max.result_bytes, used.result_bytes, quota_name, user_name, "Total result bytes"); - check(max.read_rows, used.read_rows, quota_name, user_name, "Total rows read"); - check(max.read_bytes, used.read_bytes, quota_name, user_name, "Total bytes read"); - check(max.execution_time_usec / 1000000, used.execution_time_usec / 1000000, quota_name, user_name, "Total execution time"); -} - -String QuotaForInterval::toString() const -{ - std::stringstream res; - - auto loaded_rounded_time = rounded_time.load(std::memory_order_relaxed); - - res << std::fixed << std::setprecision(3) - << "Interval: " << LocalDateTime(loaded_rounded_time) << " - " << LocalDateTime(loaded_rounded_time + duration) << ".\n" - << "Queries: " << used.queries << ".\n" - << "Errors: " << used.errors << ".\n" - << "Result rows: " << used.result_rows << ".\n" - << "Result bytes: " << used.result_bytes << ".\n" - << "Read rows: " << used.read_rows << ".\n" - << "Read bytes: " << used.read_bytes << ".\n" - << "Execution time: " << used.execution_time_usec / 1000000.0 << " sec.\n"; - - return res.str(); -} - -void QuotaForInterval::addQuery() noexcept -{ - ++used.queries; -} - -void QuotaForInterval::addError() noexcept -{ - ++used.errors; -} - -void QuotaForInterval::checkAndAddResultRowsBytes(time_t current_time, const String & quota_name, const String & user_name, size_t rows, size_t bytes) -{ - used.result_rows += rows; - used.result_bytes += bytes; - checkExceeded(current_time, quota_name, user_name); -} - -void QuotaForInterval::checkAndAddReadRowsBytes(time_t current_time, const String & quota_name, const String & user_name, size_t rows, size_t bytes) -{ - used.read_rows += rows; - used.read_bytes += bytes; - checkExceeded(current_time, quota_name, user_name); -} - -void QuotaForInterval::checkAndAddExecutionTime(time_t current_time, const String & quota_name, const String & user_name, Poco::Timespan amount) -{ - /// Information about internals of Poco::Timespan used. - used.execution_time_usec += amount.totalMicroseconds(); - checkExceeded(current_time, quota_name, user_name); -} - -void QuotaForInterval::updateTime(time_t current_time) -{ - /** If current time is greater than end of interval, - * then clear accumulated quota values and switch to next interval [rounded_time, rounded_time + duration). - */ - - auto loaded_rounded_time = rounded_time.load(std::memory_order_acquire); - while (true) - { - if (current_time < loaded_rounded_time + static_cast(duration)) - break; - - time_t new_rounded_time = (current_time - offset) / duration * duration + offset; - if (rounded_time.compare_exchange_strong(loaded_rounded_time, new_rounded_time)) - { - used.clear(); - break; - } - } -} - -void QuotaForInterval::check( - size_t max_amount, size_t used_amount, - const String & quota_name, const String & user_name, const char * resource_name) -{ - if (max_amount && used_amount > max_amount) - { - std::stringstream message; - message << "Quota for user '" << user_name << "' for "; - - if (duration == 3600) - message << "1 hour"; - else if (duration == 60) - message << "1 minute"; - else if (duration % 3600 == 0) - message << (duration / 3600) << " hours"; - else if (duration % 60 == 0) - message << (duration / 60) << " minutes"; - else - message << duration << " seconds"; - - message << " has been exceeded. " - << resource_name << ": " << used_amount << ", max: " << max_amount << ". " - << "Interval will end at " << LocalDateTime(rounded_time.load(std::memory_order_relaxed) + duration) << ". " - << "Name of quota template: '" << quota_name << "'."; - - throw Exception(message.str(), ErrorCodes::QUOTA_EXPIRED); - } -} - - -void QuotaForIntervals::initFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config, pcg64 & rng) -{ - Poco::Util::AbstractConfiguration::Keys config_keys; - config.keys(config_elem, config_keys); - - for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it) - { - if (!startsWith(*it, "interval")) - continue; - - String interval_config_elem = config_elem + "." + *it; - time_t duration = config.getInt(interval_config_elem + ".duration", 0); - time_t offset = 0; - - if (!duration) /// Skip quotas with zero duration - continue; - - bool randomize = config.getBool(interval_config_elem + ".randomize", false); - if (randomize) - offset = std::uniform_int_distribution(0, duration - 1)(rng); - - cont[duration].initFromConfig(interval_config_elem, duration, randomize, offset, config); - } -} - -void QuotaForIntervals::setMax(const QuotaForIntervals & quota) -{ - for (Container::iterator it = cont.begin(); it != cont.end();) - { - if (quota.cont.count(it->first)) - ++it; - else - cont.erase(it++); - } - - for (auto & x : quota.cont) - { - if (!cont.count(x.first)) - cont.emplace(x.first, x.second); - else - cont[x.first].max = x.second.max; - } -} - -void QuotaForIntervals::checkExceeded(time_t current_time) -{ - for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it) - it->second.checkExceeded(current_time, quota_name, user_name); -} - -void QuotaForIntervals::addQuery() noexcept -{ - for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it) - it->second.addQuery(); -} - -void QuotaForIntervals::addError() noexcept -{ - for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it) - it->second.addError(); -} - -void QuotaForIntervals::checkAndAddResultRowsBytes(time_t current_time, size_t rows, size_t bytes) -{ - for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it) - it->second.checkAndAddResultRowsBytes(current_time, quota_name, user_name, rows, bytes); -} - -void QuotaForIntervals::checkAndAddReadRowsBytes(time_t current_time, size_t rows, size_t bytes) -{ - for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it) - it->second.checkAndAddReadRowsBytes(current_time, quota_name, user_name, rows, bytes); -} - -void QuotaForIntervals::checkAndAddExecutionTime(time_t current_time, Poco::Timespan amount) -{ - for (Container::reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it) - it->second.checkAndAddExecutionTime(current_time, quota_name, user_name, amount); -} - -String QuotaForIntervals::toString() const -{ - std::stringstream res; - - for (Container::const_reverse_iterator it = cont.rbegin(); it != cont.rend(); ++it) - res << std::endl << it->second.toString(); - - return res.str(); -} - - -void Quota::loadFromConfig(const String & config_elem, const String & name_, const Poco::Util::AbstractConfiguration & config, pcg64 & rng) -{ - name = name_; - - bool new_keyed_by_ip = config.has(config_elem + ".keyed_by_ip"); - bool new_is_keyed = new_keyed_by_ip || config.has(config_elem + ".keyed"); - - if (new_is_keyed != is_keyed || new_keyed_by_ip != keyed_by_ip) - { - keyed_by_ip = new_keyed_by_ip; - is_keyed = new_is_keyed; - /// Meaning of keys has been changed. Throw away accumulated values. - quota_for_keys.clear(); - } - - ignore_key_if_not_keyed = config.has(config_elem + ".ignore_key_if_not_keyed"); - - QuotaForIntervals new_max(name, {}); - new_max.initFromConfig(config_elem, config, rng); - if (!new_max.hasEqualConfiguration(max)) - { - max = new_max; - for (auto & quota : quota_for_keys) - quota.second->setMax(max); - } -} - -QuotaForIntervalsPtr Quota::get(const String & quota_key, const String & user_name, const Poco::Net::IPAddress & ip) -{ - if (!quota_key.empty() && !ignore_key_if_not_keyed && (!is_keyed || keyed_by_ip)) - throw Exception("Quota " + name + " (for user " + user_name + ") doesn't allow client supplied keys.", - ErrorCodes::QUOTA_DOESNT_ALLOW_KEYS); - - /** Quota is calculated separately: - * - for each IP-address, if 'keyed_by_ip'; - * - otherwise for each 'quota_key', if present; - * - otherwise for each 'user_name'. - */ - - UInt64 quota_key_hashed = sipHash64( - keyed_by_ip - ? ip.toString() - : (!quota_key.empty() - ? quota_key - : user_name)); - - std::lock_guard lock(mutex); - - Container::iterator it = quota_for_keys.find(quota_key_hashed); - if (quota_for_keys.end() == it) - it = quota_for_keys.emplace(quota_key_hashed, std::make_shared(max, user_name)).first; - - return it->second; -} - - -void Quotas::loadFromConfig(const Poco::Util::AbstractConfiguration & config) -{ - pcg64 rng; - - Poco::Util::AbstractConfiguration::Keys config_keys; - config.keys("quotas", config_keys); - - /// Remove keys, that now absent in config. - std::set keys_set(config_keys.begin(), config_keys.end()); - for (Container::iterator it = cont.begin(); it != cont.end();) - { - if (keys_set.count(it->first)) - ++it; - else - cont.erase(it++); - } - - for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it) - { - if (!cont.count(*it)) - cont.try_emplace(*it); - cont[*it].loadFromConfig("quotas." + *it, *it, config, rng); - } -} - -QuotaForIntervalsPtr Quotas::get(const String & name, const String & quota_key, const String & user_name, const Poco::Net::IPAddress & ip) -{ - Container::iterator it = cont.find(name); - if (cont.end() == it) - throw Exception("Unknown quota " + name, ErrorCodes::UNKNOWN_QUOTA); - - return it->second.get(quota_key, user_name, ip); -} - -} diff --git a/dbms/src/Interpreters/Quota.h b/dbms/src/Interpreters/Quota.h deleted file mode 100644 index c1fb3f143fb..00000000000 --- a/dbms/src/Interpreters/Quota.h +++ /dev/null @@ -1,263 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - -#include -#include - -#include - -#include -#include -#include - - -namespace DB -{ - -/** Quota for resources consumption for specific interval. - * Used to limit resource usage by user. - * Quota is applied "softly" - could be slightly exceed, because it is checked usually only on each block of processed data. - * Accumulated values are not persisted and are lost on server restart. - * Quota is local to server, - * but for distributed queries, accumulated values for read rows and bytes - * are collected from all participating servers and accumulated locally. - */ - -/// Used both for maximum allowed values and for counters of current accumulated values. -template /// either size_t or std::atomic -struct QuotaValues -{ - /// Zero values (for maximums) means no limit. - Counter queries; /// Number of queries. - Counter errors; /// Number of queries with exceptions. - Counter result_rows; /// Number of rows returned as result. - Counter result_bytes; /// Number of bytes returned as result. - Counter read_rows; /// Number of rows read from tables. - Counter read_bytes; /// Number of bytes read from tables. - Counter execution_time_usec; /// Total amount of query execution time in microseconds. - - QuotaValues() - { - clear(); - } - - QuotaValues(const QuotaValues & rhs) - { - tuple() = rhs.tuple(); - } - - QuotaValues & operator=(const QuotaValues & rhs) - { - tuple() = rhs.tuple(); - return *this; - } - - void clear() - { - tuple() = std::make_tuple(0, 0, 0, 0, 0, 0, 0); - } - - void initFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config); - - bool operator== (const QuotaValues & rhs) const - { - return tuple() == rhs.tuple(); - } - -private: - auto tuple() - { - return std::forward_as_tuple(queries, errors, result_rows, result_bytes, read_rows, read_bytes, execution_time_usec); - } - - auto tuple() const - { - return std::make_tuple(queries, errors, result_rows, result_bytes, read_rows, read_bytes, execution_time_usec); - } -}; - -template <> -inline auto QuotaValues>::tuple() const -{ - return std::make_tuple( - queries.load(std::memory_order_relaxed), - errors.load(std::memory_order_relaxed), - result_rows.load(std::memory_order_relaxed), - result_bytes.load(std::memory_order_relaxed), - read_rows.load(std::memory_order_relaxed), - read_bytes.load(std::memory_order_relaxed), - execution_time_usec.load(std::memory_order_relaxed)); -} - - -/// Time, rounded down to start of interval; limits for that interval and accumulated values. -struct QuotaForInterval -{ - std::atomic rounded_time {0}; - size_t duration = 0; - bool randomize = false; - time_t offset = 0; /// Offset of interval for randomization (to avoid DoS if intervals for many users end at one time). - QuotaValues max; - QuotaValues> used; - - QuotaForInterval() = default; - QuotaForInterval(time_t duration_) : duration(duration_) {} - - void initFromConfig(const String & config_elem, time_t duration_, bool randomize_, time_t offset_, const Poco::Util::AbstractConfiguration & config); - - /// Increase current value. - void addQuery() noexcept; - void addError() noexcept; - - /// Check if quota is already exceeded. If that, throw an exception. - void checkExceeded(time_t current_time, const String & quota_name, const String & user_name); - - /// Check corresponding value. If exceeded, throw an exception. Otherwise, increase that value. - void checkAndAddResultRowsBytes(time_t current_time, const String & quota_name, const String & user_name, size_t rows, size_t bytes); - void checkAndAddReadRowsBytes(time_t current_time, const String & quota_name, const String & user_name, size_t rows, size_t bytes); - void checkAndAddExecutionTime(time_t current_time, const String & quota_name, const String & user_name, Poco::Timespan amount); - - /// Get a text, describing what quota is exceeded. - String toString() const; - - /// Only compare configuration, not accumulated (used) values or random offsets. - bool operator== (const QuotaForInterval & rhs) const - { - return randomize == rhs.randomize - && duration == rhs.duration - && max == rhs.max; - } - - QuotaForInterval & operator= (const QuotaForInterval & rhs) - { - rounded_time.store(rhs.rounded_time.load(std::memory_order_relaxed)); - duration = rhs.duration; - randomize = rhs.randomize; - offset = rhs.offset; - max = rhs.max; - used = rhs.used; - return *this; - } - - QuotaForInterval(const QuotaForInterval & rhs) - { - *this = rhs; - } - -private: - /// Reset counters of used resources, if interval for quota is expired. - void updateTime(time_t current_time); - void check(size_t max_amount, size_t used_amount, - const String & quota_name, const String & user_name, const char * resource_name); -}; - - -struct Quota; - -/// Length of interval -> quota: maximum allowed and currently accumulated values for that interval (example: 3600 -> values for current hour). -class QuotaForIntervals -{ -private: - /// While checking, will walk through intervals in order of decreasing size - from largest to smallest. - /// To report first about largest interval on what quota was exceeded. - using Container = std::map; - Container cont; - - std::string quota_name; - std::string user_name; /// user name is set only for current counters for user, not for object that contain maximum values (limits). - -public: - QuotaForIntervals(const std::string & quota_name_, const std::string & user_name_) - : quota_name(quota_name_), user_name(user_name_) {} - - QuotaForIntervals(const QuotaForIntervals & other, const std::string & user_name_) - : QuotaForIntervals(other) - { - user_name = user_name_; - } - - QuotaForIntervals() = default; - QuotaForIntervals(const QuotaForIntervals &) = default; - QuotaForIntervals & operator=(const QuotaForIntervals &) = default; - - /// Is there at least one interval for counting quota? - bool empty() const - { - return cont.empty(); - } - - void initFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config, pcg64 & rng); - - /// Set maximum values (limits) from passed argument. - /// Remove intervals that does not exist in argument. Add intervals from argument, that we don't have. - void setMax(const QuotaForIntervals & quota); - - void addQuery() noexcept; - void addError() noexcept; - - void checkExceeded(time_t current_time); - - void checkAndAddResultRowsBytes(time_t current_time, size_t rows, size_t bytes); - void checkAndAddReadRowsBytes(time_t current_time, size_t rows, size_t bytes); - void checkAndAddExecutionTime(time_t current_time, Poco::Timespan amount); - - /// Get text, describing what part of quota has been exceeded. - String toString() const; - - bool hasEqualConfiguration(const QuotaForIntervals & rhs) const - { - return cont == rhs.cont && quota_name == rhs.quota_name; - } -}; - -using QuotaForIntervalsPtr = std::shared_ptr; - - -/// Quota key -> quotas (max and current values) for intervals. If quota doesn't have keys, then values stored at key 0. -struct Quota -{ - using Container = std::unordered_map; - - String name; - - /// Maximum values from config. - QuotaForIntervals max; - /// Maximum and accumulated values for different keys. - /// For all keys, maximum values are the same and taken from 'max'. - Container quota_for_keys; - std::mutex mutex; - - bool is_keyed = false; - - /// If the quota is not keyed, but the user passed some key, ignore it instead of throwing exception. - /// For transitional periods, when you want to enable quota keys - /// - first, enable passing keys from your application, then make quota keyed in ClickHouse users config. - bool ignore_key_if_not_keyed = false; - - bool keyed_by_ip = false; - - void loadFromConfig(const String & config_elem, const String & name_, const Poco::Util::AbstractConfiguration & config, pcg64 & rng); - QuotaForIntervalsPtr get(const String & quota_key, const String & user_name, const Poco::Net::IPAddress & ip); -}; - - -class Quotas -{ -private: - /// Name of quota -> quota. - using Container = std::unordered_map; - Container cont; - -public: - void loadFromConfig(const Poco::Util::AbstractConfiguration & config); - QuotaForIntervalsPtr get(const String & name, const String & quota_key, - const String & user_name, const Poco::Net::IPAddress & ip); -}; - -} diff --git a/dbms/src/Interpreters/SelectQueryOptions.h b/dbms/src/Interpreters/SelectQueryOptions.h index 4fd94a830b8..a49245f5609 100644 --- a/dbms/src/Interpreters/SelectQueryOptions.h +++ b/dbms/src/Interpreters/SelectQueryOptions.h @@ -24,19 +24,16 @@ struct SelectQueryOptions { QueryProcessingStage::Enum to_stage; size_t subquery_depth; - bool only_analyze; - bool modify_inplace; - bool remove_duplicates; - bool ignore_limits; + bool only_analyze = false; + bool modify_inplace = false; + bool remove_duplicates = false; + bool ignore_quota = false; + bool ignore_limits = false; SelectQueryOptions(QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, size_t depth = 0) - : to_stage(stage) - , subquery_depth(depth) - , only_analyze(false) - , modify_inplace(false) - , remove_duplicates(false) - , ignore_limits(false) - {} + : to_stage(stage), subquery_depth(depth) + { + } SelectQueryOptions copy() const { return *this; } diff --git a/dbms/src/Interpreters/Users.cpp b/dbms/src/Interpreters/Users.cpp index 2ca2873e95b..e66b5119f84 100644 --- a/dbms/src/Interpreters/Users.cpp +++ b/dbms/src/Interpreters/Users.cpp @@ -49,7 +49,6 @@ User::User(const String & name_, const String & config_elem, const Poco::Util::A } profile = config.getString(config_elem + ".profile"); - quota = config.getString(config_elem + ".quota"); /// Fill list of allowed hosts. const auto config_networks = config_elem + ".networks"; @@ -130,7 +129,9 @@ User::User(const String & name_, const String & config_elem, const Poco::Util::A } } } + + if (config.has(config_elem + ".allow_quota_management")) + is_quota_management_allowed = config.getBool(config_elem + ".allow_quota_management"); } - } diff --git a/dbms/src/Interpreters/Users.h b/dbms/src/Interpreters/Users.h index e116772855a..6f9a47c4422 100644 --- a/dbms/src/Interpreters/Users.h +++ b/dbms/src/Interpreters/Users.h @@ -30,7 +30,6 @@ struct User Authentication authentication; String profile; - String quota; AllowedClientHosts allowed_client_hosts; @@ -48,6 +47,8 @@ struct User using DatabaseMap = std::unordered_map; DatabaseMap table_props; + bool is_quota_management_allowed = false; + User(const String & name_, const String & config_elem, const Poco::Util::AbstractConfiguration & config); }; diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 41c8e288ffe..2c6bf087f8d 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -24,7 +24,7 @@ #include -#include +#include #include #include #include @@ -150,7 +150,7 @@ static void logException(Context & context, QueryLogElement & elem) static void onExceptionBeforeStart(const String & query_for_logging, Context & context, time_t current_time) { /// Exception before the query execution. - context.getQuota().addError(); + context.getQuota()->used(Quota::ERRORS, 1, /* check_exceeded = */ false); const Settings & settings = context.getSettingsRef(); @@ -271,11 +271,6 @@ static std::tuple executeQueryImpl( /// Check the limits. checkASTSizeLimits(*ast, settings); - QuotaForIntervals & quota = context.getQuota(); - - quota.addQuery(); /// NOTE Seems that when new time interval has come, first query is not accounted in number of queries. - quota.checkExceeded(current_time); - /// Put query to process list. But don't put SHOW PROCESSLIST query itself. ProcessList::EntryPtr process_list_entry; if (!internal && !ast->as()) @@ -313,6 +308,21 @@ static std::tuple executeQueryImpl( auto interpreter = InterpreterFactory::get(ast, context, stage); bool use_processors = settings.experimental_use_processors && allow_processors && interpreter->canExecuteWithProcessors(); + QuotaContextPtr quota; + if (!interpreter->ignoreQuota()) + { + quota = context.getQuota(); + quota->used(Quota::QUERIES, 1); + quota->checkExceeded(Quota::ERRORS); + } + + IBlockInputStream::LocalLimits limits; + if (!interpreter->ignoreLimits()) + { + limits.mode = IBlockInputStream::LIMITS_CURRENT; + limits.size_limits = SizeLimits(settings.max_result_rows, settings.max_result_bytes, settings.result_overflow_mode); + } + if (use_processors) pipeline = interpreter->executeWithProcessors(); else @@ -339,17 +349,12 @@ static std::tuple executeQueryImpl( /// Hold element of process list till end of query execution. res.process_list_entry = process_list_entry; - IBlockInputStream::LocalLimits limits; - limits.mode = IBlockInputStream::LIMITS_CURRENT; - limits.size_limits = SizeLimits(settings.max_result_rows, settings.max_result_bytes, settings.result_overflow_mode); - if (use_processors) { - pipeline.setProgressCallback(context.getProgressCallback()); - pipeline.setProcessListElement(context.getProcessListElement()); - /// Limits on the result, the quota on the result, and also callback for progress. /// Limits apply only to the final result. + pipeline.setProgressCallback(context.getProgressCallback()); + pipeline.setProcessListElement(context.getProcessListElement()); if (stage == QueryProcessingStage::Complete) { pipeline.resize(1); @@ -363,17 +368,18 @@ static std::tuple executeQueryImpl( } else { + /// Limits on the result, the quota on the result, and also callback for progress. + /// Limits apply only to the final result. if (res.in) { res.in->setProgressCallback(context.getProgressCallback()); res.in->setProcessListElement(context.getProcessListElement()); - - /// Limits on the result, the quota on the result, and also callback for progress. - /// Limits apply only to the final result. if (stage == QueryProcessingStage::Complete) { - res.in->setLimits(limits); - res.in->setQuota(quota); + if (!interpreter->ignoreQuota()) + res.in->setQuota(quota); + if (!interpreter->ignoreLimits()) + res.in->setLimits(limits); } } @@ -484,7 +490,7 @@ static std::tuple executeQueryImpl( auto exception_callback = [elem, &context, log_queries] () mutable { - context.getQuota().addError(); + context.getQuota()->used(Quota::ERRORS, 1, /* check_exceeded = */ false); elem.type = QueryLogElement::EXCEPTION_WHILE_PROCESSING; diff --git a/dbms/src/Parsers/ASTCreateQuotaQuery.cpp b/dbms/src/Parsers/ASTCreateQuotaQuery.cpp new file mode 100644 index 00000000000..2814515d61f --- /dev/null +++ b/dbms/src/Parsers/ASTCreateQuotaQuery.cpp @@ -0,0 +1,142 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + using KeyType = Quota::KeyType; + using ResourceType = Quota::ResourceType; + using ResourceAmount = Quota::ResourceAmount; + + + void formatKeyType(const KeyType & key_type, const IAST::FormatSettings & settings) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " KEYED BY " << (settings.hilite ? IAST::hilite_none : "") << "'" + << Quota::getNameOfKeyType(key_type) << "'"; + } + + + void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") + << backQuote(new_name); + } + + + void formatLimit(ResourceType resource_type, ResourceAmount max, const IAST::FormatSettings & settings) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX " << Quota::resourceTypeToKeyword(resource_type) + << (settings.hilite ? IAST::hilite_none : ""); + + settings.ostr << (settings.hilite ? IAST::hilite_operator : "") << " = " << (settings.hilite ? IAST::hilite_none : ""); + + if (max == Quota::UNLIMITED) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ANY" << (settings.hilite ? IAST::hilite_none : ""); + else if (resource_type == Quota::EXECUTION_TIME) + settings.ostr << Quota::executionTimeToSeconds(max); + else + settings.ostr << max; + } + + + void formatLimits(const ASTCreateQuotaQuery::Limits & limits, const IAST::FormatSettings & settings) + { + auto interval_kind = IntervalKind::fromAvgSeconds(limits.duration.count()); + Int64 num_intervals = limits.duration.count() / interval_kind.toAvgSeconds(); + + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") + << " FOR" + << (limits.randomize_interval ? " RANDOMIZED" : "") + << " INTERVAL " + << (settings.hilite ? IAST::hilite_none : "") + << num_intervals << " " + << (settings.hilite ? IAST::hilite_keyword : "") + << interval_kind.toKeyword() + << (settings.hilite ? IAST::hilite_none : ""); + + if (limits.unset_tracking) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " UNSET TRACKING" << (settings.hilite ? IAST::hilite_none : ""); + } + else + { + bool limit_found = false; + for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + { + if (limits.max[resource_type]) + { + if (limit_found) + settings.ostr << ","; + limit_found = true; + formatLimit(resource_type, *limits.max[resource_type], settings); + } + } + if (!limit_found) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TRACKING" << (settings.hilite ? IAST::hilite_none : ""); + } + } + + void formatAllLimits(const std::vector & all_limits, const IAST::FormatSettings & settings) + { + bool need_comma = false; + for (auto & limits : all_limits) + { + if (need_comma) + settings.ostr << ","; + need_comma = true; + + formatLimits(limits, settings); + } + } + + void formatRoles(const ASTRoleList & roles, const IAST::FormatSettings & settings) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); + roles.format(settings); + } +} + + +String ASTCreateQuotaQuery::getID(char) const +{ + return "CreateQuotaQuery"; +} + + +ASTPtr ASTCreateQuotaQuery::clone() const +{ + return std::make_shared(*this); +} + + +void ASTCreateQuotaQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : "") << (alter ? "ALTER QUOTA" : "CREATE QUOTA") + << (settings.hilite ? hilite_none : ""); + + if (if_exists) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " IF EXISTS" << (settings.hilite ? hilite_none : ""); + else if (if_not_exists) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " IF NOT EXISTS" << (settings.hilite ? hilite_none : ""); + else if (or_replace) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " OR REPLACE" << (settings.hilite ? hilite_none : ""); + + settings.ostr << " " << backQuoteIfNeed(name); + + if (!new_name.empty()) + formatRenameTo(new_name, settings); + + if (key_type) + formatKeyType(*key_type, settings); + + formatAllLimits(all_limits, settings); + + if (roles) + formatRoles(*roles, settings); +} +} diff --git a/dbms/src/Parsers/ASTCreateQuotaQuery.h b/dbms/src/Parsers/ASTCreateQuotaQuery.h new file mode 100644 index 00000000000..056a445f23b --- /dev/null +++ b/dbms/src/Parsers/ASTCreateQuotaQuery.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class ASTRoleList; + + +/** CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name + * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + * {[SET] MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = {number | ANY} } [,...] | + * [SET] TRACKING} [,...]] + * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] + * + * ALTER QUOTA [IF EXISTS] name + * [RENAME TO new_name] + * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + * {[SET] MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = {number | ANY} } [,...] | + * [SET] TRACKING | + * UNSET TRACKING} [,...]] + * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] + */ +class ASTCreateQuotaQuery : public IAST +{ +public: + bool alter = false; + + bool if_exists = false; + bool if_not_exists = false; + bool or_replace = false; + + String name; + String new_name; + + using KeyType = Quota::KeyType; + std::optional key_type; + + using ResourceType = Quota::ResourceType; + using ResourceAmount = Quota::ResourceAmount; + static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + + struct Limits + { + std::optional max[MAX_RESOURCE_TYPE]; + bool unset_tracking = false; + std::chrono::seconds duration = std::chrono::seconds::zero(); + bool randomize_interval = false; + }; + std::vector all_limits; + + std::shared_ptr roles; + + String getID(char) const override; + ASTPtr clone() const override; + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; +} diff --git a/dbms/src/Parsers/ASTDropAccessEntityQuery.cpp b/dbms/src/Parsers/ASTDropAccessEntityQuery.cpp new file mode 100644 index 00000000000..80d69ed5316 --- /dev/null +++ b/dbms/src/Parsers/ASTDropAccessEntityQuery.cpp @@ -0,0 +1,56 @@ +#include +#include + + +namespace DB +{ +namespace +{ + using Kind = ASTDropAccessEntityQuery::Kind; + + const char * kindToKeyword(Kind kind) + { + switch (kind) + { + case Kind::QUOTA: return "QUOTA"; + } + __builtin_unreachable(); + } +} + + +ASTDropAccessEntityQuery::ASTDropAccessEntityQuery(Kind kind_) + : kind(kind_), keyword(kindToKeyword(kind_)) +{ +} + + +String ASTDropAccessEntityQuery::getID(char) const +{ + return String("DROP ") + keyword + " query"; +} + + +ASTPtr ASTDropAccessEntityQuery::clone() const +{ + return std::make_shared(*this); +} + + +void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : "") + << "DROP " << keyword + << (if_exists ? " IF EXISTS" : "") + << (settings.hilite ? hilite_none : ""); + + bool need_comma = false; + for (const auto & name : names) + { + if (need_comma) + settings.ostr << ','; + need_comma = true; + settings.ostr << ' ' << backQuoteIfNeed(name); + } +} +} diff --git a/dbms/src/Parsers/ASTDropAccessEntityQuery.h b/dbms/src/Parsers/ASTDropAccessEntityQuery.h new file mode 100644 index 00000000000..91b76253db4 --- /dev/null +++ b/dbms/src/Parsers/ASTDropAccessEntityQuery.h @@ -0,0 +1,28 @@ +#pragma once + +#include + + +namespace DB +{ + +/** DROP QUOTA [IF EXISTS] name [,...] + */ +class ASTDropAccessEntityQuery : public IAST +{ +public: + enum class Kind + { + QUOTA, + }; + const Kind kind; + const char * const keyword; + bool if_exists = false; + Strings names; + + ASTDropAccessEntityQuery(Kind kind_); + String getID(char) const override; + ASTPtr clone() const override; + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; +} diff --git a/dbms/src/Parsers/ASTRoleList.cpp b/dbms/src/Parsers/ASTRoleList.cpp new file mode 100644 index 00000000000..9e0a4fffc36 --- /dev/null +++ b/dbms/src/Parsers/ASTRoleList.cpp @@ -0,0 +1,56 @@ +#include +#include + + +namespace DB +{ +void ASTRoleList::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + if (empty()) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); + return; + } + + bool need_comma = false; + if (current_user) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "CURRENT_USER" << (settings.hilite ? IAST::hilite_none : ""); + } + + for (auto & role : roles) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << backQuoteIfNeed(role); + } + + if (all_roles) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ALL" << (settings.hilite ? IAST::hilite_none : ""); + if (except_current_user || !except_roles.empty()) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (settings.hilite ? IAST::hilite_none : ""); + need_comma = false; + + if (except_current_user) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "CURRENT_USER" << (settings.hilite ? IAST::hilite_none : ""); + } + + for (auto & except_role : except_roles) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << backQuoteIfNeed(except_role); + } + } + } +} +} diff --git a/dbms/src/Parsers/ASTRoleList.h b/dbms/src/Parsers/ASTRoleList.h new file mode 100644 index 00000000000..5e8859732c2 --- /dev/null +++ b/dbms/src/Parsers/ASTRoleList.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + + +namespace DB +{ +/// {role|CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {role|CURRENT_USER} [,...] +class ASTRoleList : public IAST +{ +public: + Strings roles; + bool current_user = false; + bool all_roles = false; + Strings except_roles; + bool except_current_user = false; + + bool empty() const { return roles.empty() && !current_user && !all_roles; } + + String getID(char) const override { return "RoleList"; } + ASTPtr clone() const override { return std::make_shared(*this); } + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; +} diff --git a/dbms/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/dbms/src/Parsers/ASTShowCreateAccessEntityQuery.cpp new file mode 100644 index 00000000000..8509a902014 --- /dev/null +++ b/dbms/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -0,0 +1,51 @@ +#include +#include + + +namespace DB +{ +namespace +{ + using Kind = ASTShowCreateAccessEntityQuery::Kind; + + const char * kindToKeyword(Kind kind) + { + switch (kind) + { + case Kind::QUOTA: return "QUOTA"; + } + __builtin_unreachable(); + } +} + + +ASTShowCreateAccessEntityQuery::ASTShowCreateAccessEntityQuery(Kind kind_) + : kind(kind_), keyword(kindToKeyword(kind_)) +{ +} + + +String ASTShowCreateAccessEntityQuery::getID(char) const +{ + return String("SHOW CREATE ") + keyword + " query"; +} + + +ASTPtr ASTShowCreateAccessEntityQuery::clone() const +{ + return std::make_shared(*this); +} + + +void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : "") + << "SHOW CREATE " << keyword + << (settings.hilite ? hilite_none : ""); + + if (current_quota) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " CURRENT" << (settings.hilite ? hilite_none : ""); + else + settings.ostr << " " << backQuoteIfNeed(name); +} +} diff --git a/dbms/src/Parsers/ASTShowCreateAccessEntityQuery.h b/dbms/src/Parsers/ASTShowCreateAccessEntityQuery.h new file mode 100644 index 00000000000..32c0ace101b --- /dev/null +++ b/dbms/src/Parsers/ASTShowCreateAccessEntityQuery.h @@ -0,0 +1,30 @@ +#pragma once + +#include + + +namespace DB +{ +/** SHOW CREATE QUOTA [name | CURRENT] + */ +class ASTShowCreateAccessEntityQuery : public ASTQueryWithOutput +{ +public: + enum class Kind + { + QUOTA, + }; + const Kind kind; + const char * const keyword; + String name; + bool current_quota = false; + + ASTShowCreateAccessEntityQuery(Kind kind_); + String getID(char) const override; + ASTPtr clone() const override; + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; + +} diff --git a/dbms/src/Parsers/ASTShowQuotasQuery.cpp b/dbms/src/Parsers/ASTShowQuotasQuery.cpp new file mode 100644 index 00000000000..ca7bd5e853f --- /dev/null +++ b/dbms/src/Parsers/ASTShowQuotasQuery.cpp @@ -0,0 +1,35 @@ +#include +#include + + +namespace DB +{ +String ASTShowQuotasQuery::getID(char) const +{ + if (usage) + return "SHOW QUOTA USAGE query"; + else + return "SHOW QUOTAS query"; +} + + +ASTPtr ASTShowQuotasQuery::clone() const +{ + return std::make_shared(*this); +} + + +void ASTShowQuotasQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + settings.ostr << (settings.hilite ? hilite_keyword : ""); + + if (usage && current) + settings.ostr << "SHOW QUOTA USAGE"; + else if (usage) + settings.ostr << "SHOW QUOTA USAGE ALL"; + else + settings.ostr << "SHOW QUOTAS"; + + settings.ostr << (settings.hilite ? hilite_none : ""); +} +} diff --git a/dbms/src/Parsers/ASTShowQuotasQuery.h b/dbms/src/Parsers/ASTShowQuotasQuery.h new file mode 100644 index 00000000000..27a08a99a54 --- /dev/null +++ b/dbms/src/Parsers/ASTShowQuotasQuery.h @@ -0,0 +1,24 @@ +#pragma once + +#include + + +namespace DB +{ +/** SHOW QUOTAS + * SHOW QUOTA USAGE [CURRENT | ALL] + */ +class ASTShowQuotasQuery : public ASTQueryWithOutput +{ +public: + bool usage = false; + bool current = false; + + String getID(char) const override; + ASTPtr clone() const override; + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; + +} diff --git a/dbms/src/Parsers/CommonParsers.h b/dbms/src/Parsers/CommonParsers.h index 2eafc1c1853..85b5217b617 100644 --- a/dbms/src/Parsers/CommonParsers.h +++ b/dbms/src/Parsers/CommonParsers.h @@ -46,99 +46,6 @@ protected: } }; -class ParserInterval: public IParserBase -{ -public: - enum class IntervalKind - { - Incorrect, - Second, - Minute, - Hour, - Day, - Week, - Month, - Quarter, - Year - }; - - IntervalKind interval_kind; - - ParserInterval() : interval_kind(IntervalKind::Incorrect) {} - - const char * getToIntervalKindFunctionName() - { - switch (interval_kind) - { - case ParserInterval::IntervalKind::Second: - return "toIntervalSecond"; - case ParserInterval::IntervalKind::Minute: - return "toIntervalMinute"; - case ParserInterval::IntervalKind::Hour: - return "toIntervalHour"; - case ParserInterval::IntervalKind::Day: - return "toIntervalDay"; - case ParserInterval::IntervalKind::Week: - return "toIntervalWeek"; - case ParserInterval::IntervalKind::Month: - return "toIntervalMonth"; - case ParserInterval::IntervalKind::Quarter: - return "toIntervalQuarter"; - case ParserInterval::IntervalKind::Year: - return "toIntervalYear"; - default: - return nullptr; - } - } - -protected: - const char * getName() const override { return "interval"; } - - bool parseImpl(Pos & pos, ASTPtr & /*node*/, Expected & expected) override - { - if (ParserKeyword("SECOND").ignore(pos, expected) || ParserKeyword("SQL_TSI_SECOND").ignore(pos, expected) - || ParserKeyword("SS").ignore(pos, expected) || ParserKeyword("S").ignore(pos, expected)) - interval_kind = IntervalKind::Second; - else if ( - ParserKeyword("MINUTE").ignore(pos, expected) || ParserKeyword("SQL_TSI_MINUTE").ignore(pos, expected) - || ParserKeyword("MI").ignore(pos, expected) || ParserKeyword("N").ignore(pos, expected)) - interval_kind = IntervalKind::Minute; - else if ( - ParserKeyword("HOUR").ignore(pos, expected) || ParserKeyword("SQL_TSI_HOUR").ignore(pos, expected) - || ParserKeyword("HH").ignore(pos, expected)) - interval_kind = IntervalKind::Hour; - else if ( - ParserKeyword("DAY").ignore(pos, expected) || ParserKeyword("SQL_TSI_DAY").ignore(pos, expected) - || ParserKeyword("DD").ignore(pos, expected) || ParserKeyword("D").ignore(pos, expected)) - interval_kind = IntervalKind::Day; - else if ( - ParserKeyword("WEEK").ignore(pos, expected) || ParserKeyword("SQL_TSI_WEEK").ignore(pos, expected) - || ParserKeyword("WK").ignore(pos, expected) || ParserKeyword("WW").ignore(pos, expected)) - interval_kind = IntervalKind::Week; - else if ( - ParserKeyword("MONTH").ignore(pos, expected) || ParserKeyword("SQL_TSI_MONTH").ignore(pos, expected) - || ParserKeyword("MM").ignore(pos, expected) || ParserKeyword("M").ignore(pos, expected)) - interval_kind = IntervalKind::Month; - else if ( - ParserKeyword("QUARTER").ignore(pos, expected) || ParserKeyword("SQL_TSI_QUARTER").ignore(pos, expected) - || ParserKeyword("QQ").ignore(pos, expected) || ParserKeyword("Q").ignore(pos, expected)) - interval_kind = IntervalKind::Quarter; - else if ( - ParserKeyword("YEAR").ignore(pos, expected) || ParserKeyword("SQL_TSI_YEAR").ignore(pos, expected) - || ParserKeyword("YYYY").ignore(pos, expected) || ParserKeyword("YY").ignore(pos, expected)) - interval_kind = IntervalKind::Year; - else - interval_kind = IntervalKind::Incorrect; - - if (interval_kind == IntervalKind::Incorrect) - { - expected.add(pos, "YEAR, QUARTER, MONTH, WEEK, DAY, HOUR, MINUTE or SECOND"); - return false; - } - /// one of ParserKeyword already made ++pos - return true; - } -}; // Parser always returns true and do nothing. class ParserNothing : public IParserBase diff --git a/dbms/src/Parsers/ExpressionElementParsers.cpp b/dbms/src/Parsers/ExpressionElementParsers.cpp index 70be49cdd04..1f1ba4edee7 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.cpp +++ b/dbms/src/Parsers/ExpressionElementParsers.cpp @@ -20,7 +20,7 @@ #include #include -#include +#include #include #include #include @@ -690,44 +690,11 @@ bool ParserExtractExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & exp ++pos; ASTPtr expr; - const char * function_name = nullptr; - ParserInterval interval_parser; - if (!interval_parser.ignore(pos, expected)) + IntervalKind interval_kind; + if (!parseIntervalKind(pos, expected, interval_kind)) return false; - switch (interval_parser.interval_kind) - { - case ParserInterval::IntervalKind::Second: - function_name = "toSecond"; - break; - case ParserInterval::IntervalKind::Minute: - function_name = "toMinute"; - break; - case ParserInterval::IntervalKind::Hour: - function_name = "toHour"; - break; - case ParserInterval::IntervalKind::Day: - function_name = "toDayOfMonth"; - break; - case ParserInterval::IntervalKind::Week: - // TODO: SELECT toRelativeWeekNum(toDate('2017-06-15')) - toRelativeWeekNum(toStartOfYear(toDate('2017-06-15'))) - // else if (ParserKeyword("WEEK").ignore(pos, expected)) - // function_name = "toRelativeWeekNum"; - return false; - case ParserInterval::IntervalKind::Month: - function_name = "toMonth"; - break; - case ParserInterval::IntervalKind::Quarter: - function_name = "toQuarter"; - break; - case ParserInterval::IntervalKind::Year: - function_name = "toYear"; - break; - default: - return false; - } - ParserKeyword s_from("FROM"); if (!s_from.ignore(pos, expected)) return false; @@ -742,7 +709,7 @@ bool ParserExtractExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & exp auto function = std::make_shared(); auto exp_list = std::make_shared(); - function->name = function_name; //"toYear"; + function->name = interval_kind.toNameOfFunctionExtractTimePart(); function->arguments = exp_list; function->children.push_back(exp_list); exp_list->children.push_back(expr); @@ -770,8 +737,8 @@ bool ParserDateAddExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & exp return false; ++pos; - ParserInterval interval_parser; - if (interval_parser.ignore(pos, expected)) + IntervalKind interval_kind; + if (parseIntervalKind(pos, expected, interval_kind)) { /// function(unit, offset, timestamp) if (pos->type != TokenType::Comma) @@ -804,20 +771,18 @@ bool ParserDateAddExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & exp if (!ParserExpression().parse(pos, offset_node, expected)) return false; - interval_parser.ignore(pos, expected); - + if (!parseIntervalKind(pos, expected, interval_kind)) + return false; } if (pos->type != TokenType::ClosingRoundBracket) return false; ++pos; - const char * interval_function_name = interval_parser.getToIntervalKindFunctionName(); - auto interval_expr_list_args = std::make_shared(); interval_expr_list_args->children = {offset_node}; auto interval_func_node = std::make_shared(); - interval_func_node->name = interval_function_name; + interval_func_node->name = interval_kind.toNameOfFunctionToIntervalDataType(); interval_func_node->arguments = std::move(interval_expr_list_args); interval_func_node->children.push_back(interval_func_node->arguments); @@ -836,7 +801,6 @@ bool ParserDateAddExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & exp bool ParserDateDiffExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - const char * interval_name = nullptr; ASTPtr left_node; ASTPtr right_node; @@ -848,40 +812,10 @@ bool ParserDateDiffExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & ex return false; ++pos; - ParserInterval interval_parser; - if (!interval_parser.ignore(pos, expected)) + IntervalKind interval_kind; + if (!parseIntervalKind(pos, expected, interval_kind)) return false; - switch (interval_parser.interval_kind) - { - case ParserInterval::IntervalKind::Second: - interval_name = "second"; - break; - case ParserInterval::IntervalKind::Minute: - interval_name = "minute"; - break; - case ParserInterval::IntervalKind::Hour: - interval_name = "hour"; - break; - case ParserInterval::IntervalKind::Day: - interval_name = "day"; - break; - case ParserInterval::IntervalKind::Week: - interval_name = "week"; - break; - case ParserInterval::IntervalKind::Month: - interval_name = "month"; - break; - case ParserInterval::IntervalKind::Quarter: - interval_name = "quarter"; - break; - case ParserInterval::IntervalKind::Year: - interval_name = "year"; - break; - default: - return false; - } - if (pos->type != TokenType::Comma) return false; ++pos; @@ -901,7 +835,7 @@ bool ParserDateDiffExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & ex ++pos; auto expr_list_args = std::make_shared(); - expr_list_args->children = {std::make_shared(interval_name), left_node, right_node}; + expr_list_args->children = {std::make_shared(interval_kind.toDateDiffUnit()), left_node, right_node}; auto func_node = std::make_shared(); func_node->name = "dateDiff"; diff --git a/dbms/src/Parsers/ExpressionListParsers.cpp b/dbms/src/Parsers/ExpressionListParsers.cpp index 6d33368d88b..060d1e89f02 100644 --- a/dbms/src/Parsers/ExpressionListParsers.cpp +++ b/dbms/src/Parsers/ExpressionListParsers.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include @@ -604,13 +604,10 @@ bool ParserIntervalOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Expec if (!ParserExpressionWithOptionalAlias(false).parse(pos, expr, expected)) return false; - - ParserInterval interval_parser; - if (!interval_parser.ignore(pos, expected)) + IntervalKind interval_kind; + if (!parseIntervalKind(pos, expected, interval_kind)) return false; - const char * function_name = interval_parser.getToIntervalKindFunctionName(); - /// the function corresponding to the operator auto function = std::make_shared(); @@ -618,7 +615,7 @@ bool ParserIntervalOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Expec auto exp_list = std::make_shared(); /// the first argument of the function is the previous element, the second is the next one - function->name = function_name; + function->name = interval_kind.toNameOfFunctionToIntervalDataType(); function->arguments = exp_list; function->children.push_back(exp_list); diff --git a/dbms/src/Parsers/IParserBase.cpp b/dbms/src/Parsers/IParserBase.cpp index 64162a595c9..e4caffa992e 100644 --- a/dbms/src/Parsers/IParserBase.cpp +++ b/dbms/src/Parsers/IParserBase.cpp @@ -12,20 +12,15 @@ namespace ErrorCodes bool IParserBase::parse(Pos & pos, ASTPtr & node, Expected & expected) { - Pos begin = pos; expected.add(pos, getName()); - pos.increaseDepth(); - bool res = parseImpl(pos, node, expected); - pos.decreaseDepth(); - - if (!res) + return wrapParseImpl(pos, IncreaseDepthTag{}, [&] { - node = nullptr; - pos = begin; - } - - return res; + bool res = parseImpl(pos, node, expected); + if (!res) + node = nullptr; + return res; + }); } } diff --git a/dbms/src/Parsers/IParserBase.h b/dbms/src/Parsers/IParserBase.h index 67b222b1b71..95951d5acb8 100644 --- a/dbms/src/Parsers/IParserBase.h +++ b/dbms/src/Parsers/IParserBase.h @@ -11,6 +11,30 @@ namespace DB class IParserBase : public IParser { public: + template + static bool wrapParseImpl(Pos & pos, const F & func) + { + Pos begin = pos; + bool res = func(); + if (!res) + pos = begin; + return res; + } + + struct IncreaseDepthTag {}; + + template + static bool wrapParseImpl(Pos & pos, IncreaseDepthTag, const F & func) + { + Pos begin = pos; + pos.increaseDepth(); + bool res = func(); + pos.decreaseDepth(); + if (!res) + pos = begin; + return res; + } + bool parse(Pos & pos, ASTPtr & node, Expected & expected); protected: diff --git a/dbms/src/Parsers/ParserCreateQuotaQuery.cpp b/dbms/src/Parsers/ParserCreateQuotaQuery.cpp new file mode 100644 index 00000000000..cc5fa4bfbcc --- /dev/null +++ b/dbms/src/Parsers/ParserCreateQuotaQuery.cpp @@ -0,0 +1,261 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int SYNTAX_ERROR; +} + + +namespace +{ + using KeyType = Quota::KeyType; + using ResourceType = Quota::ResourceType; + using ResourceAmount = Quota::ResourceAmount; + + bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_name, bool alter) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!new_name.empty() || !alter) + return false; + + if (!ParserKeyword{"RENAME TO"}.ignore(pos, expected)) + return false; + + return parseIdentifierOrStringLiteral(pos, expected, new_name); + }); + } + + bool parseKeyType(IParserBase::Pos & pos, Expected & expected, std::optional & key_type) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (key_type) + return false; + + if (!ParserKeyword{"KEYED BY"}.ignore(pos, expected)) + return false; + + ASTPtr key_type_ast; + if (!ParserStringLiteral().parse(pos, key_type_ast, expected)) + return false; + + const String & key_type_str = key_type_ast->as().value.safeGet(); + for (auto kt : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) + if (boost::iequals(Quota::getNameOfKeyType(kt), key_type_str)) + { + key_type = kt; + return true; + } + + String all_key_types_str; + for (auto kt : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) + all_key_types_str += String(all_key_types_str.empty() ? "" : ", ") + "'" + Quota::getNameOfKeyType(kt) + "'"; + String msg = "Quota cannot be keyed by '" + key_type_str + "'. Expected one of these literals: " + all_key_types_str; + throw Exception(msg, ErrorCodes::SYNTAX_ERROR); + }); + } + + bool parseLimit(IParserBase::Pos & pos, Expected & expected, ResourceType & resource_type, ResourceAmount & max) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!ParserKeyword{"MAX"}.ignore(pos, expected)) + return false; + + bool resource_type_set = false; + for (auto rt : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + { + if (ParserKeyword{Quota::resourceTypeToKeyword(rt)}.ignore(pos, expected)) + { + resource_type = rt; + resource_type_set = true; + break; + } + } + if (!resource_type_set) + return false; + + if (!ParserToken{TokenType::Equals}.ignore(pos, expected)) + return false; + + ASTPtr max_ast; + if (ParserNumber{}.parse(pos, max_ast, expected)) + { + const Field & max_field = max_ast->as().value; + if (resource_type == Quota::EXECUTION_TIME) + max = Quota::secondsToExecutionTime(applyVisitor(FieldVisitorConvertToNumber(), max_field)); + else + max = applyVisitor(FieldVisitorConvertToNumber(), max_field); + } + else if (ParserKeyword{"ANY"}.ignore(pos, expected)) + { + max = Quota::UNLIMITED; + } + else + return false; + + return true; + }); + } + + bool parseCommaAndLimit(IParserBase::Pos & pos, Expected & expected, ResourceType & resource_type, ResourceAmount & max) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!ParserToken{TokenType::Comma}.ignore(pos, expected)) + return false; + + return parseLimit(pos, expected, resource_type, max); + }); + } + + bool parseLimits(IParserBase::Pos & pos, Expected & expected, ASTCreateQuotaQuery::Limits & limits, bool alter) + { + return IParserBase::wrapParseImpl(pos, [&] + { + ASTCreateQuotaQuery::Limits new_limits; + if (!ParserKeyword{"FOR"}.ignore(pos, expected)) + return false; + + new_limits.randomize_interval = ParserKeyword{"RANDOMIZED"}.ignore(pos, expected); + + if (!ParserKeyword{"INTERVAL"}.ignore(pos, expected)) + return false; + + ASTPtr num_intervals_ast; + if (!ParserNumber{}.parse(pos, num_intervals_ast, expected)) + return false; + + double num_intervals = applyVisitor(FieldVisitorConvertToNumber(), num_intervals_ast->as().value); + + IntervalKind interval_kind; + if (!parseIntervalKind(pos, expected, interval_kind)) + return false; + + new_limits.duration = std::chrono::seconds(static_cast(num_intervals * interval_kind.toAvgSeconds())); + + if (alter && ParserKeyword{"UNSET TRACKING"}.ignore(pos, expected)) + { + new_limits.unset_tracking = true; + } + else if (ParserKeyword{"SET TRACKING"}.ignore(pos, expected) || ParserKeyword{"TRACKING"}.ignore(pos, expected)) + { + } + else + { + ParserKeyword{"SET"}.ignore(pos, expected); + ResourceType resource_type; + ResourceAmount max; + if (!parseLimit(pos, expected, resource_type, max)) + return false; + + new_limits.max[resource_type] = max; + while (parseCommaAndLimit(pos, expected, resource_type, max)) + new_limits.max[resource_type] = max; + } + + limits = new_limits; + return true; + }); + } + + bool parseAllLimits(IParserBase::Pos & pos, Expected & expected, std::vector & all_limits, bool alter) + { + return IParserBase::wrapParseImpl(pos, [&] + { + do + { + ASTCreateQuotaQuery::Limits limits; + if (!parseLimits(pos, expected, limits, alter)) + return false; + all_limits.push_back(limits); + } + while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + return true; + }); + } + + bool parseRoles(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & roles) + { + return IParserBase::wrapParseImpl(pos, [&] + { + ASTPtr node; + if (roles || !ParserKeyword{"TO"}.ignore(pos, expected) || !ParserRoleList{}.parse(pos, node, expected)) + return false; + + roles = std::static_pointer_cast(node); + return true; + }); + } +} + + +bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + bool alter; + if (ParserKeyword{"CREATE QUOTA"}.ignore(pos, expected)) + alter = false; + else if (ParserKeyword{"ALTER QUOTA"}.ignore(pos, expected)) + alter = true; + else + return false; + + bool if_exists = false; + bool if_not_exists = false; + bool or_replace = false; + if (alter) + { + if (ParserKeyword{"IF EXISTS"}.ignore(pos, expected)) + if_exists = true; + } + else + { + if (ParserKeyword{"IF NOT EXISTS"}.ignore(pos, expected)) + if_not_exists = true; + else if (ParserKeyword{"OR REPLACE"}.ignore(pos, expected)) + or_replace = true; + } + + String name; + if (!parseIdentifierOrStringLiteral(pos, expected, name)) + return false; + + String new_name; + std::optional key_type; + std::vector all_limits; + std::shared_ptr roles; + + while (parseRenameTo(pos, expected, new_name, alter) || parseKeyType(pos, expected, key_type) + || parseAllLimits(pos, expected, all_limits, alter) || parseRoles(pos, expected, roles)) + ; + + auto query = std::make_shared(); + node = query; + + query->alter = alter; + query->if_exists = if_exists; + query->if_not_exists = if_not_exists; + query->or_replace = or_replace; + query->name = std::move(name); + query->new_name = std::move(new_name); + query->key_type = key_type; + query->all_limits = std::move(all_limits); + query->roles = std::move(roles); + + return true; +} +} diff --git a/dbms/src/Parsers/ParserCreateQuotaQuery.h b/dbms/src/Parsers/ParserCreateQuotaQuery.h new file mode 100644 index 00000000000..aef33f72e67 --- /dev/null +++ b/dbms/src/Parsers/ParserCreateQuotaQuery.h @@ -0,0 +1,31 @@ +#pragma once + +#include + + +namespace DB +{ +/** Parses queries like + * CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name + * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + * {[SET] MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = {number | ANY} } [,...] | + * [SET] TRACKING} [,...]] + * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] + * + * ALTER QUOTA [IF EXISTS] name + * [RENAME TO new_name] + * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + * {[SET] MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = {number | ANY} } [,...] | + * [SET] TRACKING | + * UNSET TRACKING} [,...]] + * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] + */ +class ParserCreateQuotaQuery : public IParserBase +{ +protected: + const char * getName() const override { return "CREATE QUOTA or ALTER QUOTA query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; +} diff --git a/dbms/src/Parsers/ParserDropAccessEntityQuery.cpp b/dbms/src/Parsers/ParserDropAccessEntityQuery.cpp new file mode 100644 index 00000000000..c6d5ff889fc --- /dev/null +++ b/dbms/src/Parsers/ParserDropAccessEntityQuery.cpp @@ -0,0 +1,45 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + if (!ParserKeyword{"DROP"}.ignore(pos, expected)) + return false; + + using Kind = ASTDropAccessEntityQuery::Kind; + Kind kind; + if (ParserKeyword{"QUOTA"}.ignore(pos, expected)) + kind = Kind::QUOTA; + else + return false; + + bool if_exists = false; + if (ParserKeyword{"IF EXISTS"}.ignore(pos, expected)) + if_exists = true; + + Strings names; + do + { + String name; + if (!parseIdentifierOrStringLiteral(pos, expected, name)) + return false; + + names.push_back(std::move(name)); + } + while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + + auto query = std::make_shared(kind); + node = query; + + query->if_exists = if_exists; + query->names = std::move(names); + + return true; +} +} diff --git a/dbms/src/Parsers/ParserDropAccessEntityQuery.h b/dbms/src/Parsers/ParserDropAccessEntityQuery.h new file mode 100644 index 00000000000..f479e0d0add --- /dev/null +++ b/dbms/src/Parsers/ParserDropAccessEntityQuery.h @@ -0,0 +1,17 @@ +#pragma once + +#include + + +namespace DB +{ +/** Parses queries like + * DROP QUOTA [IF EXISTS] name [,...] + */ +class ParserDropAccessEntityQuery : public IParserBase +{ +protected: + const char * getName() const override { return "DROP QUOTA query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; +} diff --git a/dbms/src/Parsers/ParserQuery.cpp b/dbms/src/Parsers/ParserQuery.cpp index b7bdd517a43..a3bb652032e 100644 --- a/dbms/src/Parsers/ParserQuery.cpp +++ b/dbms/src/Parsers/ParserQuery.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include namespace DB @@ -22,12 +24,16 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserUseQuery use_p; ParserSetQuery set_p; ParserSystemQuery system_p; + ParserCreateQuotaQuery create_quota_p; + ParserDropAccessEntityQuery drop_access_entity_p; bool res = query_with_output_p.parse(pos, node, expected) || insert_p.parse(pos, node, expected) || use_p.parse(pos, node, expected) || set_p.parse(pos, node, expected) - || system_p.parse(pos, node, expected); + || system_p.parse(pos, node, expected) + || create_quota_p.parse(pos, node, expected) + || drop_access_entity_p.parse(pos, node, expected); return res; } diff --git a/dbms/src/Parsers/ParserQueryWithOutput.cpp b/dbms/src/Parsers/ParserQueryWithOutput.cpp index 1c44c639848..d08ae984c90 100644 --- a/dbms/src/Parsers/ParserQueryWithOutput.cpp +++ b/dbms/src/Parsers/ParserQueryWithOutput.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include namespace DB @@ -34,6 +36,8 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserOptimizeQuery optimize_p; ParserKillQueryQuery kill_query_p; ParserWatchQuery watch_p; + ParserShowCreateAccessEntityQuery show_create_access_entity_p; + ParserShowQuotasQuery show_quotas_p; ASTPtr query; @@ -49,6 +53,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec analyze_syntax = true; bool parsed = select_p.parse(pos, query, expected) + || show_create_access_entity_p.parse(pos, query, expected) /// should be before `show_tables_p` || show_tables_p.parse(pos, query, expected) || table_p.parse(pos, query, expected) || describe_table_p.parse(pos, query, expected) @@ -60,7 +65,8 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec || check_p.parse(pos, query, expected) || kill_query_p.parse(pos, query, expected) || optimize_p.parse(pos, query, expected) - || watch_p.parse(pos, query, expected); + || watch_p.parse(pos, query, expected) + || show_quotas_p.parse(pos, query, expected); if (!parsed) return false; diff --git a/dbms/src/Parsers/ParserRoleList.cpp b/dbms/src/Parsers/ParserRoleList.cpp new file mode 100644 index 00000000000..ac8914de776 --- /dev/null +++ b/dbms/src/Parsers/ParserRoleList.cpp @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +bool ParserRoleList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + Strings roles; + bool current_user = false; + bool all_roles = false; + Strings except_roles; + bool except_current_user = false; + + bool except_mode = false; + while (true) + { + if (ParserKeyword{"NONE"}.ignore(pos, expected)) + { + } + else if (ParserKeyword{"CURRENT_USER"}.ignore(pos, expected) || + ParserKeyword{"currentUser"}.ignore(pos, expected)) + { + if (ParserToken{TokenType::OpeningRoundBracket}.ignore(pos, expected)) + { + if (!ParserToken{TokenType::ClosingRoundBracket}.ignore(pos, expected)) + return false; + } + if (except_mode && !current_user) + except_current_user = true; + else + current_user = true; + } + else if (ParserKeyword{"ALL"}.ignore(pos, expected)) + { + all_roles = true; + if (ParserKeyword{"EXCEPT"}.ignore(pos, expected)) + { + except_mode = true; + continue; + } + } + else + { + String name; + if (!parseIdentifierOrStringLiteral(pos, expected, name)) + return false; + if (except_mode && (boost::range::find(roles, name) == roles.end())) + except_roles.push_back(name); + else + roles.push_back(name); + } + + if (!ParserToken{TokenType::Comma}.ignore(pos, expected)) + break; + } + + if (all_roles) + { + current_user = false; + roles.clear(); + } + + auto result = std::make_shared(); + result->roles = std::move(roles); + result->current_user = current_user; + result->all_roles = all_roles; + result->except_roles = std::move(except_roles); + result->except_current_user = except_current_user; + node = result; + return true; +} + +} diff --git a/dbms/src/Parsers/ParserRoleList.h b/dbms/src/Parsers/ParserRoleList.h new file mode 100644 index 00000000000..eca205a748c --- /dev/null +++ b/dbms/src/Parsers/ParserRoleList.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ +/** Parses a string like this: + * {role|CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {role|CURRENT_USER} [,...] + */ +class ParserRoleList : public IParserBase +{ +protected: + const char * getName() const { return "RoleList"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected); +}; + +} diff --git a/dbms/src/Parsers/ParserShowCreateAccessEntityQuery.cpp b/dbms/src/Parsers/ParserShowCreateAccessEntityQuery.cpp new file mode 100644 index 00000000000..661330ffd0b --- /dev/null +++ b/dbms/src/Parsers/ParserShowCreateAccessEntityQuery.cpp @@ -0,0 +1,47 @@ +#include +#include +#include +#include + + +namespace DB +{ +bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + if (!ParserKeyword{"SHOW CREATE"}.ignore(pos, expected)) + return false; + + using Kind = ASTShowCreateAccessEntityQuery::Kind; + Kind kind; + if (ParserKeyword{"QUOTA"}.ignore(pos, expected)) + kind = Kind::QUOTA; + else + return false; + + String name; + bool current_quota = false; + + if ((kind == Kind::QUOTA) && ParserKeyword{"CURRENT"}.ignore(pos, expected)) + { + /// SHOW CREATE QUOTA CURRENT + current_quota = true; + } + else if (parseIdentifierOrStringLiteral(pos, expected, name)) + { + /// SHOW CREATE QUOTA name + } + else + { + /// SHOW CREATE QUOTA + current_quota = true; + } + + auto query = std::make_shared(kind); + node = query; + + query->name = std::move(name); + query->current_quota = current_quota; + + return true; +} +} diff --git a/dbms/src/Parsers/ParserShowCreateAccessEntityQuery.h b/dbms/src/Parsers/ParserShowCreateAccessEntityQuery.h new file mode 100644 index 00000000000..4572b54de27 --- /dev/null +++ b/dbms/src/Parsers/ParserShowCreateAccessEntityQuery.h @@ -0,0 +1,17 @@ +#pragma once + +#include + + +namespace DB +{ +/** Parses queries like + * SHOW CREATE QUOTA [name | CURRENT] + */ +class ParserShowCreateAccessEntityQuery : public IParserBase +{ +protected: + const char * getName() const override { return "SHOW CREATE QUOTA query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; +} diff --git a/dbms/src/Parsers/ParserShowQuotasQuery.cpp b/dbms/src/Parsers/ParserShowQuotasQuery.cpp new file mode 100644 index 00000000000..69cbd352969 --- /dev/null +++ b/dbms/src/Parsers/ParserShowQuotasQuery.cpp @@ -0,0 +1,42 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +bool ParserShowQuotasQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + bool usage; + bool current; + if (ParserKeyword{"SHOW QUOTAS"}.ignore(pos, expected)) + { + usage = false; + current = false; + } + else if (ParserKeyword{"SHOW QUOTA USAGE"}.ignore(pos, expected)) + { + usage = true; + if (ParserKeyword{"ALL"}.ignore(pos, expected)) + { + current = false; + } + else + { + ParserKeyword{"CURRENT"}.ignore(pos, expected); + current = true; + } + } + else + return false; + + auto query = std::make_shared(); + query->usage = usage; + query->current = current; + node = query; + return true; +} +} diff --git a/dbms/src/Parsers/ParserShowQuotasQuery.h b/dbms/src/Parsers/ParserShowQuotasQuery.h new file mode 100644 index 00000000000..5b00b525f98 --- /dev/null +++ b/dbms/src/Parsers/ParserShowQuotasQuery.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ +/** Parses queries like + * SHOW QUOTAS + * SHOW QUOTA USAGE [CURRENT | ALL] + */ +class ParserShowQuotasQuery : public IParserBase +{ +protected: + const char * getName() const override { return "SHOW QUOTA query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; +} diff --git a/dbms/src/Parsers/parseIntervalKind.cpp b/dbms/src/Parsers/parseIntervalKind.cpp new file mode 100644 index 00000000000..7d36133e81c --- /dev/null +++ b/dbms/src/Parsers/parseIntervalKind.cpp @@ -0,0 +1,68 @@ +#include +#include +#include + + +namespace DB +{ +bool parseIntervalKind(IParser::Pos & pos, Expected & expected, IntervalKind & result) +{ + if (ParserKeyword("SECOND").ignore(pos, expected) || ParserKeyword("SQL_TSI_SECOND").ignore(pos, expected) + || ParserKeyword("SS").ignore(pos, expected) || ParserKeyword("S").ignore(pos, expected)) + { + result = IntervalKind::Second; + return true; + } + + if (ParserKeyword("MINUTE").ignore(pos, expected) || ParserKeyword("SQL_TSI_MINUTE").ignore(pos, expected) + || ParserKeyword("MI").ignore(pos, expected) || ParserKeyword("N").ignore(pos, expected)) + { + result = IntervalKind::Minute; + return true; + } + + if (ParserKeyword("HOUR").ignore(pos, expected) || ParserKeyword("SQL_TSI_HOUR").ignore(pos, expected) + || ParserKeyword("HH").ignore(pos, expected)) + { + result = IntervalKind::Hour; + return true; + } + + if (ParserKeyword("DAY").ignore(pos, expected) || ParserKeyword("SQL_TSI_DAY").ignore(pos, expected) + || ParserKeyword("DD").ignore(pos, expected) || ParserKeyword("D").ignore(pos, expected)) + { + result = IntervalKind::Day; + return true; + } + + if (ParserKeyword("WEEK").ignore(pos, expected) || ParserKeyword("SQL_TSI_WEEK").ignore(pos, expected) + || ParserKeyword("WK").ignore(pos, expected) || ParserKeyword("WW").ignore(pos, expected)) + { + result = IntervalKind::Week; + return true; + } + + if (ParserKeyword("MONTH").ignore(pos, expected) || ParserKeyword("SQL_TSI_MONTH").ignore(pos, expected) + || ParserKeyword("MM").ignore(pos, expected) || ParserKeyword("M").ignore(pos, expected)) + { + result = IntervalKind::Month; + return true; + } + + if (ParserKeyword("QUARTER").ignore(pos, expected) || ParserKeyword("SQL_TSI_QUARTER").ignore(pos, expected) + || ParserKeyword("QQ").ignore(pos, expected) || ParserKeyword("Q").ignore(pos, expected)) + { + result = IntervalKind::Quarter; + return true; + } + + if (ParserKeyword("YEAR").ignore(pos, expected) || ParserKeyword("SQL_TSI_YEAR").ignore(pos, expected) + || ParserKeyword("YYYY").ignore(pos, expected) || ParserKeyword("YY").ignore(pos, expected)) + { + result = IntervalKind::Year; + return true; + } + + return false; +} +} diff --git a/dbms/src/Parsers/parseIntervalKind.h b/dbms/src/Parsers/parseIntervalKind.h new file mode 100644 index 00000000000..59f2824dfe2 --- /dev/null +++ b/dbms/src/Parsers/parseIntervalKind.h @@ -0,0 +1,11 @@ +#pragma once + +#include +#include + + +namespace DB +{ +/// Parses an interval kind. +bool parseIntervalKind(IParser::Pos & pos, Expected & expected, IntervalKind & result); +} diff --git a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp index 5d632bdcef5..0522e7a5323 100644 --- a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp +++ b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp @@ -198,7 +198,7 @@ void TreeExecutorBlockInputStream::setLimits(const IBlockInputStream::LocalLimit source->setLimits(limits_); } -void TreeExecutorBlockInputStream::setQuota(QuotaForIntervals & quota_) +void TreeExecutorBlockInputStream::setQuota(const std::shared_ptr & quota_) { for (auto & source : sources_with_progress) source->setQuota(quota_); diff --git a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h index da1d60dd972..176fbd06af8 100644 --- a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h +++ b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h @@ -31,7 +31,7 @@ public: void setProgressCallback(const ProgressCallback & callback) final; void setProcessListElement(QueryStatus * elem) final; void setLimits(const LocalLimits & limits_) final; - void setQuota(QuotaForIntervals & quota_) final; + void setQuota(const std::shared_ptr & quota_) final; void addTotalRowsApprox(size_t value) final; protected: diff --git a/dbms/src/Processors/Pipe.cpp b/dbms/src/Processors/Pipe.cpp index b31cfd58848..17b44a48ea1 100644 --- a/dbms/src/Processors/Pipe.cpp +++ b/dbms/src/Processors/Pipe.cpp @@ -97,7 +97,7 @@ void Pipe::setLimits(const ISourceWithProgress::LocalLimits & limits) } } -void Pipe::setQuota(QuotaForIntervals & quota) +void Pipe::setQuota(const std::shared_ptr & quota) { for (auto & processor : processors) { diff --git a/dbms/src/Processors/Pipe.h b/dbms/src/Processors/Pipe.h index 72cb90c4b9e..d734c89f485 100644 --- a/dbms/src/Processors/Pipe.h +++ b/dbms/src/Processors/Pipe.h @@ -8,8 +8,6 @@ namespace DB class Pipe; using Pipes = std::vector; -class QuotaForIntervals; - /// Pipe is a set of processors which represents the part of pipeline with single output. /// All processors in pipe are connected. All ports are connected except the output one. class Pipe @@ -39,7 +37,7 @@ public: /// Specify quotas and limits for every ISourceWithProgress. void setLimits(const SourceWithProgress::LocalLimits & limits); - void setQuota(QuotaForIntervals & quota); + void setQuota(const std::shared_ptr & quota); /// Set information about preferred executor number for sources. void pinSources(size_t executor_number); diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.h b/dbms/src/Processors/Sources/SourceFromInputStream.h index 888439f15d5..8e750a33faf 100644 --- a/dbms/src/Processors/Sources/SourceFromInputStream.h +++ b/dbms/src/Processors/Sources/SourceFromInputStream.h @@ -25,7 +25,7 @@ public: /// Implementation for methods from ISourceWithProgress. void setLimits(const LocalLimits & limits_) final { stream->setLimits(limits_); } - void setQuota(QuotaForIntervals & quota_) final { stream->setQuota(quota_); } + void setQuota(const std::shared_ptr & quota_) final { stream->setQuota(quota_); } void setProcessListElement(QueryStatus * elem) final { stream->setProcessListElement(elem); } void setProgressCallback(const ProgressCallback & callback) final { stream->setProgressCallback(callback); } void addTotalRowsApprox(size_t value) final { stream->addTotalRowsApprox(value); } diff --git a/dbms/src/Processors/Sources/SourceWithProgress.cpp b/dbms/src/Processors/Sources/SourceWithProgress.cpp index 21f9d5ca9bb..fac2a53ea54 100644 --- a/dbms/src/Processors/Sources/SourceWithProgress.cpp +++ b/dbms/src/Processors/Sources/SourceWithProgress.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include namespace DB { @@ -72,10 +72,8 @@ void SourceWithProgress::progress(const Progress & value) /// It is here for compatibility with IBlockInputsStream. limits.speed_limits.throttle(progress.read_rows, progress.read_bytes, total_rows, total_elapsed_microseconds); - if (quota != nullptr && limits.mode == LimitsMode::LIMITS_TOTAL) - { - quota->checkAndAddReadRowsBytes(time(nullptr), value.read_rows, value.read_bytes); - } + if (quota && limits.mode == LimitsMode::LIMITS_TOTAL) + quota->used({Quota::READ_ROWS, value.read_rows}, {Quota::READ_BYTES, value.read_bytes}); } } diff --git a/dbms/src/Processors/Sources/SourceWithProgress.h b/dbms/src/Processors/Sources/SourceWithProgress.h index 833e5eccb6f..59e8c6afa20 100644 --- a/dbms/src/Processors/Sources/SourceWithProgress.h +++ b/dbms/src/Processors/Sources/SourceWithProgress.h @@ -21,7 +21,7 @@ public: /// Set the quota. If you set a quota on the amount of raw data, /// then you should also set mode = LIMITS_TOTAL to LocalLimits with setLimits. - virtual void setQuota(QuotaForIntervals & quota_) = 0; + virtual void setQuota(const std::shared_ptr & quota_) = 0; /// Set the pointer to the process list item. /// General information about the resources spent on the request will be written into it. @@ -49,7 +49,7 @@ public: using LimitsMode = IBlockInputStream::LimitsMode; void setLimits(const LocalLimits & limits_) final { limits = limits_; } - void setQuota(QuotaForIntervals & quota_) final { quota = "a_; } + void setQuota(const std::shared_ptr & quota_) final { quota = quota_; } void setProcessListElement(QueryStatus * elem) final { process_list_elem = elem; } void setProgressCallback(const ProgressCallback & callback) final { progress_callback = callback; } void addTotalRowsApprox(size_t value) final { total_rows_approx += value; } @@ -60,7 +60,7 @@ protected: private: LocalLimits limits; - QuotaForIntervals * quota = nullptr; + std::shared_ptr quota; ProgressCallback progress_callback; QueryStatus * process_list_elem = nullptr; diff --git a/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp b/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp index 4947d11974b..1f621439048 100644 --- a/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp +++ b/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp @@ -1,5 +1,5 @@ #include -#include +#include namespace DB { @@ -73,7 +73,7 @@ void LimitsCheckingTransform::transform(Chunk & chunk) !limits.size_limits.check(info.rows, info.bytes, "result", ErrorCodes::TOO_MANY_ROWS_OR_BYTES)) stopReading(); - if (quota != nullptr) + if (quota) checkQuota(chunk); } } @@ -100,12 +100,8 @@ void LimitsCheckingTransform::checkQuota(Chunk & chunk) case LimitsMode::LIMITS_CURRENT: { - time_t current_time = time(nullptr); - double total_elapsed = info.total_stopwatch.elapsedSeconds(); - - quota->checkAndAddResultRowsBytes(current_time, chunk.getNumRows(), chunk.bytes()); - quota->checkAndAddExecutionTime(current_time, Poco::Timespan((total_elapsed - prev_elapsed) * 1000000.0)); - + UInt64 total_elapsed = info.total_stopwatch.elapsedNanoseconds(); + quota->used({Quota::RESULT_ROWS, chunk.getNumRows()}, {Quota::RESULT_BYTES, chunk.bytes()}, {Quota::EXECUTION_TIME, total_elapsed - prev_elapsed}); prev_elapsed = total_elapsed; break; } diff --git a/dbms/src/Processors/Transforms/LimitsCheckingTransform.h b/dbms/src/Processors/Transforms/LimitsCheckingTransform.h index 53116446a75..9410301030a 100644 --- a/dbms/src/Processors/Transforms/LimitsCheckingTransform.h +++ b/dbms/src/Processors/Transforms/LimitsCheckingTransform.h @@ -36,7 +36,7 @@ public: String getName() const override { return "LimitsCheckingTransform"; } - void setQuota(QuotaForIntervals & quota_) { quota = "a_; } + void setQuota(const std::shared_ptr & quota_) { quota = quota_; } protected: void transform(Chunk & chunk) override; @@ -44,8 +44,8 @@ protected: private: LocalLimits limits; - QuotaForIntervals * quota = nullptr; - double prev_elapsed = 0; + std::shared_ptr quota; + UInt64 prev_elapsed = 0; ProcessorProfileInfo info; diff --git a/dbms/src/Storages/System/StorageSystemQuotaUsage.cpp b/dbms/src/Storages/System/StorageSystemQuotaUsage.cpp new file mode 100644 index 00000000000..8835e77eeb5 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemQuotaUsage.cpp @@ -0,0 +1,79 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +NamesAndTypesList StorageSystemQuotaUsage::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"id", std::make_shared()}, + {"key", std::make_shared()}, + {"duration", std::make_shared(std::make_shared())}, + {"end_of_interval", std::make_shared(std::make_shared())}}; + + for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + { + DataTypePtr data_type; + if (resource_type == Quota::EXECUTION_TIME) + data_type = std::make_shared(); + else + data_type = std::make_shared(); + + String column_name = Quota::resourceTypeToColumnName(resource_type); + names_and_types.push_back({column_name, std::make_shared(data_type)}); + names_and_types.push_back({String("max_") + column_name, std::make_shared(data_type)}); + } + return names_and_types; +} + + +void StorageSystemQuotaUsage::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + const auto & access_control = context.getAccessControlManager(); + for (const auto & info : access_control.getQuotaUsageInfo()) + { + for (const auto & interval : info.intervals) + { + size_t i = 0; + res_columns[i++]->insert(info.quota_name); + res_columns[i++]->insert(info.quota_id); + res_columns[i++]->insert(info.quota_key); + res_columns[i++]->insert(std::chrono::seconds{interval.duration}.count()); + res_columns[i++]->insert(std::chrono::system_clock::to_time_t(interval.end_of_interval)); + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + { + if (resource_type == Quota::EXECUTION_TIME) + { + res_columns[i++]->insert(Quota::executionTimeToSeconds(interval.used[resource_type])); + res_columns[i++]->insert(Quota::executionTimeToSeconds(interval.max[resource_type])); + } + else + { + res_columns[i++]->insert(interval.used[resource_type]); + res_columns[i++]->insert(interval.max[resource_type]); + } + } + } + + if (info.intervals.empty()) + { + size_t i = 0; + res_columns[i++]->insert(info.quota_name); + res_columns[i++]->insert(info.quota_id); + res_columns[i++]->insert(info.quota_key); + for (size_t j = 0; j != Quota::MAX_RESOURCE_TYPE * 2 + 2; ++j) + res_columns[i++]->insertDefault(); + } + } +} +} diff --git a/dbms/src/Storages/System/StorageSystemQuotaUsage.h b/dbms/src/Storages/System/StorageSystemQuotaUsage.h new file mode 100644 index 00000000000..f2151b27612 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemQuotaUsage.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class Context; + + +/** Implements the `quota_usage` system tables, which allows you to get information about + * how the quotas are used by all users. + */ +class StorageSystemQuotaUsage : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemQuotaUsage"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/dbms/src/Storages/System/StorageSystemQuotas.cpp b/dbms/src/Storages/System/StorageSystemQuotas.cpp new file mode 100644 index 00000000000..b82e348c86d --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemQuotas.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + DataTypeEnum8::Values getKeyTypeEnumValues() + { + DataTypeEnum8::Values enum_values; + for (auto key_type : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) + enum_values.push_back({Quota::getNameOfKeyType(key_type), static_cast(key_type)}); + return enum_values; + } +} + + +NamesAndTypesList StorageSystemQuotas::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"id", std::make_shared()}, + {"source", std::make_shared()}, + {"key_type", std::make_shared(getKeyTypeEnumValues())}, + {"roles", std::make_shared(std::make_shared())}, + {"all_roles", std::make_shared()}, + {"except_roles", std::make_shared(std::make_shared())}, + {"intervals.duration", std::make_shared(std::make_shared())}, + {"intervals.randomize_interval", std::make_shared(std::make_shared())}}; + + for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + { + DataTypePtr data_type; + if (resource_type == Quota::EXECUTION_TIME) + data_type = std::make_shared(); + else + data_type = std::make_shared(); + + String column_name = String("intervals.max_") + Quota::resourceTypeToColumnName(resource_type); + names_and_types.push_back({column_name, std::make_shared(data_type)}); + } + return names_and_types; +} + + +void StorageSystemQuotas::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + size_t i = 0; + auto & name_column = *res_columns[i++]; + auto & id_column = *res_columns[i++]; + auto & storage_name_column = *res_columns[i++]; + auto & key_type_column = *res_columns[i++]; + auto & roles_data = assert_cast(*res_columns[i]).getData(); + auto & roles_offsets = assert_cast(*res_columns[i++]).getOffsets(); + auto & all_roles_column = *res_columns[i++]; + auto & except_roles_data = assert_cast(*res_columns[i]).getData(); + auto & except_roles_offsets = assert_cast(*res_columns[i++]).getOffsets(); + auto & durations_data = assert_cast(*res_columns[i]).getData(); + auto & durations_offsets = assert_cast(*res_columns[i++]).getOffsets(); + auto & randomize_intervals_data = assert_cast(*res_columns[i]).getData(); + auto & randomize_intervals_offsets = assert_cast(*res_columns[i++]).getOffsets(); + IColumn * limits_data[Quota::MAX_RESOURCE_TYPE]; + ColumnArray::Offsets * limits_offsets[Quota::MAX_RESOURCE_TYPE]; + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + { + limits_data[resource_type] = &assert_cast(*res_columns[i]).getData(); + limits_offsets[resource_type] = &assert_cast(*res_columns[i++]).getOffsets(); + } + + const auto & access_control = context.getAccessControlManager(); + for (const auto & id : access_control.findAll()) + { + auto quota = access_control.tryRead(id); + if (!quota) + continue; + const auto * storage = access_control.findStorage(id); + String storage_name = storage ? storage->getStorageName() : ""; + + name_column.insert(quota->getName()); + id_column.insert(id); + storage_name_column.insert(storage_name); + key_type_column.insert(static_cast(quota->key_type)); + + for (const auto & role : quota->roles) + roles_data.insert(role); + roles_offsets.push_back(roles_data.size()); + + all_roles_column.insert(static_cast(quota->all_roles)); + + for (const auto & except_role : quota->except_roles) + except_roles_data.insert(except_role); + except_roles_offsets.push_back(except_roles_data.size()); + + for (const auto & limits : quota->all_limits) + { + durations_data.insert(std::chrono::seconds{limits.duration}.count()); + randomize_intervals_data.insert(static_cast(limits.randomize_interval)); + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + { + if (resource_type == Quota::EXECUTION_TIME) + limits_data[resource_type]->insert(Quota::executionTimeToSeconds(limits.max[resource_type])); + else + limits_data[resource_type]->insert(limits.max[resource_type]); + } + } + + durations_offsets.push_back(durations_data.size()); + randomize_intervals_offsets.push_back(randomize_intervals_data.size()); + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + limits_offsets[resource_type]->push_back(limits_data[resource_type]->size()); + } +} +} diff --git a/dbms/src/Storages/System/StorageSystemQuotas.h b/dbms/src/Storages/System/StorageSystemQuotas.h new file mode 100644 index 00000000000..0f54f193654 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemQuotas.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class Context; + + +/** Implements the `quotas` system tables, which allows you to get information about quotas. + */ +class StorageSystemQuotas : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemQuotas"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/dbms/src/Storages/System/attachSystemTables.cpp b/dbms/src/Storages/System/attachSystemTables.cpp index 528bdd06a21..2b8e630cbed 100644 --- a/dbms/src/Storages/System/attachSystemTables.cpp +++ b/dbms/src/Storages/System/attachSystemTables.cpp @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -52,6 +54,8 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("functions", StorageSystemFunctions::create("functions")); system_database.attachTable("events", StorageSystemEvents::create("events")); system_database.attachTable("settings", StorageSystemSettings::create("settings")); + system_database.attachTable("quotas", StorageSystemQuotas::create("quotas")); + system_database.attachTable("quota_usage", StorageSystemQuotaUsage::create("quota_usage")); system_database.attachTable("merge_tree_settings", SystemMergeTreeSettings::create("merge_tree_settings")); system_database.attachTable("build_options", StorageSystemBuildOptions::create("build_options")); system_database.attachTable("formats", StorageSystemFormats::create("formats")); diff --git a/dbms/tests/integration/test_quota/__init__.py b/dbms/tests/integration/test_quota/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_quota/configs/users.d/quota.xml b/dbms/tests/integration/test_quota/configs/users.d/quota.xml new file mode 120000 index 00000000000..9b12dec9c53 --- /dev/null +++ b/dbms/tests/integration/test_quota/configs/users.d/quota.xml @@ -0,0 +1 @@ +../../normal_limits.xml \ No newline at end of file diff --git a/dbms/tests/integration/test_quota/configs/users.xml b/dbms/tests/integration/test_quota/configs/users.xml new file mode 100644 index 00000000000..15a5364449b --- /dev/null +++ b/dbms/tests/integration/test_quota/configs/users.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + ::/0 + + default + myQuota + true + + + diff --git a/dbms/tests/integration/test_quota/no_quotas.xml b/dbms/tests/integration/test_quota/no_quotas.xml new file mode 100644 index 00000000000..9aba4ac0914 --- /dev/null +++ b/dbms/tests/integration/test_quota/no_quotas.xml @@ -0,0 +1,3 @@ + + + diff --git a/dbms/tests/integration/test_quota/normal_limits.xml b/dbms/tests/integration/test_quota/normal_limits.xml new file mode 100644 index 00000000000..b7c3a67b5cc --- /dev/null +++ b/dbms/tests/integration/test_quota/normal_limits.xml @@ -0,0 +1,17 @@ + + + + + + + 31556952 + + + 1000 + 0 + 1000 + 0 + + + + diff --git a/dbms/tests/integration/test_quota/simpliest.xml b/dbms/tests/integration/test_quota/simpliest.xml new file mode 100644 index 00000000000..6d51d68d8d9 --- /dev/null +++ b/dbms/tests/integration/test_quota/simpliest.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/dbms/tests/integration/test_quota/test.py b/dbms/tests/integration/test_quota/test.py new file mode 100644 index 00000000000..e7caaf5cd06 --- /dev/null +++ b/dbms/tests/integration/test_quota/test.py @@ -0,0 +1,251 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry +import os +import re +import time + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance', + config_dir="configs") + +query_from_system_quotas = "SELECT * FROM system.quotas ORDER BY name"; + +query_from_system_quota_usage = "SELECT id, key, duration, "\ + "queries, errors, result_rows, result_bytes, read_rows, read_bytes "\ + "FROM system.quota_usage ORDER BY id, key, duration"; + +def system_quotas(): + return instance.query(query_from_system_quotas).rstrip('\n') + +def system_quota_usage(): + return instance.query(query_from_system_quota_usage).rstrip('\n') + + +def copy_quota_xml(local_file_name, reload_immediately = True): + script_dir = os.path.dirname(os.path.realpath(__file__)) + instance.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/quota.xml') + if reload_immediately: + instance.query("SYSTEM RELOAD CONFIG") + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + instance.query("CREATE TABLE test_table(x UInt32) ENGINE = MergeTree ORDER BY tuple()") + instance.query("INSERT INTO test_table SELECT number FROM numbers(50)") + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def reset_quotas_and_usage_info(): + try: + yield + finally: + instance.query("DROP QUOTA IF EXISTS qA, qB") + copy_quota_xml('simpliest.xml') # To reset usage info. + copy_quota_xml('normal_limits.xml') + + +def test_quota_from_users_xml(): + assert instance.query("SELECT currentQuota()") == "myQuota\n" + assert instance.query("SELECT currentQuotaID()") == "e651da9c-a748-8703-061a-7e5e5096dae7\n" + assert instance.query("SELECT currentQuotaKey()") == "default\n" + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + + instance.query("SELECT COUNT() from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t51\t208\t50\t200" + + +def test_simpliest_quota(): + # Simpliest quota doesn't even track usage. + copy_quota_xml('simpliest.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + + +def test_tracking_quota(): + # Now we're tracking usage. + copy_quota_xml('tracking.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + + instance.query("SELECT COUNT() from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t51\t208\t50\t200" + + +def test_exceed_quota(): + # Change quota, now the limits are tiny so we will exceed the quota. + copy_quota_xml('tiny_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1]\t[1]\t[1]\t[0]\t[1]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + assert re.search("Quota.*has\ been\ exceeded", instance.query_and_get_error("SELECT * from test_table")) + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t1\t0\t0\t50\t0" + + # Change quota, now the limits are enough to execute queries. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t1\t0\t0\t50\t0" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t1\t50\t200\t100\t200" + + +def test_add_remove_interval(): + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + # Add interval. + copy_quota_xml('two_intervals.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952,63113904]\t[0,1]\t[1000,0]\t[0,0]\t[0,0]\t[0,30000]\t[1000,0]\t[0,20000]\t[0,120]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0\n"\ + "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t63113904\t0\t0\t0\t0\t0\t0" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200\n"\ + "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t63113904\t1\t0\t50\t200\t50\t200" + + # Remove interval. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t100\t400\t100\t400" + + # Remove all intervals. + copy_quota_xml('simpliest.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + + instance.query("SELECT * from test_table") + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + + # Add one interval back. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + +def test_add_remove_quota(): + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + # Add quota. + copy_quota_xml('two_quotas.xml') + assert system_quotas() ==\ + "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]\n"\ + "myQuota2\t4590510c-4d13-bf21-ec8a-c2187b092e73\tusers.xml\tclient key or user name\t[]\t0\t[]\t[3600,2629746]\t[1,0]\t[0,0]\t[0,0]\t[4000,0]\t[400000,0]\t[4000,0]\t[400000,0]\t[60,1800]" + + # Drop quota. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + + # Drop all quotas. + copy_quota_xml('no_quotas.xml') + assert system_quotas() == "" + assert system_quota_usage() == "" + + # Add one quota back. + copy_quota_xml('normal_limits.xml') + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + + +def test_reload_users_xml_by_timer(): + assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + + time.sleep(1) # The modification time of the 'quota.xml' file should be different, + # because config files are reload by timer only when the modification time is changed. + copy_quota_xml('tiny_limits.xml', reload_immediately=False) + assert_eq_with_retry(instance, query_from_system_quotas, "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t0\t[]\t[31556952]\t[0]\t[1]\t[1]\t[1]\t[0]\t[1]\t[0]\t[0]") + + +def test_dcl_introspection(): + assert instance.query("SHOW QUOTAS") == "myQuota\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000 TO default\n" + expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0/1000 errors=0 result_rows=0 result_bytes=0 read_rows=0/1000 read_bytes=0 execution_time=0" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE CURRENT")) + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE ALL")) + + instance.query("SELECT * from test_table") + expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + # Add interval. + copy_quota_xml('two_intervals.xml') + assert instance.query("SHOW QUOTAS") == "myQuota\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES = 30000, MAX READ BYTES = 20000, MAX EXECUTION TIME = 120 TO default\n" + expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*\n"\ + "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0/30000 read_rows=0 read_bytes=0/20000 execution_time=0/120" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + # Drop interval, add quota. + copy_quota_xml('two_quotas.xml') + assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES = 1000, MAX READ ROWS = 1000 TO default\n" + assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY \\'client key or user name\\' FOR RANDOMIZED INTERVAL 1 HOUR MAX RESULT ROWS = 4000, MAX RESULT BYTES = 400000, MAX READ ROWS = 4000, MAX READ BYTES = 400000, MAX EXECUTION TIME = 60, FOR INTERVAL 1 MONTH MAX EXECUTION TIME = 1800\n" + expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + +def test_dcl_management(): + copy_quota_xml('no_quotas.xml') + assert instance.query("SHOW QUOTAS") == "" + assert instance.query("SHOW QUOTA USAGE") == "" + + instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH SET MAX QUERIES = 123 TO CURRENT_USER") + assert instance.query("SHOW QUOTAS") == "qA\n" + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 5 QUARTER MAX QUERIES = 123 TO default\n" + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0/123 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("SELECT * from test_table") + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1/123 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES = 321, MAX ERRORS = 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME = 0.5") + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 30 MINUTE MAX EXECUTION TIME = 0.5, FOR INTERVAL 5 QUARTER MAX QUERIES = 321, MAX ERRORS = 10 TO default\n" + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*/0.5\n"\ + "qA key=\\\\'\\\\' interval=\[.*\] queries=1/321 errors=0/10 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH UNSET TRACKING, FOR RANDOMIZED INTERVAL 16 MONTH SET TRACKING, FOR INTERVAL 1800 SECOND UNSET TRACKING") + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING TO default\n" + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("SELECT * from test_table") + expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("ALTER QUOTA qA RENAME TO qB") + assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING TO default\n" + expected_usage = "qB key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" + assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + + instance.query("DROP QUOTA qB") + assert instance.query("SHOW QUOTAS") == "" + assert instance.query("SHOW QUOTA USAGE") == "" + + +def test_users_xml_is_readonly(): + assert re.search("storage is readonly", instance.query_and_get_error("DROP QUOTA myQuota")) diff --git a/dbms/tests/integration/test_quota/tiny_limits.xml b/dbms/tests/integration/test_quota/tiny_limits.xml new file mode 100644 index 00000000000..3ab8858738a --- /dev/null +++ b/dbms/tests/integration/test_quota/tiny_limits.xml @@ -0,0 +1,17 @@ + + + + + + + 31556952 + + + 1 + 1 + 1 + 1 + + + + diff --git a/dbms/tests/integration/test_quota/tracking.xml b/dbms/tests/integration/test_quota/tracking.xml new file mode 100644 index 00000000000..47e12bf8005 --- /dev/null +++ b/dbms/tests/integration/test_quota/tracking.xml @@ -0,0 +1,17 @@ + + + + + + + 31556952 + + + 0 + 0 + 0 + 0 + + + + diff --git a/dbms/tests/integration/test_quota/two_intervals.xml b/dbms/tests/integration/test_quota/two_intervals.xml new file mode 100644 index 00000000000..d0de605b895 --- /dev/null +++ b/dbms/tests/integration/test_quota/two_intervals.xml @@ -0,0 +1,20 @@ + + + + + + 31556952 + 1000 + 1000 + + + + true + 63113904 + 20000 + 30000 + 120 + + + + diff --git a/dbms/tests/integration/test_quota/two_quotas.xml b/dbms/tests/integration/test_quota/two_quotas.xml new file mode 100644 index 00000000000..c08cc82aca7 --- /dev/null +++ b/dbms/tests/integration/test_quota/two_quotas.xml @@ -0,0 +1,29 @@ + + + + + + 31556952 + 1000 + 1000 + + + + + + + true + 3600 + 4000 + 4000 + 400000 + 400000 + 60 + + + 2629746 + 1800 + + + + diff --git a/dbms/tests/queries/0_stateless/01033_quota_dcl.reference b/dbms/tests/queries/0_stateless/01033_quota_dcl.reference new file mode 100644 index 00000000000..7f92f992dd5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01033_quota_dcl.reference @@ -0,0 +1,2 @@ +default +CREATE QUOTA default KEYED BY \'user name\' FOR INTERVAL 1 HOUR TRACKING TO default, readonly diff --git a/dbms/tests/queries/0_stateless/01033_quota_dcl.sql b/dbms/tests/queries/0_stateless/01033_quota_dcl.sql new file mode 100644 index 00000000000..a1c7f1fc204 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01033_quota_dcl.sql @@ -0,0 +1,3 @@ +SHOW QUOTAS; +SHOW CREATE QUOTA default; +CREATE QUOTA q1; -- { serverError 497 } diff --git a/dbms/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.reference b/dbms/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.reference new file mode 100644 index 00000000000..f12dcd8258a --- /dev/null +++ b/dbms/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.reference @@ -0,0 +1,6 @@ +12 -> 102 +13 -> 103 +14 -> -1 +12(r) -> 102 +13(r) -> 103 +14(r) -> 104 diff --git a/dbms/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh b/dbms/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh new file mode 100755 index 00000000000..3181e46f205 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e -o pipefail + +# Run the client. +$CLICKHOUSE_CLIENT --multiquery <<'EOF' +DROP DATABASE IF EXISTS dictdb; +CREATE DATABASE dictdb Engine = Ordinary; +CREATE TABLE dictdb.table(x Int64, y Int64, insert_time DateTime) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO dictdb.table VALUES (12, 102, now()); + +CREATE DICTIONARY dictdb.dict +( + x Int64 DEFAULT -1, + y Int64 DEFAULT -1, + insert_time DateTime +) +PRIMARY KEY x +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table' DB 'dictdb' UPDATE_FIELD 'insert_time')) +LAYOUT(FLAT()) +LIFETIME(1); +EOF + +$CLICKHOUSE_CLIENT --query "SELECT '12 -> ', dictGetInt64('dictdb.dict', 'y', toUInt64(12))" + +$CLICKHOUSE_CLIENT --query "INSERT INTO dictdb.table VALUES (13, 103, now())" +$CLICKHOUSE_CLIENT --query "INSERT INTO dictdb.table VALUES (14, 104, now() - INTERVAL 1 DAY)" + +while [ $($CLICKHOUSE_CLIENT --query "SELECT dictGetInt64('dictdb.dict', 'y', toUInt64(13))") = -1 ] + do + sleep 0.5 + done + +$CLICKHOUSE_CLIENT --query "SELECT '13 -> ', dictGetInt64('dictdb.dict', 'y', toUInt64(13))" +$CLICKHOUSE_CLIENT --query "SELECT '14 -> ', dictGetInt64('dictdb.dict', 'y', toUInt64(14))" + +$CLICKHOUSE_CLIENT --query "SYSTEM RELOAD DICTIONARY 'dictdb.dict'" + +$CLICKHOUSE_CLIENT --query "SELECT '12(r) -> ', dictGetInt64('dictdb.dict', 'y', toUInt64(12))" +$CLICKHOUSE_CLIENT --query "SELECT '13(r) -> ', dictGetInt64('dictdb.dict', 'y', toUInt64(13))" +$CLICKHOUSE_CLIENT --query "SELECT '14(r) -> ', dictGetInt64('dictdb.dict', 'y', toUInt64(14))" + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS dictdb" diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 9f4275029c2..ab3f5b95a56 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1007,6 +1007,41 @@ Error count of each replica is capped at this value, preventing a single replica - [Table engine Distributed](../../operations/table_engines/distributed.md) - [`distributed_replica_error_half_life`](#settings-distributed_replica_error_half_life) + +## distributed_directory_monitor_sleep_time_ms {#distributed_directory_monitor_sleep_time_ms} + +Base interval of data sending by the [Distributed](../table_engines/distributed.md) table engine. Actual interval grows exponentially in case of any errors. + +Possible values: + +- Positive integer number of milliseconds. + +Default value: 100 milliseconds. + + +## distributed_directory_monitor_max_sleep_time_ms {#distributed_directory_monitor_max_sleep_time_ms} + +Maximum interval of data sending by the [Distributed](../table_engines/distributed.md) table engine. Limits exponential growth of the interval set in the [distributed_directory_monitor_sleep_time_ms](#distributed_directory_monitor_sleep_time_ms) setting. + +Possible values: + +- Positive integer number of milliseconds. + +Default value: 30000 milliseconds (30 seconds). + +## distributed_directory_monitor_batch_inserts {#distributed_directory_monitor_batch_inserts} + +Enables/disables sending of inserted data in batches. + +When batch sending is enabled, [Distributed](../table_engines/distributed.md) table engine tries to send multiple files of inserted data in one operation instead of sending them separately. Batch sending improves cluster performance by better server and network resources utilization. + +Possible values: + +- 1 — Enabled. +- 0 — Disabled. + +Defaule value: 0. + ## os_thread_priority {#setting-os_thread_priority} Sets the priority ([nice](https://en.wikipedia.org/wiki/Nice_(Unix))) for threads that execute queries. The OS scheduler considers this priority when choosing the next thread to run on each available CPU core. @@ -1022,6 +1057,53 @@ Lower values mean higher priority. Threads with low `nice` priority values are e Default value: 0. + +## query_profiler_real_time_period_ns {#query_profiler_real_time_period_ns} + +Sets the period for a real clock timer of the query profiler. Real clock timer counts wall-clock time. + +Possible values: + +- Positive integer number, in nanoseconds. + + Recommended values: + + - 10000000 (100 times a second) nanoseconds and less for single queries. + - 1000000000 (once a second) for cluster-wide profiling. + +- 0 for turning off the timer. + +Type: [UInt64](../../data_types/int_uint.md). + +Default value: 1000000000 nanoseconds (once a second). + +**See Also** + +- [system.trace_log](../system_tables.md#system_tables-trace_log) + +## query_profiler_cpu_time_period_ns {#query_profiler_cpu_time_period_ns} + +Sets the period for a CPU clock timer of the query profiler. This timer counts only CPU time. + +Possible values: + +- Positive integer number of nanoseconds. + + Recommended values: + + - 10000000 (100 times a second) nanosecods and more for for single queries. + - 1000000000 (once a second) for cluster-wide profiling. + +- 0 for turning off the timer. + +Type: [UInt64](../../data_types/int_uint.md). + +Default value: 1000000000 nanoseconds. + +**See Also** + +- [system.trace_log](../system_tables.md#system_tables-trace_log) + ## allow_introspection_functions {#settings-allow_introspection_functions} Enables of disables [introspections functions](../../query_language/functions/introspection.md) for query profiling. diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index 373b87fbf17..77964c7377f 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -551,7 +551,7 @@ You can specify an arbitrary partitioning key for the `system.query_thread_log` Contains stack traces collected by the sampling query profiler. -ClickHouse creates this table when the [trace_log](server_settings/settings.md#server_settings-trace_log) server configuration section is set. Also the `query_profiler_real_time_period_ns` and `query_profiler_cpu_time_period_ns` settings should be set. +ClickHouse creates this table when the [trace_log](server_settings/settings.md#server_settings-trace_log) server configuration section is set. Also the [query_profiler_real_time_period_ns](settings/settings.md#query_profiler_real_time_period_ns) and [query_profiler_cpu_time_period_ns](settings/settings.md#query_profiler_cpu_time_period_ns) settings should be set. To analyze logs, use the `addressToLine`, `addressToSymbol` and `demangle` introspection functions. diff --git a/docs/en/operations/table_engines/distributed.md b/docs/en/operations/table_engines/distributed.md index 38d085da568..a22fd43b34f 100644 --- a/docs/en/operations/table_engines/distributed.md +++ b/docs/en/operations/table_engines/distributed.md @@ -87,12 +87,9 @@ The Distributed engine requires writing clusters to the config file. Clusters fr There are two methods for writing data to a cluster: -First, you can define which servers to write which data to, and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table "looks at". -This is the most flexible solution – you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. -This is also the most optimal solution, since data can be written to different shards completely independently. +First, you can define which servers to write which data to, and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table "looks at". This is the most flexible solution – you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. This is also the most optimal solution, since data can be written to different shards completely independently. -Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across servers itself. -In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it doesn't have any meaning in this case. +Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across servers itself. In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it doesn't have any meaning in this case. Each shard can have a weight defined in the config file. By default, the weight is equal to one. Data is distributed across shards in the amount proportional to the shard weight. For example, if there are two shards and the first has a weight of 9 while the second has a weight of 10, the first will be sent 9 / 19 parts of the rows, and the second will be sent 10 / 19. @@ -115,7 +112,7 @@ You should be concerned about the sharding scheme in the following cases: - Queries are used that require joining data (IN or JOIN) by a specific key. If data is sharded by this key, you can use local IN or JOIN instead of GLOBAL IN or GLOBAL JOIN, which is much more efficient. - A large number of servers is used (hundreds or more) with a large number of small queries (queries of individual clients - websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, as we've done in Yandex.Metrica, you can set up bi-level sharding: divide the entire cluster into "layers", where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. Distributed tables are created for each layer, and a single shared distributed table is created for global queries. -Data is written asynchronously. For an INSERT to a Distributed table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: /var/lib/clickhouse/data/database/table/. +Data is written asynchronously. When inserted to the table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. The period of data sending is managed by the [distributed_directory_monitor_sleep_time_ms](../settings/settings.md#distributed_directory_monitor_sleep_time_ms) and [distributed_directory_monitor_max_sleep_time_ms](../settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) settings. The `Distributed` engine sends each file with inserted data separately, but you can enable batch sending of files with the [distributed_directory_monitor_batch_inserts](../settings/settings.md#distributed_directory_monitor_batch_inserts) setting. This setting improves cluster performance by better local server and network resources utilization. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: `/var/lib/clickhouse/data/database/table/`. If the server ceased to exist or had a rough restart (for example, after a device failure) after an INSERT to a Distributed table, the inserted data might be lost. If a damaged data part is detected in the table directory, it is transferred to the 'broken' subdirectory and no longer used. diff --git a/docs/fa/query_language/functions/introspection.md b/docs/fa/query_language/functions/introspection.md new file mode 120000 index 00000000000..b1a487e9c77 --- /dev/null +++ b/docs/fa/query_language/functions/introspection.md @@ -0,0 +1 @@ +../../../en/query_language/functions/introspection.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/introspection.md b/docs/ja/query_language/functions/introspection.md new file mode 120000 index 00000000000..b1a487e9c77 --- /dev/null +++ b/docs/ja/query_language/functions/introspection.md @@ -0,0 +1 @@ +../../../en/query_language/functions/introspection.md \ No newline at end of file diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index 30de03f320a..c5a2a7fd80b 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -149,6 +149,7 @@ nav: - 'Working with geographical coordinates': 'query_language/functions/geo.md' - 'Working with Nullable arguments': 'query_language/functions/functions_for_nulls.md' - 'Machine Learning Functions': 'query_language/functions/machine_learning_functions.md' + - 'Introspection': 'query_language/functions/introspection.md' - 'Other': 'query_language/functions/other_functions.md' - 'Aggregate Functions': - 'Introduction': 'query_language/agg_functions/index.md' diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index d96963953fe..8a2b32b240a 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -121,6 +121,7 @@ nav: - 'Working with geographical coordinates': 'query_language/functions/geo.md' - 'Working with Nullable arguments': 'query_language/functions/functions_for_nulls.md' - 'Machine Learning Functions': 'query_language/functions/machine_learning_functions.md' + - 'Introspection': 'query_language/functions/introspection.md' - 'Other': 'query_language/functions/other_functions.md' - 'Aggregate Functions': - 'Introduction': 'query_language/agg_functions/index.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index 3f19fe3f72b..7395dcfe145 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -148,6 +148,7 @@ nav: - 'GEO函数': 'query_language/functions/geo.md' - 'Nullable处理函数': 'query_language/functions/functions_for_nulls.md' - '机器学习函数': 'query_language/functions/machine_learning_functions.md' + - 'Introspection': 'query_language/functions/introspection.md' - '其他函数': 'query_language/functions/other_functions.md' - '聚合函数': - '介绍': 'query_language/agg_functions/index.md' diff --git a/docs/zh/query_language/functions/higher_order_functions.md b/docs/zh/query_language/functions/higher_order_functions.md index e64db0bc8d3..39c6770e5b8 100644 --- a/docs/zh/query_language/functions/higher_order_functions.md +++ b/docs/zh/query_language/functions/higher_order_functions.md @@ -12,7 +12,7 @@ 除了'arrayMap'和'arrayFilter'以外的所有其他函数，都可以省略第一个参数（lambda函数）。在这种情况下，默认返回数组元素本身。 -### arrayMap(func, arr1, ...) +### arrayMap(func, arr1, ...) {#higher_order_functions-array-map} 将arr 将从'func'函数的原始应用程序获得的数组返回到'arr'数组中的每个元素。 diff --git a/docs/zh/query_language/functions/introspection.md b/docs/zh/query_language/functions/introspection.md new file mode 120000 index 00000000000..b1a487e9c77 --- /dev/null +++ b/docs/zh/query_language/functions/introspection.md @@ -0,0 +1 @@ +../../../en/query_language/functions/introspection.md \ No newline at end of file diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 357e457b240..3e58cba0164 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -53,6 +53,7 @@ add_library (common include/common/phdr_cache.h include/ext/bit_cast.h + include/ext/chrono_io.h include/ext/collection_cast.h include/ext/enumerate.h include/ext/function_traits.h diff --git a/libs/libcommon/include/ext/chrono_io.h b/libs/libcommon/include/ext/chrono_io.h new file mode 100644 index 00000000000..8fa448b9e6a --- /dev/null +++ b/libs/libcommon/include/ext/chrono_io.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + + +namespace ext +{ + template + std::string to_string(const std::chrono::time_point & tp) + { + return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp)); + } + + template > + std::string to_string(const std::chrono::duration & dur) + { + auto seconds_as_int = std::chrono::duration_cast(dur); + if (seconds_as_int == dur) + return std::to_string(seconds_as_int.count()) + "s"; + auto seconds_as_double = std::chrono::duration_cast>(dur); + return std::to_string(seconds_as_double.count()) + "s"; + } + + template + std::ostream & operator<<(std::ostream & o, const std::chrono::time_point & tp) + { + return o << to_string(tp); + } + + template > + std::ostream & operator<<(std::ostream & o, const std::chrono::duration & dur) + { + return o << to_string(dur); + } +} diff --git a/libs/libcommon/include/ext/range.h b/libs/libcommon/include/ext/range.h index 61b644c2ce5..c379d453f7b 100644 --- a/libs/libcommon/include/ext/range.h +++ b/libs/libcommon/include/ext/range.h @@ -1,46 +1,42 @@ #pragma once #include -#include -#include -#include +#include +#include -/** Numeric range iterator, used to represent a half-closed interval [begin, end). - * In conjunction with std::reverse_iterator allows for forward and backward iteration - * over corresponding interval. - */ namespace ext { - template - using range_iterator = boost::counting_iterator; - - /** Range-based for loop adapter for (reverse_)range_iterator. - * By and large should be in conjunction with ext::range and ext::reverse_range. - */ - template - struct range_wrapper + /// For loop adaptor which is used to iterate through a half-closed interval [begin, end). + template + inline auto range(BeginType begin, EndType end) { - using value_type = typename std::remove_reference::type; - using iterator = range_iterator; + using CommonType = typename std::common_type::type; + return boost::counting_range(begin, end); + } - value_type begin_; - value_type end_; - - iterator begin() const { return iterator(begin_); } - iterator end() const { return iterator(end_); } - }; - - /** Constructs range_wrapper for forward-iteration over [begin, end) in range-based for loop. - * Usage example: - * for (const auto i : ext::range(0, 4)) print(i); - * Output: - * 0 1 2 3 - */ - template - inline range_wrapper::type> range(T1 begin, T2 end) + template + inline auto range(Type end) { - using common_type = typename std::common_type::type; - return { static_cast(begin), static_cast(end) }; + return range(static_cast(0), end); + } + + /// The same as range(), but every value is casted statically to a specified `ValueType`. + /// This is useful to iterate through all constants of a enum. + template + inline auto range_with_static_cast(BeginType begin, EndType end) + { + using CommonType = typename std::common_type::type; + if constexpr (std::is_same_v) + return boost::counting_range(begin, end); + else + return boost::counting_range(begin, end) + | boost::adaptors::transformed([](CommonType x) -> ValueType { return static_cast(x); }); + } + + template + inline auto range_with_static_cast(EndType end) + { + return range_with_static_cast(static_cast(0), end); } } diff --git a/libs/libcommon/include/ext/shared_ptr_helper.h b/libs/libcommon/include/ext/shared_ptr_helper.h index ca7219e6261..df132382fa6 100644 --- a/libs/libcommon/include/ext/shared_ptr_helper.h +++ b/libs/libcommon/include/ext/shared_ptr_helper.h @@ -20,4 +20,20 @@ struct shared_ptr_helper } }; + +template +struct is_shared_ptr +{ + static constexpr bool value = false; +}; + + +template +struct is_shared_ptr> +{ + static constexpr bool value = true; +}; + +template +inline constexpr bool is_shared_ptr_v = is_shared_ptr::value; }