Merge branch 'master' into fix-if-fixed-string

.github/workflows/anchore-analysis.yml

@@ -0,0 +1,37 @@
+# This workflow checks out code, performs an Anchore container image
+# vulnerability and compliance scan, and integrates the results with
+# GitHub Advanced Security code scanning feature.  For more information on
+# the Anchore scan action usage and parameters, see
+# https://github.com/anchore/scan-action.  For more information on
+# Anchore container image scanning in general, see
+# https://docs.anchore.com.
+
+name: Docker Container Scan (clickhouse-server)
+
+on:
+  pull_request:
+    paths: docker/server/Dockerfile
+  schedule:
+    - cron: '0 21 * * *'
+
+jobs:
+  Anchore-Build-Scan:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the code
+      uses: actions/checkout@v2
+    - name: Build the Docker image
+      run: |
+        cd docker/server
+        perl -pi -e 's|=\$version||g' Dockerfile
+        docker build . --file Dockerfile --tag localbuild/testimage:latest      
+    - name: Run the local Anchore scan action itself with GitHub Advanced Security code scanning integration enabled
+      uses: anchore/scan-action@master
+      with:
+        image-reference: "localbuild/testimage:latest"
+        dockerfile-path: "docker/server/Dockerfile"
+        acs-report-enable: true
+    - name: Upload Anchore Scan Report
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: results.sarif

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,33 @@
+name: "CodeQL Scanning"
+
+on:
+  schedule:
+    - cron: '0 19 * * *'
+jobs:
+  CodeQL-Build:
+
+    runs-on: self-hosted
+    timeout-minutes: 1440
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+        submodules: 'recursive'
+
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+      
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+
+      with:
+        languages: cpp
+
+    - run: sudo apt-get update && sudo apt-get install -y git cmake python ninja-build gcc-9 g++-9 && mkdir build
+    - run: cd build && CC=gcc-9 CXX=g++-9 cmake ..
+    - run: cd build && ninja
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "contrib/poco"]
 	path = contrib/poco
-	url = https://github.com/ClickHouse-Extras/poco
+	url = https://github.com/ClickHouse-Extras/poco.git
 	branch = clickhouse
 [submodule "contrib/zstd"]
 	path = contrib/zstd
@@ -157,6 +157,9 @@
 [submodule "contrib/openldap"]
 	path = contrib/openldap
 	url = https://github.com/openldap/openldap.git
+[submodule "contrib/AMQP-CPP"]
+	path = contrib/AMQP-CPP
+	url = https://github.com/CopernicaMarketingSoftware/AMQP-CPP.git
 [submodule "contrib/cassandra"]
 	path = contrib/cassandra
 	url = https://github.com/ClickHouse-Extras/cpp-driver.git

CHANGELOG.md

@@ -323,6 +323,81 @@
 
 ## ClickHouse release v20.3
 
+### ClickHouse release v20.3.12.112-lts 2020-06-25
+
+#### Bug Fix
+
+* Fix rare crash caused by using `Nullable` column in prewhere condition. Continuation of [#11608](https://github.com/ClickHouse/ClickHouse/issues/11608). [#11869](https://github.com/ClickHouse/ClickHouse/pull/11869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Don't allow arrayJoin inside higher order functions. It was leading to broken protocol synchronization. This closes [#3933](https://github.com/ClickHouse/ClickHouse/issues/3933). [#11846](https://github.com/ClickHouse/ClickHouse/pull/11846) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix using too many threads for queries. [#11788](https://github.com/ClickHouse/ClickHouse/pull/11788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)).
+* Now replicated fetches will be cancelled during metadata alter. [#11744](https://github.com/ClickHouse/ClickHouse/pull/11744) ([alesapin](https://github.com/alesapin)).
+* Fixed LOGICAL_ERROR caused by wrong type deduction of complex literals in Values input format. [#11732](https://github.com/ClickHouse/ClickHouse/pull/11732) ([tavplubix](https://github.com/tavplubix)).
+* Fix `ORDER BY ... WITH FILL` over const columns. [#11697](https://github.com/ClickHouse/ClickHouse/pull/11697) ([Anton Popov](https://github.com/CurtizJ)).
+* Pass proper timeouts when communicating with XDBC bridge. Recently timeouts were not respected when checking bridge liveness and receiving meta info. [#11690](https://github.com/ClickHouse/ClickHouse/pull/11690) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix error which leads to an incorrect state of `system.mutations`. It may show that whole mutation is already done but the server still has `MUTATE_PART` tasks in the replication queue and tries to execute them. This fixes [#11611](https://github.com/ClickHouse/ClickHouse/issues/11611). [#11681](https://github.com/ClickHouse/ClickHouse/pull/11681) ([alesapin](https://github.com/alesapin)).
+* Add support for regular expressions with case-insensitive flags. This fixes [#11101](https://github.com/ClickHouse/ClickHouse/issues/11101) and fixes [#11506](https://github.com/ClickHouse/ClickHouse/issues/11506). [#11649](https://github.com/ClickHouse/ClickHouse/pull/11649) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Remove trivial count query optimization if row-level security is set. In previous versions the user get total count of records in a table instead filtered. This fixes [#11352](https://github.com/ClickHouse/ClickHouse/issues/11352). [#11644](https://github.com/ClickHouse/ClickHouse/pull/11644) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix bloom filters for String (data skipping indices). [#11638](https://github.com/ClickHouse/ClickHouse/pull/11638) ([Azat Khuzhin](https://github.com/azat)).
+* Fix rare crash caused by using `Nullable` column in prewhere condition. (Probably it is connected with [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572) somehow). [#11608](https://github.com/ClickHouse/ClickHouse/pull/11608) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix error `Block structure mismatch` for queries with sampling reading from `Buffer` table. [#11602](https://github.com/ClickHouse/ClickHouse/pull/11602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix wrong exit code of the clickhouse-client, when exception.code() % 256 = 0. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)).
+* Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([tavplubix](https://github.com/tavplubix)).
+* Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)).
+* Fix shard_num/replica_num for `<node>` (breaks use_compact_format_in_distributed_parts_names). [#11528](https://github.com/ClickHouse/ClickHouse/pull/11528) ([Azat Khuzhin](https://github.com/azat)).
+* Fix memory leak when exception is thrown in the middle of aggregation with -State functions. This fixes [#8995](https://github.com/ClickHouse/ClickHouse/issues/8995). [#11496](https://github.com/ClickHouse/ClickHouse/pull/11496) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix wrong results of distributed queries when alias could override qualified column name. Fixes [#9672](https://github.com/ClickHouse/ClickHouse/issues/9672) [#9714](https://github.com/ClickHouse/ClickHouse/issues/9714). [#9972](https://github.com/ClickHouse/ClickHouse/pull/9972) ([Artem Zuikov](https://github.com/4ertus2)).
+
+
+### ClickHouse release v20.3.11.97-lts 2020-06-10
+
+#### New Feature
+
+* Now ClickHouse controls timeouts of dictionary sources on its side. Two new settings added to cache dictionary configuration: `strict_max_lifetime_seconds`, which is `max_lifetime` by default and `query_wait_timeout_milliseconds`, which is one minute by default. The first settings is also useful with `allow_read_expired_keys` settings (to forbid reading very expired keys). [#10337](https://github.com/ClickHouse/ClickHouse/pull/10337) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+
+#### Bug Fix
+
+* Fix the error `Data compressed with different methods` that can happen if `min_bytes_to_use_direct_io` is enabled and PREWHERE is active and using SAMPLE or high number of threads. This fixes [#11539](https://github.com/ClickHouse/ClickHouse/issues/11539). [#11540](https://github.com/ClickHouse/ClickHouse/pull/11540) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix return compressed size for codecs. [#11448](https://github.com/ClickHouse/ClickHouse/pull/11448) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix server crash when a column has compression codec with non-literal arguments. Fixes [#11365](https://github.com/ClickHouse/ClickHouse/issues/11365). [#11431](https://github.com/ClickHouse/ClickHouse/pull/11431) ([alesapin](https://github.com/alesapin)).
+* Fix pointInPolygon with nan as point. Fixes https://github.com/ClickHouse/ClickHouse/issues/11375. [#11421](https://github.com/ClickHouse/ClickHouse/pull/11421) ([Alexey Ilyukhov](https://github.com/livace)).
+* Fix crash in JOIN over LowCarinality(T) and Nullable(T). [#11380](https://github.com/ClickHouse/ClickHouse/issues/11380). [#11414](https://github.com/ClickHouse/ClickHouse/pull/11414) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix error code for wrong `USING` key. [#11373](https://github.com/ClickHouse/ClickHouse/issues/11373). [#11404](https://github.com/ClickHouse/ClickHouse/pull/11404) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fixed geohashesInBox with arguments outside of latitude/longitude range. [#11403](https://github.com/ClickHouse/ClickHouse/pull/11403) ([Vasily Nemkov](https://github.com/Enmk)).
+* Better errors for `joinGet()` functions. [#11389](https://github.com/ClickHouse/ClickHouse/pull/11389) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix possible `Pipeline stuck` error for queries with external sort and limit. Fixes [#11359](https://github.com/ClickHouse/ClickHouse/issues/11359). [#11366](https://github.com/ClickHouse/ClickHouse/pull/11366) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Remove redundant lock during parts send in ReplicatedMergeTree. [#11354](https://github.com/ClickHouse/ClickHouse/pull/11354) ([alesapin](https://github.com/alesapin)).
+* Fix support for `\G` (vertical output) in clickhouse-client in multiline mode. This closes [#9933](https://github.com/ClickHouse/ClickHouse/issues/9933). [#11350](https://github.com/ClickHouse/ClickHouse/pull/11350) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix crash in direct selects from StorageJoin (without JOIN) and wrong nullability. [#11340](https://github.com/ClickHouse/ClickHouse/pull/11340) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix crash in `quantilesExactWeightedArray`. [#11337](https://github.com/ClickHouse/ClickHouse/pull/11337) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Now merges stopped before change metadata in `ALTER` queries. [#11335](https://github.com/ClickHouse/ClickHouse/pull/11335) ([alesapin](https://github.com/alesapin)).
+* Make writing to `MATERIALIZED VIEW` with setting `parallel_view_processing = 1` parallel again. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#11330](https://github.com/ClickHouse/ClickHouse/pull/11330) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix visitParamExtractRaw when extracted JSON has strings with unbalanced { or [. [#11318](https://github.com/ClickHouse/ClickHouse/pull/11318) ([Ewout](https://github.com/devwout)).
+* Fix very rare race condition in ThreadPool. [#11314](https://github.com/ClickHouse/ClickHouse/pull/11314) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix potential uninitialized memory in conversion. Example: `SELECT toIntervalSecond(now64())`. [#11311](https://github.com/ClickHouse/ClickHouse/pull/11311) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix the issue when index analysis cannot work if a table has Array column in primary key and if a query is filtering by this column with `empty` or `notEmpty` functions. This fixes [#11286](https://github.com/ClickHouse/ClickHouse/issues/11286). [#11303](https://github.com/ClickHouse/ClickHouse/pull/11303) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix bug when query speed estimation can be incorrect and the limit of `min_execution_speed` may not work or work incorrectly if the query is throttled by `max_network_bandwidth`, `max_execution_speed` or `priority` settings. Change the default value of `timeout_before_checking_execution_speed` to non-zero, because otherwise the settings `min_execution_speed` and `max_execution_speed` have no effect. This fixes [#11297](https://github.com/ClickHouse/ClickHouse/issues/11297). This fixes [#5732](https://github.com/ClickHouse/ClickHouse/issues/5732). This fixes [#6228](https://github.com/ClickHouse/ClickHouse/issues/6228). Usability improvement: avoid concatenation of exception message with progress bar in `clickhouse-client`. [#11296](https://github.com/ClickHouse/ClickHouse/pull/11296) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix crash while reading malformed data in Protobuf format. This fixes https://github.com/ClickHouse/ClickHouse/issues/5957, fixes https://github.com/ClickHouse/ClickHouse/issues/11203. [#11258](https://github.com/ClickHouse/ClickHouse/pull/11258) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fixed a bug when cache-dictionary could return default value instead of normal (when there are only expired keys). This affects only string fields. [#11233](https://github.com/ClickHouse/ClickHouse/pull/11233) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fix error `Block structure mismatch in QueryPipeline` while reading from `VIEW` with constants in inner query. Fixes [#11181](https://github.com/ClickHouse/ClickHouse/issues/11181). [#11205](https://github.com/ClickHouse/ClickHouse/pull/11205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible exception `Invalid status for associated output`. [#11200](https://github.com/ClickHouse/ClickHouse/pull/11200) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible error `Cannot capture column` for higher-order functions with `Array(Array(LowCardinality))` captured argument. [#11185](https://github.com/ClickHouse/ClickHouse/pull/11185) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed S3 globbing which could fail in case of more than 1000 keys and some backends. [#11179](https://github.com/ClickHouse/ClickHouse/pull/11179) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* If data skipping index is dependent on columns that are going to be modified during background merge (for SummingMergeTree, AggregatingMergeTree as well as for TTL GROUP BY), it was calculated incorrectly. This issue is fixed by moving index calculation after merge so the index is calculated on merged data. [#11162](https://github.com/ClickHouse/ClickHouse/pull/11162) ([Azat Khuzhin](https://github.com/azat)).
+* Fix excessive reserving of threads for simple queries (optimization for reducing the number of threads, which was partly broken after changes in pipeline). [#11114](https://github.com/ClickHouse/ClickHouse/pull/11114) ([Azat Khuzhin](https://github.com/azat)).
+* Fix predicates optimization for distributed queries (`enable_optimize_predicate_expression=1`) for queries with `HAVING` section (i.e. when filtering on the server initiator is required), by preserving the order of expressions (and this is enough to fix), and also force aggregator use column names over indexes. Fixes: [#10613](https://github.com/ClickHouse/ClickHouse/issues/10613), [#11413](https://github.com/ClickHouse/ClickHouse/issues/11413). [#10621](https://github.com/ClickHouse/ClickHouse/pull/10621) ([Azat Khuzhin](https://github.com/azat)).
+* Introduce commit retry logic to decrease the possibility of getting duplicates from Kafka in rare cases when offset commit was failed. [#9884](https://github.com/ClickHouse/ClickHouse/pull/9884) ([filimonov](https://github.com/filimonov)).
+
+#### Performance Improvement
+
+* Get dictionary and check access rights only once per each call of any function reading external dictionaries. [#10928](https://github.com/ClickHouse/ClickHouse/pull/10928) ([Vitaly Baranov](https://github.com/vitlibar)).
+
+#### Build/Testing/Packaging Improvement
+
+* Fix several flaky integration tests. [#11355](https://github.com/ClickHouse/ClickHouse/pull/11355) ([alesapin](https://github.com/alesapin)).
+
 ### ClickHouse release v20.3.10.75-lts 2020-05-23
 
 #### Bug Fix

CMakeLists.txt

@@ -342,6 +342,7 @@ include (cmake/find/sparsehash.cmake)
 include (cmake/find/re2.cmake)
 include (cmake/find/libgsasl.cmake)
 include (cmake/find/rdkafka.cmake)
+include (cmake/find/amqpcpp.cmake)
 include (cmake/find/capnp.cmake)
 include (cmake/find/llvm.cmake)
 include (cmake/find/opencl.cmake)

README.md

@@ -13,3 +13,7 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Yandex.Messenger channel](https://yandex.ru/chat/#/join/20e380d9-c7be-4123-ab06-e95fb946975e) shares announcements and useful links in Russian.
 * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
 * You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
+
+## Upcoming Events		
+
+* [ClickHouse at Yandex Cloud Webinar (in Russian)](https://cloud.yandex.ru/events/144) on July 7, 2020.		

base/daemon/BaseDaemon.cpp

@@ -628,7 +628,7 @@ void BaseDaemon::initialize(Application & self)
 
     /// Create pid file.
     if (config().has("pid"))
-        pid.emplace(config().getString("pid"));
+        pid.emplace(config().getString("pid"), DB::StatusFile::write_pid);
 
     /// Change path for logging.
     if (!log_path.empty())
@@ -812,63 +812,6 @@ void BaseDaemon::defineOptions(Poco::Util::OptionSet & new_options)
     Poco::Util::ServerApplication::defineOptions(new_options);
 }
 
-bool isPidRunning(pid_t pid)
-{
-    return getpgid(pid) >= 0;
-}
-
-BaseDaemon::PID::PID(const std::string & file_)
-{
-    file = Poco::Path(file_).absolute().toString();
-    Poco::File poco_file(file);
-
-    if (poco_file.exists())
-    {
-        pid_t pid_read = 0;
-        {
-            std::ifstream in(file);
-            if (in.good())
-            {
-                in >> pid_read;
-                if (pid_read && isPidRunning(pid_read))
-                    throw Poco::Exception("Pid file exists and program running with pid = " + std::to_string(pid_read) + ", should not start daemon.");
-            }
-        }
-        std::cerr << "Old pid file exists (with pid = " << pid_read << "), removing." << std::endl;
-        poco_file.remove();
-    }
-
-    int fd = open(file.c_str(),
-        O_CREAT | O_EXCL | O_WRONLY,
-        S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
-
-    if (-1 == fd)
-    {
-        if (EEXIST == errno)
-            throw Poco::Exception("Pid file exists, should not start daemon.");
-        throw Poco::CreateFileException("Cannot create pid file.");
-    }
-
-    SCOPE_EXIT({ close(fd); });
-
-    std::stringstream s;
-    s << getpid();
-    if (static_cast<ssize_t>(s.str().size()) != write(fd, s.str().c_str(), s.str().size()))
-        throw Poco::Exception("Cannot write to pid file.");
-}
-
-BaseDaemon::PID::~PID()
-{
-    try
-    {
-        Poco::File(file).remove();
-    }
-    catch (...)
-    {
-        DB::tryLogCurrentException(__PRETTY_FUNCTION__);
-    }
-}
-
 void BaseDaemon::handleSignal(int signal_id)
 {
     if (signal_id == SIGINT ||

base/daemon/BaseDaemon.h

@@ -22,6 +22,7 @@
 #include <common/getThreadId.h>
 #include <daemon/GraphiteWriter.h>
 #include <Common/Config/ConfigProcessor.h>
+#include <Common/StatusFile.h>
 #include <loggers/Loggers.h>
 
 
@@ -163,16 +164,7 @@ protected:
 
     std::unique_ptr<Poco::TaskManager> task_manager;
 
-    /// RAII wrapper for pid file.
-    struct PID
-    {
-        std::string file;
-
-        PID(const std::string & file_);
-        ~PID();
-    };
-
-    std::optional<PID> pid;
+    std::optional<DB::StatusFile> pid;
 
     std::atomic_bool is_cancelled{false};
 

cmake/find/amqpcpp.cmake

@@ -0,0 +1,20 @@
+SET(ENABLE_AMQPCPP 1)
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/CMakeLists.txt")
+    message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init --recursive")
+    set (ENABLE_AMQPCPP 0)
+endif ()
+
+if (ENABLE_AMQPCPP)
+
+    set (USE_AMQPCPP 1)
+    set (AMQPCPP_LIBRARY AMQP-CPP)
+
+    set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include")
+
+    list (APPEND AMQPCPP_INCLUDE_DIR
+            "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include"
+            "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP")
+
+endif()
+
+message (STATUS "Using AMQP-CPP=${USE_AMQPCPP}: ${AMQPCPP_INCLUDE_DIR} : ${AMQPCPP_LIBRARY}")

contrib/AMQP-CPP

@@ -0,0 +1 @@
+Subproject commit 1c08399ab0ab9e4042ef8e2bbe9e208e5dcbc13b

contrib/CMakeLists.txt

@@ -301,6 +301,10 @@ if (USE_FASTOPS)
     add_subdirectory (fastops-cmake)
 endif()
 
+if (USE_AMQPCPP)
+    add_subdirectory (amqpcpp-cmake)
+endif()
+
 if (USE_CASSANDRA)
     add_subdirectory (libuv)
     add_subdirectory (cassandra)

contrib/amqpcpp-cmake/CMakeLists.txt

@@ -0,0 +1,44 @@
+set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP)
+
+set (SRCS
+    ${LIBRARY_DIR}/src/array.cpp
+    ${LIBRARY_DIR}/src/channel.cpp
+    ${LIBRARY_DIR}/src/channelimpl.cpp
+    ${LIBRARY_DIR}/src/connectionimpl.cpp
+    ${LIBRARY_DIR}/src/deferredcancel.cpp
+    ${LIBRARY_DIR}/src/deferredconfirm.cpp
+    ${LIBRARY_DIR}/src/deferredconsumer.cpp
+    ${LIBRARY_DIR}/src/deferredextreceiver.cpp
+    ${LIBRARY_DIR}/src/deferredget.cpp
+    ${LIBRARY_DIR}/src/deferredpublisher.cpp
+    ${LIBRARY_DIR}/src/deferredreceiver.cpp
+    ${LIBRARY_DIR}/src/field.cpp
+    ${LIBRARY_DIR}/src/flags.cpp
+    ${LIBRARY_DIR}/src/linux_tcp/openssl.cpp
+    ${LIBRARY_DIR}/src/linux_tcp/tcpconnection.cpp
+    ${LIBRARY_DIR}/src/receivedframe.cpp
+    ${LIBRARY_DIR}/src/table.cpp
+    ${LIBRARY_DIR}/src/watchable.cpp
+)
+
+add_library(amqp-cpp ${SRCS})
+
+target_compile_options (amqp-cpp
+    PUBLIC
+        -Wno-old-style-cast
+        -Wno-inconsistent-missing-destructor-override
+        -Wno-deprecated
+        -Wno-unused-parameter
+        -Wno-shadow
+        -Wno-tautological-type-limit-compare
+        -Wno-extra-semi
+# NOTE: disable all warnings at last because the warning:
+        #       "conversion function converting 'XXX' to itself will never be used"
+        #       doesn't have it's own diagnostic flag yet.
+        -w
+)
+
+target_include_directories (amqp-cpp PUBLIC ${LIBRARY_DIR}/include)
+
+target_link_libraries (amqp-cpp PUBLIC ssl)
+

contrib/jemalloc-cmake/CMakeLists.txt

@@ -22,9 +22,14 @@ if (ENABLE_JEMALLOC)
             #
             # By enabling percpu_arena number of arenas limited to number of CPUs and hence
             # this problem should go away.
-            set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0")
+            #
+            # muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to
+            # avoid spurious latencies and additional work associated with
+            # MADV_DONTNEED. See
+            # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
+            set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000")
         else()
-            set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0")
+            set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000")
         endif()
         # CACHE variable is empty, to allow changing defaults without necessity
         # to purge cache

contrib/poco

@@ -1 +1 @@
-Subproject commit be2ab90ba5dccd46919a116e3fe4fa77bb85063b
+Subproject commit 74c93443342f6028fa6402057684733b316aa737

docker/bare/Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+ADD root /

docker/bare/README.md

@@ -0,0 +1,37 @@
+## The bare minimum ClickHouse Docker image.
+
+It is intented as a showcase to check the amount of implicit dependencies of ClickHouse from the OS in addition to the OS kernel.
+
+Example usage:
+
+```
+./prepare
+docker build --tag clickhouse-bare .
+```
+
+Run clickhouse-local:
+```
+docker run -it --rm --network host clickhouse-bare /clickhouse local --query "SELECT 1"
+```
+
+Run clickhouse-client in interactive mode:
+```
+docker run -it --rm --network host clickhouse-bare /clickhouse client
+```
+
+Run clickhouse-server:
+```
+docker run -it --rm --network host clickhouse-bare /clickhouse server
+```
+
+It can be also run in chroot instead of Docker (first edit the `prepare` script to enable `proc`):
+
+```
+sudo chroot . /clickhouse server
+```
+
+## What does it miss?
+
+- creation of `clickhouse` user to run the server;
+- VOLUME for server;
+- most of the details, see other docker images for comparison.

docker/bare/prepare

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+SRC_DIR=../..
+BUILD_DIR=${SRC_DIR}/build
+
+# BTW, .so files are acceptable from any Linux distribution for the last 12 years (at least).
+# See https://presentations.clickhouse.tech/cpp_russia_2020/ for the details.
+
+mkdir root
+pushd root
+mkdir lib lib64 etc tmp root
+cp ${BUILD_DIR}/programs/clickhouse .
+cp ${SRC_DIR}/programs/server/{config,users}.xml .
+cp /lib/x86_64-linux-gnu/{libc.so.6,libdl.so.2,libm.so.6,libpthread.so.0,librt.so.1,libnss_dns.so.2,libresolv.so.2} lib
+cp /lib64/ld-linux-x86-64.so.2 lib64
+cp /etc/resolv.conf ./etc
+strip clickhouse
+
+# This is needed for chroot but not needed for Docker:
+
+# mkdir proc
+# sudo mount --bind /proc proc

docker/client/Dockerfile

@@ -6,6 +6,7 @@ ARG version=20.6.1.*
 RUN apt-get update \
     && apt-get install --yes --no-install-recommends \
         apt-transport-https \
+        ca-certificates \
         dirmngr \
         gnupg \
     && mkdir -p /etc/apt/sources.list.d \

docker/images.json

@@ -11,7 +11,8 @@
     "docker/packager/binary": {
         "name": "yandex/clickhouse-binary-builder",
         "dependent": [
-            "docker/test/split_build_smoke_test"
+            "docker/test/split_build_smoke_test",
+            "docker/test/pvs"
         ]
     },
     "docker/test/coverage": {

docker/packager/packager

@@ -31,7 +31,7 @@ def pull_image(image_name):
 def build_image(image_name, filepath):
     subprocess.check_call("docker build --network=host -t {} -f {} .".format(image_name, filepath), shell=True)
 
-def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache_dir):
+def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache_dir, docker_image_version):
     env_part = " -e ".join(env_variables)
     if env_part:
         env_part = " -e " + env_part
@@ -46,7 +46,7 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache
         ch_root=ch_root,
         ccache_dir=ccache_dir,
         env=env_part,
-        img_name=image_name,
+        img_name=image_name + ":" + docker_image_version,
         interactive=interactive
     )
 
@@ -189,6 +189,7 @@ if __name__ == "__main__":
     parser.add_argument("--alien-pkgs", nargs='+', default=[])
     parser.add_argument("--with-coverage", action="store_true")
     parser.add_argument("--with-binaries", choices=("programs", "tests", ""), default="")
+    parser.add_argument("--docker-image-version", default="latest")
 
     args = parser.parse_args()
     if not os.path.isabs(args.output_dir):
@@ -212,12 +213,14 @@ if __name__ == "__main__":
         logging.info("Should place {} to output".format(args.with_binaries))
 
     dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile")
+    image_with_version = image_name + ":" + args.docker_image_version
     if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image:
-        if not pull_image(image_name) or args.force_build_image:
-            build_image(image_name, dockerfile)
+        if not pull_image(image_with_version) or args.force_build_image:
+            build_image(image_with_version, dockerfile)
     env_prepared = parse_env_variables(
         args.build_type, args.compiler, args.sanitizer, args.package_type, image_type,
         args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.clang_tidy,
         args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries)
-    run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir)
+
+    run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir, args.docker_image_version)
     logging.info("Output placed into {}".format(args.output_dir))

docker/server/Dockerfile

@@ -7,6 +7,7 @@ ARG gosu_ver=1.10
 RUN apt-get update \
     && apt-get install --yes --no-install-recommends \
         apt-transport-https \
+        ca-certificates \
         dirmngr \
         gnupg \
     && mkdir -p /etc/apt/sources.list.d \
@@ -19,7 +20,6 @@ RUN apt-get update \
             clickhouse-client=$version \
             clickhouse-server=$version \
             locales \
-            ca-certificates \
             wget \
     && rm -rf \
         /var/lib/apt/lists/* \

docker/test/integration/compose/docker_compose_rabbitmq.yml

@@ -0,0 +1,12 @@
+version: '2.3'
+
+services:
+    rabbitmq1:
+        image: rabbitmq:3-management
+        hostname: rabbitmq1
+        ports:
+            - "5672:5672"
+            - "15672:15672"
+        environment:
+            RABBITMQ_DEFAULT_USER: "root"
+            RABBITMQ_DEFAULT_PASS: "clickhouse"

docker/test/performance-comparison/compare.sh

@@ -157,7 +157,11 @@ function run_tests
 
         TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n")
         # the grep is to filter out set -x output and keep only time output
-        { time "$script_dir/perf.py" --host localhost localhost --port 9001 9002 -- "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" || continue
+        { \
+            time "$script_dir/perf.py" --host localhost localhost --port 9001 9002 \
+                -- "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; \
+        } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" \
+            || echo "Test $test_name failed with error code $?" >> "$test_name-err.log"
     done
 
     unset TIMEFORMAT
@@ -274,10 +278,11 @@ for test_file in $(find . -maxdepth 1 -name "*-raw.tsv" -print)
 do
     test_name=$(basename "$test_file" "-raw.tsv")
     sed -n "s/^query\t/$test_name\t/p" < "$test_file" >> "analyze/query-runs.tsv"
-    sed -n "s/^client-time/$test_name/p" < "$test_file" >> "analyze/client-times.tsv"
-    sed -n "s/^report-threshold/$test_name/p" < "$test_file" >> "analyze/report-thresholds.tsv"
-    sed -n "s/^skipped/$test_name/p" < "$test_file" >> "analyze/skipped-tests.tsv"
-    sed -n "s/^display-name/$test_name/p" < "$test_file" >> "analyze/query-display-names.tsv"
+    sed -n "s/^client-time\t/$test_name\t/p" < "$test_file" >> "analyze/client-times.tsv"
+    sed -n "s/^report-threshold\t/$test_name\t/p" < "$test_file" >> "analyze/report-thresholds.tsv"
+    sed -n "s/^skipped\t/$test_name\t/p" < "$test_file" >> "analyze/skipped-tests.tsv"
+    sed -n "s/^display-name\t/$test_name\t/p" < "$test_file" >> "analyze/query-display-names.tsv"
+    sed -n "s/^partial\t/$test_name\t/p" < "$test_file" >> "analyze/partial-queries.tsv"
 done
 unset IFS
 
@@ -286,6 +291,18 @@ clickhouse-local --query "
 create view query_runs as select * from file('analyze/query-runs.tsv', TSV,
     'test text, query_index int, query_id text, version UInt8, time float');
 
+create view partial_queries as select test, query_index
+    from file('analyze/partial-queries.tsv', TSV,
+        'test text, query_index int, servers Array(int)');
+
+create table partial_query_times engine File(TSVWithNamesAndTypes,
+        'analyze/partial-query-times.tsv')
+    as select test, query_index, stddevPop(time) time_stddev, median(time) time_median
+    from query_runs
+    where (test, query_index) in partial_queries
+    group by test, query_index
+    ;
+
 create view left_query_log as select *
     from file('left-query-log.tsv', TSVWithNamesAndTypes,
         '$(cat "left-query-log.tsv.columns")');
@@ -329,6 +346,7 @@ create table query_run_metrics_full engine File(TSV, 'analyze/query-run-metrics-
     right join query_runs
         on query_logs.query_id = query_runs.query_id
             and query_logs.version = query_runs.version
+    where (test, query_index) not in partial_queries
     ;
 
 create table query_run_metrics engine File(
@@ -350,6 +368,7 @@ create table query_run_metric_names engine File(TSV, 'analyze/query-run-metric-n
 # query. We also don't have lateral joins. So I just put all runs of each
 # query into a separate file, and then compute randomization distribution
 # for each file. I do this in parallel using GNU parallel.
+( set +x # do not bloat the log
 IFS=$'\n'
 for prefix in $(cut -f1,2 "analyze/query-run-metrics.tsv" | sort | uniq)
 do
@@ -366,6 +385,7 @@ do
 done
 wait
 unset IFS
+)
 
 parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log
 }
@@ -389,12 +409,20 @@ create view query_display_names as select * from
         'test text, query_index int, query_display_name text')
     ;
 
+create table partial_queries_report engine File(TSV, 'report/partial-queries-report.tsv')
+    as select floor(time_median, 3) m, floor(time_stddev / time_median, 3) v,
+        test, query_index, query_display_name
+    from file('analyze/partial-query-times.tsv', TSVWithNamesAndTypes,
+        'test text, query_index int, time_stddev float, time_median float') t
+    join query_display_names using (test, query_index)
+    order by test, query_index
+    ;
+
 -- WITH, ARRAY JOIN and CROSS JOIN do not like each other:
 --  https://github.com/ClickHouse/ClickHouse/issues/11868
 --  https://github.com/ClickHouse/ClickHouse/issues/11757
 -- Because of this, we make a view with arrays first, and then apply all the
 -- array joins.
-
 create view query_metric_stat_arrays as
     with (select * from file('analyze/query-run-metric-names.tsv',
         TSV, 'n Array(String)')) as metric_name
@@ -766,9 +794,16 @@ done
 wait
 unset IFS
 
-# Remember that grep sets error code when nothing is found, hence the bayan
-# operator.
-grep -H -m2 -i '\(Exception\|Error\):[^:]' ./*-err.log | sed 's/:/\t/' >> run-errors.tsv ||:
+# Prefer to grep for clickhouse_driver exception messages, but if there are none,
+# just show a couple of lines from the log.
+for log in *-err.log
+do
+    test=$(basename "$log" "-err.log")
+    {
+        grep -H -m2 -i '\(Exception\|Error\):[^:]' "$log" \
+            || head -2 "$log"
+    } | sed "s/^/$test\t/" >> run-errors.tsv ||:
+done
 }
 
 function report_metrics
@@ -858,10 +893,6 @@ case "$stage" in
         cat "/proc/$pid/smaps" > "$pid-smaps.txt" ||:
     done
 
-    # Sleep for five minutes to see how the servers enter a quiescent state (e.g.
-    # how fast the memory usage drops).
-    sleep 300
-
     # We had a bug where getting profiles froze sometimes, so try to save some
     # logs if this happens again. Give the servers some time to collect all info,
     # then trace and kill. Start in a subshell, so that both function don't

docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml

@@ -20,4 +20,6 @@
     </metric_log>
 
     <uncompressed_cache_size>1000000000</uncompressed_cache_size>
+
+    <asynchronous_metrics_update_period_s>10</asynchronous_metrics_update_period_s>
 </yandex>

docker/test/performance-comparison/download.sh

@@ -24,14 +24,32 @@ dataset_paths["values"]="https://clickhouse-datasets.s3.yandex.net/values_with_e
 
 function download
 {
+    # Historically there were various path for the performance test package.
+    # Test all of them.
+    for path in "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/"{,clickhouse_build_check/}"performance/performance.tgz"
+    do
+        if curl --fail --head "$path"
+        then
+            left_path="$path"
+        fi
+    done
+
+    for path in "https://clickhouse-builds.s3.yandex.net/$right_pr/$right_sha/"{,clickhouse_build_check/}"performance/performance.tgz"
+    do
+        if curl --fail --head "$path"
+        then
+            right_path="$path"
+        fi
+    done
+
     # might have the same version on left and right
-    if ! [ "$left_sha" = "$right_sha" ]
+    if ! [ "$left_path" = "$right_path" ]
     then
-        wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/clickhouse_build_check/performance/performance.tgz" -O- | tar -C left --strip-components=1 -zxv  &
-        wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$right_pr/$right_sha/clickhouse_build_check/performance/performance.tgz" -O- | tar -C right --strip-components=1 -zxv &
+        wget -nv -nd -c "$left_path" -O- | tar -C left --strip-components=1 -zxv  &
+        wget -nv -nd -c "$right_path" -O- | tar -C right --strip-components=1 -zxv &
     else
         mkdir right ||:
-        wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/clickhouse_build_check/performance/performance.tgz" -O- | tar -C left --strip-components=1 -zxv && cp -a left/* right &
+        wget -nv -nd -c "$left_path" -O- | tar -C left --strip-components=1 -zxv && cp -a left/* right &
     fi
 
     for dataset_name in $datasets

docker/test/performance-comparison/entrypoint.sh

@@ -50,10 +50,18 @@ function find_reference_sha
 
         # FIXME sometimes we have testing tags on commits without published builds --
         # normally these are documentation commits. Loop to skip them.
-        if curl --fail --head "https://clickhouse-builds.s3.yandex.net/0/$REF_SHA/clickhouse_build_check/performance/performance.tgz"
-        then
-            break
-        fi
+        # Historically there were various path for the performance test package.
+        # Test all of them.
+        unset found
+        for path in "https://clickhouse-builds.s3.yandex.net/0/$REF_SHA/"{,clickhouse_build_check/}"performance/performance.tgz"
+        do
+            if curl --fail --head "$path"
+            then
+                found="$path"
+                break
+            fi
+        done
+        if [ -n "$found" ] ; then break; fi
 
         start_ref="$REF_SHA~"
     done

docker/test/performance-comparison/perf.py

@@ -7,6 +7,7 @@ import clickhouse_driver
 import xml.etree.ElementTree as et
 import argparse
 import pprint
+import re
 import string
 import time
 import traceback
@@ -102,10 +103,11 @@ for s in servers:
 # connection loses the changes in settings.
 drop_query_templates = [q.text for q in root.findall('drop_query')]
 drop_queries = substitute_parameters(drop_query_templates)
-for c in connections:
+for conn_index, c in enumerate(connections):
     for q in drop_queries:
         try:
             c.execute(q)
+            print(f'drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
         except:
             pass
 
@@ -117,10 +119,12 @@ for c in connections:
 # configurable). So the end result is uncertain, but hopefully we'll be able to
 # run at least some queries.
 settings = root.findall('settings/*')
-for c in connections:
+for conn_index, c in enumerate(connections):
     for s in settings:
         try:
-            c.execute("set {} = '{}'".format(s.tag, s.text))
+            q = f"set {s.tag} = '{s.text}'"
+            c.execute(q)
+            print(f'set\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
         except:
             print(traceback.format_exc(), file=sys.stderr)
 
@@ -139,16 +143,28 @@ for t in tables:
 # Run create queries
 create_query_templates = [q.text for q in root.findall('create_query')]
 create_queries = substitute_parameters(create_query_templates)
-for c in connections:
+
+# Disallow temporary tables, because the clickhouse_driver reconnects on errors,
+# and temporary tables are destroyed. We want to be able to continue after some
+# errors.
+for q in create_queries:
+    if re.search('create temporary table', q, flags=re.IGNORECASE):
+        print(f"Temporary tables are not allowed in performance tests: '{q}'",
+            file = sys.stderr)
+        sys.exit(1)
+
+for conn_index, c in enumerate(connections):
     for q in create_queries:
         c.execute(q)
+        print(f'create\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
 
 # Run fill queries
 fill_query_templates = [q.text for q in root.findall('fill_query')]
 fill_queries = substitute_parameters(fill_query_templates)
-for c in connections:
+for conn_index, c in enumerate(connections):
     for q in fill_queries:
         c.execute(q)
+        print(f'fill\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
 
 # Run test queries
 for query_index, q in enumerate(test_queries):
@@ -165,31 +181,47 @@ for query_index, q in enumerate(test_queries):
 
     # Prewarm: run once on both servers. Helps to bring the data into memory,
     # precompile the queries, etc.
-    try:
-        for conn_index, c in enumerate(connections):
+    # A query might not run on the old server if it uses a function added in the
+    # new one. We want to run them on the new server only, so that the PR author
+    # can ensure that the test works properly. Remember the errors we had on
+    # each server.
+    query_error_on_connection = [None] * len(connections);
+    for conn_index, c in enumerate(connections):
+        try:
             prewarm_id = f'{query_prefix}.prewarm0'
             res = c.execute(q, query_id = prewarm_id)
             print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}')
-    except KeyboardInterrupt:
-        raise
-    except:
-        # If prewarm fails for some query -- skip it, and try to test the others.
-        # This might happen if the new test introduces some function that the
-        # old server doesn't support. Still, report it as an error.
-        # FIXME the driver reconnects on error and we lose settings, so this might
-        # lead to further errors or unexpected behavior.
-        print(traceback.format_exc(), file=sys.stderr)
+        except KeyboardInterrupt:
+            raise
+        except:
+            # FIXME the driver reconnects on error and we lose settings, so this
+            # might lead to further errors or unexpected behavior.
+            query_error_on_connection[conn_index] = traceback.format_exc();
+            continue
+
+    # If prewarm fails for the query on both servers -- report the error, skip
+    # the query and continue testing the next query.
+    if query_error_on_connection.count(None) == 0:
+        print(query_error_on_connection[0], file = sys.stderr)
         continue
 
+    # If prewarm fails on one of the servers, run the query on the rest of them.
+    # Useful for queries that use new functions added in the new server version.
+    if query_error_on_connection.count(None) < len(query_error_on_connection):
+        no_error = [i for i, e in enumerate(query_error_on_connection) if not e]
+        print(f'partial\t{query_index}\t{no_error}')
+
     # Now, perform measured runs.
-    # Track the time spent by the client to process this query, so that we can notice
-    # out the queries that take long to process on the client side, e.g. by sending
-    # excessive data.
+    # Track the time spent by the client to process this query, so that we can
+    # notice the queries that take long to process on the client side, e.g. by
+    # sending excessive data.
     start_seconds = time.perf_counter()
     server_seconds = 0
     for run in range(0, args.runs):
         run_id = f'{query_prefix}.run{run}'
         for conn_index, c in enumerate(connections):
+            if query_error_on_connection[conn_index]:
+                continue
             res = c.execute(q, query_id = run_id)
             print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}')
             server_seconds += c.last_query.elapsed
@@ -198,8 +230,8 @@ for query_index, q in enumerate(test_queries):
     print(f'client-time\t{query_index}\t{client_seconds}\t{server_seconds}')
 
 # Run drop queries
-drop_query_templates = [q.text for q in root.findall('drop_query')]
 drop_queries = substitute_parameters(drop_query_templates)
-for c in connections:
+for conn_index, c in enumerate(connections):
     for q in drop_queries:
         c.execute(q)
+        print(f'drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')

docker/test/performance-comparison/report.py

@@ -7,6 +7,7 @@ import csv
 import itertools
 import json
 import os
+import os.path
 import pprint
 import sys
 import traceback
@@ -23,6 +24,7 @@ faster_queries = 0
 slower_queries = 0
 unstable_queries = 0
 very_unstable_queries = 0
+unstable_partial_queries = 0
 
 # max seconds to run one query by itself, not counting preparation
 allowed_single_run_time = 2
@@ -194,6 +196,31 @@ if args.report == 'main':
                      ['Client time, s', 'Server time, s', 'Ratio', 'Test', 'Query'],
                      slow_on_client_rows)
 
+    def print_partial():
+        rows = tsvRows('report/partial-queries-report.tsv')
+        if not rows:
+            return
+        global unstable_partial_queries, slow_average_tests
+        print(tableStart('Partial queries'))
+        columns = ['Median time, s', 'Relative time variance', 'Test', '#', 'Query']
+        print(tableHeader(columns))
+        attrs = ['' for c in columns]
+        for row in rows:
+            if float(row[1]) > 0.10:
+                attrs[1] = f'style="background: {color_bad}"'
+                unstable_partial_queries += 1
+            else:
+                attrs[1] = ''
+            if float(row[0]) > allowed_single_run_time:
+                attrs[0] = f'style="background: {color_bad}"'
+                slow_average_tests += 1
+            else:
+                attrs[0] = ''
+            print(tableRow(row, attrs))
+        print(tableEnd())
+
+    print_partial()
+
     def print_changes():
         rows = tsvRows('report/changed-perf.tsv')
         if not rows:
@@ -324,6 +351,9 @@ if args.report == 'main':
     print_test_times()
 
     def print_benchmark_results():
+        if not os.path.isfile('benchmark/website-left.json'):
+            return
+
         json_reports = [json.load(open(f'benchmark/website-{x}.json')) for x in ['left', 'right']]
         stats = [next(iter(x.values()))["statistics"] for x in json_reports]
         qps = [x["QPS"] for x in stats]
@@ -417,6 +447,11 @@ if args.report == 'main':
             status = 'failure'
         message_array.append(str(slower_queries) + ' slower')
 
+    if unstable_partial_queries:
+        unstable_queries += unstable_partial_queries
+        error_tests += unstable_partial_queries
+        status = 'failure'
+
     if unstable_queries:
         message_array.append(str(unstable_queries) + ' unstable')
 

docker/test/pvs/Dockerfile

@@ -25,7 +25,7 @@ ENV PKG_VERSION="pvs-studio-7.08.39365.50-amd64.deb"
 RUN wget "https://files.viva64.com/$PKG_VERSION"
 RUN sudo dpkg -i "$PKG_VERSION"
 
-CMD cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic  \
+CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic  \
     && cmake . && ninja re2_st && \
     pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
     plog-converter -a GA:1,2 -t fullhtml -o /test_output/pvs-studio-html-report pvs-studio.log; \

docker/test/stateful/Dockerfile

@@ -53,4 +53,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
     && clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" \
     && clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" \
     && clickhouse-client --query "SHOW TABLES FROM test" \
-    && clickhouse-test --testname --shard --zookeeper --no-stateless $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+    && clickhouse-test --testname --shard --zookeeper --no-stateless --use-skip-list $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt

docker/test/stateful_with_coverage/run.sh

@@ -105,7 +105,7 @@ LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABL
 LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
 LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
 LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
-LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-test --testname --shard --zookeeper --no-stateless $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-test --testname --shard --zookeeper --no-stateless --use-skip-list $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
 
 kill_clickhouse
 

docker/test/stateless/Dockerfile

@@ -83,4 +83,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
     if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/; fi; \
     ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml; \
     service zookeeper start; sleep 5; \
-    service clickhouse-server start && sleep 5 && clickhouse-test --testname --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+    service clickhouse-server start && sleep 5 && clickhouse-test --testname --shard --zookeeper --use-skip-list $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt

docker/test/stateless_with_coverage/run.sh

@@ -76,7 +76,7 @@ start_clickhouse
 
 sleep 10
 
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard --zookeeper --use-skip-list $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
 
 kill_clickhouse
 

docker/test/stress/stress

@@ -17,13 +17,13 @@ def run_perf_test(cmd, xmls_path, output_folder):
 def run_func_test(cmd, output_prefix, num_processes, skip_tests_option):
     output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)]
     f = open(output_paths[0], 'w')
-    main_command = "{} {}".format(cmd, skip_tests_option)
+    main_command = "{} --use-skip-list {}".format(cmd, skip_tests_option)
     logging.info("Run func tests main cmd '%s'", main_command)
     pipes = [Popen(main_command, shell=True, stdout=f, stderr=f)]
     for output_path in output_paths[1:]:
         time.sleep(0.5)
         f = open(output_path, 'w')
-        full_command = "{} --order=random {}".format(cmd, skip_tests_option)
+        full_command = "{} --use-skip-list --order=random {}".format(cmd, skip_tests_option)
         logging.info("Run func tests '%s'", full_command)
         p = Popen(full_command, shell=True, stdout=f, stderr=f)
         pipes.append(p)

docs/en/development/architecture.md

@@ -120,7 +120,7 @@ There are ordinary functions and aggregate functions. For aggregate functions, s
 
 Ordinary functions don’t change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`’s of data to implement vectorized query execution.
 
-There are some miscellaneous functions, like [blockSize](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), and [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate), that exploit block processing and violate the independence of rows.
+There are some miscellaneous functions, like [blockSize](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), and [runningAccumulate](../sql-reference/functions/other-functions.md#runningaccumulatexploit block processing and violate the independence of rows.
 
 ClickHouse has strong typing, so there’s no implicit type conversion. If a function doesn't support a specific combination of types, it throws an exception. But functions can work (be overloaded) for many different combinations of types. For example, the `plus` function (to implement the `+` operator) works for any combination of numeric types: `UInt8` + `Float32`, `UInt16` + `Int8`, and so on. Also, some variadic functions can accept any number of arguments, such as the `concat` function.
 

docs/en/interfaces/formats.md

@@ -1055,11 +1055,11 @@ Each Avro message embeds a schema id that can be resolved to the actual schema w
 
 Schemas are cached once resolved.
 
-Schema Registry URL is configured with [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#settings-format_avro_schema_registry_url)
+Schema Registry URL is configured with [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#format_avro_schema_registry_url).
 
 ### Data Types Matching {#data_types-matching-1}
 
-Same as [Avro](#data-format-avro)
+Same as [Avro](#data-format-avro).
 
 ### Usage {#usage}
 
@@ -1093,7 +1093,7 @@ SELECT * FROM topic1_stream;
 ```
 
 !!! note "Warning"
-    Setting `format_avro_schema_registry_url` needs to be configured in `users.xml` to maintain it’s value after a restart.
+    Setting `format_avro_schema_registry_url` needs to be configured in `users.xml` to maintain it’s value after a restart. Also you can use the `format_avro_schema_registry_url` setting of the `Kafka` table engine.
 
 ## Parquet {#data-format-parquet}
 

docs/en/introduction/adopters.md

@@ -50,7 +50,8 @@ toc_title: Adopters
 | <a href="http://www.pragma-innovation.fr/" class="favicon">Pragma Innovation</a>               | Telemetry and Big Data Analysis | Main product          | —                                                          | —                                                                            | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf)                                                                                  |
 | <a href="https://www.qingcloud.com/" class="favicon">QINGCLOUD</a>                             | Cloud services                  | Main product          | —                                                          | —                                                                            | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf)                                        |
 | <a href="https://qrator.net" class="favicon">Qrator</a>                                        | DDoS protection                 | Main product          | —                                                          | —                                                                            | [Blog Post, March 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/)                                                                                                                                      |
-| <a href="https://www.percent.cn/" class="favicon">Percent 百分点</a>                           | Analytics                       | Main Product          | —                                                          | —                                                                            | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf)                                                            |
+| <a href="https://www.percent.cn/" class="favicon">Percent 百分点</a>                            | Analytics                       | Main Product          | —                                                          | —                                                                            | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf)                                                            |
+| <a href="https://plausible.io/" class="favicon">Plausible</a>                                  | Analytics                       | Main Product          | —                                                          | —                                                                            | [Blog post, June 2020](https://twitter.com/PlausibleHQ/status/1273889629087969280)                                                                                                                                      |
 | <a href="https://rambler.ru" class="favicon">Rambler</a>                                       | Internet services               | Analytics             | —                                                          | —                                                                            | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141)                                                                                                |
 | <a href="https://www.tencent.com" class="favicon">Tencent</a>                                  | Messaging                       | Logging               | —                                                          | —                                                                            | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050)                                                                                                                                                   |
 | <a href="https://trafficstars.com/" class="favicon">Traffic Stars</a>                          | AD network                      | —                     | —                                                          | —                                                                            | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf)                                                                                          |

docs/en/operations/access-rights.md

@@ -34,7 +34,7 @@ By default, the ClickHouse server provides the `default` user account which is n
 If you just started using ClickHouse, consider the following scenario:
 
 1.  [Enable](#enabling-access-control) SQL-driven access control and account management for the `default` user.
-2.  Log in to the `default` user account and create all the required users. Don’t forget to create an administrator account (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`).
+2.  Log in to the `default` user account and create all the required users. Don’t forget to create an administrator account (`GRANT ALL ON *.* TO admin_user_account WITH GRANT OPTION`).
 3.  [Restrict permissions](../operations/settings/permissions-for-queries.md#permissions_for_queries) for the `default` user and disable SQL-driven access control and account management for it.
 
 ### Properties of Current Solution {#access-control-properties}

docs/en/operations/server-configuration-parameters/settings.md

@@ -398,6 +398,27 @@ The cache is shared for the server and memory is allocated as needed. The cache
 <mark_cache_size>5368709120</mark_cache_size>
 ```
 
+
+## max_server_memory_usage {#max_server_memory_usage}
+
+Limits total RAM usage by the ClickHouse server. You can specify it only for the default profile.
+
+Possible values:
+
+-   Positive integer.
+-   0 — Unlimited.
+
+Default value: `0`.
+
+**Additional Info**
+
+On hosts with low RAM and swap, you possibly need setting `max_server_memory_usage_to_ram_ratio > 1`.
+
+**See also**
+
+-   [max_memory_usage](../settings/query-complexity.md#settings_max_memory_usage)
+
+
 ## max\_concurrent\_queries {#max-concurrent-queries}
 
 The maximum number of simultaneously processed requests.

docs/en/operations/settings/query-complexity.md

@@ -36,7 +36,7 @@ Memory usage is not monitored for the states of certain aggregate functions.
 
 Memory usage is not fully tracked for states of the aggregate functions `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` from `String` and `Array` arguments.
 
-Memory consumption is also restricted by the parameters `max_memory_usage_for_user` and `max_memory_usage_for_all_queries`.
+Memory consumption is also restricted by the parameters `max_memory_usage_for_user` and [max_server_memory_usage](../server-configuration-parameters/settings.md#max_server_memory_usage).
 
 ## max\_memory\_usage\_for\_user {#max-memory-usage-for-user}
 
@@ -46,18 +46,10 @@ Default values are defined in [Settings.h](https://github.com/ClickHouse/ClickHo
 
 See also the description of [max\_memory\_usage](#settings_max_memory_usage).
 
-## max\_memory\_usage\_for\_all\_queries {#max-memory-usage-for-all-queries}
-
-The maximum amount of RAM to use for running all queries on a single server.
-
-Default values are defined in [Settings.h](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L289). By default, the amount is not restricted (`max_memory_usage_for_all_queries = 0`).
-
-See also the description of [max\_memory\_usage](#settings_max_memory_usage).
 
 ## max\_rows\_to\_read {#max-rows-to-read}
 
 The following restrictions can be checked on each block (instead of on each row). That is, the restrictions can be broken a little.
-When running a query in multiple threads, the following restrictions apply to each thread separately.
 
 A maximum number of rows that can be read from a table when running a query.
 

docs/en/operations/settings/settings.md

@@ -727,6 +727,17 @@ The INSERT query also contains data for INSERT that is processed by a separate s
 
 Default value: 256 KiB.
 
+## max\_parser\_depth {#max_parser_depth}
+
+Limits maximum recursion depth in the recursive descent parser. Allows to control stack size.
+
+Possible values:
+
+- Positive integer.
+- 0 — Recursion depth is unlimited.
+
+Default value: 1000.
+
 ## interactive\_delay {#interactive-delay}
 
 The interval in microseconds for checking whether request execution has been cancelled and sending the progress.
@@ -1368,13 +1379,11 @@ Possible values: 32 (32 bytes) - 1073741824 (1 GiB)
 
 Default value: 32768 (32 KiB)
 
-## format\_avro\_schema\_registry\_url {#settings-format_avro_schema_registry_url}
+## format\_avro\_schema\_registry\_url {#format_avro_schema_registry_url}
 
-Sets Confluent Schema Registry URL to use with [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent) format
+Sets [Confluent Schema Registry](https://docs.confluent.io/current/schema-registry/index.html) URL to use with [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent) format.
 
-Type: URL
-
-Default value: Empty
+Default value: `Empty`.
 
 ## background\_pool\_size {#background_pool_size}
 
@@ -1418,6 +1427,23 @@ Possible values:
 
 Default value: 16.
 
+## always_fetch_merged_part {#always_fetch_merged_part}
+
+Prohibits data parts merging in [Replicated*MergeTree](../../engines/table-engines/mergetree-family/replication.md)-engine tables.
+
+When merging is prohibited, the replica never merges parts and always downloads merged parts from other replicas. If there is no required data yet, the replica waits for it. CPU and disk load on the replica server decreases, but the network load on cluster increases. This setting can be useful on servers with relatively weak CPUs or slow disks, such as servers for backups storage.
+
+Possible values:
+
+-   0 — `Replicated*MergeTree`-engine tables merge data parts at the replica.
+-   1 — `Replicated*MergeTree`-engine tables don't merge data parts at the replica. The tables download merged data parts from other replicas.
+
+Default value: 0.
+
+**See Also** 
+
+-   [Data Replication](../../engines/table-engines/mergetree-family/replication.md)
+
 ## background\_distributed\_schedule\_pool\_size {#background_distributed_schedule_pool_size}
 
 Sets the number of threads performing background tasks for [distributed](../../engines/table-engines/special/distributed.md) sends. This setting is applied at ClickHouse server start and can’t be changed in a user session.

docs/en/operations/tips.md

@@ -120,6 +120,7 @@ zoo.cfg:
 tickTime=2000
 # The number of ticks that the initial
 # synchronization phase can take
+# This value is not quite motivated
 initLimit=30000
 # The number of ticks that can pass between
 # sending a request and getting an acknowledgement
@@ -127,6 +128,9 @@ syncLimit=10
 
 maxClientCnxns=2000
 
+# It is the maximum value that client may request and the server will accept.
+# It is Ok to have high maxSessionTimeout on server to allow clients to work with high session timeout if they want.
+# But we request session timeout of 30 seconds by default (you can change it with session_timeout_ms in ClickHouse config).
 maxSessionTimeout=60000000
 # the directory where the snapshot is stored.
 dataDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '}}' }}/data

docs/en/sql-reference/aggregate-functions/combinators.md

@@ -33,7 +33,7 @@ To work with these states, use:
 
 -   [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) table engine.
 -   [finalizeAggregation](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) function.
--   [runningAccumulate](../../sql-reference/functions/other-functions.md#function-runningaccumulate) function.
+-   [runningAccumulate](../../sql-reference/functions/other-functions.md#runningaccumulate) function.
 -   [-Merge](#aggregate_functions_combinators-merge) combinator.
 -   [-MergeState](#aggregate_functions_combinators-mergestate) combinator.
 

docs/en/sql-reference/functions/other-functions.md

@@ -1054,11 +1054,110 @@ Result:
 
 Takes state of aggregate function. Returns result of aggregation (finalized state).
 
-## runningAccumulate {#function-runningaccumulate}
+## runningAccumulate {#runningaccumulate}
 
-Takes the states of the aggregate function and returns a column with values, are the result of the accumulation of these states for a set of block lines, from the first to the current line.
-For example, takes state of aggregate function (example runningAccumulate(uniqState(UserID))), and for each row of block, return result of aggregate function on merge of states of all previous rows and current row.
-So, result of function depends on partition of data to blocks and on order of data in block.
+Accumulates states of an aggregate function for each row of a data block. 
+
+!!! warning "Warning"
+    The state is reset for each new data block.
+
+**Syntax**
+
+```sql
+runningAccumulate(agg_state[, grouping]);
+```
+
+**Parameters**
+
+- `agg_state` — State of the aggregate function. [AggregateFunction](../../sql-reference/data-types/aggregatefunction.md#data-type-aggregatefunction).
+- `grouping` — Grouping key. Optional. The state of the function is reset if the `grouping` value is changed. It can be any of the [supported data types](../../sql-reference/data-types/index.md) for which the equality operator is defined.
+
+**Returned value**
+
+- Each resulting row contains a result of the aggregate function, accumulated for all the input rows from 0 to the current position. `runningAccumulate` resets states for each new data block or when the `grouping` value changes.
+
+Type depends on the aggregate function used.
+
+**Examples**
+
+Consider how you can use `runningAccumulate` to find the cumulative sum of numbers without and with grouping.
+
+Query:
+
+```sql
+SELECT k, runningAccumulate(sum_k) AS res FROM (SELECT number as k, sumState(k) AS sum_k FROM numbers(10) GROUP BY k ORDER BY k);
+```
+
+Result:
+
+```text
+┌─k─┬─res─┐
+│ 0 │   0 │
+│ 1 │   1 │
+│ 2 │   3 │
+│ 3 │   6 │
+│ 4 │  10 │
+│ 5 │  15 │
+│ 6 │  21 │
+│ 7 │  28 │
+│ 8 │  36 │
+│ 9 │  45 │
+└───┴─────┘
+```
+
+The subquery generates `sumState` for every number from `0` to `9`. `sumState` returns the state of the [sum](../aggregate-functions/reference/sum.md) function that contains the sum of a single number.
+
+The whole query does the following:
+
+1. For the first row, `runningAccumulate` takes `sumState(0)` and returns `0`. 
+2. For the second row, the function merges `sumState(0)` and `sumState(1)` resulting in `sumState(0 + 1)`, and returns `1` as a result.
+3. For the third row, the function merges `sumState(0 + 1)` and `sumState(2)` resulting in `sumState(0 + 1 + 2)`, and returns `3` as a result.
+4. The actions are repeated until the block ends.
+
+The following example shows the `groupping` parameter usage:
+
+Query:
+
+```sql
+SELECT 
+    grouping,
+    item,
+    runningAccumulate(state, grouping) AS res
+FROM 
+(
+    SELECT 
+        toInt8(number / 4) AS grouping,
+        number AS item,
+        sumState(number) AS state
+    FROM numbers(15)
+    GROUP BY item
+    ORDER BY item ASC
+);
+```
+
+Result:
+
+```text
+┌─grouping─┬─item─┬─res─┐
+│        0 │    0 │   0 │
+│        0 │    1 │   1 │
+│        0 │    2 │   3 │
+│        0 │    3 │   6 │
+│        1 │    4 │   4 │
+│        1 │    5 │   9 │
+│        1 │    6 │  15 │
+│        1 │    7 │  22 │
+│        2 │    8 │   8 │
+│        2 │    9 │  17 │
+│        2 │   10 │  27 │
+│        2 │   11 │  38 │
+│        3 │   12 │  12 │
+│        3 │   13 │  25 │
+│        3 │   14 │  39 │
+└──────────┴──────┴─────┘
+```
+
+As you can see, `runningAccumulate` merges states for each group of rows separately.
 
 ## joinGet {#joinget}
 

docs/es/operations/settings/query-complexity.md

@@ -59,7 +59,6 @@ Ver también la descripción de [Método de codificación de datos:](#settings_m
 ## ¿Qué puedes encontrar en Neodigit {#max-rows-to-read}
 
 Las siguientes restricciones se pueden verificar en cada bloque (en lugar de en cada fila). Es decir, las restricciones se pueden romper un poco.
-Al ejecutar una consulta en varios subprocesos, las siguientes restricciones se aplican a cada subproceso por separado.
 
 Un número máximo de filas que se pueden leer de una tabla al ejecutar una consulta.
 

docs/ru/development/architecture.md

@@ -116,7 +116,7 @@ ClickHouse - полноценная колоночная СУБД. Данные
 
 Обычный функции не изменяют число строк и  работают так, как если бы обрабатывали каждую строку независимо. В действительности же, функции вызываются не к отдельным строкам, а блокам данных для реализации векторизованного выполнения запросов.
 
-Некоторые функции, такие как [blockSize](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), и [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate), эксплуатируют блочную обработку и нарушают независимость строк.
+Некоторые функции, такие как [blockSize](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), и [runningAccumulate](../sql-reference/functions/other-functions.md#runningaccumulate), эксплуатируют блочную обработку и нарушают независимость строк.
 
 ClickHouse имеет сильную типизацию, поэтому нет никакого неявного преобразования типов. Если функция не поддерживает определенную комбинацию типов, она создает исключение. Но функции могут работать (перегружаться) для многих различных комбинаций типов. Например, функция `plus` (для реализации `+` оператор) работает для любой комбинации числовых типов: `UInt8` + `Float32`, `UInt16` + `Int8` и так далее. Кроме того, некоторые вариадические функции, такие как `concat`, могут принимать любое количество аргументов.
 

docs/ru/engines/table-engines/index.md

@@ -53,7 +53,7 @@
 -   [Distributed](special/distributed.md#distributed)
 -   [MaterializedView](special/materializedview.md#materializedview)
 -   [Dictionary](special/dictionary.md#dictionary)
--   [Merge](special/merge.md#merge
+-   [Merge](special/merge.md#merge)
 -   [File](special/file.md#file)
 -   [Null](special/null.md#null)
 -   [Set](special/set.md#set)

docs/ru/interfaces/formats.md

@@ -945,6 +945,55 @@ message MessageType {
 ClickHouse пишет и читает сообщения `Protocol Buffers` в формате `length-delimited`. Это означает, что перед каждым сообщением пишется его длина
 в формате [varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). См. также [как читать и записывать сообщения Protocol Buffers в формате length-delimited в различных языках программирования](https://cwiki.apache.org/confluence/display/GEODE/Delimiting+Protobuf+Messages).
 
+## Avro {#data-format-avro}
+
+## AvroConfluent {#data-format-avro-confluent}
+
+Для формата `AvroConfluent` ClickHouse поддерживает декодирование сообщений `Avro` с одним объектом. Такие сообщения используются с [Kafka] (http://kafka.apache.org/) и  реестром схем [Confluent](https://docs.confluent.io/current/schema-registry/index.html). 
+
+Каждое сообщение `Avro` содержит идентификатор схемы, который может быть разрешен для фактической схемы с помощью реестра схем.
+
+Схемы кэшируются после разрешения.
+
+URL-адрес реестра схем настраивается с помощью [format\_avro\_schema\_registry\_url](../operations/settings/settings.md#format_avro_schema_registry_url).
+
+### Соответствие типов данных {#sootvetstvie-tipov-dannykh-0}
+
+Такое же, как в [Avro](#data-format-avro).
+
+### Использование {#ispolzovanie}
+
+Чтобы быстро проверить разрешение схемы, используйте [kafkacat](https://github.com/edenhill/kafkacat) с языком запросов [clickhouse-local](../operations/utilities/clickhouse-local.md): 
+
+``` bash
+$ kafkacat -b kafka-broker  -C -t topic1 -o beginning -f '%s' -c 3 | clickhouse-local   --input-format AvroConfluent --format_avro_schema_registry_url 'http://schema-registry' -S "field1 Int64, field2 String"  -q 'select *  from table'
+1 a
+2 b
+3 c
+```
+
+Чтобы использовать `AvroConfluent` с [Kafka](../engines/table-engines/integrations/kafka.md):
+
+``` sql
+CREATE TABLE topic1_stream
+(
+    field1 String,
+    field2 String
+)
+ENGINE = Kafka()
+SETTINGS
+kafka_broker_list = 'kafka-broker',
+kafka_topic_list = 'topic1',
+kafka_group_name = 'group1',
+kafka_format = 'AvroConfluent';
+
+SET format_avro_schema_registry_url = 'http://schema-registry';
+
+SELECT * FROM topic1_stream;
+```
+!!! note "Внимание"
+    `format_avro_schema_registry_url` необходимо настроить в `users.xml`, чтобы сохранить значение после перезапуска. Также можно использовать настройку `format_avro_schema_registry_url` табличного движка `Kafka`.
+
 ## Parquet {#data-format-parquet}
 
 [Apache Parquet](http://parquet.apache.org/) — формат поколоночного хранения данных, который распространён в экосистеме Hadoop. Для формата `Parquet` ClickHouse поддерживает операции чтения и записи.

docs/ru/operations/access-rights.md

@@ -31,7 +31,7 @@ ClickHouse поддерживает управление доступом на
 Если вы начали пользоваться ClickHouse недавно, попробуйте следующий сценарий:
 
 1. [Включите](#enabling-access-control) SQL-ориентированное управление доступом для пользователя `default`.
-2. Войдите под пользователем `default` и создайте всех необходимых пользователей. Не забудьте создать аккаунт администратора (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`).
+2. Войдите под пользователем `default` и создайте всех необходимых пользователей. Не забудьте создать аккаунт администратора (`GRANT ALL ON *.* TO admin_user_account WITH GRANT OPTION`).
 3. [Ограничьте разрешения](settings/permissions-for-queries.md#permissions_for_queries) для пользователя `default` и отключите для него SQL-ориентированное управление доступом.
 
 ### Особенности реализации {#access-control-properties}

docs/ru/operations/server-configuration-parameters/settings.md

@@ -372,6 +372,25 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat
 <max_concurrent_queries>100</max_concurrent_queries>
 ```
 
+## max_server_memory_usage {#max_server_memory_usage}
+
+Ограничивает объём оперативной памяти, используемой сервером ClickHouse. Настройка может быть задана только для профиля `default`.
+
+Возможные значения:
+
+-   Положительное целое число.
+-   0 — объём используемой памяти не ограничен.
+
+Значение по умолчанию: `0`.
+
+**Дополнительная информация**
+
+На серверах с небольшим объёмом RAM и файла подкачки может потребоваться настройка `max_server_memory_usage_to_ram_ratio > 1`.
+
+**См. также**
+
+-   [max_memory_usage](../settings/query-complexity.md#settings_max_memory_usage)
+
 ## max\_connections {#max-connections}
 
 Максимальное количество входящих соединений.

docs/ru/operations/settings/query-complexity.md

@@ -1,4 +1,4 @@
-# Ограничения на сложность запроса {#ogranicheniia-na-slozhnost-zaprosa}
+# Ограничения на сложность запроса {#restrictions-on-query-complexity}
 
 Ограничения на сложность запроса - часть настроек.
 Используются, чтобы обеспечить более безопасное исполнение запросов из пользовательского интерфейса.
@@ -32,7 +32,7 @@
 
 Потребление памяти не полностью учитывается для состояний агрегатных функций `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` от аргументов `String` и `Array`.
 
-Потребление памяти ограничивается также параметрами `max_memory_usage_for_user` и `max_memory_usage_for_all_queries`.
+Потребление памяти ограничивается также параметрами `max_memory_usage_for_user` и [max_server_memory_usage](../server-configuration-parameters/settings.md#max_server_memory_usage).
 
 ## max\_memory\_usage\_for\_user {#max-memory-usage-for-user}
 
@@ -42,18 +42,9 @@
 
 Смотрите также описание настройки [max\_memory\_usage](#settings_max_memory_usage).
 
-## max\_memory\_usage\_for\_all\_queries {#max-memory-usage-for-all-queries}
-
-Максимальный возможный объём оперативной памяти для всех запросов на одном сервере.
-
-Значения по умолчанию определены в файле [Settings.h](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L289). По умолчанию размер не ограничен (`max_memory_usage_for_all_queries = 0`).
-
-Смотрите также описание настройки [max\_memory\_usage](#settings_max_memory_usage).
-
 ## max\_rows\_to\_read {#max-rows-to-read}
 
 Следующие ограничения могут проверяться на каждый блок (а не на каждую строку). То есть, ограничения могут быть немного нарушены.
-При выполнении запроса в несколько потоков, следующие ограничения действуют в каждом потоке по отдельности.
 
 Максимальное количество строчек, которое можно прочитать из таблицы при выполнении запроса.
 

docs/ru/operations/settings/settings.md

@@ -644,6 +644,17 @@ log_query_threads=1
 
 Значение по умолчанию: 256 Кб.
 
+## max\_parser\_depth {#max_parser_depth}
+
+Ограничивает максимальную глубину рекурсии в парсере рекурсивного спуска. Позволяет контролировать размер стека.
+
+Возможные значения:
+
+- Положительное целое число.
+- 0 — Глубина рекурсии не ограничена.
+
+Значение по умолчанию: 1000.
+
 ## interactive\_delay {#interactive-delay}
 
 Интервал в микросекундах для проверки, не запрошена ли остановка выполнения запроса, и отправки прогресса.
@@ -1205,6 +1216,23 @@ Default value: 0.
 
 Значение по умолчанию: 16.
 
+## always_fetch_merged_part {#always_fetch_merged_part}
+
+Запрещает слияние данных для таблиц семейства [Replicated*MergeTree](../../engines/table-engines/mergetree-family/replication.md).
+
+Если слияние запрещено, реплика никогда не выполняет слияние отдельных кусков данных, а всегда загружает объединённые данные из других реплик. Если объединённых данных пока нет, реплика ждет их появления. Нагрузка на процессор и диски на реплике уменьшается, но нагрузка на сеть в кластере возрастает. Настройка может быть полезна на репликах с относительно слабыми процессорами или медленными дисками, например, на репликах для хранения архивных данных.
+
+Возможные значения:
+
+-   0 — таблицы семейства `Replicated*MergeTree` выполняют слияние данных на реплике.
+-   1 — таблицы семейства `Replicated*MergeTree` не выполняют слияние данных на реплике, а загружают объединённые данные из других реплик.
+
+Значение по умолчанию: 0.
+
+**См. также:** 
+
+-   [Репликация данных](../../engines/table-engines/mergetree-family/replication.md)
+
 ## transform_null_in {#transform_null_in}
 
 Разрешает сравнивать значения [NULL](../../sql-reference/syntax.md#null-literal) в операторе [IN](../../sql-reference/operators/in.md).
@@ -1313,6 +1341,12 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1;
 
 Значение по умолчанию: 16.
 
+## format\_avro\_schema\_registry\_url {#format_avro_schema_registry_url}
+
+Задает URL реестра схем [Confluent](https://docs.confluent.io/current/schema-registry/index.html) для использования с форматом [AvroConfluent](../../interfaces/formats.md#data-format-avro-confluent).
+
+Значение по умолчанию: `Пустая строка`.
+
 ## min_insert_block_size_rows_for_materialized_views {#min-insert-block-size-rows-for-materialized-views}
 
 Устанавливает минимальное количество строк в блоке, который может быть вставлен в таблицу запросом `INSERT`. Блоки меньшего размера склеиваются в блоки большего размера. Настройка применяется только для блоков, вставляемых в [материализованное представление](../../sql-reference/statements/create.md#create-view). Настройка позволяет избежать избыточного потребления памяти.

docs/ru/sql-reference/aggregate-functions/combinators.md

@@ -29,7 +29,7 @@
 
 -   Движок таблиц [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md).
 -   Функция [finalizeAggregation](../../sql-reference/aggregate-functions/combinators.md#function-finalizeaggregation).
--   Функция [runningAccumulate](../../sql-reference/aggregate-functions/combinators.md#function-runningaccumulate).
+-   Функция [runningAccumulate](../../sql-reference/aggregate-functions/combinators.md#runningaccumulate).
 -   Комбинатор [-Merge](#aggregate_functions_combinators-merge).
 -   Комбинатор [-MergeState](#aggregate_functions_combinators-mergestate).
 

docs/ru/sql-reference/functions/array-functions.md

@@ -948,7 +948,7 @@ flatten(array_of_arrays)
 
 **Параметры**
 
--   `array_of_arrays` — [Массивов](../../sql-reference/functions/array-functions.md) массивов. Например, `[[1,2,3], [4,5]]`.
+-   `array_of_arrays` — [Массив](../../sql-reference/functions/array-functions.md) массивов. Например, `[[1,2,3], [4,5]]`.
 
 **Примеры**
 

docs/ru/sql-reference/functions/other-functions.md

@@ -1,4 +1,4 @@
-# Прочие функции {#prochie-funktsii}
+# Прочие функции {#other-functions}
 
 ## hostName() {#hostname}
 
@@ -1036,9 +1036,110 @@ SELECT formatReadableSize(filesystemCapacity()) AS "Capacity", toTypeName(filesy
 
 Принимает состояние агрегатной функции. Возвращает результат агрегирования.
 
-## runningAccumulate {#function-runningaccumulate}
+## runningAccumulate {#runningaccumulate}
 
-Принимает на вход состояния агрегатной функции и возвращает столбец со значениями, которые представляют собой результат мёржа этих состояний для выборки строк из блока от первой до текущей строки. Например, принимает состояние агрегатной функции (например, `runningAccumulate(uniqState(UserID))`), и для каждой строки блока возвращает результат агрегатной функции после мёржа состояний функции для всех предыдущих строк и текущей. Таким образом, результат зависит от разбиения данных по блокам и от порядка данных в блоке.
+Накапливает состояния агрегатной функции для каждой строки блока данных.
+
+!!! warning "Warning"
+    Функция обнуляет состояние для каждого нового блока.
+
+**Синтаксис**
+
+```sql
+runningAccumulate(agg_state[, grouping]);
+```
+
+**Параметры**
+
+- `agg_state` — Состояние агрегатной функции. [AggregateFunction](../../sql-reference/data-types/aggregatefunction.md#data-type-aggregatefunction).
+- `grouping` — Ключ группировки. Опциональный параметр. Состояние функции обнуляется, если значение `grouping` меняется. Параметр может быть любого [поддерживаемого типа данных](../../sql-reference/data-types/index.md), для которого определен оператор равенства.
+
+**Возвращаемое значение**
+
+- Каждая результирующая строка содержит результат агрегатной функции, накопленный для всех входных строк от 0 до текущей позиции. `runningAccumulate` обнуляет состояния для каждого нового блока данных или при изменении значения `grouping`.
+
+Тип зависит от используемой агрегатной функции.
+
+**Примеры**
+
+Рассмотрим примеры использования `runningAccumulate` для нахождения кумулятивной суммы чисел без и с группировкой.
+
+Запрос:
+
+```sql
+SELECT k, runningAccumulate(sum_k) AS res FROM (SELECT number as k, sumState(k) AS sum_k FROM numbers(10) GROUP BY k ORDER BY k);
+```
+
+Результат:
+
+```text
+┌─k─┬─res─┐
+│ 0 │   0 │
+│ 1 │   1 │
+│ 2 │   3 │
+│ 3 │   6 │
+│ 4 │  10 │
+│ 5 │  15 │
+│ 6 │  21 │
+│ 7 │  28 │
+│ 8 │  36 │
+│ 9 │  45 │
+└───┴─────┘
+```
+
+Подзапрос формирует `sumState` для каждого числа от `0` до `9`. `sumState` возвращает состояние функции [sum](../../sql-reference/aggregate-functions/reference.md#agg_function-sum), содержащее сумму одного числа.
+
+Весь запрос делает следующее:
+
+1. Для первой строки `runningAccumulate` берет `sumState(0)` и возвращает `0`.
+2. Для второй строки функция объединяет `sumState (0)` и `sumState (1)`, что приводит к `sumState (0 + 1)`, и возвращает в результате `1`.
+3. Для третьей строки функция объединяет `sumState (0 + 1)` и `sumState (2)`, что приводит к `sumState (0 + 1 + 2)`, и в результате возвращает `3`.
+4. Действия повторяются до тех пор, пока не закончится блок.
+
+В следующем примере показано использование параметра `grouping`:
+
+Запрос:
+
+```sql
+SELECT 
+    grouping,
+    item,
+    runningAccumulate(state, grouping) AS res
+FROM 
+(
+    SELECT 
+        toInt8(number / 4) AS grouping,
+        number AS item,
+        sumState(number) AS state
+    FROM numbers(15)
+    GROUP BY item
+    ORDER BY item ASC
+);
+```
+
+Результат:
+
+```text
+┌─grouping─┬─item─┬─res─┐
+│        0 │    0 │   0 │
+│        0 │    1 │   1 │
+│        0 │    2 │   3 │
+│        0 │    3 │   6 │
+│        1 │    4 │   4 │
+│        1 │    5 │   9 │
+│        1 │    6 │  15 │
+│        1 │    7 │  22 │
+│        2 │    8 │   8 │
+│        2 │    9 │  17 │
+│        2 │   10 │  27 │
+│        2 │   11 │  38 │
+│        3 │   12 │  12 │
+│        3 │   13 │  25 │
+│        3 │   14 │  39 │
+└──────────┴──────┴─────┘
+```
+
+Как вы можете видеть, `runningAccumulate` объединяет состояния для каждой группы строк отдельно.
 
 ## joinGet {#joinget}
 

docs/tools/blog.py

@@ -80,7 +80,8 @@ def build_for_lang(lang, args):
                 includes_dir=os.path.join(os.path.dirname(__file__), '..', '_includes'),
                 is_amp=False,
                 is_blog=True,
-                post_meta=post_meta
+                post_meta=post_meta,
+                today=datetime.date.today().isoformat()
             )
         )
 
@@ -89,6 +90,13 @@ def build_for_lang(lang, args):
 
         redirects.build_blog_redirects(args)
 
+        env = util.init_jinja2_env(args)
+        with open(os.path.join(args.website_dir, 'templates', 'blog', 'rss.xml'), 'rb') as f:
+            rss_template_string = f.read().decode('utf-8').strip()
+        rss_template = env.from_string(rss_template_string)
+        with open(os.path.join(args.blog_output_dir, lang, 'rss.xml'), 'w') as f:
+            f.write(rss_template.render({'config': raw_config}))
+
         # TODO: AMP for blog
         # if not args.skip_amp:
         #     amp.build_amp(lang, args, cfg)

docs/tools/mdx_clickhouse.py

@@ -14,9 +14,6 @@ import macros.plugin
 
 import slugify as slugify_impl
 
-import amp
-import website
-
 
 def slugify(value, separator):
     return slugify_impl.slugify(value, separator=separator, word_boundary=True, save_order=True)
@@ -119,6 +116,7 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin):
         ])
 
     def on_env(self, env, config, files):
+        import util
         env.add_extension('jinja2.ext.i18n')
         dirname = os.path.join(config.data['theme'].dirs[0], 'locale')
         lang = config.data['theme']['language']
@@ -126,10 +124,7 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin):
             get_translations(dirname, lang),
             newstyle=True
         )
-        chunk_size = 10240
-        env.filters['chunks'] = lambda line: [line[i:i+chunk_size] for i in range(0, len(line), chunk_size)]
-        env.filters['html_to_amp'] = amp.html_to_amp
-        env.filters['adjust_markdown_html'] = website.adjust_markdown_html
+        util.init_jinja2_filters(env)
         return env
 
     def render(self, markdown):

docs/tools/requirements.txt

@@ -25,7 +25,7 @@ protobuf==3.12.2
 numpy==1.18.5
 Pygments==2.5.2
 pymdown-extensions==7.1
-python-slugify==1.2.6
+python-slugify==4.0.1
 PyYAML==5.3.1
 repackage==0.7.3
 requests==2.24.0

docs/tools/translate/requirements.txt

@@ -5,7 +5,7 @@ googletrans==3.0.0
 idna==2.10
 Jinja2==2.11.2
 pandocfilters==1.4.2
-python-slugify==4.0.0
+python-slugify==4.0.1
 PyYAML==5.3.1
 requests==2.24.0
 text-unidecode==1.3

docs/tools/util.py

@@ -1,5 +1,6 @@
 import collections
 import contextlib
+import datetime
 import multiprocessing
 import os
 import shutil
@@ -8,6 +9,7 @@ import socket
 import tempfile
 import threading
 
+import jinja2
 import yaml
 
 
@@ -111,3 +113,35 @@ def represent_ordereddict(dumper, data):
 
 
 yaml.add_representer(collections.OrderedDict, represent_ordereddict)
+
+
+def init_jinja2_filters(env):
+    import amp
+    import website
+    chunk_size = 10240
+    env.filters['chunks'] = lambda line: [line[i:i + chunk_size] for i in range(0, len(line), chunk_size)]
+    env.filters['html_to_amp'] = amp.html_to_amp
+    env.filters['adjust_markdown_html'] = website.adjust_markdown_html
+    env.filters['to_rfc882'] = lambda d: datetime.datetime.strptime(d, '%Y-%m-%d').strftime('%a, %d %b %Y %H:%M:%S GMT')
+
+
+def init_jinja2_env(args):
+    import mdx_clickhouse
+    env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader([
+            args.website_dir,
+            os.path.join(args.docs_dir, '_includes')
+        ]),
+        extensions=[
+            'jinja2.ext.i18n',
+            'jinja2_highlight.HighlightExtension'
+        ]
+    )
+    env.extend(jinja2_highlight_cssclass='syntax p-3 my-3')
+    translations_dir = os.path.join(args.website_dir, 'locale')
+    env.install_gettext_translations(
+        mdx_clickhouse.get_translations(translations_dir, 'en'),
+        newstyle=True
+    )
+    init_jinja2_filters(env)
+    return env

docs/tools/website.py

@@ -11,15 +11,21 @@ import bs4
 import closure
 import cssmin
 import htmlmin
-import jinja2
 import jsmin
 
-import mdx_clickhouse
+import util
 
 
 def handle_iframe(iframe, soup):
-    if not iframe.attrs['src'].startswith('https://www.youtube.com/'):
-        raise RuntimeError('iframes are allowed only for YouTube')
+    allowed_domains = ['https://www.youtube.com/', 'https://datalens.yandex/']
+    illegal_domain = True
+    iframe_src = iframe.attrs['src']
+    for domain in allowed_domains:
+        if iframe_src.startswith(domain):
+            illegal_domain = False
+            break
+    if illegal_domain:
+        raise RuntimeError(f'iframe from illegal domain: {iframe_src}')
     wrapper = soup.new_tag('div')
     wrapper.attrs['class'] = ['embed-responsive', 'embed-responsive-16by9']
     iframe.insert_before(wrapper)
@@ -43,8 +49,11 @@ def adjust_markdown_html(content):
 
     for a in soup.find_all('a'):
         a_class = a.attrs.get('class')
+        a_href = a.attrs.get('href')
         if a_class and 'headerlink' in a_class:
             a.string = '\xa0'
+        if a_href and a_href.startswith('http'):
+            a.attrs['target'] = '_blank'
 
     for iframe in soup.find_all('iframe'):
         handle_iframe(iframe, soup)
@@ -121,22 +130,7 @@ def minify_html(content):
 
 def build_website(args):
     logging.info('Building website')
-    env = jinja2.Environment(
-        loader=jinja2.FileSystemLoader([
-            args.website_dir,
-            os.path.join(args.docs_dir, '_includes')
-        ]),
-        extensions=[
-            'jinja2.ext.i18n',
-            'jinja2_highlight.HighlightExtension'
-        ]
-    )
-    env.extend(jinja2_highlight_cssclass='syntax p-3 my-3')
-    translations_dir = os.path.join(args.website_dir, 'locale')
-    env.install_gettext_translations(
-        mdx_clickhouse.get_translations(translations_dir, 'en'),
-        newstyle=True
-    )
+    env = util.init_jinja2_env(args)
 
     shutil.copytree(
         args.website_dir,

docs/zh/development/browse-code.md

@@ -1,14 +1,12 @@
 ---
-machine_translated: true
-machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
 toc_priority: 63
 toc_title: "\u6D4F\u89C8\u6E90\u4EE3\u7801"
 ---
 
 # 浏览ClickHouse源代码 {#browse-clickhouse-source-code}
 
-您可以使用 **Woboq** 在线代码浏览器可用 [这里](https://clickhouse.tech/codebrowser/html_report/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示，搜索和索引。 代码快照每天更新。
+您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.tech/codebrowser/html_report/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。
 
-此外，您还可以浏览源 [GitHub](https://github.com/ClickHouse/ClickHouse) 像往常一样
+此外，您还可以像往常一样浏览源代码 [GitHub](https://github.com/ClickHouse/ClickHouse) 
 
-如果你有兴趣使用什么样的IDE，我们建议CLion，QT Creator，VS Code和KDevelop（有注意事项）。 您可以使用任何喜欢的IDE。 Vim和Emacs也算数。
+如果你希望了解哪种IDE较好，我们推荐使用CLion，QT Creator，VS Code和KDevelop（有注意事项）。 您可以使用任何您喜欢的IDE。 Vim和Emacs也可以。

docs/zh/development/build-osx.md

@@ -6,13 +6,13 @@ ClickHouse 支持在 Mac OS X 10.12 版本中编译。若您在用更早的操
 ## 安装 Homebrew {#an-zhuang-homebrew}
 
 ``` bash
-/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
 ```
 
 ## 安装编译器，工具库 {#an-zhuang-bian-yi-qi-gong-ju-ku}
 
 ``` bash
-brew install cmake ninja gcc icu4c mariadb-connector-c openssl libtool gettext
+$ brew install cmake ninja libtool gettext
 ```
 
 ## 拉取 ClickHouse 源码 {#la-qu-clickhouse-yuan-ma}
@@ -27,11 +27,11 @@ cd ClickHouse
 ## 编译 ClickHouse {#bian-yi-clickhouse}
 
 ``` bash
-mkdir build
-cd build
-cmake .. -DCMAKE_CXX_COMPILER=`which g++-8` -DCMAKE_C_COMPILER=`which gcc-8`
-ninja
-cd ..
+$ mkdir build
+$ cd build
+$ cmake .. -DCMAKE_CXX_COMPILER=`which clang++` -DCMAKE_C_COMPILER=`which clang`
+$ ninja
+$ cd ..
 ```
 
 ## 注意事项 {#zhu-yi-shi-xiang}

docs/zh/operations/settings/query-complexity.md

@@ -60,7 +60,6 @@ Restrictions on the «maximum amount of something» can take the value 0, which
 ## max\_rows\_to\_read {#max-rows-to-read}
 
 可以在每个块（而不是每行）上检查以下限制。 也就是说，限制可以打破一点。
-在多个线程中运行查询时，以下限制单独应用于每个线程。
 
 运行查询时可从表中读取的最大行数。
 

programs/copier/ClusterCopierApp.cpp

@@ -1,4 +1,6 @@
 #include "ClusterCopierApp.h"
+#include <Common/StatusFile.h>
+
 
 namespace DB
 {
@@ -91,7 +93,7 @@ void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options)
 
 void ClusterCopierApp::mainImpl()
 {
-    StatusFile status_file(process_path + "/status");
+    StatusFile status_file(process_path + "/status", StatusFile::write_full_info);
     ThreadStatus thread_status;
 
     auto * log = &logger();

programs/copier/Internals.h

@@ -66,7 +66,6 @@
 #include <Dictionaries/registerDictionaries.h>
 #include <Disks/registerDisks.h>
 #include <Databases/DatabaseMemory.h>
-#include <Common/StatusFile.h>
 
 #include "Aliases.h"
 

programs/local/LocalServer.cpp

@@ -248,7 +248,7 @@ try
     if (!context->getPath().empty())
     {
         /// Lock path directory before read
-        status.emplace(context->getPath() + "status");
+        status.emplace(context->getPath() + "status", StatusFile::write_full_info);
 
         LOG_DEBUG(log, "Loading metadata from {}", context->getPath());
         loadMetadataSystem(*context);

programs/server/Server.cpp

@@ -378,7 +378,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
 
     global_context->setPath(path);
 
-    StatusFile status{path + "status"};
+    StatusFile status{path + "status", StatusFile::write_full_info};
 
     SCOPE_EXIT({
         /** Ask to cancel background jobs all table engines,

programs/server/config.xml

@@ -66,7 +66,6 @@
     <https_port>8443</https_port>
     <tcp_port_secure>9440</tcp_port_secure>
     -->
-
     <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
     <openSSL>
         <server> <!-- Used for https server AND secure tcp port -->

src/Access/AccessControlManager.cpp

@@ -40,27 +40,8 @@ class AccessControlManager::ContextAccessCache
 public:
     explicit ContextAccessCache(const AccessControlManager & manager_) : manager(manager_) {}
 
-    std::shared_ptr<const ContextAccess> getContextAccess(
-        const UUID & user_id,
-        const boost::container::flat_set<UUID> & current_roles,
-        bool use_default_roles,
-        const Settings & settings,
-        const String & current_database,
-        const ClientInfo & client_info)
+    std::shared_ptr<const ContextAccess> getContextAccess(const ContextAccessParams & params)
     {
-        ContextAccess::Params params;
-        params.user_id = user_id;
-        params.current_roles = current_roles;
-        params.use_default_roles = use_default_roles;
-        params.current_database = current_database;
-        params.readonly = settings.readonly;
-        params.allow_ddl = settings.allow_ddl;
-        params.allow_introspection = settings.allow_introspection_functions;
-        params.interface = client_info.interface;
-        params.http_method = client_info.http_method;
-        params.address = client_info.current_address.host();
-        params.quota_key = client_info.quota_key;
-
         std::lock_guard lock{mutex};
         auto x = cache.get(params);
         if (x)
@@ -119,7 +100,25 @@ std::shared_ptr<const ContextAccess> AccessControlManager::getContextAccess(
     const String & current_database,
     const ClientInfo & client_info) const
 {
-    return context_access_cache->getContextAccess(user_id, current_roles, use_default_roles, settings, current_database, client_info);
+    ContextAccessParams params;
+    params.user_id = user_id;
+    params.current_roles = current_roles;
+    params.use_default_roles = use_default_roles;
+    params.current_database = current_database;
+    params.readonly = settings.readonly;
+    params.allow_ddl = settings.allow_ddl;
+    params.allow_introspection = settings.allow_introspection_functions;
+    params.interface = client_info.interface;
+    params.http_method = client_info.http_method;
+    params.address = client_info.current_address.host();
+    params.quota_key = client_info.quota_key;
+    return getContextAccess(params);
+}
+
+
+std::shared_ptr<const ContextAccess> AccessControlManager::getContextAccess(const ContextAccessParams & params) const
+{
+    return context_access_cache->getContextAccess(params);
 }
 
 

src/Access/AccessControlManager.h

@@ -21,6 +21,7 @@ namespace Poco
 namespace DB
 {
 class ContextAccess;
+struct ContextAccessParams;
 struct User;
 using UserPtr = std::shared_ptr<const User>;
 class EnabledRoles;
@@ -58,6 +59,8 @@ public:
         const String & current_database,
         const ClientInfo & client_info) const;
 
+    std::shared_ptr<const ContextAccess> getContextAccess(const ContextAccessParams & params) const;
+
     std::shared_ptr<const EnabledRoles> getEnabledRoles(
         const boost::container::flat_set<UUID> & current_roles,
         const boost::container::flat_set<UUID> & current_roles_with_admin_option) const;

src/Access/AccessRights.cpp

@@ -1,7 +1,9 @@
 #include <Access/AccessRights.h>
 #include <Common/Exception.h>
 #include <common/logger_useful.h>
+#include <boost/container/small_vector.hpp>
 #include <boost/range/adaptor/map.hpp>
+#include <boost/range/algorithm/sort.hpp>
 #include <unordered_map>
 
 namespace DB
@@ -9,7 +11,6 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int INVALID_GRANT;
-    extern const int LOGICAL_ERROR;
 }
 
 
@@ -58,12 +59,194 @@ namespace
         const AccessFlags system_reload_embedded_dictionaries = AccessType::SYSTEM_RELOAD_EMBEDDED_DICTIONARIES;
     };
 
-    std::string_view checkCurrentDatabase(const std::string_view & current_database)
+    using Kind = AccessRightsElementWithOptions::Kind;
+
+    struct ProtoElement
     {
-        if (current_database.empty())
-            throw Exception("No current database", ErrorCodes::LOGICAL_ERROR);
-        return current_database;
-    }
+        AccessFlags access_flags;
+        boost::container::small_vector<std::string_view, 3> full_name;
+        bool grant_option = false;
+        Kind kind = Kind::GRANT;
+
+        friend bool operator<(const ProtoElement & left, const ProtoElement & right)
+        {
+            static constexpr auto compare_name = [](const boost::container::small_vector<std::string_view, 3> & left_name,
+                                                    const boost::container::small_vector<std::string_view, 3> & right_name,
+                                                    size_t i)
+            {
+                if (i < left_name.size())
+                {
+                    if (i < right_name.size())
+                        return left_name[i].compare(right_name[i]);
+                    else
+                        return 1; /// left_name is longer => left_name > right_name
+                }
+                else if (i < right_name.size())
+                    return 1; /// right_name is longer => left < right
+                else
+                    return 0; /// left_name == right_name
+            };
+
+            if (int cmp = compare_name(left.full_name, right.full_name, 0))
+                return cmp < 0;
+
+            if (int cmp = compare_name(left.full_name, right.full_name, 1))
+                return cmp < 0;
+
+            if (left.kind != right.kind)
+                return (left.kind == Kind::GRANT);
+
+            if (left.grant_option != right.grant_option)
+                return right.grant_option;
+
+            if (int cmp = compare_name(left.full_name, right.full_name, 2))
+                return cmp < 0;
+
+            return (left.access_flags < right.access_flags);
+        }
+
+        AccessRightsElementWithOptions getResult() const
+        {
+            AccessRightsElementWithOptions res;
+            res.access_flags = access_flags;
+            res.grant_option = grant_option;
+            res.kind = kind;
+            switch (full_name.size())
+            {
+                case 0:
+                {
+                    res.any_database = true;
+                    res.any_table = true;
+                    res.any_column = true;
+                    break;
+                }
+                case 1:
+                {
+                    res.any_database = false;
+                    res.database = full_name[0];
+                    res.any_table = true;
+                    res.any_column = true;
+                    break;
+                }
+                case 2:
+                {
+                    res.any_database = false;
+                    res.database = full_name[0];
+                    res.any_table = false;
+                    res.table = full_name[1];
+                    res.any_column = true;
+                    break;
+                }
+                case 3:
+                {
+                    res.any_database = false;
+                    res.database = full_name[0];
+                    res.any_table = false;
+                    res.table = full_name[1];
+                    res.any_column = false;
+                    res.columns.emplace_back(full_name[2]);
+                    break;
+                }
+            }
+            return res;
+        }
+    };
+
+    class ProtoElements : public std::vector<ProtoElement>
+    {
+    public:
+        AccessRightsElementsWithOptions getResult() const
+        {
+            ProtoElements sorted = *this;
+            boost::range::sort(sorted);
+            AccessRightsElementsWithOptions res;
+            res.reserve(sorted.size());
+
+            for (size_t i = 0; i != sorted.size();)
+            {
+                size_t count_elements_with_diff_columns = sorted.countElementsWithDifferenceInColumnOnly(i);
+                if (count_elements_with_diff_columns == 1)
+                {
+                    /// Easy case: one Element is converted to one AccessRightsElement.
+                    const auto & element = sorted[i];
+                    if (element.access_flags)
+                        res.emplace_back(element.getResult());
+                    ++i;
+                }
+                else
+                {
+                    /// Difficult case: multiple Elements are converted to one or multiple AccessRightsElements.
+                    sorted.appendResultWithElementsWithDifferenceInColumnOnly(i, count_elements_with_diff_columns, res);
+                    i += count_elements_with_diff_columns;
+                }
+            }
+            return res;
+        }
+
+    private:
+        size_t countElementsWithDifferenceInColumnOnly(size_t start) const
+        {
+            const auto & start_element = (*this)[start];
+            if ((start_element.full_name.size() != 3) || (start == size() - 1))
+                return 1;
+
+            auto it = std::find_if(begin() + start + 1, end(), [&](const ProtoElement & element)
+            {
+                return (element.full_name.size() != 3) || (element.full_name[0] != start_element.full_name[0])
+                    || (element.full_name[1] != start_element.full_name[1]) || (element.grant_option != start_element.grant_option)
+                    || (element.kind != start_element.kind);
+            });
+
+            return it - (begin() + start);
+        }
+
+        /// Collects columns together to write multiple columns into one AccessRightsElement.
+        /// That procedure allows to output access rights in more compact way,
+        /// e.g. "SELECT(x, y)" instead of "SELECT(x), SELECT(y)".
+        void appendResultWithElementsWithDifferenceInColumnOnly(size_t start, size_t count, AccessRightsElementsWithOptions & res) const
+        {
+            const auto * pbegin = data() + start;
+            const auto * pend = pbegin + count;
+            AccessFlags handled_flags;
+
+            while (pbegin < pend)
+            {
+                while (pbegin < pend && !(pbegin->access_flags - handled_flags))
+                    ++pbegin;
+
+                while (pbegin < pend && !((pend - 1)->access_flags - handled_flags))
+                    --pend;
+
+                if (pbegin >= pend)
+                    break;
+
+                AccessFlags common_flags = (pbegin->access_flags - handled_flags);
+                for (const auto * element = pbegin + 1; element != pend; ++element)
+                {
+                    if (auto new_common_flags = (element->access_flags - handled_flags) & common_flags)
+                        common_flags = new_common_flags;
+                }
+
+                res.emplace_back();
+                auto & back = res.back();
+                back.grant_option = pbegin->grant_option;
+                back.kind = pbegin->kind;
+                back.any_database = false;
+                back.database = pbegin->full_name[0];
+                back.any_table = false;
+                back.table = pbegin->full_name[1];
+                back.any_column = false;
+                back.access_flags = common_flags;
+                for (const auto * element = pbegin; element != pend; ++element)
+                {
+                    if (((element->access_flags - handled_flags) & common_flags) == common_flags)
+                        back.columns.emplace_back(element->full_name[2]);
+                }
+
+                handled_flags |= common_flags;
+            }
+        }
+    };
 }
 
 
@@ -243,23 +426,43 @@ public:
 
     friend bool operator!=(const Node & left, const Node & right) { return !(left == right); }
 
-    void merge(const Node & other, const Helper & helper)
+    void makeUnion(const Node & other, const Helper & helper)
     {
-        mergeAccessRec(other);
+        makeUnionRec(other);
         calculateFinalAccessRec(helper);
     }
 
-    void logTree(Poco::Logger * log) const
+    void makeIntersection(const Node & other, const Helper & helper)
     {
-        LOG_TRACE(log, "Tree({}): name={}, access={}, final_access={}, min_access={}, max_access={}, num_children={}",
-            level, node_name ? *node_name : "NULL", access.toString(),
+        makeIntersectionRec(other);
+        calculateFinalAccessRec(helper);
+    }
+
+    ProtoElements getElements() const
+    {
+        ProtoElements res;
+        getElementsRec(res, {}, *this, {});
+        return res;
+    }
+
+    static ProtoElements getElements(const Node * node, const Node * node_with_grant_option)
+    {
+        ProtoElements res;
+        getElementsRec(res, {}, node, {}, node_with_grant_option, {});
+        return res;
+    }
+
+    void logTree(Poco::Logger * log, const String & title) const
+    {
+        LOG_TRACE(log, "Tree({}): level={}, name={}, access={}, final_access={}, min_access={}, max_access={}, num_children={}",
+            title, level, node_name ? *node_name : "NULL", access.toString(),
             final_access.toString(), min_access.toString(), max_access.toString(),
             (children ? children->size() : 0));
 
         if (children)
         {
             for (auto & child : *children | boost::adaptors::map_values)
-                child.logTree(log);
+                child.logTree(log, title);
         }
     }
 
@@ -342,6 +545,93 @@ private:
         }
     }
 
+    static void getElementsRec(
+        ProtoElements & res,
+        const boost::container::small_vector<std::string_view, 3> & full_name,
+        const Node & node,
+        const AccessFlags & parent_access)
+    {
+        auto access = node.access;
+        auto revokes = parent_access - access;
+        auto grants = access - parent_access;
+
+        if (revokes)
+            res.push_back(ProtoElement{revokes, full_name, false, Kind::REVOKE});
+
+        if (grants)
+            res.push_back(ProtoElement{grants, full_name, false, Kind::GRANT});
+
+        if (node.children)
+        {
+            for (const auto & [child_name, child] : *node.children)
+            {
+                boost::container::small_vector<std::string_view, 3> child_full_name = full_name;
+                child_full_name.push_back(child_name);
+                getElementsRec(res, child_full_name, child, access);
+            }
+        }
+    }
+
+    static void getElementsRec(
+        ProtoElements & res,
+        const boost::container::small_vector<std::string_view, 3> & full_name,
+        const Node * node,
+        const AccessFlags & parent_access,
+        const Node * node_go,
+        const AccessFlags & parent_access_go)
+    {
+        auto access = node ? node->access : parent_access;
+        auto access_go = node_go ? node_go->access : parent_access_go;
+        auto revokes = parent_access - access;
+        auto revokes_go = parent_access_go - access_go - revokes;
+        auto grants_go = access_go - parent_access_go;
+        auto grants = access - parent_access - grants_go;
+
+        if (revokes)
+            res.push_back(ProtoElement{revokes, full_name, false, Kind::REVOKE});
+
+        if (revokes_go)
+            res.push_back(ProtoElement{revokes_go, full_name, true, Kind::REVOKE});
+
+        if (grants)
+            res.push_back(ProtoElement{grants, full_name, false, Kind::GRANT});
+
+        if (grants_go)
+            res.push_back(ProtoElement{grants_go, full_name, true, Kind::GRANT});
+
+        if (node && node->children)
+        {
+            for (const auto & [child_name, child] : *node->children)
+            {
+                boost::container::small_vector<std::string_view, 3> child_full_name = full_name;
+                child_full_name.push_back(child_name);
+                const Node * child_node = &child;
+                const Node * child_node_go = nullptr;
+                if (node_go && node_go->children)
+                {
+                    auto it = node_go->children->find(child_name);
+                    if (it != node_go->children->end())
+                        child_node_go = &it->second;
+                }
+                getElementsRec(res, child_full_name, child_node, access, child_node_go, access_go);
+            }
+
+        }
+        if (node_go && node_go->children)
+        {
+            for (const auto & [child_name, child] : *node_go->children)
+            {
+                if (node && node->children && node->children->count(child_name))
+                    continue; /// already processed
+                boost::container::small_vector<std::string_view, 3> child_full_name = full_name;
+                child_full_name.push_back(child_name);
+                const Node * child_node = nullptr;
+                const Node * child_node_go = &child;
+                getElementsRec(res, child_full_name, child_node, access, child_node_go, access_go);
+            }
+        }
+    }
+
     void calculateFinalAccessRec(const Helper & helper)
     {
         /// Traverse tree.
@@ -438,12 +728,12 @@ private:
         max_access = final_access | max_access_among_children;
     }
 
-    void mergeAccessRec(const Node & rhs)
+    void makeUnionRec(const Node & rhs)
     {
         if (rhs.children)
         {
             for (const auto & [rhs_childname, rhs_child] : *rhs.children)
-                getChild(rhs_childname).mergeAccessRec(rhs_child);
+                getChild(rhs_childname).makeUnionRec(rhs_child);
         }
         access |= rhs.access;
         if (children)
@@ -455,6 +745,24 @@ private:
             }
         }
     }
+
+    void makeIntersectionRec(const Node & rhs)
+    {
+        if (rhs.children)
+        {
+            for (const auto & [rhs_childname, rhs_child] : *rhs.children)
+                getChild(rhs_childname).makeIntersectionRec(rhs_child);
+        }
+        access &= rhs.access;
+        if (children)
+        {
+            for (auto & [lhs_childname, lhs_child] : *children)
+            {
+                if (!rhs.tryGetChild(lhs_childname))
+                    lhs_child.access &= rhs.access;
+            }
+        }
+    }
 };
 
 
@@ -476,6 +784,10 @@ AccessRights & AccessRights::operator =(const AccessRights & src)
         root = std::make_unique<Node>(*src.root);
     else
         root = nullptr;
+    if (src.root_with_grant_option)
+        root_with_grant_option = std::make_unique<Node>(*src.root_with_grant_option);
+    else
+        root_with_grant_option = nullptr;
     return *this;
 }
 
@@ -488,302 +800,267 @@ AccessRights::AccessRights(const AccessFlags & access)
 
 bool AccessRights::isEmpty() const
 {
-    return !root;
+    return !root && !root_with_grant_option;
 }
 
 
 void AccessRights::clear()
 {
     root = nullptr;
+    root_with_grant_option = nullptr;
 }
 
 
-template <typename... Args>
+template <bool with_grant_option, typename... Args>
 void AccessRights::grantImpl(const AccessFlags & flags, const Args &... args)
 {
-    if (!root)
-        root = std::make_unique<Node>();
-    root->grant(flags, Helper::instance(), args...);
-    if (!root->access && !root->children)
-        root = nullptr;
+    auto helper = [&](std::unique_ptr<Node> & root_node)
+    {
+        if (!root_node)
+            root_node = std::make_unique<Node>();
+        root_node->grant(flags, Helper::instance(), args...);
+        if (!root_node->access && !root_node->children)
+            root_node = nullptr;
+    };
+    helper(root);
+
+    if constexpr (with_grant_option)
+        helper(root_with_grant_option);
 }
 
-void AccessRights::grant(const AccessFlags & flags) { grantImpl(flags); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl(flags, database); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl(flags, database, table); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl(flags, database, table, column); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl(flags, database, table, columns); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl(flags, database, table, columns); }
-
-void AccessRights::grant(const AccessRightsElement & element, std::string_view current_database)
+template <bool with_grant_option>
+void AccessRights::grantImpl(const AccessRightsElement & element)
 {
     if (element.any_database)
-    {
-        grant(element.access_flags);
-    }
+        grantImpl<with_grant_option>(element.access_flags);
     else if (element.any_table)
-    {
-        if (element.database.empty())
-            grant(element.access_flags, checkCurrentDatabase(current_database));
-        else
-            grant(element.access_flags, element.database);
-    }
+        grantImpl<with_grant_option>(element.access_flags, element.database);
     else if (element.any_column)
-    {
-        if (element.database.empty())
-            grant(element.access_flags, checkCurrentDatabase(current_database), element.table);
-        else
-            grant(element.access_flags, element.database, element.table);
-    }
+        grantImpl<with_grant_option>(element.access_flags, element.database, element.table);
     else
-    {
-        if (element.database.empty())
-            grant(element.access_flags, checkCurrentDatabase(current_database), element.table, element.columns);
-        else
-            grant(element.access_flags, element.database, element.table, element.columns);
-    }
+        grantImpl<with_grant_option>(element.access_flags, element.database, element.table, element.columns);
 }
 
-void AccessRights::grant(const AccessRightsElements & elements, std::string_view current_database)
+template <bool with_grant_option>
+void AccessRights::grantImpl(const AccessRightsElements & elements)
 {
     for (const auto & element : elements)
-        grant(element, current_database);
+        grantImpl<with_grant_option>(element);
 }
 
+void AccessRights::grant(const AccessFlags & flags) { grantImpl<false>(flags); }
+void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl<false>(flags, database); }
+void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<false>(flags, database, table); }
+void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<false>(flags, database, table, column); }
+void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
+void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
+void AccessRights::grant(const AccessRightsElement & element) { grantImpl<false>(element); }
+void AccessRights::grant(const AccessRightsElements & elements) { grantImpl<false>(elements); }
 
-template <typename... Args>
+void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl<true>(flags); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl<true>(flags, database); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<true>(flags, database, table); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<true>(flags, database, table, column); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
+void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl<true>(element); }
+void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl<true>(elements); }
+
+
+template <bool grant_option, typename... Args>
 void AccessRights::revokeImpl(const AccessFlags & flags, const Args &... args)
 {
-    if (!root)
-        return;
-    root->revoke(flags, Helper::instance(), args...);
-    if (!root->access && !root->children)
-        root = nullptr;
+    auto helper = [&](std::unique_ptr<Node> & root_node)
+    {
+        if (!root_node)
+            return;
+        root_node->revoke(flags, Helper::instance(), args...);
+        if (!root_node->access && !root_node->children)
+            root_node = nullptr;
+    };
+    helper(root_with_grant_option);
+
+    if constexpr (!grant_option)
+        helper(root);
 }
 
-void AccessRights::revoke(const AccessFlags & flags) { revokeImpl(flags); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl(flags, database); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl(flags, database, table); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl(flags, database, table, column); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl(flags, database, table, columns); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl(flags, database, table, columns); }
-
-
-void AccessRights::revoke(const AccessRightsElement & element, std::string_view current_database)
+template <bool grant_option>
+void AccessRights::revokeImpl(const AccessRightsElement & element)
 {
     if (element.any_database)
-    {
-        revoke(element.access_flags);
-    }
+        revokeImpl<grant_option>(element.access_flags);
     else if (element.any_table)
-    {
-        if (element.database.empty())
-            revoke(element.access_flags, checkCurrentDatabase(current_database));
-        else
-            revoke(element.access_flags, element.database);
-    }
+        revokeImpl<grant_option>(element.access_flags, element.database);
     else if (element.any_column)
-    {
-        if (element.database.empty())
-            revoke(element.access_flags, checkCurrentDatabase(current_database), element.table);
-        else
-            revoke(element.access_flags, element.database, element.table);
-    }
+        revokeImpl<grant_option>(element.access_flags, element.database, element.table);
     else
-    {
-        if (element.database.empty())
-            revoke(element.access_flags, checkCurrentDatabase(current_database), element.table, element.columns);
-        else
-            revoke(element.access_flags, element.database, element.table, element.columns);
-    }
+        revokeImpl<grant_option>(element.access_flags, element.database, element.table, element.columns);
 }
 
-void AccessRights::revoke(const AccessRightsElements & elements, std::string_view current_database)
+template <bool grant_option>
+void AccessRights::revokeImpl(const AccessRightsElements & elements)
 {
     for (const auto & element : elements)
-        revoke(element, current_database);
+        revokeImpl<grant_option>(element);
 }
 
+void AccessRights::revoke(const AccessFlags & flags) { revokeImpl<false>(flags); }
+void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl<false>(flags, database); }
+void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<false>(flags, database, table); }
+void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<false>(flags, database, table, column); }
+void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
+void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
+void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl<false>(element); }
+void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl<false>(elements); }
 
-AccessRightsElements AccessRights::getGrants() const
-{
-    AccessRightsElements grants;
-    getGrantsAndPartialRevokesImpl(&grants, nullptr);
-    return grants;
-}
-
-AccessRightsElements AccessRights::getPartialRevokes() const
-{
-    AccessRightsElements partial_revokes;
-    getGrantsAndPartialRevokesImpl(nullptr, &partial_revokes);
-    return partial_revokes;
-}
-
-AccessRights::GrantsAndPartialRevokes AccessRights::getGrantsAndPartialRevokes() const
-{
-    GrantsAndPartialRevokes res;
-    getGrantsAndPartialRevokesImpl(&res.grants, &res.revokes);
-    return res;
-}
-
-
-void AccessRights::getGrantsAndPartialRevokesImpl(AccessRightsElements * out_grants, AccessRightsElements * out_partial_revokes) const
+void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl<true>(flags); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl<true>(flags, database); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<true>(flags, database, table); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<true>(flags, database, table, column); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
+void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl<true>(element); }
+void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl<true>(elements); }
+
+
+AccessRightsElementsWithOptions AccessRights::getElements() const
 {
+#if 0
+    logTree();
+#endif
     if (!root)
-        return;
-    auto global_access = root->access;
-    if (out_grants && global_access)
-        out_grants->push_back({global_access});
-    if (root->children)
-    {
-        for (const auto & [db_name, db_node] : *root->children)
-        {
-            if (out_grants)
-            {
-                if (auto db_grants = db_node.access - global_access)
-                    out_grants->push_back({db_grants, db_name});
-            }
-            if (out_partial_revokes)
-            {
-                if (auto db_partial_revokes = global_access - db_node.access)
-                    out_partial_revokes->push_back({db_partial_revokes, db_name});
-            }
-            if (db_node.children)
-            {
-                for (const auto & [table_name, table_node] : *db_node.children)
-                {
-                    if (out_grants)
-                    {
-                        if (auto table_grants = table_node.access - db_node.access)
-                            out_grants->push_back({table_grants, db_name, table_name});
-                    }
-                    if (out_partial_revokes)
-                    {
-                        if (auto table_partial_revokes = db_node.access - table_node.access)
-                            out_partial_revokes->push_back({table_partial_revokes, db_name, table_name});
-                    }
-                    if (table_node.children)
-                    {
-                        for (const auto & [column_name, column_node] : *table_node.children)
-                        {
-                            if (out_grants)
-                            {
-                                if (auto column_grants = column_node.access - table_node.access)
-                                    out_grants->push_back({column_grants, db_name, table_name, column_name});
-                            }
-                            if (out_partial_revokes)
-                            {
-                                if (auto column_partial_revokes = table_node.access - column_node.access)
-                                    out_partial_revokes->push_back({column_partial_revokes, db_name, table_name, column_name});
-                            }
-                        }
-
-                    }
-                }
-            }
-        }
-    }
+        return {};
+    if (!root_with_grant_option)
+        return root->getElements().getResult();
+    return Node::getElements(root.get(), root_with_grant_option.get()).getResult();
 }
 
 
 String AccessRights::toString() const
 {
-    String res;
-    auto gr = getGrantsAndPartialRevokes();
-    if (!gr.grants.empty())
-    {
-        res += "GRANT ";
-        res += gr.grants.toString();
-    }
-    if (!gr.revokes.empty())
-    {
-        if (!res.empty())
-            res += ", ";
-        res += "REVOKE ";
-        res += gr.revokes.toString();
-    }
-    if (res.empty())
-        res = "GRANT USAGE ON *.*";
-    return res;
+    return getElements().toString();
 }
 
 
-template <typename... Args>
+template <bool grant_option, typename... Args>
 bool AccessRights::isGrantedImpl(const AccessFlags & flags, const Args &... args) const
 {
-    if (!root)
-        return flags.isEmpty();
-    return root->isGranted(flags, args...);
+    auto helper = [&](const std::unique_ptr<Node> & root_node) -> bool
+    {
+        if (!root_node)
+            return flags.isEmpty();
+        return root_node->isGranted(flags, args...);
+    };
+    if constexpr (grant_option)
+        return helper(root_with_grant_option);
+    else
+        return helper(root);
 }
 
-bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl(flags); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl(flags, database); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(flags, database, table); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(flags, database, table, column); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl(flags, database, table, columns); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); }
-
-bool AccessRights::isGranted(const AccessRightsElement & element, std::string_view current_database) const
+template <bool grant_option>
+bool AccessRights::isGrantedImpl(const AccessRightsElement & element) const
 {
     if (element.any_database)
-    {
-        return isGranted(element.access_flags);
-    }
+        return isGrantedImpl<grant_option>(element.access_flags);
     else if (element.any_table)
-    {
-        if (element.database.empty())
-            return isGranted(element.access_flags, checkCurrentDatabase(current_database));
-        else
-            return isGranted(element.access_flags, element.database);
-    }
+        return isGrantedImpl<grant_option>(element.access_flags, element.database);
     else if (element.any_column)
-    {
-        if (element.database.empty())
-            return isGranted(element.access_flags, checkCurrentDatabase(current_database), element.table);
-        else
-            return isGranted(element.access_flags, element.database, element.table);
-    }
+        return isGrantedImpl<grant_option>(element.access_flags, element.database, element.table);
     else
-    {
-        if (element.database.empty())
-            return isGranted(element.access_flags, checkCurrentDatabase(current_database), element.table, element.columns);
-        else
-            return isGranted(element.access_flags, element.database, element.table, element.columns);
-    }
+        return isGrantedImpl<grant_option>(element.access_flags, element.database, element.table, element.columns);
 }
 
-bool AccessRights::isGranted(const AccessRightsElements & elements, std::string_view current_database) const
+template <bool grant_option>
+bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const
 {
     for (const auto & element : elements)
-        if (!isGranted(element, current_database))
+        if (!isGrantedImpl<grant_option>(element))
             return false;
     return true;
 }
 
+bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl<false>(flags); }
+bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<false>(flags, database); }
+bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<false>(flags, database, table); }
+bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<false>(flags, database, table, column); }
+bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl<false>(element); }
+bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl<false>(elements); }
+
+bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl<true>(flags); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<true>(flags, database); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<true>(flags, database, table); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<true>(flags, database, table, column); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
+bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }
+
 
 bool operator ==(const AccessRights & left, const AccessRights & right)
 {
-    if (!left.root)
-        return !right.root;
-    if (!right.root)
-        return false;
-    return *left.root == *right.root;
+    auto helper = [](const std::unique_ptr<AccessRights::Node> & left_node, const std::unique_ptr<AccessRights::Node> & right_node)
+    {
+        if (!left_node)
+            return !right_node;
+        if (!right_node)
+            return false;
+        return *left_node == *right_node;
+    };
+    return helper(left.root, right.root) && helper(left.root_with_grant_option, right.root_with_grant_option);
 }
 
 
-void AccessRights::merge(const AccessRights & other)
+void AccessRights::makeUnion(const AccessRights & other)
 {
-    if (!root)
+    auto helper = [](std::unique_ptr<Node> & root_node, const std::unique_ptr<Node> & other_root_node)
     {
-        *this = other;
-        return;
-    }
-    if (other.root)
+        if (!root_node)
+        {
+            if (other_root_node)
+                root_node = std::make_unique<Node>(*other_root_node);
+            return;
+        }
+        if (other_root_node)
+        {
+            root_node->makeUnion(*other_root_node, Helper::instance());
+            if (!root_node->access && !root_node->children)
+                root_node = nullptr;
+        }
+    };
+    helper(root, other.root);
+    helper(root_with_grant_option, other.root_with_grant_option);
+}
+
+
+void AccessRights::makeIntersection(const AccessRights & other)
+{
+    auto helper = [](std::unique_ptr<Node> & root_node, const std::unique_ptr<Node> & other_root_node)
     {
-        root->merge(*other.root, Helper::instance());
-        if (!root->access && !root->children)
-            root = nullptr;
-    }
+        if (!root_node)
+        {
+            if (other_root_node)
+                root_node = std::make_unique<Node>(*other_root_node);
+            return;
+        }
+        if (other_root_node)
+        {
+            root_node->makeIntersection(*other_root_node, Helper::instance());
+            if (!root_node->access && !root_node->children)
+                root_node = nullptr;
+        }
+    };
+    helper(root, other.root);
+    helper(root_with_grant_option, other.root_with_grant_option);
+}
+
+
+AccessRights AccessRights::getFullAccess()
+{
+    AccessRights res;
+    res.grantWithGrantOption(AccessType::ALL);
+    return res;
 }
 
 
@@ -791,7 +1068,11 @@ void AccessRights::logTree() const
 {
     auto * log = &Poco::Logger::get("AccessRights");
     if (root)
-        root->logTree(log);
+    {
+        root->logTree(log, "");
+        if (root_with_grant_option)
+            root->logTree(log, "go");
+    }
     else
         LOG_TRACE(log, "Tree: NULL");
 }

src/Access/AccessRights.h

@@ -26,6 +26,12 @@ public:
     /// Revokes everything. It's the same as revoke(AccessType::ALL).
     void clear();
 
+    /// Returns the information about all the access granted as a string.
+    String toString() const;
+
+    /// Returns the information about all the access granted.
+    AccessRightsElementsWithOptions getElements() const;
+
     /// Grants access on a specified database/table/column.
     /// Does nothing if the specified access has been already granted.
     void grant(const AccessFlags & flags);
@@ -34,8 +40,17 @@ public:
     void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
     void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
     void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
-    void grant(const AccessRightsElement & element, std::string_view current_database = {});
-    void grant(const AccessRightsElements & elements, std::string_view current_database = {});
+    void grant(const AccessRightsElement & element);
+    void grant(const AccessRightsElements & elements);
+
+    void grantWithGrantOption(const AccessFlags & flags);
+    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database);
+    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
+    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
+    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
+    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void grantWithGrantOption(const AccessRightsElement & element);
+    void grantWithGrantOption(const AccessRightsElements & elements);
 
     /// Revokes a specified access granted earlier on a specified database/table/column.
     /// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear();
@@ -45,21 +60,17 @@ public:
     void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
     void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
     void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
-    void revoke(const AccessRightsElement & element, std::string_view current_database = {});
-    void revoke(const AccessRightsElements & elements, std::string_view current_database = {});
+    void revoke(const AccessRightsElement & element);
+    void revoke(const AccessRightsElements & elements);
 
-    /// Returns the information about all the access granted.
-    struct GrantsAndPartialRevokes
-    {
-        AccessRightsElements grants;
-        AccessRightsElements revokes;
-    };
-    AccessRightsElements getGrants() const;
-    AccessRightsElements getPartialRevokes() const;
-    GrantsAndPartialRevokes getGrantsAndPartialRevokes() const;
-
-    /// Returns the information about all the access granted as a string.
-    String toString() const;
+    void revokeGrantOption(const AccessFlags & flags);
+    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database);
+    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
+    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
+    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
+    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void revokeGrantOption(const AccessRightsElement & element);
+    void revokeGrantOption(const AccessRightsElements & elements);
 
     /// Whether a specified access granted.
     bool isGranted(const AccessFlags & flags) const;
@@ -68,38 +79,62 @@ public:
     bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
     bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
     bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
-    bool isGranted(const AccessRightsElement & element, std::string_view current_database = {}) const;
-    bool isGranted(const AccessRightsElements & elements, std::string_view current_database = {}) const;
+    bool isGranted(const AccessRightsElement & element) const;
+    bool isGranted(const AccessRightsElements & elements) const;
+
+    bool hasGrantOption(const AccessFlags & flags) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool hasGrantOption(const AccessRightsElement & element) const;
+    bool hasGrantOption(const AccessRightsElements & elements) const;
+
+    /// Merges two sets of access rights together.
+    /// It's used to combine access rights from multiple roles.
+    void makeUnion(const AccessRights & other);
+
+    void makeIntersection(const AccessRights & other);
 
     friend bool operator ==(const AccessRights & left, const AccessRights & right);
     friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); }
 
-    /// Merges two sets of access rights together.
-    /// It's used to combine access rights from multiple roles.
-    void merge(const AccessRights & other);
+    static AccessRights getFullAccess();
 
 private:
-    template <typename... Args>
+    template <bool with_grant_option, typename... Args>
     void grantImpl(const AccessFlags & flags, const Args &... args);
 
-    template <typename... Args>
+    template <bool with_grant_options>
+    void grantImpl(const AccessRightsElement & element);
+
+    template <bool with_grant_options>
+    void grantImpl(const AccessRightsElements & elements);
+
+    template <bool grant_option, typename... Args>
     void revokeImpl(const AccessFlags & flags, const Args &... args);
 
-    template <typename... Args>
+    template <bool grant_option>
+    void revokeImpl(const AccessRightsElement & element);
+
+    template <bool grant_option>
+    void revokeImpl(const AccessRightsElements & elements);
+
+    template <bool grant_option, typename... Args>
     bool isGrantedImpl(const AccessFlags & flags, const Args &... args) const;
 
-    bool isGrantedImpl(const AccessRightsElement & element, std::string_view current_database) const;
-    bool isGrantedImpl(const AccessRightsElements & elements, std::string_view current_database) const;
+    template <bool grant_option>
+    bool isGrantedImpl(const AccessRightsElement & element) const;
 
-    template <typename... Args>
-    AccessFlags getAccessImpl(const Args &... args) const;
-
-    void getGrantsAndPartialRevokesImpl(AccessRightsElements * grants, AccessRightsElements * partial_revokes) const;
+    template <bool grant_option>
+    bool isGrantedImpl(const AccessRightsElements & elements) const;
 
     void logTree() const;
 
     struct Node;
     std::unique_ptr<Node> root;
+    std::unique_ptr<Node> root_with_grant_option;
 };
 
 }

src/Access/AccessRightsElement.cpp

@@ -12,222 +12,158 @@ namespace DB
 {
 namespace
 {
-    size_t groupElements(AccessRightsElements & elements, size_t start)
+    using Kind = AccessRightsElementWithOptions::Kind;
+
+    String formatOptions(bool grant_option, Kind kind, const String & inner_part)
     {
-        auto & start_element = elements[start];
-        auto it = std::find_if(elements.begin() + start + 1, elements.end(),
-                               [&](const AccessRightsElement & element)
+        if (kind == Kind::REVOKE)
         {
-            return (element.database != start_element.database) ||
-                   (element.any_database != start_element.any_database) ||
-                   (element.table != start_element.table) ||
-                   (element.any_table != start_element.any_table) ||
-                   (element.any_column != start_element.any_column);
-        });
-        size_t end = it - elements.begin();
-
-        /// All the elements at indices from start to end here specify
-        /// the same database and table.
-
-        if (start_element.any_column)
-        {
-            /// Easy case: the elements don't specify columns.
-            /// All we need is to combine the access flags.
-            for (size_t i = start + 1; i != end; ++i)
-            {
-                start_element.access_flags |= elements[i].access_flags;
-                elements[i].access_flags = {};
-            }
-            return end;
+            if (grant_option)
+                return "REVOKE GRANT OPTION " + inner_part;
+            else
+                return "REVOKE " + inner_part;
         }
-
-        /// Difficult case: the elements specify columns.
-        /// We have to find groups of columns with common access flags.
-        for (size_t i = start; i != end; ++i)
+        else
         {
-            if (!elements[i].access_flags)
-                continue;
-
-            AccessFlags common_flags = elements[i].access_flags;
-            size_t num_elements_with_common_flags = 1;
-            for (size_t j = i + 1; j != end; ++j)
-            {
-                auto new_common_flags = common_flags & elements[j].access_flags;
-                if (new_common_flags)
-                {
-                    common_flags = new_common_flags;
-                    ++num_elements_with_common_flags;
-                }
-            }
-
-            if (num_elements_with_common_flags == 1)
-                continue;
-
-            if (elements[i].access_flags != common_flags)
-            {
-                elements.insert(elements.begin() + i + 1, elements[i]);
-                elements[i].access_flags = common_flags;
-                elements[i].columns.clear();
-                ++end;
-            }
-
-            for (size_t j = i + 1; j != end; ++j)
-            {
-                if ((elements[j].access_flags & common_flags) == common_flags)
-                {
-                    boost::range::push_back(elements[i].columns, elements[j].columns);
-                    elements[j].access_flags -= common_flags;
-                }
-            }
+            if (grant_option)
+                return "GRANT " + inner_part + " WITH GRANT OPTION";
+            else
+                return "GRANT " + inner_part;
         }
-
-        return end;
     }
 
-    /// Tries to combine elements to decrease their number.
-    void groupElements(AccessRightsElements & elements)
+
+    String formatONClause(const String & database, bool any_database, const String & table, bool any_table)
     {
-        if (!boost::range::is_sorted(elements))
-            boost::range::sort(elements); /// Algorithm in groupElement() requires elements to be sorted.
-        for (size_t start = 0; start != elements.size();)
-            start = groupElements(elements, start);
+        String msg = "ON ";
+
+        if (any_database)
+            msg += "*.";
+        else if (!database.empty())
+            msg += backQuoteIfNeed(database) + ".";
+
+        if (any_table)
+            msg += "*";
+        else
+            msg += backQuoteIfNeed(table);
+        return msg;
     }
 
-    /// Removes unnecessary elements, sorts elements and makes them unique.
-    void sortElementsAndMakeUnique(AccessRightsElements & elements)
+
+    String formatAccessFlagsWithColumns(const AccessFlags & access_flags, const Strings & columns, bool any_column)
     {
-        /// Remove empty elements.
-        boost::range::remove_erase_if(elements, [](const AccessRightsElement & element)
+        String columns_in_parentheses;
+        if (!any_column)
         {
-            return !element.access_flags || (!element.any_column && element.columns.empty());
-        });
-
-        /// Sort columns and make them unique.
-        for (auto & element : elements)
-        {
-            if (element.any_column)
-                continue;
-
-            if (!boost::range::is_sorted(element.columns))
-                boost::range::sort(element.columns);
-            element.columns.erase(std::unique(element.columns.begin(), element.columns.end()), element.columns.end());
+            if (columns.empty())
+                return "USAGE";
+            for (const auto & column : columns)
+            {
+                columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", ";
+                columns_in_parentheses += backQuoteIfNeed(column);
+            }
+            columns_in_parentheses += ")";
         }
 
-        /// Sort elements themselves.
-        boost::range::sort(elements);
-        elements.erase(std::unique(elements.begin(), elements.end()), elements.end());
+        auto keywords = access_flags.toKeywords();
+        if (keywords.empty())
+            return "USAGE";
+
+        String msg;
+        for (const std::string_view & keyword : keywords)
+        {
+            if (!msg.empty())
+                msg += ", ";
+            msg += String{keyword} + columns_in_parentheses;
+        }
+        return msg;
     }
 }
 
-void AccessRightsElement::setDatabase(const String & new_database)
-{
-    database = new_database;
-    any_database = false;
-}
-
-
-void AccessRightsElement::replaceEmptyDatabase(const String & new_database)
-{
-    if (isEmptyDatabase())
-        setDatabase(new_database);
-}
-
-
-bool AccessRightsElement::isEmptyDatabase() const
-{
-    return !any_database && database.empty();
-}
-
 
 String AccessRightsElement::toString() const
 {
-    String msg = toStringWithoutON();
-    msg += " ON ";
-
-    if (any_database)
-        msg += "*.";
-    else if (!database.empty())
-        msg += backQuoteIfNeed(database) + ".";
-
-    if (any_table)
-        msg += "*";
-    else
-        msg += backQuoteIfNeed(table);
-    return msg;
+    return formatAccessFlagsWithColumns(access_flags, columns, any_column) + " " + formatONClause(database, any_database, table, any_table);
 }
 
-String AccessRightsElement::toStringWithoutON() const
+String AccessRightsElementWithOptions::toString() const
 {
-    String columns_in_parentheses;
-    if (!any_column)
-    {
-        if (columns.empty())
-            return "USAGE";
-        for (const auto & column : columns)
-        {
-            columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", ";
-            columns_in_parentheses += backQuoteIfNeed(column);
-        }
-        columns_in_parentheses += ")";
-    }
-
-    auto keywords = access_flags.toKeywords();
-    if (keywords.empty())
-        return "USAGE";
-
-    String msg;
-    for (const std::string_view & keyword : keywords)
-    {
-        if (!msg.empty())
-            msg += ", ";
-        msg += String{keyword} + columns_in_parentheses;
-    }
-    return msg;
+    return formatOptions(grant_option, kind, AccessRightsElement::toString());
 }
 
-
-void AccessRightsElements::replaceEmptyDatabase(const String & new_database)
+String AccessRightsElements::toString() const
 {
-    for (auto & element : *this)
-        element.replaceEmptyDatabase(new_database);
-}
-
-
-String AccessRightsElements::toString()
-{
-    normalize();
-
     if (empty())
         return "USAGE ON *.*";
 
-    String msg;
-    bool need_comma = false;
+    String res;
+    String inner_part;
+
     for (size_t i = 0; i != size(); ++i)
     {
         const auto & element = (*this)[i];
-        if (std::exchange(need_comma, true))
-            msg += ", ";
-        bool next_element_on_same_db_and_table = false;
+
+        if (!inner_part.empty())
+            inner_part += ", ";
+        inner_part += formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column);
+
+        bool next_element_uses_same_table = false;
         if (i != size() - 1)
         {
             const auto & next_element = (*this)[i + 1];
-            if ((element.database == next_element.database) && (element.any_database == next_element.any_database)
-                && (element.table == next_element.table) && (element.any_table == next_element.any_table))
-                next_element_on_same_db_and_table = true;
+            if (element.sameDatabaseAndTable(next_element))
+                next_element_uses_same_table = true;
+        }
+
+        if (!next_element_uses_same_table)
+        {
+            if (!res.empty())
+                res += ", ";
+            res += inner_part + " " + formatONClause(element.database, element.any_database, element.table, element.any_table);
+            inner_part.clear();
         }
-        if (next_element_on_same_db_and_table)
-            msg += element.toStringWithoutON();
-        else
-            msg += element.toString();
     }
-    return msg;
+
+    return res;
 }
 
-
-void AccessRightsElements::normalize()
+String AccessRightsElementsWithOptions::toString() const
 {
-    groupElements(*this);
-    sortElementsAndMakeUnique(*this);
+    if (empty())
+        return "GRANT USAGE ON *.*";
+
+    String res;
+    String inner_part;
+
+    for (size_t i = 0; i != size(); ++i)
+    {
+        const auto & element = (*this)[i];
+
+        if (!inner_part.empty())
+            inner_part += ", ";
+        inner_part += formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column);
+
+        bool next_element_uses_same_mode_and_table = false;
+        if (i != size() - 1)
+        {
+            const auto & next_element = (*this)[i + 1];
+            if (element.sameDatabaseAndTable(next_element) && element.sameOptions(next_element))
+                next_element_uses_same_mode_and_table = true;
+        }
+
+        if (!next_element_uses_same_mode_and_table)
+        {
+            if (!res.empty())
+                res += ", ";
+            res += formatOptions(
+                element.grant_option,
+                element.kind,
+                inner_part + " " + formatONClause(element.database, element.any_database, element.table, element.any_table));
+            inner_part.clear();
+        }
+    }
+
+    return res;
 }
 
 }

src/Access/AccessRightsElement.h

@@ -71,26 +71,48 @@ struct AccessRightsElement
     {
     }
 
-    auto toTuple() const { return std::tie(access_flags, database, any_database, table, any_table, columns, any_column); }
+    auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns); }
     friend bool operator==(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() == right.toTuple(); }
-    friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() != right.toTuple(); }
-    friend bool operator<(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() < right.toTuple(); }
-    friend bool operator>(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() > right.toTuple(); }
-    friend bool operator<=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() <= right.toTuple(); }
-    friend bool operator>=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() >= right.toTuple(); }
+    friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return !(left == right); }
 
-    /// Sets the database.
-    void setDatabase(const String & new_database);
+    bool sameDatabaseAndTable(const AccessRightsElement & other) const
+    {
+        return (database == other.database) && (any_database == other.any_database) && (table == other.table)
+            && (any_table == other.any_table);
+    }
+
+    bool isEmptyDatabase() const { return !any_database && database.empty(); }
 
     /// If the database is empty, replaces it with `new_database`. Otherwise does nothing.
     void replaceEmptyDatabase(const String & new_database);
 
-    bool isEmptyDatabase() const;
-
     /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table".
-    /// The returned string isn't prefixed with the "GRANT" keyword.
     String toString() const;
-    String toStringWithoutON() const;
+};
+
+
+struct AccessRightsElementWithOptions : public AccessRightsElement
+{
+    bool grant_option = false;
+
+    enum class Kind
+    {
+        GRANT,
+        REVOKE,
+    };
+    Kind kind = Kind::GRANT;
+
+    bool sameOptions(const AccessRightsElementWithOptions & other) const
+    {
+        return (grant_option == other.grant_option) && (kind == other.kind);
+    }
+
+    auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns, grant_option, kind); }
+    friend bool operator==(const AccessRightsElementWithOptions & left, const AccessRightsElementWithOptions & right) { return left.toTuple() == right.toTuple(); }
+    friend bool operator!=(const AccessRightsElementWithOptions & left, const AccessRightsElementWithOptions & right) { return !(left == right); }
+
+    /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table".
+    String toString() const;
 };
 
 
@@ -101,13 +123,38 @@ public:
     /// Replaces the empty database with `new_database`.
     void replaceEmptyDatabase(const String & new_database);
 
-    /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table".
-    /// The returned string isn't prefixed with the "GRANT" keyword.
-    String toString() const { return AccessRightsElements(*this).toString(); }
-    String toString();
-
-    /// Reorder and group elements to show them in more readable form.
-    void normalize();
+    /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table".
+    String toString() const;
 };
 
+
+class AccessRightsElementsWithOptions : public std::vector<AccessRightsElementWithOptions>
+{
+public:
+    /// Replaces the empty database with `new_database`.
+    void replaceEmptyDatabase(const String & new_database);
+
+    /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table".
+    String toString() const;
+};
+
+
+inline void AccessRightsElement::replaceEmptyDatabase(const String & new_database)
+{
+    if (isEmptyDatabase())
+        database = new_database;
+}
+
+inline void AccessRightsElements::replaceEmptyDatabase(const String & new_database)
+{
+    for (auto & element : *this)
+        element.replaceEmptyDatabase(new_database);
+}
+
+inline void AccessRightsElementsWithOptions::replaceEmptyDatabase(const String & new_database)
+{
+    for (auto & element : *this)
+        element.replaceEmptyDatabase(new_database);
+}
+
 }

src/Access/ContextAccess.cpp

@@ -15,8 +15,6 @@
 #include <Poco/Logger.h>
 #include <common/logger_useful.h>
 #include <boost/algorithm/string/join.hpp>
-#include <boost/smart_ptr/make_shared_object.hpp>
-#include <boost/range/algorithm/fill.hpp>
 #include <boost/range/algorithm/set_algorithm.hpp>
 #include <assert.h>
 
@@ -35,61 +33,66 @@ namespace ErrorCodes
 
 namespace
 {
-    enum CheckAccessRightsMode
+    std::shared_ptr<AccessRights> mixAccessRightsFromUserAndRoles(const User & user, const EnabledRolesInfo & roles_info)
     {
-        RETURN_FALSE_IF_ACCESS_DENIED,
-        LOG_WARNING_IF_ACCESS_DENIED,
-        THROW_IF_ACCESS_DENIED,
-    };
-
-
-    String formatSkippedMessage()
-    {
-        return "";
+        auto res = std::make_shared<AccessRights>(user.access);
+        res->makeUnion(roles_info.access);
+        return res;
     }
 
-    String formatSkippedMessage(const std::string_view & database)
+    std::shared_ptr<AccessRights> applyParamsToAccessRights(const AccessRights & access, const ContextAccessParams & params)
     {
-        return ". Skipped database " + backQuoteIfNeed(database);
-    }
+        auto res = std::make_shared<AccessRights>(access);
 
-    String formatSkippedMessage(const std::string_view & database, const std::string_view & table)
-    {
-        String str = ". Skipped table ";
-        if (!database.empty())
-            str += backQuoteIfNeed(database) + ".";
-        str += backQuoteIfNeed(table);
-        return str;
-    }
+        static const AccessFlags table_ddl = AccessType::CREATE_DATABASE | AccessType::CREATE_TABLE | AccessType::CREATE_VIEW
+            | AccessType::ALTER_TABLE | AccessType::ALTER_VIEW | AccessType::DROP_DATABASE | AccessType::DROP_TABLE | AccessType::DROP_VIEW
+            | AccessType::TRUNCATE;
 
-    String formatSkippedMessage(const std::string_view & database, const std::string_view & table, const std::string_view & column)
-    {
-        String str = ". Skipped column " + backQuoteIfNeed(column) + " ON ";
-        if (!database.empty())
-            str += backQuoteIfNeed(database) + ".";
-        str += backQuoteIfNeed(table);
-        return str;
-    }
+        static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY;
+        static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
+        static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
+        static const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS;
 
-    template <typename StringT>
-    String formatSkippedMessage(const std::string_view & database, const std::string_view & table, const std::vector<StringT> & columns)
-    {
-        if (columns.size() == 1)
-            return formatSkippedMessage(database, table, columns[0]);
+        if (params.readonly)
+            res->revoke(write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY);
 
-        String str = ". Skipped columns ";
-        bool need_comma = false;
-        for (const auto & column : columns)
+        if (params.readonly == 1)
         {
-            if (std::exchange(need_comma, true))
-                str += ", ";
-            str += backQuoteIfNeed(column);
+            /// Table functions are forbidden in readonly mode.
+            /// For example, for readonly = 2 - allowed.
+            res->revoke(AccessType::CREATE_TEMPORARY_TABLE);
         }
-        str += " ON ";
-        if (!database.empty())
-            str += backQuoteIfNeed(database) + ".";
-        str += backQuoteIfNeed(table);
-        return str;
+
+        if (!params.allow_ddl)
+            res->revoke(table_and_dictionary_ddl);
+
+        if (!params.allow_introspection)
+            res->revoke(AccessType::INTROSPECTION);
+
+        /// Anyone has access to the "system" database.
+        res->grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
+
+        if (params.readonly != 1)
+        {
+            /// User has access to temporary or external table if such table was resolved in session or query context
+            res->grant(AccessFlags::allTableFlags() | AccessFlags::allColumnFlags(), DatabaseCatalog::TEMPORARY_DATABASE);
+        }
+
+        if (params.readonly)
+        {
+            /// No grant option in readonly mode.
+            res->revokeGrantOption(AccessType::ALL);
+        }
+
+        return res;
+    }
+
+
+    std::array<UUID, 1> to_array(const UUID & id)
+    {
+        std::array<UUID, 1> ids;
+        ids[0] = id;
+        return ids;
     }
 }
 
@@ -116,8 +119,12 @@ void ContextAccess::setUser(const UserPtr & user_) const
     if (!user)
     {
         /// User has been dropped.
-        auto nothing_granted = boost::make_shared<AccessRights>();
-        boost::range::fill(result_access, nothing_granted);
+        auto nothing_granted = std::make_shared<AccessRights>();
+        access = nothing_granted;
+        access_without_readonly = nothing_granted;
+        access_with_allow_ddl = nothing_granted;
+        access_with_allow_introspection = nothing_granted;
+        access_from_user_and_roles = nothing_granted;
         subscription_for_user_change = {};
         subscription_for_roles_changes = {};
         enabled_roles = nullptr;
@@ -169,10 +176,33 @@ void ContextAccess::setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> &
 {
     assert(roles_info_);
     roles_info = roles_info_;
-    boost::range::fill(result_access, nullptr /* need recalculate */);
     enabled_row_policies = manager->getEnabledRowPolicies(*params.user_id, roles_info->enabled_roles);
     enabled_quota = manager->getEnabledQuota(*params.user_id, user_name, roles_info->enabled_roles, params.address, params.quota_key);
     enabled_settings = manager->getEnabledSettings(*params.user_id, user->settings, roles_info->enabled_roles, roles_info->settings_from_enabled_roles);
+    calculateAccessRights();
+}
+
+
+void ContextAccess::calculateAccessRights() const
+{
+    access_from_user_and_roles = mixAccessRightsFromUserAndRoles(*user, *roles_info);
+    access = applyParamsToAccessRights(*access_from_user_and_roles, params);
+
+    access_without_readonly = nullptr;
+    access_with_allow_ddl = nullptr;
+    access_with_allow_introspection = nullptr;
+
+    if (trace_log)
+    {
+        if (roles_info && !roles_info->getCurrentRolesNames().empty())
+        {
+            LOG_TRACE(trace_log, "Current_roles: {}, enabled_roles: {}",
+                boost::algorithm::join(roles_info->getCurrentRolesNames(), ", "),
+                boost::algorithm::join(roles_info->getEnabledRolesNames(), ", "));
+        }
+        LOG_TRACE(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", params.readonly, params.allow_ddl, params.allow_introspection);
+        LOG_TRACE(trace_log, "List of all grants: {}", access->toString());
+    }
 }
 
 
@@ -193,284 +223,6 @@ bool ContextAccess::isClientHostAllowed() const
 }
 
 
-template <int mode, bool grant_option, typename... Args>
-bool ContextAccess::calculateResultAccessAndCheck(Poco::Logger * log_, const AccessFlags & flags, const Args &... args) const
-{
-    auto access = calculateResultAccess(grant_option);
-    bool is_granted = access->isGranted(flags, args...);
-
-    if (trace_log)
-        LOG_TRACE(trace_log, "Access {}: {}", (is_granted ? "granted" : "denied"), (AccessRightsElement{flags, args...}.toString()));
-
-    if (is_granted)
-        return true;
-
-    if constexpr (mode == RETURN_FALSE_IF_ACCESS_DENIED)
-        return false;
-
-    if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED)
-    {
-        if (!log_)
-            return false;
-    }
-
-    auto show_error = [&](const String & msg, [[maybe_unused]] int error_code)
-    {
-        if constexpr (mode == THROW_IF_ACCESS_DENIED)
-            throw Exception(user_name + ": " + msg, error_code);
-        else if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED)
-            LOG_WARNING(log_, "{}: {}{}", user_name, msg, formatSkippedMessage(args...));
-    };
-
-    if (!user)
-    {
-        show_error("User has been dropped", ErrorCodes::UNKNOWN_USER);
-    }
-    else if (grant_option && calculateResultAccess(false, params.readonly, params.allow_ddl, params.allow_introspection)->isGranted(flags, args...))
-    {
-        show_error(
-            "Not enough privileges. "
-            "The required privileges have been granted, but without grant option. "
-            "To execute this query it's necessary to have the grant "
-                + AccessRightsElement{flags, args...}.toString() + " WITH GRANT OPTION",
-            ErrorCodes::ACCESS_DENIED);
-    }
-    else if (params.readonly && calculateResultAccess(false, false, params.allow_ddl, params.allow_introspection)->isGranted(flags, args...))
-    {
-        if (params.interface == ClientInfo::Interface::HTTP && params.http_method == ClientInfo::HTTPMethod::GET)
-            show_error(
-                "Cannot execute query in readonly mode. "
-                "For queries over HTTP, method GET implies readonly. You should use method POST for modifying queries",
-                ErrorCodes::READONLY);
-        else
-            show_error("Cannot execute query in readonly mode", ErrorCodes::READONLY);
-    }
-    else if (!params.allow_ddl && calculateResultAccess(false, params.readonly, true, params.allow_introspection)->isGranted(flags, args...))
-    {
-        show_error("Cannot execute query. DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
-    }
-    else if (!params.allow_introspection && calculateResultAccess(false, params.readonly, params.allow_ddl, true)->isGranted(flags, args...))
-    {
-        show_error("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED);
-    }
-    else
-    {
-        show_error(
-            "Not enough privileges. To execute this query it's necessary to have the grant "
-                + AccessRightsElement{flags, args...}.toString() + (grant_option ? " WITH GRANT OPTION" : ""),
-            ErrorCodes::ACCESS_DENIED);
-    }
-
-    return false;
-}
-
-
-template <int mode, bool grant_option>
-bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags) const
-{
-    return calculateResultAccessAndCheck<mode, grant_option>(log_, flags);
-}
-
-template <int mode, bool grant_option, typename... Args>
-bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const Args &... args) const
-{
-    if (database.empty())
-        return calculateResultAccessAndCheck<mode, grant_option>(log_, flags, params.current_database, args...);
-    else
-        return calculateResultAccessAndCheck<mode, grant_option>(log_, flags, database, args...);
-}
-
-
-template <int mode, bool grant_option>
-bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessRightsElement & element) const
-{
-    if (element.any_database)
-    {
-        return checkAccessImpl<mode, grant_option>(log_, element.access_flags);
-    }
-    else if (element.any_table)
-    {
-        return checkAccessImpl<mode, grant_option>(log_, element.access_flags, element.database);
-    }
-    else if (element.any_column)
-    {
-        return checkAccessImpl<mode, grant_option>(log_, element.access_flags, element.database, element.table);
-    }
-    else
-    {
-        return checkAccessImpl<mode, grant_option>(log_, element.access_flags, element.database, element.table, element.columns);
-    }
-}
-
-
-template <int mode, bool grant_option>
-bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessRightsElements & elements) const
-{
-    for (const auto & element : elements)
-        if (!checkAccessImpl<mode, grant_option>(log_, element))
-            return false;
-    return true;
-}
-
-
-void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, false>(nullptr, flags); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, false>(nullptr, flags, database); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, false>(nullptr, flags, database, table); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, false>(nullptr, flags, database, table, column); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, false>(nullptr, flags, database, table, columns); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, false>(nullptr, flags, database, table, columns); }
-void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, false>(nullptr, element); }
-void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, false>(nullptr, elements); }
-
-bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<RETURN_FALSE_IF_ACCESS_DENIED, false>(nullptr, flags); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<RETURN_FALSE_IF_ACCESS_DENIED, false>(nullptr, flags, database); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<RETURN_FALSE_IF_ACCESS_DENIED, false>(nullptr, flags, database, table); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<RETURN_FALSE_IF_ACCESS_DENIED, false>(nullptr, flags, database, table, column); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<RETURN_FALSE_IF_ACCESS_DENIED, false>(nullptr, flags, database, table, columns); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<RETURN_FALSE_IF_ACCESS_DENIED, false>(nullptr, flags, database, table, columns); }
-bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<RETURN_FALSE_IF_ACCESS_DENIED, false>(nullptr, element); }
-bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<RETURN_FALSE_IF_ACCESS_DENIED, false>(nullptr, elements); }
-
-bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags) const { return checkAccessImpl<LOG_WARNING_IF_ACCESS_DENIED, false>(log_, flags); }
-bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<LOG_WARNING_IF_ACCESS_DENIED, false>(log_, flags, database); }
-bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<LOG_WARNING_IF_ACCESS_DENIED, false>(log_, flags, database, table); }
-bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<LOG_WARNING_IF_ACCESS_DENIED, false>(log_, flags, database, table, column); }
-bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<LOG_WARNING_IF_ACCESS_DENIED, false>(log_, flags, database, table, columns); }
-bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<LOG_WARNING_IF_ACCESS_DENIED, false>(log_, flags, database, table, columns); }
-bool ContextAccess::isGranted(Poco::Logger * log_, const AccessRightsElement & element) const { return checkAccessImpl<LOG_WARNING_IF_ACCESS_DENIED, false>(log_, element); }
-bool ContextAccess::isGranted(Poco::Logger * log_, const AccessRightsElements & elements) const { return checkAccessImpl<LOG_WARNING_IF_ACCESS_DENIED, false>(log_, elements); }
-
-void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, true>(nullptr, flags); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, true>(nullptr, flags, database); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, true>(nullptr, flags, database, table); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, true>(nullptr, flags, database, table, column); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, true>(nullptr, flags, database, table, columns); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, true>(nullptr, flags, database, table, columns); }
-void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, true>(nullptr, element); }
-void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<THROW_IF_ACCESS_DENIED, true>(nullptr, elements); }
-
-
-void ContextAccess::checkAdminOption(const UUID & role_id) const
-{
-    if (isGranted(AccessType::ROLE_ADMIN))
-        return;
-
-    auto info = getRolesInfo();
-    if (info && info->enabled_roles_with_admin_option.count(role_id))
-        return;
-
-    if (!user)
-        throw Exception(user_name + ": User has been dropped", ErrorCodes::UNKNOWN_USER);
-
-    std::optional<String> role_name = manager->readName(role_id);
-    if (!role_name)
-        role_name = "ID {" + toString(role_id) + "}";
-    throw Exception(
-        getUserName() + ": Not enough privileges. To execute this query it's necessary to have the grant " + backQuoteIfNeed(*role_name)
-            + " WITH ADMIN OPTION ",
-        ErrorCodes::ACCESS_DENIED);
-}
-
-
-boost::shared_ptr<const AccessRights> ContextAccess::calculateResultAccess(bool grant_option) const
-{
-    return calculateResultAccess(grant_option, params.readonly, params.allow_ddl, params.allow_introspection);
-}
-
-
-boost::shared_ptr<const AccessRights> ContextAccess::calculateResultAccess(bool grant_option, UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const
-{
-    size_t index = static_cast<size_t>(readonly_ != params.readonly)
-                       + static_cast<size_t>(allow_ddl_ != params.allow_ddl) * 2 +
-                       + static_cast<size_t>(allow_introspection_ != params.allow_introspection) * 3
-                       + static_cast<size_t>(grant_option) * 4;
-    assert(index < std::size(result_access));
-    auto res = result_access[index].load();
-    if (res)
-        return res;
-
-    std::lock_guard lock{mutex};
-    res = result_access[index].load();
-    if (res)
-        return res;
-
-    auto merged_access = boost::make_shared<AccessRights>();
-
-    if (grant_option)
-    {
-        *merged_access = user->access.access_with_grant_option;
-        if (roles_info)
-            merged_access->merge(roles_info->access_with_grant_option);
-    }
-    else
-    {
-        *merged_access = user->access.access;
-        if (roles_info)
-            merged_access->merge(roles_info->access);
-    }
-
-    static const AccessFlags table_ddl = AccessType::CREATE_DATABASE | AccessType::CREATE_TABLE | AccessType::CREATE_VIEW
-        | AccessType::ALTER_TABLE | AccessType::ALTER_VIEW | AccessType::DROP_DATABASE | AccessType::DROP_TABLE | AccessType::DROP_VIEW
-        | AccessType::TRUNCATE;
-
-    static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY;
-    static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
-    static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
-    static const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS;
-
-    if (readonly_)
-        merged_access->revoke(write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY);
-
-    if (readonly_ == 1)
-    {
-        /// Table functions are forbidden in readonly mode.
-        /// For example, for readonly = 2 - allowed.
-        merged_access->revoke(AccessType::CREATE_TEMPORARY_TABLE);
-    }
-
-    if (!allow_ddl_)
-        merged_access->revoke(table_and_dictionary_ddl);
-
-    if (!allow_introspection_ && !grant_option)
-        merged_access->revoke(AccessType::INTROSPECTION);
-
-    /// Anyone has access to the "system" database.
-    merged_access->grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
-
-    if (readonly_ != 1)
-    {
-        /// User has access to temporary or external table if such table was resolved in session or query context
-        merged_access->grant(AccessFlags::allTableFlags() | AccessFlags::allColumnFlags(), DatabaseCatalog::TEMPORARY_DATABASE);
-    }
-
-    if (readonly_ && grant_option)
-    {
-        /// No grant option in readonly mode.
-        merged_access->revoke(AccessType::ALL);
-    }
-
-    if (trace_log && (params.readonly == readonly_) && (params.allow_ddl == allow_ddl_) && (params.allow_introspection == allow_introspection_))
-    {
-        if (grant_option)
-            LOG_TRACE(trace_log, "List of all grants: {} WITH GRANT OPTION", merged_access->toString());
-        else
-            LOG_TRACE(trace_log, "List of all grants: {}", merged_access->toString());
-
-        if (roles_info && !roles_info->getCurrentRolesNames().empty())
-        {
-            LOG_TRACE(trace_log, "Current_roles: {}, enabled_roles: {}",
-                boost::algorithm::join(roles_info->getCurrentRolesNames(), ", "),
-                boost::algorithm::join(roles_info->getEnabledRolesNames(), ", "));
-        }
-        LOG_TRACE(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", readonly_, allow_ddl_, allow_introspection_);
-    }
-
-    res = std::move(merged_access);
-    result_access[index].store(res);
-    return res;
-}
-
-
 UserPtr ContextAccess::getUser() const
 {
     std::lock_guard lock{mutex};
@@ -520,9 +272,7 @@ std::shared_ptr<const ContextAccess> ContextAccess::getFullAccess()
     static const std::shared_ptr<const ContextAccess> res = []
     {
         auto full_access = std::shared_ptr<ContextAccess>(new ContextAccess);
-        auto everything_granted = boost::make_shared<AccessRights>();
-        everything_granted->grant(AccessType::ALL);
-        boost::range::fill(full_access->result_access, everything_granted);
+        full_access->access = std::make_shared<AccessRights>(AccessRights::getFullAccess());
         full_access->enabled_quota = EnabledQuota::getUnlimitedQuota();
         return full_access;
     }();
@@ -543,4 +293,284 @@ std::shared_ptr<const SettingsConstraints> ContextAccess::getSettingsConstraints
     return enabled_settings ? enabled_settings->getConstraints() : nullptr;
 }
 
+
+std::shared_ptr<const AccessRights> ContextAccess::getAccess() const
+{
+    std::lock_guard lock{mutex};
+    return access;
+}
+
+
+template <bool grant_option, typename... Args>
+bool ContextAccess::isGrantedImpl2(const AccessFlags & flags, const Args &... args) const
+{
+    bool access_granted;
+    if constexpr (grant_option)
+        access_granted = getAccess()->hasGrantOption(flags, args...);
+    else
+        access_granted = getAccess()->isGranted(flags, args...);
+
+    if (trace_log)
+        LOG_TRACE(trace_log, "Access {}: {}{}", (access_granted ? "granted" : "denied"), (AccessRightsElement{flags, args...}.toString()),
+                  (grant_option ? " WITH GRANT OPTION" : ""));
+
+    return access_granted;
+}
+
+template <bool grant_option>
+bool ContextAccess::isGrantedImpl(const AccessFlags & flags) const
+{
+    return isGrantedImpl2<grant_option>(flags);
+}
+
+template <bool grant_option, typename... Args>
+bool ContextAccess::isGrantedImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const
+{
+    return isGrantedImpl2<grant_option>(flags, database.empty() ? params.current_database : database, args...);
+}
+
+template <bool grant_option>
+bool ContextAccess::isGrantedImpl(const AccessRightsElement & element) const
+{
+    if (element.any_database)
+        return isGrantedImpl<grant_option>(element.access_flags);
+    else if (element.any_table)
+        return isGrantedImpl<grant_option>(element.access_flags, element.database);
+    else if (element.any_column)
+        return isGrantedImpl<grant_option>(element.access_flags, element.database, element.table);
+    else
+        return isGrantedImpl<grant_option>(element.access_flags, element.database, element.table, element.columns);
+}
+
+template <bool grant_option>
+bool ContextAccess::isGrantedImpl(const AccessRightsElements & elements) const
+{
+    for (const auto & element : elements)
+        if (!isGrantedImpl<grant_option>(element))
+            return false;
+    return true;
+}
+
+bool ContextAccess::isGranted(const AccessFlags & flags) const { return isGrantedImpl<false>(flags); }
+bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<false>(flags, database); }
+bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<false>(flags, database, table); }
+bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<false>(flags, database, table, column); }
+bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool ContextAccess::isGranted(const AccessRightsElement & element) const { return isGrantedImpl<false>(element); }
+bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl<false>(elements); }
+
+bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl<true>(flags); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<true>(flags, database); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<true>(flags, database, table); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<true>(flags, database, table, column); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
+bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }
+
+
+template <bool grant_option, typename... Args>
+void ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &... args) const
+{
+    if constexpr (grant_option)
+    {
+        if (hasGrantOption(flags, args...))
+            return;
+    }
+    else
+    {
+        if (isGranted(flags, args...))
+            return;
+    }
+
+    auto show_error = [&](const String & msg, int error_code)
+    {
+        throw Exception(user_name + ": " + msg, error_code);
+    };
+
+    std::lock_guard lock{mutex};
+
+    if (!user)
+        show_error("User has been dropped", ErrorCodes::UNKNOWN_USER);
+
+    if (grant_option && access->isGranted(flags, args...))
+    {
+        show_error(
+            "Not enough privileges. "
+            "The required privileges have been granted, but without grant option. "
+            "To execute this query it's necessary to have the grant "
+                + AccessRightsElement{flags, args...}.toString() + " WITH GRANT OPTION",
+            ErrorCodes::ACCESS_DENIED);
+    }
+
+    if (params.readonly)
+    {
+        if (!access_without_readonly)
+        {
+            Params changed_params = params;
+            changed_params.readonly = 0;
+            access_without_readonly = applyParamsToAccessRights(*access_from_user_and_roles, changed_params);
+        }
+
+        if (access_without_readonly->isGranted(flags, args...))
+        {
+            if (params.interface == ClientInfo::Interface::HTTP && params.http_method == ClientInfo::HTTPMethod::GET)
+                show_error(
+                    "Cannot execute query in readonly mode. "
+                    "For queries over HTTP, method GET implies readonly. You should use method POST for modifying queries",
+                    ErrorCodes::READONLY);
+            else
+                show_error("Cannot execute query in readonly mode", ErrorCodes::READONLY);
+        }
+    }
+
+    if (!params.allow_ddl)
+    {
+        if (!access_with_allow_ddl)
+        {
+            Params changed_params = params;
+            changed_params.allow_ddl = true;
+            access_with_allow_ddl = applyParamsToAccessRights(*access_from_user_and_roles, changed_params);
+        }
+
+        if (access_with_allow_ddl->isGranted(flags, args...))
+        {
+            show_error("Cannot execute query. DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
+        }
+    }
+
+    if (!params.allow_introspection)
+    {
+        if (!access_with_allow_introspection)
+        {
+            Params changed_params = params;
+            changed_params.allow_introspection = true;
+            access_with_allow_introspection = applyParamsToAccessRights(*access_from_user_and_roles, changed_params);
+        }
+
+        if (access_with_allow_introspection->isGranted(flags, args...))
+        {
+            show_error("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED);
+        }
+    }
+
+    show_error(
+        "Not enough privileges. To execute this query it's necessary to have the grant "
+            + AccessRightsElement{flags, args...}.toString() + (grant_option ? " WITH GRANT OPTION" : ""),
+        ErrorCodes::ACCESS_DENIED);
+}
+
+template <bool grant_option>
+void ContextAccess::checkAccessImpl(const AccessFlags & flags) const
+{
+    checkAccessImpl2<grant_option>(flags);
+}
+
+template <bool grant_option, typename... Args>
+void ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const
+{
+    checkAccessImpl2<grant_option>(flags, database.empty() ? params.current_database : database, args...);
+}
+
+template <bool grant_option>
+void ContextAccess::checkAccessImpl(const AccessRightsElement & element) const
+{
+    if (element.any_database)
+        checkAccessImpl<grant_option>(element.access_flags);
+    else if (element.any_table)
+        checkAccessImpl<grant_option>(element.access_flags, element.database);
+    else if (element.any_column)
+        checkAccessImpl<grant_option>(element.access_flags, element.database, element.table);
+    else
+        checkAccessImpl<grant_option>(element.access_flags, element.database, element.table, element.columns);
+}
+
+template <bool grant_option>
+void ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
+{
+    for (const auto & element : elements)
+        checkAccessImpl<grant_option>(element);
+}
+
+void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<false>(flags); }
+void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<false>(flags, database); }
+void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<false>(flags, database, table); }
+void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<false>(flags, database, table, column); }
+void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<false>(flags, database, table, columns); }
+void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<false>(flags, database, table, columns); }
+void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<false>(element); }
+void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<false>(elements); }
+
+void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true>(flags); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true>(flags, database); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true>(flags, database, table); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true>(flags, database, table, column); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true>(flags, database, table, columns); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true>(flags, database, table, columns); }
+void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true>(element); }
+void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true>(elements); }
+
+
+template <typename Container, typename GetNameFunction>
+void ContextAccess::checkAdminOptionImpl(const Container & role_ids, const GetNameFunction & get_name_function) const
+{
+    if (isGranted(AccessType::ROLE_ADMIN))
+        return;
+
+    auto info = getRolesInfo();
+    if (!info)
+    {
+        if (!user)
+            throw Exception(user_name + ": User has been dropped", ErrorCodes::UNKNOWN_USER);
+        return;
+    }
+
+    size_t i = 0;
+    for (auto it = std::begin(role_ids); it != std::end(role_ids); ++it, ++i)
+    {
+        const UUID & role_id = *it;
+        if (info->enabled_roles_with_admin_option.count(role_id))
+            continue;
+
+        auto role_name = get_name_function(role_id, i);
+        if (!role_name)
+            role_name = "ID {" + toString(role_id) + "}";
+        String msg = "To execute this query it's necessary to have the role " + backQuoteIfNeed(*role_name) + " granted with ADMIN option";
+        if (info->enabled_roles.count(role_id))
+            msg = "Role " + backQuote(*role_name) + " is granted, but without ADMIN option. " + msg;
+        throw Exception(getUserName() + ": Not enough privileges. " + msg, ErrorCodes::ACCESS_DENIED);
+    }
+}
+
+void ContextAccess::checkAdminOption(const UUID & role_id) const
+{
+    checkAdminOptionImpl(to_array(role_id), [this](const UUID & id, size_t) { return manager->tryReadName(id); });
+}
+
+void ContextAccess::checkAdminOption(const UUID & role_id, const String & role_name) const
+{
+    checkAdminOptionImpl(to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional<String>{role_name}; });
+}
+
+void ContextAccess::checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const
+{
+    checkAdminOptionImpl(to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
+}
+
+void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids) const
+{
+    checkAdminOptionImpl(role_ids, [this](const UUID & id, size_t) { return manager->tryReadName(id); });
+}
+
+void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const
+{
+    checkAdminOptionImpl(role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional<String>{names_of_roles[i]}; });
+}
+
+void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const
+{
+    checkAdminOptionImpl(role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
+}
+
 }

src/Access/ContextAccess.h

@@ -6,7 +6,6 @@
 #include <Core/UUID.h>
 #include <ext/scope_guard.h>
 #include <ext/shared_ptr_helper.h>
-#include <boost/smart_ptr/atomic_shared_ptr.hpp>
 #include <boost/container/flat_set.hpp>
 #include <mutex>
 
@@ -30,32 +29,34 @@ class IAST;
 using ASTPtr = std::shared_ptr<IAST>;
 
 
+struct ContextAccessParams
+{
+    std::optional<UUID> user_id;
+    boost::container::flat_set<UUID> current_roles;
+    bool use_default_roles = false;
+    UInt64 readonly = 0;
+    bool allow_ddl = false;
+    bool allow_introspection = false;
+    String current_database;
+    ClientInfo::Interface interface = ClientInfo::Interface::TCP;
+    ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN;
+    Poco::Net::IPAddress address;
+    String quota_key;
+
+    auto toTuple() const { return std::tie(user_id, current_roles, use_default_roles, readonly, allow_ddl, allow_introspection, current_database, interface, http_method, address, quota_key); }
+    friend bool operator ==(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return lhs.toTuple() == rhs.toTuple(); }
+    friend bool operator !=(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return !(lhs == rhs); }
+    friend bool operator <(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return lhs.toTuple() < rhs.toTuple(); }
+    friend bool operator >(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return rhs < lhs; }
+    friend bool operator <=(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return !(rhs < lhs); }
+    friend bool operator >=(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return !(lhs < rhs); }
+};
+
+
 class ContextAccess
 {
 public:
-    struct Params
-    {
-        std::optional<UUID> user_id;
-        boost::container::flat_set<UUID> current_roles;
-        bool use_default_roles = false;
-        UInt64 readonly = 0;
-        bool allow_ddl = false;
-        bool allow_introspection = false;
-        String current_database;
-        ClientInfo::Interface interface = ClientInfo::Interface::TCP;
-        ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN;
-        Poco::Net::IPAddress address;
-        String quota_key;
-
-        auto toTuple() const { return std::tie(user_id, current_roles, use_default_roles, readonly, allow_ddl, allow_introspection, current_database, interface, http_method, address, quota_key); }
-        friend bool operator ==(const Params & lhs, const Params & rhs) { return lhs.toTuple() == rhs.toTuple(); }
-        friend bool operator !=(const Params & lhs, const Params & rhs) { return !(lhs == rhs); }
-        friend bool operator <(const Params & lhs, const Params & rhs) { return lhs.toTuple() < rhs.toTuple(); }
-        friend bool operator >(const Params & lhs, const Params & rhs) { return rhs < lhs; }
-        friend bool operator <=(const Params & lhs, const Params & rhs) { return !(rhs < lhs); }
-        friend bool operator >=(const Params & lhs, const Params & rhs) { return !(lhs < rhs); }
-    };
-
+    using Params = ContextAccessParams;
     const Params & getParams() const { return params; }
 
     /// Returns the current user. The function can return nullptr.
@@ -90,16 +91,8 @@ public:
     /// The function returns nullptr if there are no constraints.
     std::shared_ptr<const SettingsConstraints> getSettingsConstraints() const;
 
-    /// Checks if a specified access is granted, and throws an exception if not.
-    /// Empty database means the current database.
-    void checkAccess(const AccessFlags & flags) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
-    void checkAccess(const AccessRightsElement & element) const;
-    void checkAccess(const AccessRightsElements & elements) const;
+    /// Returns the current access rights.
+    std::shared_ptr<const AccessRights> getAccess() const;
 
     /// Checks if a specified access is granted.
     bool isGranted(const AccessFlags & flags) const;
@@ -111,17 +104,26 @@ public:
     bool isGranted(const AccessRightsElement & element) const;
     bool isGranted(const AccessRightsElements & elements) const;
 
-    /// Checks if a specified access is granted, and logs a warning if not.
-    bool isGranted(Poco::Logger * log_, const AccessFlags & flags) const;
-    bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database) const;
-    bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
-    bool isGranted(Poco::Logger * log_, const AccessRightsElement & element) const;
-    bool isGranted(Poco::Logger * log_, const AccessRightsElements & elements) const;
+    bool hasGrantOption(const AccessFlags & flags) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool hasGrantOption(const AccessRightsElement & element) const;
+    bool hasGrantOption(const AccessRightsElements & elements) const;
+
+    /// Checks if a specified access is granted, and throws an exception if not.
+    /// Empty database means the current database.
+    void checkAccess(const AccessFlags & flags) const;
+    void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
+    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
+    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
+    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
+    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    void checkAccess(const AccessRightsElement & element) const;
+    void checkAccess(const AccessRightsElements & elements) const;
 
-    /// Checks if a specified access is granted with grant option, and throws an exception if not.
     void checkGrantOption(const AccessFlags & flags) const;
     void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const;
     void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
@@ -133,6 +135,11 @@ public:
 
     /// Checks if a specified role is granted with admin option, and throws an exception if not.
     void checkAdminOption(const UUID & role_id) const;
+    void checkAdminOption(const UUID & role_id, const String & role_name) const;
+    void checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
+    void checkAdminOption(const std::vector<UUID> & role_ids) const;
+    void checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
+    void checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
 
     /// Makes an instance of ContextAccess which provides full access to everything
     /// without any limitations. This is used for the global context.
@@ -146,24 +153,40 @@ private:
     void setUser(const UserPtr & user_) const;
     void setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> & roles_info_) const;
     void setSettingsAndConstraints() const;
+    void calculateAccessRights() const;
 
-    template <int mode, bool grant_option>
-    bool checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags) const;
+    template <bool grant_option>
+    bool isGrantedImpl(const AccessFlags & flags) const;
 
-    template <int mode, bool grant_option, typename... Args>
-    bool checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
+    template <bool grant_option, typename... Args>
+    bool isGrantedImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
 
-    template <int mode, bool grant_option>
-    bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElement & element) const;
+    template <bool grant_option>
+    bool isGrantedImpl(const AccessRightsElement & element) const;
 
-    template <int mode, bool grant_option>
-    bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElements & elements) const;
+    template <bool grant_option>
+    bool isGrantedImpl(const AccessRightsElements & elements) const;
 
-    template <int mode, bool grant_option, typename... Args>
-    bool calculateResultAccessAndCheck(Poco::Logger * log_, const AccessFlags & flags, const Args &... args) const;
+    template <bool grant_option, typename... Args>
+    bool isGrantedImpl2(const AccessFlags & flags, const Args &... args) const;
 
-    boost::shared_ptr<const AccessRights> calculateResultAccess(bool grant_option) const;
-    boost::shared_ptr<const AccessRights> calculateResultAccess(bool grant_option, UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const;
+    template <bool grant_option>
+    void checkAccessImpl(const AccessFlags & flags) const;
+
+    template <bool grant_option, typename... Args>
+    void checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
+
+    template <bool grant_option>
+    void checkAccessImpl(const AccessRightsElement & element) const;
+
+    template <bool grant_option>
+    void checkAccessImpl(const AccessRightsElements & elements) const;
+
+    template <bool grant_option, typename... Args>
+    void checkAccessImpl2(const AccessFlags & flags, const Args &... args) const;
+
+    template <typename Container, typename GetNameFunction>
+    void checkAdminOptionImpl(const Container & role_ids, const GetNameFunction & get_name_function) const;
 
     const AccessControlManager * manager = nullptr;
     const Params params;
@@ -174,10 +197,14 @@ private:
     mutable std::shared_ptr<const EnabledRoles> enabled_roles;
     mutable ext::scope_guard subscription_for_roles_changes;
     mutable std::shared_ptr<const EnabledRolesInfo> roles_info;
-    mutable boost::atomic_shared_ptr<const AccessRights> result_access[7];
+    mutable std::shared_ptr<const AccessRights> access;
     mutable std::shared_ptr<const EnabledRowPolicies> enabled_row_policies;
     mutable std::shared_ptr<const EnabledQuota> enabled_quota;
     mutable std::shared_ptr<const EnabledSettings> enabled_settings;
+    mutable std::shared_ptr<const AccessRights> access_without_readonly;
+    mutable std::shared_ptr<const AccessRights> access_with_allow_ddl;
+    mutable std::shared_ptr<const AccessRights> access_with_allow_introspection;
+    mutable std::shared_ptr<const AccessRights> access_from_user_and_roles;
     mutable std::mutex mutex;
 };
 

src/Access/EnabledRolesInfo.cpp

@@ -28,8 +28,7 @@ bool operator==(const EnabledRolesInfo & lhs, const EnabledRolesInfo & rhs)
 {
     return (lhs.current_roles == rhs.current_roles) && (lhs.enabled_roles == rhs.enabled_roles)
         && (lhs.enabled_roles_with_admin_option == rhs.enabled_roles_with_admin_option) && (lhs.names_of_roles == rhs.names_of_roles)
-        && (lhs.access == rhs.access) && (lhs.access_with_grant_option == rhs.access_with_grant_option)
-        && (lhs.settings_from_enabled_roles == rhs.settings_from_enabled_roles);
+        && (lhs.access == rhs.access) && (lhs.settings_from_enabled_roles == rhs.settings_from_enabled_roles);
 }
 
 }

src/Access/EnabledRolesInfo.h

@@ -18,7 +18,6 @@ struct EnabledRolesInfo
     boost::container::flat_set<UUID> enabled_roles_with_admin_option;
     std::unordered_map<UUID, String> names_of_roles;
     AccessRights access;
-    AccessRights access_with_grant_option;
     SettingsProfileElements settings_from_enabled_roles;
 
     Strings getCurrentRolesNames() const;

src/Access/GrantedAccess.cpp

@@ -1,22 +0,0 @@
-#include <Access/GrantedAccess.h>
-
-
-namespace DB
-{
-
-GrantedAccess::GrantsAndPartialRevokes GrantedAccess::getGrantsAndPartialRevokes() const
-{
-    GrantsAndPartialRevokes res;
-    res.grants_with_grant_option = access_with_grant_option.getGrants();
-    AccessRights access_without_gg = access;
-    access_without_gg.revoke(res.grants_with_grant_option);
-    auto gr = access_without_gg.getGrantsAndPartialRevokes();
-    res.grants = std::move(gr.grants);
-    res.revokes = std::move(gr.revokes);
-    AccessRights access_with_grant_options_without_r = access_with_grant_option;
-    access_with_grant_options_without_r.grant(res.revokes);
-    res.revokes_grant_option = access_with_grant_options_without_r.getPartialRevokes();
-    return res;
-}
-
-}

src/Access/GrantedAccess.h

@@ -1,55 +0,0 @@
-#pragma once
-
-#include <Access/AccessRights.h>
-
-
-namespace DB
-{
-/// Access rights as they are granted to a role or user.
-/// Stores both the access rights themselves and the access rights with grant option.
-struct GrantedAccess
-{
-    AccessRights access;
-    AccessRights access_with_grant_option;
-
-    template <typename... Args>
-    void grant(const Args &... args)
-    {
-        access.grant(args...);
-    }
-
-    template <typename... Args>
-    void grantWithGrantOption(const Args &... args)
-    {
-        access.grant(args...);
-        access_with_grant_option.grant(args...);
-    }
-
-    template <typename... Args>
-    void revoke(const Args &... args)
-    {
-        access.revoke(args...);
-        access_with_grant_option.revoke(args...);
-    }
-
-    template <typename... Args>
-    void revokeGrantOption(const Args &... args)
-    {
-        access_with_grant_option.revoke(args...);
-    }
-
-    struct GrantsAndPartialRevokes
-    {
-        AccessRightsElements grants;
-        AccessRightsElements revokes;
-        AccessRightsElements grants_with_grant_option;
-        AccessRightsElements revokes_grant_option;
-    };
-
-    /// Retrieves the information about grants and partial revokes.
-    GrantsAndPartialRevokes getGrantsAndPartialRevokes() const;
-
-    friend bool operator ==(const GrantedAccess & left, const GrantedAccess & right) { return (left.access == right.access) && (left.access_with_grant_option == right.access_with_grant_option); }
-    friend bool operator !=(const GrantedAccess & left, const GrantedAccess & right) { return !(left == right); }
-};
-}

src/Access/Role.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <Access/IAccessEntity.h>
-#include <Access/GrantedAccess.h>
+#include <Access/AccessRights.h>
 #include <Access/GrantedRoles.h>
 #include <Access/SettingsProfileElement.h>
 
@@ -11,7 +11,7 @@ namespace DB
 
 struct Role : public IAccessEntity
 {
-    GrantedAccess access;
+    AccessRights access;
     GrantedRoles granted_roles;
     SettingsProfileElements settings;
 

src/Access/RoleCache.cpp

@@ -43,8 +43,7 @@ namespace
             roles_info.enabled_roles_with_admin_option.emplace(role_id);
 
         roles_info.names_of_roles[role_id] = role->getName();
-        roles_info.access.merge(role->access.access);
-        roles_info.access_with_grant_option.merge(role->access.access_with_grant_option);
+        roles_info.access.makeUnion(role->access);
         roles_info.settings_from_enabled_roles.merge(role->settings);
 
         for (const auto & granted_role : role->granted_roles.roles)

src/Access/User.h

@@ -1,9 +1,9 @@
 #pragma once
 
 #include <Access/IAccessEntity.h>
+#include <Access/AccessRights.h>
 #include <Access/Authentication.h>
 #include <Access/AllowedClientHosts.h>
-#include <Access/GrantedAccess.h>
 #include <Access/GrantedRoles.h>
 #include <Access/RolesOrUsersSet.h>
 #include <Access/SettingsProfileElement.h>
@@ -17,7 +17,7 @@ struct User : public IAccessEntity
 {
     Authentication authentication;
     AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{};
-    GrantedAccess access;
+    AccessRights access;
     GrantedRoles granted_roles;
     RolesOrUsersSet default_roles = RolesOrUsersSet::AllTag{};
     SettingsProfileElements settings;

src/Access/ya.make

@@ -17,7 +17,6 @@ SRCS(
     EnabledRolesInfo.cpp
     EnabledRowPolicies.cpp
     EnabledSettings.cpp
-    GrantedAccess.cpp
     GrantedRoles.cpp
     IAccessEntity.cpp
     IAccessStorage.cpp

src/AggregateFunctions/AggregateFunctionTopK.cpp

@@ -100,7 +100,8 @@ AggregateFunctionPtr createAggregateFunctionTopK(const std::string & name, const
         threshold = k;
     }
 
-    AggregateFunctionPtr res(createWithNumericType<AggregateFunctionTopK, is_weighted>(*argument_types[0], threshold, load_factor, argument_types, params));
+    AggregateFunctionPtr res(createWithNumericType<AggregateFunctionTopK, is_weighted>(
+        *argument_types[0], threshold, load_factor, argument_types, params));
 
     if (!res)
         res = AggregateFunctionPtr(createWithExtraTypes<is_weighted>(argument_types[0], threshold, load_factor, params));

src/AggregateFunctions/AggregateFunctionTopK.h

@@ -47,7 +47,7 @@ public:
 
     DataTypePtr getReturnType() const override
     {
-        return std::make_shared<DataTypeArray>(std::make_shared<DataTypeNumber<T>>());
+        return std::make_shared<DataTypeArray>(this->argument_types[0]);
     }
 
     void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override

src/CMakeLists.txt

@@ -73,6 +73,10 @@ if(USE_RDKAFKA)
     add_headers_and_sources(dbms Storages/Kafka)
 endif()
 
+if (USE_AMQPCPP)
+    add_headers_and_sources(dbms Storages/RabbitMQ)
+endif()
+
 if (USE_AWS_S3)
     add_headers_and_sources(dbms Common/S3)
     add_headers_and_sources(dbms Disks/S3)
@@ -262,6 +266,9 @@ if (USE_RDKAFKA)
     endif()
 endif()
 
+if (USE_AMQPCPP)
+    dbms_target_link_libraries(PUBLIC amqp-cpp)
+endif()
 
 if(RE2_INCLUDE_DIR)
     target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})

src/Columns/ColumnConst.cpp

@@ -120,7 +120,9 @@ void ColumnConst::getPermutation(bool /*reverse*/, size_t /*limit*/, int /*nan_d
         res[i] = i;
 }
 
-void ColumnConst::updatePermutation(bool, size_t, int, Permutation &, EqualRanges &) const {}
+void ColumnConst::updatePermutation(bool, size_t, int, Permutation &, EqualRanges &) const
+{
+}
 
 void ColumnConst::updateWeakHash32(WeakHash32 & hash) const
 {

src/Common/ErrorCodes.cpp

@@ -496,6 +496,7 @@ namespace ErrorCodes
     extern const int NO_SUITABLE_FUNCTION_IMPLEMENTATION = 527;
     extern const int CASSANDRA_INTERNAL_ERROR = 528;
     extern const int NOT_A_LEADER = 529;
+    extern const int CANNOT_CONNECT_RABBITMQ = 530;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

src/Common/ProfileEvents.cpp

@@ -196,8 +196,12 @@
     M(PerfCpuMigrations, "Number of times the process has migrated to a new CPU") \
     M(PerfAlignmentFaults, "Number of alignment faults. These happen when unaligned memory accesses happen; the kernel can handle these but it reduces performance. This happens only on some architectures (never on x86).") \
     M(PerfEmulationFaults, "Number of emulation faults. The kernel sometimes traps on unimplemented instructions and emulates them for user space. This can negatively impact performance.") \
-    M(PerfPageFaultsMinor, "This counts the number of minor page faults. These did not require disk I/O to handle.") \
-    M(PerfPageFaultsMajor, "This counts the number of major page faults. These required disk I/O to handle.") \
+    M(PerfMinEnabledTime, "For all events, minimum time that an event was enabled. Used to track event multiplexing influence") \
+    M(PerfMinEnabledRunningTime, "Running time for event with minimum enabled time. Used to track the amount of event multiplexing") \
+    M(PerfDataTLBReferences, "Data TLB references") \
+    M(PerfDataTLBMisses, "Data TLB misses") \
+    M(PerfInstructionTLBReferences, "Instruction TLB references") \
+    M(PerfInstructionTLBMisses, "Instruction TLB misses") \
     \
     M(CreatedHTTPConnections, "Total amount of created HTTP connections (closed or opened).") \
     \

src/Common/StatusFile.cpp

@@ -30,8 +30,21 @@ namespace ErrorCodes
 }
 
 
-StatusFile::StatusFile(const std::string & path_)
-    : path(path_)
+StatusFile::FillFunction StatusFile::write_pid = [](WriteBuffer & out)
+{
+    out << getpid();
+};
+
+StatusFile::FillFunction StatusFile::write_full_info = [](WriteBuffer & out)
+{
+    out << "PID: " << getpid() << "\n"
+        << "Started at: " << LocalDateTime(time(nullptr)) << "\n"
+        << "Revision: " << ClickHouseRevision::get() << "\n";
+};
+
+
+StatusFile::StatusFile(std::string path_, FillFunction fill_)
+    : path(std::move(path_)), fill(std::move(fill_))
 {
     /// If file already exists. NOTE Minor race condition.
     if (Poco::File(path).exists())
@@ -72,13 +85,8 @@ StatusFile::StatusFile(const std::string & path_)
             throwFromErrnoWithPath("Cannot lseek " + path, path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
 
         /// Write information about current server instance to the file.
-        {
-            WriteBufferFromFileDescriptor out(fd, 1024);
-            out
-                << "PID: " << getpid() << "\n"
-                << "Started at: " << LocalDateTime(time(nullptr)) << "\n"
-                << "Revision: " << ClickHouseRevision::get() << "\n";
-        }
+        WriteBufferFromFileDescriptor out(fd, 1024);
+        fill(out);
     }
     catch (...)
     {

src/Common/StatusFile.h

@@ -1,23 +1,33 @@
 #pragma once
 
 #include <string>
+#include <functional>
 #include <boost/noncopyable.hpp>
 
 
 namespace DB
 {
 
+class WriteBuffer;
+
 
 /** Provides that no more than one server works with one data directory.
   */
 class StatusFile : private boost::noncopyable
 {
 public:
-    explicit StatusFile(const std::string & path_);
+    using FillFunction = std::function<void(WriteBuffer&)>;
+
+    StatusFile(std::string path_, FillFunction fill_);
     ~StatusFile();
 
+    /// You can use one of these functions to fill the file or provide your own.
+    static FillFunction write_pid;
+    static FillFunction write_full_info;
+
 private:
     const std::string path;
+    FillFunction fill;
     int fd = -1;
 };
 

src/Common/ThreadProfileEvents.cpp

@@ -147,6 +147,19 @@ thread_local PerfEventsCounters current_thread_counters;
         .settings_name = #LOCAL_NAME \
     }
 
+// One event for cache accesses and one for cache misses.
+// Type is ACCESS or MISS
+#define CACHE_EVENT(PERF_NAME, LOCAL_NAME, TYPE) \
+    PerfEventInfo \
+    { \
+        .event_type = perf_type_id::PERF_TYPE_HW_CACHE, \
+        .event_config = (PERF_NAME) \
+            | (PERF_COUNT_HW_CACHE_OP_READ << 8) \
+            | (PERF_COUNT_HW_CACHE_RESULT_ ## TYPE << 16), \
+        .profile_event = ProfileEvents::LOCAL_NAME, \
+        .settings_name = #LOCAL_NAME \
+    }
+
 // descriptions' source: http://man7.org/linux/man-pages/man2/perf_event_open.2.html
 static const PerfEventInfo raw_events_info[] = {
     HARDWARE_EVENT(PERF_COUNT_HW_CPU_CYCLES, PerfCpuCycles),
@@ -167,8 +180,19 @@ static const PerfEventInfo raw_events_info[] = {
     SOFTWARE_EVENT(PERF_COUNT_SW_CPU_MIGRATIONS, PerfCpuMigrations),
     SOFTWARE_EVENT(PERF_COUNT_SW_ALIGNMENT_FAULTS, PerfAlignmentFaults),
     SOFTWARE_EVENT(PERF_COUNT_SW_EMULATION_FAULTS, PerfEmulationFaults),
-    SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MIN, PerfPageFaultsMinor),
-    SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MAJ, PerfPageFaultsMajor)
+
+    // Don't add them -- they are the same as SoftPageFaults and HardPageFaults,
+    // match well numerically.
+    // SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MIN, PerfPageFaultsMinor),
+    // SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MAJ, PerfPageFaultsMajor),
+
+    CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBReferences, ACCESS),
+    CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBMisses, MISS),
+
+    // Apparently it doesn't make sense to treat these values as relative:
+    // https://stackoverflow.com/questions/49933319/how-to-interpret-perf-itlb-loads-itlb-load-misses
+    CACHE_EVENT(PERF_COUNT_HW_CACHE_ITLB, PerfInstructionTLBReferences, ACCESS),
+    CACHE_EVENT(PERF_COUNT_HW_CACHE_ITLB, PerfInstructionTLBMisses, MISS),
 };
 
 static_assert(sizeof(raw_events_info) / sizeof(raw_events_info[0]) == NUMBER_OF_RAW_EVENTS);
@@ -455,7 +479,12 @@ void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile
         }
     }
 
-    // actually process counters' values
+    // Actually process counters' values. Track the minimal time that a performance
+    // counter was enabled, and the corresponding running time, to give some idea
+    // about the amount of counter multiplexing.
+    UInt64 min_enabled_time = -1;
+    UInt64 running_time_for_min_enabled_time = 0;
+
     for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i)
     {
         int fd = thread_events_descriptors_holder.descriptors[i];
@@ -469,14 +498,30 @@ void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile
         // Account for counter multiplexing. time_running and time_enabled are
         // not reset by PERF_EVENT_IOC_RESET, so we don't use it and calculate
         // deltas from old values.
+        const auto enabled = current_value.time_enabled - previous_value.time_enabled;
+        const auto running = current_value.time_running - previous_value.time_running;
         const UInt64 delta = (current_value.value - previous_value.value)
-            * (current_value.time_enabled - previous_value.time_enabled)
-            / std::max(1.f,
-                float(current_value.time_running - previous_value.time_running));
+            * enabled / std::max(1.f, float(running));
+
+        if (min_enabled_time > enabled)
+        {
+            min_enabled_time = enabled;
+            running_time_for_min_enabled_time = running;
+        }
 
         profile_events.increment(info.profile_event, delta);
     }
 
+    // If we had at least one enabled event, also show multiplexing-related
+    // statistics.
+    if (min_enabled_time != UInt64(-1))
+    {
+        profile_events.increment(ProfileEvents::PerfMinEnabledTime,
+            min_enabled_time);
+        profile_events.increment(ProfileEvents::PerfMinEnabledRunningTime,
+            running_time_for_min_enabled_time);
+    }
+
     // Store current counter values for the next profiling period.
     memcpy(previous_values, current_values, sizeof(current_values));
 }

src/Common/ThreadProfileEvents.h

@@ -53,8 +53,12 @@ namespace ProfileEvents
     extern const Event PerfCpuMigrations;
     extern const Event PerfAlignmentFaults;
     extern const Event PerfEmulationFaults;
-    extern const Event PerfPageFaultsMinor;
-    extern const Event PerfPageFaultsMajor;
+    extern const Event PerfMinEnabledTime;
+    extern const Event PerfMinEnabledRunningTime;
+    extern const Event PerfDataTLBReferences;
+    extern const Event PerfDataTLBMisses;
+    extern const Event PerfInstructionTLBReferences;
+    extern const Event PerfInstructionTLBMisses;
 #endif
 }
 
@@ -158,7 +162,7 @@ struct PerfEventValue
     UInt64 time_running = 0;
 };
 
-static constexpr size_t NUMBER_OF_RAW_EVENTS = 18;
+static constexpr size_t NUMBER_OF_RAW_EVENTS = 20;
 
 struct PerfDescriptorsHolder : boost::noncopyable
 {

src/Core/Settings.h

@@ -74,6 +74,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when the connection pool is full.", 0) \
     M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.", 0) \
     M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.", 0) \
+    M(SettingMilliseconds, rabbitmq_max_wait_ms, 5000, "The wait time for reading from RabbitMQ before retry.", 0) \
     M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.", 0) \
     M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \
     M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.", 0) \
@@ -335,6 +336,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingBool, enable_unaligned_array_join, false, "Allow ARRAY JOIN with multiple arrays that have different sizes. When this settings is enabled, arrays will be resized to the longest one.", 0) \
     M(SettingBool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \
     M(SettingBool, optimize_aggregation_in_order, false, "Enable GROUP BY optimization for aggregating data in corresponding order in MergeTree tables.", 0) \
+    M(SettingUInt64, read_in_order_two_level_merge_threshold, 100, "Minimal number of parts to read to run preliminary merge step during multithread reading in order of primary key.", 0) \
     M(SettingBool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \
     M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \
     M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \

src/Core/config_core.h.in

@@ -5,6 +5,7 @@
 #cmakedefine01 USE_ICU
 #cmakedefine01 USE_MYSQL
 #cmakedefine01 USE_RDKAFKA
+#cmakedefine01 USE_AMQPCPP
 #cmakedefine01 USE_EMBEDDED_COMPILER
 #cmakedefine01 USE_INTERNAL_LLVM_LIBRARY
 #cmakedefine01 USE_SSL

src/Databases/DatabaseOnDisk.cpp

@@ -389,6 +389,9 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati
         }
     };
 
+    /// Metadata files to load: name and flag for .tmp_drop files
+    std::set<std::pair<String, bool>> metadata_files;
+
     Poco::DirectoryIterator dir_end;
     for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
     {
@@ -404,7 +407,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati
         if (endsWith(dir_it.name(), tmp_drop_ext))
         {
             /// There are files that we tried to delete previously
-            process_tmp_drop_metadata_file(dir_it.name());
+            metadata_files.emplace(dir_it.name(), false);
         }
         else if (endsWith(dir_it.name(), ".sql.tmp"))
         {
@@ -415,12 +418,26 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati
         else if (endsWith(dir_it.name(), ".sql"))
         {
             /// The required files have names like `table_name.sql`
-            process_metadata_file(dir_it.name());
+            metadata_files.emplace(dir_it.name(), true);
         }
         else
             throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(),
                 ErrorCodes::INCORRECT_FILE_NAME);
     }
+
+    /// Read and parse metadata in parallel
+    ThreadPool pool(SettingMaxThreads().getAutoValue());
+    for (const auto & file : metadata_files)
+    {
+        pool.scheduleOrThrowOnError([&]()
+        {
+            if (file.second)
+                process_metadata_file(file.first);
+            else
+                process_tmp_drop_metadata_file(file.first);
+        });
+    }
+    pool.wait();
 }
 
 ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * logger, const Context & context, const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/)

src/Databases/DatabaseOrdinary.cpp

@@ -112,11 +112,12 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto
       *  which does not correspond to order tables creation and does not correspond to order of their location on disk.
       */
     using FileNames = std::map<std::string, ASTPtr>;
+    std::mutex file_names_mutex;
     FileNames file_names;
 
     size_t total_dictionaries = 0;
 
-    auto process_metadata = [&context, &file_names, &total_dictionaries, this](const String & file_name)
+    auto process_metadata = [&context, &file_names, &total_dictionaries, &file_names_mutex, this](const String & file_name)
     {
         fs::path path(getMetadataPath());
         fs::path file_path(file_name);
@@ -128,6 +129,7 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto
             if (ast)
             {
                 auto * create_query = ast->as<ASTCreateQuery>();
+                std::lock_guard lock{file_names_mutex};
                 file_names[file_name] = ast;
                 total_dictionaries += create_query->is_dictionary;
             }

src/Functions/array/FunctionArrayMapped.h

@@ -73,7 +73,7 @@ public:
             if (!array_type)
                 throw Exception("Argument " + toString(i + 2) + " of function " + getName() + " must be array. Found "
                                 + arguments[i + 1]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-            nested_types[i] = removeLowCardinality(array_type->getNestedType());
+            nested_types[i] = recursiveRemoveLowCardinality(array_type->getNestedType());
         }
 
         const DataTypeFunction * function_type = checkAndGetDataType<DataTypeFunction>(arguments[0].get());
@@ -190,9 +190,7 @@ public:
                     const ColumnConst * column_const_array = checkAndGetColumnConst<ColumnArray>(column_array_ptr.get());
                     if (!column_const_array)
                         throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN);
-                    column_array_ptr = column_const_array->convertToFullColumn();
-                    if (column_array_ptr->lowCardinality())
-                        column_array_ptr = column_array_ptr->convertToFullColumnIfLowCardinality();
+                    column_array_ptr = recursiveRemoveLowCardinality(column_const_array->convertToFullColumn());
                     column_array = checkAndGetColumn<ColumnArray>(column_array_ptr.get());
                 }
 
@@ -218,7 +216,7 @@ public:
                 }
 
                 arrays.emplace_back(ColumnWithTypeAndName(column_array->getDataPtr(),
-                                                          removeLowCardinality(array_type->getNestedType()),
+                                                          recursiveRemoveLowCardinality(array_type->getNestedType()),
                                                           array_with_type_and_name.name));
             }