" folders in GUI-based IDEs.
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
diff --git a/contrib/c-ares b/contrib/c-ares
new file mode 160000
index 00000000000..afee6748b0b
--- /dev/null
+++ b/contrib/c-ares
@@ -0,0 +1 @@
+Subproject commit afee6748b0b99acf4509d42fa37ac8422262f91b
diff --git a/contrib/c-ares-cmake/CMakeLists.txt b/contrib/c-ares-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..603c1f8b65c
--- /dev/null
+++ b/contrib/c-ares-cmake/CMakeLists.txt
@@ -0,0 +1,35 @@
+# Choose to build static or shared library for c-ares.
+if (USE_STATIC_LIBRARIES)
+ set(CARES_STATIC ON CACHE BOOL "" FORCE)
+ set(CARES_SHARED OFF CACHE BOOL "" FORCE)
+else ()
+ set(CARES_STATIC OFF CACHE BOOL "" FORCE)
+ set(CARES_SHARED ON CACHE BOOL "" FORCE)
+endif ()
+
+# Disable looking for libnsl on a platforms that has gethostbyname in glibc
+#
+# c-ares searching for gethostbyname in the libnsl library, however in the
+# version that shipped with gRPC it doing it wrong [1], since it uses
+# CHECK_LIBRARY_EXISTS(), which will return TRUE even if the function exists in
+# another dependent library. The upstream already contains correct macro [2],
+# but it is not included in gRPC (even upstream gRPC, not the one that is
+# shipped with clickhousee).
+#
+# [1]: https://github.com/c-ares/c-ares/blob/e982924acee7f7313b4baa4ee5ec000c5e373c30/CMakeLists.txt#L125
+# [2]: https://github.com/c-ares/c-ares/blob/44fbc813685a1fa8aa3f27fcd7544faf612d376a/CMakeLists.txt#L146
+#
+# And because if you by some reason have libnsl [3] installed, clickhouse will
+# reject to start w/o it. While this is completelly different library.
+#
+# [3]: https://packages.debian.org/bullseye/libnsl2
+if (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+ set(HAVE_LIBNSL OFF CACHE BOOL "" FORCE)
+endif()
+
+# Force use of c-ares inet_net_pton instead of libresolv one
+set(HAVE_INET_NET_PTON OFF CACHE BOOL "" FORCE)
+
+add_subdirectory("../c-ares/" "../c-ares/")
+
+add_library(ch_contrib::c-ares ALIAS c-ares)
\ No newline at end of file
diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt
index 520e04d198e..b1ed7e464b6 100644
--- a/contrib/grpc-cmake/CMakeLists.txt
+++ b/contrib/grpc-cmake/CMakeLists.txt
@@ -45,38 +45,11 @@ set(_gRPC_SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL)
# Use abseil-cpp from ClickHouse contrib, not from gRPC third_party.
set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE)
-# Choose to build static or shared library for c-ares.
-if (USE_STATIC_LIBRARIES)
- set(CARES_STATIC ON CACHE BOOL "" FORCE)
- set(CARES_SHARED OFF CACHE BOOL "" FORCE)
-else ()
- set(CARES_STATIC OFF CACHE BOOL "" FORCE)
- set(CARES_SHARED ON CACHE BOOL "" FORCE)
-endif ()
-
-# Disable looking for libnsl on a platforms that has gethostbyname in glibc
-#
-# c-ares searching for gethostbyname in the libnsl library, however in the
-# version that shipped with gRPC it doing it wrong [1], since it uses
-# CHECK_LIBRARY_EXISTS(), which will return TRUE even if the function exists in
-# another dependent library. The upstream already contains correct macro [2],
-# but it is not included in gRPC (even upstream gRPC, not the one that is
-# shipped with clickhousee).
-#
-# [1]: https://github.com/c-ares/c-ares/blob/e982924acee7f7313b4baa4ee5ec000c5e373c30/CMakeLists.txt#L125
-# [2]: https://github.com/c-ares/c-ares/blob/44fbc813685a1fa8aa3f27fcd7544faf612d376a/CMakeLists.txt#L146
-#
-# And because if you by some reason have libnsl [3] installed, clickhouse will
-# reject to start w/o it. While this is completelly different library.
-#
-# [3]: https://packages.debian.org/bullseye/libnsl2
-if (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
- set(HAVE_LIBNSL OFF CACHE BOOL "" FORCE)
-endif()
-
# We don't want to build C# extensions.
set(gRPC_BUILD_CSHARP_EXT OFF)
+set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares)
+set(gRPC_CARES_PROVIDER "clickhouse" CACHE STRING "" FORCE)
add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}")
# The contrib/grpc/CMakeLists.txt redefined the PROTOBUF_GENERATE_GRPC_CPP() function for its own purposes,
diff --git a/contrib/llvm-cmake/CMakeLists.txt b/contrib/llvm-cmake/CMakeLists.txt
index a108e6537c9..4a4a5cef62e 100644
--- a/contrib/llvm-cmake/CMakeLists.txt
+++ b/contrib/llvm-cmake/CMakeLists.txt
@@ -93,6 +93,18 @@ set (CMAKE_CXX_STANDARD 17)
set (LLVM_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm")
set (LLVM_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm")
add_subdirectory ("${LLVM_SOURCE_DIR}" "${LLVM_BINARY_DIR}")
+set_directory_properties (PROPERTIES
+ # due to llvm crosscompile cmake does not know how to clean it, and on clean
+ # will lead to the following error:
+ #
+ # ninja: error: remove(contrib/llvm/llvm/NATIVE): Directory not empty
+ #
+ ADDITIONAL_CLEAN_FILES "${LLVM_BINARY_DIR}"
+ # llvm's cmake configuring this file only when cmake runs,
+ # and after clean cmake will not know that it should re-run,
+ # add explicitly depends from llvm-config.h
+ CMAKE_CONFIGURE_DEPENDS "${LLVM_BINARY_DIR}/include/llvm/Config/llvm-config.h"
+)
add_library (_llvm INTERFACE)
target_link_libraries (_llvm INTERFACE ${REQUIRED_LLVM_LIBRARIES})
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 2bbdd978e5e..6b8109a15b2 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -135,6 +135,7 @@ function clone_submodules
contrib/replxx
contrib/wyhash
contrib/hashidsxx
+ contrib/c-ares
)
git submodule sync
diff --git a/docker/test/integration/runner/compose/docker_compose_coredns.yml b/docker/test/integration/runner/compose/docker_compose_coredns.yml
new file mode 100644
index 00000000000..b329d4e0a46
--- /dev/null
+++ b/docker/test/integration/runner/compose/docker_compose_coredns.yml
@@ -0,0 +1,9 @@
+version: "2.3"
+
+services:
+ coredns:
+ image: coredns/coredns:latest
+ restart: always
+ volumes:
+ - ${COREDNS_CONFIG_DIR}/example.com:/example.com
+ - ${COREDNS_CONFIG_DIR}/Corefile:/Corefile
diff --git a/docker/test/stress/stress b/docker/test/stress/stress
index ab25d13695b..6d90b9d5437 100755
--- a/docker/test/stress/stress
+++ b/docker/test/stress/stress
@@ -46,6 +46,9 @@ def get_options(i, backward_compatibility_check):
if i == 13:
client_options.append("memory_tracker_fault_probability=0.001")
+ if i % 2 == 1 and not backward_compatibility_check:
+ client_options.append("group_by_use_nulls=1")
+
if client_options:
options.append(" --client-option " + " ".join(client_options))
diff --git a/docs/README.md b/docs/README.md
index b328a3ee125..fa8b6bed85c 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -38,9 +38,9 @@ Writing the docs is extremely useful for project's users and developers, and gro
The documentation contains information about all the aspects of the ClickHouse lifecycle: developing, testing, installing, operating, and using. The base language of the documentation is English. The English version is the most actual. All other languages are supported as much as they can by contributors from different countries.
-At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, Chinese, Japanese. We store the documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs).
+At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, and Chinese. We store the reference documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs), and user guides in a separate repo [Clickhouse/clickhouse-docs](https://github.com/ClickHouse/clickhouse-docs).
-Each language lays in the corresponding folder. Files that are not translated from English are the symbolic links to the English ones.
+Each language lies in the corresponding folder. Files that are not translated from English are symbolic links to the English ones.
@@ -48,9 +48,9 @@ Each language lays in the corresponding folder. Files that are not translated fr
You can contribute to the documentation in many ways, for example:
-- Fork the ClickHouse repository, edit, commit, push, and open a pull request.
+- Fork the ClickHouse and ClickHouse-docs repositories, edit, commit, push, and open a pull request.
- Add the `documentation` label to this pull request for proper automatic checks applying. If you have no permissions for adding labels, the reviewer of your PR adds it.
+ Add the `pr-documentation` label to this pull request for proper automatic checks applying. If you do not have permission to add labels, then the reviewer of your PR will add it.
- Open a required file in the ClickHouse repository and edit it from the GitHub web interface.
@@ -158,15 +158,15 @@ When everything is ready, we will add the new language to the website.
-### Documentation for Different Audience
+### Documentation for Different Audiences
-When writing documentation, think about people who read it. Each audience has specific requirements for terms they use in communications.
+When writing documentation, think about the people who read it. Each audience has specific requirements for terms they use in communications.
-ClickHouse documentation can be divided by the audience for the following parts:
+ClickHouse documentation can be divided up by the audience for the following parts:
-- Conceptual topics in [Introduction](https://clickhouse.com/docs/en/), tutorials and overviews, changelog.
+- Conceptual topics like tutorials and overviews.
- These topics are for the most common auditory. When editing text in them, use the most common terms that are comfortable for the audience with basic technical skills.
+ These topics are for the most common audience. When editing text in them, use the most common terms that are comfortable for the audience with basic technical skills.
- Query language reference and related topics.
diff --git a/docs/en/development/build.md b/docs/en/development/build.md
index dbb90f8e537..e12884b61c4 100644
--- a/docs/en/development/build.md
+++ b/docs/en/development/build.md
@@ -75,7 +75,7 @@ This will create the `programs/clickhouse` executable, which can be used with `c
The build requires the following components:
- Git (is used only to checkout the sources, it’s not needed for the build)
-- CMake 3.14 or newer
+- CMake 3.15 or newer
- Ninja
- C++ compiler: clang-14 or newer
- Linker: lld
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 9f66d5d29a9..ed1f139f482 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -3329,6 +3329,15 @@ Read more about [memory overcommit](memory-overcommit.md).
Default value: `1GiB`.
+## compatibility {#compatibility}
+
+This setting changes other settings according to provided ClickHouse version.
+If a behaviour in ClickHouse was changed by using a different default value for some setting, this compatibility setting allows you to use default values from previous versions for all the settings that were not set by the user.
+
+This setting takes ClickHouse version number as a string, like `21.3`, `21.8`. Empty value means that this setting is disabled.
+
+Disabled by default.
+
# Format settings {#format-settings}
## input_format_skip_unknown_fields {#input_format_skip_unknown_fields}
diff --git a/docs/tools/README.md b/docs/tools/README.md
index 163600804c6..7cf3540d108 100644
--- a/docs/tools/README.md
+++ b/docs/tools/README.md
@@ -1,50 +1,94 @@
-## How ClickHouse documentation is generated? {#how-clickhouse-documentation-is-generated}
+## Generating ClickHouse documentation {#how-clickhouse-documentation-is-generated}
-ClickHouse documentation is built using [build.py](build.py) script that uses [mkdocs](https://www.mkdocs.org) library and it’s dependencies to separately build all version of documentations (all languages in either single and multi page mode) as static HTMLs for each single page version. The results are then put in the correct directory structure. It is recommended to use Python 3.7 to run this script.
+ClickHouse documentation is built using [Docusaurus](https://docusaurus.io).
-[release.sh](release.sh) also pulls static files needed for [official ClickHouse website](https://clickhouse.com) from [../../website](../../website) folder then pushes to specified GitHub repo to be served via [GitHub Pages](https://pages.github.com).
+## Check the look of your documentation changes {#how-to-check-if-the-documentation-will-look-fine}
-## How to check if the documentation will look fine? {#how-to-check-if-the-documentation-will-look-fine}
+There are a few options that are all useful depending on how large or complex your edits are.
-There are few options that are all useful depending on how large or complex your edits are.
+### Use the GitHub web interface to edit
-### Use GitHub web interface to edit
+Every page in the docs has an **Edit this page** link that opens the page in the GitHub editor. GitHub has Markdown support with a preview feature. The details of GitHub Markdown and the documentation Markdown are a bit different but generally this is close enough, and the person merging your PR will build the docs and check them.
-GitHub has Markdown support with preview feature, but the details of GitHub Markdown dialect are a bit different in ClickHouse documentation.
+### Install a Markdown editor or plugin for your IDE {#install-markdown-editor-or-plugin-for-your-ide}
-### Install Markdown editor or plugin for your IDE {#install-markdown-editor-or-plugin-for-your-ide}
+Usually, these plugins provide a preview of how the markdown will render, and they catch basic errors like unclosed tags very early.
-Usually those also have some way to preview how Markdown will look like, which allows to catch basic errors like unclosed tags very early.
-### Use build.py {#use-build-py}
+## Build the docs locally {#use-build-py}
-It’ll take some effort to go through, but the result will be very close to production documentation.
+You can build the docs locally. It takes a few minutes to set up, but once you have done it the first time, the process is very simple.
-For the first time you’ll need to:
+### Clone the repos
-#### 1. Set up virtualenv
+The documentation is in two repos, clone both of them:
+- [ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse)
+- [ClickHouse/ClickHouse-docs](https://github.com/ClickHouse/clickhouse-docs)
-``` bash
-$ cd ClickHouse/docs/tools
-$ mkdir venv
-$ virtualenv -p $(which python3) venv
-$ source venv/bin/activate
-$ pip3 install -r requirements.txt
+### Install Node.js
+
+The documentation is built with Docusaurus, which requires Node.js. We recommend version 16. Install [Node.js](https://nodejs.org/en/download/).
+
+### Copy files into place
+
+Docusaurus expects all of the markdown files to be located in the directory tree `clickhouse-docs/docs/`. This is not the way our repos are set up, so some copying of files is needed to build the docs:
+
+```bash
+# from the parent directory of both the ClickHouse/ClickHouse and ClickHouse-clickhouse-docs repos:
+cp -r ClickHouse/docs/en/development clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/engines clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/getting-started clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/interfaces clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/operations clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/sql-reference clickhouse-docs/docs/en/
+
+cp -r ClickHouse/docs/ru/* clickhouse-docs/docs/ru/
+cp -r ClickHouse/docs/zh clickhouse-docs/docs/
```
-#### 2. Run build.py
+#### Note: Symlinks will not work.
+### Setup Docusaurus
-When all prerequisites are installed, running `build.py` without args (there are some, check `build.py --help`) will generate `ClickHouse/docs/build` folder with complete static html website.
+There are two commands that you may need to use with Docusaurus:
+- `yarn install`
+- `yarn start`
-The easiest way to see the result is to use `--livereload=8888` argument of build.py. Alternatively, you can manually launch a HTTP server to serve the docs, for example by running `cd ClickHouse/docs/build && python3 -m http.server 8888`. Then go to http://localhost:8888 in browser. Feel free to use any other port instead of 8888.
+#### Install Docusaurus and its dependencies:
+
+```bash
+cd clickhouse-docs
+yarn install
+```
+
+#### Start a development Docusaurus environment
+
+This command will start Docusaurus in development mode, which means that as you edit source (for example, `.md` files) files the changes will be rendered into HTML files and served by the Docusaurus development server.
+
+```bash
+yarn start
+```
+
+### Make your changes to the markdown files
+
+Edit your files. Remember that if you are editing files in the `ClickHouse/ClickHouse` repo then you should edit them
+in that repo and then copy the edited file into the `ClickHouse/clickhouse-docs/` directory structure so that they are updated in your develoment environment.
+
+`yarn start` probably opened a browser for you when you ran it; if not, open a browser to `http://localhost:3000/docs/en/intro` and navigate to the documentation that you are changing. If you have already made the changes, you can verify them here; if not, make them, and you will see the page update as you save the changes.
## How to change code highlighting? {#how-to-change-code-hl}
-ClickHouse does not use mkdocs `highlightjs` feature. It uses modified pygments styles instead.
-If you want to change code highlighting, edit the `website/css/highlight.css` file.
-Currently, an [eighties](https://github.com/idleberg/base16-pygments/blob/master/css/base16-eighties.dark.css) theme
-is used.
+Code highlighting is based on the language chosen for your code blocks. Specify the language when you start the code block:
+```sql
+SELECT firstname from imdb.actors;
+```
+
+
+```sql
+SELECT firstname from imdb.actors;
+```
+
+If you need a language supported then open an issue in [ClickHouse-docs](https://github.com/ClickHouse/clickhouse-docs/issues).
## How to subscribe on documentation changes? {#how-to-subscribe-on-documentation-changes}
At the moment there’s no easy way to do just that, but you can consider:
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index cf9b7cbafea..584806951cf 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -102,9 +102,34 @@ void Client::processError(const String & query) const
}
+void Client::showWarnings()
+{
+ try
+ {
+ std::vector messages = loadWarningMessages();
+ if (!messages.empty())
+ {
+ std::cout << "Warnings:" << std::endl;
+ for (const auto & message : messages)
+ std::cout << " * " << message << std::endl;
+ std::cout << std::endl;
+ }
+ }
+ catch (...)
+ {
+ /// Ignore exception
+ }
+}
+
/// Make query to get all server warnings
std::vector Client::loadWarningMessages()
{
+ /// Older server versions cannot execute the query loading warnings.
+ constexpr UInt64 min_server_revision_to_load_warnings = DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED;
+
+ if (server_revision < min_server_revision_to_load_warnings)
+ return {};
+
std::vector messages;
connection->sendQuery(connection_parameters.timeouts,
"SELECT * FROM viewIfPermitted(SELECT message FROM system.warnings ELSE null('message String'))",
@@ -226,25 +251,9 @@ try
connect();
- /// Load Warnings at the beginning of connection
+ /// Show warnings at the beginning of connection.
if (is_interactive && !config().has("no-warnings"))
- {
- try
- {
- std::vector messages = loadWarningMessages();
- if (!messages.empty())
- {
- std::cout << "Warnings:" << std::endl;
- for (const auto & message : messages)
- std::cout << " * " << message << std::endl;
- std::cout << std::endl;
- }
- }
- catch (...)
- {
- /// Ignore exception
- }
- }
+ showWarnings();
if (is_interactive && !delayed_interactive)
{
@@ -370,7 +379,7 @@ void Client::connect()
}
server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch);
- load_suggestions = is_interactive && (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false));
+ load_suggestions = is_interactive && (server_revision >= Suggest::MIN_SERVER_REVISION) && !config().getBool("disable_suggestion", false);
if (server_display_name = connection->getServerDisplayName(connection_parameters.timeouts); server_display_name.empty())
server_display_name = config().getString("host", "localhost");
diff --git a/programs/client/Client.h b/programs/client/Client.h
index 164b8e2ebaa..1fec282be51 100644
--- a/programs/client/Client.h
+++ b/programs/client/Client.h
@@ -45,6 +45,7 @@ protected:
private:
void printChangedSettings() const;
+ void showWarnings();
std::vector loadWarningMessages();
};
}
diff --git a/programs/disks/CommandCopy.cpp b/programs/disks/CommandCopy.cpp
index f9cd7444287..1e5852fe651 100644
--- a/programs/disks/CommandCopy.cpp
+++ b/programs/disks/CommandCopy.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandLink.cpp b/programs/disks/CommandLink.cpp
index 6e9a7e64324..af48f0de097 100644
--- a/programs/disks/CommandLink.cpp
+++ b/programs/disks/CommandLink.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandList.cpp b/programs/disks/CommandList.cpp
index 8c6bfac3a9b..e76bb9e65fb 100644
--- a/programs/disks/CommandList.cpp
+++ b/programs/disks/CommandList.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandListDisks.cpp b/programs/disks/CommandListDisks.cpp
index 2bcbb045d67..22cffdd21fd 100644
--- a/programs/disks/CommandListDisks.cpp
+++ b/programs/disks/CommandListDisks.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandMove.cpp b/programs/disks/CommandMove.cpp
index 4a377cc7225..6322cf4b47d 100644
--- a/programs/disks/CommandMove.cpp
+++ b/programs/disks/CommandMove.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandRead.cpp b/programs/disks/CommandRead.cpp
index aa472fa217e..6b77a27e918 100644
--- a/programs/disks/CommandRead.cpp
+++ b/programs/disks/CommandRead.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandRemove.cpp b/programs/disks/CommandRemove.cpp
index d9925fbd93e..c1d3129bb8d 100644
--- a/programs/disks/CommandRemove.cpp
+++ b/programs/disks/CommandRemove.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandWrite.cpp b/programs/disks/CommandWrite.cpp
index c8ae91ea8d5..0b1c5823c81 100644
--- a/programs/disks/CommandWrite.cpp
+++ b/programs/disks/CommandWrite.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/src/Access/Common/AllowedClientHosts.cpp b/src/Access/Common/AllowedClientHosts.cpp
index 85d7065d823..efbdf3924e8 100644
--- a/src/Access/Common/AllowedClientHosts.cpp
+++ b/src/Access/Common/AllowedClientHosts.cpp
@@ -110,18 +110,24 @@ namespace
}
/// Returns the host name by its address.
- String getHostByAddress(const IPAddress & address)
+ Strings getHostsByAddress(const IPAddress & address)
{
- String host = DNSResolver::instance().reverseResolve(address);
+ auto hosts = DNSResolver::instance().reverseResolve(address);
- /// Check that PTR record is resolved back to client address
- if (!isAddressOfHost(address, host))
- throw Exception("Host " + String(host) + " isn't resolved back to " + address.toString(), ErrorCodes::DNS_ERROR);
+ if (hosts.empty())
+ throw Exception(ErrorCodes::DNS_ERROR, "{} could not be resolved", address.toString());
- return host;
+
+ for (const auto & host : hosts)
+ {
+ /// Check that PTR record is resolved back to client address
+ if (!isAddressOfHost(address, host))
+ throw Exception(ErrorCodes::DNS_ERROR, "Host {} isn't resolved back to {}", host, address.toString());
+ }
+
+ return hosts;
}
-
void parseLikePatternIfIPSubnet(const String & pattern, IPSubnet & subnet, IPAddress::Family address_family)
{
size_t slash = pattern.find('/');
@@ -520,20 +526,29 @@ bool AllowedClientHosts::contains(const IPAddress & client_address) const
return true;
/// Check `name_regexps`.
- std::optional resolved_host;
+ std::optional resolved_hosts;
auto check_name_regexp = [&](const String & name_regexp_)
{
try
{
if (boost::iequals(name_regexp_, "localhost"))
return is_client_local();
- if (!resolved_host)
- resolved_host = getHostByAddress(client_v6);
- if (resolved_host->empty())
- return false;
- Poco::RegularExpression re(name_regexp_);
- Poco::RegularExpression::Match match;
- return re.match(*resolved_host, match) != 0;
+ if (!resolved_hosts)
+ {
+ resolved_hosts = getHostsByAddress(client_address);
+ }
+
+ for (const auto & host : resolved_hosts.value())
+ {
+ Poco::RegularExpression re(name_regexp_);
+ Poco::RegularExpression::Match match;
+ if (re.match(host, match) != 0)
+ {
+ return true;
+ }
+ }
+
+ return false;
}
catch (const Exception & e)
{
diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp
index 994abc7b53a..0cbe420f345 100644
--- a/src/Access/DiskAccessStorage.cpp
+++ b/src/Access/DiskAccessStorage.cpp
@@ -15,6 +15,7 @@
#include
#include
#include
+#include
#include
#include
diff --git a/src/AggregateFunctions/parseAggregateFunctionParameters.h b/src/AggregateFunctions/parseAggregateFunctionParameters.h
index a67bc081303..41a04324f6d 100644
--- a/src/AggregateFunctions/parseAggregateFunctionParameters.h
+++ b/src/AggregateFunctions/parseAggregateFunctionParameters.h
@@ -8,6 +8,8 @@
namespace DB
{
+struct Array;
+
Array getAggregateFunctionParametersArray(
const ASTPtr & expression_list,
const std::string & error_context,
diff --git a/src/Backups/registerBackupEnginesFileAndDisk.cpp b/src/Backups/registerBackupEnginesFileAndDisk.cpp
index 050a51939b6..380ae36a8e3 100644
--- a/src/Backups/registerBackupEnginesFileAndDisk.cpp
+++ b/src/Backups/registerBackupEnginesFileAndDisk.cpp
@@ -7,6 +7,7 @@
#include
#include
#include
+#include
namespace DB
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 2424f2c42d8..5c02905172c 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -376,7 +376,7 @@ if (TARGET ch_contrib::rdkafka)
endif()
if (TARGET ch_contrib::nats_io)
- dbms_target_link_libraries(PRIVATE ch_contrib::nats_io)
+ dbms_target_link_libraries(PRIVATE ch_contrib::nats_io ch_contrib::uv)
endif()
if (TARGET ch_contrib::sasl2)
@@ -447,6 +447,9 @@ if (TARGET ch_contrib::avrocpp)
dbms_target_link_libraries(PRIVATE ch_contrib::avrocpp)
endif ()
+set_source_files_properties(Common/CaresPTRResolver.cpp PROPERTIES COMPILE_FLAGS -Wno-reserved-identifier)
+target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::c-ares)
+
if (TARGET OpenSSL::Crypto)
dbms_target_link_libraries (PRIVATE OpenSSL::Crypto)
target_link_libraries (clickhouse_common_io PRIVATE OpenSSL::Crypto)
diff --git a/src/Client/Suggest.h b/src/Client/Suggest.h
index 65b60ceffc4..25d45f7ffaf 100644
--- a/src/Client/Suggest.h
+++ b/src/Client/Suggest.h
@@ -28,8 +28,8 @@ public:
template
void load(ContextPtr context, const ConnectionParameters & connection_parameters, Int32 suggestion_limit);
- /// Older server versions cannot execute the query above.
- static constexpr int MIN_SERVER_REVISION = 54406;
+ /// Older server versions cannot execute the query loading suggestions.
+ static constexpr int MIN_SERVER_REVISION = DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED;
private:
void fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query);
diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp
index d8e98ec9406..8d61f6e726a 100644
--- a/src/Columns/ColumnNullable.cpp
+++ b/src/Columns/ColumnNullable.cpp
@@ -793,4 +793,18 @@ ColumnPtr makeNullable(const ColumnPtr & column)
return ColumnNullable::create(column, ColumnUInt8::create(column->size(), 0));
}
+ColumnPtr makeNullableSafe(const ColumnPtr & column)
+{
+ if (isColumnNullable(*column))
+ return column;
+
+ if (isColumnConst(*column))
+ return ColumnConst::create(makeNullableSafe(assert_cast(*column).getDataColumnPtr()), column->size());
+
+ if (column->canBeInsideNullable())
+ return makeNullable(column);
+
+ return column;
+}
+
}
diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h
index 52e57f7f0d0..e832f6d20e5 100644
--- a/src/Columns/ColumnNullable.h
+++ b/src/Columns/ColumnNullable.h
@@ -223,5 +223,6 @@ private:
};
ColumnPtr makeNullable(const ColumnPtr & column);
+ColumnPtr makeNullableSafe(const ColumnPtr & column);
}
diff --git a/src/Common/CaresPTRResolver.cpp b/src/Common/CaresPTRResolver.cpp
new file mode 100644
index 00000000000..f6228e97c02
--- /dev/null
+++ b/src/Common/CaresPTRResolver.cpp
@@ -0,0 +1,109 @@
+#include "CaresPTRResolver.h"
+#include
+#include
+#include
+#include "ares.h"
+#include "netdb.h"
+
+namespace DB
+{
+
+ namespace ErrorCodes
+ {
+ extern const int DNS_ERROR;
+ }
+
+ static void callback(void * arg, int status, int, struct hostent * host)
+ {
+ auto * ptr_records = reinterpret_cast*>(arg);
+ if (status == ARES_SUCCESS && host->h_aliases)
+ {
+ int i = 0;
+ while (auto * ptr_record = host->h_aliases[i])
+ {
+ ptr_records->emplace_back(ptr_record);
+ i++;
+ }
+ }
+ }
+
+ CaresPTRResolver::CaresPTRResolver(CaresPTRResolver::provider_token) : channel(nullptr)
+ {
+ /*
+ * ares_library_init is not thread safe. Currently, the only other usage of c-ares seems to be in grpc.
+ * In grpc, ares_library_init seems to be called only in Windows.
+ * See https://github.com/grpc/grpc/blob/master/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc#L1187
+ * That means it's safe to init it here, but we should be cautious when introducing new code that depends on c-ares and even updates
+ * to grpc. As discussed in https://github.com/ClickHouse/ClickHouse/pull/37827#discussion_r919189085, c-ares should be adapted to be atomic
+ * */
+ if (ares_library_init(ARES_LIB_INIT_ALL) != ARES_SUCCESS || ares_init(&channel) != ARES_SUCCESS)
+ {
+ throw DB::Exception("Failed to initialize c-ares", DB::ErrorCodes::DNS_ERROR);
+ }
+ }
+
+ CaresPTRResolver::~CaresPTRResolver()
+ {
+ ares_destroy(channel);
+ ares_library_cleanup();
+ }
+
+ std::vector CaresPTRResolver::resolve(const std::string & ip)
+ {
+ std::vector ptr_records;
+
+ resolve(ip, ptr_records);
+ wait();
+
+ return ptr_records;
+ }
+
+ std::vector CaresPTRResolver::resolve_v6(const std::string & ip)
+ {
+ std::vector ptr_records;
+
+ resolve_v6(ip, ptr_records);
+ wait();
+
+ return ptr_records;
+ }
+
+ void CaresPTRResolver::resolve(const std::string & ip, std::vector & response)
+ {
+ in_addr addr;
+
+ inet_pton(AF_INET, ip.c_str(), &addr);
+
+ ares_gethostbyaddr(channel, reinterpret_cast(&addr), sizeof(addr), AF_INET, callback, &response);
+ }
+
+ void CaresPTRResolver::resolve_v6(const std::string & ip, std::vector & response)
+ {
+ in6_addr addr;
+ inet_pton(AF_INET6, ip.c_str(), &addr);
+
+ ares_gethostbyaddr(channel, reinterpret_cast(&addr), sizeof(addr), AF_INET6, callback, &response);
+ }
+
+ void CaresPTRResolver::wait()
+ {
+ timeval * tvp, tv;
+ fd_set read_fds;
+ fd_set write_fds;
+ int nfds;
+
+ for (;;)
+ {
+ FD_ZERO(&read_fds);
+ FD_ZERO(&write_fds);
+ nfds = ares_fds(channel, &read_fds,&write_fds);
+ if (nfds == 0)
+ {
+ break;
+ }
+ tvp = ares_timeout(channel, nullptr, &tv);
+ select(nfds, &read_fds, &write_fds, nullptr, tvp);
+ ares_process(channel, &read_fds, &write_fds);
+ }
+ }
+}
diff --git a/src/Common/CaresPTRResolver.h b/src/Common/CaresPTRResolver.h
new file mode 100644
index 00000000000..fd6a1cf7bc5
--- /dev/null
+++ b/src/Common/CaresPTRResolver.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#include "DNSPTRResolver.h"
+
+using ares_channel = struct ares_channeldata *;
+
+namespace DB
+{
+
+ /*
+ * Implements reverse DNS resolution using c-ares lib. System reverse DNS resolution via
+ * gethostbyaddr or getnameinfo does not work reliably because in some systems
+ * it returns all PTR records for a given IP and in others it returns only one.
+ * */
+ class CaresPTRResolver : public DNSPTRResolver
+ {
+ friend class DNSPTRResolverProvider;
+
+ /*
+ * Allow only DNSPTRProvider to instantiate this class
+ * */
+ struct provider_token {};
+
+ public:
+ explicit CaresPTRResolver(provider_token);
+ ~CaresPTRResolver() override;
+
+ std::vector resolve(const std::string & ip) override;
+
+ std::vector resolve_v6(const std::string & ip) override;
+
+ private:
+ void wait();
+
+ void resolve(const std::string & ip, std::vector & response);
+
+ void resolve_v6(const std::string & ip, std::vector & response);
+
+ ares_channel channel;
+ };
+}
+
diff --git a/src/Common/DNSPTRResolver.h b/src/Common/DNSPTRResolver.h
new file mode 100644
index 00000000000..e6cce83f79d
--- /dev/null
+++ b/src/Common/DNSPTRResolver.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include
+#include
+
+namespace DB
+{
+ struct DNSPTRResolver
+ {
+
+ virtual ~DNSPTRResolver() = default;
+
+ virtual std::vector resolve(const std::string & ip) = 0;
+
+ virtual std::vector resolve_v6(const std::string & ip) = 0;
+
+ };
+}
diff --git a/src/Common/DNSPTRResolverProvider.cpp b/src/Common/DNSPTRResolverProvider.cpp
new file mode 100644
index 00000000000..41c73f4f36f
--- /dev/null
+++ b/src/Common/DNSPTRResolverProvider.cpp
@@ -0,0 +1,13 @@
+#include "DNSPTRResolverProvider.h"
+#include "CaresPTRResolver.h"
+
+namespace DB
+{
+ std::shared_ptr DNSPTRResolverProvider::get()
+ {
+ static auto cares_resolver = std::make_shared(
+ CaresPTRResolver::provider_token {}
+ );
+ return cares_resolver;
+ }
+}
diff --git a/src/Common/DNSPTRResolverProvider.h b/src/Common/DNSPTRResolverProvider.h
new file mode 100644
index 00000000000..a7f534749e3
--- /dev/null
+++ b/src/Common/DNSPTRResolverProvider.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include
+#include "DNSPTRResolver.h"
+
+namespace DB
+{
+ /*
+ * Provides a ready-to-use DNSPTRResolver instance.
+ * It hides 3rd party lib dependencies, handles initialization and lifetime.
+ * Since `get` function is static, it can be called from any context. Including cached static functions.
+ * */
+ class DNSPTRResolverProvider
+ {
+ public:
+ static std::shared_ptr get();
+ };
+}
diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp
index 0616e324b73..10797b7a809 100644
--- a/src/Common/DNSResolver.cpp
+++ b/src/Common/DNSResolver.cpp
@@ -12,6 +12,7 @@
#include
#include
#include
+#include "DNSPTRResolverProvider.h"
namespace ProfileEvents
{
@@ -138,16 +139,17 @@ static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
return addresses;
}
-static String reverseResolveImpl(const Poco::Net::IPAddress & address)
+static Strings reverseResolveImpl(const Poco::Net::IPAddress & address)
{
- Poco::Net::SocketAddress sock_addr(address, 0);
+ auto ptr_resolver = DB::DNSPTRResolverProvider::get();
- /// Resolve by hand, because Poco::Net::DNS::hostByAddress(...) does getaddrinfo(...) after getnameinfo(...)
- char host[1024];
- int err = getnameinfo(sock_addr.addr(), sock_addr.length(), host, sizeof(host), nullptr, 0, NI_NAMEREQD);
- if (err)
- throw Exception("Cannot getnameinfo(" + address.toString() + "): " + gai_strerror(err), ErrorCodes::DNS_ERROR);
- return host;
+ if (address.family() == Poco::Net::IPAddress::Family::IPv4)
+ {
+ return ptr_resolver->resolve(address.toString());
+ } else
+ {
+ return ptr_resolver->resolve_v6(address.toString());
+ }
}
struct DNSResolver::Impl
@@ -235,7 +237,7 @@ std::vector DNSResolver::resolveAddressList(const std:
return addresses;
}
-String DNSResolver::reverseResolve(const Poco::Net::IPAddress & address)
+Strings DNSResolver::reverseResolve(const Poco::Net::IPAddress & address)
{
if (impl->disable_cache)
return reverseResolveImpl(address);
diff --git a/src/Common/DNSResolver.h b/src/Common/DNSResolver.h
index fdd9799f96f..84c88586636 100644
--- a/src/Common/DNSResolver.h
+++ b/src/Common/DNSResolver.h
@@ -36,8 +36,8 @@ public:
std::vector resolveAddressList(const std::string & host, UInt16 port);
- /// Accepts host IP and resolves its host name
- String reverseResolve(const Poco::Net::IPAddress & address);
+ /// Accepts host IP and resolves its host names
+ Strings reverseResolve(const Poco::Net::IPAddress & address);
/// Get this server host name
String getHostName();
diff --git a/src/Common/LRUFileCache.cpp b/src/Common/LRUFileCache.cpp
index 0ce76dbdec6..6306b6de059 100644
--- a/src/Common/LRUFileCache.cpp
+++ b/src/Common/LRUFileCache.cpp
@@ -45,7 +45,7 @@ void LRUFileCache::initialize()
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
- return;
+ throw;
}
}
else
@@ -841,7 +841,11 @@ void LRUFileCache::loadCacheInfoIntoMemory(std::lock_guard & cache_l
/// cache_base_path / key_prefix / key / offset
if (!files.empty())
- throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache already initialized");
+ throw Exception(
+ ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
+ "Cache initialization is partially made. "
+ "This can be a result of a failed first attempt to initialize cache. "
+ "Please, check log for error messages");
fs::directory_iterator key_prefix_it{cache_base_path};
for (; key_prefix_it != fs::directory_iterator(); ++key_prefix_it)
diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp
index cfc364929a3..60efab69433 100644
--- a/src/Common/OptimizedRegularExpression.cpp
+++ b/src/Common/OptimizedRegularExpression.cpp
@@ -342,6 +342,23 @@ OptimizedRegularExpressionImpl::OptimizedRegularExpressionImpl(cons
}
}
+template
+OptimizedRegularExpressionImpl::OptimizedRegularExpressionImpl(OptimizedRegularExpressionImpl && rhs) noexcept
+ : is_trivial(rhs.is_trivial)
+ , required_substring_is_prefix(rhs.required_substring_is_prefix)
+ , is_case_insensitive(rhs.is_case_insensitive)
+ , required_substring(std::move(rhs.required_substring))
+ , re2(std::move(rhs.re2))
+ , number_of_subpatterns(rhs.number_of_subpatterns)
+{
+ if (!required_substring.empty())
+ {
+ if (is_case_insensitive)
+ case_insensitive_substring_searcher.emplace(required_substring.data(), required_substring.size());
+ else
+ case_sensitive_substring_searcher.emplace(required_substring.data(), required_substring.size());
+ }
+}
template
bool OptimizedRegularExpressionImpl::match(const char * subject, size_t subject_size) const
diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h
index eaa7b06e309..dad8706a50d 100644
--- a/src/Common/OptimizedRegularExpression.h
+++ b/src/Common/OptimizedRegularExpression.h
@@ -56,6 +56,9 @@ public:
using StringPieceType = std::conditional_t;
OptimizedRegularExpressionImpl(const std::string & regexp_, int options = 0); /// NOLINT
+ /// StringSearcher store pointers to required_substring, it must be updated on move.
+ OptimizedRegularExpressionImpl(OptimizedRegularExpressionImpl && rhs) noexcept;
+ OptimizedRegularExpressionImpl(const OptimizedRegularExpressionImpl & rhs) = delete;
bool match(const std::string & subject) const
{
diff --git a/src/Common/ShellCommand.h b/src/Common/ShellCommand.h
index 190b5bc664e..dfc4a826f62 100644
--- a/src/Common/ShellCommand.h
+++ b/src/Common/ShellCommand.h
@@ -3,6 +3,7 @@
#include
#include
#include
+#include
namespace DB
diff --git a/src/Common/tests/gtest_lsan.cpp b/src/Common/tests/gtest_lsan.cpp
deleted file mode 100644
index f6e1984ec58..00000000000
--- a/src/Common/tests/gtest_lsan.cpp
+++ /dev/null
@@ -1,33 +0,0 @@
-#include // ADDRESS_SANITIZER
-
-#ifdef ADDRESS_SANITIZER
-
-#include
-#include
-
-#include
-#include
-
-/// Test that ensures that LSan works.
-///
-/// Regression test for the case when it may not work,
-/// because of broken getauxval() [1].
-///
-/// [1]: https://github.com/ClickHouse/ClickHouse/pull/33957
-TEST(Common, LSan)
-{
- int sanitizers_exit_code = 1;
-
- ASSERT_EXIT({
- std::thread leak_in_thread([]()
- {
- void * leak = malloc(4096);
- ASSERT_NE(leak, nullptr);
- });
- leak_in_thread.join();
-
- __lsan_do_leak_check();
- }, ::testing::ExitedWithCode(sanitizers_exit_code), ".*LeakSanitizer: detected memory leaks.*");
-}
-
-#endif
diff --git a/src/Coordination/CoordinationSettings.cpp b/src/Coordination/CoordinationSettings.cpp
index 34d69967828..4733adcf67a 100644
--- a/src/Coordination/CoordinationSettings.cpp
+++ b/src/Coordination/CoordinationSettings.cpp
@@ -1,5 +1,4 @@
#include
-#include
#include
#include
#include
diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp
index 7c6ed227a06..8261f5d1e26 100644
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@@ -21,6 +21,7 @@
#include
#include
#include
+#include
namespace DB
{
@@ -111,7 +112,7 @@ KeeperServer::KeeperServer(
configuration_and_settings_->snapshot_storage_path,
coordination_settings,
checkAndGetSuperdigest(configuration_and_settings_->super_digest),
- config.getBool("keeper_server.digest_enabled", true)))
+ config.getBool("keeper_server.digest_enabled", false)))
, state_manager(nuraft::cs_new(
server_id, "keeper_server", configuration_and_settings_->log_storage_path, configuration_and_settings_->state_file_path, config, coordination_settings))
, log(&Poco::Logger::get("KeeperServer"))
diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h
index 3638a036098..7b56367769e 100644
--- a/src/Core/BaseSettings.h
+++ b/src/Core/BaseSettings.h
@@ -43,9 +43,16 @@ class BaseSettings : public TTraits::Data
{
using CustomSettingMap = std::unordered_map, SettingFieldCustom>>;
public:
+ BaseSettings() = default;
+ BaseSettings(const BaseSettings &) = default;
+ BaseSettings(BaseSettings &&) noexcept = default;
+ BaseSettings & operator=(const BaseSettings &) = default;
+ BaseSettings & operator=(BaseSettings &&) noexcept = default;
+ virtual ~BaseSettings() = default;
+
using Traits = TTraits;
- void set(std::string_view name, const Field & value);
+ virtual void set(std::string_view name, const Field & value);
Field get(std::string_view name) const;
void setString(std::string_view name, const String & value);
@@ -62,6 +69,8 @@ public:
/// Resets all the settings to their default values.
void resetToDefault();
+ /// Resets specified setting to its default value.
+ void resetToDefault(std::string_view name);
bool has(std::string_view name) const { return hasBuiltin(name) || hasCustom(name); }
static bool hasBuiltin(std::string_view name);
@@ -315,6 +324,14 @@ void BaseSettings::resetToDefault()
custom_settings_map.clear();
}
+template
+void BaseSettings::resetToDefault(std::string_view name)
+{
+ const auto & accessor = Traits::Accessor::instance();
+ if (size_t index = accessor.find(name); index != static_cast(-1))
+ accessor.resetValueToDefault(*this, index);
+}
+
template
bool BaseSettings::hasBuiltin(std::string_view name)
{
diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h
index 2df48a79776..584720694d7 100644
--- a/src/Core/ProtocolDefines.h
+++ b/src/Core/ProtocolDefines.h
@@ -52,8 +52,10 @@
/// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION,
/// later is just a number for server version (one number instead of commit SHA)
/// for simplicity (sometimes it may be more convenient in some use cases).
-#define DBMS_TCP_PROTOCOL_VERSION 54456
+#define DBMS_TCP_PROTOCOL_VERSION 54457
#define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449
#define DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT 54456
+
+#define DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED 54457
diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp
index 5251569505e..7bac3f04fc6 100644
--- a/src/Core/Settings.cpp
+++ b/src/Core/Settings.cpp
@@ -1,5 +1,6 @@
#include "Settings.h"
+#include
#include
#include
#include
@@ -145,6 +146,53 @@ std::vector Settings::getAllRegisteredNames() const
return all_settings;
}
+void Settings::set(std::string_view name, const Field & value)
+{
+ BaseSettings::set(name, value);
+
+ if (name == "compatibility")
+ applyCompatibilitySetting();
+ /// If we change setting that was changed by compatibility setting before
+ /// we should remove it from settings_changed_by_compatibility_setting,
+ /// otherwise the next time we will change compatibility setting
+ /// this setting will be changed too (and we don't want it).
+ else if (settings_changed_by_compatibility_setting.contains(name))
+ settings_changed_by_compatibility_setting.erase(name);
+}
+
+void Settings::applyCompatibilitySetting()
+{
+ /// First, revert all changes applied by previous compatibility setting
+ for (const auto & setting_name : settings_changed_by_compatibility_setting)
+ resetToDefault(setting_name);
+
+ settings_changed_by_compatibility_setting.clear();
+ String compatibility = getString("compatibility");
+ /// If setting value is empty, we don't need to change settings
+ if (compatibility.empty())
+ return;
+
+ ClickHouseVersion version(compatibility);
+ /// Iterate through ClickHouse version in descending order and apply reversed
+ /// changes for each version that is higher that version from compatibility setting
+ for (auto it = settings_changes_history.rbegin(); it != settings_changes_history.rend(); ++it)
+ {
+ if (version >= it->first)
+ break;
+
+ /// Apply reversed changes from this version.
+ for (const auto & change : it->second)
+ {
+ /// If this setting was changed manually, we don't change it
+ if (isChanged(change.name) && !settings_changed_by_compatibility_setting.contains(change.name))
+ continue;
+
+ BaseSettings::set(change.name, change.previous_value);
+ settings_changed_by_compatibility_setting.insert(change.name);
+ }
+ }
+}
+
IMPLEMENT_SETTINGS_TRAITS(FormatFactorySettingsTraits, FORMAT_FACTORY_SETTINGS)
}
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 0ad23f719fb..4e8e21db0dc 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -35,6 +35,10 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
*
* `flags` can be either 0 or IMPORTANT.
* A setting is "IMPORTANT" if it affects the results of queries and can't be ignored by older versions.
+ *
+ * When adding new settings that control some backward incompatible changes or when changing some settings values,
+ * consider adding them to settings changes history in SettingsChangesHistory.h for special `compatibility` setting
+ * to work correctly.
*/
#define COMMON_SETTINGS(M) \
@@ -132,6 +136,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \
M(Bool, enable_positional_arguments, true, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \
\
+ M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \
+ \
M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \
M(UInt64, parallel_replicas_count, 0, "", 0) \
M(UInt64, parallel_replica_offset, 0, "", 0) \
@@ -599,6 +605,11 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \
M(Bool, allow_deprecated_syntax_for_merge_tree, false, "Allow to create *MergeTree tables with deprecated engine definition syntax", 0) \
\
+ M(String, compatibility, "", "Changes other settings according to provided ClickHouse version. If we know that we changed some behaviour in ClickHouse by changing some settings in some version, this compatibility setting will control these settings", 0) \
+ \
+ M(Map, additional_table_filters, "", "Additional filter expression which would be applied after reading from specified table. Syntax: {'table1': 'expression', 'database.table2': 'expression'}", 0) \
+ M(String, additional_result_filter, "", "Additional filter expression which would be applied to query result", 0) \
+ \
/** Experimental functions */ \
M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \
M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \
@@ -652,7 +663,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
#define FORMAT_FACTORY_SETTINGS(M) \
M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \
- M(Bool, format_csv_allow_single_quotes, true, "If it is set to true, allow strings in single quotes.", 0) \
+ M(Bool, format_csv_allow_single_quotes, false, "If it is set to true, allow strings in single quotes.", 0) \
M(Bool, format_csv_allow_double_quotes, true, "If it is set to true, allow strings in double quotes.", 0) \
M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \
M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices", 0) \
@@ -762,7 +773,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(Bool, output_format_pretty_row_numbers, false, "Add row numbers before each row for pretty output format", 0) \
M(Bool, insert_distributed_one_random_shard, false, "If setting is enabled, inserting into distributed table will choose a random shard to write when there is no sharding key", 0) \
\
- M(UInt64, cross_to_inner_join_rewrite, 2, "Use inner join instead of comma/cross join if possible. Possible values: 0 - no rewrite, 1 - apply if possible for comma/cross, 2 - force rewrite all comma joins, cross - if possible", 0) \
+ M(UInt64, cross_to_inner_join_rewrite, 1, "Use inner join instead of comma/cross join if there're joining expressions in the WHERE section. Values: 0 - no rewrite, 1 - apply if possible for comma/cross, 2 - force rewrite all comma joins, cross - if possible", 0) \
\
M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \
M(Bool, output_format_arrow_string_as_string, false, "Use Arrow String type instead of Binary for String columns", 0) \
@@ -829,6 +840,13 @@ struct Settings : public BaseSettings, public IHints<2, Settings
void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field);
void addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field);
+
+ void set(std::string_view name, const Field & value) override;
+
+private:
+ void applyCompatibilitySetting();
+
+ std::unordered_set settings_changed_by_compatibility_setting;
};
/*
diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h
new file mode 100644
index 00000000000..ba60fb99308
--- /dev/null
+++ b/src/Core/SettingsChangesHistory.h
@@ -0,0 +1,114 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include