" > /test_output/bc_check_error_messages.txt \
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
diff --git a/docker/test/stress/stress b/docker/test/stress/stress
index ab25d13695b..6d90b9d5437 100755
--- a/docker/test/stress/stress
+++ b/docker/test/stress/stress
@@ -46,6 +46,9 @@ def get_options(i, backward_compatibility_check):
if i == 13:
client_options.append("memory_tracker_fault_probability=0.001")
+ if i % 2 == 1 and not backward_compatibility_check:
+ client_options.append("group_by_use_nulls=1")
+
if client_options:
options.append(" --client-option " + " ".join(client_options))
diff --git a/docs/README.md b/docs/README.md
index b328a3ee125..fa8b6bed85c 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -38,9 +38,9 @@ Writing the docs is extremely useful for project's users and developers, and gro
The documentation contains information about all the aspects of the ClickHouse lifecycle: developing, testing, installing, operating, and using. The base language of the documentation is English. The English version is the most actual. All other languages are supported as much as they can by contributors from different countries.
-At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, Chinese, Japanese. We store the documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs).
+At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, and Chinese. We store the reference documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs), and user guides in a separate repo [Clickhouse/clickhouse-docs](https://github.com/ClickHouse/clickhouse-docs).
-Each language lays in the corresponding folder. Files that are not translated from English are the symbolic links to the English ones.
+Each language lies in the corresponding folder. Files that are not translated from English are symbolic links to the English ones.
@@ -48,9 +48,9 @@ Each language lays in the corresponding folder. Files that are not translated fr
You can contribute to the documentation in many ways, for example:
-- Fork the ClickHouse repository, edit, commit, push, and open a pull request.
+- Fork the ClickHouse and ClickHouse-docs repositories, edit, commit, push, and open a pull request.
- Add the `documentation` label to this pull request for proper automatic checks applying. If you have no permissions for adding labels, the reviewer of your PR adds it.
+ Add the `pr-documentation` label to this pull request for proper automatic checks applying. If you do not have permission to add labels, then the reviewer of your PR will add it.
- Open a required file in the ClickHouse repository and edit it from the GitHub web interface.
@@ -158,15 +158,15 @@ When everything is ready, we will add the new language to the website.
-### Documentation for Different Audience
+### Documentation for Different Audiences
-When writing documentation, think about people who read it. Each audience has specific requirements for terms they use in communications.
+When writing documentation, think about the people who read it. Each audience has specific requirements for terms they use in communications.
-ClickHouse documentation can be divided by the audience for the following parts:
+ClickHouse documentation can be divided up by the audience for the following parts:
-- Conceptual topics in [Introduction](https://clickhouse.com/docs/en/), tutorials and overviews, changelog.
+- Conceptual topics like tutorials and overviews.
- These topics are for the most common auditory. When editing text in them, use the most common terms that are comfortable for the audience with basic technical skills.
+ These topics are for the most common audience. When editing text in them, use the most common terms that are comfortable for the audience with basic technical skills.
- Query language reference and related topics.
diff --git a/docs/en/development/build.md b/docs/en/development/build.md
index dbb90f8e537..e12884b61c4 100644
--- a/docs/en/development/build.md
+++ b/docs/en/development/build.md
@@ -75,7 +75,7 @@ This will create the `programs/clickhouse` executable, which can be used with `c
The build requires the following components:
- Git (is used only to checkout the sources, it’s not needed for the build)
-- CMake 3.14 or newer
+- CMake 3.15 or newer
- Ninja
- C++ compiler: clang-14 or newer
- Linker: lld
diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index 5d8ed9cdacd..e499849426b 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -1632,6 +1632,8 @@ kafka_topic_list = 'topic1',
kafka_group_name = 'group1',
kafka_format = 'AvroConfluent';
+-- for debug purposes you can set format_avro_schema_registry_url in a session.
+-- this way cannot be used in production
SET format_avro_schema_registry_url = 'http://schema-registry';
SELECT * FROM topic1_stream;
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 75c2aa57b32..ed1f139f482 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -2626,7 +2626,7 @@ Possible values:
- Any positive integer.
- 0 - Disabled (infinite timeout).
-Default value: 1800.
+Default value: 180.
## http_receive_timeout {#http_receive_timeout}
@@ -2637,7 +2637,7 @@ Possible values:
- Any positive integer.
- 0 - Disabled (infinite timeout).
-Default value: 1800.
+Default value: 180.
## check_query_single_value_result {#check_query_single_value_result}
@@ -3329,6 +3329,15 @@ Read more about [memory overcommit](memory-overcommit.md).
Default value: `1GiB`.
+## compatibility {#compatibility}
+
+This setting changes other settings according to provided ClickHouse version.
+If a behaviour in ClickHouse was changed by using a different default value for some setting, this compatibility setting allows you to use default values from previous versions for all the settings that were not set by the user.
+
+This setting takes ClickHouse version number as a string, like `21.3`, `21.8`. Empty value means that this setting is disabled.
+
+Disabled by default.
+
# Format settings {#format-settings}
## input_format_skip_unknown_fields {#input_format_skip_unknown_fields}
diff --git a/docs/tools/README.md b/docs/tools/README.md
index 163600804c6..7cf3540d108 100644
--- a/docs/tools/README.md
+++ b/docs/tools/README.md
@@ -1,50 +1,94 @@
-## How ClickHouse documentation is generated? {#how-clickhouse-documentation-is-generated}
+## Generating ClickHouse documentation {#how-clickhouse-documentation-is-generated}
-ClickHouse documentation is built using [build.py](build.py) script that uses [mkdocs](https://www.mkdocs.org) library and it’s dependencies to separately build all version of documentations (all languages in either single and multi page mode) as static HTMLs for each single page version. The results are then put in the correct directory structure. It is recommended to use Python 3.7 to run this script.
+ClickHouse documentation is built using [Docusaurus](https://docusaurus.io).
-[release.sh](release.sh) also pulls static files needed for [official ClickHouse website](https://clickhouse.com) from [../../website](../../website) folder then pushes to specified GitHub repo to be served via [GitHub Pages](https://pages.github.com).
+## Check the look of your documentation changes {#how-to-check-if-the-documentation-will-look-fine}
-## How to check if the documentation will look fine? {#how-to-check-if-the-documentation-will-look-fine}
+There are a few options that are all useful depending on how large or complex your edits are.
-There are few options that are all useful depending on how large or complex your edits are.
+### Use the GitHub web interface to edit
-### Use GitHub web interface to edit
+Every page in the docs has an **Edit this page** link that opens the page in the GitHub editor. GitHub has Markdown support with a preview feature. The details of GitHub Markdown and the documentation Markdown are a bit different but generally this is close enough, and the person merging your PR will build the docs and check them.
-GitHub has Markdown support with preview feature, but the details of GitHub Markdown dialect are a bit different in ClickHouse documentation.
+### Install a Markdown editor or plugin for your IDE {#install-markdown-editor-or-plugin-for-your-ide}
-### Install Markdown editor or plugin for your IDE {#install-markdown-editor-or-plugin-for-your-ide}
+Usually, these plugins provide a preview of how the markdown will render, and they catch basic errors like unclosed tags very early.
-Usually those also have some way to preview how Markdown will look like, which allows to catch basic errors like unclosed tags very early.
-### Use build.py {#use-build-py}
+## Build the docs locally {#use-build-py}
-It’ll take some effort to go through, but the result will be very close to production documentation.
+You can build the docs locally. It takes a few minutes to set up, but once you have done it the first time, the process is very simple.
-For the first time you’ll need to:
+### Clone the repos
-#### 1. Set up virtualenv
+The documentation is in two repos, clone both of them:
+- [ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse)
+- [ClickHouse/ClickHouse-docs](https://github.com/ClickHouse/clickhouse-docs)
-``` bash
-$ cd ClickHouse/docs/tools
-$ mkdir venv
-$ virtualenv -p $(which python3) venv
-$ source venv/bin/activate
-$ pip3 install -r requirements.txt
+### Install Node.js
+
+The documentation is built with Docusaurus, which requires Node.js. We recommend version 16. Install [Node.js](https://nodejs.org/en/download/).
+
+### Copy files into place
+
+Docusaurus expects all of the markdown files to be located in the directory tree `clickhouse-docs/docs/`. This is not the way our repos are set up, so some copying of files is needed to build the docs:
+
+```bash
+# from the parent directory of both the ClickHouse/ClickHouse and ClickHouse-clickhouse-docs repos:
+cp -r ClickHouse/docs/en/development clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/engines clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/getting-started clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/interfaces clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/operations clickhouse-docs/docs/en/
+cp -r ClickHouse/docs/en/sql-reference clickhouse-docs/docs/en/
+
+cp -r ClickHouse/docs/ru/* clickhouse-docs/docs/ru/
+cp -r ClickHouse/docs/zh clickhouse-docs/docs/
```
-#### 2. Run build.py
+#### Note: Symlinks will not work.
+### Setup Docusaurus
-When all prerequisites are installed, running `build.py` without args (there are some, check `build.py --help`) will generate `ClickHouse/docs/build` folder with complete static html website.
+There are two commands that you may need to use with Docusaurus:
+- `yarn install`
+- `yarn start`
-The easiest way to see the result is to use `--livereload=8888` argument of build.py. Alternatively, you can manually launch a HTTP server to serve the docs, for example by running `cd ClickHouse/docs/build && python3 -m http.server 8888`. Then go to http://localhost:8888 in browser. Feel free to use any other port instead of 8888.
+#### Install Docusaurus and its dependencies:
+
+```bash
+cd clickhouse-docs
+yarn install
+```
+
+#### Start a development Docusaurus environment
+
+This command will start Docusaurus in development mode, which means that as you edit source (for example, `.md` files) files the changes will be rendered into HTML files and served by the Docusaurus development server.
+
+```bash
+yarn start
+```
+
+### Make your changes to the markdown files
+
+Edit your files. Remember that if you are editing files in the `ClickHouse/ClickHouse` repo then you should edit them
+in that repo and then copy the edited file into the `ClickHouse/clickhouse-docs/` directory structure so that they are updated in your develoment environment.
+
+`yarn start` probably opened a browser for you when you ran it; if not, open a browser to `http://localhost:3000/docs/en/intro` and navigate to the documentation that you are changing. If you have already made the changes, you can verify them here; if not, make them, and you will see the page update as you save the changes.
## How to change code highlighting? {#how-to-change-code-hl}
-ClickHouse does not use mkdocs `highlightjs` feature. It uses modified pygments styles instead.
-If you want to change code highlighting, edit the `website/css/highlight.css` file.
-Currently, an [eighties](https://github.com/idleberg/base16-pygments/blob/master/css/base16-eighties.dark.css) theme
-is used.
+Code highlighting is based on the language chosen for your code blocks. Specify the language when you start the code block:
+```sql
+SELECT firstname from imdb.actors;
+```
+
+
+```sql
+SELECT firstname from imdb.actors;
+```
+
+If you need a language supported then open an issue in [ClickHouse-docs](https://github.com/ClickHouse/clickhouse-docs/issues).
## How to subscribe on documentation changes? {#how-to-subscribe-on-documentation-changes}
At the moment there’s no easy way to do just that, but you can consider:
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index cf9b7cbafea..584806951cf 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -102,9 +102,34 @@ void Client::processError(const String & query) const
}
+void Client::showWarnings()
+{
+ try
+ {
+ std::vector messages = loadWarningMessages();
+ if (!messages.empty())
+ {
+ std::cout << "Warnings:" << std::endl;
+ for (const auto & message : messages)
+ std::cout << " * " << message << std::endl;
+ std::cout << std::endl;
+ }
+ }
+ catch (...)
+ {
+ /// Ignore exception
+ }
+}
+
/// Make query to get all server warnings
std::vector Client::loadWarningMessages()
{
+ /// Older server versions cannot execute the query loading warnings.
+ constexpr UInt64 min_server_revision_to_load_warnings = DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED;
+
+ if (server_revision < min_server_revision_to_load_warnings)
+ return {};
+
std::vector messages;
connection->sendQuery(connection_parameters.timeouts,
"SELECT * FROM viewIfPermitted(SELECT message FROM system.warnings ELSE null('message String'))",
@@ -226,25 +251,9 @@ try
connect();
- /// Load Warnings at the beginning of connection
+ /// Show warnings at the beginning of connection.
if (is_interactive && !config().has("no-warnings"))
- {
- try
- {
- std::vector messages = loadWarningMessages();
- if (!messages.empty())
- {
- std::cout << "Warnings:" << std::endl;
- for (const auto & message : messages)
- std::cout << " * " << message << std::endl;
- std::cout << std::endl;
- }
- }
- catch (...)
- {
- /// Ignore exception
- }
- }
+ showWarnings();
if (is_interactive && !delayed_interactive)
{
@@ -370,7 +379,7 @@ void Client::connect()
}
server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch);
- load_suggestions = is_interactive && (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false));
+ load_suggestions = is_interactive && (server_revision >= Suggest::MIN_SERVER_REVISION) && !config().getBool("disable_suggestion", false);
if (server_display_name = connection->getServerDisplayName(connection_parameters.timeouts); server_display_name.empty())
server_display_name = config().getString("host", "localhost");
diff --git a/programs/client/Client.h b/programs/client/Client.h
index 164b8e2ebaa..1fec282be51 100644
--- a/programs/client/Client.h
+++ b/programs/client/Client.h
@@ -45,6 +45,7 @@ protected:
private:
void printChangedSettings() const;
+ void showWarnings();
std::vector loadWarningMessages();
};
}
diff --git a/programs/disks/CommandCopy.cpp b/programs/disks/CommandCopy.cpp
index f9cd7444287..1e5852fe651 100644
--- a/programs/disks/CommandCopy.cpp
+++ b/programs/disks/CommandCopy.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandLink.cpp b/programs/disks/CommandLink.cpp
index 6e9a7e64324..af48f0de097 100644
--- a/programs/disks/CommandLink.cpp
+++ b/programs/disks/CommandLink.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandList.cpp b/programs/disks/CommandList.cpp
index 8c6bfac3a9b..e76bb9e65fb 100644
--- a/programs/disks/CommandList.cpp
+++ b/programs/disks/CommandList.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandListDisks.cpp b/programs/disks/CommandListDisks.cpp
index 2bcbb045d67..22cffdd21fd 100644
--- a/programs/disks/CommandListDisks.cpp
+++ b/programs/disks/CommandListDisks.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandMove.cpp b/programs/disks/CommandMove.cpp
index 4a377cc7225..6322cf4b47d 100644
--- a/programs/disks/CommandMove.cpp
+++ b/programs/disks/CommandMove.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandRead.cpp b/programs/disks/CommandRead.cpp
index aa472fa217e..6b77a27e918 100644
--- a/programs/disks/CommandRead.cpp
+++ b/programs/disks/CommandRead.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandRemove.cpp b/programs/disks/CommandRemove.cpp
index d9925fbd93e..c1d3129bb8d 100644
--- a/programs/disks/CommandRemove.cpp
+++ b/programs/disks/CommandRemove.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/programs/disks/CommandWrite.cpp b/programs/disks/CommandWrite.cpp
index c8ae91ea8d5..0b1c5823c81 100644
--- a/programs/disks/CommandWrite.cpp
+++ b/programs/disks/CommandWrite.cpp
@@ -1,6 +1,7 @@
#pragma once
#include "ICommand.h"
+#include
namespace DB
{
diff --git a/src/Access/Common/AllowedClientHosts.cpp b/src/Access/Common/AllowedClientHosts.cpp
index 85d7065d823..efbdf3924e8 100644
--- a/src/Access/Common/AllowedClientHosts.cpp
+++ b/src/Access/Common/AllowedClientHosts.cpp
@@ -110,18 +110,24 @@ namespace
}
/// Returns the host name by its address.
- String getHostByAddress(const IPAddress & address)
+ Strings getHostsByAddress(const IPAddress & address)
{
- String host = DNSResolver::instance().reverseResolve(address);
+ auto hosts = DNSResolver::instance().reverseResolve(address);
- /// Check that PTR record is resolved back to client address
- if (!isAddressOfHost(address, host))
- throw Exception("Host " + String(host) + " isn't resolved back to " + address.toString(), ErrorCodes::DNS_ERROR);
+ if (hosts.empty())
+ throw Exception(ErrorCodes::DNS_ERROR, "{} could not be resolved", address.toString());
- return host;
+
+ for (const auto & host : hosts)
+ {
+ /// Check that PTR record is resolved back to client address
+ if (!isAddressOfHost(address, host))
+ throw Exception(ErrorCodes::DNS_ERROR, "Host {} isn't resolved back to {}", host, address.toString());
+ }
+
+ return hosts;
}
-
void parseLikePatternIfIPSubnet(const String & pattern, IPSubnet & subnet, IPAddress::Family address_family)
{
size_t slash = pattern.find('/');
@@ -520,20 +526,29 @@ bool AllowedClientHosts::contains(const IPAddress & client_address) const
return true;
/// Check `name_regexps`.
- std::optional resolved_host;
+ std::optional resolved_hosts;
auto check_name_regexp = [&](const String & name_regexp_)
{
try
{
if (boost::iequals(name_regexp_, "localhost"))
return is_client_local();
- if (!resolved_host)
- resolved_host = getHostByAddress(client_v6);
- if (resolved_host->empty())
- return false;
- Poco::RegularExpression re(name_regexp_);
- Poco::RegularExpression::Match match;
- return re.match(*resolved_host, match) != 0;
+ if (!resolved_hosts)
+ {
+ resolved_hosts = getHostsByAddress(client_address);
+ }
+
+ for (const auto & host : resolved_hosts.value())
+ {
+ Poco::RegularExpression re(name_regexp_);
+ Poco::RegularExpression::Match match;
+ if (re.match(host, match) != 0)
+ {
+ return true;
+ }
+ }
+
+ return false;
}
catch (const Exception & e)
{
diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp
index 994abc7b53a..0cbe420f345 100644
--- a/src/Access/DiskAccessStorage.cpp
+++ b/src/Access/DiskAccessStorage.cpp
@@ -15,6 +15,7 @@
#include
#include
#include
+#include
#include
#include
diff --git a/src/AggregateFunctions/parseAggregateFunctionParameters.h b/src/AggregateFunctions/parseAggregateFunctionParameters.h
index a67bc081303..41a04324f6d 100644
--- a/src/AggregateFunctions/parseAggregateFunctionParameters.h
+++ b/src/AggregateFunctions/parseAggregateFunctionParameters.h
@@ -8,6 +8,8 @@
namespace DB
{
+struct Array;
+
Array getAggregateFunctionParametersArray(
const ASTPtr & expression_list,
const std::string & error_context,
diff --git a/src/Backups/registerBackupEnginesFileAndDisk.cpp b/src/Backups/registerBackupEnginesFileAndDisk.cpp
index 050a51939b6..380ae36a8e3 100644
--- a/src/Backups/registerBackupEnginesFileAndDisk.cpp
+++ b/src/Backups/registerBackupEnginesFileAndDisk.cpp
@@ -7,6 +7,7 @@
#include
#include
#include
+#include
namespace DB
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 01d30876dec..f4d3be14da6 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -382,7 +382,7 @@ if (TARGET ch_contrib::rdkafka)
endif()
if (TARGET ch_contrib::nats_io)
- dbms_target_link_libraries(PRIVATE ch_contrib::nats_io)
+ dbms_target_link_libraries(PRIVATE ch_contrib::nats_io ch_contrib::uv)
endif()
if (TARGET ch_contrib::sasl2)
@@ -453,6 +453,9 @@ if (TARGET ch_contrib::avrocpp)
dbms_target_link_libraries(PRIVATE ch_contrib::avrocpp)
endif ()
+set_source_files_properties(Common/CaresPTRResolver.cpp PROPERTIES COMPILE_FLAGS -Wno-reserved-identifier)
+target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::c-ares)
+
if (TARGET OpenSSL::Crypto)
dbms_target_link_libraries (PRIVATE OpenSSL::Crypto)
target_link_libraries (clickhouse_common_io PRIVATE OpenSSL::Crypto)
diff --git a/src/Client/Suggest.h b/src/Client/Suggest.h
index 65b60ceffc4..25d45f7ffaf 100644
--- a/src/Client/Suggest.h
+++ b/src/Client/Suggest.h
@@ -28,8 +28,8 @@ public:
template
void load(ContextPtr context, const ConnectionParameters & connection_parameters, Int32 suggestion_limit);
- /// Older server versions cannot execute the query above.
- static constexpr int MIN_SERVER_REVISION = 54406;
+ /// Older server versions cannot execute the query loading suggestions.
+ static constexpr int MIN_SERVER_REVISION = DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED;
private:
void fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query);
diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp
index d8e98ec9406..8d61f6e726a 100644
--- a/src/Columns/ColumnNullable.cpp
+++ b/src/Columns/ColumnNullable.cpp
@@ -793,4 +793,18 @@ ColumnPtr makeNullable(const ColumnPtr & column)
return ColumnNullable::create(column, ColumnUInt8::create(column->size(), 0));
}
+ColumnPtr makeNullableSafe(const ColumnPtr & column)
+{
+ if (isColumnNullable(*column))
+ return column;
+
+ if (isColumnConst(*column))
+ return ColumnConst::create(makeNullableSafe(assert_cast(*column).getDataColumnPtr()), column->size());
+
+ if (column->canBeInsideNullable())
+ return makeNullable(column);
+
+ return column;
+}
+
}
diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h
index 52e57f7f0d0..e832f6d20e5 100644
--- a/src/Columns/ColumnNullable.h
+++ b/src/Columns/ColumnNullable.h
@@ -223,5 +223,6 @@ private:
};
ColumnPtr makeNullable(const ColumnPtr & column);
+ColumnPtr makeNullableSafe(const ColumnPtr & column);
}
diff --git a/src/Common/CaresPTRResolver.cpp b/src/Common/CaresPTRResolver.cpp
new file mode 100644
index 00000000000..f6228e97c02
--- /dev/null
+++ b/src/Common/CaresPTRResolver.cpp
@@ -0,0 +1,109 @@
+#include "CaresPTRResolver.h"
+#include
+#include
+#include
+#include "ares.h"
+#include "netdb.h"
+
+namespace DB
+{
+
+ namespace ErrorCodes
+ {
+ extern const int DNS_ERROR;
+ }
+
+ static void callback(void * arg, int status, int, struct hostent * host)
+ {
+ auto * ptr_records = reinterpret_cast*>(arg);
+ if (status == ARES_SUCCESS && host->h_aliases)
+ {
+ int i = 0;
+ while (auto * ptr_record = host->h_aliases[i])
+ {
+ ptr_records->emplace_back(ptr_record);
+ i++;
+ }
+ }
+ }
+
+ CaresPTRResolver::CaresPTRResolver(CaresPTRResolver::provider_token) : channel(nullptr)
+ {
+ /*
+ * ares_library_init is not thread safe. Currently, the only other usage of c-ares seems to be in grpc.
+ * In grpc, ares_library_init seems to be called only in Windows.
+ * See https://github.com/grpc/grpc/blob/master/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc#L1187
+ * That means it's safe to init it here, but we should be cautious when introducing new code that depends on c-ares and even updates
+ * to grpc. As discussed in https://github.com/ClickHouse/ClickHouse/pull/37827#discussion_r919189085, c-ares should be adapted to be atomic
+ * */
+ if (ares_library_init(ARES_LIB_INIT_ALL) != ARES_SUCCESS || ares_init(&channel) != ARES_SUCCESS)
+ {
+ throw DB::Exception("Failed to initialize c-ares", DB::ErrorCodes::DNS_ERROR);
+ }
+ }
+
+ CaresPTRResolver::~CaresPTRResolver()
+ {
+ ares_destroy(channel);
+ ares_library_cleanup();
+ }
+
+ std::vector CaresPTRResolver::resolve(const std::string & ip)
+ {
+ std::vector ptr_records;
+
+ resolve(ip, ptr_records);
+ wait();
+
+ return ptr_records;
+ }
+
+ std::vector CaresPTRResolver::resolve_v6(const std::string & ip)
+ {
+ std::vector ptr_records;
+
+ resolve_v6(ip, ptr_records);
+ wait();
+
+ return ptr_records;
+ }
+
+ void CaresPTRResolver::resolve(const std::string & ip, std::vector & response)
+ {
+ in_addr addr;
+
+ inet_pton(AF_INET, ip.c_str(), &addr);
+
+ ares_gethostbyaddr(channel, reinterpret_cast(&addr), sizeof(addr), AF_INET, callback, &response);
+ }
+
+ void CaresPTRResolver::resolve_v6(const std::string & ip, std::vector & response)
+ {
+ in6_addr addr;
+ inet_pton(AF_INET6, ip.c_str(), &addr);
+
+ ares_gethostbyaddr(channel, reinterpret_cast(&addr), sizeof(addr), AF_INET6, callback, &response);
+ }
+
+ void CaresPTRResolver::wait()
+ {
+ timeval * tvp, tv;
+ fd_set read_fds;
+ fd_set write_fds;
+ int nfds;
+
+ for (;;)
+ {
+ FD_ZERO(&read_fds);
+ FD_ZERO(&write_fds);
+ nfds = ares_fds(channel, &read_fds,&write_fds);
+ if (nfds == 0)
+ {
+ break;
+ }
+ tvp = ares_timeout(channel, nullptr, &tv);
+ select(nfds, &read_fds, &write_fds, nullptr, tvp);
+ ares_process(channel, &read_fds, &write_fds);
+ }
+ }
+}
diff --git a/src/Common/CaresPTRResolver.h b/src/Common/CaresPTRResolver.h
new file mode 100644
index 00000000000..fd6a1cf7bc5
--- /dev/null
+++ b/src/Common/CaresPTRResolver.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#include "DNSPTRResolver.h"
+
+using ares_channel = struct ares_channeldata *;
+
+namespace DB
+{
+
+ /*
+ * Implements reverse DNS resolution using c-ares lib. System reverse DNS resolution via
+ * gethostbyaddr or getnameinfo does not work reliably because in some systems
+ * it returns all PTR records for a given IP and in others it returns only one.
+ * */
+ class CaresPTRResolver : public DNSPTRResolver
+ {
+ friend class DNSPTRResolverProvider;
+
+ /*
+ * Allow only DNSPTRProvider to instantiate this class
+ * */
+ struct provider_token {};
+
+ public:
+ explicit CaresPTRResolver(provider_token);
+ ~CaresPTRResolver() override;
+
+ std::vector resolve(const std::string & ip) override;
+
+ std::vector resolve_v6(const std::string & ip) override;
+
+ private:
+ void wait();
+
+ void resolve(const std::string & ip, std::vector & response);
+
+ void resolve_v6(const std::string & ip, std::vector & response);
+
+ ares_channel channel;
+ };
+}
+
diff --git a/src/Common/DNSPTRResolver.h b/src/Common/DNSPTRResolver.h
new file mode 100644
index 00000000000..e6cce83f79d
--- /dev/null
+++ b/src/Common/DNSPTRResolver.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include
+#include
+
+namespace DB
+{
+ struct DNSPTRResolver
+ {
+
+ virtual ~DNSPTRResolver() = default;
+
+ virtual std::vector resolve(const std::string & ip) = 0;
+
+ virtual std::vector resolve_v6(const std::string & ip) = 0;
+
+ };
+}
diff --git a/src/Common/DNSPTRResolverProvider.cpp b/src/Common/DNSPTRResolverProvider.cpp
new file mode 100644
index 00000000000..41c73f4f36f
--- /dev/null
+++ b/src/Common/DNSPTRResolverProvider.cpp
@@ -0,0 +1,13 @@
+#include "DNSPTRResolverProvider.h"
+#include "CaresPTRResolver.h"
+
+namespace DB
+{
+ std::shared_ptr DNSPTRResolverProvider::get()
+ {
+ static auto cares_resolver = std::make_shared(
+ CaresPTRResolver::provider_token {}
+ );
+ return cares_resolver;
+ }
+}
diff --git a/src/Common/DNSPTRResolverProvider.h b/src/Common/DNSPTRResolverProvider.h
new file mode 100644
index 00000000000..a7f534749e3
--- /dev/null
+++ b/src/Common/DNSPTRResolverProvider.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include
+#include "DNSPTRResolver.h"
+
+namespace DB
+{
+ /*
+ * Provides a ready-to-use DNSPTRResolver instance.
+ * It hides 3rd party lib dependencies, handles initialization and lifetime.
+ * Since `get` function is static, it can be called from any context. Including cached static functions.
+ * */
+ class DNSPTRResolverProvider
+ {
+ public:
+ static std::shared_ptr get();
+ };
+}
diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp
index 0616e324b73..10797b7a809 100644
--- a/src/Common/DNSResolver.cpp
+++ b/src/Common/DNSResolver.cpp
@@ -12,6 +12,7 @@
#include
#include
#include
+#include "DNSPTRResolverProvider.h"
namespace ProfileEvents
{
@@ -138,16 +139,17 @@ static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
return addresses;
}
-static String reverseResolveImpl(const Poco::Net::IPAddress & address)
+static Strings reverseResolveImpl(const Poco::Net::IPAddress & address)
{
- Poco::Net::SocketAddress sock_addr(address, 0);
+ auto ptr_resolver = DB::DNSPTRResolverProvider::get();
- /// Resolve by hand, because Poco::Net::DNS::hostByAddress(...) does getaddrinfo(...) after getnameinfo(...)
- char host[1024];
- int err = getnameinfo(sock_addr.addr(), sock_addr.length(), host, sizeof(host), nullptr, 0, NI_NAMEREQD);
- if (err)
- throw Exception("Cannot getnameinfo(" + address.toString() + "): " + gai_strerror(err), ErrorCodes::DNS_ERROR);
- return host;
+ if (address.family() == Poco::Net::IPAddress::Family::IPv4)
+ {
+ return ptr_resolver->resolve(address.toString());
+ } else
+ {
+ return ptr_resolver->resolve_v6(address.toString());
+ }
}
struct DNSResolver::Impl
@@ -235,7 +237,7 @@ std::vector DNSResolver::resolveAddressList(const std:
return addresses;
}
-String DNSResolver::reverseResolve(const Poco::Net::IPAddress & address)
+Strings DNSResolver::reverseResolve(const Poco::Net::IPAddress & address)
{
if (impl->disable_cache)
return reverseResolveImpl(address);
diff --git a/src/Common/DNSResolver.h b/src/Common/DNSResolver.h
index fdd9799f96f..84c88586636 100644
--- a/src/Common/DNSResolver.h
+++ b/src/Common/DNSResolver.h
@@ -36,8 +36,8 @@ public:
std::vector resolveAddressList(const std::string & host, UInt16 port);
- /// Accepts host IP and resolves its host name
- String reverseResolve(const Poco::Net::IPAddress & address);
+ /// Accepts host IP and resolves its host names
+ Strings reverseResolve(const Poco::Net::IPAddress & address);
/// Get this server host name
String getHostName();
diff --git a/src/Common/LRUFileCache.cpp b/src/Common/LRUFileCache.cpp
index 0ce76dbdec6..6306b6de059 100644
--- a/src/Common/LRUFileCache.cpp
+++ b/src/Common/LRUFileCache.cpp
@@ -45,7 +45,7 @@ void LRUFileCache::initialize()
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
- return;
+ throw;
}
}
else
@@ -841,7 +841,11 @@ void LRUFileCache::loadCacheInfoIntoMemory(std::lock_guard & cache_l
/// cache_base_path / key_prefix / key / offset
if (!files.empty())
- throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache already initialized");
+ throw Exception(
+ ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
+ "Cache initialization is partially made. "
+ "This can be a result of a failed first attempt to initialize cache. "
+ "Please, check log for error messages");
fs::directory_iterator key_prefix_it{cache_base_path};
for (; key_prefix_it != fs::directory_iterator(); ++key_prefix_it)
diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp
index cfc364929a3..60efab69433 100644
--- a/src/Common/OptimizedRegularExpression.cpp
+++ b/src/Common/OptimizedRegularExpression.cpp
@@ -342,6 +342,23 @@ OptimizedRegularExpressionImpl::OptimizedRegularExpressionImpl(cons
}
}
+template
+OptimizedRegularExpressionImpl::OptimizedRegularExpressionImpl(OptimizedRegularExpressionImpl && rhs) noexcept
+ : is_trivial(rhs.is_trivial)
+ , required_substring_is_prefix(rhs.required_substring_is_prefix)
+ , is_case_insensitive(rhs.is_case_insensitive)
+ , required_substring(std::move(rhs.required_substring))
+ , re2(std::move(rhs.re2))
+ , number_of_subpatterns(rhs.number_of_subpatterns)
+{
+ if (!required_substring.empty())
+ {
+ if (is_case_insensitive)
+ case_insensitive_substring_searcher.emplace(required_substring.data(), required_substring.size());
+ else
+ case_sensitive_substring_searcher.emplace(required_substring.data(), required_substring.size());
+ }
+}
template
bool OptimizedRegularExpressionImpl::match(const char * subject, size_t subject_size) const
diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h
index eaa7b06e309..dad8706a50d 100644
--- a/src/Common/OptimizedRegularExpression.h
+++ b/src/Common/OptimizedRegularExpression.h
@@ -56,6 +56,9 @@ public:
using StringPieceType = std::conditional_t;
OptimizedRegularExpressionImpl(const std::string & regexp_, int options = 0); /// NOLINT
+ /// StringSearcher store pointers to required_substring, it must be updated on move.
+ OptimizedRegularExpressionImpl(OptimizedRegularExpressionImpl && rhs) noexcept;
+ OptimizedRegularExpressionImpl(const OptimizedRegularExpressionImpl & rhs) = delete;
bool match(const std::string & subject) const
{
diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp
index 86adeeaf7e5..0050288b1cf 100644
--- a/src/Common/ShellCommand.cpp
+++ b/src/Common/ShellCommand.cpp
@@ -1,9 +1,7 @@
#include
#include
-#include
#include
#include
-#include
#include
#include
@@ -13,6 +11,7 @@
#include
#include
#include
+#include
namespace
@@ -94,53 +93,15 @@ ShellCommand::~ShellCommand()
bool ShellCommand::tryWaitProcessWithTimeout(size_t timeout_in_seconds)
{
- int status = 0;
-
LOG_TRACE(getLogger(), "Try wait for shell command pid {} with timeout {}", pid, timeout_in_seconds);
wait_called = true;
- struct timespec interval {.tv_sec = 1, .tv_nsec = 0};
in.close();
out.close();
err.close();
- if (timeout_in_seconds == 0)
- {
- /// If there is no timeout before signal try to waitpid 1 time without block so we can avoid sending
- /// signal if process is already normally terminated.
-
- int waitpid_res = waitpid(pid, &status, WNOHANG);
- bool process_terminated_normally = (waitpid_res == pid);
- return process_terminated_normally;
- }
-
- /// If timeout is positive try waitpid without block in loop until
- /// process is normally terminated or waitpid return error
-
- while (timeout_in_seconds != 0)
- {
- int waitpid_res = waitpid(pid, &status, WNOHANG);
- bool process_terminated_normally = (waitpid_res == pid);
-
- if (process_terminated_normally)
- {
- return true;
- }
- else if (waitpid_res == 0)
- {
- --timeout_in_seconds;
- nanosleep(&interval, nullptr);
-
- continue;
- }
- else if (waitpid_res == -1 && errno != EINTR)
- {
- return false;
- }
- }
-
- return false;
+ return waitForPid(pid, timeout_in_seconds);
}
void ShellCommand::logCommand(const char * filename, char * const argv[])
diff --git a/src/Common/ShellCommand.h b/src/Common/ShellCommand.h
index 190b5bc664e..dfc4a826f62 100644
--- a/src/Common/ShellCommand.h
+++ b/src/Common/ShellCommand.h
@@ -3,6 +3,7 @@
#include
#include
#include
+#include
namespace DB
diff --git a/src/Common/tests/gtest_lsan.cpp b/src/Common/tests/gtest_lsan.cpp
deleted file mode 100644
index f6e1984ec58..00000000000
--- a/src/Common/tests/gtest_lsan.cpp
+++ /dev/null
@@ -1,33 +0,0 @@
-#include // ADDRESS_SANITIZER
-
-#ifdef ADDRESS_SANITIZER
-
-#include
-#include
-
-#include
-#include
-
-/// Test that ensures that LSan works.
-///
-/// Regression test for the case when it may not work,
-/// because of broken getauxval() [1].
-///
-/// [1]: https://github.com/ClickHouse/ClickHouse/pull/33957
-TEST(Common, LSan)
-{
- int sanitizers_exit_code = 1;
-
- ASSERT_EXIT({
- std::thread leak_in_thread([]()
- {
- void * leak = malloc(4096);
- ASSERT_NE(leak, nullptr);
- });
- leak_in_thread.join();
-
- __lsan_do_leak_check();
- }, ::testing::ExitedWithCode(sanitizers_exit_code), ".*LeakSanitizer: detected memory leaks.*");
-}
-
-#endif
diff --git a/src/Common/waitForPid.cpp b/src/Common/waitForPid.cpp
new file mode 100644
index 00000000000..38f43ae2f6a
--- /dev/null
+++ b/src/Common/waitForPid.cpp
@@ -0,0 +1,192 @@
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wgnu-statement-expression"
+#define HANDLE_EINTR(x) ({ \
+ decltype(x) eintr_wrapper_result; \
+ do { \
+ eintr_wrapper_result = (x); \
+ } while (eintr_wrapper_result == -1 && errno == EINTR); \
+ eintr_wrapper_result; \
+})
+
+#if defined(OS_LINUX)
+
+#include
+#include
+
+#if !defined(__NR_pidfd_open)
+ #if defined(__x86_64__)
+ #define SYS_pidfd_open 434
+ #elif defined(__aarch64__)
+ #define SYS_pidfd_open 434
+ #elif defined(__ppc64__)
+ #define SYS_pidfd_open 434
+ #elif defined(__riscv)
+ #define SYS_pidfd_open 434
+ #else
+ #error "Unsupported architecture"
+ #endif
+#else
+ #define SYS_pidfd_open __NR_pidfd_open
+#endif
+
+namespace DB
+{
+
+static int syscall_pidfd_open(pid_t pid)
+{
+ // pidfd_open cannot be interrupted, no EINTR handling
+ return syscall(SYS_pidfd_open, pid, 0);
+}
+
+static int dir_pidfd_open(pid_t pid)
+{
+ std::string path = "/proc/" + std::to_string(pid);
+ return HANDLE_EINTR(open(path.c_str(), O_DIRECTORY));
+}
+
+static bool supportsPidFdOpen()
+{
+ VersionNumber pidfd_open_minimal_version(5, 3, 0);
+ VersionNumber linux_version(Poco::Environment::osVersion());
+ return linux_version >= pidfd_open_minimal_version;
+}
+
+static int pidFdOpen(pid_t pid)
+{
+ // use pidfd_open or just plain old /proc/[pid] open for Linux
+ if (supportsPidFdOpen())
+ {
+ return syscall_pidfd_open(pid);
+ }
+ else
+ {
+ return dir_pidfd_open(pid);
+ }
+}
+
+static int pollPid(pid_t pid, int timeout_in_ms)
+{
+ struct pollfd pollfd;
+
+ int pid_fd = pidFdOpen(pid);
+ if (pid_fd == -1)
+ {
+ return false;
+ }
+ pollfd.fd = pid_fd;
+ pollfd.events = POLLIN;
+ int ready = poll(&pollfd, 1, timeout_in_ms);
+ int save_errno = errno;
+ close(pid_fd);
+ errno = save_errno;
+ return ready;
+}
+#elif defined(OS_DARWIN) || defined(OS_FREEBSD)
+
+#include
+#include
+
+namespace DB
+{
+
+static int pollPid(pid_t pid, int timeout_in_ms)
+{
+ int status = 0;
+ int kq = HANDLE_EINTR(kqueue());
+ if (kq == -1)
+ {
+ return false;
+ }
+ struct kevent change = {.ident = NULL};
+ EV_SET(&change, pid, EVFILT_PROC, EV_ADD, NOTE_EXIT, 0, NULL);
+ int result = HANDLE_EINTR(kevent(kq, &change, 1, NULL, 0, NULL));
+ if (result == -1)
+ {
+ if (errno != ESRCH)
+ {
+ return false;
+ }
+ // check if pid already died while we called kevent()
+ if (waitpid(pid, &status, WNOHANG) == pid)
+ {
+ return true;
+ }
+ return false;
+ }
+
+ struct kevent event = {.ident = NULL};
+ struct timespec remaining_timespec = {.tv_sec = timeout_in_ms / 1000, .tv_nsec = (timeout_in_ms % 1000) * 1000000};
+ int ready = kevent(kq, nullptr, 0, &event, 1, &remaining_timespec);
+ int save_errno = errno;
+ close(kq);
+ errno = save_errno;
+ return ready;
+}
+#else
+ #error "Unsupported OS type"
+#endif
+
+bool waitForPid(pid_t pid, size_t timeout_in_seconds)
+{
+ int status = 0;
+
+ Stopwatch watch;
+
+ if (timeout_in_seconds == 0)
+ {
+ /// If there is no timeout before signal try to waitpid 1 time without block so we can avoid sending
+ /// signal if process is already normally terminated.
+
+ int waitpid_res = waitpid(pid, &status, WNOHANG);
+ bool process_terminated_normally = (waitpid_res == pid);
+ return process_terminated_normally;
+ }
+
+ /// If timeout is positive try waitpid without block in loop until
+ /// process is normally terminated or waitpid return error
+
+ int timeout_in_ms = timeout_in_seconds * 1000;
+ while (timeout_in_ms > 0)
+ {
+ int waitpid_res = waitpid(pid, &status, WNOHANG);
+ bool process_terminated_normally = (waitpid_res == pid);
+ if (process_terminated_normally)
+ {
+ return true;
+ }
+ else if (waitpid_res == 0)
+ {
+ watch.restart();
+ int ready = pollPid(pid, timeout_in_ms);
+ if (ready <= 0)
+ {
+ if (errno == EINTR || errno == EAGAIN)
+ {
+ timeout_in_ms -= watch.elapsedMilliseconds();
+ }
+ else
+ {
+ return false;
+ }
+ }
+ continue;
+ }
+ else if (waitpid_res == -1 && errno != EINTR)
+ {
+ return false;
+ }
+ }
+ return false;
+}
+
+}
+#pragma GCC diagnostic pop
diff --git a/src/Common/waitForPid.h b/src/Common/waitForPid.h
new file mode 100644
index 00000000000..71c1a74712c
--- /dev/null
+++ b/src/Common/waitForPid.h
@@ -0,0 +1,12 @@
+#pragma once
+#include
+
+namespace DB
+{
+/*
+ * Waits for a specific pid with timeout, using modern Linux and OSX facilities
+ * Returns `true` if process terminated successfully or `false` otherwise
+ */
+bool waitForPid(pid_t pid, size_t timeout_in_seconds);
+
+}
diff --git a/src/Coordination/CoordinationSettings.cpp b/src/Coordination/CoordinationSettings.cpp
index 34d69967828..4733adcf67a 100644
--- a/src/Coordination/CoordinationSettings.cpp
+++ b/src/Coordination/CoordinationSettings.cpp
@@ -1,5 +1,4 @@
#include
-#include
#include
#include
#include
diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp
index 7c6ed227a06..8261f5d1e26 100644
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@@ -21,6 +21,7 @@
#include
#include
#include
+#include
namespace DB
{
@@ -111,7 +112,7 @@ KeeperServer::KeeperServer(
configuration_and_settings_->snapshot_storage_path,
coordination_settings,
checkAndGetSuperdigest(configuration_and_settings_->super_digest),
- config.getBool("keeper_server.digest_enabled", true)))
+ config.getBool("keeper_server.digest_enabled", false)))
, state_manager(nuraft::cs_new(
server_id, "keeper_server", configuration_and_settings_->log_storage_path, configuration_and_settings_->state_file_path, config, coordination_settings))
, log(&Poco::Logger::get("KeeperServer"))
diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h
index 3638a036098..7b56367769e 100644
--- a/src/Core/BaseSettings.h
+++ b/src/Core/BaseSettings.h
@@ -43,9 +43,16 @@ class BaseSettings : public TTraits::Data
{
using CustomSettingMap = std::unordered_map, SettingFieldCustom>>;
public:
+ BaseSettings() = default;
+ BaseSettings(const BaseSettings &) = default;
+ BaseSettings(BaseSettings &&) noexcept = default;
+ BaseSettings & operator=(const BaseSettings &) = default;
+ BaseSettings & operator=(BaseSettings &&) noexcept = default;
+ virtual ~BaseSettings() = default;
+
using Traits = TTraits;
- void set(std::string_view name, const Field & value);
+ virtual void set(std::string_view name, const Field & value);
Field get(std::string_view name) const;
void setString(std::string_view name, const String & value);
@@ -62,6 +69,8 @@ public:
/// Resets all the settings to their default values.
void resetToDefault();
+ /// Resets specified setting to its default value.
+ void resetToDefault(std::string_view name);
bool has(std::string_view name) const { return hasBuiltin(name) || hasCustom(name); }
static bool hasBuiltin(std::string_view name);
@@ -315,6 +324,14 @@ void BaseSettings::resetToDefault()
custom_settings_map.clear();
}
+template
+void BaseSettings::resetToDefault(std::string_view name)
+{
+ const auto & accessor = Traits::Accessor::instance();
+ if (size_t index = accessor.find(name); index != static_cast(-1))
+ accessor.resetValueToDefault(*this, index);
+}
+
template
bool BaseSettings::hasBuiltin(std::string_view name)
{
diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h
index 2df48a79776..584720694d7 100644
--- a/src/Core/ProtocolDefines.h
+++ b/src/Core/ProtocolDefines.h
@@ -52,8 +52,10 @@
/// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION,
/// later is just a number for server version (one number instead of commit SHA)
/// for simplicity (sometimes it may be more convenient in some use cases).
-#define DBMS_TCP_PROTOCOL_VERSION 54456
+#define DBMS_TCP_PROTOCOL_VERSION 54457
#define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449
#define DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT 54456
+
+#define DBMS_MIN_PROTOCOL_VERSION_WITH_VIEW_IF_PERMITTED 54457
diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp
index 5251569505e..7bac3f04fc6 100644
--- a/src/Core/Settings.cpp
+++ b/src/Core/Settings.cpp
@@ -1,5 +1,6 @@
#include "Settings.h"
+#include
#include
#include
#include
@@ -145,6 +146,53 @@ std::vector Settings::getAllRegisteredNames() const
return all_settings;
}
+void Settings::set(std::string_view name, const Field & value)
+{
+ BaseSettings::set(name, value);
+
+ if (name == "compatibility")
+ applyCompatibilitySetting();
+ /// If we change setting that was changed by compatibility setting before
+ /// we should remove it from settings_changed_by_compatibility_setting,
+ /// otherwise the next time we will change compatibility setting
+ /// this setting will be changed too (and we don't want it).
+ else if (settings_changed_by_compatibility_setting.contains(name))
+ settings_changed_by_compatibility_setting.erase(name);
+}
+
+void Settings::applyCompatibilitySetting()
+{
+ /// First, revert all changes applied by previous compatibility setting
+ for (const auto & setting_name : settings_changed_by_compatibility_setting)
+ resetToDefault(setting_name);
+
+ settings_changed_by_compatibility_setting.clear();
+ String compatibility = getString("compatibility");
+ /// If setting value is empty, we don't need to change settings
+ if (compatibility.empty())
+ return;
+
+ ClickHouseVersion version(compatibility);
+ /// Iterate through ClickHouse version in descending order and apply reversed
+ /// changes for each version that is higher that version from compatibility setting
+ for (auto it = settings_changes_history.rbegin(); it != settings_changes_history.rend(); ++it)
+ {
+ if (version >= it->first)
+ break;
+
+ /// Apply reversed changes from this version.
+ for (const auto & change : it->second)
+ {
+ /// If this setting was changed manually, we don't change it
+ if (isChanged(change.name) && !settings_changed_by_compatibility_setting.contains(change.name))
+ continue;
+
+ BaseSettings::set(change.name, change.previous_value);
+ settings_changed_by_compatibility_setting.insert(change.name);
+ }
+ }
+}
+
IMPLEMENT_SETTINGS_TRAITS(FormatFactorySettingsTraits, FORMAT_FACTORY_SETTINGS)
}
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index bda72f089eb..17e4d27bbcd 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -35,6 +35,10 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
*
* `flags` can be either 0 or IMPORTANT.
* A setting is "IMPORTANT" if it affects the results of queries and can't be ignored by older versions.
+ *
+ * When adding new settings that control some backward incompatible changes or when changing some settings values,
+ * consider adding them to settings changes history in SettingsChangesHistory.h for special `compatibility` setting
+ * to work correctly.
*/
#define COMMON_SETTINGS(M) \
@@ -132,6 +136,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \
M(Bool, enable_positional_arguments, true, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \
\
+ M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \
+ \
M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \
M(UInt64, parallel_replicas_count, 0, "", 0) \
M(UInt64, parallel_replica_offset, 0, "", 0) \
@@ -599,6 +605,11 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \
M(Bool, allow_deprecated_syntax_for_merge_tree, false, "Allow to create *MergeTree tables with deprecated engine definition syntax", 0) \
\
+ M(String, compatibility, "", "Changes other settings according to provided ClickHouse version. If we know that we changed some behaviour in ClickHouse by changing some settings in some version, this compatibility setting will control these settings", 0) \
+ \
+ M(Map, additional_table_filters, "", "Additional filter expression which would be applied after reading from specified table. Syntax: {'table1': 'expression', 'database.table2': 'expression'}", 0) \
+ M(String, additional_result_filter, "", "Additional filter expression which would be applied to query result", 0) \
+ \
/** Experimental functions */ \
M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \
M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \
@@ -652,7 +663,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
#define FORMAT_FACTORY_SETTINGS(M) \
M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \
- M(Bool, format_csv_allow_single_quotes, true, "If it is set to true, allow strings in single quotes.", 0) \
+ M(Bool, format_csv_allow_single_quotes, false, "If it is set to true, allow strings in single quotes.", 0) \
M(Bool, format_csv_allow_double_quotes, true, "If it is set to true, allow strings in double quotes.", 0) \
M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \
M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices", 0) \
@@ -758,7 +769,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(Bool, output_format_pretty_row_numbers, false, "Add row numbers before each row for pretty output format", 0) \
M(Bool, insert_distributed_one_random_shard, false, "If setting is enabled, inserting into distributed table will choose a random shard to write when there is no sharding key", 0) \
\
- M(UInt64, cross_to_inner_join_rewrite, 1, "Use inner join instead of comma/cross join if possible. Possible values: 0 - no rewrite, 1 - apply if possible, 2 - force rewrite all cross joins", 0) \
+ M(UInt64, cross_to_inner_join_rewrite, 1, "Use inner join instead of comma/cross join if there're joining expressions in the WHERE section. Values: 0 - no rewrite, 1 - apply if possible for comma/cross, 2 - force rewrite all comma joins, cross - if possible", 0) \
\
M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \
M(Bool, output_format_arrow_string_as_string, false, "Use Arrow String type instead of Binary for String columns", 0) \
@@ -825,6 +836,13 @@ struct Settings : public BaseSettings, public IHints<2, Settings
void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field);
void addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field);
+
+ void set(std::string_view name, const Field & value) override;
+
+private:
+ void applyCompatibilitySetting();
+
+ std::unordered_set settings_changed_by_compatibility_setting;
};
/*
diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h
new file mode 100644
index 00000000000..ba60fb99308
--- /dev/null
+++ b/src/Core/SettingsChangesHistory.h
@@ -0,0 +1,114 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include