diff --git a/.github/ISSUE_TEMPLATE/40_bug-report.md b/.github/ISSUE_TEMPLATE/40_bug-report.md index 97137366189..5c8611d47e6 100644 --- a/.github/ISSUE_TEMPLATE/40_bug-report.md +++ b/.github/ISSUE_TEMPLATE/40_bug-report.md @@ -10,12 +10,26 @@ assignees: '' You have to provide the following information whenever possible. **Describe the bug** + A clear and concise description of what works not as it is supposed to. **Does it reproduce on recent release?** + [The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv) +**Enable crash reporting** + +If possible, change "enabled" to true in "send_crash_reports" section in `config.xml`: + +``` + + + + false +``` + **How to reproduce** + * Which ClickHouse server version to use * Which interface to use, if matters * Non-default settings, if any @@ -24,10 +38,13 @@ A clear and concise description of what works not as it is supposed to. * Queries to run that lead to unexpected result **Expected behavior** + A clear and concise description of what you expected to happen. **Error message and/or stacktrace** + If applicable, add screenshots to help explain your problem. **Additional context** + Add any other context about the problem here. diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 01e700ebba3..6aa65942445 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -26,8 +26,6 @@ #include #include #include -#include -#include #include #include #include @@ -59,6 +57,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include @@ -70,6 +69,7 @@ #endif #include +namespace fs = std::filesystem; DB::PipeFDs signal_pipe; @@ -437,11 +437,11 @@ static void sanitizerDeathCallback() static std::string createDirectory(const std::string & file) { - auto path = Poco::Path(file).makeParent(); - if (path.toString().empty()) + fs::path path = fs::path(file).parent_path(); + if (path.empty()) return ""; - Poco::File(path).createDirectories(); - return path.toString(); + fs::create_directories(path); + return path; }; @@ -449,7 +449,7 @@ static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path { try { - Poco::File(path).createDirectories(); + fs::create_directories(path); return true; } catch (...) @@ -470,7 +470,7 @@ void BaseDaemon::reloadConfiguration() */ config_path = config().getString("config-file", getDefaultConfigFileName()); DB::ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); + config_processor.setConfigPath(fs::path(config_path).parent_path()); loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true); if (last_configuration != nullptr) @@ -524,18 +524,20 @@ std::string BaseDaemon::getDefaultConfigFileName() const void BaseDaemon::closeFDs() { #if defined(OS_FREEBSD) || defined(OS_DARWIN) - Poco::File proc_path{"/dev/fd"}; + fs::path proc_path{"/dev/fd"}; #else - Poco::File proc_path{"/proc/self/fd"}; + fs::path proc_path{"/proc/self/fd"}; #endif - if (proc_path.isDirectory()) /// Hooray, proc exists + if (fs::is_directory(proc_path)) /// Hooray, proc exists { - std::vector fds; - /// in /proc/self/fd directory filenames are numeric file descriptors - proc_path.list(fds); - for (const auto & fd_str : fds) + /// in /proc/self/fd directory filenames are numeric file descriptors. + /// Iterate directory separately from closing fds to avoid closing iterated directory fd. + std::vector fds; + for (const auto & path : fs::directory_iterator(proc_path)) + fds.push_back(DB::parse(path.path().filename())); + + for (const auto & fd : fds) { - int fd = DB::parse(fd_str); if (fd > 2 && fd != signal_pipe.fds_rw[0] && fd != signal_pipe.fds_rw[1]) ::close(fd); } @@ -597,7 +599,7 @@ void BaseDaemon::initialize(Application & self) { /** When creating pid file and looking for config, will search for paths relative to the working path of the program when started. */ - std::string path = Poco::Path(config().getString("application.path")).setFileName("").toString(); + std::string path = fs::path(config().getString("application.path")).replace_filename(""); if (0 != chdir(path.c_str())) throw Poco::Exception("Cannot change directory to " + path); } @@ -645,7 +647,7 @@ void BaseDaemon::initialize(Application & self) std::string log_path = config().getString("logger.log", ""); if (!log_path.empty()) - log_path = Poco::Path(log_path).setFileName("").toString(); + log_path = fs::path(log_path).replace_filename(""); /** Redirect stdout, stderr to separate files in the log directory (or in the specified file). * Some libraries write to stderr in case of errors in debug mode, @@ -708,8 +710,7 @@ void BaseDaemon::initialize(Application & self) tryCreateDirectories(&logger(), core_path); - Poco::File cores = core_path; - if (!(cores.exists() && cores.isDirectory())) + if (!(fs::exists(core_path) && fs::is_directory(core_path))) { core_path = !log_path.empty() ? log_path : "/opt/"; tryCreateDirectories(&logger(), core_path); diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 1028dc7d2dc..3571c64edd6 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -1,6 +1,5 @@ #include -#include #include #include @@ -25,6 +24,7 @@ # include # include +namespace fs = std::filesystem; namespace { @@ -53,8 +53,7 @@ void setExtras() sentry_set_extra("physical_cpu_cores", sentry_value_new_int32(getNumberOfPhysicalCPUCores())); if (!server_data_path.empty()) - sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix( - Poco::File(server_data_path).freeSpace()).c_str())); + sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix(fs::space(server_data_path).free).c_str())); } void sentry_logger(sentry_level_e level, const char * message, va_list args, void *) @@ -110,12 +109,12 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) if (enabled) { server_data_path = config.getString("path", ""); - const std::filesystem::path & default_tmp_path = std::filesystem::path(config.getString("tmp_path", Poco::Path::temp())) / "sentry"; + const std::filesystem::path & default_tmp_path = fs::path(config.getString("tmp_path", fs::temp_directory_path())) / "sentry"; const std::string & endpoint = config.getString("send_crash_reports.endpoint"); const std::string & temp_folder_path = config.getString("send_crash_reports.tmp_path", default_tmp_path); - Poco::File(temp_folder_path).createDirectories(); + fs::create_directories(temp_folder_path); sentry_options_t * options = sentry_options_new(); /// will be freed by sentry_init or sentry_shutdown sentry_options_set_release(options, VERSION_STRING_SHORT); diff --git a/base/loggers/Loggers.cpp b/base/loggers/Loggers.cpp index 913deaf1eb8..80e62d0a6d6 100644 --- a/base/loggers/Loggers.cpp +++ b/base/loggers/Loggers.cpp @@ -6,10 +6,11 @@ #include "OwnFormattingChannel.h" #include "OwnPatternFormatter.h" #include -#include #include #include -#include +#include + +namespace fs = std::filesystem; namespace DB { @@ -20,11 +21,11 @@ namespace DB // TODO: move to libcommon static std::string createDirectory(const std::string & file) { - auto path = Poco::Path(file).makeParent(); - if (path.toString().empty()) + auto path = fs::path(file).parent_path(); + if (path.empty()) return ""; - Poco::File(path).createDirectories(); - return path.toString(); + fs::create_directories(path); + return path; }; void Loggers::setTextLog(std::shared_ptr log, int max_priority) @@ -70,7 +71,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log // Set up two channel chains. log_file = new Poco::FileChannel; - log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString()); + log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(log_path)); log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); @@ -102,7 +103,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log std::cerr << "Logging errors to " << errorlog_path << std::endl; error_log_file = new Poco::FileChannel; - error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString()); + error_log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(errorlog_path)); error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); diff --git a/contrib/avro b/contrib/avro index 92caca2d42f..1ee16d8c5a7 160000 --- a/contrib/avro +++ b/contrib/avro @@ -1 +1 @@ -Subproject commit 92caca2d42fc9a97e34e95f963593539d32ed331 +Subproject commit 1ee16d8c5a7808acff5cf0475f771195d9aa3faa diff --git a/contrib/cassandra b/contrib/cassandra index c097fb5c7e6..eb9b68dadbb 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit c097fb5c7e63cc430016d9a8b240d8e63fbefa52 +Subproject commit eb9b68dadbb4417a2c132ad4a1c2fa76e65e6fc1 diff --git a/docker/packager/packager b/docker/packager/packager index 836f30dec42..81474166cc9 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -154,6 +154,10 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if clang_tidy: cmake_flags.append('-DENABLE_CLANG_TIDY=1') + cmake_flags.append('-DENABLE_UTILS=1') + cmake_flags.append('-DUSE_GTEST=1') + cmake_flags.append('-DENABLE_TESTS=1') + cmake_flags.append('-DENABLE_EXAMPLES=1') # Don't stop on first error to find more clang-tidy errors in one run. result.append('NINJA_FLAGS=-k0') diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 626bedb453c..670fc9e58b3 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -56,17 +56,19 @@ function watchdog sleep 3600 echo "Fuzzing run has timed out" - killall clickhouse-client ||: for _ in {1..10} do - if ! pgrep -f clickhouse-client + # Only kill by pid the particular client that runs the fuzzing, or else + # we can kill some clickhouse-client processes this script starts later, + # e.g. for checking server liveness. + if ! kill $fuzzer_pid then break fi sleep 1 done - killall -9 clickhouse-client ||: + kill -9 -- $fuzzer_pid ||: } function filter_exists @@ -85,7 +87,7 @@ function fuzz { # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests. # Don't overwrite the NEW_TESTS_OPT so that it can be set from the environment. - NEW_TESTS="$(grep -P 'tests/queries/0_stateless/.*\.sql' ci-changed-files.txt | sed -r -e 's!^!ch/!' | sort -R)" + NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\)$!ch/\1!p' ci-changed-files.txt | sort -R)" # ci-changed-files.txt contains also files that has been deleted/renamed, filter them out. NEW_TESTS="$(filter_exists $NEW_TESTS)" if [[ -n "$NEW_TESTS" ]] @@ -115,17 +117,49 @@ continue gdb -batch -command script.gdb -p "$(pidof clickhouse-server)" & - fuzzer_exit_code=0 # SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric. # SC2046: Quote this to prevent word splitting. Actually I need word splitting. # shellcheck disable=SC2012,SC2046 clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) $NEW_TESTS_OPT \ > >(tail -n 100000 > fuzzer.log) \ - 2>&1 \ - || fuzzer_exit_code=$? + 2>&1 & + fuzzer_pid=$! + echo "Fuzzer pid is $fuzzer_pid" + # Start a watchdog that should kill the fuzzer on timeout. + # The shell won't kill the child sleep when we kill it, so we have to put it + # into a separate process group so that we can kill them all. + set -m + watchdog & + watchdog_pid=$! + set +m + # Check that the watchdog has started. + kill -0 $watchdog_pid + + # Wait for the fuzzer to complete. + # Note that the 'wait || ...' thing is required so that the script doesn't + # exit because of 'set -e' when 'wait' returns nonzero code. + fuzzer_exit_code=0 + wait "$fuzzer_pid" || fuzzer_exit_code=$? echo "Fuzzer exit code is $fuzzer_exit_code" + kill -- -$watchdog_pid ||: + + # If the server dies, most often the fuzzer returns code 210: connetion + # refused, and sometimes also code 32: attempt to read after eof. For + # simplicity, check again whether the server is accepting connections, using + # clickhouse-client. We don't check for existence of server process, because + # the process is still present while the server is terminating and not + # accepting the connections anymore. + if clickhouse-client --query "select 1 format Null" + then + server_died=0 + else + echo "Server live check returns $?" + server_died=1 + fi + + # Stop the server. clickhouse-client --query "select elapsed, query from system.processes" ||: killall clickhouse-server ||: for _ in {1..10} @@ -137,6 +171,41 @@ continue sleep 1 done killall -9 clickhouse-server ||: + + # Debug. + date + sleep 10 + jobs + pstree -aspgT + + # Make files with status and description we'll show for this check on Github. + task_exit_code=$fuzzer_exit_code + if [ "$server_died" == 1 ] + then + # The server has died. + task_exit_code=210 + echo "failure" > status.txt + if ! grep -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: AddressSanitizer:.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*\|.*_LIBCPP_ASSERT.*" server.log > description.txt + then + echo "Lost connection to server. See the logs." > description.txt + fi + elif [ "$fuzzer_exit_code" == "143" ] || [ "$fuzzer_exit_code" == "0" ] + then + # Variants of a normal run: + # 0 -- fuzzing ended earlier than timeout. + # 143 -- SIGTERM -- the fuzzer was killed by timeout. + task_exit_code=0 + echo "success" > status.txt + echo "OK" > description.txt + else + # The server was alive, but the fuzzer returned some error. Probably this + # is a problem in the fuzzer itself. Don't grep the server log in this + # case, because we will find a message about normal server termination + # (Received signal 15), which is confusing. + task_exit_code=$fuzzer_exit_code + echo "failure" > status.txt + echo "Fuzzer failed ($fuzzer_exit_code). See the logs." > description.txt + fi } case "$stage" in @@ -165,50 +234,7 @@ case "$stage" in time configure ;& "fuzz") - # Start a watchdog that should kill the fuzzer on timeout. - # The shell won't kill the child sleep when we kill it, so we have to put it - # into a separate process group so that we can kill them all. - set -m - watchdog & - watchdog_pid=$! - set +m - # Check that the watchdog has started - kill -0 $watchdog_pid - - fuzzer_exit_code=0 - time fuzz || fuzzer_exit_code=$? - kill -- -$watchdog_pid ||: - - # Debug - date - sleep 10 - jobs - pstree -aspgT - - # Make files with status and description we'll show for this check on Github - task_exit_code=$fuzzer_exit_code - if [ "$fuzzer_exit_code" == 143 ] - then - # SIGTERM -- the fuzzer was killed by timeout, which means a normal run. - echo "success" > status.txt - echo "OK" > description.txt - task_exit_code=0 - elif [ "$fuzzer_exit_code" == 210 ] - then - # Lost connection to the server. This probably means that the server died - # with abort. - echo "failure" > status.txt - if ! grep -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: AddressSanitizer:.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*\|.*_LIBCPP_ASSERT.*" server.log > description.txt - then - echo "Lost connection to server. See the logs." > description.txt - fi - else - # Something different -- maybe the fuzzer itself died? Don't grep the - # server log in this case, because we will find a message about normal - # server termination (Received signal 15), which is confusing. - echo "failure" > status.txt - echo "Fuzzer failed ($fuzzer_exit_code). See the logs." > description.txt - fi + time fuzz ;& "report") cat > report.html <(c) << std::endl; The same is true for small methods in any classes or structs. -For templated classes and structs, don’t separate the method declarations from the implementation (because otherwise they must be defined in the same translation unit). +For templated classes and structs, do not separate the method declarations from the implementation (because otherwise they must be defined in the same translation unit). **31.** You can wrap lines at 140 characters, instead of 80. @@ -442,7 +442,7 @@ Use `RAII` and see above. **3.** Error handling. -Use exceptions. In most cases, you only need to throw an exception, and don’t need to catch it (because of `RAII`). +Use exceptions. In most cases, you only need to throw an exception, and do not need to catch it (because of `RAII`). In offline data processing applications, it’s often acceptable to not catch exceptions. @@ -599,7 +599,7 @@ public: There is no need to use a separate `namespace` for application code. -Small libraries don’t need this, either. +Small libraries do not need this, either. For medium to large libraries, put everything in a `namespace`. @@ -755,9 +755,9 @@ If there is a good solution already available, then use it, even if it means you (But be prepared to remove bad libraries from code.) -**3.** You can install a library that isn’t in the packages, if the packages don’t have what you need or have an outdated version or the wrong type of compilation. +**3.** You can install a library that isn’t in the packages, if the packages do not have what you need or have an outdated version or the wrong type of compilation. -**4.** If the library is small and doesn’t have its own complex build system, put the source files in the `contrib` folder. +**4.** If the library is small and does not have its own complex build system, put the source files in the `contrib` folder. **5.** Preference is always given to libraries that are already in use. diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 7547497b9af..4231bda6c35 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -35,7 +35,7 @@ Tests should use (create, drop, etc) only tables in `test` database that is assu ### Choosing the Test Name -The name of the test starts with a five-digit prefix followed by a descriptive name, such as `00422_hash_function_constexpr.sql`. To choose the prefix, find the largest prefix already present in the directory, and increment it by one. In the meantime, some other tests might be added with the same numeric prefix, but this is OK and doesn't lead to any problems, you don't have to change it later. +The name of the test starts with a five-digit prefix followed by a descriptive name, such as `00422_hash_function_constexpr.sql`. To choose the prefix, find the largest prefix already present in the directory, and increment it by one. In the meantime, some other tests might be added with the same numeric prefix, but this is OK and does not lead to any problems, you don't have to change it later. Some tests are marked with `zookeeper`, `shard` or `long` in their names. `zookeeper` is for tests that are using ZooKeeper. `shard` is for tests that requires server to listen `127.0.0.*`; `distributed` or `global` have the same meaning. `long` is for tests that run slightly longer that one second. You can disable these groups of tests using `--no-zookeeper`, `--no-shard` and `--no-long` options, respectively. Make sure to add a proper prefix to your test name if it needs ZooKeeper or distributed queries. @@ -51,7 +51,7 @@ Do not check for a particular wording of error message, it may change in the fut ### Testing a Distributed Query -If you want to use distributed queries in functional tests, you can leverage `remote` table function with `127.0.0.{1..2}` addresses for the server to query itself; or you can use predefined test clusters in server configuration file like `test_shard_localhost`. Remember to add the words `shard` or `distributed` to the test name, so that it is ran in CI in correct configurations, where the server is configured to support distributed queries. +If you want to use distributed queries in functional tests, you can leverage `remote` table function with `127.0.0.{1..2}` addresses for the server to query itself; or you can use predefined test clusters in server configuration file like `test_shard_localhost`. Remember to add the words `shard` or `distributed` to the test name, so that it is run in CI in correct configurations, where the server is configured to support distributed queries. ## Known Bugs {#known-bugs} @@ -60,11 +60,11 @@ If we know some bugs that can be easily reproduced by functional tests, we place ## Integration Tests {#integration-tests} -Integration tests allow to test ClickHouse in clustered configuration and ClickHouse interaction with other servers like MySQL, Postgres, MongoDB. They are useful to emulate network splits, packet drops, etc. These tests are run under Docker and create multiple containers with various software. +Integration tests allow testing ClickHouse in clustered configuration and ClickHouse interaction with other servers like MySQL, Postgres, MongoDB. They are useful to emulate network splits, packet drops, etc. These tests are run under Docker and create multiple containers with various software. See `tests/integration/README.md` on how to run these tests. -Note that integration of ClickHouse with third-party drivers is not tested. Also we currently don’t have integration tests with our JDBC and ODBC drivers. +Note that integration of ClickHouse with third-party drivers is not tested. Also, we currently do not have integration tests with our JDBC and ODBC drivers. ## Unit Tests {#unit-tests} @@ -123,7 +123,7 @@ Example with gdb: $ sudo -u clickhouse gdb --args /usr/bin/clickhouse server --config-file /etc/clickhouse-server/config.xml ``` -If the system clickhouse-server is already running and you don’t want to stop it, you can change port numbers in your `config.xml` (or override them in a file in `config.d` directory), provide appropriate data path, and run it. +If the system clickhouse-server is already running and you do not want to stop it, you can change port numbers in your `config.xml` (or override them in a file in `config.d` directory), provide appropriate data path, and run it. `clickhouse` binary has almost no dependencies and works across wide range of Linux distributions. To quick and dirty test your changes on a server, you can simply `scp` your fresh built `clickhouse` binary to your server and then run it as in examples above. @@ -161,7 +161,7 @@ $ clickhouse benchmark --concurrency 16 < queries.tsv Then leave it for a night or weekend and go take a rest. -You should check that `clickhouse-server` doesn’t crash, memory footprint is bounded and performance not degrading over time. +You should check that `clickhouse-server` does not crash, memory footprint is bounded and performance not degrading over time. Precise query execution timings are not recorded and not compared due to high variability of queries and environment. @@ -230,7 +230,7 @@ Fuzzers are not built by default. To build fuzzers both `-DENABLE_FUZZING=1` and We recommend to disable Jemalloc while building fuzzers. Configuration used to integrate ClickHouse fuzzing to Google OSS-Fuzz can be found at `docker/fuzz`. -We also use simple fuzz test to generate random SQL queries and to check that the server doesn’t die executing them. +We also use simple fuzz test to generate random SQL queries and to check that the server does not die executing them. You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer). We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. You can learn more about this fuzzer in [this blog article](https://clickhouse.tech/blog/en/2021/fuzzing-clickhouse/). @@ -332,7 +332,7 @@ We run tests with Yandex internal CI and job automation system named “Sandbox Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored for several months. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you. -We don’t use Travis CI due to the limit on time and computational power. -We don’t use Jenkins. It was used before and now we are happy we are not using Jenkins. +We do not use Travis CI due to the limit on time and computational power. +We do not use Jenkins. It was used before and now we are happy we are not using Jenkins. [Original article](https://clickhouse.tech/docs/en/development/tests/) diff --git a/docs/en/engines/database-engines/atomic.md b/docs/en/engines/database-engines/atomic.md index d897631dd6e..4f5f69a5ab7 100644 --- a/docs/en/engines/database-engines/atomic.md +++ b/docs/en/engines/database-engines/atomic.md @@ -47,7 +47,7 @@ EXCHANGE TABLES new_table AND old_table; ### ReplicatedMergeTree in Atomic Database {#replicatedmergetree-in-atomic-database} -For [ReplicatedMergeTree](../table-engines/mergetree-family/replication.md#table_engines-replication) tables is recomended do not specify parameters of engine - path in ZooKeeper and replica name. In this case will be used parameters of the configuration [default_replica_path](../../operations/server-configuration-parameters/settings.md#default_replica_path) and [default_replica_name](../../operations/server-configuration-parameters/settings.md#default_replica_name). If you want specify parameters of engine explicitly than recomended to use {uuid} macros. This is useful so that unique paths are automatically generated for each table in the ZooKeeper. +For [ReplicatedMergeTree](../table-engines/mergetree-family/replication.md#table_engines-replication) tables, it is recommended to not specify engine parameters - path in ZooKeeper and replica name. In this case, configuration parameters will be used [default_replica_path](../../operations/server-configuration-parameters/settings.md#default_replica_path) and [default_replica_name](../../operations/server-configuration-parameters/settings.md#default_replica_name). If you want to specify engine parameters explicitly, it is recommended to use {uuid} macros. This is useful so that unique paths are automatically generated for each table in ZooKeeper. ## See Also diff --git a/docs/en/engines/table-engines/index.md b/docs/en/engines/table-engines/index.md index eb4fc583f88..13b3395e15b 100644 --- a/docs/en/engines/table-engines/index.md +++ b/docs/en/engines/table-engines/index.md @@ -82,8 +82,8 @@ Virtual column is an integral table engine attribute that is defined in the engi You shouldn’t specify virtual columns in the `CREATE TABLE` query and you can’t see them in `SHOW CREATE TABLE` and `DESCRIBE TABLE` query results. Virtual columns are also read-only, so you can’t insert data into virtual columns. -To select data from a virtual column, you must specify its name in the `SELECT` query. `SELECT *` doesn’t return values from virtual columns. +To select data from a virtual column, you must specify its name in the `SELECT` query. `SELECT *` does not return values from virtual columns. -If you create a table with a column that has the same name as one of the table virtual columns, the virtual column becomes inaccessible. We don’t recommend doing this. To help avoid conflicts, virtual column names are usually prefixed with an underscore. +If you create a table with a column that has the same name as one of the table virtual columns, the virtual column becomes inaccessible. We do not recommend doing this. To help avoid conflicts, virtual column names are usually prefixed with an underscore. [Original article](https://clickhouse.tech/docs/en/engines/table-engines/) diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 2eebf5bdb92..a3a13f9d152 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -40,7 +40,7 @@ Required parameters: - `kafka_broker_list` — A comma-separated list of brokers (for example, `localhost:9092`). - `kafka_topic_list` — A list of Kafka topics. -- `kafka_group_name` — A group of Kafka consumers. Reading margins are tracked for each group separately. If you don’t want messages to be duplicated in the cluster, use the same group name everywhere. +- `kafka_group_name` — A group of Kafka consumers. Reading margins are tracked for each group separately. If you do not want messages to be duplicated in the cluster, use the same group name everywhere. - `kafka_format` — Message format. Uses the same notation as the SQL `FORMAT` function, such as `JSONEachRow`. For more information, see the [Formats](../../../interfaces/formats.md) section. Optional parameters: diff --git a/docs/en/engines/table-engines/log-family/index.md b/docs/en/engines/table-engines/log-family/index.md index 1f6d88c20e3..8cdde239f44 100644 --- a/docs/en/engines/table-engines/log-family/index.md +++ b/docs/en/engines/table-engines/log-family/index.md @@ -38,7 +38,7 @@ Engines: ## Differences {#differences} -The `TinyLog` engine is the simplest in the family and provides the poorest functionality and lowest efficiency. The `TinyLog` engine doesn’t support parallel data reading by several threads in a single query. It reads data slower than other engines in the family that support parallel reading from a single query and it uses almost as many file descriptors as the `Log` engine because it stores each column in a separate file. Use it only in simple scenarios. +The `TinyLog` engine is the simplest in the family and provides the poorest functionality and lowest efficiency. The `TinyLog` engine does not support parallel data reading by several threads in a single query. It reads data slower than other engines in the family that support parallel reading from a single query and it uses almost as many file descriptors as the `Log` engine because it stores each column in a separate file. Use it only in simple scenarios. The `Log` and `StripeLog` engines support parallel data reading. When reading data, ClickHouse uses multiple threads. Each thread processes a separate data block. The `Log` engine uses a separate file for each column of the table. `StripeLog` stores all the data in one file. As a result, the `StripeLog` engine uses fewer file descriptors, but the `Log` engine provides higher efficiency when reading data. diff --git a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md index ea0b265d652..4ec976eda30 100644 --- a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -126,7 +126,7 @@ Also when there are at least 2 more “state” rows than “cancel” rows, or Thus, collapsing should not change the results of calculating statistics. Changes gradually collapsed so that in the end only the last state of almost every object left. -The `Sign` is required because the merging algorithm doesn’t guarantee that all of the rows with the same sorting key will be in the same resulting data part and even on the same physical server. ClickHouse process `SELECT` queries with multiple threads, and it can not predict the order of rows in the result. The aggregation is required if there is a need to get completely “collapsed” data from `CollapsingMergeTree` table. +The `Sign` is required because the merging algorithm does not guarantee that all of the rows with the same sorting key will be in the same resulting data part and even on the same physical server. ClickHouse process `SELECT` queries with multiple threads, and it can not predict the order of rows in the result. The aggregation is required if there is a need to get completely “collapsed” data from `CollapsingMergeTree` table. To finalize collapsing, write a query with `GROUP BY` clause and aggregate functions that account for the sign. For example, to calculate quantity, use `sum(Sign)` instead of `count()`. To calculate the sum of something, use `sum(Sign * x)` instead of `sum(x)`, and so on, and also add `HAVING sum(Sign) > 0`. diff --git a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md index 855d5fdadf4..535922875ef 100644 --- a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -33,6 +33,8 @@ ORDER BY (CounterID, StartDate, intHash32(UserID)); In this example, we set partitioning by the event types that occurred during the current week. +By default, the floating-point partition key is not supported. To use it enable the setting [allow_floating_point_partition_key](../../../operations/settings/merge-tree-settings.md#allow_floating_point_partition_key). + When inserting new data to a table, this data is stored as a separate part (chunk) sorted by the primary key. In 10-15 minutes after inserting, the parts of the same partition are merged into the entire part. !!! info "Info" diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index 14df9ae130e..3ead798503d 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -7,7 +7,7 @@ toc_title: GraphiteMergeTree This engine is designed for thinning and aggregating/averaging (rollup) [Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite. -You can use any ClickHouse table engine to store the Graphite data if you don’t need rollup, but if you need a rollup use `GraphiteMergeTree`. The engine reduces the volume of storage and increases the efficiency of queries from Graphite. +You can use any ClickHouse table engine to store the Graphite data if you do not need rollup, but if you need a rollup use `GraphiteMergeTree`. The engine reduces the volume of storage and increases the efficiency of queries from Graphite. The engine inherits properties from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md). diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 8743090df41..e385b234cd8 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -64,7 +64,7 @@ For a description of parameters, see the [CREATE query description](../../../sql ClickHouse uses the sorting key as a primary key if the primary key is not defined obviously by the `PRIMARY KEY` clause. - Use the `ORDER BY tuple()` syntax, if you don’t need sorting. See [Selecting the Primary Key](#selecting-the-primary-key). + Use the `ORDER BY tuple()` syntax, if you do not need sorting. See [Selecting the Primary Key](#selecting-the-primary-key). - `PARTITION BY` — The [partitioning key](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md). Optional. @@ -162,7 +162,7 @@ Data parts can be stored in `Wide` or `Compact` format. In `Wide` format each co Data storing format is controlled by the `min_bytes_for_wide_part` and `min_rows_for_wide_part` settings of the table engine. If the number of bytes or rows in a data part is less then the corresponding setting's value, the part is stored in `Compact` format. Otherwise it is stored in `Wide` format. If none of these settings is set, data parts are stored in `Wide` format. -Each data part is logically divided into granules. A granule is the smallest indivisible data set that ClickHouse reads when selecting data. ClickHouse doesn’t split rows or values, so each granule always contains an integer number of rows. The first row of a granule is marked with the value of the primary key for the row. For each data part, ClickHouse creates an index file that stores the marks. For each column, whether it’s in the primary key or not, ClickHouse also stores the same marks. These marks let you find data directly in column files. +Each data part is logically divided into granules. A granule is the smallest indivisible data set that ClickHouse reads when selecting data. ClickHouse does not split rows or values, so each granule always contains an integer number of rows. The first row of a granule is marked with the value of the primary key for the row. For each data part, ClickHouse creates an index file that stores the marks. For each column, whether it’s in the primary key or not, ClickHouse also stores the same marks. These marks let you find data directly in column files. The granule size is restricted by the `index_granularity` and `index_granularity_bytes` settings of the table engine. The number of rows in a granule lays in the `[1, index_granularity]` range, depending on the size of the rows. The size of a granule can exceed `index_granularity_bytes` if the size of a single row is greater than the value of the setting. In this case, the size of the granule equals the size of the row. @@ -227,7 +227,7 @@ This feature is helpful when using the [SummingMergeTree](../../../engines/table In this case it makes sense to leave only a few columns in the primary key that will provide efficient range scans and add the remaining dimension columns to the sorting key tuple. -[ALTER](../../../sql-reference/statements/alter/index.md) of the sorting key is a lightweight operation because when a new column is simultaneously added to the table and to the sorting key, existing data parts don’t need to be changed. Since the old sorting key is a prefix of the new sorting key and there is no data in the newly added column, the data is sorted by both the old and new sorting keys at the moment of table modification. +[ALTER](../../../sql-reference/statements/alter/index.md) of the sorting key is a lightweight operation because when a new column is simultaneously added to the table and to the sorting key, existing data parts do not need to be changed. Since the old sorting key is a prefix of the new sorting key and there is no data in the newly added column, the data is sorted by both the old and new sorting keys at the moment of table modification. ### Use of Indexes and Partitions in Queries {#use-of-indexes-and-partitions-in-queries} @@ -265,7 +265,7 @@ The key for partitioning by month allows reading only those data blocks which co Consider, for example, the days of the month. They form a [monotonic sequence](https://en.wikipedia.org/wiki/Monotonic_function) for one month, but not monotonic for more extended periods. This is a partially-monotonic sequence. If a user creates the table with partially-monotonic primary key, ClickHouse creates a sparse index as usual. When a user selects data from this kind of table, ClickHouse analyzes the query conditions. If the user wants to get data between two marks of the index and both these marks fall within one month, ClickHouse can use the index in this particular case because it can calculate the distance between the parameters of a query and index marks. -ClickHouse cannot use an index if the values of the primary key in the query parameter range don’t represent a monotonic sequence. In this case, ClickHouse uses the full scan method. +ClickHouse cannot use an index if the values of the primary key in the query parameter range do not represent a monotonic sequence. In this case, ClickHouse uses the full scan method. ClickHouse uses this logic not only for days of the month sequences, but for any primary key that represents a partially-monotonic sequence. diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md index b82bc65afc2..ca0db24e640 100644 --- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md @@ -7,9 +7,9 @@ toc_title: ReplacingMergeTree The engine differs from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree) in that it removes duplicate entries with the same [sorting key](../../../engines/table-engines/mergetree-family/mergetree.md) value (`ORDER BY` table section, not `PRIMARY KEY`). -Data deduplication occurs only during a merge. Merging occurs in the background at an unknown time, so you can’t plan for it. Some of the data may remain unprocessed. Although you can run an unscheduled merge using the `OPTIMIZE` query, don’t count on using it, because the `OPTIMIZE` query will read and write a large amount of data. +Data deduplication occurs only during a merge. Merging occurs in the background at an unknown time, so you can’t plan for it. Some of the data may remain unprocessed. Although you can run an unscheduled merge using the `OPTIMIZE` query, do not count on using it, because the `OPTIMIZE` query will read and write a large amount of data. -Thus, `ReplacingMergeTree` is suitable for clearing out duplicate data in the background in order to save space, but it doesn’t guarantee the absence of duplicates. +Thus, `ReplacingMergeTree` is suitable for clearing out duplicate data in the background in order to save space, but it does not guarantee the absence of duplicates. ## Creating a Table {#creating-a-table} @@ -34,7 +34,7 @@ For a description of request parameters, see [statement description](../../../sq **ReplacingMergeTree Parameters** -- `ver` — column with version. Type `UInt*`, `Date` or `DateTime`. Optional parameter. +- `ver` — column with the version number. Type `UInt*`, `Date`, `DateTime` or `DateTime64`. Optional parameter. When merging, `ReplacingMergeTree` from all the rows with the same sorting key leaves only one: @@ -66,5 +66,3 @@ All of the parameters excepting `ver` have the same meaning as in `MergeTree`. - `ver` - column with the version. Optional parameter. For a description, see the text above. - -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/replacingmergetree/) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 8e4698b9bf8..2db6686beb7 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -95,7 +95,7 @@ If ZooKeeper isn’t set in the config file, you can’t create replicated table ZooKeeper is not used in `SELECT` queries because replication does not affect the performance of `SELECT` and queries run just as fast as they do for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../../../operations/settings/settings.md#settings-max_replica_delay_for_distributed_queries) and [fallback_to_stale_replicas_for_distributed_queries](../../../operations/settings/settings.md#settings-fallback_to_stale_replicas_for_distributed_queries). -For each `INSERT` query, approximately ten entries are added to ZooKeeper through several transactions. (To be more precise, this is for each inserted block of data; an INSERT query contains one block or one block per `max_insert_block_size = 1048576` rows.) This leads to slightly longer latencies for `INSERT` compared to non-replicated tables. But if you follow the recommendations to insert data in batches of no more than one `INSERT` per second, it doesn’t create any problems. The entire ClickHouse cluster used for coordinating one ZooKeeper cluster has a total of several hundred `INSERTs` per second. The throughput on data inserts (the number of rows per second) is just as high as for non-replicated data. +For each `INSERT` query, approximately ten entries are added to ZooKeeper through several transactions. (To be more precise, this is for each inserted block of data; an INSERT query contains one block or one block per `max_insert_block_size = 1048576` rows.) This leads to slightly longer latencies for `INSERT` compared to non-replicated tables. But if you follow the recommendations to insert data in batches of no more than one `INSERT` per second, it does not create any problems. The entire ClickHouse cluster used for coordinating one ZooKeeper cluster has a total of several hundred `INSERTs` per second. The throughput on data inserts (the number of rows per second) is just as high as for non-replicated data. For very large clusters, you can use different ZooKeeper clusters for different shards. However, this hasn’t proven necessary on the Yandex.Metrica cluster (approximately 300 servers). @@ -107,7 +107,7 @@ By default, an INSERT query waits for confirmation of writing the data from only Each block of data is written atomically. The INSERT query is divided into blocks up to `max_insert_block_size = 1048576` rows. In other words, if the `INSERT` query has less than 1048576 rows, it is made atomically. -Data blocks are deduplicated. For multiple writes of the same data block (data blocks of the same size containing the same rows in the same order), the block is only written once. The reason for this is in case of network failures when the client application doesn’t know if the data was written to the DB, so the `INSERT` query can simply be repeated. It doesn’t matter which replica INSERTs were sent to with identical data. `INSERTs` are idempotent. Deduplication parameters are controlled by [merge_tree](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) server settings. +Data blocks are deduplicated. For multiple writes of the same data block (data blocks of the same size containing the same rows in the same order), the block is only written once. The reason for this is in case of network failures when the client application does not know if the data was written to the DB, so the `INSERT` query can simply be repeated. It does not matter which replica INSERTs were sent to with identical data. `INSERTs` are idempotent. Deduplication parameters are controlled by [merge_tree](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree) server settings. During replication, only the source data to insert is transferred over the network. Further data transformation (merging) is coordinated and performed on all the replicas in the same way. This minimizes network usage, which means that replication works well when replicas reside in different datacenters. (Note that duplicating data in different datacenters is the main goal of replication.) @@ -174,7 +174,7 @@ In this case, the path consists of the following parts: `{layer}-{shard}` is the shard identifier. In this example it consists of two parts, since the Yandex.Metrica cluster uses bi-level sharding. For most tasks, you can leave just the {shard} substitution, which will be expanded to the shard identifier. -`table_name` is the name of the node for the table in ZooKeeper. It is a good idea to make it the same as the table name. It is defined explicitly, because in contrast to the table name, it doesn’t change after a RENAME query. +`table_name` is the name of the node for the table in ZooKeeper. It is a good idea to make it the same as the table name. It is defined explicitly, because in contrast to the table name, it does not change after a RENAME query. *HINT*: you could add a database name in front of `table_name` as well. E.g. `db_name.table_name` The two built-in substitutions `{database}` and `{table}` can be used, they expand into the table name and the database name respectively (unless these macros are defined in the `macros` section). So the zookeeper path can be specified as `'/clickhouse/tables/{layer}-{shard}/{database}/{table}'`. diff --git a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md index 1f23e4daf51..9bfd1816d32 100644 --- a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md @@ -7,7 +7,7 @@ toc_title: SummingMergeTree The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree). The difference is that when merging data parts for `SummingMergeTree` tables ClickHouse replaces all the rows with the same primary key (or more accurately, with the same [sorting key](../../../engines/table-engines/mergetree-family/mergetree.md)) with one row which contains summarized values for the columns with the numeric data type. If the sorting key is composed in a way that a single key value corresponds to large number of rows, this significantly reduces storage volume and speeds up data selection. -We recommend to use the engine together with `MergeTree`. Store complete data in `MergeTree` table, and use `SummingMergeTree` for aggregated data storing, for example, when preparing reports. Such an approach will prevent you from losing valuable data due to an incorrectly composed primary key. +We recommend using the engine together with `MergeTree`. Store complete data in `MergeTree` table, and use `SummingMergeTree` for aggregated data storing, for example, when preparing reports. Such an approach will prevent you from losing valuable data due to an incorrectly composed primary key. ## Creating a Table {#creating-a-table} diff --git a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index b23139b402b..93c35344e24 100644 --- a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -133,7 +133,7 @@ When ClickHouse inserts data, it orders rows by the primary key. If the `Version ## Selecting Data {#selecting-data} -ClickHouse doesn’t guarantee that all of the rows with the same primary key will be in the same resulting data part or even on the same physical server. This is true both for writing the data and for subsequent merging of the data parts. In addition, ClickHouse processes `SELECT` queries with multiple threads, and it cannot predict the order of rows in the result. This means that aggregation is required if there is a need to get completely “collapsed” data from a `VersionedCollapsingMergeTree` table. +ClickHouse does not guarantee that all of the rows with the same primary key will be in the same resulting data part or even on the same physical server. This is true both for writing the data and for subsequent merging of the data parts. In addition, ClickHouse processes `SELECT` queries with multiple threads, and it cannot predict the order of rows in the result. This means that aggregation is required if there is a need to get completely “collapsed” data from a `VersionedCollapsingMergeTree` table. To finalize collapsing, write a query with a `GROUP BY` clause and aggregate functions that account for the sign. For example, to calculate quantity, use `sum(Sign)` instead of `count()`. To calculate the sum of something, use `sum(Sign * x)` instead of `sum(x)`, and add `HAVING sum(Sign) > 0`. @@ -219,7 +219,7 @@ HAVING sum(Sign) > 0 └─────────────────────┴───────────┴──────────┴─────────┘ ``` -If we don’t need aggregation and want to force collapsing, we can use the `FINAL` modifier for the `FROM` clause. +If we do not need aggregation and want to force collapsing, we can use the `FINAL` modifier for the `FROM` clause. ``` sql SELECT * FROM UAct FINAL diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index 8245cd19e8c..cacb310a15c 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -20,11 +20,11 @@ Engine parameters: Optional engine parameters: -- `flush_time`, `flush_rows`, `flush_bytes` – Conditions for flushing data from the buffer, that will happen only in background (ommited or zero means no `flush*` parameters). +- `flush_time`, `flush_rows`, `flush_bytes` – Conditions for flushing data from the buffer, that will happen only in background (omitted or zero means no `flush*` parameters). Data is flushed from the buffer and written to the destination table if all the `min*` conditions or at least one `max*` condition are met. -Also if at least one `flush*` condition are met flush initiated in background, this is different from `max*`, since `flush*` allows you to configure background flushes separately to avoid adding latency for `INSERT` (into `Buffer`) queries. +Also, if at least one `flush*` condition are met flush initiated in background, this is different from `max*`, since `flush*` allows you to configure background flushes separately to avoid adding latency for `INSERT` (into `Buffer`) queries. - `min_time`, `max_time`, `flush_time` – Condition for the time in seconds from the moment of the first write to the buffer. - `min_rows`, `max_rows`, `flush_rows` – Condition for the number of rows in the buffer. @@ -49,12 +49,12 @@ You can set empty strings in single quotation marks for the database and table n When reading from a Buffer table, data is processed both from the buffer and from the destination table (if there is one). Note that the Buffer tables does not support an index. In other words, data in the buffer is fully scanned, which might be slow for large buffers. (For data in a subordinate table, the index that it supports will be used.) -If the set of columns in the Buffer table doesn’t match the set of columns in a subordinate table, a subset of columns that exist in both tables is inserted. +If the set of columns in the Buffer table does not match the set of columns in a subordinate table, a subset of columns that exist in both tables is inserted. -If the types don’t match for one of the columns in the Buffer table and a subordinate table, an error message is entered in the server log and the buffer is cleared. -The same thing happens if the subordinate table doesn’t exist when the buffer is flushed. +If the types do not match for one of the columns in the Buffer table and a subordinate table, an error message is entered in the server log, and the buffer is cleared. +The same thing happens if the subordinate table does not exist when the buffer is flushed. -If you need to run ALTER for a subordinate table and the Buffer table, we recommend first deleting the Buffer table, running ALTER for the subordinate table, then creating the Buffer table again. +If you need to run ALTER for a subordinate table, and the Buffer table, we recommend first deleting the Buffer table, running ALTER for the subordinate table, then creating the Buffer table again. If the server is restarted abnormally, the data in the buffer is lost. @@ -70,6 +70,6 @@ Due to these disadvantages, we can only recommend using a Buffer table in rare c A Buffer table is used when too many INSERTs are received from a large number of servers over a unit of time and data can’t be buffered before insertion, which means the INSERTs can’t run fast enough. -Note that it doesn’t make sense to insert data one row at a time, even for Buffer tables. This will only produce a speed of a few thousand rows per second, while inserting larger blocks of data can produce over a million rows per second (see the section “Performance”). +Note that it does not make sense to insert data one row at a time, even for Buffer tables. This will only produce a speed of a few thousand rows per second, while inserting larger blocks of data can produce over a million rows per second (see the section “Performance”). [Original article](https://clickhouse.tech/docs/en/operations/table_engines/buffer/) diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index c47e0c27cd2..6de6602a216 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -25,7 +25,7 @@ The Distributed engine accepts parameters: - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) setting - [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) for the examples -Also it accept the following settings: +Also, it accepts the following settings: - `fsync_after_insert` - do the `fsync` for the file data after asynchronous insert to Distributed. Guarantees that the OS flushed the whole inserted data to a file **on the initiator node** disk. @@ -124,7 +124,7 @@ Replicas are duplicating servers (in order to read all the data, you can access Cluster names must not contain dots. The parameters `host`, `port`, and optionally `user`, `password`, `secure`, `compression` are specified for each server: -- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn’t start. If you change the DNS record, restart the server. +- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server does not start. If you change the DNS record, restart the server. - `port` – The TCP port for messenger activity (`tcp_port` in the config, usually set to 9000). Do not confuse it with http_port. - `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [Access rights](../../../operations/access-rights.md). - `password` – The password for connecting to a remote server (not masked). Default value: empty string. @@ -143,13 +143,13 @@ To view your clusters, use the `system.clusters` table. The Distributed engine allows working with a cluster like a local server. However, the cluster is inextensible: you must write its configuration in the server config file (even better, for all the cluster’s servers). -The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don’t need to create a Distributed table – use the `remote` table function instead. See the section [Table functions](../../../sql-reference/table-functions/index.md). +The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you do not need to create a Distributed table – use the `remote` table function instead. See the section [Table functions](../../../sql-reference/table-functions/index.md). There are two methods for writing data to a cluster: First, you can define which servers to write which data to and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table “looks at”. This is the most flexible solution as you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. This is also the most optimal solution since data can be written to different shards completely independently. -Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across the servers itself. In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it doesn’t mean anything in this case. +Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across the servers itself. In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it does not mean anything in this case. Each shard can have a weight defined in the config file. By default, the weight is equal to one. Data is distributed across shards in the amount proportional to the shard weight. For example, if there are two shards and the first has a weight of 9 while the second has a weight of 10, the first will be sent 9 / 19 parts of the rows, and the second will be sent 10 / 19. @@ -165,7 +165,7 @@ The sharding expression can be any expression from constants and table columns t A simple reminder from the division is a limited solution for sharding and isn’t always appropriate. It works for medium and large volumes of data (dozens of servers), but not for very large volumes of data (hundreds of servers or more). In the latter case, use the sharding scheme required by the subject area, rather than using entries in Distributed tables. -SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don’t have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. +SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you do not have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. You should be concerned about the sharding scheme in the following cases: diff --git a/docs/en/engines/table-engines/special/external-data.md b/docs/en/engines/table-engines/special/external-data.md index 88d76b3805e..b5429ccad12 100644 --- a/docs/en/engines/table-engines/special/external-data.md +++ b/docs/en/engines/table-engines/special/external-data.md @@ -9,7 +9,7 @@ ClickHouse allows sending a server the data that is needed for processing a quer For example, if you have a text file with important user identifiers, you can upload it to the server along with a query that uses filtration by this list. -If you need to run more than one query with a large volume of external data, don’t use this feature. It is better to upload the data to the DB ahead of time. +If you need to run more than one query with a large volume of external data, do not use this feature. It is better to upload the data to the DB ahead of time. External data can be uploaded using the command-line client (in non-interactive mode), or using the HTTP interface. diff --git a/docs/en/engines/table-engines/special/file.md b/docs/en/engines/table-engines/special/file.md index 2acec40ef02..17eef2b4941 100644 --- a/docs/en/engines/table-engines/special/file.md +++ b/docs/en/engines/table-engines/special/file.md @@ -24,7 +24,7 @@ The `Format` parameter specifies one of the available file formats. To perform `INSERT` queries – for output. The available formats are listed in the [Formats](../../../interfaces/formats.md#formats) section. -ClickHouse does not allow to specify filesystem path for`File`. It will use folder defined by [path](../../../operations/server-configuration-parameters/settings.md) setting in server configuration. +ClickHouse does not allow specifying filesystem path for`File`. It will use folder defined by [path](../../../operations/server-configuration-parameters/settings.md) setting in server configuration. When creating table using `File(Format)` it creates empty subdirectory in that folder. When data is written to that table, it’s put into `data.Format` file in that subdirectory. diff --git a/docs/en/engines/table-engines/special/join.md b/docs/en/engines/table-engines/special/join.md index 30dbec73939..4cd1c741352 100644 --- a/docs/en/engines/table-engines/special/join.md +++ b/docs/en/engines/table-engines/special/join.md @@ -28,7 +28,7 @@ See the detailed description of the [CREATE TABLE](../../../sql-reference/statem - `join_type` – [JOIN type](../../../sql-reference/statements/select/join.md#select-join-types). - `k1[, k2, ...]` – Key columns from the `USING` clause that the `JOIN` operation is made with. -Enter `join_strictness` and `join_type` parameters without quotes, for example, `Join(ANY, LEFT, col1)`. They must match the `JOIN` operation that the table will be used for. If the parameters don’t match, ClickHouse doesn’t throw an exception and may return incorrect data. +Enter `join_strictness` and `join_type` parameters without quotes, for example, `Join(ANY, LEFT, col1)`. They must match the `JOIN` operation that the table will be used for. If the parameters do not match, ClickHouse does not throw an exception and may return incorrect data. ## Table Usage {#table-usage} diff --git a/docs/en/engines/table-engines/special/memory.md b/docs/en/engines/table-engines/special/memory.md index a6c833ebdba..b6402c2030b 100644 --- a/docs/en/engines/table-engines/special/memory.md +++ b/docs/en/engines/table-engines/special/memory.md @@ -6,7 +6,7 @@ toc_title: Memory # Memory Table Engine {#memory} The Memory engine stores data in RAM, in uncompressed form. Data is stored in exactly the same form as it is received when read. In other words, reading from this table is completely free. -Concurrent data access is synchronized. Locks are short: read and write operations don’t block each other. +Concurrent data access is synchronized. Locks are short: read and write operations do not block each other. Indexes are not supported. Reading is parallelized. Maximal productivity (over 10 GB/sec) is reached on simple queries, because there is no reading from the disk, decompressing, or deserializing data. (We should note that in many cases, the productivity of the MergeTree engine is almost as high.) diff --git a/docs/en/faq/general/columnar-database.md b/docs/en/faq/general/columnar-database.md index 1c6a2bc2989..e30b4a94a87 100644 --- a/docs/en/faq/general/columnar-database.md +++ b/docs/en/faq/general/columnar-database.md @@ -22,4 +22,4 @@ Here is the illustration of the difference between traditional row-oriented syst **Columnar** ![Columnar](https://clickhouse.tech/docs/en/images/column-oriented.gif#) -A columnar database is a preferred choice for analytical applications because it allows to have many columns in a table just in case, but don’t pay the cost for unused columns on read query execution time. Column-oriented databases are designed for big data processing because and data warehousing, they often natively scale using distributed clusters of low-cost hardware to increase throughput. ClickHouse does it with combination of [distributed](../../engines/table-engines/special/distributed.md) and [replicated](../../engines/table-engines/mergetree-family/replication.md) tables. +A columnar database is a preferred choice for analytical applications because it allows to have many columns in a table just in case, but do not pay the cost for unused columns on read query execution time. Column-oriented databases are designed for big data processing because and data warehousing, they often natively scale using distributed clusters of low-cost hardware to increase throughput. ClickHouse does it with combination of [distributed](../../engines/table-engines/special/distributed.md) and [replicated](../../engines/table-engines/mergetree-family/replication.md) tables. diff --git a/docs/en/faq/general/dbms-naming.md b/docs/en/faq/general/dbms-naming.md index 88a66659ab3..d4e87ff450a 100644 --- a/docs/en/faq/general/dbms-naming.md +++ b/docs/en/faq/general/dbms-naming.md @@ -6,7 +6,7 @@ toc_priority: 10 # What Does “ClickHouse” Mean? {#what-does-clickhouse-mean} -It’s a combination of “**Click**stream” and “Data ware**House**”. It comes from the original use case at Yandex.Metrica, where ClickHouse was supposed to keep records of all clicks by people from all over the Internet and it still does the job. You can read more about this use case on [ClickHouse history](../../introduction/history.md) page. +It’s a combination of “**Click**stream” and “Data ware**House**”. It comes from the original use case at Yandex.Metrica, where ClickHouse was supposed to keep records of all clicks by people from all over the Internet, and it still does the job. You can read more about this use case on [ClickHouse history](../../introduction/history.md) page. This two-part meaning has two consequences: diff --git a/docs/en/faq/general/ne-tormozit.md b/docs/en/faq/general/ne-tormozit.md index 44fe686d670..17c5479fa6d 100644 --- a/docs/en/faq/general/ne-tormozit.md +++ b/docs/en/faq/general/ne-tormozit.md @@ -15,9 +15,9 @@ One of the following batches of those t-shirts was supposed to be given away on So, what does it mean? Here are some ways to translate *“не тормозит”*: -- If you translate it literally, it’d be something like *“ClickHouse doesn’t press the brake pedal”*. +- If you translate it literally, it’d be something like *“ClickHouse does not press the brake pedal”*. - If you’d want to express it as close to how it sounds to a Russian person with IT background, it’d be something like *“If your larger system lags, it’s not because it uses ClickHouse”*. -- Shorter, but not so precise versions could be *“ClickHouse is not slow”*, *“ClickHouse doesn’t lag”* or just *“ClickHouse is fast”*. +- Shorter, but not so precise versions could be *“ClickHouse is not slow”*, *“ClickHouse does not lag”* or just *“ClickHouse is fast”*. If you haven’t seen one of those t-shirts in person, you can check them out online in many ClickHouse-related videos. For example, this one: diff --git a/docs/en/faq/general/olap.md b/docs/en/faq/general/olap.md index f023b8c3524..1f6df183f8c 100644 --- a/docs/en/faq/general/olap.md +++ b/docs/en/faq/general/olap.md @@ -31,7 +31,7 @@ All database management systems could be classified into two groups: OLAP (Onlin In practice OLAP and OLTP are not categories, it’s more like a spectrum. Most real systems usually focus on one of them but provide some solutions or workarounds if the opposite kind of workload is also desired. This situation often forces businesses to operate multiple storage systems integrated, which might be not so big deal but having more systems make it more expensive to maintain. So the trend of recent years is HTAP (**Hybrid Transactional/Analytical Processing**) when both kinds of the workload are handled equally well by a single database management system. -Even if a DBMS started as a pure OLAP or pure OLTP, they are forced to move towards that HTAP direction to keep up with their competition. And ClickHouse is no exception, initially, it has been designed as [fast-as-possible OLAP system](../../faq/general/why-clickhouse-is-so-fast.md) and it still doesn’t have full-fledged transaction support, but some features like consistent read/writes and mutations for updating/deleting data had to be added. +Even if a DBMS started as a pure OLAP or pure OLTP, they are forced to move towards that HTAP direction to keep up with their competition. And ClickHouse is no exception, initially, it has been designed as [fast-as-possible OLAP system](../../faq/general/why-clickhouse-is-so-fast.md) and it still does not have full-fledged transaction support, but some features like consistent read/writes and mutations for updating/deleting data had to be added. The fundamental trade-off between OLAP and OLTP systems remains: diff --git a/docs/en/faq/general/who-is-using-clickhouse.md b/docs/en/faq/general/who-is-using-clickhouse.md index 2ae07507123..b7ff867d726 100644 --- a/docs/en/faq/general/who-is-using-clickhouse.md +++ b/docs/en/faq/general/who-is-using-clickhouse.md @@ -6,9 +6,9 @@ toc_priority: 9 # Who Is Using ClickHouse? {#who-is-using-clickhouse} -Being an open-source product makes this question not so straightforward to answer. You don’t have to tell anyone if you want to start using ClickHouse, you just go grab source code or pre-compiled packages. There’s no contract to sign and the [Apache 2.0 license](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) allows for unconstrained software distribution. +Being an open-source product makes this question not so straightforward to answer. You do not have to tell anyone if you want to start using ClickHouse, you just go grab source code or pre-compiled packages. There’s no contract to sign and the [Apache 2.0 license](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) allows for unconstrained software distribution. -Also, the technology stack is often in a grey zone of what’s covered by an NDA. Some companies consider technologies they use as a competitive advantage even if they are open-source and don’t allow employees to share any details publicly. Some see some PR risks and allow employees to share implementation details only with their PR department approval. +Also, the technology stack is often in a grey zone of what’s covered by an NDA. Some companies consider technologies they use as a competitive advantage even if they are open-source and do not allow employees to share any details publicly. Some see some PR risks and allow employees to share implementation details only with their PR department approval. So how to tell who is using ClickHouse? diff --git a/docs/en/faq/index.md b/docs/en/faq/index.md index 1ae71cf680c..1e9c3b8ae64 100644 --- a/docs/en/faq/index.md +++ b/docs/en/faq/index.md @@ -39,7 +39,7 @@ Question candidates: - How to kill a process (query) in ClickHouse? - How to implement pivot (like in pandas)? - How to remove the default ClickHouse user through users.d? -- Importing MySQL dump to Clickhouse +- Importing MySQL dump to ClickHouse - Window function workarounds (row_number, lag/lead, running diff/sum/average) ##} diff --git a/docs/en/faq/operations/delete-old-data.md b/docs/en/faq/operations/delete-old-data.md index fdf1f1f290e..32fc485e98a 100644 --- a/docs/en/faq/operations/delete-old-data.md +++ b/docs/en/faq/operations/delete-old-data.md @@ -12,7 +12,7 @@ The short answer is “yes”. ClickHouse has multiple mechanisms that allow fre ClickHouse allows to automatically drop values when some condition happens. This condition is configured as an expression based on any columns, usually just static offset for any timestamp column. -The key advantage of this approach is that it doesn’t need any external system to trigger, once TTL is configured, data removal happens automatically in background. +The key advantage of this approach is that it does not need any external system to trigger, once TTL is configured, data removal happens automatically in background. !!! note "Note" TTL can also be used to move data not only to [/dev/null](https://en.wikipedia.org/wiki/Null_device), but also between different storage systems, like from SSD to HDD. @@ -21,7 +21,7 @@ More details on [configuring TTL](../../engines/table-engines/mergetree-family/m ## ALTER DELETE {#alter-delete} -ClickHouse doesn’t have real-time point deletes like in [OLTP](https://en.wikipedia.org/wiki/Online_transaction_processing) databases. The closest thing to them are mutations. They are issued as `ALTER ... DELETE` or `ALTER ... UPDATE` queries to distinguish from normal `DELETE` or `UPDATE` as they are asynchronous batch operations, not immediate modifications. The rest of syntax after `ALTER TABLE` prefix is similar. +ClickHouse does not have real-time point deletes like in [OLTP](https://en.wikipedia.org/wiki/Online_transaction_processing) databases. The closest thing to them are mutations. They are issued as `ALTER ... DELETE` or `ALTER ... UPDATE` queries to distinguish from normal `DELETE` or `UPDATE` as they are asynchronous batch operations, not immediate modifications. The rest of syntax after `ALTER TABLE` prefix is similar. `ALTER DELETE` can be issued to flexibly remove old data. If you need to do it regularly, the main downside will be the need to have an external system to submit the query. There are also some performance considerations since mutation rewrite complete parts even there’s only a single row to be deleted. diff --git a/docs/en/faq/operations/production.md b/docs/en/faq/operations/production.md index 77f7a76f2f9..52ca300ced0 100644 --- a/docs/en/faq/operations/production.md +++ b/docs/en/faq/operations/production.md @@ -25,7 +25,7 @@ Here’re some key points to get reasonable fidelity in a pre-production environ - Don’t make it read-only with some frozen data. - Don’t make it write-only with just copying data without building some typical reports. - Don’t wipe it clean instead of applying schema migrations. -- Use a sample of real production data and queries. Try to choose a sample that’s still representative and makes `SELECT` queries return reasonable results. Use obfuscation if your data is sensitive and internal policies don’t allow it to leave the production environment. +- Use a sample of real production data and queries. Try to choose a sample that’s still representative and makes `SELECT` queries return reasonable results. Use obfuscation if your data is sensitive and internal policies do not allow it to leave the production environment. - Make sure that pre-production is covered by your monitoring and alerting software the same way as your production environment does. - If your production spans across multiple datacenters or regions, make your pre-production does the same. - If your production uses complex features like replication, distributed table, cascading materialize views, make sure they are configured similarly in pre-production. @@ -61,8 +61,8 @@ For production use, there are two key options: `stable` and `lts`. Here is some - `stable` is the kind of package we recommend by default. They are released roughly monthly (and thus provide new features with reasonable delay) and three latest stable releases are supported in terms of diagnostics and backporting of bugfixes. - `lts` are released twice a year and are supported for a year after their initial release. You might prefer them over `stable` in the following cases: - - Your company has some internal policies that don’t allow for frequent upgrades or using non-LTS software. - - You are using ClickHouse in some secondary products that either doesn’t require any complex ClickHouse features and don’t have enough resources to keep it updated. + - Your company has some internal policies that do not allow for frequent upgrades or using non-LTS software. + - You are using ClickHouse in some secondary products that either does not require any complex ClickHouse features and do not have enough resources to keep it updated. Many teams who initially thought that `lts` is the way to go, often switch to `stable` anyway because of some recent feature that’s important for their product. diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index c444264b71f..9a4848a3ef0 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -132,7 +132,7 @@ To start the server as a daemon, run: $ sudo service clickhouse-server start ``` -If you don’t have `service` command, run as +If you do not have `service` command, run as ``` bash $ sudo /etc/init.d/clickhouse-server start @@ -140,7 +140,7 @@ $ sudo /etc/init.d/clickhouse-server start See the logs in the `/var/log/clickhouse-server/` directory. -If the server doesn’t start, check the configurations in the file `/etc/clickhouse-server/config.xml`. +If the server does not start, check the configurations in the file `/etc/clickhouse-server/config.xml`. You can also manually launch the server from the console: @@ -149,7 +149,7 @@ $ clickhouse-server --config-file=/etc/clickhouse-server/config.xml ``` In this case, the log will be printed to the console, which is convenient during development. -If the configuration file is in the current directory, you don’t need to specify the `--config-file` parameter. By default, it uses `./config.xml`. +If the configuration file is in the current directory, you do not need to specify the `--config-file` parameter. By default, it uses `./config.xml`. ClickHouse supports access restriction settings. They are located in the `users.xml` file (next to `config.xml`). By default, access is allowed from anywhere for the `default` user, without a password. See `user/default/networks`. diff --git a/docs/en/getting-started/tutorial.md b/docs/en/getting-started/tutorial.md index fe697972dff..694a82e100e 100644 --- a/docs/en/getting-started/tutorial.md +++ b/docs/en/getting-started/tutorial.md @@ -105,7 +105,7 @@ Syntax for creating tables is way more complicated compared to databases (see [r 2. Table schema, i.e. list of columns and their [data types](../sql-reference/data-types/index.md). 3. [Table engine](../engines/table-engines/index.md) and its settings, which determines all the details on how queries to this table will be physically executed. -Yandex.Metrica is a web analytics service, and sample dataset doesn’t cover its full functionality, so there are only two tables to create: +Yandex.Metrica is a web analytics service, and sample dataset does not cover its full functionality, so there are only two tables to create: - `hits` is a table with each action done by all users on all websites covered by the service. - `visits` is a table that contains pre-built sessions instead of individual actions. diff --git a/docs/en/guides/apply-catboost-model.md b/docs/en/guides/apply-catboost-model.md index 7c2c8a575ec..ec3ecc92141 100644 --- a/docs/en/guides/apply-catboost-model.md +++ b/docs/en/guides/apply-catboost-model.md @@ -20,7 +20,7 @@ For more information about training CatBoost models, see [Training and applying ## Prerequisites {#prerequisites} -If you don’t have the [Docker](https://docs.docker.com/install/) yet, install it. +If you do not have the [Docker](https://docs.docker.com/install/) yet, install it. !!! note "Note" [Docker](https://www.docker.com) is a software platform that allows you to create containers that isolate a CatBoost and ClickHouse installation from the rest of the system. diff --git a/docs/en/index.md b/docs/en/index.md index 676fd444995..12e72ebdf3b 100644 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -54,7 +54,7 @@ The higher the load on the system, the more important it is to customize the sys - There is one large table per query. All tables are small, except for one. - A query result is significantly smaller than the source data. In other words, data is filtered or aggregated, so the result fits in a single server’s RAM. -It is easy to see that the OLAP scenario is very different from other popular scenarios (such as OLTP or Key-Value access). So it doesn’t make sense to try to use OLTP or a Key-Value DB for processing analytical queries if you want to get decent performance. For example, if you try to use MongoDB or Redis for analytics, you will get very poor performance compared to OLAP databases. +It is easy to see that the OLAP scenario is very different from other popular scenarios (such as OLTP or Key-Value access). So it does not make sense to try to use OLTP or a Key-Value DB for processing analytical queries if you want to get decent performance. For example, if you try to use MongoDB or Redis for analytics, you will get very poor performance compared to OLAP databases. ## Why Column-Oriented Databases Work Better in the OLAP Scenario {#why-column-oriented-databases-work-better-in-the-olap-scenario} @@ -80,15 +80,15 @@ For example, the query “count the number of records for each advertising platf ### CPU {#cpu} -Since executing a query requires processing a large number of rows, it helps to dispatch all operations for entire vectors instead of for separate rows, or to implement the query engine so that there is almost no dispatching cost. If you don’t do this, with any half-decent disk subsystem, the query interpreter inevitably stalls the CPU. It makes sense to both store data in columns and process it, when possible, by columns. +Since executing a query requires processing a large number of rows, it helps to dispatch all operations for entire vectors instead of for separate rows, or to implement the query engine so that there is almost no dispatching cost. If you do not do this, with any half-decent disk subsystem, the query interpreter inevitably stalls the CPU. It makes sense to both store data in columns and process it, when possible, by columns. There are two ways to do this: -1. A vector engine. All operations are written for vectors, instead of for separate values. This means you don’t need to call operations very often, and dispatching costs are negligible. Operation code contains an optimized internal cycle. +1. A vector engine. All operations are written for vectors, instead of for separate values. This means you do not need to call operations very often, and dispatching costs are negligible. Operation code contains an optimized internal cycle. 2. Code generation. The code generated for the query has all the indirect calls in it. -This is not done in “normal” databases, because it doesn’t make sense when running simple queries. However, there are exceptions. For example, MemSQL uses code generation to reduce latency when processing SQL queries. (For comparison, analytical DBMSs require optimization of throughput, not latency.) +This is not done in “normal” databases, because it does not make sense when running simple queries. However, there are exceptions. For example, MemSQL uses code generation to reduce latency when processing SQL queries. (For comparison, analytical DBMSs require optimization of throughput, not latency.) Note that for CPU efficiency, the query language must be declarative (SQL or MDX), or at least a vector (J, K). The query should only contain implicit loops, allowing for optimization. diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index 7e072e366dc..8457ea41857 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -66,7 +66,7 @@ When processing a query, the client shows: 3. The result in the specified format. 4. The number of lines in the result, the time passed, and the average speed of query processing. -You can cancel a long query by pressing Ctrl+C. However, you will still need to wait for a little for the server to abort the request. It is not possible to cancel a query at certain stages. If you don’t wait and press Ctrl+C a second time, the client will exit. +You can cancel a long query by pressing Ctrl+C. However, you will still need to wait for a little for the server to abort the request. It is not possible to cancel a query at certain stages. If you do not wait and press Ctrl+C a second time, the client will exit. The command-line client allows passing external data (external temporary tables) for querying. For more information, see the section “External data for query processing”. diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 5987ba0f676..c616d843173 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -662,7 +662,7 @@ ClickHouse allows: - Any order of key-value pairs in the object. - Omitting some values. -ClickHouse ignores spaces between elements and commas after the objects. You can pass all the objects in one line. You don’t have to separate them with line breaks. +ClickHouse ignores spaces between elements and commas after the objects. You can pass all the objects in one line. You do not have to separate them with line breaks. **Omitted values processing** @@ -770,9 +770,9 @@ SELECT * FROM json_each_row_nested ## Native {#native} -The most efficient format. Data is written and read by blocks in binary format. For each block, the number of rows, number of columns, column names and types, and parts of columns in this block are recorded one after another. In other words, this format is “columnar” – it doesn’t convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. +The most efficient format. Data is written and read by blocks in binary format. For each block, the number of rows, number of columns, column names and types, and parts of columns in this block are recorded one after another. In other words, this format is “columnar” – it does not convert columns to rows. This is the format used in the native interface for interaction between servers, for using the command-line client, and for C++ clients. -You can use this format to quickly generate dumps that can only be read by the ClickHouse DBMS. It doesn’t make sense to work with this format yourself. +You can use this format to quickly generate dumps that can only be read by the ClickHouse DBMS. It does not make sense to work with this format yourself. ## Null {#null} @@ -1039,7 +1039,7 @@ struct Message { } ``` -Deserialization is effective and usually doesn’t increase the system load. +Deserialization is effective and usually does not increase the system load. See also [Format Schema](#formatschema). @@ -1312,7 +1312,7 @@ ClickHouse supports configurable precision of the `Decimal` type. The `INSERT` q Unsupported ORC data types: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -The data types of ClickHouse table columns don’t have to match the corresponding ORC data fields. When inserting data, ClickHouse interprets data types according to the table above and then [casts](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) the data to the data type set for the ClickHouse table column. +The data types of ClickHouse table columns do not have to match the corresponding ORC data fields. When inserting data, ClickHouse interprets data types according to the table above and then [casts](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) the data to the data type set for the ClickHouse table column. ### Inserting Data {#inserting-data-2} diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 18533cfc6c2..dec3c839020 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -52,7 +52,7 @@ X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","writ ``` As you can see, curl is somewhat inconvenient in that spaces must be URL escaped. -Although wget escapes everything itself, we don’t recommend using it because it doesn’t work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. +Although wget escapes everything itself, we do not recommend using it because it does not work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. ``` bash $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- @@ -146,7 +146,7 @@ Deleting the table. $ echo 'DROP TABLE t' | curl 'http://localhost:8123/' --data-binary @- ``` -For successful requests that don’t return a data table, an empty response body is returned. +For successful requests that do not return a data table, an empty response body is returned. ## Compression {#compression} @@ -273,7 +273,7 @@ Possible header fields: - `written_rows` — Number of rows written. - `written_bytes` — Volume of data written in bytes. -Running requests don’t stop automatically if the HTTP connection is lost. Parsing and data formatting are performed on the server-side, and using the network might be ineffective. +Running requests do not stop automatically if the HTTP connection is lost. Parsing and data formatting are performed on the server-side, and using the network might be ineffective. The optional ‘query_id’ parameter can be passed as the query ID (any string). For more information, see the section “Settings, replace_running_query”. The optional ‘quota_key’ parameter can be passed as the quota key (any string). For more information, see the section “Quotas”. diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index f5c85289171..835e3c6e98a 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -42,6 +42,8 @@ toc_title: Client Libraries - Ruby - [ClickHouse (Ruby)](https://github.com/shlima/click_house) - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) +- Rust + - [Klickhouse](https://github.com/Protryon/klickhouse) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickHouse](https://github.com/IMSMWU/RClickHouse) diff --git a/docs/en/introduction/distinctive-features.md b/docs/en/introduction/distinctive-features.md index be7c2d2e7c1..ee9f9484c67 100644 --- a/docs/en/introduction/distinctive-features.md +++ b/docs/en/introduction/distinctive-features.md @@ -61,7 +61,7 @@ Unlike other database management systems, secondary indexes in ClickHouse does n ## Suitable for Online Queries {#suitable-for-online-queries} -Most OLAP database management systems don’t aim for online queries with sub-second latencies. In alternative systems, report building time of tens of seconds or even minutes is often considered acceptable. Sometimes it takes even more which forces to prepare reports offline (in advance or by responding with “come back later”). +Most OLAP database management systems do not aim for online queries with sub-second latencies. In alternative systems, report building time of tens of seconds or even minutes is often considered acceptable. Sometimes it takes even more which forces to prepare reports offline (in advance or by responding with “come back later”). In ClickHouse low latency means that queries can be processed without delay and without trying to prepare an answer in advance, right at the same moment while the user interface page is loading. In other words, online. diff --git a/docs/en/operations/access-rights.md b/docs/en/operations/access-rights.md index 9f7d2a0b95b..8d48218f417 100644 --- a/docs/en/operations/access-rights.md +++ b/docs/en/operations/access-rights.md @@ -31,7 +31,7 @@ To see all users, roles, profiles, etc. and all their grants use [SHOW ACCESS](. ## Usage {#access-control-usage} -By default, the ClickHouse server provides the `default` user account which is not allowed using SQL-driven access control and account management but has all the rights and permissions. The `default` user account is used in any cases when the username is not defined, for example, at login from client or in distributed queries. In distributed query processing a default user account is used, if the configuration of the server or cluster doesn’t specify the [user and password](../engines/table-engines/special/distributed.md) properties. +By default, the ClickHouse server provides the `default` user account which is not allowed using SQL-driven access control and account management but has all the rights and permissions. The `default` user account is used in any cases when the username is not defined, for example, at login from client or in distributed queries. In distributed query processing a default user account is used, if the configuration of the server or cluster does not specify the [user and password](../engines/table-engines/special/distributed.md) properties. If you just started using ClickHouse, consider the following scenario: diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index f4206f5d70c..9c8f5389ccd 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -5,7 +5,7 @@ toc_title: Data Backup # Data Backup {#data-backup} -While [replication](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [you can’t just drop tables with a MergeTree-like engine containing more than 50 Gb of data](server-configuration-parameters/settings.md#max-table-size-to-drop). However, these safeguards don’t cover all possible cases and can be circumvented. +While [replication](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [you can’t just drop tables with a MergeTree-like engine containing more than 50 Gb of data](server-configuration-parameters/settings.md#max-table-size-to-drop). However, these safeguards do not cover all possible cases and can be circumvented. In order to effectively mitigate possible human errors, you should carefully prepare a strategy for backing up and restoring your data **in advance**. @@ -30,7 +30,7 @@ For smaller volumes of data, a simple `INSERT INTO ... SELECT ...` to remote tab ## Manipulations with Parts {#manipulations-with-parts} -ClickHouse allows using the `ALTER TABLE ... FREEZE PARTITION ...` query to create a local copy of table partitions. This is implemented using hardlinks to the `/var/lib/clickhouse/shadow/` folder, so it usually does not consume extra disk space for old data. The created copies of files are not handled by ClickHouse server, so you can just leave them there: you will have a simple backup that doesn’t require any additional external system, but it will still be prone to hardware issues. For this reason, it’s better to remotely copy them to another location and then remove the local copies. Distributed filesystems and object stores are still a good options for this, but normal attached file servers with a large enough capacity might work as well (in this case the transfer will occur via the network filesystem or maybe [rsync](https://en.wikipedia.org/wiki/Rsync)). +ClickHouse allows using the `ALTER TABLE ... FREEZE PARTITION ...` query to create a local copy of table partitions. This is implemented using hardlinks to the `/var/lib/clickhouse/shadow/` folder, so it usually does not consume extra disk space for old data. The created copies of files are not handled by ClickHouse server, so you can just leave them there: you will have a simple backup that does not require any additional external system, but it will still be prone to hardware issues. For this reason, it’s better to remotely copy them to another location and then remove the local copies. Distributed filesystems and object stores are still a good options for this, but normal attached file servers with a large enough capacity might work as well (in this case the transfer will occur via the network filesystem or maybe [rsync](https://en.wikipedia.org/wiki/Rsync)). Data can be restored from backup using the `ALTER TABLE ... ATTACH PARTITION ...` For more information about queries related to partition manipulations, see the [ALTER documentation](../sql-reference/statements/alter/partition.md#alter_manipulations-with-partitions). diff --git a/docs/en/operations/optimizing-performance/sampling-query-profiler.md b/docs/en/operations/optimizing-performance/sampling-query-profiler.md index 0c075180530..9244592d515 100644 --- a/docs/en/operations/optimizing-performance/sampling-query-profiler.md +++ b/docs/en/operations/optimizing-performance/sampling-query-profiler.md @@ -11,13 +11,13 @@ To use profiler: - Setup the [trace_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) section of the server configuration. - This section configures the [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) system table containing the results of the profiler functioning. It is configured by default. Remember that data in this table is valid only for a running server. After the server restart, ClickHouse doesn’t clean up the table and all the stored virtual memory address may become invalid. + This section configures the [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) system table containing the results of the profiler functioning. It is configured by default. Remember that data in this table is valid only for a running server. After the server restart, ClickHouse does not clean up the table and all the stored virtual memory address may become invalid. - Setup the [query_profiler_cpu_time_period_ns](../../operations/settings/settings.md#query_profiler_cpu_time_period_ns) or [query_profiler_real_time_period_ns](../../operations/settings/settings.md#query_profiler_real_time_period_ns) settings. Both settings can be used simultaneously. These settings allow you to configure profiler timers. As these are the session settings, you can get different sampling frequency for the whole server, individual users or user profiles, for your interactive session, and for each individual query. -The default sampling frequency is one sample per second and both CPU and real timers are enabled. This frequency allows collecting enough information about ClickHouse cluster. At the same time, working with this frequency, profiler doesn’t affect ClickHouse server’s performance. If you need to profile each individual query try to use higher sampling frequency. +The default sampling frequency is one sample per second and both CPU and real timers are enabled. This frequency allows collecting enough information about ClickHouse cluster. At the same time, working with this frequency, profiler does not affect ClickHouse server’s performance. If you need to profile each individual query try to use higher sampling frequency. To analyze the `trace_log` system table: diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md index 56c3eaf6455..bbea735cdba 100644 --- a/docs/en/operations/quotas.md +++ b/docs/en/operations/quotas.md @@ -72,7 +72,7 @@ The resource consumption calculated for each interval is output to the server lo ``` -For the ‘statbox’ quota, restrictions are set for every hour and for every 24 hours (86,400 seconds). The time interval is counted, starting from an implementation-defined fixed moment in time. In other words, the 24-hour interval doesn’t necessarily begin at midnight. +For the ‘statbox’ quota, restrictions are set for every hour and for every 24 hours (86,400 seconds). The time interval is counted, starting from an implementation-defined fixed moment in time. In other words, the 24-hour interval does not necessarily begin at midnight. When the interval ends, all collected values are cleared. For the next hour, the quota calculation starts over. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 19671b523e3..801a1d27add 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -502,12 +502,12 @@ The default `max_server_memory_usage` value is calculated as `memory_amount * ma ## max_server_memory_usage_to_ram_ratio {#max_server_memory_usage_to_ram_ratio} -Defines the fraction of total physical RAM amount, available to the Clickhouse server. If the server tries to utilize more, the memory is cut down to the appropriate amount. +Defines the fraction of total physical RAM amount, available to the ClickHouse server. If the server tries to utilize more, the memory is cut down to the appropriate amount. Possible values: - Positive double. -- 0 — The Clickhouse server can use all available RAM. +- 0 — The ClickHouse server can use all available RAM. Default value: `0`. @@ -826,7 +826,7 @@ Use the following parameters to configure logging: - `engine` - [MergeTree Engine Definition](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) for a system table. Can't be used if `partition_by` defined. - `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. -If the table doesn’t exist, ClickHouse will create it. If the structure of the query log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. +If the table does not exist, ClickHouse will create it. If the structure of the query log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. **Example** @@ -853,7 +853,7 @@ Use the following parameters to configure logging: - `engine` - [MergeTree Engine Definition](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) for a system table. Can't be used if `partition_by` defined. - `flush_interval_milliseconds` – Interval for flushing data from the buffer in memory to the table. -If the table doesn’t exist, ClickHouse will create it. If the structure of the query thread log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. +If the table does not exist, ClickHouse will create it. If the structure of the query thread log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. **Example** @@ -1155,7 +1155,7 @@ This setting only applies to the `MergeTree` family. It can be specified: If `use_minimalistic_part_header_in_zookeeper = 1`, then [replicated](../../engines/table-engines/mergetree-family/replication.md) tables store the headers of the data parts compactly using a single `znode`. If the table contains many columns, this storage method significantly reduces the volume of the data stored in Zookeeper. !!! attention "Attention" - After applying `use_minimalistic_part_header_in_zookeeper = 1`, you can’t downgrade the ClickHouse server to a version that doesn’t support this setting. Be careful when upgrading ClickHouse on servers in a cluster. Don’t upgrade all the servers at once. It is safer to test new versions of ClickHouse in a test environment, or on just a few servers of a cluster. + After applying `use_minimalistic_part_header_in_zookeeper = 1`, you can’t downgrade the ClickHouse server to a version that does not support this setting. Be careful when upgrading ClickHouse on servers in a cluster. Don’t upgrade all the servers at once. It is safer to test new versions of ClickHouse in a test environment, or on just a few servers of a cluster. Data part headers already stored with this setting can't be restored to their previous (non-compact) representation. diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 6e3d0bc0fde..10ea46098d4 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -251,4 +251,15 @@ Possible values: Default value: -1 (unlimited). +## allow_floating_point_partition_key {#allow_floating_point_partition_key} + +Enables to allow floating-point number as a partition key. + +Possible values: + +- 0 — Floating-point partition key not allowed. +- 1 — Floating-point partition key allowed. + +Default value: `0`. + [Original article](https://clickhouse.tech/docs/en/operations/settings/merge_tree_settings/) diff --git a/docs/en/operations/settings/query-complexity.md b/docs/en/operations/settings/query-complexity.md index 2ecf50762d5..d60aa170907 100644 --- a/docs/en/operations/settings/query-complexity.md +++ b/docs/en/operations/settings/query-complexity.md @@ -19,7 +19,7 @@ It can take one of two values: `throw` or `break`. Restrictions on aggregation ( `break` – Stop executing the query and return the partial result, as if the source data ran out. -`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but don’t add new keys to the set. +`any (only for group_by_overflow_mode)` – Continuing aggregation for the keys that got into the set, but do not add new keys to the set. ## max_memory_usage {#settings_max_memory_usage} @@ -27,7 +27,7 @@ The maximum amount of RAM to use for running a query on a single server. In the default configuration file, the maximum is 10 GB. -The setting doesn’t consider the volume of available memory or the total volume of memory on the machine. +The setting does not consider the volume of available memory or the total volume of memory on the machine. The restriction applies to a single query within a single server. You can use `SHOW PROCESSLIST` to see the current memory consumption for each query. Besides, the peak memory consumption is tracked for each query and written to the log. @@ -288,7 +288,7 @@ Defines what action ClickHouse performs when any of the following join limits is Possible values: - `THROW` — ClickHouse throws an exception and breaks operation. -- `BREAK` — ClickHouse breaks operation and doesn’t throw an exception. +- `BREAK` — ClickHouse breaks operation and does not throw an exception. Default value: `THROW`. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 1a0202ad45a..10461eacbff 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -365,13 +365,37 @@ throws an exception. ## input_format_null_as_default {#settings-input-format-null-as-default} -Enables or disables using default values if input data contain `NULL`, but the data type of the corresponding column in not `Nullable(T)` (for text input formats). +Enables or disables the initialization of [NULL](../../sql-reference/syntax.md#null-literal) fields with [default values](../../sql-reference/statements/create/table.md#create-default-values), if data type of these fields is not [nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). +If column type is not nullable and this setting is disabled, then inserting `NULL` causes an exception. If column type is nullable, then `NULL` values are inserted as is, regardless of this setting. + +This setting is applicable to [INSERT ... VALUES](../../sql-reference/statements/insert-into.md) queries for text input formats. + +Possible values: + +- 0 — Inserting `NULL` into a not nullable column causes an exception. +- 1 — `NULL` fields are initialized with default column values. + +Default value: `1`. + +## insert_null_as_default {#insert_null_as_default} + +Enables or disables the insertion of [default values](../../sql-reference/statements/create/table.md#create-default-values) instead of [NULL](../../sql-reference/syntax.md#null-literal) into columns with not [nullable](../../sql-reference/data-types/nullable.md#data_type-nullable) data type. +If column type is not nullable and this setting is disabled, then inserting `NULL` causes an exception. If column type is nullable, then `NULL` values are inserted as is, regardless of this setting. + +This setting is applicable to [INSERT ... SELECT](../../sql-reference/statements/insert-into.md#insert_query_insert-select) queries. Note that `SELECT` subqueries may be concatenated with `UNION ALL` clause. + +Possible values: + +- 0 — Inserting `NULL` into a not nullable column causes an exception. +- 1 — Default column value is inserted instead of `NULL`. + +Default value: `1`. ## input_format_skip_unknown_fields {#settings-input-format-skip-unknown-fields} Enables or disables skipping insertion of extra data. -When writing data, ClickHouse throws an exception if input data contain columns that do not exist in the target table. If skipping is enabled, ClickHouse doesn’t insert extra data and doesn’t throw an exception. +When writing data, ClickHouse throws an exception if input data contain columns that do not exist in the target table. If skipping is enabled, ClickHouse does not insert extra data and does not throw an exception. Supported formats: @@ -428,7 +452,7 @@ Default value: 1. Allows choosing a parser of the text representation of date and time. -The setting doesn’t apply to [date and time functions](../../sql-reference/functions/date-time-functions.md). +The setting does not apply to [date and time functions](../../sql-reference/functions/date-time-functions.md). Possible values: @@ -455,15 +479,15 @@ Possible values: - `simple` - Simple output format. - Clickhouse output date and time `YYYY-MM-DD hh:mm:ss` format. For example, `2019-08-20 10:18:56`. The calculation is performed according to the data type's time zone (if present) or server time zone. + ClickHouse output date and time `YYYY-MM-DD hh:mm:ss` format. For example, `2019-08-20 10:18:56`. The calculation is performed according to the data type's time zone (if present) or server time zone. - `iso` - ISO output format. - Clickhouse output date and time in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) `YYYY-MM-DDThh:mm:ssZ` format. For example, `2019-08-20T10:18:56Z`. Note that output is in UTC (`Z` means UTC). + ClickHouse output date and time in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) `YYYY-MM-DDThh:mm:ssZ` format. For example, `2019-08-20T10:18:56Z`. Note that output is in UTC (`Z` means UTC). - `unix_timestamp` - Unix timestamp output format. - Clickhouse output date and time in [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) format. For example `1566285536`. + ClickHouse output date and time in [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) format. For example `1566285536`. Default value: `simple`. @@ -662,7 +686,7 @@ Default value: 8. ## merge_tree_max_rows_to_use_cache {#setting-merge-tree-max-rows-to-use-cache} -If ClickHouse should read more than `merge_tree_max_rows_to_use_cache` rows in one query, it doesn’t use the cache of uncompressed blocks. +If ClickHouse should read more than `merge_tree_max_rows_to_use_cache` rows in one query, it does not use the cache of uncompressed blocks. The cache of uncompressed blocks stores data extracted for queries. ClickHouse uses this cache to speed up responses to repeated small queries. This setting protects the cache from trashing by queries that read a large amount of data. The [uncompressed_cache_size](../../operations/server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) server setting defines the size of the cache of uncompressed blocks. @@ -674,7 +698,7 @@ Default value: 128 ✕ 8192. ## merge_tree_max_bytes_to_use_cache {#setting-merge-tree-max-bytes-to-use-cache} -If ClickHouse should read more than `merge_tree_max_bytes_to_use_cache` bytes in one query, it doesn’t use the cache of uncompressed blocks. +If ClickHouse should read more than `merge_tree_max_bytes_to_use_cache` bytes in one query, it does not use the cache of uncompressed blocks. The cache of uncompressed blocks stores data extracted for queries. ClickHouse uses this cache to speed up responses to repeated small queries. This setting protects the cache from trashing by queries that read a large amount of data. The [uncompressed_cache_size](../../operations/server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size) server setting defines the size of the cache of uncompressed blocks. @@ -816,8 +840,8 @@ Result: The size of blocks (in a count of rows) to form for insertion into a table. This setting only applies in cases when the server forms the blocks. For example, for an INSERT via the HTTP interface, the server parses the data format and forms blocks of the specified size. -But when using clickhouse-client, the client parses the data itself, and the ‘max_insert_block_size’ setting on the server doesn’t affect the size of the inserted blocks. -The setting also doesn’t have a purpose when using INSERT SELECT, since data is inserted using the same blocks that are formed after SELECT. +But when using clickhouse-client, the client parses the data itself, and the ‘max_insert_block_size’ setting on the server does not affect the size of the inserted blocks. +The setting also does not have a purpose when using INSERT SELECT, since data is inserted using the same blocks that are formed after SELECT. Default value: 1,048,576. @@ -887,7 +911,7 @@ Higher values will lead to higher memory usage. The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). Specifying smaller block size generally leads to slightly reduced compression ratio, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced. !!! note "Warning" - This is an expert-level setting, and you shouldn't change it if you're just getting started with Clickhouse. + This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse. Don’t confuse blocks for compression (a chunk of memory consisting of bytes) with blocks for query processing (a set of rows from a table). @@ -904,7 +928,7 @@ We are writing a UInt32-type column (4 bytes per value). When writing 8192 rows, We are writing a URL column with the String type (average size of 60 bytes per value). When writing 8192 rows, the average will be slightly less than 500 KB of data. Since this is more than 65,536, a compressed block will be formed for each mark. In this case, when reading data from the disk in the range of a single mark, extra data won’t be decompressed. !!! note "Warning" - This is an expert-level setting, and you shouldn't change it if you're just getting started with Clickhouse. + This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse. ## max_query_size {#settings-max_query_size} @@ -1018,7 +1042,7 @@ For queries that read at least a somewhat large volume of data (one million rows When using the HTTP interface, the ‘query_id’ parameter can be passed. This is any string that serves as the query identifier. If a query from the same user with the same ‘query_id’ already exists at this time, the behaviour depends on the ‘replace_running_query’ parameter. -`0` (default) – Throw an exception (don’t allow the query to run if a query with the same ‘query_id’ is already running). +`0` (default) – Throw an exception (do not allow the query to run if a query with the same ‘query_id’ is already running). `1` – Cancel the old query and start running the new one. @@ -1077,7 +1101,7 @@ load_balancing = nearest_hostname The number of errors is counted for each replica. Every 5 minutes, the number of errors is integrally divided by 2. Thus, the number of errors is calculated for a recent time with exponential smoothing. If there is one replica with a minimal number of errors (i.e. errors occurred recently on the other replicas), the query is sent to it. If there are multiple replicas with the same minimal number of errors, the query is sent to the replica with a hostname that is most similar to the server’s hostname in the config file (for the number of different characters in identical positions, up to the minimum length of both hostnames). For instance, example01-01-1 and example01-01-2.yandex.ru are different in one position, while example01-01-1 and example01-02-2 differ in two places. -This method might seem primitive, but it doesn’t require external data about network topology, and it doesn’t compare IP addresses, which would be complicated for our IPv6 addresses. +This method might seem primitive, but it does not require external data about network topology, and it does not compare IP addresses, which would be complicated for our IPv6 addresses. Thus, if there are equivalent replicas, the closest one by name is preferred. We can also assume that when sending a query to the same server, in the absence of failures, a distributed query will also go to the same servers. So even if different data is placed on the replicas, the query will return mostly the same results. @@ -1149,7 +1173,7 @@ Default value: `1`. This setting is useful for replicated tables with a sampling key. A query may be processed faster if it is executed on several servers in parallel. But the query performance may degrade in the following cases: -- The position of the sampling key in the partitioning key doesn't allow efficient range scans. +- The position of the sampling key in the partitioning key does not allow efficient range scans. - Adding a sampling key to the table makes filtering by other columns less efficient. - The sampling key is an expression that is expensive to calculate. - The cluster latency distribution has a long tail, so that querying more servers increases the query overall latency. @@ -1171,7 +1195,7 @@ For testing, the value can be set to 0: compilation runs synchronously and the q If the value is 1 or more, compilation occurs asynchronously in a separate thread. The result will be used as soon as it is ready, including queries that are currently running. Compiled code is required for each different combination of aggregate functions used in the query and the type of keys in the GROUP BY clause. -The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don’t use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. +The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they do not use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted. ## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers} @@ -1505,7 +1529,7 @@ Possible values: - 1 — skipping enabled. - If a shard is unavailable, ClickHouse returns a result based on partial data and doesn’t report node availability issues. + If a shard is unavailable, ClickHouse returns a result based on partial data and does not report node availability issues. - 0 — skipping disabled. @@ -1613,7 +1637,7 @@ Enables or disables query execution if [optimize_skip_unused_shards](#optimize-s Possible values: -- 0 — Disabled. ClickHouse doesn’t throw an exception. +- 0 — Disabled. ClickHouse does not throw an exception. - 1 — Enabled. Query execution is disabled only if the table has a sharding key. - 2 — Enabled. Query execution is disabled regardless of whether a sharding key is defined for the table. @@ -1758,7 +1782,7 @@ Default value: 0. Sets the priority ([nice](https://en.wikipedia.org/wiki/Nice_(Unix))) for threads that execute queries. The OS scheduler considers this priority when choosing the next thread to run on each available CPU core. !!! warning "Warning" - To use this setting, you need to set the `CAP_SYS_NICE` capability. The `clickhouse-server` package sets it up during installation. Some virtual environments don’t allow you to set the `CAP_SYS_NICE` capability. In this case, `clickhouse-server` shows a message about it at the start. + To use this setting, you need to set the `CAP_SYS_NICE` capability. The `clickhouse-server` package sets it up during installation. Some virtual environments do not allow you to set the `CAP_SYS_NICE` capability. In this case, `clickhouse-server` shows a message about it at the start. Possible values: @@ -2053,7 +2077,7 @@ When merging is prohibited, the replica never merges parts and always downloads Possible values: - 0 — `Replicated*MergeTree`-engine tables merge data parts at the replica. -- 1 — `Replicated*MergeTree`-engine tables don’t merge data parts at the replica. The tables download merged data parts from other replicas. +- 1 — `Replicated*MergeTree`-engine tables do not merge data parts at the replica. The tables download merged data parts from other replicas. Default value: 0. @@ -2184,7 +2208,7 @@ Allows or restricts using the [LowCardinality](../../sql-reference/data-types/lo If usage of `LowCardinality` is restricted, ClickHouse server converts `LowCardinality`-columns to ordinary ones for `SELECT` queries, and convert ordinary columns to `LowCardinality`-columns for `INSERT` queries. -This setting is required mainly for third-party clients which don’t support `LowCardinality` data type. +This setting is required mainly for third-party clients which do not support `LowCardinality` data type. Possible values: @@ -2672,7 +2696,7 @@ Possible values: - `'DISTINCT'` — ClickHouse outputs rows as a result of combining queries removing duplicate rows. - `'ALL'` — ClickHouse outputs all rows as a result of combining queries including duplicate rows. -- `''` — Clickhouse generates an exception when used with `UNION`. +- `''` — ClickHouse generates an exception when used with `UNION`. Default value: `''`. @@ -2999,7 +3023,6 @@ SET limit = 5; SET offset = 7; SELECT * FROM test LIMIT 10 OFFSET 100; ``` - Result: ``` text @@ -3010,4 +3033,36 @@ Result: └─────┘ ``` +## optimize_fuse_sum_count_avg {#optimize_fuse_sum_count_avg} + +Enables to fuse aggregate functions with identical argument. It rewrites query contains at least two aggregate functions from [sum](../../sql-reference/aggregate-functions/reference/sum.md#agg_function-sum), [count](../../sql-reference/aggregate-functions/reference/count.md#agg_function-count) or [avg](../../sql-reference/aggregate-functions/reference/avg.md#agg_function-avg) with identical argument to [sumCount](../../sql-reference/aggregate-functions/reference/sumcount.md#agg_function-sumCount). + +Possible values: + +- 0 — Functions with identical argument are not fused. +- 1 — Functions with identical argument are fused. + +Default value: `0`. + +**Example** + +Query: + +``` sql +CREATE TABLE fuse_tbl(a Int8, b Int8) Engine = Log; +SET optimize_fuse_sum_count_avg = 1; +EXPLAIN SYNTAX SELECT sum(a), sum(b), count(b), avg(b) from fuse_tbl FORMAT TSV; +``` + +Result: + +``` text +SELECT + sum(a), + sumCount(b).1, + sumCount(b).2, + (sumCount(b).1) / (sumCount(b).2) +FROM fuse_tbl +``` + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index e66f082167e..ab3ba25493a 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -59,7 +59,7 @@ For collecting system metrics ClickHouse server uses: **procfs** -If ClickHouse server doesn’t have `CAP_NET_ADMIN` capability, it tries to fall back to `ProcfsMetricsProvider`. `ProcfsMetricsProvider` allows collecting per-query system metrics (for CPU and I/O). +If ClickHouse server does not have `CAP_NET_ADMIN` capability, it tries to fall back to `ProcfsMetricsProvider`. `ProcfsMetricsProvider` allows collecting per-query system metrics (for CPU and I/O). If procfs is supported and enabled on the system, ClickHouse server collects these metrics: diff --git a/docs/en/operations/system-tables/one.md b/docs/en/operations/system-tables/one.md index a85e01bc75a..51316dfbc44 100644 --- a/docs/en/operations/system-tables/one.md +++ b/docs/en/operations/system-tables/one.md @@ -2,7 +2,7 @@ This table contains a single row with a single `dummy` UInt8 column containing the value 0. -This table is used if a `SELECT` query doesn’t specify the `FROM` clause. +This table is used if a `SELECT` query does not specify the `FROM` clause. This is similar to the `DUAL` table found in other DBMSs. diff --git a/docs/en/operations/system-tables/parts.md b/docs/en/operations/system-tables/parts.md index f02d1ebc114..5a4715a4513 100644 --- a/docs/en/operations/system-tables/parts.md +++ b/docs/en/operations/system-tables/parts.md @@ -26,7 +26,7 @@ Columns: - `active` ([UInt8](../../sql-reference/data-types/int-uint.md)) – Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging. -- `marks` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` by the index granularity (usually 8192) (this hint doesn’t work for adaptive granularity). +- `marks` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The number of marks. To get the approximate number of rows in a data part, multiply `marks` by the index granularity (usually 8192) (this hint does not work for adaptive granularity). - `rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The number of rows. @@ -66,7 +66,7 @@ Columns: - `primary_key_bytes_in_memory_allocated` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The amount of memory (in bytes) reserved for primary key values. -- `is_frozen` ([UInt8](../../sql-reference/data-types/int-uint.md)) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup doesn’t exist. For more details, see [FREEZE PARTITION](../../sql-reference/statements/alter/partition.md#alter_freeze-partition) +- `is_frozen` ([UInt8](../../sql-reference/data-types/int-uint.md)) – Flag that shows that a partition data backup exists. 1, the backup exists. 0, the backup does not exist. For more details, see [FREEZE PARTITION](../../sql-reference/statements/alter/partition.md#alter_freeze-partition) - `database` ([String](../../sql-reference/data-types/string.md)) – Name of the database. diff --git a/docs/en/operations/system-tables/parts_columns.md b/docs/en/operations/system-tables/parts_columns.md index 5c3dd7155f7..293abb18a50 100644 --- a/docs/en/operations/system-tables/parts_columns.md +++ b/docs/en/operations/system-tables/parts_columns.md @@ -26,7 +26,7 @@ Columns: - `active` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Flag that indicates whether the data part is active. If a data part is active, it’s used in a table. Otherwise, it’s deleted. Inactive data parts remain after merging. -- `marks` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of marks. To get the approximate number of rows in a data part, multiply `marks` by the index granularity (usually 8192) (this hint doesn’t work for adaptive granularity). +- `marks` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of marks. To get the approximate number of rows in a data part, multiply `marks` by the index granularity (usually 8192) (this hint does not work for adaptive granularity). - `rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of rows. diff --git a/docs/en/operations/system-tables/processes.md b/docs/en/operations/system-tables/processes.md index a379fc4a07a..9ef3c648006 100644 --- a/docs/en/operations/system-tables/processes.md +++ b/docs/en/operations/system-tables/processes.md @@ -11,7 +11,7 @@ Columns: - `bytes_read` (UInt64) – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. - `total_rows_approx` (UInt64) – The approximation of the total number of rows that should be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. - `memory_usage` (UInt64) – Amount of RAM the request uses. It might not include some types of dedicated memory. See the [max_memory_usage](../../operations/settings/query-complexity.md#settings_max_memory_usage) setting. -- `query` (String) – The query text. For `INSERT`, it doesn’t include the data to insert. +- `query` (String) – The query text. For `INSERT`, it does not include the data to insert. - `query_id` (String) – Query ID, if defined. diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md index 6cf87ee1f17..85f0679fe37 100644 --- a/docs/en/operations/system-tables/query_log.md +++ b/docs/en/operations/system-tables/query_log.md @@ -3,15 +3,15 @@ Contains information about executed queries, for example, start time, duration of processing, error messages. !!! note "Note" - This table doesn’t contain the ingested data for `INSERT` queries. + This table does not contain the ingested data for `INSERT` queries. You can change settings of queries logging in the [query_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query-log) section of the server configuration. -You can disable queries logging by setting [log_queries = 0](../../operations/settings/settings.md#settings-log-queries). We don’t recommend to turn off logging because information in this table is important for solving issues. +You can disable queries logging by setting [log_queries = 0](../../operations/settings/settings.md#settings-log-queries). We do not recommend to turn off logging because information in this table is important for solving issues. The flushing period of data is set in `flush_interval_milliseconds` parameter of the [query_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing, use the [SYSTEM FLUSH LOGS](../../sql-reference/statements/system.md#query_language-system-flush_logs) query. -ClickHouse doesn’t delete data from the table automatically. See [Introduction](../../operations/system-tables/index.md#system-tables-introduction) for more details. +ClickHouse does not delete data from the table automatically. See [Introduction](../../operations/system-tables/index.md#system-tables-introduction) for more details. The `system.query_log` table registers two kinds of queries: @@ -37,8 +37,8 @@ Columns: - `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Start time of query execution. - `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Start time of query execution with microsecond precision. - `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution in milliseconds. -- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number of rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_rows` includes the total number of rows read at all replicas. Each replica sends it’s `read_rows` value, and the server-initiator of the query summarizes all received and local values. The cache volumes don’t affect this value. -- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number of bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_bytes` includes the total number of rows read at all replicas. Each replica sends it’s `read_bytes` value, and the server-initiator of the query summarizes all received and local values. The cache volumes don’t affect this value. +- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number of rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_rows` includes the total number of rows read at all replicas. Each replica sends it’s `read_rows` value, and the server-initiator of the query summarizes all received and local values. The cache volumes do not affect this value. +- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number of bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_bytes` includes the total number of rows read at all replicas. Each replica sends it’s `read_bytes` value, and the server-initiator of the query summarizes all received and local values. The cache volumes do not affect this value. - `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. - `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. - `result_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of rows in a result of the `SELECT` query, or a number of rows in the `INSERT` query. diff --git a/docs/en/operations/system-tables/query_thread_log.md b/docs/en/operations/system-tables/query_thread_log.md index 0ae2e7d5d3b..296a33259b3 100644 --- a/docs/en/operations/system-tables/query_thread_log.md +++ b/docs/en/operations/system-tables/query_thread_log.md @@ -9,7 +9,7 @@ To start logging: The flushing period of data is set in `flush_interval_milliseconds` parameter of the [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server settings section. To force flushing, use the [SYSTEM FLUSH LOGS](../../sql-reference/statements/system.md#query_language-system-flush_logs) query. -ClickHouse doesn’t delete data from the table automatically. See [Introduction](../../operations/system-tables/index.md#system-tables-introduction) for more details. +ClickHouse does not delete data from the table automatically. See [Introduction](../../operations/system-tables/index.md#system-tables-introduction) for more details. Columns: diff --git a/docs/en/operations/system-tables/replicas.md b/docs/en/operations/system-tables/replicas.md index 8da68d2d2ab..63a2141e399 100644 --- a/docs/en/operations/system-tables/replicas.md +++ b/docs/en/operations/system-tables/replicas.md @@ -57,7 +57,7 @@ Columns: Note that writes can be performed to any replica that is available and has a session in ZK, regardless of whether it is a leader. - `can_become_leader` (`UInt8`) - Whether the replica can be a leader. - `is_readonly` (`UInt8`) - Whether the replica is in read-only mode. - This mode is turned on if the config doesn’t have sections with ZooKeeper, if an unknown error occurred when reinitializing sessions in ZooKeeper, and during session reinitialization in ZooKeeper. + This mode is turned on if the config does not have sections with ZooKeeper, if an unknown error occurred when reinitializing sessions in ZooKeeper, and during session reinitialization in ZooKeeper. - `is_session_expired` (`UInt8`) - the session with ZooKeeper has expired. Basically the same as `is_readonly`. - `future_parts` (`UInt32`) - The number of data parts that will appear as the result of INSERTs or merges that haven’t been done yet. - `parts_to_check` (`UInt32`) - The number of data parts in the queue for verification. A part is put in the verification queue if there is suspicion that it might be damaged. @@ -84,7 +84,7 @@ The next 4 columns have a non-zero value only where there is an active session w - `active_replicas` (`UInt8`) - The number of replicas of this table that have a session in ZooKeeper (i.e., the number of functioning replicas). If you request all the columns, the table may work a bit slowly, since several reads from ZooKeeper are made for each row. -If you don’t request the last 4 columns (log_max_index, log_pointer, total_replicas, active_replicas), the table works quickly. +If you do not request the last 4 columns (log_max_index, log_pointer, total_replicas, active_replicas), the table works quickly. For example, you can check that everything is working correctly like this: @@ -118,7 +118,7 @@ WHERE OR active_replicas < total_replicas ``` -If this query doesn’t return anything, it means that everything is fine. +If this query does not return anything, it means that everything is fine. [Original article](https://clickhouse.tech/docs/en/operations/system_tables/replicas) diff --git a/docs/en/operations/system-tables/replication_queue.md b/docs/en/operations/system-tables/replication_queue.md index 539a29432ac..965774b81bf 100644 --- a/docs/en/operations/system-tables/replication_queue.md +++ b/docs/en/operations/system-tables/replication_queue.md @@ -77,7 +77,7 @@ parts_to_merge: ['20201130_121373_121378_1','20201130_121379_121379_0',' is_detach: 0 is_currently_executing: 0 num_tries: 36 -last_exception: Code: 226, e.displayText() = DB::Exception: Marks file '/opt/clickhouse/data/merge/visits_v2/tmp_fetch_20201130_121373_121384_2/CounterID.mrk' doesn't exist (version 20.8.7.15 (official build)) +last_exception: Code: 226, e.displayText() = DB::Exception: Marks file '/opt/clickhouse/data/merge/visits_v2/tmp_fetch_20201130_121373_121384_2/CounterID.mrk' does not exist (version 20.8.7.15 (official build)) last_attempt_time: 2020-12-08 17:35:54 num_postponed: 0 postpone_reason: diff --git a/docs/en/operations/system-tables/stack_trace.md b/docs/en/operations/system-tables/stack_trace.md index 44b13047cc3..eb1824a6f66 100644 --- a/docs/en/operations/system-tables/stack_trace.md +++ b/docs/en/operations/system-tables/stack_trace.md @@ -6,6 +6,7 @@ To analyze stack frames, use the `addressToLine`, `addressToSymbol` and `demangl Columns: +- `thread_name` ([String](../../sql-reference/data-types/string.md)) — Thread name. - `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Thread identifier. - `query_id` ([String](../../sql-reference/data-types/string.md)) — Query identifier that can be used to get details about a query that was running from the [query_log](../system-tables/query_log.md) system table. - `trace` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — A [stack trace](https://en.wikipedia.org/wiki/Stack_trace) which represents a list of physical addresses where the called methods are stored. @@ -21,12 +22,14 @@ SET allow_introspection_functions = 1; Getting symbols from ClickHouse object files: ``` sql -WITH arrayMap(x -> demangle(addressToSymbol(x)), trace) AS all SELECT thread_id, query_id, arrayStringConcat(all, '\n') AS res FROM system.stack_trace LIMIT 1 \G +WITH arrayMap(x -> demangle(addressToSymbol(x)), trace) AS all SELECT thread_name, thread_id, query_id, arrayStringConcat(all, '\n') AS res FROM system.stack_trace LIMIT 1 \G; ``` ``` text Row 1: ────── +thread_name: clickhouse-serv + thread_id: 686 query_id: 1a11f70b-626d-47c1-b948-f9c7b206395d res: sigqueue @@ -51,12 +54,14 @@ __clone Getting filenames and line numbers in ClickHouse source code: ``` sql -WITH arrayMap(x -> addressToLine(x), trace) AS all, arrayFilter(x -> x LIKE '%/dbms/%', all) AS dbms SELECT thread_id, query_id, arrayStringConcat(notEmpty(dbms) ? dbms : all, '\n') AS res FROM system.stack_trace LIMIT 1 \G +WITH arrayMap(x -> addressToLine(x), trace) AS all, arrayFilter(x -> x LIKE '%/dbms/%', all) AS dbms SELECT thread_name, thread_id, query_id, arrayStringConcat(notEmpty(dbms) ? dbms : all, '\n') AS res FROM system.stack_trace LIMIT 1 \G; ``` ``` text Row 1: ────── +thread_name: clickhouse-serv + thread_id: 686 query_id: cad353e7-1c29-4b2e-949f-93e597ab7a54 res: /lib/x86_64-linux-gnu/libc-2.27.so @@ -84,6 +89,3 @@ res: /lib/x86_64-linux-gnu/libc-2.27.so - [system.trace_log](../system-tables/trace_log.md) — Contains stack traces collected by the sampling query profiler. - [arrayMap](../../sql-reference/functions/array-functions.md#array-map) — Description and usage example of the `arrayMap` function. - [arrayFilter](../../sql-reference/functions/array-functions.md#array-filter) — Description and usage example of the `arrayFilter` function. - - -[Original article](https://clickhouse.tech/docs/en/operations/system-tables/stack_trace) diff --git a/docs/en/operations/system-tables/tables.md b/docs/en/operations/system-tables/tables.md index ccc9ab94f8b..480db3087f6 100644 --- a/docs/en/operations/system-tables/tables.md +++ b/docs/en/operations/system-tables/tables.md @@ -54,6 +54,9 @@ Columns: - `lifetime_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of bytes INSERTed since server start (only for `Buffer` tables). +- `comment` ([String](../../sql-reference/data-types/string.md)) - The comment for the table. + + The `system.tables` table is used in `SHOW TABLES` query implementation. **Example** @@ -65,47 +68,53 @@ SELECT * FROM system.tables LIMIT 2 FORMAT Vertical; ```text Row 1: ────── -database: system -name: aggregate_function_combinators -uuid: 00000000-0000-0000-0000-000000000000 -engine: SystemAggregateFunctionCombinators +database: base +name: t1 +uuid: 81b1c20a-b7c6-4116-a2ce-7583fb6b6736 +engine: MergeTree is_temporary: 0 -data_paths: [] -metadata_path: /var/lib/clickhouse/metadata/system/aggregate_function_combinators.sql -metadata_modification_time: 1970-01-01 03:00:00 +data_paths: ['/var/lib/clickhouse/store/81b/81b1c20a-b7c6-4116-a2ce-7583fb6b6736/'] +metadata_path: /var/lib/clickhouse/store/461/461cf698-fd0b-406d-8c01-5d8fd5748a91/t1.sql +metadata_modification_time: 2021-01-25 19:14:32 dependencies_database: [] dependencies_table: [] -create_table_query: -engine_full: -partition_key: -sorting_key: -primary_key: -sampling_key: -storage_policy: -total_rows: ᴺᵁᴸᴸ -total_bytes: ᴺᵁᴸᴸ +create_table_query: CREATE TABLE base.t1 (`n` UInt64) ENGINE = MergeTree ORDER BY n SETTINGS index_granularity = 8192 +engine_full: MergeTree ORDER BY n SETTINGS index_granularity = 8192 +partition_key: +sorting_key: n +primary_key: n +sampling_key: +storage_policy: default +total_rows: 1 +total_bytes: 99 +lifetime_rows: ᴺᵁᴸᴸ +lifetime_bytes: ᴺᵁᴸᴸ +comment: Row 2: ────── -database: system -name: asynchronous_metrics +database: default +name: 53r93yleapyears uuid: 00000000-0000-0000-0000-000000000000 -engine: SystemAsynchronousMetrics +engine: MergeTree is_temporary: 0 -data_paths: [] -metadata_path: /var/lib/clickhouse/metadata/system/asynchronous_metrics.sql -metadata_modification_time: 1970-01-01 03:00:00 +data_paths: ['/var/lib/clickhouse/data/default/53r93yleapyears/'] +metadata_path: /var/lib/clickhouse/metadata/default/53r93yleapyears.sql +metadata_modification_time: 2020-09-23 09:05:36 dependencies_database: [] dependencies_table: [] -create_table_query: -engine_full: -partition_key: -sorting_key: -primary_key: -sampling_key: -storage_policy: -total_rows: ᴺᵁᴸᴸ -total_bytes: ᴺᵁᴸᴸ +create_table_query: CREATE TABLE default.`53r93yleapyears` (`id` Int8, `febdays` Int8) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192 +engine_full: MergeTree ORDER BY id SETTINGS index_granularity = 8192 +partition_key: +sorting_key: id +primary_key: id +sampling_key: +storage_policy: default +total_rows: 2 +total_bytes: 155 +lifetime_rows: ᴺᵁᴸᴸ +lifetime_bytes: ᴺᵁᴸᴸ +comment: ``` [Original article](https://clickhouse.tech/docs/en/operations/system_tables/tables) diff --git a/docs/en/operations/system-tables/zookeeper.md b/docs/en/operations/system-tables/zookeeper.md index 82ace5e81dc..3b8db14934e 100644 --- a/docs/en/operations/system-tables/zookeeper.md +++ b/docs/en/operations/system-tables/zookeeper.md @@ -5,10 +5,10 @@ The query must either have a ‘path =’ condition or a `path IN` condition The query `SELECT * FROM system.zookeeper WHERE path = '/clickhouse'` outputs data for all children on the `/clickhouse` node. To output data for all root nodes, write path = ‘/’. -If the path specified in ‘path’ doesn’t exist, an exception will be thrown. +If the path specified in ‘path’ does not exist, an exception will be thrown. The query `SELECT * FROM system.zookeeper WHERE path IN ('/', '/clickhouse')` outputs data for all children on the `/` and `/clickhouse` node. -If in the specified ‘path’ collection has doesn't exist path, an exception will be thrown. +If in the specified ‘path’ collection has does not exist path, an exception will be thrown. It can be used to do a batch of ZooKeeper path queries. Columns: diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md index 865fe58d7cd..0b74ae95b06 100644 --- a/docs/en/operations/tips.md +++ b/docs/en/operations/tips.md @@ -52,7 +52,7 @@ But for storing archives with rare queries, shelves will work. ## RAID {#raid} When using HDD, you can combine their RAID-10, RAID-5, RAID-6 or RAID-50. -For Linux, software RAID is better (with `mdadm`). We don’t recommend using LVM. +For Linux, software RAID is better (with `mdadm`). We do not recommend using LVM. When creating RAID-10, select the `far` layout. If your budget allows, choose RAID-10. diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index 39449afccef..f2695ce8437 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -55,7 +55,7 @@ If `clickhouse-server` start failed with a configuration error, you should see t 2019.01.11 15:23:25.549505 [ 45 ] {} ExternalDictionaries: Failed reloading 'event2id' external dictionary: Poco::Exception. Code: 1000, e.code() = 111, e.displayText() = Connection refused, e.what() = Connection refused ``` -If you don’t see an error at the end of the file, look through the entire file starting from the string: +If you do not see an error at the end of the file, look through the entire file starting from the string: ``` text Application: starting up. @@ -79,7 +79,7 @@ Revision: 54413 **See system.d logs** -If you don’t find any useful information in `clickhouse-server` logs or there aren’t any logs, you can view `system.d` logs using the command: +If you do not find any useful information in `clickhouse-server` logs or there aren’t any logs, you can view `system.d` logs using the command: ``` bash $ sudo journalctl -u clickhouse-server diff --git a/docs/en/operations/utilities/clickhouse-obfuscator.md b/docs/en/operations/utilities/clickhouse-obfuscator.md index 7fd608fcac0..b01a7624b56 100644 --- a/docs/en/operations/utilities/clickhouse-obfuscator.md +++ b/docs/en/operations/utilities/clickhouse-obfuscator.md @@ -1,42 +1,42 @@ -# ClickHouse obfuscator - -A simple tool for table data obfuscation. - -It reads an input table and produces an output table, that retains some properties of input, but contains different data. -It allows publishing almost real production data for usage in benchmarks. - -It is designed to retain the following properties of data: -- cardinalities of values (number of distinct values) for every column and every tuple of columns; -- conditional cardinalities: number of distinct values of one column under the condition on the value of another column; -- probability distributions of the absolute value of integers; the sign of signed integers; exponent and sign for floats; -- probability distributions of the length of strings; -- probability of zero values of numbers; empty strings and arrays, `NULL`s; - -- data compression ratio when compressed with LZ77 and entropy family of codecs; -- continuity (magnitude of difference) of time values across the table; continuity of floating-point values; -- date component of `DateTime` values; - -- UTF-8 validity of string values; -- string values look natural. - -Most of the properties above are viable for performance testing: - -reading data, filtering, aggregatio, and sorting will work at almost the same speed -as on original data due to saved cardinalities, magnitudes, compression ratios, etc. - -It works in a deterministic fashion: you define a seed value and the transformation is determined by input data and by seed. -Some transformations are one to one and could be reversed, so you need to have a large seed and keep it in secret. - -It uses some cryptographic primitives to transform data but from the cryptographic point of view, it doesn't do it properly, that is why you should not consider the result as secure unless you have another reason. The result may retain some data you don't want to publish. - - -It always leaves 0, 1, -1 numbers, dates, lengths of arrays, and null flags exactly as in source data. -For example, you have a column `IsMobile` in your table with values 0 and 1. In transformed data, it will have the same value. - -So, the user will be able to count the exact ratio of mobile traffic. - -Let's give another example. When you have some private data in your table, like user email and you don't want to publish any single email address. -If your table is large enough and contains multiple different emails and no email has a very high frequency than all others, it will anonymize all data. But if you have a small number of different values in a column, it can reproduce some of them. -You should look at the working algorithm of this tool works, and fine-tune its command line parameters. - -This tool works fine only with an average amount of data (at least 1000s of rows). +# ClickHouse obfuscator + +A simple tool for table data obfuscation. + +It reads an input table and produces an output table, that retains some properties of input, but contains different data. +It allows publishing almost real production data for usage in benchmarks. + +It is designed to retain the following properties of data: +- cardinalities of values (number of distinct values) for every column and every tuple of columns; +- conditional cardinalities: number of distinct values of one column under the condition on the value of another column; +- probability distributions of the absolute value of integers; the sign of signed integers; exponent and sign for floats; +- probability distributions of the length of strings; +- probability of zero values of numbers; empty strings and arrays, `NULL`s; + +- data compression ratio when compressed with LZ77 and entropy family of codecs; +- continuity (magnitude of difference) of time values across the table; continuity of floating-point values; +- date component of `DateTime` values; + +- UTF-8 validity of string values; +- string values look natural. + +Most of the properties above are viable for performance testing: + +reading data, filtering, aggregatio, and sorting will work at almost the same speed +as on original data due to saved cardinalities, magnitudes, compression ratios, etc. + +It works in a deterministic fashion: you define a seed value and the transformation is determined by input data and by seed. +Some transformations are one to one and could be reversed, so you need to have a large seed and keep it in secret. + +It uses some cryptographic primitives to transform data but from the cryptographic point of view, it does not do it properly, that is why you should not consider the result as secure unless you have another reason. The result may retain some data you don't want to publish. + + +It always leaves 0, 1, -1 numbers, dates, lengths of arrays, and null flags exactly as in source data. +For example, you have a column `IsMobile` in your table with values 0 and 1. In transformed data, it will have the same value. + +So, the user will be able to count the exact ratio of mobile traffic. + +Let's give another example. When you have some private data in your table, like user email and you don't want to publish any single email address. +If your table is large enough and contains multiple different emails and no email has a very high frequency than all others, it will anonymize all data. But if you have a small number of different values in a column, it can reproduce some of them. +You should look at the working algorithm of this tool works, and fine-tune its command line parameters. + +This tool works fine only with an average amount of data (at least 1000s of rows). diff --git a/docs/en/sql-reference/aggregate-functions/combinators.md b/docs/en/sql-reference/aggregate-functions/combinators.md index 259202805d3..3fc5121ebcc 100644 --- a/docs/en/sql-reference/aggregate-functions/combinators.md +++ b/docs/en/sql-reference/aggregate-functions/combinators.md @@ -61,7 +61,7 @@ Result: ## -State {#agg-functions-combinator-state} -If you apply this combinator, the aggregate function doesn’t return the resulting value (such as the number of unique values for the [uniq](../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function), but an intermediate state of the aggregation (for `uniq`, this is the hash table for calculating the number of unique values). This is an `AggregateFunction(...)` that can be used for further processing or stored in a table to finish aggregating later. +If you apply this combinator, the aggregate function does not return the resulting value (such as the number of unique values for the [uniq](../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function), but an intermediate state of the aggregation (for `uniq`, this is the hash table for calculating the number of unique values). This is an `AggregateFunction(...)` that can be used for further processing or stored in a table to finish aggregating later. To work with these states, use: @@ -77,7 +77,7 @@ If you apply this combinator, the aggregate function takes the intermediate aggr ## -MergeState {#aggregate_functions_combinators-mergestate} -Merges the intermediate aggregation states in the same way as the -Merge combinator. However, it doesn’t return the resulting value, but an intermediate aggregation state, similar to the -State combinator. +Merges the intermediate aggregation states in the same way as the -Merge combinator. However, it does not return the resulting value, but an intermediate aggregation state, similar to the -State combinator. ## -ForEach {#agg-functions-combinator-foreach} @@ -92,7 +92,7 @@ Examples: `sum(DISTINCT x)`, `groupArray(DISTINCT x)`, `corrStableDistinct(DISTI Changes behavior of an aggregate function. -If an aggregate function doesn’t have input values, with this combinator it returns the default value for its return data type. Applies to the aggregate functions that can take empty input data. +If an aggregate function does not have input values, with this combinator it returns the default value for its return data type. Applies to the aggregate functions that can take empty input data. `-OrDefault` can be used with other combinators. @@ -222,7 +222,7 @@ Lets you divide data into groups, and then separately aggregates the data in tho **Arguments** - `start` — Starting value of the whole required interval for `resampling_key` values. -- `stop` — Ending value of the whole required interval for `resampling_key` values. The whole interval doesn’t include the `stop` value `[start, stop)`. +- `stop` — Ending value of the whole required interval for `resampling_key` values. The whole interval does not include the `stop` value `[start, stop)`. - `step` — Step for separating the whole interval into subintervals. The `aggFunction` is executed over each of those subintervals independently. - `resampling_key` — Column whose values are used for separating data into intervals. - `aggFunction_params` — `aggFunction` parameters. diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 0edb1601023..8e97171d31b 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -9,7 +9,7 @@ Some aggregate functions can accept not only argument columns (used for compress ## histogram {#histogram} -Calculates an adaptive histogram. It doesn’t guarantee precise results. +Calculates an adaptive histogram. It does not guarantee precise results. ``` sql histogram(number_of_bins)(values) @@ -79,7 +79,7 @@ FROM └────────┴───────┘ ``` -In this case, you should remember that you don’t know the histogram bin borders. +In this case, you should remember that you do not know the histogram bin borders. ## sequenceMatch(pattern)(timestamp, cond1, cond2, …) {#function-sequencematch} @@ -114,7 +114,7 @@ Type: `UInt8`. - `(?N)` — Matches the condition argument at position `N`. Conditions are numbered in the `[1, 32]` range. For example, `(?1)` matches the argument passed to the `cond1` parameter. -- `.*` — Matches any number of events. You don’t need conditional arguments to match this element of the pattern. +- `.*` — Matches any number of events. You do not need conditional arguments to match this element of the pattern. - `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` matches events that occur more than 1800 seconds from each other. An arbitrary number of any events can lay between these events. You can use the `>=`, `>`, `<`, `<=` operators. @@ -172,7 +172,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM ## sequenceCount(pattern)(time, cond1, cond2, …) {#function-sequencecount} -Counts the number of event chains that matched the pattern. The function searches event chains that don’t overlap. It starts to search for the next chain after the current chain is matched. +Counts the number of event chains that matched the pattern. The function searches event chains that do not overlap. It starts to search for the next chain after the current chain is matched. !!! warning "Warning" Events that occur at the same second may lay in the sequence in an undefined order affecting the result. diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md index 32d174136e0..70f30f3a480 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md @@ -18,10 +18,10 @@ When using multiple `quantile*` functions with different levels in a query, the **Syntax** ``` sql -quantileTDigest(level)(expr) +quantileTDigestWeighted(level)(expr, weight) ``` -Alias: `medianTDigest`. +Alias: `medianTDigestWeighted`. **Arguments** diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletiming.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletiming.md index 58ce6495a96..dd545c1a485 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiletiming.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletiming.md @@ -6,7 +6,7 @@ toc_priority: 204 With the determined precision computes the [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence. -The result is deterministic (it doesn’t depend on the query processing order). The function is optimized for working with sequences which describe distributions like loading web pages times or backend response times. +The result is deterministic (it does not depend on the query processing order). The function is optimized for working with sequences which describe distributions like loading web pages times or backend response times. When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function. @@ -31,7 +31,7 @@ Alias: `medianTiming`. The calculation is accurate if: -- Total number of values doesn’t exceed 5670. +- Total number of values does not exceed 5670. - Total number of values exceeds 5670, but the page loading time is less than 1024ms. Otherwise, the result of the calculation is rounded to the nearest multiple of 16 ms. diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted.md index fb3b9dbf4d2..25846cde636 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted.md @@ -6,7 +6,7 @@ toc_priority: 205 With the determined precision computes the [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence according to the weight of each sequence member. -The result is deterministic (it doesn’t depend on the query processing order). The function is optimized for working with sequences which describe distributions like loading web pages times or backend response times. +The result is deterministic (it does not depend on the query processing order). The function is optimized for working with sequences which describe distributions like loading web pages times or backend response times. When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function. @@ -33,7 +33,7 @@ Alias: `medianTimingWeighted`. The calculation is accurate if: -- Total number of values doesn’t exceed 5670. +- Total number of values does not exceed 5670. - Total number of values exceeds 5670, but the page loading time is less than 1024ms. Otherwise, the result of the calculation is rounded to the nearest multiple of 16 ms. diff --git a/docs/en/sql-reference/aggregate-functions/reference/rankCorr.md b/docs/en/sql-reference/aggregate-functions/reference/rankCorr.md index 55ee1b8289b..b364317c22b 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/rankCorr.md +++ b/docs/en/sql-reference/aggregate-functions/reference/rankCorr.md @@ -1,4 +1,8 @@ -## rankCorr {#agg_function-rankcorr} +--- +toc_priority: 145 +--- + +# rankCorr {#agg_function-rankcorr} Computes a rank correlation coefficient. diff --git a/docs/en/sql-reference/aggregate-functions/reference/sumcount.md b/docs/en/sql-reference/aggregate-functions/reference/sumcount.md new file mode 100644 index 00000000000..80e87663f89 --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/sumcount.md @@ -0,0 +1,46 @@ +--- +toc_priority: 144 +--- + +# sumCount {#agg_function-sumCount} + +Calculates the sum of the numbers and counts the number of rows at the same time. + +**Syntax** + +``` sql +sumCount(x) +``` + +**Arguments** + +- `x` — Input value, must be [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md), or [Decimal](../../../sql-reference/data-types/decimal.md). + +**Returned value** + +- Tuple `(sum, count)`, where `sum` is the sum of numbers and `count` is the number of rows with not-NULL values. + +Type: [Tuple](../../../sql-reference/data-types/tuple.md). + +**Example** + +Query: + +``` sql +CREATE TABLE s_table (x Int8) Engine = Log; +INSERT INTO s_table SELECT number FROM numbers(0, 20); +INSERT INTO s_table VALUES (NULL); +SELECT sumCount(x) from s_table; +``` + +Result: + +``` text +┌─sumCount(x)─┐ +│ (190,20) │ +└─────────────┘ +``` + +**See also** + +- [optimize_fuse_sum_count_avg](../../../operations/settings/settings.md#optimize_fuse_sum_count_avg) setting. diff --git a/docs/en/sql-reference/aggregate-functions/reference/topk.md b/docs/en/sql-reference/aggregate-functions/reference/topk.md index b9bea013ea8..7e6d0db4946 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/topk.md +++ b/docs/en/sql-reference/aggregate-functions/reference/topk.md @@ -12,7 +12,7 @@ Implements the [Filtered Space-Saving](http://www.l2f.inesc-id.pt/~fmmb/wiki/upl topK(N)(column) ``` -This function doesn’t provide a guaranteed result. In certain situations, errors might occur and it might return frequent values that aren’t the most frequent values. +This function does not provide a guaranteed result. In certain situations, errors might occur and it might return frequent values that aren’t the most frequent values. We recommend using the `N < 10` value; performance is reduced with large `N` values. Maximum value of `N = 65536`. diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniq.md b/docs/en/sql-reference/aggregate-functions/reference/uniq.md index 94771c59cc8..598af24c0de 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniq.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniq.md @@ -28,7 +28,7 @@ Function: This algorithm is very accurate and very efficient on the CPU. When the query contains several of these functions, using `uniq` is almost as fast as using other aggregate functions. -- Provides the result deterministically (it doesn’t depend on the query processing order). +- Provides the result deterministically (it does not depend on the query processing order). We recommend using this function in almost all scenarios. diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md index 8ae916961f9..623c43ae10c 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md @@ -32,7 +32,7 @@ Function: For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. -- Provides the result deterministically (it doesn’t depend on the query processing order). +- Provides the result deterministically (it does not depend on the query processing order). !!! note "Note" Since it uses 32-bit hash for non-`String` type, the result will have very high error for cardinalities significantly larger than `UINT_MAX` (error will raise quickly after a few tens of billions of distinct values), hence in this case you should use [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md b/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md index 80b1e935b55..1d619ab7d93 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md +++ b/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md @@ -28,9 +28,9 @@ Function: 2^12 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). -- Provides the determinate result (it doesn’t depend on the query processing order). +- Provides the determinate result (it does not depend on the query processing order). -We don’t recommend using this function. In most cases, use the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) or [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined) function. +We do not recommend using this function. In most cases, use the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) or [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined) function. **See Also** diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md index 9ed328e0de6..50093053686 100644 --- a/docs/en/sql-reference/data-types/geo.md +++ b/docs/en/sql-reference/data-types/geo.md @@ -5,7 +5,7 @@ toc_title: Geo # Geo Data Types {#geo-data-types} -Clickhouse supports data types for representing geographical objects — locations, lands, etc. +ClickHouse supports data types for representing geographical objects — locations, lands, etc. !!! warning "Warning" Currently geo data types are an experimental feature. To work with them you must set `allow_experimental_geo_types = 1`. diff --git a/docs/en/sql-reference/data-types/lowcardinality.md b/docs/en/sql-reference/data-types/lowcardinality.md index e0a483973e6..5f0f400ce43 100644 --- a/docs/en/sql-reference/data-types/lowcardinality.md +++ b/docs/en/sql-reference/data-types/lowcardinality.md @@ -55,7 +55,7 @@ Functions: ## See Also {#see-also} - [A Magical Mystery Tour of the LowCardinality Data Type](https://www.altinity.com/blog/2019/3/27/low-cardinality). -- [Reducing Clickhouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). +- [Reducing ClickHouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). - [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). [Original article](https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/) diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 2cf5e41867e..4207e389734 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -5,7 +5,7 @@ toc_title: Nullable # Nullable(typename) {#data_type-nullable} -Allows to store special marker ([NULL](../../sql-reference/syntax.md)) that denotes “missing value” alongside normal values allowed by `TypeName`. For example, a `Nullable(Int8)` type column can store `Int8` type values, and the rows that don’t have a value will store `NULL`. +Allows to store special marker ([NULL](../../sql-reference/syntax.md)) that denotes “missing value” alongside normal values allowed by `TypeName`. For example, a `Nullable(Int8)` type column can store `Int8` type values, and the rows that do not have a value will store `NULL`. For a `TypeName`, you can’t use composite data types [Array](../../sql-reference/data-types/array.md) and [Tuple](../../sql-reference/data-types/tuple.md). Composite data types can contain `Nullable` type values, such as `Array(Nullable(Int8))`. diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md index f3a245e9627..8138d4a4103 100644 --- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md @@ -1,6 +1,6 @@ # SimpleAggregateFunction {#data-type-simpleaggregatefunction} -`SimpleAggregateFunction(name, types_of_arguments…)` data type stores current value of the aggregate function, and does not store its full state as [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) does. This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. This property guarantees that partial aggregation results are enough to compute the combined one, so we don’t have to store and process any extra data. +`SimpleAggregateFunction(name, types_of_arguments…)` data type stores current value of the aggregate function, and does not store its full state as [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) does. This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. This property guarantees that partial aggregation results are enough to compute the combined one, so we do not have to store and process any extra data. The common way to produce an aggregate function value is by calling the aggregate function with the [-SimpleState](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-simplestate) suffix. diff --git a/docs/en/sql-reference/data-types/string.md b/docs/en/sql-reference/data-types/string.md index 42d8798c8a3..e72ce8f0b5a 100644 --- a/docs/en/sql-reference/data-types/string.md +++ b/docs/en/sql-reference/data-types/string.md @@ -12,7 +12,7 @@ When creating tables, numeric parameters for string fields can be set (e.g. `VAR ## Encodings {#encodings} -ClickHouse doesn’t have the concept of encodings. Strings can contain an arbitrary set of bytes, which are stored and output as-is. +ClickHouse does not have the concept of encodings. Strings can contain an arbitrary set of bytes, which are stored and output as-is. If you need to store texts, we recommend using UTF-8 encoding. At the very least, if your terminal uses UTF-8 (as recommended), you can read and write your values without making conversions. Similarly, certain functions for working with strings have separate variations that work under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. For example, the ‘length’ function calculates the string length in bytes, while the ‘lengthUTF8’ function calculates the string length in Unicode code points, assuming that the value is UTF-8 encoded. diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 8217fb8da3a..d229336c58d 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -43,7 +43,7 @@ The dictionary configuration file has the following format: You can [configure](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md) any number of dictionaries in the same file. -[DDL queries for dictionaries](../../../sql-reference/statements/create/dictionary.md) doesn’t require any additional records in server configuration. They allow to work with dictionaries as first-class entities, like tables or views. +[DDL queries for dictionaries](../../../sql-reference/statements/create/dictionary.md) does not require any additional records in server configuration. They allow to work with dictionaries as first-class entities, like tables or views. !!! attention "Attention" You can convert values for a small dictionary by describing it in a `SELECT` query (see the [transform](../../../sql-reference/functions/other-functions.md) function). This functionality is not related to external dictionaries. diff --git a/docs/en/sql-reference/dictionaries/internal-dicts.md b/docs/en/sql-reference/dictionaries/internal-dicts.md index 472351a19a4..9f142c4207d 100644 --- a/docs/en/sql-reference/dictionaries/internal-dicts.md +++ b/docs/en/sql-reference/dictionaries/internal-dicts.md @@ -31,7 +31,7 @@ You can also create these files yourself. The file format is as follows: - region ID (`UInt32`) - parent region ID (`UInt32`) -- region type (`UInt8`): 1 - continent, 3 - country, 4 - federal district, 5 - region, 6 - city; other types don’t have values +- region type (`UInt8`): 1 - continent, 3 - country, 4 - federal district, 5 - region, 6 - city; other types do not have values - population (`UInt32`) — optional column `regions_names_*.txt`: TabSeparated (no header), columns: diff --git a/docs/en/sql-reference/functions/arithmetic-functions.md b/docs/en/sql-reference/functions/arithmetic-functions.md index faa03dfc9d3..3187f13b5b9 100644 --- a/docs/en/sql-reference/functions/arithmetic-functions.md +++ b/docs/en/sql-reference/functions/arithmetic-functions.md @@ -70,7 +70,7 @@ Calculates a number with the reverse sign. The result is always signed. ## abs(a) {#arithm_func-abs} -Calculates the absolute value of the number (a). That is, if a \< 0, it returns -a. For unsigned types it doesn’t do anything. For signed integer types, it returns an unsigned number. +Calculates the absolute value of the number (a). That is, if a \< 0, it returns -a. For unsigned types it does not do anything. For signed integer types, it returns an unsigned number. ## gcd(a, b) {#gcda-b} diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index 7d4fcf29476..611f620ed9f 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -125,7 +125,7 @@ hasAll(set, subset) - An empty array is a subset of any array. - `Null` processed as a value. -- Order of values in both of arrays doesn’t matter. +- Order of values in both of arrays does not matter. **Examples** @@ -162,7 +162,7 @@ hasAny(array1, array2) **Peculiar properties** - `Null` processed as a value. -- Order of values in both of arrays doesn’t matter. +- Order of values in both of arrays does not matter. **Examples** @@ -602,7 +602,7 @@ SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res; └────────────────────┘ ``` -Here, the elements that are passed in the second array (\[2, 1\]) define a sorting key for the corresponding element from the source array (\[‘hello’, ‘world’\]), that is, \[‘hello’ –\> 2, ‘world’ –\> 1\]. Since the lambda function doesn’t use `x`, actual values of the source array don’t affect the order in the result. So, ‘hello’ will be the second element in the result, and ‘world’ will be the first. +Here, the elements that are passed in the second array (\[2, 1\]) define a sorting key for the corresponding element from the source array (\[‘hello’, ‘world’\]), that is, \[‘hello’ –\> 2, ‘world’ –\> 1\]. Since the lambda function does not use `x`, actual values of the source array do not affect the order in the result. So, ‘hello’ will be the second element in the result, and ‘world’ will be the first. Other examples are shown below. diff --git a/docs/en/sql-reference/functions/array-join.md b/docs/en/sql-reference/functions/array-join.md index f35e0d10117..e87d0bca4bb 100644 --- a/docs/en/sql-reference/functions/array-join.md +++ b/docs/en/sql-reference/functions/array-join.md @@ -7,7 +7,7 @@ toc_title: arrayJoin This is a very unusual function. -Normal functions don’t change a set of rows, but just change the values in each row (map). +Normal functions do not change a set of rows, but just change the values in each row (map). Aggregate functions compress a set of rows (fold or reduce). The ‘arrayJoin’ function takes each row and generates a set of rows (unfold). diff --git a/docs/en/sql-reference/functions/bit-functions.md b/docs/en/sql-reference/functions/bit-functions.md index e07f28c0f24..57e55a7da56 100644 --- a/docs/en/sql-reference/functions/bit-functions.md +++ b/docs/en/sql-reference/functions/bit-functions.md @@ -228,7 +228,7 @@ bitCount(x) - Number of bits set to one in the input number. -The function doesn’t convert input value to a larger type ([sign extension](https://en.wikipedia.org/wiki/Sign_extension)). So, for example, `bitCount(toUInt8(-1)) = 8`. +The function does not convert input value to a larger type ([sign extension](https://en.wikipedia.org/wiki/Sign_extension)). So, for example, `bitCount(toUInt8(-1)) = 8`. Type: `UInt8`. diff --git a/docs/en/sql-reference/functions/bitmap-functions.md b/docs/en/sql-reference/functions/bitmap-functions.md index 4875532605e..c695c894784 100644 --- a/docs/en/sql-reference/functions/bitmap-functions.md +++ b/docs/en/sql-reference/functions/bitmap-functions.md @@ -140,7 +140,7 @@ bitmapContains(haystack, needle) **Returned values** -- 0 — If `haystack` doesn’t contain `needle`. +- 0 — If `haystack` does not contain `needle`. - 1 — If `haystack` contains `needle`. Type: `UInt8`. diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 69baf64ef55..afbaed2b413 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -460,7 +460,7 @@ For mode values with a meaning of “with 4 or more days this year,” weeks are - Otherwise, it is the last week of the previous year, and the next week is week 1. -For mode values with a meaning of “contains January 1”, the week contains January 1 is week 1. It doesn’t matter how many days in the new year the week contained, even if it contained only one day. +For mode values with a meaning of “contains January 1”, the week contains January 1 is week 1. It does not matter how many days in the new year the week contained, even if it contained only one day. ``` sql toWeek(date, [, mode][, Timezone]) diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index 6b72d3c2269..167afdabb80 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -87,8 +87,6 @@ The function is using uppercase letters `A-F` and not using any prefixes (like ` For integer arguments, it prints hex digits (“nibbles”) from the most significant to least significant (big endian or “human readable” order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints both digits of every byte even if leading digit is zero. -Example: - **Example** Query: @@ -151,10 +149,62 @@ Result: └──────────────────┘ ``` -## unhex(str) {#unhexstr} +## unhex {#unhexstr} -Accepts a string containing any number of hexadecimal digits, and returns a string containing the corresponding bytes. Supports both uppercase and lowercase letters A-F. The number of hexadecimal digits does not have to be even. If it is odd, the last digit is interpreted as the least significant half of the 00-0F byte. If the argument string contains anything other than hexadecimal digits, some implementation-defined result is returned (an exception isn’t thrown). -If you want to convert the result to a number, you can use the ‘reverse’ and ‘reinterpretAsType’ functions. +Performs the opposite operation of [hex](#hex). It interprets each pair of hexadecimal digits (in the argument) as a number and converts it to the byte represented by the number. The return value is a binary string (BLOB). + +If you want to convert the result to a number, you can use the [reverse](../../sql-reference/functions/string-functions.md#reverse) and [reinterpretAs](../../sql-reference/functions/type-conversion-functions.md#type-conversion-functions) functions. + +!!! note "Note" + If `unhex` is invoked from within the `clickhouse-client`, binary strings display using UTF-8. + +Alias: `UNHEX`. + +**Syntax** + +``` sql +unhex(arg) +``` + +**Arguments** + +- `arg` — A string containing any number of hexadecimal digits. Type: [String](../../sql-reference/data-types/string.md). + +Supports both uppercase and lowercase letters `A-F`. The number of hexadecimal digits does not have to be even. If it is odd, the last digit is interpreted as the least significant half of the `00-0F` byte. If the argument string contains anything other than hexadecimal digits, some implementation-defined result is returned (an exception isn’t thrown). For a numeric argument the inverse of hex(N) is not performed by unhex(). + +**Returned value** + +- A binary string (BLOB). + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: +``` sql +SELECT unhex('303132'), UNHEX('4D7953514C'); +``` + +Result: +``` text +┌─unhex('303132')─┬─unhex('4D7953514C')─┐ +│ 012 │ MySQL │ +└─────────────────┴─────────────────────┘ +``` + +Query: + +``` sql +SELECT reinterpretAsUInt64(reverse(unhex('FFF'))) AS num; +``` + +Result: + +``` text +┌──num─┐ +│ 4095 │ +└──────┘ +``` ## UUIDStringToNum(str) {#uuidstringtonumstr} @@ -171,4 +221,3 @@ Accepts an integer. Returns a string containing the list of powers of two that t ## bitmaskToArray(num) {#bitmasktoarraynum} Accepts an integer. Returns an array of UInt64 numbers containing the list of powers of two that total the source number when summed. Numbers in the array are in ascending order. - diff --git a/docs/en/sql-reference/functions/ext-dict-functions.md b/docs/en/sql-reference/functions/ext-dict-functions.md index 8eb10bd0208..7c0fe11ae64 100644 --- a/docs/en/sql-reference/functions/ext-dict-functions.md +++ b/docs/en/sql-reference/functions/ext-dict-functions.md @@ -4,7 +4,7 @@ toc_title: External Dictionaries --- !!! attention "Attention" - `dict_name` parameter must be fully qualified for dictionaries created with DDL queries. Eg. `.`. + For dictionaries, created with [DDL queries](../../sql-reference/statements/create/dictionary.md), the `dict_name` parameter must be fully specified, like `.`. Otherwise, the current database is used. # Functions for Working with External Dictionaries {#ext_dict_functions} @@ -25,7 +25,7 @@ dictGetOrNull('dict_name', attr_name, id_expr) - `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). - `attr_names` — Name of the column of the dictionary, [String literal](../../sql-reference/syntax.md#syntax-string-literal), or tuple of column names, [Tuple](../../sql-reference/data-types/tuple.md)([String literal](../../sql-reference/syntax.md#syntax-string-literal)). - `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration. -- `default_value_expr` — Values returned if the dictionary doesn’t contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) or [Tuple](../../sql-reference/data-types/tuple.md)([Expression](../../sql-reference/syntax.md#syntax-expressions)), returning the value (or values) in the data types configured for the `attr_names` attribute. +- `default_value_expr` — Values returned if the dictionary does not contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) or [Tuple](../../sql-reference/data-types/tuple.md)([Expression](../../sql-reference/syntax.md#syntax-expressions)), returning the value (or values) in the data types configured for the `attr_names` attribute. **Returned value** @@ -37,7 +37,7 @@ dictGetOrNull('dict_name', attr_name, id_expr) - `dictGetOrDefault` returns the value passed as the `default_value_expr` parameter. - `dictGetOrNull` returns `NULL` in case key was not found in dictionary. -ClickHouse throws an exception if it cannot parse the value of the attribute or the value doesn’t match the attribute data type. +ClickHouse throws an exception if it cannot parse the value of the attribute or the value does not match the attribute data type. **Example for simple key dictionary** @@ -288,6 +288,119 @@ dictIsIn('dict_name', child_id_expr, ancestor_id_expr) Type: `UInt8`. +## dictGetChildren {#dictgetchildren} + +Returns first-level children as an array of indexes. It is the inverse transformation for [dictGetHierarchy](#dictgethierarchy). + +**Syntax** + +``` sql +dictGetChildren(dict_name, key) +``` + +**Arguments** + +- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). +- `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value. + +**Returned values** + +- First-level descendants for the key. + +Type: [Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md)). + +**Example** + +Consider the hierarchic dictionary: + +``` text +┌─id─┬─parent_id─┐ +│ 1 │ 0 │ +│ 2 │ 1 │ +│ 3 │ 1 │ +│ 4 │ 2 │ +└────┴───────────┘ +``` + +First-level children: + +``` sql +SELECT dictGetChildren('hierarchy_flat_dictionary', number) FROM system.numbers LIMIT 4; +``` + +``` text +┌─dictGetChildren('hierarchy_flat_dictionary', number)─┐ +│ [1] │ +│ [2,3] │ +│ [4] │ +│ [] │ +└──────────────────────────────────────────────────────┘ +``` + +## dictGetDescendant {#dictgetdescendant} + +Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively. + +**Syntax** + +``` sql +dictGetDescendants(dict_name, key, level) +``` + +**Arguments** + +- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). +- `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value. +- `level` — Hierarchy level. If `level = 0` returns all descendants to the end. [UInt8](../../sql-reference/data-types/int-uint.md). + +**Returned values** + +- Descendants for the key. + +Type: [Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md)). + +**Example** + +Consider the hierarchic dictionary: + +``` text +┌─id─┬─parent_id─┐ +│ 1 │ 0 │ +│ 2 │ 1 │ +│ 3 │ 1 │ +│ 4 │ 2 │ +└────┴───────────┘ +``` +All descendants: + +``` sql +SELECT dictGetDescendants('hierarchy_flat_dictionary', number) FROM system.numbers LIMIT 4; +``` + +``` text +┌─dictGetDescendants('hierarchy_flat_dictionary', number)─┐ +│ [1,2,3,4] │ +│ [2,3,4] │ +│ [4] │ +│ [] │ +└─────────────────────────────────────────────────────────┘ +``` + +First-level descendants: + +``` sql +SELECT dictGetDescendants('hierarchy_flat_dictionary', number, 1) FROM system.numbers LIMIT 4; +``` + +``` text +┌─dictGetDescendants('hierarchy_flat_dictionary', number, 1)─┐ +│ [1] │ +│ [2,3] │ +│ [4] │ +│ [] │ +└────────────────────────────────────────────────────────────┘ +``` + ## Other Functions {#ext_dict_functions-other} ClickHouse supports specialized functions that convert dictionary attribute values to a specific data type regardless of the dictionary configuration. @@ -316,7 +429,7 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) - `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). - `attr_name` — Name of the column of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). - `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration. -- `default_value_expr` — Value returned if the dictionary doesn’t contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning the value in the data type configured for the `attr_name` attribute. +- `default_value_expr` — Value returned if the dictionary does not contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning the value in the data type configured for the `attr_name` attribute. **Returned value** @@ -327,4 +440,4 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) - `dictGet[Type]` returns the content of the `` element specified for the attribute in the dictionary configuration. - `dictGet[Type]OrDefault` returns the value passed as the `default_value_expr` parameter. -ClickHouse throws an exception if it cannot parse the value of the attribute or the value doesn’t match the attribute data type. +ClickHouse throws an exception if it cannot parse the value of the attribute or the value does not match the attribute data type. diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 0ea4cfd6fbe..35a42c49a41 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -43,7 +43,7 @@ SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00') ## MD5 {#hash_functions-md5} Calculates the MD5 from a string and returns the resulting set of bytes as FixedString(16). -If you don’t need MD5 in particular, but you need a decent cryptographic 128-bit hash, use the ‘sipHash128’ function instead. +If you do not need MD5 in particular, but you need a decent cryptographic 128-bit hash, use the ‘sipHash128’ function instead. If you want to get the same result as output by the md5sum utility, use lower(hex(MD5(s))). ## sipHash64 {#hash_functions-siphash64} diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md index 32408759b98..58e0994a11d 100644 --- a/docs/en/sql-reference/functions/index.md +++ b/docs/en/sql-reference/functions/index.md @@ -6,7 +6,7 @@ toc_title: Introduction # Functions {#functions} -There are at least\* two types of functions - regular functions (they are just called “functions”) and aggregate functions. These are completely different concepts. Regular functions work as if they are applied to each row separately (for each row, the result of the function doesn’t depend on the other rows). Aggregate functions accumulate a set of values from various rows (i.e. they depend on the entire set of rows). +There are at least\* two types of functions - regular functions (they are just called “functions”) and aggregate functions. These are completely different concepts. Regular functions work as if they are applied to each row separately (for each row, the result of the function does not depend on the other rows). Aggregate functions accumulate a set of values from various rows (i.e. they depend on the entire set of rows). In this section we discuss regular functions. For aggregate functions, see the section “Aggregate functions”. @@ -14,7 +14,7 @@ In this section we discuss regular functions. For aggregate functions, see the s ## Strong Typing {#strong-typing} -In contrast to standard SQL, ClickHouse has strong typing. In other words, it doesn’t make implicit conversions between types. Each function works for a specific set of types. This means that sometimes you need to use type conversion functions. +In contrast to standard SQL, ClickHouse has strong typing. In other words, it does not make implicit conversions between types. Each function works for a specific set of types. This means that sometimes you need to use type conversion functions. ## Common Subexpression Elimination {#common-subexpression-elimination} @@ -78,7 +78,7 @@ For example, in the query `SELECT f(sum(g(x))) FROM distributed_table GROUP BY h - if a `distributed_table` has at least two shards, the functions ‘g’ and ‘h’ are performed on remote servers, and the function ‘f’ is performed on the requestor server. - if a `distributed_table` has only one shard, all the ‘f’, ‘g’, and ‘h’ functions are performed on this shard’s server. -The result of a function usually doesn’t depend on which server it is performed on. However, sometimes this is important. +The result of a function usually does not depend on which server it is performed on. However, sometimes this is important. For example, functions that work with dictionaries use the dictionary that exists on the server they are running on. Another example is the `hostName` function, which returns the name of the server it is running on in order to make `GROUP BY` by servers in a `SELECT` query. diff --git a/docs/en/sql-reference/functions/ip-address-functions.md b/docs/en/sql-reference/functions/ip-address-functions.md index d37ef2e8f1a..137ebc2407d 100644 --- a/docs/en/sql-reference/functions/ip-address-functions.md +++ b/docs/en/sql-reference/functions/ip-address-functions.md @@ -48,7 +48,7 @@ LIMIT 10 └────────────────┴───────┘ ``` -Since using ‘xxx’ is highly unusual, this may be changed in the future. We recommend that you don’t rely on the exact format of this fragment. +Since using ‘xxx’ is highly unusual, this may be changed in the future. We recommend that you do not rely on the exact format of this fragment. ### IPv6NumToString(x) {#ipv6numtostringx} diff --git a/docs/en/sql-reference/functions/json-functions.md b/docs/en/sql-reference/functions/json-functions.md index d545a0ae4e6..e731180c393 100644 --- a/docs/en/sql-reference/functions/json-functions.md +++ b/docs/en/sql-reference/functions/json-functions.md @@ -12,7 +12,7 @@ The following assumptions are made: 1. The field name (function argument) must be a constant. 2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, but `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` 3. Fields are searched for on any nesting level, indiscriminately. If there are multiple matching fields, the first occurrence is used. -4. The JSON doesn’t have space characters outside of string literals. +4. The JSON does not have space characters outside of string literals. ## visitParamHas(params, name) {#visitparamhasparams-name} @@ -22,7 +22,7 @@ Alias: `simpleJSONHas`. ## visitParamExtractUInt(params, name) {#visitparamextractuintparams-name} -Parses UInt64 from the value of the field named `name`. If this is a string field, it tries to parse a number from the beginning of the string. If the field doesn’t exist, or it exists but doesn’t contain a number, it returns 0. +Parses UInt64 from the value of the field named `name`. If this is a string field, it tries to parse a number from the beginning of the string. If the field does not exist, or it exists but does not contain a number, it returns 0. Alias: `simpleJSONExtractUInt`. @@ -106,7 +106,7 @@ SELECT JSONHas('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4) = 0 - Positive integer = access the n-th member/key from the beginning. - Negative integer = access the n-th member/key from the end. -Minimum index of the element is 1. Thus the element 0 doesn’t exist. +Minimum index of the element is 1. Thus the element 0 does not exist. You may use integers to access both JSON arrays and JSON objects. diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index ecb7b982157..8163650efab 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1654,7 +1654,7 @@ joinGet(join_storage_table_name, `value_column`, join_keys) Returns list of values corresponded to list of keys. -If certain doesn’t exist in source table then `0` or `null` will be returned based on [join_use_nulls](../../operations/settings/settings.md#join_use_nulls) setting. +If certain does not exist in source table then `0` or `null` will be returned based on [join_use_nulls](../../operations/settings/settings.md#join_use_nulls) setting. More info about `join_use_nulls` in [Join operation](../../engines/table-engines/special/join.md). @@ -1714,7 +1714,7 @@ Code: 395. DB::Exception: Received from localhost:9000. DB::Exception: Too many. ## identity {#identity} -Returns the same value that was used as its argument. Used for debugging and testing, allows to cancel using index, and get the query performance of a full scan. When query is analyzed for possible use of index, the analyzer doesn’t look inside `identity` functions. Also constant folding is not applied too. +Returns the same value that was used as its argument. Used for debugging and testing, allows to cancel using index, and get the query performance of a full scan. When query is analyzed for possible use of index, the analyzer does not look inside `identity` functions. Also constant folding is not applied too. **Syntax** diff --git a/docs/en/sql-reference/functions/rounding-functions.md b/docs/en/sql-reference/functions/rounding-functions.md index c0bd44a6467..5f74c6329d1 100644 --- a/docs/en/sql-reference/functions/rounding-functions.md +++ b/docs/en/sql-reference/functions/rounding-functions.md @@ -14,7 +14,7 @@ Returns the largest round number that is less than or equal to `x`. A round numb Examples: `floor(123.45, 1) = 123.4, floor(123.45, -1) = 120.` `x` is any numeric type. The result is a number of the same type. -For integer arguments, it makes sense to round with a negative `N` value (for non-negative `N`, the function doesn’t do anything). +For integer arguments, it makes sense to round with a negative `N` value (for non-negative `N`, the function does not do anything). If rounding causes overflow (for example, floor(-128, -1)), an implementation-specific result is returned. ## ceil(x\[, N\]), ceiling(x\[, N\]) {#ceilx-n-ceilingx-n} diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 85570cb408d..5074f478bc0 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -29,17 +29,17 @@ The function also works for arrays. ## lengthUTF8 {#lengthutf8} -Returns the length of a string in Unicode code points (not in characters), assuming that the string contains a set of bytes that make up UTF-8 encoded text. If this assumption is not met, it returns some result (it doesn’t throw an exception). +Returns the length of a string in Unicode code points (not in characters), assuming that the string contains a set of bytes that make up UTF-8 encoded text. If this assumption is not met, it returns some result (it does not throw an exception). The result type is UInt64. ## char_length, CHAR_LENGTH {#char-length} -Returns the length of a string in Unicode code points (not in characters), assuming that the string contains a set of bytes that make up UTF-8 encoded text. If this assumption is not met, it returns some result (it doesn’t throw an exception). +Returns the length of a string in Unicode code points (not in characters), assuming that the string contains a set of bytes that make up UTF-8 encoded text. If this assumption is not met, it returns some result (it does not throw an exception). The result type is UInt64. ## character_length, CHARACTER_LENGTH {#character-length} -Returns the length of a string in Unicode code points (not in characters), assuming that the string contains a set of bytes that make up UTF-8 encoded text. If this assumption is not met, it returns some result (it doesn’t throw an exception). +Returns the length of a string in Unicode code points (not in characters), assuming that the string contains a set of bytes that make up UTF-8 encoded text. If this assumption is not met, it returns some result (it does not throw an exception). The result type is UInt64. ## lower, lcase {#lower} @@ -53,14 +53,14 @@ Converts ASCII Latin symbols in a string to uppercase. ## lowerUTF8 {#lowerutf8} Converts a string to lowercase, assuming the string contains a set of bytes that make up a UTF-8 encoded text. -It doesn’t detect the language. So for Turkish the result might not be exactly correct. +It does not detect the language. So for Turkish the result might not be exactly correct. If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point. If the string contains a set of bytes that is not UTF-8, then the behavior is undefined. ## upperUTF8 {#upperutf8} Converts a string to uppercase, assuming the string contains a set of bytes that make up a UTF-8 encoded text. -It doesn’t detect the language. So for Turkish the result might not be exactly correct. +It does not detect the language. So for Turkish the result might not be exactly correct. If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point. If the string contains a set of bytes that is not UTF-8, then the behavior is undefined. @@ -139,7 +139,7 @@ Reverses the string (as a sequence of bytes). ## reverseUTF8 {#reverseutf8} -Reverses a sequence of Unicode code points, assuming that the string contains a set of bytes representing a UTF-8 text. Otherwise, it does something else (it doesn’t throw an exception). +Reverses a sequence of Unicode code points, assuming that the string contains a set of bytes representing a UTF-8 text. Otherwise, it does something else (it does not throw an exception). ## format(pattern, s0, s1, …) {#format} @@ -264,7 +264,7 @@ Returns a substring starting with the byte from the ‘offset’ index that is ## substringUTF8(s, offset, length) {#substringutf8} -The same as ‘substring’, but for Unicode code points. Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, it returns some result (it doesn’t throw an exception). +The same as ‘substring’, but for Unicode code points. Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, it returns some result (it does not throw an exception). ## appendTrailingCharIfAbsent(s, c) {#appendtrailingcharifabsent} @@ -305,7 +305,7 @@ SELECT startsWith('Spider-Man', 'Spi'); **Returned values** - 1, if the string starts with the specified prefix. -- 0, if the string doesn’t start with the specified prefix. +- 0, if the string does not start with the specified prefix. **Example** @@ -363,7 +363,7 @@ Result: ## trimLeft {#trimleft} -Removes all consecutive occurrences of common whitespace (ASCII character 32) from the beginning of a string. It doesn’t remove other kinds of whitespace characters (tab, no-break space, etc.). +Removes all consecutive occurrences of common whitespace (ASCII character 32) from the beginning of a string. It does not remove other kinds of whitespace characters (tab, no-break space, etc.). **Syntax** @@ -401,7 +401,7 @@ Result: ## trimRight {#trimright} -Removes all consecutive occurrences of common whitespace (ASCII character 32) from the end of a string. It doesn’t remove other kinds of whitespace characters (tab, no-break space, etc.). +Removes all consecutive occurrences of common whitespace (ASCII character 32) from the end of a string. It does not remove other kinds of whitespace characters (tab, no-break space, etc.). **Syntax** @@ -439,7 +439,7 @@ Result: ## trimBoth {#trimboth} -Removes all consecutive occurrences of common whitespace (ASCII character 32) from both ends of a string. It doesn’t remove other kinds of whitespace characters (tab, no-break space, etc.). +Removes all consecutive occurrences of common whitespace (ASCII character 32) from both ends of a string. It does not remove other kinds of whitespace characters (tab, no-break space, etc.). **Syntax** diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 01b1dd2d004..551c4aee8f0 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -126,7 +126,7 @@ Result: The same as [position](#position) returns the position (in bytes) of the found substring in the string, starting from 1. Use the function for a case-insensitive search. -Works under the assumption that the string contains a set of bytes representing a single-byte encoded text. If this assumption is not met and a character can’t be represented using a single byte, the function doesn’t throw an exception and returns some unexpected result. If character can be represented using two bytes, it will use two bytes and so on. +Works under the assumption that the string contains a set of bytes representing a single-byte encoded text. If this assumption is not met and a character can’t be represented using a single byte, the function does not throw an exception and returns some unexpected result. If character can be represented using two bytes, it will use two bytes and so on. **Syntax** @@ -167,7 +167,7 @@ Result: Returns the position (in Unicode points) of the found substring in the string, starting from 1. -Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, the function doesn’t throw an exception and returns some unexpected result. If character can be represented using two Unicode points, it will use two and so on. +Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, the function does not throw an exception and returns some unexpected result. If character can be represented using two Unicode points, it will use two and so on. For a case-insensitive search, use the function [positionCaseInsensitiveUTF8](#positioncaseinsensitiveutf8). @@ -242,7 +242,7 @@ Result: The same as [positionUTF8](#positionutf8), but is case-insensitive. Returns the position (in Unicode points) of the found substring in the string, starting from 1. -Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, the function doesn’t throw an exception and returns some unexpected result. If character can be represented using two Unicode points, it will use two and so on. +Works under the assumption that the string contains a set of bytes representing a UTF-8 encoded text. If this assumption is not met, the function does not throw an exception and returns some unexpected result. If character can be represented using two Unicode points, it will use two and so on. **Syntax** @@ -349,7 +349,7 @@ For a case-insensitive search or/and in UTF-8 format use functions `multiSearchA Checks whether the string matches the `pattern` regular expression. A `re2` regular expression. The [syntax](https://github.com/google/re2/wiki/Syntax) of the `re2` regular expressions is more limited than the syntax of the Perl regular expressions. -Returns 0 if it doesn’t match, or 1 if it matches. +Returns 0 if it does not match, or 1 if it matches. Note that the backslash symbol (`\`) is used for escaping in the regular expression. The same symbol is used for escaping in string literals. So in order to escape the symbol in a regular expression, you must write two backslashes (\\) in a string literal. @@ -391,11 +391,11 @@ The same as `multiFuzzyMatchAny`, but returns the array of all indices in any or ## extract(haystack, pattern) {#extracthaystack-pattern} -Extracts a fragment of a string using a regular expression. If ‘haystack’ doesn’t match the ‘pattern’ regex, an empty string is returned. If the regex doesn’t contain subpatterns, it takes the fragment that matches the entire regex. Otherwise, it takes the fragment that matches the first subpattern. +Extracts a fragment of a string using a regular expression. If ‘haystack’ does not match the ‘pattern’ regex, an empty string is returned. If the regex does not contain subpatterns, it takes the fragment that matches the entire regex. Otherwise, it takes the fragment that matches the first subpattern. ## extractAll(haystack, pattern) {#extractallhaystack-pattern} -Extracts all the fragments of a string using a regular expression. If ‘haystack’ doesn’t match the ‘pattern’ regex, an empty string is returned. Returns an array of strings consisting of all matches to the regex. In general, the behavior is the same as the ‘extract’ function (it takes the first subpattern, or the entire expression if there isn’t a subpattern). +Extracts all the fragments of a string using a regular expression. If ‘haystack’ does not match the ‘pattern’ regex, an empty string is returned. Returns an array of strings consisting of all matches to the regex. In general, the behavior is the same as the ‘extract’ function (it takes the first subpattern, or the entire expression if there isn’t a subpattern). ## extractAllGroupsHorizontal {#extractallgroups-horizontal} @@ -419,7 +419,7 @@ extractAllGroupsHorizontal(haystack, pattern) - Type: [Array](../../sql-reference/data-types/array.md). -If `haystack` doesn’t match the `pattern` regex, an array of empty arrays is returned. +If `haystack` does not match the `pattern` regex, an array of empty arrays is returned. **Example** @@ -460,7 +460,7 @@ extractAllGroupsVertical(haystack, pattern) - Type: [Array](../../sql-reference/data-types/array.md). -If `haystack` doesn’t match the `pattern` regex, an empty array is returned. +If `haystack` does not match the `pattern` regex, an empty array is returned. **Example** @@ -513,7 +513,7 @@ ilike(haystack, pattern) **Arguments** - `haystack` — Input string. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `pattern` — If `pattern` doesn't contain percent signs or underscores, then the `pattern` only represents the string itself. An underscore (`_`) in `pattern` stands for (matches) any single character. A percent sign (`%`) matches any sequence of zero or more characters. +- `pattern` — If `pattern` does not contain percent signs or underscores, then the `pattern` only represents the string itself. An underscore (`_`) in `pattern` stands for (matches) any single character. A percent sign (`%`) matches any sequence of zero or more characters. Some `pattern` examples: @@ -527,7 +527,7 @@ Some `pattern` examples: **Returned values** - True, if the string matches `pattern`. -- False, if the string doesn't match `pattern`. +- False, if the string does not match `pattern`. **Example** diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 86442835425..4189d0feeb5 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -153,7 +153,7 @@ Result: Can be used with [MinHash](../../sql-reference/functions/hash-functions.md#ngramminhash) functions for detection of semi-duplicate strings: ``` sql -SELECT tupleHammingDistance(wordShingleMinHash(string), wordShingleMinHashCaseInsensitive(string)) as HammingDistance FROM (SELECT 'Clickhouse is a column-oriented database management system for online analytical processing of queries.' AS string); +SELECT tupleHammingDistance(wordShingleMinHash(string), wordShingleMinHashCaseInsensitive(string)) as HammingDistance FROM (SELECT 'ClickHouse is a column-oriented database management system for online analytical processing of queries.' AS string); ``` Result: diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index 9feb7a3c711..397ae45ec71 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -5,7 +5,7 @@ toc_title: URLs # Functions for Working with URLs {#functions-for-working-with-urls} -All these functions don’t follow the RFC. They are maximally simplified for improved performance. +All these functions do not follow the RFC. They are maximally simplified for improved performance. ## Functions that Extract Parts of a URL {#functions-that-extract-parts-of-a-url} @@ -398,7 +398,7 @@ Result: ## Functions that Remove Part of a URL {#functions-that-remove-part-of-a-url} -If the URL doesn’t have anything similar, the URL remains unchanged. +If the URL does not have anything similar, the URL remains unchanged. ### cutWWW {#cutwww} diff --git a/docs/en/sql-reference/functions/ym-dict-functions.md b/docs/en/sql-reference/functions/ym-dict-functions.md index 941f75ff006..f947c81c7a9 100644 --- a/docs/en/sql-reference/functions/ym-dict-functions.md +++ b/docs/en/sql-reference/functions/ym-dict-functions.md @@ -136,7 +136,7 @@ In the Yandex geobase, the population might be recorded for child regions, but n ### regionIn(lhs, rhs\[, geobase\]) {#regioninlhs-rhs-geobase} -Checks whether a ‘lhs’ region belongs to a ‘rhs’ region. Returns a UInt8 number equal to 1 if it belongs, or 0 if it doesn’t belong. +Checks whether a ‘lhs’ region belongs to a ‘rhs’ region. Returns a UInt8 number equal to 1 if it belongs, or 0 if it does not belong. The relationship is reflexive – any region also belongs to itself. ### regionHierarchy(id\[, geobase\]) {#regionhierarchyid-geobase} @@ -146,7 +146,7 @@ Example: `regionHierarchy(toUInt32(213)) = [213,1,3,225,10001,10000]`. ### regionToName(id\[, lang\]) {#regiontonameid-lang} -Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID doesn’t exist, an empty string is returned. +Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID does not exist, an empty string is returned. `ua` and `uk` both mean Ukrainian. diff --git a/docs/en/sql-reference/operators/in.md b/docs/en/sql-reference/operators/in.md index 0abeabc7f57..3d8d2673468 100644 --- a/docs/en/sql-reference/operators/in.md +++ b/docs/en/sql-reference/operators/in.md @@ -208,10 +208,10 @@ and the temporary table `_data1` will be sent to every remote server with the qu This is more optimal than using the normal IN. However, keep the following points in mind: -1. When creating a temporary table, data is not made unique. To reduce the volume of data transmitted over the network, specify DISTINCT in the subquery. (You don’t need to do this for a normal IN.) +1. When creating a temporary table, data is not made unique. To reduce the volume of data transmitted over the network, specify DISTINCT in the subquery. (You do not need to do this for a normal IN.) 2. The temporary table will be sent to all the remote servers. Transmission does not account for network topology. For example, if 10 remote servers reside in a datacenter that is very remote in relation to the requestor server, the data will be sent 10 times over the channel to the remote datacenter. Try to avoid large data sets when using GLOBAL IN. 3. When transmitting data to remote servers, restrictions on network bandwidth are not configurable. You might overload the network. -4. Try to distribute data across servers so that you don’t need to use GLOBAL IN on a regular basis. +4. Try to distribute data across servers so that you do not need to use GLOBAL IN on a regular basis. 5. If you need to use GLOBAL IN often, plan the location of the ClickHouse cluster so that a single group of replicas resides in no more than one data center with a fast network between them, so that a query can be processed entirely within a single data center. It also makes sense to specify a local table in the `GLOBAL IN` clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers. @@ -236,4 +236,4 @@ where M is between 1 and 3 depending on which replica the local query is executi Therefore adding the max_parallel_replicas setting will only produce correct results if both tables have the same replication scheme and are sampled by UserID or a subkey of it. In particular, if local_table_2 does not have a sampling key, incorrect results will be produced. The same rule applies to JOIN. -One workaround if local_table_2 doesn't meet the requirements, is to use `GLOBAL IN` or `GLOBAL JOIN`. +One workaround if local_table_2 does not meet the requirements, is to use `GLOBAL IN` or `GLOBAL JOIN`. diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index e073d5f23f0..31fce7f72b3 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -250,7 +250,7 @@ The following operators do not have a priority since they are brackets: ## Associativity {#associativity} All binary operators have left associativity. For example, `1 + 2 + 3` is transformed to `plus(plus(1, 2), 3)`. -Sometimes this doesn’t work the way you expect. For example, `SELECT 4 > 2 > 3` will result in 0. +Sometimes this does not work the way you expect. For example, `SELECT 4 > 2 > 3` will result in 0. For efficiency, the `and` and `or` functions accept any number of arguments. The corresponding chains of `AND` and `OR` operators are transformed into a single call of these functions. diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index d661bd4cd59..2e7cd1be952 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -39,7 +39,7 @@ Adds a new column to the table with the specified `name`, `type`, [`codec`](../. If the `IF NOT EXISTS` clause is included, the query won’t return an error if the column already exists. If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. If you want to add a column to the beginning of the table use the `FIRST` clause. Otherwise, the column is added to the end of the table. For a chain of actions, `name_after` can be the name of a column that is added in one of the previous actions. -Adding a column just changes the table structure, without performing any actions with data. The data doesn’t appear on the disk after `ALTER`. If the data is missing for a column when reading from the table, it is filled in with default values (by performing the default expression if there is one, or using zeros or empty strings). The column appears on the disk after merging data parts (see [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md)). +Adding a column just changes the table structure, without performing any actions with data. The data does not appear on the disk after `ALTER`. If the data is missing for a column when reading from the table, it is filled in with default values (by performing the default expression if there is one, or using zeros or empty strings). The column appears on the disk after merging data parts (see [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md)). This approach allows us to complete the `ALTER` query instantly, without increasing the volume of old data. @@ -70,7 +70,7 @@ Added3 UInt32 DROP COLUMN [IF EXISTS] name ``` -Deletes the column with the name `name`. If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist. +Deletes the column with the name `name`. If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist. Deletes data from the file system. Since this deletes entire files, the query is completed almost instantly. @@ -89,7 +89,7 @@ ALTER TABLE visits DROP COLUMN browser RENAME COLUMN [IF EXISTS] name to new_name ``` -Renames the column `name` to `new_name`. If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist. Since renaming does not involve the underlying data, the query is completed almost instantly. +Renames the column `name` to `new_name`. If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist. Since renaming does not involve the underlying data, the query is completed almost instantly. **NOTE**: Columns specified in the key expression of the table (either with `ORDER BY` or `PRIMARY KEY`) cannot be renamed. Trying to change these columns will produce `SQL Error [524]`. @@ -107,7 +107,7 @@ CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name Resets all data in a column for a specified partition. Read more about setting the partition name in the section [How to specify the partition expression](#alter-how-to-specify-part-expr). -If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist. +If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist. Example: @@ -121,7 +121,7 @@ ALTER TABLE visits CLEAR COLUMN browser IN PARTITION tuple() COMMENT COLUMN [IF EXISTS] name 'comment' ``` -Adds a comment to the column. If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist. +Adds a comment to the column. If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist. Each column can have one comment. If a comment already exists for the column, a new comment overwrites the previous comment. @@ -149,11 +149,11 @@ This query changes the `name` column properties: For examples of columns TTL modifying, see [Column TTL](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-column-ttl). -If the `IF EXISTS` clause is specified, the query won’t return an error if the column doesn’t exist. +If the `IF EXISTS` clause is specified, the query won’t return an error if the column does not exist. The query also can change the order of the columns using `FIRST | AFTER` clause, see [ADD COLUMN](#alter_add-column) description. -When changing the type, values are converted as if the [toType](../../../sql-reference/functions/type-conversion-functions.md) functions were applied to them. If only the default expression is changed, the query doesn’t do anything complex, and is completed almost instantly. +When changing the type, values are converted as if the [toType](../../../sql-reference/functions/type-conversion-functions.md) functions were applied to them. If only the default expression is changed, the query does not do anything complex, and is completed almost instantly. Example: @@ -213,4 +213,4 @@ If the `ALTER` query is not sufficient to make the table changes you need, you c The `ALTER` query blocks all reads and writes for the table. In other words, if a long `SELECT` is running at the time of the `ALTER` query, the `ALTER` query will wait for it to complete. At the same time, all new queries to the same table will wait while this `ALTER` is running. -For tables that don’t store data themselves (such as `Merge` and `Distributed`), `ALTER` just changes the table structure, and does not change the structure of subordinate tables. For example, when running ALTER for a `Distributed` table, you will also need to run `ALTER` for the tables on all remote servers. +For tables that do not store data themselves (such as `Merge` and `Distributed`), `ALTER` just changes the table structure, and does not change the structure of subordinate tables. For example, when running ALTER for a `Distributed` table, you will also need to run `ALTER` for the tables on all remote servers. diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index b22f89928b9..b38643d6027 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -184,7 +184,7 @@ To restore data from a backup, do the following: 2. Copy the data from the `data/database/table/` directory inside the backup to the `/var/lib/clickhouse/data/database/table/detached/` directory. 3. Run `ALTER TABLE t ATTACH PARTITION` queries to add the data to a table. -Restoring from a backup doesn’t require stopping the server. +Restoring from a backup does not require stopping the server. For more information about backups and restoring data, see the [Data Backup](../../../operations/backup.md) section. diff --git a/docs/en/sql-reference/statements/check-table.md b/docs/en/sql-reference/statements/check-table.md index 65e6238ebbc..d40fe263b1a 100644 --- a/docs/en/sql-reference/statements/check-table.md +++ b/docs/en/sql-reference/statements/check-table.md @@ -28,7 +28,7 @@ The `CHECK TABLE` query supports the following table engines: Performed over the tables with another table engines causes an exception. -Engines from the `*Log` family don’t provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner. +Engines from the `*Log` family do not provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner. ## Checking the MergeTree Family Tables {#checking-mergetree-tables} diff --git a/docs/en/sql-reference/statements/create/database.md b/docs/en/sql-reference/statements/create/database.md index bdb31d44b0b..3c6f73d54db 100644 --- a/docs/en/sql-reference/statements/create/database.md +++ b/docs/en/sql-reference/statements/create/database.md @@ -15,7 +15,7 @@ CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(.. ### IF NOT EXISTS {#if-not-exists} -If the `db_name` database already exists, then ClickHouse doesn’t create a new database and: +If the `db_name` database already exists, then ClickHouse does not create a new database and: - Doesn’t throw an exception if clause is specified. - Throws an exception if clause isn’t specified. diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 5f1f0151350..70ac9acd186 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -96,13 +96,13 @@ If the default expression is defined, the column type is optional. If there isn If the data type and default expression are defined explicitly, this expression will be cast to the specified type using type casting functions. Example: `Hits UInt32 DEFAULT 0` means the same thing as `Hits UInt32 DEFAULT toUInt32(0)`. -Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions don’t contain loops. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. +Default expressions may be defined as an arbitrary expression from table constants and columns. When creating and changing the table structure, it checks that expressions do not contain loops. For INSERT, it checks that expressions are resolvable – that all columns they can be calculated from have been passed. ### DEFAULT {#default} `DEFAULT expr` -Normal default value. If the INSERT query doesn’t specify the corresponding column, it will be filled in by computing the corresponding expression. +Normal default value. If the INSERT query does not specify the corresponding column, it will be filled in by computing the corresponding expression. ### MATERIALIZED {#materialized} @@ -234,14 +234,14 @@ High compression levels are useful for asymmetric scenarios, like compress once, ### Specialized Codecs {#create-query-specialized-codecs} -These codecs are designed to make compression more effective by using specific features of data. Some of these codecs don’t compress data themself. Instead, they prepare the data for a common purpose codec, which compresses it better than without this preparation. +These codecs are designed to make compression more effective by using specific features of data. Some of these codecs do not compress data themself. Instead, they prepare the data for a common purpose codec, which compresses it better than without this preparation. Specialized codecs: - `Delta(delta_bytes)` — Compression approach in which raw values are replaced by the difference of two neighboring values, except for the first value that stays unchanged. Up to `delta_bytes` are used for storing delta values, so `delta_bytes` is the maximum size of raw values. Possible `delta_bytes` values: 1, 2, 4, 8. The default value for `delta_bytes` is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it’s 1. - `DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). - `Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). -- `T64` — Compression approach that crops unused high bits of values in integer data types (including `Enum`, `Date` and `DateTime`). At each step of its algorithm, codec takes a block of 64 values, puts them into 64x64 bit matrix, transposes it, crops the unused bits of values and returns the rest as a sequence. Unused bits are the bits, that don’t differ between maximum and minimum values in the whole data part for which the compression is used. +- `T64` — Compression approach that crops unused high bits of values in integer data types (including `Enum`, `Date` and `DateTime`). At each step of its algorithm, codec takes a block of 64 values, puts them into 64x64 bit matrix, transposes it, crops the unused bits of values and returns the rest as a sequence. Unused bits are the bits, that do not differ between maximum and minimum values in the whole data part for which the compression is used. `DoubleDelta` and `Gorilla` codecs are used in Gorilla TSDB as the components of its compressing algorithm. Gorilla approach is effective in scenarios when there is a sequence of slowly changing values with their timestamps. Timestamps are effectively compressed by the `DoubleDelta` codec, and values are effectively compressed by the `Gorilla` codec. For example, to get an effectively stored table, you can create it in the following configuration: @@ -287,7 +287,7 @@ It’s possible to use tables with [ENGINE = Memory](../../../engines/table-engi !!!note "Note" This query is supported only for [Atomic](../../../engines/database-engines/atomic.md) database engine. -If you need to delete some data from a table, you can create a new table and fill it with a `SELECT` statement that doesn't retrieve unwanted data, then drop the old table and rename the new one: +If you need to delete some data from a table, you can create a new table and fill it with a `SELECT` statement that does not retrieve unwanted data, then drop the old table and rename the new one: ```sql CREATE TABLE myNewTable AS myOldTable; @@ -354,3 +354,39 @@ SELECT * FROM base.t1; │ 3 │ └───┘ ``` + +## COMMENT Clause {#comment-table} + +You can add a comment to the table when you creating it. + +!!!note "Note" + The comment is supported for all table engines except [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) and [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md). + + +**Syntax** + +``` sql +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ... +) +ENGINE = engine +COMMENT 'Comment' +``` + +**Example** + +Query: + +``` sql +CREATE TABLE t1 (x String) ENGINE = Memory COMMENT 'The temporary table'; +SELECT name, comment FROM system.tables WHERE name = 't1'; +``` + +Result: + +```text +┌─name─┬─comment─────────────┐ +│ t1 │ The temporary table │ +└──────┴─────────────────────┘ +``` diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 456adc4bb13..ad9f203b768 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -52,7 +52,7 @@ Another way of specifying host is to use `@` syntax following the username. Exam - `CREATE USER mira@'192.168.%.%'` — Equivalent to the `HOST LIKE` syntax. !!! info "Warning" - ClickHouse treats `user_name@'address'` as a username as a whole. Thus, technically you can create multiple users with the same `user_name` and different constructions after `@`. However, we don’t recommend to do so. + ClickHouse treats `user_name@'address'` as a username as a whole. Thus, technically you can create multiple users with the same `user_name` and different constructions after `@`. However, we do not recommend to do so. ## GRANTEES Clause {#grantees} diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index 633db355d4a..4b51bb8b067 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -15,7 +15,7 @@ Syntax: CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] AS SELECT ... ``` -Normal views don’t store any data. They just perform a read from another table on each access. In other words, a normal view is nothing more than a saved query. When reading from a view, this saved query is used as a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. +Normal views do not store any data. They just perform a read from another table on each access. In other words, a normal view is nothing more than a saved query. When reading from a view, this saved query is used as a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. As an example, assume you’ve created a view: @@ -50,9 +50,9 @@ When creating a materialized view with `TO [db].[table]`, you must not use `POPU A materialized view is implemented as follows: when inserting data to the table specified in `SELECT`, part of the inserted data is converted by this `SELECT` query, and the result is inserted in the view. !!! important "Important" - Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) doesn’t change the materialized view. + Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view. -If you specify `POPULATE`, the existing table data is inserted in the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We **don’t recommend** using POPULATE, since data inserted in the table during the view creation will not be inserted in it. +If you specify `POPULATE`, the existing table data is inserted in the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We **do not recommend** using POPULATE, since data inserted in the table during the view creation will not be inserted in it. A `SELECT` query can contain `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`… Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` is set, data is aggregated during insertion, but only within a single packet of inserted data. The data won’t be further aggregated. The exception is when using an `ENGINE` that independently performs data aggregation, such as `SummingMergeTree`. @@ -229,7 +229,7 @@ WATCH lv ``` ``` -Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.lv doesn't exist.. +Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.lv does not exist.. ``` ### Usage {#live-view-usage} diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 59f5b297ece..a181dd8deee 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -17,7 +17,7 @@ Detaching does not delete the data or metadata for the table or materialized vie Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md). System log tables can be also attached back (e.g. `query_log`, `text_log`, etc). Other system tables can't be reattached. On the next server launch the server will recall those tables again. -`ATTACH MATERIALIZED VIEW` doesn't work with short syntax (without `SELECT`), but you can attach it using the `ATTACH TABLE` query. +`ATTACH MATERIALIZED VIEW` does not work with short syntax (without `SELECT`), but you can attach it using the `ATTACH TABLE` query. Note that you can not detach permanently the table which is already detached (temporary). But you can attach it back and then detach permanently again. @@ -64,7 +64,7 @@ Result: ``` text Received exception from server (version 21.4.1): -Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.test doesn't exist. +Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.test does not exist. ``` [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 4317a20419e..90a2a46c7cf 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -5,7 +5,7 @@ toc_title: DROP # DROP Statements {#drop} -Deletes existing entity. If the `IF EXISTS` clause is specified, these queries don’t return an error if the entity doesn’t exist. +Deletes existing entity. If the `IF EXISTS` clause is specified, these queries do not return an error if the entity does not exist. ## DROP DATABASE {#drop-database} @@ -97,4 +97,4 @@ Syntax: DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -[Оriginal article](https://clickhouse.tech/docs/en/sql-reference/statements/drop/) \ No newline at end of file +[Оriginal article](https://clickhouse.tech/docs/en/sql-reference/statements/drop/) diff --git a/docs/en/sql-reference/statements/exists.md b/docs/en/sql-reference/statements/exists.md index 3b0f4b66343..b7c4a487791 100644 --- a/docs/en/sql-reference/statements/exists.md +++ b/docs/en/sql-reference/statements/exists.md @@ -9,4 +9,4 @@ toc_title: EXISTS EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format] ``` -Returns a single `UInt8`-type column, which contains the single value `0` if the table or database doesn’t exist, or `1` if the table exists in the specified database. +Returns a single `UInt8`-type column, which contains the single value `0` if the table or database does not exist, or `1` if the table exists in the specified database. diff --git a/docs/en/sql-reference/statements/explain.md b/docs/en/sql-reference/statements/explain.md index 1c19adcdcd8..f22f92c625a 100644 --- a/docs/en/sql-reference/statements/explain.md +++ b/docs/en/sql-reference/statements/explain.md @@ -111,9 +111,11 @@ Dump query plan steps. Settings: -- `header` — Prints output header for step. Default: 0. -- `description` — Prints step description. Default: 1. -- `actions` — Prints detailed information about step actions. Default: 0. +- `header` — Prints output header for step. Default: 0. +- `description` — Prints step description. Default: 1. +- `indexes` — Shows used indexes, the number of filtered parts and the number of filtered granules for every index applied. Default: 0. Supported for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables. +- `actions` — Prints detailed information about step actions. Default: 0. +- `json` — Prints query plan steps as a row in [JSON](../../interfaces/formats.md#json) format. Default: 0. It is recommended to use [TSVRaw](../../interfaces/formats.md#tabseparatedraw) format to avoid unnecessary escaping. Example: @@ -134,6 +136,225 @@ Union !!! note "Note" Step and query cost estimation is not supported. +When `json = 1`, the query plan is represented in JSON format. Every node is a dictionary that always has the keys `Node Type` and `Plans`. `Node Type` is a string with a step name. `Plans` is an array with child step descriptions. Other optional keys may be added depending on node type and settings. + +Example: + +```sql +EXPLAIN json = 1, description = 0 SELECT 1 UNION ALL SELECT 2 FORMAT TSVRaw; +``` + +```json +[ + { + "Plan": { + "Node Type": "Union", + "Plans": [ + { + "Node Type": "Expression", + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Plans": [ + { + "Node Type": "ReadFromStorage" + } + ] + } + ] + }, + { + "Node Type": "Expression", + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Plans": [ + { + "Node Type": "ReadFromStorage" + } + ] + } + ] + } + ] + } + } +] +``` + +With `description` = 1, the `Description` key is added to the step: + +```json +{ + "Node Type": "ReadFromStorage", + "Description": "SystemOne" +} +``` + +With `header` = 1, the `Header` key is added to the step as an array of columns. + +Example: + +```sql +EXPLAIN json = 1, description = 0, header = 1 SELECT 1, 2 + dummy; +``` + +```json +[ + { + "Plan": { + "Node Type": "Expression", + "Header": [ + { + "Name": "1", + "Type": "UInt8" + }, + { + "Name": "plus(2, dummy)", + "Type": "UInt16" + } + ], + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Header": [ + { + "Name": "dummy", + "Type": "UInt8" + } + ], + "Plans": [ + { + "Node Type": "ReadFromStorage", + "Header": [ + { + "Name": "dummy", + "Type": "UInt8" + } + ] + } + ] + } + ] + } + } +] +``` + +With `indexes` = 1, the `Indexes` key is added. It contains an array of used indexes. Each index is described as JSON with `Type` key (a string `MinMax`, `Partition`, `PrimaryKey` or `Skip`) and optional keys: + +- `Name` — An index name (for now, is used only for `Skip` index). +- `Keys` — An array of columns used by the index. +- `Condition` — A string with condition used. +- `Description` — An index (for now, is used only for `Skip` index). +- `Initial Parts` — A number of parts before the index is applied. +- `Selected Parts` — A number of parts after the index is applied. +- `Initial Granules` — A number of granules before the index is applied. +- `Selected Granulesis` — A number of granules after the index is applied. + +Example: + +```json +"Node Type": "ReadFromMergeTree", +"Indexes": [ + { + "Type": "MinMax", + "Keys": ["y"], + "Condition": "(y in [1, +inf))", + "Initial Parts": 5, + "Selected Parts": 4, + "Initial Granules": 12, + "Selected Granules": 11 + }, + { + "Type": "Partition", + "Keys": ["y", "bitAnd(z, 3)"], + "Condition": "and((bitAnd(z, 3) not in [1, 1]), and((y in [1, +inf)), (bitAnd(z, 3) not in [1, 1])))", + "Initial Parts": 4, + "Selected Parts": 3, + "Initial Granules": 11, + "Selected Granules": 10 + }, + { + "Type": "PrimaryKey", + "Keys": ["x", "y"], + "Condition": "and((x in [11, +inf)), (y in [1, +inf)))", + "Initial Parts": 3, + "Selected Parts": 2, + "Initial Granules": 10, + "Selected Granules": 6 + }, + { + "Type": "Skip", + "Name": "t_minmax", + "Description": "minmax GRANULARITY 2", + "Initial Parts": 2, + "Selected Parts": 1, + "Initial Granules": 6, + "Selected Granules": 2 + }, + { + "Type": "Skip", + "Name": "t_set", + "Description": "set GRANULARITY 2", + "Initial Parts": 1, + "Selected Parts": 1, + "Initial Granules": 2, + "Selected Granules": 1 + } +] +``` + +With `actions` = 1, added keys depend on step type. + +Example: + +```sql +EXPLAIN json = 1, actions = 1, description = 0 SELECT 1 FORMAT TSVRaw; +``` + +```json +[ + { + "Plan": { + "Node Type": "Expression", + "Expression": { + "Inputs": [], + "Actions": [ + { + "Node Type": "Column", + "Result Type": "UInt8", + "Result Type": "Column", + "Column": "Const(UInt8)", + "Arguments": [], + "Removed Arguments": [], + "Result": 0 + } + ], + "Outputs": [ + { + "Name": "1", + "Type": "UInt8" + } + ], + "Positions": [0], + "Project Input": true + }, + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Plans": [ + { + "Node Type": "ReadFromStorage" + } + ] + } + ] + } + } +] +``` + ### EXPLAIN PIPELINE {#explain-pipeline} Settings: @@ -141,7 +362,6 @@ Settings: - `header` — Prints header for each output port. Default: 0. - `graph` — Prints a graph described in the [DOT](https://en.wikipedia.org/wiki/DOT_(graph_description_language)) graph description language. Default: 0. - `compact` — Prints graph in compact mode if `graph` setting is enabled. Default: 1. -- `indexes` — Shows used indexes, the number of filtered parts, and granules for every index applied. Default: 0. Supported for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables. Example: diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 068eac1d083..8ca2b25ce66 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -49,7 +49,7 @@ It means that `john` has the permission to execute: - `SELECT x FROM db.table`. - `SELECT y FROM db.table`. -`john` can’t execute `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn’t return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns. In this case ClickHouse returns all the data. +`john` can’t execute `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse does not return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns. In this case ClickHouse returns all the data. Also `john` has the `GRANT OPTION` privilege, so it can grant other users with privileges of the same or smaller scope. @@ -230,7 +230,7 @@ Consider the following privilege: GRANT SELECT(x,y) ON db.table TO john ``` -This privilege allows `john` to execute any `SELECT` query that involves data from the `x` and/or `y` columns in `db.table`, for example, `SELECT x FROM db.table`. `john` can’t execute `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn’t return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns, in this case ClickHouse returns all the data. +This privilege allows `john` to execute any `SELECT` query that involves data from the `x` and/or `y` columns in `db.table`, for example, `SELECT x FROM db.table`. `john` can’t execute `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse does not return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns, in this case ClickHouse returns all the data. ### INSERT {#grant-insert} @@ -240,7 +240,7 @@ Privilege level: `COLUMN`. **Description** -User granted with this privilege can execute `INSERT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query doesn’t insert any data. +User granted with this privilege can execute `INSERT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query does not insert any data. **Example** @@ -292,7 +292,7 @@ Examples of how this hierarchy is treated: **Notes** -- The `MODIFY SETTING` privilege allows modifying table engine settings. It doesn’t affect settings or server configuration parameters. +- The `MODIFY SETTING` privilege allows modifying table engine settings. It does not affect settings or server configuration parameters. - The `ATTACH` operation needs the [CREATE](#grant-create) privilege. - The `DETACH` operation needs the [DROP](#grant-drop) privilege. - To stop mutation by the [KILL MUTATION](../../sql-reference/statements/misc.md#kill-mutation) query, you need to have a privilege to start this mutation. For example, if you want to stop the `ALTER UPDATE` query, you need the `ALTER UPDATE`, `ALTER TABLE`, or `ALTER` privilege. diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index 66effcccc3f..db10ddd47c6 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -57,7 +57,7 @@ SELECT * FROM insert_select_testtable; In this example, we see that the second inserted row has `a` and `c` columns filled by the passed values, and `b` filled with value by default. -If a list of columns doesn't include all existing columns, the rest of the columns are filled with: +If a list of columns does not include all existing columns, the rest of the columns are filled with: - The values calculated from the `DEFAULT` expressions specified in the table definition. - Zeros and empty strings, if `DEFAULT` expressions are not defined. @@ -105,6 +105,8 @@ However, you can delete old data using `ALTER TABLE ... DROP PARTITION`. `FORMAT` clause must be specified in the end of query if `SELECT` clause contains table function [input()](../../sql-reference/table-functions/input.md). +To insert a default value instead of `NULL` into a column with not nullable data type, enable [insert_null_as_default](../../operations/settings/settings.md#insert_null_as_default) setting. + ### Performance Considerations {#performance-considerations} `INSERT` sorts the input data by primary key and splits them into partitions by a partition key. If you insert data into several partitions at once, it can significantly reduce the performance of the `INSERT` query. To avoid this: diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 6aa09cca4ef..eab6f602c4a 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -31,7 +31,7 @@ KILL QUERY WHERE user='username' SYNC Read-only users can only stop their own queries. -By default, the asynchronous version of queries is used (`ASYNC`), which doesn’t wait for confirmation that queries have stopped. +By default, the asynchronous version of queries is used (`ASYNC`), which does not wait for confirmation that queries have stopped. The synchronous version (`SYNC`) waits for all queries to stop and displays information about each process as it stops. The response contains the `kill_status` column, which can take the following values: diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 247252d3f4e..5eaf0558d7b 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -20,7 +20,7 @@ The `OPTMIZE` query is supported for [MergeTree](../../engines/table-engines/mer When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all nodes (if the `replication_alter_partitions_sync` setting is enabled). -- If `OPTIMIZE` doesn’t perform a merge for any reason, it doesn’t notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting. +- If `OPTIMIZE` does not perform a merge for any reason, it does not notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting. - If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter/index.md#alter-how-to-specify-part-expr). - If you specify `FINAL`, optimization is performed even when all the data is already in one part. Also merge is forced even if concurrent merges are performed. - If you specify `DEDUPLICATE`, then completely identical rows (unless by-clause is specified) will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine. @@ -31,15 +31,17 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engin If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key). !!! note "Note" - Notice that `*` behaves just like in `SELECT`: `MATERIALIZED` and `ALIAS` columns are not used for expansion. - Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an ALIAS column. + Notice that `*` behaves just like in `SELECT`: [MATERIALIZED](../../sql-reference/statements/create/table.md#materialized) and [ALIAS](../../sql-reference/statements/create/table.md#alias) columns are not used for expansion. + Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an `ALIAS` column. + +**Syntax** ``` sql -OPTIMIZE TABLE table DEDUPLICATE; -- the old one -OPTIMIZE TABLE table DEDUPLICATE BY *; -- not the same as the old one, excludes MATERIALIZED columns (see the note above) +OPTIMIZE TABLE table DEDUPLICATE; -- all columns +OPTIMIZE TABLE table DEDUPLICATE BY *; -- excludes MATERIALIZED and ALIAS columns +OPTIMIZE TABLE table DEDUPLICATE BY colX,colY,colZ; OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT colX; OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT (colX, colY); -OPTIMIZE TABLE table DEDUPLICATE BY col1,col2,col3; OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex'); OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT colX; OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (colX, colY); @@ -47,7 +49,7 @@ OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (c **Examples** -Create a table: +Consider the table: ``` sql CREATE TABLE example ( @@ -62,32 +64,129 @@ CREATE TABLE example ( PARTITION BY partition_key ORDER BY (primary_key, secondary_key); ``` +``` sql +INSERT INTO example (primary_key, secondary_key, value, partition_key) +VALUES (0, 0, 0, 0), (0, 0, 0, 0), (1, 1, 2, 2), (1, 1, 2, 3), (1, 1, 3, 3); +``` +``` sql +SELECT * FROM example; +``` +Result: +``` -The 'old' deduplicate, all columns are taken into account, i.e. row is removed only if all values in all columns are equal to corresponding values in previous row. +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +When columns for deduplication are not specified, all of them are taken into account. Row is removed only if all values in all columns are equal to corresponding values in previous row: ``` sql OPTIMIZE TABLE example FINAL DEDUPLICATE; ``` - -Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key`, and `materialized_value` columns. - ``` sql +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +When columns are specified implicitly, the table is deduplicated by all columns that are not `ALIAS` or `MATERIALIZED`. Considering the table above, these are `primary_key`, `secondary_key`, `value`, and `partition_key` columns: +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY *; ``` - -Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED` and explicitly not `materialized_value`: `primary_key`, `secondary_key`, `value`, and `partition_key` columns. - ``` sql -OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT materialized_value; +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ ``` -Deduplicate explicitly by `primary_key`, `secondary_key`, and `partition_key` columns. +Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED` and explicitly not `value`: `primary_key`, `secondary_key`, and `partition_key` columns. + ``` sql +OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT value; +``` +``` sql +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +Deduplicate explicitly by `primary_key`, `secondary_key`, and `partition_key` columns: +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY primary_key, secondary_key, partition_key; ``` - -Deduplicate by any column matching a regex: `primary_key`, `secondary_key`, and `partition_key` columns. - ``` sql +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +Deduplicate by any column matching a regex: `primary_key`, `secondary_key`, and `partition_key` columns: +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY COLUMNS('.*_key'); ``` +``` sql +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` diff --git a/docs/en/sql-reference/statements/rename.md b/docs/en/sql-reference/statements/rename.md index a9dda6ed3b2..4f454626493 100644 --- a/docs/en/sql-reference/statements/rename.md +++ b/docs/en/sql-reference/statements/rename.md @@ -19,4 +19,4 @@ Renames one or more tables. RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... [ON CLUSTER cluster] ``` -Renaming tables is a light operation. If you indicated another database after `TO`, the table will be moved to this database. However, the directories with databases must reside in the same file system (otherwise, an error is returned). If you rename multiple tables in one query, this is a non-atomic operation, it may be partially executed, queries in other sessions may receive the error `Table ... doesn't exist ..`. +Renaming tables is a light operation. If you indicated another database after `TO`, the table will be moved to this database. However, the directories with databases must reside in the same file system (otherwise, an error is returned). If you rename multiple tables in one query, this is a non-atomic operation, it may be partially executed, queries in other sessions may receive the error `Table ... does not exist ..`. diff --git a/docs/en/sql-reference/statements/select/from.md b/docs/en/sql-reference/statements/select/from.md index 3ecb5096ab8..7c5ea732122 100644 --- a/docs/en/sql-reference/statements/select/from.md +++ b/docs/en/sql-reference/statements/select/from.md @@ -29,7 +29,7 @@ Now `SELECT` queries with `FINAL` are executed in parallel and slightly faster. ### Drawbacks {#drawbacks} -Queries that use `FINAL` are executed slightly slower than similar queries that don’t, because: +Queries that use `FINAL` are executed slightly slower than similar queries that do not, because: - Data is merged during query execution. - Queries with `FINAL` read primary key columns in addition to the columns specified in the query. diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md index a07c810fae8..e6affc07b78 100644 --- a/docs/en/sql-reference/statements/select/group-by.md +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -208,7 +208,7 @@ This extra row is only produced in `JSON*`, `TabSeparated*`, and `Pretty*` forma ### Configuring Totals Processing {#configuring-totals-processing} -By default, `totals_mode = 'before_having'`. In this case, ‘totals’ is calculated across all rows, including the ones that don’t pass through HAVING and `max_rows_to_group_by`. +By default, `totals_mode = 'before_having'`. In this case, ‘totals’ is calculated across all rows, including the ones that do not pass through HAVING and `max_rows_to_group_by`. The other alternatives include only the rows that pass through HAVING in ‘totals’, and behave differently with the setting `max_rows_to_group_by` and `group_by_overflow_mode = 'any'`. @@ -274,4 +274,4 @@ When merging data flushed to the disk, as well as when merging results from remo When external aggregation is enabled, if there was less than `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times). -If you have an [ORDER BY](../../../sql-reference/statements/select/order-by.md) with a [LIMIT](../../../sql-reference/statements/select/limit.md) after `GROUP BY`, then the amount of used RAM depends on the amount of data in `LIMIT`, not in the whole table. But if the `ORDER BY` doesn’t have `LIMIT`, don’t forget to enable external sorting (`max_bytes_before_external_sort`). +If you have an [ORDER BY](../../../sql-reference/statements/select/order-by.md) with a [LIMIT](../../../sql-reference/statements/select/limit.md) after `GROUP BY`, then the amount of used RAM depends on the amount of data in `LIMIT`, not in the whole table. But if the `ORDER BY` does not have `LIMIT`, do not forget to enable external sorting (`max_bytes_before_external_sort`). diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 0712ea8daa7..2f2ce943225 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -101,7 +101,7 @@ SELECT COLUMNS('a'), COLUMNS('c'), toTypeName(COLUMNS('c')) FROM col_names └────┴────┴────┴────────────────┘ ``` -Each column returned by the `COLUMNS` expression is passed to the function as a separate argument. Also you can pass other arguments to the function if it supports them. Be careful when using functions. If a function doesn’t support the number of arguments you have passed to it, ClickHouse throws an exception. +Each column returned by the `COLUMNS` expression is passed to the function as a separate argument. Also you can pass other arguments to the function if it supports them. Be careful when using functions. If a function does not support the number of arguments you have passed to it, ClickHouse throws an exception. For example: @@ -111,12 +111,12 @@ SELECT COLUMNS('a') + COLUMNS('c') FROM col_names ``` text Received exception from server (version 19.14.1): -Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus doesn't match: passed 3, should be 2. +Code: 42. DB::Exception: Received from localhost:9000. DB::Exception: Number of arguments for function plus does not match: passed 3, should be 2. ``` In this example, `COLUMNS('a')` returns two columns: `aa` and `ab`. `COLUMNS('c')` returns the `bc` column. The `+` operator can’t apply to 3 arguments, so ClickHouse throws an exception with the relevant message. -Columns that matched the `COLUMNS` expression can have different data types. If `COLUMNS` doesn’t match any columns and is the only expression in `SELECT`, ClickHouse throws an exception. +Columns that matched the `COLUMNS` expression can have different data types. If `COLUMNS` does not match any columns and is the only expression in `SELECT`, ClickHouse throws an exception. ### Asterisk {#asterisk} @@ -128,7 +128,7 @@ You can put an asterisk in any part of a query instead of an expression. When th - When there is strong filtration on a small number of columns using `PREWHERE`. - In subqueries (since columns that aren’t needed for the external query are excluded from subqueries). -In all other cases, we don’t recommend using the asterisk, since it only gives you the drawbacks of a columnar DBMS instead of the advantages. In other words using the asterisk is not recommended. +In all other cases, we do not recommend using the asterisk, since it only gives you the drawbacks of a columnar DBMS instead of the advantages. In other words using the asterisk is not recommended. ### Extreme Values {#extreme-values} diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index f19a785c6b7..a8fec5cfa26 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -252,11 +252,11 @@ External sorting works much less effectively than sorting in RAM. If `ORDER BY` expression has a prefix that coincides with the table sorting key, you can optimize the query by using the [optimize_read_in_order](../../../operations/settings/settings.md#optimize_read_in_order) setting. - When the `optimize_read_in_order` setting is enabled, the Clickhouse server uses the table index and reads the data in order of the `ORDER BY` key. This allows to avoid reading all data in case of specified [LIMIT](../../../sql-reference/statements/select/limit.md). So queries on big data with small limit are processed faster. + When the `optimize_read_in_order` setting is enabled, the ClickHouse server uses the table index and reads the data in order of the `ORDER BY` key. This allows to avoid reading all data in case of specified [LIMIT](../../../sql-reference/statements/select/limit.md). So queries on big data with small limit are processed faster. -Optimization works with both `ASC` and `DESC` and doesn't work together with [GROUP BY](../../../sql-reference/statements/select/group-by.md) clause and [FINAL](../../../sql-reference/statements/select/from.md#select-from-final) modifier. +Optimization works with both `ASC` and `DESC` and does not work together with [GROUP BY](../../../sql-reference/statements/select/group-by.md) clause and [FINAL](../../../sql-reference/statements/select/from.md#select-from-final) modifier. -When the `optimize_read_in_order` setting is disabled, the Clickhouse server does not use the table index while processing `SELECT` queries. +When the `optimize_read_in_order` setting is disabled, the ClickHouse server does not use the table index while processing `SELECT` queries. Consider disabling `optimize_read_in_order` manually, when running queries that have `ORDER BY` clause, large `LIMIT` and [WHERE](../../../sql-reference/statements/select/where.md) condition that requires to read huge amount of records before queried data is found. @@ -265,7 +265,7 @@ Optimization is supported in the following table engines: - [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) - [Merge](../../../engines/table-engines/special/merge.md), [Buffer](../../../engines/table-engines/special/buffer.md), and [MaterializedView](../../../engines/table-engines/special/materializedview.md) table engines over `MergeTree`-engine tables -In `MaterializedView`-engine tables the optimization works with views like `SELECT ... FROM merge_tree_table ORDER BY pk`. But it is not supported in the queries like `SELECT ... FROM view ORDER BY pk` if the view query doesn't have the `ORDER BY` clause. +In `MaterializedView`-engine tables the optimization works with views like `SELECT ... FROM merge_tree_table ORDER BY pk`. But it is not supported in the queries like `SELECT ... FROM view ORDER BY pk` if the view query does not have the `ORDER BY` clause. ## ORDER BY Expr WITH FILL Modifier {#orderby-with-fill} @@ -364,7 +364,7 @@ returns └────────────┴────────────┴──────────┘ ``` -Field `d1` doesn’t fill and use default value cause we don’t have repeated values for `d2` value, and sequence for `d1` can’t be properly calculated. +Field `d1` does not fill and use default value cause we do not have repeated values for `d2` value, and sequence for `d1` can’t be properly calculated. The following query with a changed field in `ORDER BY` diff --git a/docs/en/sql-reference/statements/select/sample.md b/docs/en/sql-reference/statements/select/sample.md index 55c1919b81d..2ed0a804736 100644 --- a/docs/en/sql-reference/statements/select/sample.md +++ b/docs/en/sql-reference/statements/select/sample.md @@ -11,7 +11,7 @@ When data sampling is enabled, the query is not performed on all the data, but o Approximated query processing can be useful in the following cases: - When you have strict timing requirements (like \<100ms) but you can’t justify the cost of additional hardware resources to meet them. -- When your raw data is not accurate, so approximation doesn’t noticeably degrade the quality. +- When your raw data is not accurate, so approximation does not noticeably degrade the quality. - Business requirements target approximate results (for cost-effectiveness, or to market exact results to premium users). !!! note "Note" @@ -59,7 +59,7 @@ In this case, the query is executed on a sample of at least `n` rows (but not si Since the minimum unit for data reading is one granule (its size is set by the `index_granularity` setting), it makes sense to set a sample that is much larger than the size of the granule. -When using the `SAMPLE n` clause, you don’t know which relative percent of data was processed. So you don’t know the coefficient the aggregate functions should be multiplied by. Use the `_sample_factor` virtual column to get the approximate result. +When using the `SAMPLE n` clause, you do not know which relative percent of data was processed. So you do not know the coefficient the aggregate functions should be multiplied by. Use the `_sample_factor` virtual column to get the approximate result. The `_sample_factor` column contains relative coefficients that are calculated dynamically. This column is created automatically when you [create](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) a table with the specified sampling key. The usage examples of the `_sample_factor` column are shown below. @@ -79,7 +79,7 @@ FROM visits SAMPLE 10000000 ``` -The example below shows how to calculate the average session duration. Note that you don’t need to use the relative coefficient to calculate the average values. +The example below shows how to calculate the average session duration. Note that you do not need to use the relative coefficient to calculate the average values. ``` sql SELECT avg(Duration) diff --git a/docs/en/sql-reference/statements/select/union.md b/docs/en/sql-reference/statements/select/union.md index cf18ff7a4a2..6cedfb89787 100644 --- a/docs/en/sql-reference/statements/select/union.md +++ b/docs/en/sql-reference/statements/select/union.md @@ -78,4 +78,10 @@ Result: Queries that are parts of `UNION/UNION ALL/UNION DISTINCT` can be run simultaneously, and their results can be mixed together. +**See Also** + +- [insert_null_as_default](../../../operations/settings/settings.md#insert_null_as_default) setting. +- [union_default_mode](../../../operations/settings/settings.md#union-default-mode) setting. + + [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/union/) diff --git a/docs/en/sql-reference/statements/select/with.md b/docs/en/sql-reference/statements/select/with.md index 6a0564a8ede..0958f651847 100644 --- a/docs/en/sql-reference/statements/select/with.md +++ b/docs/en/sql-reference/statements/select/with.md @@ -4,7 +4,7 @@ toc_title: WITH # WITH Clause {#with-clause} -Clickhouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), that is provides to use results of `WITH` clause in the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression. +ClickHouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), that is provides to use results of `WITH` clause in the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression. ## Syntax diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 7b3f709b876..a78ef38241f 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -240,7 +240,7 @@ If user is not specified, the query returns privileges for the current user. Shows parameters that were used at a [user creation](../../sql-reference/statements/create/user.md). -`SHOW CREATE USER` doesn’t output user passwords. +`SHOW CREATE USER` does not output user passwords. ### Syntax {#show-create-user-syntax} diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 0d8d9de3306..9397d7002fd 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -169,7 +169,7 @@ SYSTEM START MERGES [ON VOLUME | [db.]merge_tree_family_table_name ### STOP TTL MERGES {#query_language-stop-ttl-merges} Provides possibility to stop background delete old data according to [TTL expression](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) for tables in the MergeTree family: -Returns `Ok.` even if table doesn’t exist or table has not MergeTree engine. Returns error when database doesn’t exist: +Returns `Ok.` even if table does not exist or table has not MergeTree engine. Returns error when database does not exist: ``` sql SYSTEM STOP TTL MERGES [[db.]merge_tree_family_table_name] @@ -178,7 +178,7 @@ SYSTEM STOP TTL MERGES [[db.]merge_tree_family_table_name] ### START TTL MERGES {#query_language-start-ttl-merges} Provides possibility to start background delete old data according to [TTL expression](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) for tables in the MergeTree family: -Returns `Ok.` even if table doesn’t exist. Returns error when database doesn’t exist: +Returns `Ok.` even if table does not exist. Returns error when database does not exist: ``` sql SYSTEM START TTL MERGES [[db.]merge_tree_family_table_name] @@ -187,7 +187,7 @@ SYSTEM START TTL MERGES [[db.]merge_tree_family_table_name] ### STOP MOVES {#query_language-stop-moves} Provides possibility to stop background move data according to [TTL table expression with TO VOLUME or TO DISK clause](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family: -Returns `Ok.` even if table doesn’t exist. Returns error when database doesn’t exist: +Returns `Ok.` even if table does not exist. Returns error when database does not exist: ``` sql SYSTEM STOP MOVES [[db.]merge_tree_family_table_name] @@ -196,7 +196,7 @@ SYSTEM STOP MOVES [[db.]merge_tree_family_table_name] ### START MOVES {#query_language-start-moves} Provides possibility to start background move data according to [TTL table expression with TO VOLUME and TO DISK clause](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family: -Returns `Ok.` even if table doesn’t exist. Returns error when database doesn’t exist: +Returns `Ok.` even if table does not exist. Returns error when database does not exist: ``` sql SYSTEM START MOVES [[db.]merge_tree_family_table_name] @@ -209,7 +209,7 @@ ClickHouse can manage background replication related processes in [ReplicatedMer ### STOP FETCHES {#query_language-system-stop-fetches} Provides possibility to stop background fetches for inserted parts for tables in the `ReplicatedMergeTree` family: -Always returns `Ok.` regardless of the table engine and even if table or database doesn’t exist. +Always returns `Ok.` regardless of the table engine and even if table or database does not exist. ``` sql SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name] @@ -218,7 +218,7 @@ SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name] ### START FETCHES {#query_language-system-start-fetches} Provides possibility to start background fetches for inserted parts for tables in the `ReplicatedMergeTree` family: -Always returns `Ok.` regardless of the table engine and even if table or database doesn’t exist. +Always returns `Ok.` regardless of the table engine and even if table or database does not exist. ``` sql SYSTEM START FETCHES [[db.]replicated_merge_tree_family_table_name] diff --git a/docs/en/sql-reference/table-functions/cluster.md b/docs/en/sql-reference/table-functions/cluster.md index b85542d784f..2856e66db9b 100644 --- a/docs/en/sql-reference/table-functions/cluster.md +++ b/docs/en/sql-reference/table-functions/cluster.md @@ -24,7 +24,7 @@ clusterAllReplicas('cluster_name', db, table[, sharding_key]) `sharding_key` - When insert into cluster function with more than one shard, sharding_key need to be provided. -Using the `cluster` and `clusterAllReplicas` table functions are less efficient than creating a `Distributed` table because in this case, the server connection is re-established for every request. When processing a large number of queries, please always create the `Distributed` table ahead of time, and don’t use the `cluster` and `clusterAllReplicas` table functions. +Using the `cluster` and `clusterAllReplicas` table functions are less efficient than creating a `Distributed` table because in this case, the server connection is re-established for every request. When processing a large number of queries, please always create the `Distributed` table ahead of time, and do not use the `cluster` and `clusterAllReplicas` table functions. The `cluster` and `clusterAllReplicas` table functions can be useful in the following cases: diff --git a/docs/en/sql-reference/table-functions/remote.md b/docs/en/sql-reference/table-functions/remote.md index e80e58a76aa..ae399c7e612 100644 --- a/docs/en/sql-reference/table-functions/remote.md +++ b/docs/en/sql-reference/table-functions/remote.md @@ -42,7 +42,7 @@ The dataset from remote servers. **Usage** -Using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and don’t use the `remote` table function. +Using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function. The `remote` table function can be useful in the following cases: diff --git a/docs/en/sql-reference/table-functions/view.md b/docs/en/sql-reference/table-functions/view.md index e49a9f5218b..18323ec4e92 100644 --- a/docs/en/sql-reference/table-functions/view.md +++ b/docs/en/sql-reference/table-functions/view.md @@ -5,7 +5,7 @@ toc_title: view ## view {#view} -Turns a subquery into a table. The function implements views (see [CREATE VIEW](https://clickhouse.tech/docs/en/sql-reference/statements/create/view/#create-view)). The resulting table doesn't store data, but only stores the specified `SELECT` query. When reading from the table, ClickHouse executes the query and deletes all unnecessary columns from the result. +Turns a subquery into a table. The function implements views (see [CREATE VIEW](https://clickhouse.tech/docs/en/sql-reference/statements/create/view/#create-view)). The resulting table does not store data, but only stores the specified `SELECT` query. When reading from the table, ClickHouse executes the query and deletes all unnecessary columns from the result. **Syntax** diff --git a/docs/en/whats-new/changelog/2017.md b/docs/en/whats-new/changelog/2017.md index 17d3efe7bab..9ceca2b6c4a 100644 --- a/docs/en/whats-new/changelog/2017.md +++ b/docs/en/whats-new/changelog/2017.md @@ -7,7 +7,7 @@ toc_title: '2017' This release contains bug fixes for the previous release 1.1.54318: -- Fixed bug with possible race condition in replication that could lead to data loss. This issue affects versions 1.1.54310 and 1.1.54318. If you use one of these versions with Replicated tables, the update is strongly recommended. This issue shows in logs in Warning messages like `Part ... from own log doesn't exist.` The issue is relevant even if you don’t see these messages in logs. +- Fixed bug with possible race condition in replication that could lead to data loss. This issue affects versions 1.1.54310 and 1.1.54318. If you use one of these versions with Replicated tables, the update is strongly recommended. This issue shows in logs in Warning messages like `Part ... from own log does not exist.` The issue is relevant even if you do not see these messages in logs. ### ClickHouse Release 1.1.54318, 2017-11-30 {#clickhouse-release-1-1-54318-2017-11-30} @@ -50,7 +50,7 @@ This release contains bug fixes for the previous release 1.1.54310: - Fixed nonatomic adding and removing of parts in Replicated tables. - Data inserted into a materialized view is not subjected to unnecessary deduplication. - Executing a query to a Distributed table for which the local replica is lagging and remote replicas are unavailable does not result in an error anymore. -- Users don’t need access permissions to the `default` database to create temporary tables anymore. +- Users do not need access permissions to the `default` database to create temporary tables anymore. - Fixed crashing when specifying the Array type without arguments. - Fixed hangups when the disk volume containing server logs is full. - Fixed an overflow in the toRelativeWeekNum function for the first week of the Unix epoch. @@ -138,7 +138,7 @@ This release contains bug fixes for the previous release 1.1.54310: #### Please Note When Upgrading: {#please-note-when-upgrading} -- There is now a higher default value for the MergeTree setting `max_bytes_to_merge_at_max_space_in_pool` (the maximum total size of data parts to merge, in bytes): it has increased from 100 GiB to 150 GiB. This might result in large merges running after the server upgrade, which could cause an increased load on the disk subsystem. If the free space available on the server is less than twice the total amount of the merges that are running, this will cause all other merges to stop running, including merges of small data parts. As a result, INSERT queries will fail with the message “Merges are processing significantly slower than inserts.” Use the `SELECT * FROM system.merges` query to monitor the situation. You can also check the `DiskSpaceReservedForMerge` metric in the `system.metrics` table, or in Graphite. You don’t need to do anything to fix this, since the issue will resolve itself once the large merges finish. If you find this unacceptable, you can restore the previous value for the `max_bytes_to_merge_at_max_space_in_pool` setting. To do this, go to the `` section in config.xml, set ``` ``107374182400 ``` and restart the server. +- There is now a higher default value for the MergeTree setting `max_bytes_to_merge_at_max_space_in_pool` (the maximum total size of data parts to merge, in bytes): it has increased from 100 GiB to 150 GiB. This might result in large merges running after the server upgrade, which could cause an increased load on the disk subsystem. If the free space available on the server is less than twice the total amount of the merges that are running, this will cause all other merges to stop running, including merges of small data parts. As a result, INSERT queries will fail with the message “Merges are processing significantly slower than inserts.” Use the `SELECT * FROM system.merges` query to monitor the situation. You can also check the `DiskSpaceReservedForMerge` metric in the `system.metrics` table, or in Graphite. You do not need to do anything to fix this, since the issue will resolve itself once the large merges finish. If you find this unacceptable, you can restore the previous value for the `max_bytes_to_merge_at_max_space_in_pool` setting. To do this, go to the `` section in config.xml, set ``` ``107374182400 ``` and restart the server. ### ClickHouse Release 1.1.54284, 2017-08-29 {#clickhouse-release-1-1-54284-2017-08-29} @@ -181,7 +181,7 @@ This release contains bug fixes for the previous release 1.1.54276: - Added the `output_format_json_quote_denormals` setting, which enables outputting nan and inf values in JSON format. - Optimized stream allocation when reading from a Distributed table. -- Settings can be configured in readonly mode if the value doesn’t change. +- Settings can be configured in readonly mode if the value does not change. - Added the ability to retrieve non-integer granules of the MergeTree engine in order to meet restrictions on the block size specified in the preferred_block_size_bytes setting. The purpose is to reduce the consumption of RAM and increase cache locality when processing queries from tables with large columns. - Efficient use of indexes that contain expressions like `toStartOfHour(x)` for conditions like `toStartOfHour(x) op сonstexpr.` - Added new settings for MergeTree engines (the merge_tree section in config.xml): diff --git a/docs/en/whats-new/changelog/2018.md b/docs/en/whats-new/changelog/2018.md index b0c4e147352..3544c9a9b49 100644 --- a/docs/en/whats-new/changelog/2018.md +++ b/docs/en/whats-new/changelog/2018.md @@ -32,7 +32,7 @@ toc_title: '2018' - Now you can use a parameter to configure the precision of the `uniqCombined` aggregate function (select the number of HyperLogLog cells). [#3406](https://github.com/ClickHouse/ClickHouse/pull/3406) - Added the `system.contributors` table that contains the names of everyone who made commits in ClickHouse. [#3452](https://github.com/ClickHouse/ClickHouse/pull/3452) - Added the ability to omit the partition for the `ALTER TABLE ... FREEZE` query in order to back up all partitions at once. [#3514](https://github.com/ClickHouse/ClickHouse/pull/3514) -- Added `dictGet` and `dictGetOrDefault` functions that don’t require specifying the type of return value. The type is determined automatically from the dictionary description. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3564) +- Added `dictGet` and `dictGetOrDefault` functions that do not require specifying the type of return value. The type is determined automatically from the dictionary description. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3564) - Now you can specify comments for a column in the table description and change it using `ALTER`. [#3377](https://github.com/ClickHouse/ClickHouse/pull/3377) - Reading is supported for `Join` type tables with simple keys. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3728) - Now you can specify the options `join_use_nulls`, `max_rows_in_join`, `max_bytes_in_join`, and `join_overflow_mode` when creating a `Join` type table. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/3728) @@ -70,7 +70,7 @@ toc_title: '2018' #### Improvements: {#improvements-1} -- The server does not write the processed configuration files to the `/etc/clickhouse-server/` directory. Instead, it saves them in the `preprocessed_configs` directory inside `path`. This means that the `/etc/clickhouse-server/` directory doesn’t have write access for the `clickhouse` user, which improves security. [#2443](https://github.com/ClickHouse/ClickHouse/pull/2443) +- The server does not write the processed configuration files to the `/etc/clickhouse-server/` directory. Instead, it saves them in the `preprocessed_configs` directory inside `path`. This means that the `/etc/clickhouse-server/` directory does not have write access for the `clickhouse` user, which improves security. [#2443](https://github.com/ClickHouse/ClickHouse/pull/2443) - The `min_merge_bytes_to_use_direct_io` option is set to 10 GiB by default. A merge that forms large parts of tables from the MergeTree family will be performed in `O_DIRECT` mode, which prevents excessive page cache eviction. [#3504](https://github.com/ClickHouse/ClickHouse/pull/3504) - Accelerated server start when there is a very large number of tables. [#3398](https://github.com/ClickHouse/ClickHouse/pull/3398) - Added a connection pool and HTTP `Keep-Alive` for connections between replicas. [#3594](https://github.com/ClickHouse/ClickHouse/pull/3594) @@ -291,7 +291,7 @@ toc_title: '2018' - Fixed an error when using `FINAL` with `PREWHERE`. [#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) - Fixed an error when using `PREWHERE` over columns that were added during `ALTER`. [#3298](https://github.com/ClickHouse/ClickHouse/pull/3298) - Added a check for the absence of `arrayJoin` for `DEFAULT` and `MATERIALIZED` expressions. Previously, `arrayJoin` led to an error when inserting data. [#3337](https://github.com/ClickHouse/ClickHouse/pull/3337) -- Added a check for the absence of `arrayJoin` in a `PREWHERE` clause. Previously, this led to messages like `Size ... doesn't match` or `Unknown compression method` when executing queries. [#3357](https://github.com/ClickHouse/ClickHouse/pull/3357) +- Added a check for the absence of `arrayJoin` in a `PREWHERE` clause. Previously, this led to messages like `Size ... does not match` or `Unknown compression method` when executing queries. [#3357](https://github.com/ClickHouse/ClickHouse/pull/3357) - Fixed segfault that could occur in rare cases after optimization that replaced AND chains from equality evaluations with the corresponding IN expression. [liuyimin-bytedance](https://github.com/ClickHouse/ClickHouse/pull/3339) - Minor corrections to `clickhouse-benchmark`: previously, client information was not sent to the server; now the number of queries executed is calculated more accurately when shutting down and for limiting the number of iterations. [#3351](https://github.com/ClickHouse/ClickHouse/pull/3351) [#3352](https://github.com/ClickHouse/ClickHouse/pull/3352) @@ -392,7 +392,7 @@ toc_title: '2018' - The operation timeout can now be configured when working with ZooKeeper. [urykhy](https://github.com/ClickHouse/ClickHouse/pull/2971) - You can specify an offset for `LIMIT n, m` as `LIMIT n OFFSET m`. [#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) - You can use the `SELECT TOP n` syntax as an alternative for `LIMIT`. [#2840](https://github.com/ClickHouse/ClickHouse/pull/2840) -- Increased the size of the queue to write to system tables, so the `SystemLog parameter queue is full` error doesn’t happen as often. +- Increased the size of the queue to write to system tables, so the `SystemLog parameter queue is full` error does not happen as often. - The `windowFunnel` aggregate function now supports events that meet multiple conditions. [Amos Bird](https://github.com/ClickHouse/ClickHouse/pull/2801) - Duplicate columns can be used in a `USING` clause for `JOIN`. [#3006](https://github.com/ClickHouse/ClickHouse/pull/3006) - `Pretty` formats now have a limit on column alignment by width. Use the `output_format_pretty_max_column_pad_width` setting. If a value is wider, it will still be displayed in its entirety, but the other cells in the table will not be too wide. [#3003](https://github.com/ClickHouse/ClickHouse/pull/3003) @@ -405,7 +405,7 @@ toc_title: '2018' #### Bug Fixes: {#bug-fixes-13} -- Fixed an issue with `Dictionary` tables (throws the `Size of offsets doesn't match size of column` or `Unknown compression method` exception). This bug appeared in version 18.10.3. [#2913](https://github.com/ClickHouse/ClickHouse/issues/2913) +- Fixed an issue with `Dictionary` tables (throws the `Size of offsets does not match size of column` or `Unknown compression method` exception). This bug appeared in version 18.10.3. [#2913](https://github.com/ClickHouse/ClickHouse/issues/2913) - Fixed a bug when merging `CollapsingMergeTree` tables if one of the data parts is empty (these parts are formed during merge or `ALTER DELETE` if all data was deleted), and the `vertical` algorithm was used for the merge. [#3049](https://github.com/ClickHouse/ClickHouse/pull/3049) - Fixed a race condition during `DROP` or `TRUNCATE` for `Memory` tables with a simultaneous `SELECT`, which could lead to server crashes. This bug appeared in version 1.1.54388. [#3038](https://github.com/ClickHouse/ClickHouse/pull/3038) - Fixed the possibility of data loss when inserting in `Replicated` tables if the `Session is expired` error is returned (data loss can be detected by the `ReplicatedDataLoss` metric). This error occurred in version 1.1.54378. [#2939](https://github.com/ClickHouse/ClickHouse/pull/2939) [#2949](https://github.com/ClickHouse/ClickHouse/pull/2949) [#2964](https://github.com/ClickHouse/ClickHouse/pull/2964) @@ -715,7 +715,7 @@ toc_title: '2018' #### Backward Incompatible Changes: {#backward-incompatible-changes-7} - Removed escaping in `Vertical` and `Pretty*` formats and deleted the `VerticalRaw` format. -- If servers with version 1.1.54388 (or newer) and servers with an older version are used simultaneously in a distributed query and the query has the `cast(x, 'Type')` expression without the `AS` keyword and doesn’t have the word `cast` in uppercase, an exception will be thrown with a message like `Not found column cast(0, 'UInt8') in block`. Solution: Update the server on the entire cluster. +- If servers with version 1.1.54388 (or newer) and servers with an older version are used simultaneously in a distributed query and the query has the `cast(x, 'Type')` expression without the `AS` keyword and does not have the word `cast` in uppercase, an exception will be thrown with a message like `Not found column cast(0, 'UInt8') in block`. Solution: Update the server on the entire cluster. ### ClickHouse Release 1.1.54385, 2018-06-01 {#clickhouse-release-1-1-54385-2018-06-01} @@ -1044,7 +1044,7 @@ This release contains bug fixes for the previous release 1.1.54337: #### Backward Incompatible Changes: {#backward-incompatible-changes-11} -- The format for marks in `Log` type tables that contain `Nullable` columns was changed in a backward incompatible way. If you have these tables, you should convert them to the `TinyLog` type before starting up the new server version. To do this, replace `ENGINE = Log` with `ENGINE = TinyLog` in the corresponding `.sql` file in the `metadata` directory. If your table doesn’t have `Nullable` columns or if the type of your table is not `Log`, then you don’t need to do anything. +- The format for marks in `Log` type tables that contain `Nullable` columns was changed in a backward incompatible way. If you have these tables, you should convert them to the `TinyLog` type before starting up the new server version. To do this, replace `ENGINE = Log` with `ENGINE = TinyLog` in the corresponding `.sql` file in the `metadata` directory. If your table does not have `Nullable` columns or if the type of your table is not `Log`, then you do not need to do anything. - Removed the `experimental_allow_extended_storage_definition_syntax` setting. Now this feature is enabled by default. - The `runningIncome` function was renamed to `runningDifferenceStartingWithFirstvalue` to avoid confusion. - Removed the `FROM ARRAY JOIN arr` syntax when ARRAY JOIN is specified directly after FROM with no table (Amos Bird). diff --git a/docs/en/whats-new/changelog/2019.md b/docs/en/whats-new/changelog/2019.md index eacd522390f..bd86bf6ce8b 100644 --- a/docs/en/whats-new/changelog/2019.md +++ b/docs/en/whats-new/changelog/2019.md @@ -11,16 +11,16 @@ toc_title: '2019' - Fixed potential buffer overflow in decompress. Malicious user can pass fabricated compressed data that could cause read after buffer. This issue was found by Eldar Zaitov from Yandex information security team. [#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed possible server crash (`std::terminate`) when the server cannot send or write data in JSON or XML format with values of String data type (that require UTF-8 validation) or when compressing result data with Brotli algorithm or in some other rare cases. [#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed dictionaries with source from a clickhouse `VIEW`, now reading such dictionaries doesn’t cause the error `There is no query`. [#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fixed dictionaries with source from a clickhouse `VIEW`, now reading such dictionaries does not cause the error `There is no query`. [#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Fixed checking if a client host is allowed by host_regexp specified in users.xml. [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241), [#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) - `RENAME TABLE` for a distributed table now renames the folder containing inserted data before sending to shards. This fixes an issue with successive renames `tableA->tableB`, `tableC->tableA`. [#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) - `range_hashed` external dictionaries created by DDL queries now allow ranges of arbitrary numeric types. [#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) - Fixed `INSERT INTO table SELECT ... FROM mysql(...)` table function. [#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -- Fixed segfault in `INSERT INTO TABLE FUNCTION file()` while inserting into a file which doesn’t exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) +- Fixed segfault in `INSERT INTO TABLE FUNCTION file()` while inserting into a file which does not exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) - Fixed bitmapAnd error when intersecting an aggregated bitmap and a scalar bitmap. [#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) - Fixed segfault when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier, just like `EXISTS t`. [#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed return type for functions `rand` and `randConstant` in case of nullable argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -- Fixed `DROP DICTIONARY IF EXISTS db.dict`, now it doesn’t throw exception if `db` doesn’t exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) +- Fixed `DROP DICTIONARY IF EXISTS db.dict`, now it does not throw exception if `db` does not exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) - If a table wasn’t completely dropped because of server crash, the server will try to restore and load it [#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) - Fixed a trivial count query for a distributed table if there are more than two shard local table. [#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) - Fixed bug that lead to a data race in DB::BlockStreamProfileInfo::calculateRowsBeforeLimit() [#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) @@ -36,7 +36,7 @@ toc_title: '2019' - Removed the mutation number from a part name in case there were no mutations. This removing improved the compatibility with older versions. [#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) - Fixed the bug that mutations are skipped for some attached parts due to their data_version are larger than the table mutation version. [#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) - Allow starting the server with redundant copies of parts after moving them to another device. [#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -- Fixed the error “Sizes of columns doesn’t match” that might appear when using aggregate function columns. [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) +- Fixed the error “Sizes of columns does not match” that might appear when using aggregate function columns. [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) - Now an exception will be thrown in case of using WITH TIES alongside LIMIT BY. And now it’s possible to use TOP with LIMIT BY. [#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) - Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) @@ -52,7 +52,7 @@ toc_title: '2019' - Make `bloom_filter` type of index supporting `LowCardinality` and `Nullable` [#7363](https://github.com/ClickHouse/ClickHouse/issues/7363) [#7561](https://github.com/ClickHouse/ClickHouse/pull/7561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Add function `isValidJSON` to check that passed string is a valid json. [#5910](https://github.com/ClickHouse/ClickHouse/issues/5910) [#7293](https://github.com/ClickHouse/ClickHouse/pull/7293) ([Vdimir](https://github.com/Vdimir)) - Implement `arrayCompact` function [#7328](https://github.com/ClickHouse/ClickHouse/pull/7328) ([Memo](https://github.com/Joeywzr)) -- Created function `hex` for Decimal numbers. It works like `hex(reinterpretAsString())`, but doesn’t delete last zero bytes. [#7355](https://github.com/ClickHouse/ClickHouse/pull/7355) ([Mikhail Korotov](https://github.com/millb)) +- Created function `hex` for Decimal numbers. It works like `hex(reinterpretAsString())`, but does not delete last zero bytes. [#7355](https://github.com/ClickHouse/ClickHouse/pull/7355) ([Mikhail Korotov](https://github.com/millb)) - Add `arrayFill` and `arrayReverseFill` functions, which replace elements by other elements in front/back of them in the array. [#7380](https://github.com/ClickHouse/ClickHouse/pull/7380) ([hcz](https://github.com/hczhcz)) - Add `CRC32IEEE()`/`CRC64()` support [#7480](https://github.com/ClickHouse/ClickHouse/pull/7480) ([Azat Khuzhin](https://github.com/azat)) - Implement `char` function similar to one in [mysql](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char) [#7486](https://github.com/ClickHouse/ClickHouse/pull/7486) ([sundyli](https://github.com/sundy-li)) @@ -609,12 +609,12 @@ toc_title: '2019' - Fixed the possibility of hanging queries when server is overloaded and global thread pool becomes near full. This have higher chance to happen on clusters with large number of shards (hundreds), because distributed queries allocate a thread per connection to each shard. For example, this issue may reproduce if a cluster of 330 shards is processing 30 concurrent distributed queries. This issue affects all versions starting from 19.2. [#6301](https://github.com/ClickHouse/ClickHouse/pull/6301) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed logic of `arrayEnumerateUniqRanked` function. [#6423](https://github.com/ClickHouse/ClickHouse/pull/6423) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fix segfault when decoding symbol table. [#6603](https://github.com/ClickHouse/ClickHouse/pull/6603) ([Amos Bird](https://github.com/amosbird)) -- Fixed irrelevant exception in cast of `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fixed irrelevant exception in cast of `LowCardinality(Nullable)` to not-Nullable column in case if it does not contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Removed extra quoting of description in `system.settings` table. [#6696](https://github.com/ClickHouse/ClickHouse/issues/6696) [#6699](https://github.com/ClickHouse/ClickHouse/pull/6699) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Avoid possible deadlock in `TRUNCATE` of Replicated table. [#6695](https://github.com/ClickHouse/ClickHouse/pull/6695) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fix reading in order of sorting key. [#6189](https://github.com/ClickHouse/ClickHouse/pull/6189) ([Anton Popov](https://github.com/CurtizJ)) - Fix `ALTER TABLE ... UPDATE` query for tables with `enable_mixed_granularity_parts=1`. [#6543](https://github.com/ClickHouse/ClickHouse/pull/6543) ([alesapin](https://github.com/alesapin)) -- Fix bug opened by [#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) (since 19.4.0). Reproduces in queries to Distributed tables over MergeTree tables when we doesn’t query any columns (`SELECT 1`). [#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alesapin](https://github.com/alesapin)) +- Fix bug opened by [#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) (since 19.4.0). Reproduces in queries to Distributed tables over MergeTree tables when we does not query any columns (`SELECT 1`). [#6236](https://github.com/ClickHouse/ClickHouse/pull/6236) ([alesapin](https://github.com/alesapin)) - Fixed overflow in integer division of signed type to unsigned type. The behaviour was exactly as in C or C++ language (integer promotion rules) that may be surprising. Please note that the overflow is still possible when dividing large signed number to large unsigned number or vice-versa (but that case is less usual). The issue existed in all server versions. [#6214](https://github.com/ClickHouse/ClickHouse/issues/6214) [#6233](https://github.com/ClickHouse/ClickHouse/pull/6233) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Limit maximum sleep time for throttling when `max_execution_speed` or `max_execution_speed_bytes` is set. Fixed false errors like `Estimated query execution time (inf seconds) is too long`. [#5547](https://github.com/ClickHouse/ClickHouse/issues/5547) [#6232](https://github.com/ClickHouse/ClickHouse/pull/6232) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed issues about using `MATERIALIZED` columns and aliases in `MaterializedView`. [#448](https://github.com/ClickHouse/ClickHouse/issues/448) [#3484](https://github.com/ClickHouse/ClickHouse/issues/3484) [#3450](https://github.com/ClickHouse/ClickHouse/issues/3450) [#2878](https://github.com/ClickHouse/ClickHouse/issues/2878) [#2285](https://github.com/ClickHouse/ClickHouse/issues/2285) [#3796](https://github.com/ClickHouse/ClickHouse/pull/3796) ([Amos Bird](https://github.com/amosbird)) [#6316](https://github.com/ClickHouse/ClickHouse/pull/6316) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -639,7 +639,7 @@ toc_title: '2019' - Allow to `ATTACH` live views (for example, at the server startup) regardless to `allow_experimental_live_view` setting. [#6754](https://github.com/ClickHouse/ClickHouse/pull/6754) ([alexey-milovidov](https://github.com/alexey-milovidov)) - For stack traces gathered by query profiler, do not include stack frames generated by the query profiler itself. [#6250](https://github.com/ClickHouse/ClickHouse/pull/6250) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Now table functions `values`, `file`, `url`, `hdfs` have support for ALIAS columns. [#6255](https://github.com/ClickHouse/ClickHouse/pull/6255) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Throw an exception if `config.d` file doesn’t have the corresponding root element as the config file. [#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) +- Throw an exception if `config.d` file does not have the corresponding root element as the config file. [#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) - Print extra info in exception message for `no space left on device`. [#6182](https://github.com/ClickHouse/ClickHouse/issues/6182), [#6252](https://github.com/ClickHouse/ClickHouse/issues/6252) [#6352](https://github.com/ClickHouse/ClickHouse/pull/6352) ([tavplubix](https://github.com/tavplubix)) - When determining shards of a `Distributed` table to be covered by a read query (for `optimize_skip_unused_shards` = 1) ClickHouse now checks conditions from both `prewhere` and `where` clauses of select statement. [#6521](https://github.com/ClickHouse/ClickHouse/pull/6521) ([Alexander Kazakov](https://github.com/Akazz)) - Enabled `SIMDJSON` for machines without AVX2 but with SSE 4.2 and PCLMUL instruction set. [#6285](https://github.com/ClickHouse/ClickHouse/issues/6285) [#6320](https://github.com/ClickHouse/ClickHouse/pull/6320) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -657,7 +657,7 @@ toc_title: '2019' - Fixed possible deadlock of distributed queries when one of shards is localhost but the query is sent via network connection. [#6759](https://github.com/ClickHouse/ClickHouse/pull/6759) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Changed semantic of multiple tables `RENAME` to avoid possible deadlocks. [#6757](https://github.com/ClickHouse/ClickHouse/issues/6757). [#6756](https://github.com/ClickHouse/ClickHouse/pull/6756) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Rewritten MySQL compatibility server to prevent loading full packet payload in memory. Decreased memory consumption for each connection to approximately `2 * DBMS_DEFAULT_BUFFER_SIZE` (read/write buffers). [#5811](https://github.com/ClickHouse/ClickHouse/pull/5811) ([Yuriy Baranov](https://github.com/yurriy)) -- Move AST alias interpreting logic out of parser that doesn’t have to know anything about query semantics. [#6108](https://github.com/ClickHouse/ClickHouse/pull/6108) ([Artem Zuikov](https://github.com/4ertus2)) +- Move AST alias interpreting logic out of parser that does not have to know anything about query semantics. [#6108](https://github.com/ClickHouse/ClickHouse/pull/6108) ([Artem Zuikov](https://github.com/4ertus2)) - Slightly more safe parsing of `NamesAndTypesList`. [#6408](https://github.com/ClickHouse/ClickHouse/issues/6408). [#6410](https://github.com/ClickHouse/ClickHouse/pull/6410) ([alexey-milovidov](https://github.com/alexey-milovidov)) - `clickhouse-copier`: Allow use `where_condition` from config with `partition_key` alias in query for checking partition existence (Earlier it was used only in reading data queries). [#6577](https://github.com/ClickHouse/ClickHouse/pull/6577) ([proller](https://github.com/proller)) - Added optional message argument in `throwIf`. ([#5772](https://github.com/ClickHouse/ClickHouse/issues/5772)) [#6329](https://github.com/ClickHouse/ClickHouse/pull/6329) ([Vdimir](https://github.com/Vdimir)) @@ -869,7 +869,7 @@ toc_title: '2019' #### Improvement {#improvement-4} -- Throws an exception if `config.d` file doesn’t have the corresponding root element as the config file [#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) +- Throws an exception if `config.d` file does not have the corresponding root element as the config file [#6123](https://github.com/ClickHouse/ClickHouse/pull/6123) ([dimarub2000](https://github.com/dimarub2000)) #### Performance Improvement {#performance-improvement-3} @@ -986,7 +986,7 @@ toc_title: '2019' - Fix segfault in ExternalLoader::reloadOutdated(). [#6082](https://github.com/ClickHouse/ClickHouse/pull/6082) ([Vitaly Baranov](https://github.com/vitlibar)) - Fixed the case when server may close listening sockets but not shutdown and continue serving remaining queries. You may end up with two running clickhouse-server processes. Sometimes, the server may return an error `bad_function_call` for remaining queries. [#6231](https://github.com/ClickHouse/ClickHouse/pull/6231) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed useless and incorrect condition on update field for initial loading of external dictionaries via ODBC, MySQL, ClickHouse and HTTP. This fixes [#6069](https://github.com/ClickHouse/ClickHouse/issues/6069) [#6083](https://github.com/ClickHouse/ClickHouse/pull/6083) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed irrelevant exception in cast of `LowCardinality(Nullable)` to not-Nullable column in case if it doesn’t contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +- Fixed irrelevant exception in cast of `LowCardinality(Nullable)` to not-Nullable column in case if it does not contain Nulls (e.g. in query like `SELECT CAST(CAST('Hello' AS LowCardinality(Nullable(String))) AS String)`. [#6094](https://github.com/ClickHouse/ClickHouse/issues/6094) [#6119](https://github.com/ClickHouse/ClickHouse/pull/6119) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - Fix non-deterministic result of “uniq” aggregate function in extreme rare cases. The bug was present in all ClickHouse versions. [#6058](https://github.com/ClickHouse/ClickHouse/pull/6058) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Segfault when we set a little bit too high CIDR on the function `IPv6CIDRToRange`. [#6068](https://github.com/ClickHouse/ClickHouse/pull/6068) ([Guillaume Tassery](https://github.com/YiuRULE)) - Fixed small memory leak when server throw many exceptions from many different contexts. [#6144](https://github.com/ClickHouse/ClickHouse/pull/6144) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -1159,7 +1159,7 @@ toc_title: '2019' #### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-8} - Implemented `TestKeeper` as an implementation of ZooKeeper interface used for testing [#5643](https://github.com/ClickHouse/ClickHouse/pull/5643) ([alexey-milovidov](https://github.com/alexey-milovidov)) ([levushkin aleksej](https://github.com/alexey-milovidov)) -- From now on `.sql` tests can be run isolated by server, in parallel, with random database. It allows to run them faster, add new tests with custom server configurations, and be sure that different tests doesn’t affect each other. [#5554](https://github.com/ClickHouse/ClickHouse/pull/5554) ([Ivan](https://github.com/abyss7)) +- From now on `.sql` tests can be run isolated by server, in parallel, with random database. It allows to run them faster, add new tests with custom server configurations, and be sure that different tests does not affect each other. [#5554](https://github.com/ClickHouse/ClickHouse/pull/5554) ([Ivan](https://github.com/abyss7)) - Remove `` and `` from performance tests [#5672](https://github.com/ClickHouse/ClickHouse/pull/5672) ([Olga Khvostikova](https://github.com/stavrolia)) - Fixed “select_format” performance test for `Pretty` formats [#5642](https://github.com/ClickHouse/ClickHouse/pull/5642) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -1201,7 +1201,7 @@ toc_title: '2019' - Fixed UInt32 overflow bug in linear models. Allow eval ML model for non-const model argument. [#5516](https://github.com/ClickHouse/ClickHouse/pull/5516) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) - `ALTER TABLE ... DROP INDEX IF EXISTS ...` should not raise an exception if provided index does not exist [#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) - Fix segfault with `bitmapHasAny` in scalar subquery [#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) -- Fixed error when replication connection pool doesn’t retry to resolve host, even when DNS cache was dropped. [#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) +- Fixed error when replication connection pool does not retry to resolve host, even when DNS cache was dropped. [#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) - Fixed `ALTER ... MODIFY TTL` on ReplicatedMergeTree. [#5539](https://github.com/ClickHouse/ClickHouse/pull/5539) ([Anton Popov](https://github.com/CurtizJ)) - Fix INSERT into Distributed table with MATERIALIZED column [#5429](https://github.com/ClickHouse/ClickHouse/pull/5429) ([Azat Khuzhin](https://github.com/azat)) - Fix bad alloc when truncate Join storage [#5437](https://github.com/ClickHouse/ClickHouse/pull/5437) ([TCeason](https://github.com/TCeason)) @@ -1261,8 +1261,8 @@ toc_title: '2019' - Added `max_parts_in_total` setting for MergeTree family of tables (default: 100 000) that prevents unsafe specification of partition key #5166. [#5171](https://github.com/ClickHouse/ClickHouse/pull/5171) ([alexey-milovidov](https://github.com/alexey-milovidov)) - `clickhouse-obfuscator`: derive seed for individual columns by combining initial seed with column name, not column position. This is intended to transform datasets with multiple related tables, so that tables will remain JOINable after transformation. [#5178](https://github.com/ClickHouse/ClickHouse/pull/5178) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Added functions `JSONExtractRaw`, `JSONExtractKeyAndValues`. Renamed functions `jsonExtract` to `JSONExtract`. When something goes wrong these functions return the correspondent values, not `NULL`. Modified function `JSONExtract`, now it gets the return type from its last parameter and doesn’t inject nullables. Implemented fallback to RapidJSON in case AVX2 instructions are not available. Simdjson library updated to a new version. [#5235](https://github.com/ClickHouse/ClickHouse/pull/5235) ([Vitaly Baranov](https://github.com/vitlibar)) -- Now `if` and `multiIf` functions don’t rely on the condition’s `Nullable`, but rely on the branches for sql compatibility. [#5238](https://github.com/ClickHouse/ClickHouse/pull/5238) ([Jian Wu](https://github.com/janplus)) +- Added functions `JSONExtractRaw`, `JSONExtractKeyAndValues`. Renamed functions `jsonExtract` to `JSONExtract`. When something goes wrong these functions return the correspondent values, not `NULL`. Modified function `JSONExtract`, now it gets the return type from its last parameter and does not inject nullables. Implemented fallback to RapidJSON in case AVX2 instructions are not available. Simdjson library updated to a new version. [#5235](https://github.com/ClickHouse/ClickHouse/pull/5235) ([Vitaly Baranov](https://github.com/vitlibar)) +- Now `if` and `multiIf` functions do not rely on the condition’s `Nullable`, but rely on the branches for sql compatibility. [#5238](https://github.com/ClickHouse/ClickHouse/pull/5238) ([Jian Wu](https://github.com/janplus)) - `In` predicate now generates `Null` result from `Null` input like the `Equal` function. [#5152](https://github.com/ClickHouse/ClickHouse/pull/5152) ([Jian Wu](https://github.com/janplus)) - Check the time limit every (flush_interval / poll_timeout) number of rows from Kafka. This allows to break the reading from Kafka consumer more frequently and to check the time limits for the top-level streams [#5249](https://github.com/ClickHouse/ClickHouse/pull/5249) ([Ivan](https://github.com/abyss7)) - Link rdkafka with bundled SASL. It should allow to use SASL SCRAM authentication [#5253](https://github.com/ClickHouse/ClickHouse/pull/5253) ([Ivan](https://github.com/abyss7)) @@ -1347,13 +1347,13 @@ toc_title: '2019' - Fixed bitmap functions produce wrong result. [#5359](https://github.com/ClickHouse/ClickHouse/pull/5359) ([Andy Yang](https://github.com/andyyzh)) - Fix element_count for hashed dictionary (do not include duplicates) [#5440](https://github.com/ClickHouse/ClickHouse/pull/5440) ([Azat Khuzhin](https://github.com/azat)) - Use contents of environment variable TZ as the name for timezone. It helps to correctly detect default timezone in some cases.[#5443](https://github.com/ClickHouse/ClickHouse/pull/5443) ([Ivan](https://github.com/abyss7)) -- Do not try to convert integers in `dictGetT` functions, because it doesn’t work correctly. Throw an exception instead. [#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) +- Do not try to convert integers in `dictGetT` functions, because it does not work correctly. Throw an exception instead. [#5446](https://github.com/ClickHouse/ClickHouse/pull/5446) ([Artem Zuikov](https://github.com/4ertus2)) - Fix settings in ExternalData HTTP request. [#5455](https://github.com/ClickHouse/ClickHouse/pull/5455) ([Danila Kutenin](https://github.com/danlark1)) - Fix bug when parts were removed only from FS without dropping them from Zookeeper. [#5520](https://github.com/ClickHouse/ClickHouse/pull/5520) ([alesapin](https://github.com/alesapin)) - Fix segmentation fault in `bitmapHasAny` function. [#5528](https://github.com/ClickHouse/ClickHouse/pull/5528) ([Zhichang Yu](https://github.com/yuzhichang)) -- Fixed error when replication connection pool doesn’t retry to resolve host, even when DNS cache was dropped. [#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) -- Fixed `DROP INDEX IF EXISTS` query. Now `ALTER TABLE ... DROP INDEX IF EXISTS ...` query doesn’t raise an exception if provided index does not exist. [#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) +- Fixed error when replication connection pool does not retry to resolve host, even when DNS cache was dropped. [#5534](https://github.com/ClickHouse/ClickHouse/pull/5534) ([alesapin](https://github.com/alesapin)) +- Fixed `DROP INDEX IF EXISTS` query. Now `ALTER TABLE ... DROP INDEX IF EXISTS ...` query does not raise an exception if provided index does not exist. [#5524](https://github.com/ClickHouse/ClickHouse/pull/5524) ([Gleb Novikov](https://github.com/NanoBjorn)) - Fix union all supertype column. There were cases with inconsistent data and column types of resulting columns. [#5503](https://github.com/ClickHouse/ClickHouse/pull/5503) ([Artem Zuikov](https://github.com/4ertus2)) - Skip ZNONODE during DDL query processing. Before if another node removes the znode in task queue, the one that did not process it, but already get list of children, will terminate the DDLWorker thread. [#5489](https://github.com/ClickHouse/ClickHouse/pull/5489) ([Azat Khuzhin](https://github.com/azat)) @@ -1712,7 +1712,7 @@ toc_title: '2019' - Improved heuristics of “move to PREWHERE” optimization. [#4405](https://github.com/ClickHouse/ClickHouse/pull/4405) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Use proper lookup tables that uses HashTable’s API for 8-bit and 16-bit keys. [#4536](https://github.com/ClickHouse/ClickHouse/pull/4536) ([Amos Bird](https://github.com/amosbird)) - Improved performance of string comparison. [#4564](https://github.com/ClickHouse/ClickHouse/pull/4564) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Cleanup distributed DDL queue in a separate thread so that it doesn’t slow down the main loop that processes distributed DDL tasks. [#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([Alex Zatelepin](https://github.com/ztlpn)) +- Cleanup distributed DDL queue in a separate thread so that it does not slow down the main loop that processes distributed DDL tasks. [#4502](https://github.com/ClickHouse/ClickHouse/pull/4502) ([Alex Zatelepin](https://github.com/ztlpn)) - When `min_bytes_to_use_direct_io` is set to 1, not every file was opened with O_DIRECT mode because the data size to read was sometimes underestimated by the size of one compressed block. [#4526](https://github.com/ClickHouse/ClickHouse/pull/4526) ([alexey-milovidov](https://github.com/alexey-milovidov)) #### Build/Testing/Packaging Improvement {#buildtestingpackaging-improvement-12} @@ -1839,7 +1839,7 @@ toc_title: '2019' - Fixed incorrect result when `Date` and `DateTime` arguments are used in branches of conditional operator (function `if`). Added generic case for function `if`. [#4243](https://github.com/ClickHouse/ClickHouse/pull/4243) ([alexey-milovidov](https://github.com/alexey-milovidov)) - ClickHouse dictionaries now load within `clickhouse` process. [#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed deadlock when `SELECT` from a table with `File` engine was retried after `No such file or directory` error. [#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed race condition when selecting from `system.tables` may give `table doesn't exist` error. [#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fixed race condition when selecting from `system.tables` may give `table does not exist` error. [#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) - `clickhouse-client` can segfault on exit while loading data for command line suggestions if it was run in interactive mode. [#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed a bug when the execution of mutations containing `IN` operators was producing incorrect results. [#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) - Fixed error: if there is a database with `Dictionary` engine, all dictionaries forced to load at server startup, and if there is a dictionary with ClickHouse source from localhost, the dictionary cannot load. [#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -1941,7 +1941,7 @@ This release contains exactly the same set of patches as 19.3.6. - Fixed error: if there is a database with `Dictionary` engine, all dictionaries forced to load at server startup, and if there is a dictionary with ClickHouse source from localhost, the dictionary cannot load. [#4255](https://github.com/ClickHouse/ClickHouse/pull/4255) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed a bug when the execution of mutations containing `IN` operators was producing incorrect results. [#4099](https://github.com/ClickHouse/ClickHouse/pull/4099) ([Alex Zatelepin](https://github.com/ztlpn)) - `clickhouse-client` can segfault on exit while loading data for command line suggestions if it was run in interactive mode. [#4317](https://github.com/ClickHouse/ClickHouse/pull/4317) ([alexey-milovidov](https://github.com/alexey-milovidov)) -- Fixed race condition when selecting from `system.tables` may give `table doesn't exist` error. [#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fixed race condition when selecting from `system.tables` may give `table does not exist` error. [#4313](https://github.com/ClickHouse/ClickHouse/pull/4313) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed deadlock when `SELECT` from a table with `File` engine was retried after `No such file or directory` error. [#4161](https://github.com/ClickHouse/ClickHouse/pull/4161) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed an issue: local ClickHouse dictionaries are loaded via TCP, but should load within process. [#4166](https://github.com/ClickHouse/ClickHouse/pull/4166) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed `No message received` error when interacting with PostgreSQL ODBC Driver through TLS connection. Also fixes segfault when using MySQL ODBC Driver. [#4170](https://github.com/ClickHouse/ClickHouse/pull/4170) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -2030,8 +2030,8 @@ This release contains exactly the same set of patches as 19.3.6. #### Performance Improvements {#performance-improvements-5} -- Add a MergeTree setting `use_minimalistic_part_header_in_zookeeper`. If enabled, Replicated tables will store compact part metadata in a single part znode. This can dramatically reduce ZooKeeper snapshot size (especially if the tables have a lot of columns). Note that after enabling this setting you will not be able to downgrade to a version that doesn’t support it. [#3960](https://github.com/ClickHouse/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn)) -- Add an DFA-based implementation for functions `sequenceMatch` and `sequenceCount` in case pattern doesn’t contain time. [#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) +- Add a MergeTree setting `use_minimalistic_part_header_in_zookeeper`. If enabled, Replicated tables will store compact part metadata in a single part znode. This can dramatically reduce ZooKeeper snapshot size (especially if the tables have a lot of columns). Note that after enabling this setting you will not be able to downgrade to a version that does not support it. [#3960](https://github.com/ClickHouse/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn)) +- Add an DFA-based implementation for functions `sequenceMatch` and `sequenceCount` in case pattern does not contain time. [#4004](https://github.com/ClickHouse/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) - Performance improvement for integer numbers serialization. [#3968](https://github.com/ClickHouse/ClickHouse/pull/3968) ([Amos Bird](https://github.com/amosbird)) - Zero left padding PODArray so that -1 element is always valid and zeroed. It’s used for branchless calculation of offsets. [#3920](https://github.com/ClickHouse/ClickHouse/pull/3920) ([Amos Bird](https://github.com/amosbird)) - Reverted `jemalloc` version which lead to performance degradation. [#4018](https://github.com/ClickHouse/ClickHouse/pull/4018) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -2055,7 +2055,7 @@ This release contains exactly the same set of patches as 19.3.6. - Fixed bugs found by PVS-Studio. [#4013](https://github.com/ClickHouse/ClickHouse/pull/4013) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fixed glibc compatibility issues. [#4100](https://github.com/ClickHouse/ClickHouse/pull/4100) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Move Docker images to 18.10 and add compatibility file for glibc \>= 2.28 [#3965](https://github.com/ClickHouse/ClickHouse/pull/3965) ([alesapin](https://github.com/alesapin)) -- Add env variable if user don’t want to chown directories in server Docker image. [#3967](https://github.com/ClickHouse/ClickHouse/pull/3967) ([alesapin](https://github.com/alesapin)) +- Add env variable if user do not want to chown directories in server Docker image. [#3967](https://github.com/ClickHouse/ClickHouse/pull/3967) ([alesapin](https://github.com/alesapin)) - Enabled most of the warnings from `-Weverything` in clang. Enabled `-Wpedantic`. [#3986](https://github.com/ClickHouse/ClickHouse/pull/3986) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Added a few more warnings that are available only in clang 8. [#3993](https://github.com/ClickHouse/ClickHouse/pull/3993) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Link to `libLLVM` rather than to individual LLVM libs when using shared linking. [#3989](https://github.com/ClickHouse/ClickHouse/pull/3989) ([Orivej Desh](https://github.com/orivej)) diff --git a/docs/en/whats-new/changelog/2020.md b/docs/en/whats-new/changelog/2020.md index bf4e4fb0fcc..7fb1f5d9377 100644 --- a/docs/en/whats-new/changelog/2020.md +++ b/docs/en/whats-new/changelog/2020.md @@ -14,7 +14,7 @@ toc_title: '2020' * Restrict merges from wide to compact parts. In case of vertical merge it led to broken result part. [#18381](https://github.com/ClickHouse/ClickHouse/pull/18381) ([Anton Popov](https://github.com/CurtizJ)). * Fix filling table `system.settings_profile_elements`. This PR fixes [#18231](https://github.com/ClickHouse/ClickHouse/issues/18231). [#18379](https://github.com/ClickHouse/ClickHouse/pull/18379) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix possible crashes in aggregate functions with combinator `Distinct`, while using two-level aggregation. Fixes [#17682](https://github.com/ClickHouse/ClickHouse/issues/17682). [#18365](https://github.com/ClickHouse/ClickHouse/pull/18365) ([Anton Popov](https://github.com/CurtizJ)). -* Fix error when query `MODIFY COLUMN ... REMOVE TTL` doesn't actually remove column TTL. [#18130](https://github.com/ClickHouse/ClickHouse/pull/18130) ([alesapin](https://github.com/alesapin)). +* Fix error when query `MODIFY COLUMN ... REMOVE TTL` does not actually remove column TTL. [#18130](https://github.com/ClickHouse/ClickHouse/pull/18130) ([alesapin](https://github.com/alesapin)). #### Build/Testing/Packaging Improvement @@ -88,7 +88,7 @@ toc_title: '2020' * Fix `optimize_distributed_group_by_sharding_key` setting (that is disabled by default) for query with OFFSET only. [#16996](https://github.com/ClickHouse/ClickHouse/pull/16996) ([Azat Khuzhin](https://github.com/azat)). * Fix for Merge tables over Distributed tables with JOIN. [#16993](https://github.com/ClickHouse/ClickHouse/pull/16993) ([Azat Khuzhin](https://github.com/azat)). * Fixed wrong result in big integers (128, 256 bit) when casting from double. Big integers support is experimental. [#16986](https://github.com/ClickHouse/ClickHouse/pull/16986) ([Mike](https://github.com/myrrc)). -* Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter does not finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). * Blame info was not calculated correctly in `clickhouse-git-import`. [#16959](https://github.com/ClickHouse/ClickHouse/pull/16959) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix order by optimization with monotonous functions. Fixes [#16107](https://github.com/ClickHouse/ClickHouse/issues/16107). [#16956](https://github.com/ClickHouse/ClickHouse/pull/16956) ([Anton Popov](https://github.com/CurtizJ)). * Fix optimization of group by with enabled setting `optimize_aggregators_of_group_by_keys` and joins. Fixes [#12604](https://github.com/ClickHouse/ClickHouse/issues/12604). [#16951](https://github.com/ClickHouse/ClickHouse/pull/16951) ([Anton Popov](https://github.com/CurtizJ)). @@ -212,7 +212,7 @@ toc_title: '2020' * `SELECT count() FROM table` now can be executed if only one any column can be selected from the `table`. This PR fixes [#10639](https://github.com/ClickHouse/ClickHouse/issues/10639). [#18233](https://github.com/ClickHouse/ClickHouse/pull/18233) ([Vitaly Baranov](https://github.com/vitlibar)). * `SELECT JOIN` now requires the `SELECT` privilege on each of the joined tables. This PR fixes [#17654](https://github.com/ClickHouse/ClickHouse/issues/17654). [#18232](https://github.com/ClickHouse/ClickHouse/pull/18232) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix possible incomplete query result while reading from `MergeTree*` in case of read backoff (message ` MergeTreeReadPool: Will lower number of threads` in logs). Was introduced in [#16423](https://github.com/ClickHouse/ClickHouse/issues/16423). Fixes [#18137](https://github.com/ClickHouse/ClickHouse/issues/18137). [#18216](https://github.com/ClickHouse/ClickHouse/pull/18216) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix error when query `MODIFY COLUMN ... REMOVE TTL` doesn't actually remove column TTL. [#18130](https://github.com/ClickHouse/ClickHouse/pull/18130) ([alesapin](https://github.com/alesapin)). +* Fix error when query `MODIFY COLUMN ... REMOVE TTL` does not actually remove column TTL. [#18130](https://github.com/ClickHouse/ClickHouse/pull/18130) ([alesapin](https://github.com/alesapin)). * Fix indeterministic functions with predicate optimizer. This fixes [#17244](https://github.com/ClickHouse/ClickHouse/issues/17244). [#17273](https://github.com/ClickHouse/ClickHouse/pull/17273) ([Winter Zhang](https://github.com/zhang2014)). * Mutation might hang waiting for some non-existent part after `MOVE` or `REPLACE PARTITION` or, in rare cases, after `DETACH` or `DROP PARTITION`. It's fixed. [#15537](https://github.com/ClickHouse/ClickHouse/pull/15537) ([tavplubix](https://github.com/tavplubix)). @@ -253,7 +253,7 @@ toc_title: '2020' * Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). * Fixed wrong result in big integers (128, 256 bit) when casting from double. [#16986](https://github.com/ClickHouse/ClickHouse/pull/16986) ([Mike](https://github.com/myrrc)). * Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). -* Fixed possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Fixed possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter does not finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). * Blame info was not calculated correctly in `clickhouse-git-import`. [#16959](https://github.com/ClickHouse/ClickHouse/pull/16959) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed order by optimization with monotonous functions. Fixes [#16107](https://github.com/ClickHouse/ClickHouse/issues/16107). [#16956](https://github.com/ClickHouse/ClickHouse/pull/16956) ([Anton Popov](https://github.com/CurtizJ)). * Fixed optimization of group by with enabled setting `optimize_aggregators_of_group_by_keys` and joins. This fixes [#12604](https://github.com/ClickHouse/ClickHouse/issues/12604). [#16951](https://github.com/ClickHouse/ClickHouse/pull/16951) ([Anton Popov](https://github.com/CurtizJ)). @@ -424,7 +424,7 @@ toc_title: '2020' * Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). * Fixed wrong result in big integers (128, 256 bit) when casting from double. [#16986](https://github.com/ClickHouse/ClickHouse/pull/16986) ([Mike](https://github.com/myrrc)). * Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). -* Fixed possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Fixed possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter does not finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). * Blame info was not calculated correctly in `clickhouse-git-import`. [#16959](https://github.com/ClickHouse/ClickHouse/pull/16959) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed order by optimization with monotonous functions. This fixes [#16107](https://github.com/ClickHouse/ClickHouse/issues/16107). [#16956](https://github.com/ClickHouse/ClickHouse/pull/16956) ([Anton Popov](https://github.com/CurtizJ)). * Fixrf optimization of group by with enabled setting `optimize_aggregators_of_group_by_keys` and joins. This fixes [#12604](https://github.com/ClickHouse/ClickHouse/issues/12604). [#16951](https://github.com/ClickHouse/ClickHouse/pull/16951) ([Anton Popov](https://github.com/CurtizJ)). @@ -514,7 +514,7 @@ toc_title: '2020' * Fix rare segfaults when inserting into or selecting from MaterializedView and concurrently dropping target table (for Atomic database engine). [#15984](https://github.com/ClickHouse/ClickHouse/pull/15984) ([tavplubix](https://github.com/tavplubix)). * Fix ambiguity in parsing of settings profiles: `CREATE USER ... SETTINGS profile readonly` is now considered as using a profile named `readonly`, not a setting named `profile` with the readonly constraint. This fixes [#15628](https://github.com/ClickHouse/ClickHouse/issues/15628). [#15982](https://github.com/ClickHouse/ClickHouse/pull/15982) ([Vitaly Baranov](https://github.com/vitlibar)). * `MaterializeMySQL` (experimental feature): Fix crash on create database failure. [#15954](https://github.com/ClickHouse/ClickHouse/pull/15954) ([Winter Zhang](https://github.com/zhang2014)). -* Fixed `DROP TABLE IF EXISTS` failure with `Table ... doesn't exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) - Fixed `DROP/DETACH DATABASE` failure with `Table ... doesn't exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([tavplubix](https://github.com/tavplubix)). +* Fixed `DROP TABLE IF EXISTS` failure with `Table ... does not exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) - Fixed `DROP/DETACH DATABASE` failure with `Table ... does not exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([tavplubix](https://github.com/tavplubix)). * Fix incorrect empty result for query from `Distributed` table if query has `WHERE`, `PREWHERE` and `GLOBAL IN`. Fixes [#15792](https://github.com/ClickHouse/ClickHouse/issues/15792). [#15933](https://github.com/ClickHouse/ClickHouse/pull/15933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fixes [#12513](https://github.com/ClickHouse/ClickHouse/issues/12513): difference expressions with same alias when query is reanalyzed. [#15886](https://github.com/ClickHouse/ClickHouse/pull/15886) ([Winter Zhang](https://github.com/zhang2014)). * Fix possible very rare deadlocks in RBAC implementation. [#15875](https://github.com/ClickHouse/ClickHouse/pull/15875) ([Vitaly Baranov](https://github.com/vitlibar)). @@ -535,7 +535,7 @@ toc_title: '2020' * Fixed `Element ... is not a constant expression` error when using `JSON*` function result in `VALUES`, `LIMIT` or right side of `IN` operator. [#15589](https://github.com/ClickHouse/ClickHouse/pull/15589) ([tavplubix](https://github.com/tavplubix)). * Query will finish faster in case of exception. Cancel execution on remote replicas if exception happens. [#15578](https://github.com/ClickHouse/ClickHouse/pull/15578) ([Azat Khuzhin](https://github.com/azat)). * Prevent the possibility of error message `Could not calculate available disk space (statvfs), errno: 4, strerror: Interrupted system call`. This fixes [#15541](https://github.com/ClickHouse/ClickHouse/issues/15541). [#15557](https://github.com/ClickHouse/ClickHouse/pull/15557) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix `Database doesn't exist.` in queries with IN and Distributed table when there's no database on initiator. [#15538](https://github.com/ClickHouse/ClickHouse/pull/15538) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix `Database does not exist.` in queries with IN and Distributed table when there's no database on initiator. [#15538](https://github.com/ClickHouse/ClickHouse/pull/15538) ([Artem Zuikov](https://github.com/4ertus2)). * Mutation might hang waiting for some non-existent part after `MOVE` or `REPLACE PARTITION` or, in rare cases, after `DETACH` or `DROP PARTITION`. It's fixed. [#15537](https://github.com/ClickHouse/ClickHouse/pull/15537) ([tavplubix](https://github.com/tavplubix)). * Fix bug when `ILIKE` operator stops being case insensitive if `LIKE` with the same pattern was executed. [#15536](https://github.com/ClickHouse/ClickHouse/pull/15536) ([alesapin](https://github.com/alesapin)). * Fix `Missing columns` errors when selecting columns which absent in data, but depend on other columns which also absent in data. Fixes [#15530](https://github.com/ClickHouse/ClickHouse/issues/15530). [#15532](https://github.com/ClickHouse/ClickHouse/pull/15532) ([alesapin](https://github.com/alesapin)). @@ -552,7 +552,7 @@ toc_title: '2020' * Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix 'Unknown identifier' in GROUP BY when query has JOIN over Merge table. [#15242](https://github.com/ClickHouse/ClickHouse/pull/15242) ([Artem Zuikov](https://github.com/4ertus2)). * Fix instance crash when using `joinGet` with `LowCardinality` types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). -* Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Fix bug in table engine `Buffer` which does not allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). * Adjust Decimal field size in MySQL column definition packet. [#15152](https://github.com/ClickHouse/ClickHouse/pull/15152) ([maqroll](https://github.com/maqroll)). * Fixes `Data compressed with different methods` in `join_algorithm='auto'`. Keep LowCardinality as type for left table join key in `join_algorithm='partial_merge'`. [#15088](https://github.com/ClickHouse/ClickHouse/pull/15088) ([Artem Zuikov](https://github.com/4ertus2)). * Update `jemalloc` to fix `percpu_arena` with affinity mask. [#15035](https://github.com/ClickHouse/ClickHouse/pull/15035) ([Azat Khuzhin](https://github.com/azat)). [#14957](https://github.com/ClickHouse/ClickHouse/pull/14957) ([Azat Khuzhin](https://github.com/azat)). @@ -711,7 +711,7 @@ toc_title: '2020' * Fix bug when `ON CLUSTER` queries may hang forever for non-leader ReplicatedMergeTreeTables. [#17089](https://github.com/ClickHouse/ClickHouse/pull/17089) ([alesapin](https://github.com/alesapin)). * Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). -* Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter does not finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). * Install script should always create subdirs in config folders. This is only relevant for Docker build with custom config. [#16936](https://github.com/ClickHouse/ClickHouse/pull/16936) ([filimonov](https://github.com/filimonov)). * Fix possible error `Illegal type of argument` for queries with `ORDER BY`. Fixes [#16580](https://github.com/ClickHouse/ClickHouse/issues/16580). [#16928](https://github.com/ClickHouse/ClickHouse/pull/16928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Abort multipart upload if no data was written to WriteBufferFromS3. [#16840](https://github.com/ClickHouse/ClickHouse/pull/16840) ([Pavel Kovalenko](https://github.com/Jokser)). @@ -751,7 +751,7 @@ toc_title: '2020' * Fix rare segfaults when inserting into or selecting from MaterializedView and concurrently dropping target table (for Atomic database engine). [#15984](https://github.com/ClickHouse/ClickHouse/pull/15984) ([tavplubix](https://github.com/tavplubix)). * Fix ambiguity in parsing of settings profiles: `CREATE USER ... SETTINGS profile readonly` is now considered as using a profile named `readonly`, not a setting named `profile` with the readonly constraint. This fixes [#15628](https://github.com/ClickHouse/ClickHouse/issues/15628). [#15982](https://github.com/ClickHouse/ClickHouse/pull/15982) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix a crash when database creation fails. [#15954](https://github.com/ClickHouse/ClickHouse/pull/15954) ([Winter Zhang](https://github.com/zhang2014)). -* Fixed `DROP TABLE IF EXISTS` failure with `Table ... doesn't exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... doesn't exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([tavplubix](https://github.com/tavplubix)). +* Fixed `DROP TABLE IF EXISTS` failure with `Table ... does not exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... does not exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([tavplubix](https://github.com/tavplubix)). * Fix incorrect empty result for query from `Distributed` table if query has `WHERE`, `PREWHERE` and `GLOBAL IN`. Fixes [#15792](https://github.com/ClickHouse/ClickHouse/issues/15792). [#15933](https://github.com/ClickHouse/ClickHouse/pull/15933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix possible deadlocks in RBAC. [#15875](https://github.com/ClickHouse/ClickHouse/pull/15875) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix exception `Block structure mismatch` in `SELECT ... ORDER BY DESC` queries which were executed after `ALTER MODIFY COLUMN` query. Fixes [#15800](https://github.com/ClickHouse/ClickHouse/issues/15800). [#15852](https://github.com/ClickHouse/ClickHouse/pull/15852) ([alesapin](https://github.com/alesapin)). @@ -789,7 +789,7 @@ toc_title: '2020' * Fix rare race condition on server startup when system.logs are enabled. [#15300](https://github.com/ClickHouse/ClickHouse/pull/15300) ([alesapin](https://github.com/alesapin)). * Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix instance crash when using joinGet with LowCardinality types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). -* Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Fix bug in table engine `Buffer` which does not allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). * Adjust decimals field size in mysql column definition packet. [#15152](https://github.com/ClickHouse/ClickHouse/pull/15152) ([maqroll](https://github.com/maqroll)). * Fixed `Cannot rename ... errno: 22, strerror: Invalid argument` error on DDL query execution in Atomic database when running clickhouse-server in docker on Mac OS. [#15024](https://github.com/ClickHouse/ClickHouse/pull/15024) ([tavplubix](https://github.com/tavplubix)). * Fix to make predicate push down work when subquery contains finalizeAggregation function. Fixes [#14847](https://github.com/ClickHouse/ClickHouse/issues/14847). [#14937](https://github.com/ClickHouse/ClickHouse/pull/14937) ([filimonov](https://github.com/filimonov)). @@ -909,7 +909,7 @@ toc_title: '2020' * Fixed bug when `ON CLUSTER` queries may hang forever for non-leader ReplicatedMergeTreeTables. [#17089](https://github.com/ClickHouse/ClickHouse/pull/17089) ([alesapin](https://github.com/alesapin)). * Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). * Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). -* Fixed possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Fixed possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter does not finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). * Install script should always create subdirs in config folders. This is only relevant for Docker build with custom config. [#16936](https://github.com/ClickHouse/ClickHouse/pull/16936) ([filimonov](https://github.com/filimonov)). * Fixed possible error `Illegal type of argument` for queries with `ORDER BY`. Fixes [#16580](https://github.com/ClickHouse/ClickHouse/issues/16580). [#16928](https://github.com/ClickHouse/ClickHouse/pull/16928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Abort multipart upload if no data was written to WriteBufferFromS3. [#16840](https://github.com/ClickHouse/ClickHouse/pull/16840) ([Pavel Kovalenko](https://github.com/Jokser)). @@ -949,7 +949,7 @@ toc_title: '2020' * Fix rare segfaults when inserting into or selecting from MaterializedView and concurrently dropping target table (for Atomic database engine). [#15984](https://github.com/ClickHouse/ClickHouse/pull/15984) ([tavplubix](https://github.com/tavplubix)). * Fix ambiguity in parsing of settings profiles: `CREATE USER ... SETTINGS profile readonly` is now considered as using a profile named `readonly`, not a setting named `profile` with the readonly constraint. This fixes [#15628](https://github.com/ClickHouse/ClickHouse/issues/15628). [#15982](https://github.com/ClickHouse/ClickHouse/pull/15982) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix a crash when database creation fails. [#15954](https://github.com/ClickHouse/ClickHouse/pull/15954) ([Winter Zhang](https://github.com/zhang2014)). -* Fixed `DROP TABLE IF EXISTS` failure with `Table ... doesn't exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... doesn't exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([tavplubix](https://github.com/tavplubix)). +* Fixed `DROP TABLE IF EXISTS` failure with `Table ... does not exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... does not exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([tavplubix](https://github.com/tavplubix)). * Fix incorrect empty result for query from `Distributed` table if query has `WHERE`, `PREWHERE` and `GLOBAL IN`. Fixes [#15792](https://github.com/ClickHouse/ClickHouse/issues/15792). [#15933](https://github.com/ClickHouse/ClickHouse/pull/15933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix possible deadlocks in RBAC. [#15875](https://github.com/ClickHouse/ClickHouse/pull/15875) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix exception `Block structure mismatch` in `SELECT ... ORDER BY DESC` queries which were executed after `ALTER MODIFY COLUMN` query. Fixes [#15800](https://github.com/ClickHouse/ClickHouse/issues/15800). [#15852](https://github.com/ClickHouse/ClickHouse/pull/15852) ([alesapin](https://github.com/alesapin)). @@ -984,7 +984,7 @@ toc_title: '2020' * Fix rare race condition on server startup when system.logs are enabled. [#15300](https://github.com/ClickHouse/ClickHouse/pull/15300) ([alesapin](https://github.com/alesapin)). * Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix instance crash when using joinGet with LowCardinality types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). -* Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Fix bug in table engine `Buffer` which does not allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). * Adjust decimals field size in mysql column definition packet. [#15152](https://github.com/ClickHouse/ClickHouse/pull/15152) ([maqroll](https://github.com/maqroll)). * We already use padded comparison between String and FixedString (https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsComparison.h#L333). This PR applies the same logic to field comparison which corrects the usage of FixedString as primary keys. This fixes [#14908](https://github.com/ClickHouse/ClickHouse/issues/14908). [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). * If function `bar` was called with specifically crafted arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([alexey-milovidov](https://github.com/alexey-milovidov)). @@ -1028,7 +1028,7 @@ toc_title: '2020' #### Backward Incompatible Change -* Now `OPTIMIZE FINAL` query doesn't recalculate TTL for parts that were added before TTL was created. Use `ALTER TABLE ... MATERIALIZE TTL` once to calculate them, after that `OPTIMIZE FINAL` will evaluate TTL's properly. This behavior never worked for replicated tables. [#14220](https://github.com/ClickHouse/ClickHouse/pull/14220) ([alesapin](https://github.com/alesapin)). +* Now `OPTIMIZE FINAL` query does not recalculate TTL for parts that were added before TTL was created. Use `ALTER TABLE ... MATERIALIZE TTL` once to calculate them, after that `OPTIMIZE FINAL` will evaluate TTL's properly. This behavior never worked for replicated tables. [#14220](https://github.com/ClickHouse/ClickHouse/pull/14220) ([alesapin](https://github.com/alesapin)). * Extend `parallel_distributed_insert_select` setting, adding an option to run `INSERT` into local table. The setting changes type from `Bool` to `UInt64`, so the values `false` and `true` are no longer supported. If you have these values in server configuration, the server will not start. Please replace them with `0` and `1`, respectively. [#14060](https://github.com/ClickHouse/ClickHouse/pull/14060) ([Azat Khuzhin](https://github.com/azat)). * Remove support for the `ODBCDriver` input/output format. This was a deprecated format once used for communication with the ClickHouse ODBC driver, now long superseded by the `ODBCDriver2` format. Resolves [#13629](https://github.com/ClickHouse/ClickHouse/issues/13629). [#13847](https://github.com/ClickHouse/ClickHouse/pull/13847) ([hexiaoting](https://github.com/hexiaoting)). * When upgrading from versions older than 20.5, if rolling update is performed and cluster contains both versions 20.5 or greater and less than 20.5, if ClickHouse nodes with old versions are restarted and old version has been started up in presence of newer versions, it may lead to `Part ... intersects previous part` errors. To prevent this error, first install newer clickhouse-server packages on all cluster nodes and then do restarts (so, when clickhouse-server is restarted, it will start up with the new version). @@ -1257,7 +1257,7 @@ toc_title: '2020' * Fixed [#10572](https://github.com/ClickHouse/ClickHouse/issues/10572) fix bloom filter index with const expression. [#12659](https://github.com/ClickHouse/ClickHouse/pull/12659) ([Winter Zhang](https://github.com/zhang2014)). * Fix SIGSEGV in StorageKafka when broker is unavailable (and not only). [#12658](https://github.com/ClickHouse/ClickHouse/pull/12658) ([Azat Khuzhin](https://github.com/azat)). * Add support for function `if` with `Array(UUID)` arguments. This fixes [#11066](https://github.com/ClickHouse/ClickHouse/issues/11066). [#12648](https://github.com/ClickHouse/ClickHouse/pull/12648) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* CREATE USER IF NOT EXISTS now doesn't throw exception if the user exists. This fixes [#12507](https://github.com/ClickHouse/ClickHouse/issues/12507). [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)). +* CREATE USER IF NOT EXISTS now does not throw exception if the user exists. This fixes [#12507](https://github.com/ClickHouse/ClickHouse/issues/12507). [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)). * Exception `There is no supertype...` can be thrown during `ALTER ... UPDATE` in unexpected cases (e.g. when subtracting from UInt64 column). This fixes [#7306](https://github.com/ClickHouse/ClickHouse/issues/7306). This fixes [#4165](https://github.com/ClickHouse/ClickHouse/issues/4165). [#12633](https://github.com/ClickHouse/ClickHouse/pull/12633) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix possible `Pipeline stuck` error for queries with external sorting. Fixes [#12617](https://github.com/ClickHouse/ClickHouse/issues/12617). [#12618](https://github.com/ClickHouse/ClickHouse/pull/12618) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix error `Output of TreeExecutor is not sorted` for `OPTIMIZE DEDUPLICATE`. Fixes [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572). [#12613](https://github.com/ClickHouse/ClickHouse/pull/12613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). @@ -1398,7 +1398,7 @@ toc_title: '2020' * Fixed bloom filter index with const expression. This fixes [#10572](https://github.com/ClickHouse/ClickHouse/issues/10572). [#12659](https://github.com/ClickHouse/ClickHouse/pull/12659) ([Winter Zhang](https://github.com/zhang2014)). * Fixed `SIGSEGV` in `StorageKafka` when broker is unavailable (and not only). [#12658](https://github.com/ClickHouse/ClickHouse/pull/12658) ([Azat Khuzhin](https://github.com/azat)). * Added support for function `if` with `Array(UUID)` arguments. This fixes [#11066](https://github.com/ClickHouse/ClickHouse/issues/11066). [#12648](https://github.com/ClickHouse/ClickHouse/pull/12648) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* `CREATE USER IF NOT EXISTS` now doesn't throw exception if the user exists. This fixes [#12507](https://github.com/ClickHouse/ClickHouse/issues/12507). [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)). +* `CREATE USER IF NOT EXISTS` now does not throw exception if the user exists. This fixes [#12507](https://github.com/ClickHouse/ClickHouse/issues/12507). [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)). * Better exception message in disk access storage. [#12625](https://github.com/ClickHouse/ClickHouse/pull/12625) ([alesapin](https://github.com/alesapin)). * The function `groupArrayMoving*` was not working for distributed queries. It's result was calculated within incorrect data type (without promotion to the largest type). The function `groupArrayMovingAvg` was returning integer number that was inconsistent with the `avg` function. This fixes [#12568](https://github.com/ClickHouse/ClickHouse/issues/12568). [#12622](https://github.com/ClickHouse/ClickHouse/pull/12622) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed lack of aliases with function `any`. [#12593](https://github.com/ClickHouse/ClickHouse/pull/12593) ([Anton Popov](https://github.com/CurtizJ)). @@ -1436,7 +1436,7 @@ toc_title: '2020' * Cap max_memory_usage* limits to the process resident memory. [#12182](https://github.com/ClickHouse/ClickHouse/pull/12182) ([Azat Khuzhin](https://github.com/azat)). * Fix dictGet arguments check during `GROUP BY` injective functions elimination. [#12179](https://github.com/ClickHouse/ClickHouse/pull/12179) ([Azat Khuzhin](https://github.com/azat)). * Fixed the behaviour when `SummingMergeTree` engine sums up columns from partition key. Added an exception in case of explicit definition of columns to sum which intersects with partition key columns. This fixes [#7867](https://github.com/ClickHouse/ClickHouse/issues/7867). [#12173](https://github.com/ClickHouse/ClickHouse/pull/12173) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Don't split the dictionary source's table name into schema and table name itself if ODBC connection doesn't support schema. [#12165](https://github.com/ClickHouse/ClickHouse/pull/12165) ([Vitaly Baranov](https://github.com/vitlibar)). +* Don't split the dictionary source's table name into schema and table name itself if ODBC connection does not support schema. [#12165](https://github.com/ClickHouse/ClickHouse/pull/12165) ([Vitaly Baranov](https://github.com/vitlibar)). * Fixed wrong logic in `ALTER DELETE` that leads to deleting of records when condition evaluates to NULL. This fixes [#9088](https://github.com/ClickHouse/ClickHouse/issues/9088). This closes [#12106](https://github.com/ClickHouse/ClickHouse/issues/12106). [#12153](https://github.com/ClickHouse/ClickHouse/pull/12153) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed transform of query to send to external DBMS (e.g. MySQL, ODBC) in presense of aliases. This fixes [#12032](https://github.com/ClickHouse/ClickHouse/issues/12032). [#12151](https://github.com/ClickHouse/ClickHouse/pull/12151) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed bad code in redundant ORDER BY optimization. The bug was introduced in [#10067](https://github.com/ClickHouse/ClickHouse/issues/10067). [#12148](https://github.com/ClickHouse/ClickHouse/pull/12148) ([alexey-milovidov](https://github.com/alexey-milovidov)). @@ -1467,7 +1467,7 @@ toc_title: '2020' * Added `KILL QUERY [connection_id]` for the MySQL client/driver to cancel the long query, issue [#12038](https://github.com/ClickHouse/ClickHouse/issues/12038). [#12152](https://github.com/ClickHouse/ClickHouse/pull/12152) ([BohuTANG](https://github.com/BohuTANG)). * Added support for `%g` (two digit ISO year) and `%G` (four digit ISO year) substitutions in `formatDateTime` function. [#12136](https://github.com/ClickHouse/ClickHouse/pull/12136) ([vivarum](https://github.com/vivarum)). * Added 'type' column in system.disks. [#12115](https://github.com/ClickHouse/ClickHouse/pull/12115) ([ianton-ru](https://github.com/ianton-ru)). -* Improved `REVOKE` command: now it requires grant/admin option for only access which will be revoked. For example, to execute `REVOKE ALL ON *.* FROM user1` now it doesn't require to have full access rights granted with grant option. Added command `REVOKE ALL FROM user1` - it revokes all granted roles from `user1`. [#12083](https://github.com/ClickHouse/ClickHouse/pull/12083) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improved `REVOKE` command: now it requires grant/admin option for only access which will be revoked. For example, to execute `REVOKE ALL ON *.* FROM user1` now it does not require to have full access rights granted with grant option. Added command `REVOKE ALL FROM user1` - it revokes all granted roles from `user1`. [#12083](https://github.com/ClickHouse/ClickHouse/pull/12083) ([Vitaly Baranov](https://github.com/vitlibar)). * Added replica priority for load_balancing (for manual prioritization of the load balancing). [#11995](https://github.com/ClickHouse/ClickHouse/pull/11995) ([Azat Khuzhin](https://github.com/azat)). * Switched paths in S3 metadata to relative which allows to handle S3 blobs more easily. [#11892](https://github.com/ClickHouse/ClickHouse/pull/11892) ([Vladimir Chebotarev](https://github.com/excitoon)). @@ -1617,7 +1617,7 @@ toc_title: '2020' * Fix wrong result of comparison of FixedString with constant String. This fixes [#11393](https://github.com/ClickHouse/ClickHouse/issues/11393). This bug appeared in version 20.4. [#11828](https://github.com/ClickHouse/ClickHouse/pull/11828) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix wrong result for `if` with NULLs in condition. [#11807](https://github.com/ClickHouse/ClickHouse/pull/11807) ([Artem Zuikov](https://github.com/4ertus2)). * Fix using too many threads for queries. [#11788](https://github.com/ClickHouse/ClickHouse/pull/11788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fixed `Scalar doesn't exist` exception when using `WITH ...` in `SELECT ... FROM merge_tree_table ...` [#11621](https://github.com/ClickHouse/ClickHouse/issues/11621). [#11767](https://github.com/ClickHouse/ClickHouse/pull/11767) ([Amos Bird](https://github.com/amosbird)). +* Fixed `Scalar does not exist` exception when using `WITH ...` in `SELECT ... FROM merge_tree_table ...` [#11621](https://github.com/ClickHouse/ClickHouse/issues/11621). [#11767](https://github.com/ClickHouse/ClickHouse/pull/11767) ([Amos Bird](https://github.com/amosbird)). * Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)). * Now replicated fetches will be cancelled during metadata alter. [#11744](https://github.com/ClickHouse/ClickHouse/pull/11744) ([alesapin](https://github.com/alesapin)). * Parse metadata stored in zookeeper before checking for equality. [#11739](https://github.com/ClickHouse/ClickHouse/pull/11739) ([Azat Khuzhin](https://github.com/azat)). @@ -1637,7 +1637,7 @@ toc_title: '2020' * Fix wrong exit code of the clickhouse-client, when `exception.code() % 256 == 0`. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)). * Fix race conditions in CREATE/DROP of different replicas of ReplicatedMergeTree. Continue to work if the table was not removed completely from ZooKeeper or not created successfully. This fixes [#11432](https://github.com/ClickHouse/ClickHouse/issues/11432). [#11592](https://github.com/ClickHouse/ClickHouse/pull/11592) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Size of offsets does not match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fixed rare segfault in `SHOW CREATE TABLE` Fixes [#11490](https://github.com/ClickHouse/ClickHouse/issues/11490). [#11579](https://github.com/ClickHouse/ClickHouse/pull/11579) ([tavplubix](https://github.com/tavplubix)). * All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([tavplubix](https://github.com/tavplubix)). * Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)). @@ -1758,7 +1758,7 @@ toc_title: '2020' * Multiple names are now allowed in commands: CREATE USER, CREATE ROLE, ALTER USER, SHOW CREATE USER, SHOW GRANTS and so on. [#11670](https://github.com/ClickHouse/ClickHouse/pull/11670) ([Vitaly Baranov](https://github.com/vitlibar)). * Add support for distributed DDL (`UPDATE/DELETE/DROP PARTITION`) on cross replication clusters. [#11508](https://github.com/ClickHouse/ClickHouse/pull/11508) ([frank lee](https://github.com/etah000)). * Clear password from command line in `clickhouse-client` and `clickhouse-benchmark` if the user has specified it with explicit value. This prevents password exposure by `ps` and similar tools. [#11665](https://github.com/ClickHouse/ClickHouse/pull/11665) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Don't use debug info from ELF file if it doesn't correspond to the running binary. It is needed to avoid printing wrong function names and source locations in stack traces. This fixes [#7514](https://github.com/ClickHouse/ClickHouse/issues/7514). [#11657](https://github.com/ClickHouse/ClickHouse/pull/11657) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Don't use debug info from ELF file if it does not correspond to the running binary. It is needed to avoid printing wrong function names and source locations in stack traces. This fixes [#7514](https://github.com/ClickHouse/ClickHouse/issues/7514). [#11657](https://github.com/ClickHouse/ClickHouse/pull/11657) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Return NULL/zero when value is not parsed completely in parseDateTimeBestEffortOrNull/Zero functions. This fixes [#7876](https://github.com/ClickHouse/ClickHouse/issues/7876). [#11653](https://github.com/ClickHouse/ClickHouse/pull/11653) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Skip empty parameters in requested URL. They may appear when you write `http://localhost:8123/?&a=b` or `http://localhost:8123/?a=b&&c=d`. This closes [#10749](https://github.com/ClickHouse/ClickHouse/issues/10749). [#11651](https://github.com/ClickHouse/ClickHouse/pull/11651) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Allow using `groupArrayArray` and `groupUniqArrayArray` as `SimpleAggregateFunction`. [#11650](https://github.com/ClickHouse/ClickHouse/pull/11650) ([Volodymyr Kuznetsov](https://github.com/ksvladimir)). @@ -1933,7 +1933,7 @@ toc_title: '2020' * Fixed logical functions for UInt8 values when they are not equal to 0 or 1. [#12196](https://github.com/ClickHouse/ClickHouse/pull/12196) ([Alexander Kazakov](https://github.com/Akazz)). * Cap max_memory_usage* limits to the process resident memory. [#12182](https://github.com/ClickHouse/ClickHouse/pull/12182) ([Azat Khuzhin](https://github.com/azat)). * Fixed `dictGet` arguments check during GROUP BY injective functions elimination. [#12179](https://github.com/ClickHouse/ClickHouse/pull/12179) ([Azat Khuzhin](https://github.com/azat)). -* Don't split the dictionary source's table name into schema and table name itself if ODBC connection doesn't support schema. [#12165](https://github.com/ClickHouse/ClickHouse/pull/12165) ([Vitaly Baranov](https://github.com/vitlibar)). +* Don't split the dictionary source's table name into schema and table name itself if ODBC connection does not support schema. [#12165](https://github.com/ClickHouse/ClickHouse/pull/12165) ([Vitaly Baranov](https://github.com/vitlibar)). * Fixed wrong logic in `ALTER DELETE` that leads to deleting of records when condition evaluates to NULL. This fixes [#9088](https://github.com/ClickHouse/ClickHouse/issues/9088). This closes [#12106](https://github.com/ClickHouse/ClickHouse/issues/12106). [#12153](https://github.com/ClickHouse/ClickHouse/pull/12153) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed transform of query to send to external DBMS (e.g. MySQL, ODBC) in presense of aliases. This fixes [#12032](https://github.com/ClickHouse/ClickHouse/issues/12032). [#12151](https://github.com/ClickHouse/ClickHouse/pull/12151) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed potential overflow in integer division. This fixes [#12119](https://github.com/ClickHouse/ClickHouse/issues/12119). [#12140](https://github.com/ClickHouse/ClickHouse/pull/12140) ([alexey-milovidov](https://github.com/alexey-milovidov)). @@ -1997,7 +1997,7 @@ toc_title: '2020' * Fix error `Block structure mismatch` for queries with sampling reading from `Buffer` table. [#11602](https://github.com/ClickHouse/ClickHouse/pull/11602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix wrong exit code of the clickhouse-client, when exception.code() % 256 = 0. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)). * Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Size of offsets does not match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fixed rare segfault in `SHOW CREATE TABLE` Fixes [#11490](https://github.com/ClickHouse/ClickHouse/issues/11490). [#11579](https://github.com/ClickHouse/ClickHouse/pull/11579) ([tavplubix](https://github.com/tavplubix)). * All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([tavplubix](https://github.com/tavplubix)). * Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)). @@ -2213,7 +2213,7 @@ No changes compared to v20.4.3.16-stable. * Fix Distributed-over-Distributed with the only one shard in a nested table [#9997](https://github.com/ClickHouse/ClickHouse/pull/9997) ([Azat Khuzhin](https://github.com/azat)) * Fix possible rows loss for queries with `JOIN` and `UNION ALL`. Fixes [#9826](https://github.com/ClickHouse/ClickHouse/issues/9826), [#10113](https://github.com/ClickHouse/ClickHouse/issues/10113). ... [#10099](https://github.com/ClickHouse/ClickHouse/pull/10099) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fix bug in dictionary when local clickhouse server is used as source. It may caused memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)) -* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node doesn't exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)) +* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node does not exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)) * Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with mixed single and two-level aggregation from different shards. [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fix deadlock when database with materialized view failed attach at start [#10054](https://github.com/ClickHouse/ClickHouse/pull/10054) ([Azat Khuzhin](https://github.com/azat)) * Fix a segmentation fault that could occur in GROUP BY over string keys containing trailing zero bytes ([#8636](https://github.com/ClickHouse/ClickHouse/issues/8636), [#8925](https://github.com/ClickHouse/ClickHouse/issues/8925)). ... [#10025](https://github.com/ClickHouse/ClickHouse/pull/10025) ([Alexander Kuzmenkov](https://github.com/akuzm)) @@ -2228,7 +2228,7 @@ No changes compared to v20.4.3.16-stable. * Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)) * Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)) * Fix `DISTINCT` for Distributed when `optimize_skip_unused_shards` is set. [#9808](https://github.com/ClickHouse/ClickHouse/pull/9808) ([Azat Khuzhin](https://github.com/azat)) -* Fix "scalar doesn't exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). ... [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)) +* Fix "scalar does not exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). ... [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)) * Fix error with qualified names in `distributed_product_mode=\'local\'`. Fixes [#4756](https://github.com/ClickHouse/ClickHouse/issues/4756) [#9891](https://github.com/ClickHouse/ClickHouse/pull/9891) ([Artem Zuikov](https://github.com/4ertus2)) * For INSERT queries shards now do clamp the settings from the initiator to their constraints instead of throwing an exception. This fix allows to send INSERT queries to a shard with another constraints. This change improves fix [#9447](https://github.com/ClickHouse/ClickHouse/issues/9447). [#9852](https://github.com/ClickHouse/ClickHouse/pull/9852) ([Vitaly Baranov](https://github.com/vitlibar)) * Add some retries when commiting offsets to Kafka broker, since it can reject commit if during `offsets.commit.timeout.ms` there were no enough replicas available for the `__consumer_offsets` topic [#9884](https://github.com/ClickHouse/ClickHouse/pull/9884) ([filimonov](https://github.com/filimonov)) @@ -2248,11 +2248,11 @@ No changes compared to v20.4.3.16-stable. * Fix possible permanent "Cannot schedule a task" error. [#9154](https://github.com/ClickHouse/ClickHouse/pull/9154) ([Azat Khuzhin](https://github.com/azat)) * Fix bug in backquoting in external dictionaries DDL. Fixes [#9619](https://github.com/ClickHouse/ClickHouse/issues/9619). [#9734](https://github.com/ClickHouse/ClickHouse/pull/9734) ([alesapin](https://github.com/alesapin)) * Fixed data race in `text_log`. It does not correspond to any real bug. [#9726](https://github.com/ClickHouse/ClickHouse/pull/9726) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix bug in a replication that doesn't allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)) +* Fix bug in a replication that does not allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)) * Fixed incorrect internal function names for `sumKahan` and `sumWithOverflow`. It led to exception while using this functions in remote queries. [#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)) * Add setting `use_compact_format_in_distributed_parts_names` which allows to write files for `INSERT` queries into `Distributed` table with more compact format. This fixes [#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)) * Fix RIGHT and FULL JOIN with LowCardinality in JOIN keys. [#9610](https://github.com/ClickHouse/ClickHouse/pull/9610) ([Artem Zuikov](https://github.com/4ertus2)) -* Fix possible exceptions `Size of filter doesn't match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) +* Fix possible exceptions `Size of filter does not match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) * Allow `ALTER ON CLUSTER` of Distributed tables with internal replication. This fixes [#3268](https://github.com/ClickHouse/ClickHouse/issues/3268) [#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)) * Fix issue when timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [#5743](https://github.com/ClickHouse/ClickHouse/issues/5743) [#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -2533,7 +2533,7 @@ No changes compared to v20.4.3.16-stable. * Fix error `Block structure mismatch` for queries with sampling reading from `Buffer` table. [#11602](https://github.com/ClickHouse/ClickHouse/pull/11602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix wrong exit code of the clickhouse-client, when exception.code() % 256 = 0. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)). * Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Size of offsets does not match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([tavplubix](https://github.com/tavplubix)). * Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)). * Fix shard_num/replica_num for `` (breaks use_compact_format_in_distributed_parts_names). [#11528](https://github.com/ClickHouse/ClickHouse/pull/11528) ([Azat Khuzhin](https://github.com/azat)). @@ -2692,7 +2692,7 @@ No changes compared to v20.4.3.16-stable. * Fix incorrect `index_granularity_bytes` check while creating new replica. Fixes [#10098](https://github.com/ClickHouse/ClickHouse/issues/10098). [#10121](https://github.com/ClickHouse/ClickHouse/pull/10121) ([alesapin](https://github.com/alesapin)). * Fix SIGSEGV on INSERT into Distributed table when its structure differs from the underlying tables. [#10105](https://github.com/ClickHouse/ClickHouse/pull/10105) ([Azat Khuzhin](https://github.com/azat)). * Fix possible rows loss for queries with `JOIN` and `UNION ALL`. Fixes [#9826](https://github.com/ClickHouse/ClickHouse/issues/9826), [#10113](https://github.com/ClickHouse/ClickHouse/issues/10113). [#10099](https://github.com/ClickHouse/ClickHouse/pull/10099) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node doesn't exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)). +* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node does not exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)). * Add some arguments check and support identifier arguments for MySQL Database Engine. [#10077](https://github.com/ClickHouse/ClickHouse/pull/10077) ([Winter Zhang](https://github.com/zhang2014)). * Fix bug in clickhouse dictionary source from localhost clickhouse server. The bug may lead to memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)). * Fix bug in `CHECK TABLE` query when table contain skip indices. [#10068](https://github.com/ClickHouse/ClickHouse/pull/10068) ([alesapin](https://github.com/alesapin)). @@ -2704,7 +2704,7 @@ No changes compared to v20.4.3.16-stable. * Fix a bug with `ON CLUSTER` DDL queries freezing on server startup. [#9927](https://github.com/ClickHouse/ClickHouse/pull/9927) ([Gagan Arneja](https://github.com/garneja)). * Fix parsing multiple hosts set in the CREATE USER command, e.g. `CREATE USER user6 HOST NAME REGEXP 'lo.?*host', NAME REGEXP 'lo*host'`. [#9924](https://github.com/ClickHouse/ClickHouse/pull/9924) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)). -* Fix "scalar doesn't exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)). +* Fix "scalar does not exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)). * Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)). * Fix error with qualified names in `distributed_product_mode='local'`. Fixes [#4756](https://github.com/ClickHouse/ClickHouse/issues/4756). [#9891](https://github.com/ClickHouse/ClickHouse/pull/9891) ([Artem Zuikov](https://github.com/4ertus2)). * Fix calculating grants for introspection functions from the setting 'allow_introspection_functions'. [#9840](https://github.com/ClickHouse/ClickHouse/pull/9840) ([Vitaly Baranov](https://github.com/vitlibar)). @@ -2745,7 +2745,7 @@ No changes compared to v20.4.3.16-stable. #### Bug Fix * This release also contains all bug fixes from 20.1.7.38 -* Fix bug in a replication that doesn't allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. +* Fix bug in a replication that does not allow replication to work if the user has executed mutations on the previous version. This fixes [#9645](https://github.com/ClickHouse/ClickHouse/issues/9645). [#9652](https://github.com/ClickHouse/ClickHouse/pull/9652) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. * Add setting `use_compact_format_in_distributed_parts_names` which allows to write files for `INSERT` queries into `Distributed` table with more compact format. This fixes [#9647](https://github.com/ClickHouse/ClickHouse/issues/9647). [#9653](https://github.com/ClickHouse/ClickHouse/pull/9653) ([alesapin](https://github.com/alesapin)). It makes version 20.3 backward compatible again. ### ClickHouse release v20.3.2.1, 2020-03-12 @@ -2760,7 +2760,7 @@ No changes compared to v20.4.3.16-stable. * Remove `findClusterIndex`, `findClusterValue` functions. This fixes [#8641](https://github.com/ClickHouse/ClickHouse/issues/8641). If you were using these functions, send an email to `clickhouse-feedback@yandex-team.com` [#9543](https://github.com/ClickHouse/ClickHouse/pull/9543) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Now it's not allowed to create columns or add columns with `SELECT` subquery as default expression. [#9481](https://github.com/ClickHouse/ClickHouse/pull/9481) ([alesapin](https://github.com/alesapin)) * Require aliases for subqueries in JOIN. [#9274](https://github.com/ClickHouse/ClickHouse/pull/9274) ([Artem Zuikov](https://github.com/4ertus2)) -* Improved `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression doesn't change type of column and `MODIFY` type doesn't loose default expression value. Fixes [#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([alesapin](https://github.com/alesapin)) +* Improved `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression does not change type of column and `MODIFY` type does not loose default expression value. Fixes [#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) ([alesapin](https://github.com/alesapin)) * Require server to be restarted to apply the changes in logging configuration. This is a temporary workaround to avoid the bug where the server logs to a deleted log file (see [#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) * The setting `experimental_use_processors` is enabled by default. This setting enables usage of the new query pipeline. This is internal refactoring and we expect no visible changes. If you will see any issues, set it to back zero. [#8768](https://github.com/ClickHouse/ClickHouse/pull/8768) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -2811,7 +2811,7 @@ No changes compared to v20.4.3.16-stable. * Found keys were counted as missed in metrics of cache dictionaries. [#9411](https://github.com/ClickHouse/ClickHouse/pull/9411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) * Fix replication protocol incompatibility introduced in [#8598](https://github.com/ClickHouse/ClickHouse/issues/8598). [#9412](https://github.com/ClickHouse/ClickHouse/pull/9412) ([alesapin](https://github.com/alesapin)) * Fixed race condition on `queue_task_handle` at the startup of `ReplicatedMergeTree` tables. [#9552](https://github.com/ClickHouse/ClickHouse/pull/9552) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* The token `NOT` didn't work in `SHOW TABLES NOT LIKE` query [#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* The token `NOT` did not work in `SHOW TABLES NOT LIKE` query [#8727](https://github.com/ClickHouse/ClickHouse/issues/8727) [#8940](https://github.com/ClickHouse/ClickHouse/pull/8940) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Added range check to function `h3EdgeLengthM`. Without this check, buffer overflow is possible. [#8945](https://github.com/ClickHouse/ClickHouse/pull/8945) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed up a bug in batched calculations of ternary logical OPs on multiple arguments (more than 10). [#8718](https://github.com/ClickHouse/ClickHouse/pull/8718) ([Alexander Kazakov](https://github.com/Akazz)) * Fix error of PREWHERE optimization, which could lead to segfaults or `Inconsistent number of columns got from MergeTreeRangeReader` exception. [#9024](https://github.com/ClickHouse/ClickHouse/pull/9024) ([Anton Popov](https://github.com/CurtizJ)) @@ -2872,7 +2872,7 @@ No changes compared to v20.4.3.16-stable. * Fix bug in which a misleading error message was shown when running `SHOW CREATE TABLE a_table_that_does_not_exist`. [#8899](https://github.com/ClickHouse/ClickHouse/pull/8899) ([achulkov2](https://github.com/achulkov2)) * Fixed `Parameters are out of bound` exception in some rare cases when we have a constant in the `SELECT` clause when we have an `ORDER BY` and a `LIMIT` clause. [#8892](https://github.com/ClickHouse/ClickHouse/pull/8892) ([Guillaume Tassery](https://github.com/YiuRULE)) * Fix mutations finalization, when already done mutation can have status `is_done=0`. [#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) ([alesapin](https://github.com/alesapin)) -* Prevent from executing `ALTER ADD INDEX` for MergeTree tables with old syntax, because it doesn't work. [#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([Mikhail Korotov](https://github.com/millb)) +* Prevent from executing `ALTER ADD INDEX` for MergeTree tables with old syntax, because it does not work. [#8822](https://github.com/ClickHouse/ClickHouse/pull/8822) ([Mikhail Korotov](https://github.com/millb)) * During server startup do not access table, which `LIVE VIEW` depends on, so server will be able to start. Also remove `LIVE VIEW` dependencies when detaching `LIVE VIEW`. `LIVE VIEW` is an experimental feature. [#8824](https://github.com/ClickHouse/ClickHouse/pull/8824) ([tavplubix](https://github.com/tavplubix)) * Fix possible segfault in `MergeTreeRangeReader`, while executing `PREWHERE`. [#9106](https://github.com/ClickHouse/ClickHouse/pull/9106) ([Anton Popov](https://github.com/CurtizJ)) * Fix possible mismatched checksums with column TTLs. [#9451](https://github.com/ClickHouse/ClickHouse/pull/9451) ([Anton Popov](https://github.com/CurtizJ)) @@ -3020,7 +3020,7 @@ No changes compared to v20.4.3.16-stable. #### Bug Fix -* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Size of offsets does not match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). ### ClickHouse release v20.1.13.105-stable 2020-06-10 @@ -3116,7 +3116,7 @@ No changes compared to v20.4.3.16-stable. * Fix `'Not found column in block'` error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839). [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)). * Fix a bug with `ON CLUSTER` DDL queries freezing on server startup. [#9927](https://github.com/ClickHouse/ClickHouse/pull/9927) ([Gagan Arneja](https://github.com/garneja)). * Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)). -* Fix `'scalar doesn't exist'` error in ALTER queries ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)). +* Fix `'scalar does not exist'` error in ALTER queries ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)). * Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)). * Fixed `DeleteOnDestroy` logic in `ATTACH PART` which could lead to automatic removal of attached part and added few tests. [#9410](https://github.com/ClickHouse/ClickHouse/pull/9410) ([Vladimir Chebotarev](https://github.com/excitoon)). @@ -3155,7 +3155,7 @@ No changes compared to v20.4.3.16-stable. #### Bug Fix * Fixed incorrect internal function names for `sumKahan` and `sumWithOverflow`. I lead to exception while using this functions in remote queries. [#9636](https://github.com/ClickHouse/ClickHouse/pull/9636) ([Azat Khuzhin](https://github.com/azat)). This issue was in all ClickHouse releases. * Allow `ALTER ON CLUSTER` of `Distributed` tables with internal replication. This fixes [#3268](https://github.com/ClickHouse/ClickHouse/issues/3268). [#9617](https://github.com/ClickHouse/ClickHouse/pull/9617) ([shinoi2](https://github.com/shinoi2)). This issue was in all ClickHouse releases. -* Fix possible exceptions `Size of filter doesn't match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. Fixes [#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) +* Fix possible exceptions `Size of filter does not match size of column` and `Invalid number of rows in Chunk` in `MergeTreeRangeReader`. They could appear while executing `PREWHERE` in some cases. Fixes [#9132](https://github.com/ClickHouse/ClickHouse/issues/9132). [#9612](https://github.com/ClickHouse/ClickHouse/pull/9612) ([Anton Popov](https://github.com/CurtizJ)) * Fixed the issue: timezone was not preserved if you write a simple arithmetic expression like `time + 1` (in contrast to an expression like `time + INTERVAL 1 SECOND`). This fixes [#5743](https://github.com/ClickHouse/ClickHouse/issues/5743). [#9323](https://github.com/ClickHouse/ClickHouse/pull/9323) ([alexey-milovidov](https://github.com/alexey-milovidov)). This issue was in all ClickHouse releases. * Now it's not possible to create or add columns with simple cyclic aliases like `a DEFAULT b, b DEFAULT a`. [#9603](https://github.com/ClickHouse/ClickHouse/pull/9603) ([alesapin](https://github.com/alesapin)) * Fixed the issue when padding at the end of base64 encoded value can be malformed. Update base64 library. This fixes [#9491](https://github.com/ClickHouse/ClickHouse/issues/9491), closes [#9492](https://github.com/ClickHouse/ClickHouse/issues/9492) [#9500](https://github.com/ClickHouse/ClickHouse/pull/9500) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -3199,7 +3199,7 @@ No changes compared to v20.4.3.16-stable. [#9231](https://github.com/ClickHouse/ClickHouse/pull/9231) [(abyss7)](https://github.com/abyss7) * Allow comma join with `IN()` inside. Fixes [#7314](https://github.com/ClickHouse/ClickHouse/issues/7314). [#9251](https://github.com/ClickHouse/ClickHouse/pull/9251) [(4ertus2)](https://github.com/4ertus2) -* Improve `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression doesn't change type of column and `MODIFY` type doesn't loose default expression value. Fixes [#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). +* Improve `ALTER MODIFY/ADD` queries logic. Now you cannot `ADD` column without type, `MODIFY` default expression does not change type of column and `MODIFY` type does not loose default expression value. Fixes [#8669](https://github.com/ClickHouse/ClickHouse/issues/8669). [#9227](https://github.com/ClickHouse/ClickHouse/pull/9227) [(alesapin)](https://github.com/alesapin) * Fix mutations finalization, when already done mutation can have status is_done=0. [#9217](https://github.com/ClickHouse/ClickHouse/pull/9217) [(alesapin)](https://github.com/alesapin) @@ -3283,7 +3283,7 @@ No changes compared to v20.4.3.16-stable. * Fix error "Mismatch column sizes" when inserting default `Tuple` from `JSONEachRow`. This fixes [#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) * Now an exception will be thrown in case of using `WITH TIES` alongside `LIMIT BY`. Also add ability to use `TOP` with `LIMIT BY`. This fixes [#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) * Fix unintendent dependency from fresh glibc version in `clickhouse-odbc-bridge` binary. [#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Bird](https://github.com/amosbird)) -* Fix bug in check function of `*MergeTree` engines family. Now it doesn't fail in case when we have equal amount of rows in last granule and last mark (non-final). [#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) +* Fix bug in check function of `*MergeTree` engines family. Now it does not fail in case when we have equal amount of rows in last granule and last mark (non-final). [#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) * Fix insert into `Enum*` columns after `ALTER` query, when underlying numeric type is equal to table specified type. This fixes [#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) * Allowed non-constant negative "size" argument for function `substring`. It was not allowed by mistake. This fixes [#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix parsing bug when wrong number of arguments passed to `(O|J)DBC` table engine. [#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) @@ -3296,7 +3296,7 @@ No changes compared to v20.4.3.16-stable. * Fix function `IN` inside `WHERE` statement when row-level table filter is present. Fixes [#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) * Now an exception is thrown if the integral value is not parsed completely for settings values. [#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) * Fix exception when aggregate function is used in query to distributed table with more than two local shards. [#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -* Now bloom filter can handle zero length arrays and doesn't perform redundant calculations. [#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) +* Now bloom filter can handle zero length arrays and does not perform redundant calculations. [#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) * Fixed checking if a client host is allowed by matching the client host to `host_regexp` specified in `users.xml`. [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) * Relax ambiguous column check that leads to false positives in multiple `JOIN ON` section. [#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) * Fixed possible server crash (`std::terminate`) when the server cannot send or write data in `JSON` or `XML` format with values of `String` data type (that require `UTF-8` validation) or when compressing result data with Brotli algorithm or in some other rare cases. This fixes [#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -3323,19 +3323,19 @@ No changes compared to v20.4.3.16-stable. * Fix error in background merge of columns with `SimpleAggregateFunction(LowCardinality)` type. [#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fixed type check in function `toDateTime64`. [#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) * Now server do not crash on `LEFT` or `FULL JOIN` with and Join engine and unsupported `join_use_nulls` settings. [#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) -* Now `DROP DICTIONARY IF EXISTS db.dict` query doesn't throw exception if `db` doesn't exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) +* Now `DROP DICTIONARY IF EXISTS db.dict` query does not throw exception if `db` does not exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) * Fix possible crashes in table functions (`file`, `mysql`, `remote`) caused by usage of reference to removed `IStorage` object. Fix incorrect parsing of columns specified at insertion into table function. [#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) * Ensure network be up before starting `clickhouse-server`. This fixes [#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) -* Fix timeouts handling for secure connections, so queries doesn't hang indefenitely. This fixes [#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix timeouts handling for secure connections, so queries does not hang indefenitely. This fixes [#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix `clickhouse-copier`'s redundant contention between concurrent workers. [#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -* Now mutations doesn't skip attached parts, even if their mutation version were larger than current mutation version. [#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) +* Now mutations does not skip attached parts, even if their mutation version were larger than current mutation version. [#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) * Ignore redundant copies of `*MergeTree` data parts after move to another disk and server restart. [#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) * Fix crash in `FULL JOIN` with `LowCardinality` in `JOIN` key. [#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) * Forbidden to use column name more than once in insert query like `INSERT INTO tbl (x, y, x)`. This fixes [#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) * Added fallback for detection the number of physical CPU cores for unknown CPUs (using the number of logical CPU cores). This fixes [#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix `There's no column` error for materialized and alias columns. [#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) * Fixed sever crash when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier. Just like `EXISTS t`. This fixes [#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). This bug was introduced in version 19.17. [#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix rare bug with error `"Sizes of columns doesn't match"` that might appear when using `SimpleAggregateFunction` column. [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) +* Fix rare bug with error `"Sizes of columns does not match"` that might appear when using `SimpleAggregateFunction` column. [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) * Fix bug where user with empty `allow_databases` got access to all databases (and same for `allow_dictionaries`). [#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) * Fix client crash when server already disconnected from client. [#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) * Fix `ORDER BY` behaviour in case of sorting by primary key prefix and non primary key suffix. [#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) @@ -3367,7 +3367,7 @@ No changes compared to v20.4.3.16-stable. * Fix `CHECK TABLE` query for `*MergeTree` tables without key. Fixes [#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) * Fixed conversion of `Float64` to MySQL type. [#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) * Now if table was not completely dropped because of server crash, server will try to restore and load it. [#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -* Fixed crash in table function `file` while inserting into file that doesn't exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) +* Fixed crash in table function `file` while inserting into file that does not exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) * Fix rare deadlock which can happen when `trace_log` is in enabled. [#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) * Add ability to work with different types besides `Date` in `RangeHashed` external dictionary created from DDL query. Fixes [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) * Fixes crash when `now64()` is called with result of another function. [#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) @@ -3465,7 +3465,7 @@ No changes compared to v20.4.3.16-stable. * Add performance tests for short string optimized hash tables. [#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Bird](https://github.com/amosbird)) * Now ClickHouse will build on `AArch64` even if `MADV_FREE` is not available. This fixes [#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Bird](https://github.com/amosbird)) * Update `zlib-ng` to fix memory sanitizer problems. [#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Enable internal MySQL library on non-Linux system, because usage of OS packages is very fragile and usually doesn't work at all. This fixes [#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enable internal MySQL library on non-Linux system, because usage of OS packages is very fragile and usually does not work at all. This fixes [#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed build on some systems after enabling `libc++`. This supersedes [#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Make `Field` methods more type-safe to find more errors. [#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) * Added missing files to the `libc-headers` submodule. [#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -3494,7 +3494,7 @@ No changes compared to v20.4.3.16-stable. * Fix mode of default password file for `.deb` linux distros. [#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) * Improved expression for getting `clickhouse-server` PID in `clickhouse-test`. [#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) * Updated contrib/googletest to v1.10.0. [#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) -* Fixed ThreadSaninitizer report in `base64` library. Also updated this library to the latest version, but it doesn't matter. This fixes [#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed ThreadSaninitizer report in `base64` library. Also updated this library to the latest version, but it does not matter. This fixes [#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix `00600_replace_running_query` for processors. [#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Remove support for `tcmalloc` to make `CMakeLists.txt` simpler. [#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Release gcc builds now use `libc++` instead of `libstdc++`. Recently `libc++` was used only with clang. This will improve consistency of build configurations and portability. [#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexey-milovidov](https://github.com/alexey-milovidov)) diff --git a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md index 9a09618e508..4f0206158f1 100644 --- a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -34,6 +34,8 @@ ORDER BY (CounterID, StartDate, intHash32(UserID)); В этом примере задано партиционирование по типам событий, произошедших в течение текущей недели. +По умолчанию, ключ партиционирования с плавающей запятой не поддерживается. Чтобы использовать его, включите настройку [allow_floating_point_partition_key](../../../operations/settings/merge-tree-settings.md#allow_floating_point_partition_key). + Каждая партиция состоит из отдельных фрагментов или так называемых *кусков данных*. Каждый кусок отсортирован по первичному ключу. При вставке данных в таблицу каждая отдельная запись сохраняется в виде отдельного куска. Через некоторое время после вставки (обычно до 10 минут), ClickHouse выполняет в фоновом режиме слияние данных — в результате куски для одной и той же партиции будут объединены в более крупный кусок. !!! info "Info" diff --git a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md index ec0b339e8c9..ebd7875179d 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md @@ -33,11 +33,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **Параметры ReplacingMergeTree** -- `ver` — столбец с версией, тип `UInt*`, `Date` или `DateTime`. Необязательный параметр. +- `ver` — столбец с номером версии. Тип `UInt*`, `Date`, `DateTime` или `DateTime64`. Необязательный параметр. При слиянии `ReplacingMergeTree` оставляет только строку для каждого уникального ключа сортировки: - - Последнюю в выборке, если `ver` не задан. Под выборкой здесь понимается набор строк в наборе партов, участвующих в слиянии. Последний по времени создания парт (последний инсерт) будет последним в выборке. Таким образом, после дедупликации для каждого значения ключа сортировки останется самая последняя строка из самого последнего инсерта. + - Последнюю в выборке, если `ver` не задан. Под выборкой здесь понимается набор строк в наборе кусков данных, участвующих в слиянии. Последний по времени создания кусок (последняя вставка) будет последним в выборке. Таким образом, после дедупликации для каждого значения ключа сортировки останется самая последняя строка из самой последней вставки. - С максимальной версией, если `ver` задан. **Секции запроса** @@ -62,7 +62,6 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Все параметры, кроме `ver` имеют то же значение, что в и `MergeTree`. -- `ver` — столбец с версией. Необязательный параметр. Описание смотрите выше по тексту. +- `ver` — столбец с номером версии. Необязательный параметр. Описание смотрите выше по тексту. - diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index e58948b0148..2af99bb8026 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -246,4 +246,15 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: -1 (неограниченно). +## allow_floating_point_partition_key {#allow_floating_point_partition_key} + +Позволяет использовать число с плавающей запятой в качестве ключа партиционирования. + +Возможные значения: + +- 0 — Ключ партиционирования с плавающей запятой не разрешен. +- 1 — Ключ партиционирования с плавающей запятой разрешен. + +Значение по умолчанию: `0`. + [Original article](https://clickhouse.tech/docs/ru/operations/settings/merge_tree_settings/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index f84dd1e3ddb..ada8ee91293 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -347,7 +347,31 @@ INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2; ## input_format_null_as_default {#settings-input-format-null-as-default} -Включает или отключает использование значений по умолчанию в случаях, когда во входных данных содержится `NULL`, но тип соответствующего столбца не `Nullable(T)` (для текстовых форматов). +Включает или отключает инициализацию [значениями по умолчанию](../../sql-reference/statements/create/table.md#create-default-values) ячеек с [NULL](../../sql-reference/syntax.md#null-literal), если тип данных столбца не позволяет [хранить NULL](../../sql-reference/data-types/nullable.md#data_type-nullable). +Если столбец не позволяет хранить `NULL` и эта настройка отключена, то вставка `NULL` приведет к возникновению исключения. Если столбец позволяет хранить `NULL`, то значения `NULL` вставляются независимо от этой настройки. + +Эта настройка используется для запросов [INSERT ... VALUES](../../sql-reference/statements/insert-into.md) для текстовых входных форматов. + +Возможные значения: + +- 0 — вставка `NULL` в столбец, не позволяющий хранить `NULL`, приведет к возникновению исключения. +- 1 — ячейки с `NULL` инициализируются значением столбца по умолчанию. + +Значение по умолчанию: `1`. + +## insert_null_as_default {#insert_null_as_default} + +Включает или отключает вставку [значений по умолчанию](../../sql-reference/statements/create/table.md#create-default-values) вместо [NULL](../../sql-reference/syntax.md#null-literal) в столбцы, которые не позволяют [хранить NULL](../../sql-reference/data-types/nullable.md#data_type-nullable). +Если столбец не позволяет хранить `NULL` и эта настройка отключена, то вставка `NULL` приведет к возникновению исключения. Если столбец позволяет хранить `NULL`, то значения `NULL` вставляются независимо от этой настройки. + +Эта настройка используется для запросов [INSERT ... SELECT](../../sql-reference/statements/insert-into.md#insert_query_insert-select). При этом подзапросы `SELECT` могут объединяться с помощью `UNION ALL`. + +Возможные значения: + +- 0 — вставка `NULL` в столбец, не позволяющий хранить `NULL`, приведет к возникновению исключения. +- 1 — вместо `NULL` вставляется значение столбца по умолчанию. + +Значение по умолчанию: `1`. ## input_format_skip_unknown_fields {#settings-input-format-skip-unknown-fields} @@ -2900,4 +2924,36 @@ SELECT * FROM test LIMIT 10 OFFSET 100; Значение по умолчанию: `1800`. +## optimize_fuse_sum_count_avg {#optimize_fuse_sum_count_avg} + +Позволяет объединить агрегатные функции с одинаковым аргументом. Запрос, содержащий по крайней мере две агрегатные функции: [sum](../../sql-reference/aggregate-functions/reference/sum.md#agg_function-sum), [count](../../sql-reference/aggregate-functions/reference/count.md#agg_function-count) или [avg](../../sql-reference/aggregate-functions/reference/avg.md#agg_function-avg) с одинаковым аргументом, перезаписывается как [sumCount](../../sql-reference/aggregate-functions/reference/sumcount.md#agg_function-sumCount). + +Возможные значения: + +- 0 — функции с одинаковым аргументом не объединяются. +- 1 — функции с одинаковым аргументом объединяются. + +Значение по умолчанию: `0`. + +**Пример** + +Запрос: + +``` sql +CREATE TABLE fuse_tbl(a Int8, b Int8) Engine = Log; +SET optimize_fuse_sum_count_avg = 1; +EXPLAIN SYNTAX SELECT sum(a), sum(b), count(b), avg(b) from fuse_tbl FORMAT TSV; +``` + +Результат: + +``` text +SELECT + sum(a), + sumCount(b).1, + sumCount(b).2, + (sumCount(b).1) / (sumCount(b).2) +FROM fuse_tbl +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) diff --git a/docs/ru/operations/system-tables/stack_trace.md b/docs/ru/operations/system-tables/stack_trace.md index 58d0a1c4b6a..338c14534cf 100644 --- a/docs/ru/operations/system-tables/stack_trace.md +++ b/docs/ru/operations/system-tables/stack_trace.md @@ -6,9 +6,10 @@ Столбцы: -- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Идентификатор потока. -- `query_id` ([String](../../sql-reference/data-types/string.md)) — Идентификатор запроса. Может быть использован для получения подробной информации о выполненном запросе из системной таблицы [query_log](#system_tables-query_log). -- `trace` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — [Трассировка стека](https://en.wikipedia.org/wiki/Stack_trace). Представляет собой список физических адресов, по которым расположены вызываемые методы. +- `thread_name` ([String](../../sql-reference/data-types/string.md)) — имя потока. +- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — идентификатор потока. +- `query_id` ([String](../../sql-reference/data-types/string.md)) — идентификатор запроса. Может быть использован для получения подробной информации о выполненном запросе из системной таблицы [query_log](#system_tables-query_log). +- `trace` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — [трассировка стека](https://en.wikipedia.org/wiki/Stack_trace). Представляет собой список физических адресов, по которым расположены вызываемые методы. **Пример** @@ -21,12 +22,14 @@ SET allow_introspection_functions = 1; Получение символов из объектных файлов ClickHouse: ``` sql -WITH arrayMap(x -> demangle(addressToSymbol(x)), trace) AS all SELECT thread_id, query_id, arrayStringConcat(all, '\n') AS res FROM system.stack_trace LIMIT 1 \G +WITH arrayMap(x -> demangle(addressToSymbol(x)), trace) AS all SELECT thread_name, thread_id, query_id, arrayStringConcat(all, '\n') AS res FROM system.stack_trace LIMIT 1 \G; ``` ``` text Row 1: ────── +thread_name: clickhouse-serv + thread_id: 686 query_id: 1a11f70b-626d-47c1-b948-f9c7b206395d res: sigqueue @@ -51,12 +54,14 @@ __clone Получение имен файлов и номеров строк в исходном коде ClickHouse: ``` sql -WITH arrayMap(x -> addressToLine(x), trace) AS all, arrayFilter(x -> x LIKE '%/dbms/%', all) AS dbms SELECT thread_id, query_id, arrayStringConcat(notEmpty(dbms) ? dbms : all, '\n') AS res FROM system.stack_trace LIMIT 1 \G +WITH arrayMap(x -> addressToLine(x), trace) AS all, arrayFilter(x -> x LIKE '%/dbms/%', all) AS dbms SELECT thread_name, thread_id, query_id, arrayStringConcat(notEmpty(dbms) ? dbms : all, '\n') AS res FROM system.stack_trace LIMIT 1 \G; ``` ``` text Row 1: ────── +thread_name: clickhouse-serv + thread_id: 686 query_id: cad353e7-1c29-4b2e-949f-93e597ab7a54 res: /lib/x86_64-linux-gnu/libc-2.27.so @@ -78,10 +83,9 @@ res: /lib/x86_64-linux-gnu/libc-2.27.so /lib/x86_64-linux-gnu/libc-2.27.so ``` -**См. также** - -- [Функции интроспекции](../../sql-reference/functions/introspection.md) — Что такое функции интроспекции и как их использовать. -- [system.trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) — Содержит трассировки стека, собранные профилировщиком выборочных запросов. -- [arrayMap](../../sql-reference/functions/array-functions.md#array-map) — Описание и пример использования функции `arrayMap`. -- [arrayFilter](../../sql-reference/functions/array-functions.md#array-filter) — Описание и пример использования функции `arrayFilter`. +**Смотрите также** +- [Функции интроспекции](../../sql-reference/functions/introspection.md) — описание функций интроспекции и примеры использования. +- [system.trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) — системная таблица, содержащая трассировки стека, собранные профилировщиком выборочных запросов. +- [arrayMap](../../sql-reference/functions/array-functions.md#array-map) — описание и пример использования функции `arrayMap`. +- [arrayFilter](../../sql-reference/functions/array-functions.md#array-filter) — описание и пример использования функции `arrayFilter`. diff --git a/docs/ru/operations/system-tables/tables.md b/docs/ru/operations/system-tables/tables.md index 11bb6a9eda2..3dec1e7d940 100644 --- a/docs/ru/operations/system-tables/tables.md +++ b/docs/ru/operations/system-tables/tables.md @@ -39,6 +39,8 @@ - `lifetime_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - общее количество байт, добавленных оператором `INSERT` с момента запуска сервера (только для таблиц `Buffer`). +- `comment` ([String](../../sql-reference/data-types/string.md)) — комментарий к таблице. + Таблица `system.tables` используется при выполнении запроса `SHOW TABLES`. **Пример** @@ -50,45 +52,51 @@ SELECT * FROM system.tables LIMIT 2 FORMAT Vertical; ```text Row 1: ────── -database: system -name: aggregate_function_combinators -uuid: 00000000-0000-0000-0000-000000000000 -engine: SystemAggregateFunctionCombinators +database: base +name: t1 +uuid: 81b1c20a-b7c6-4116-a2ce-7583fb6b6736 +engine: MergeTree is_temporary: 0 -data_paths: [] -metadata_path: /var/lib/clickhouse/metadata/system/aggregate_function_combinators.sql -metadata_modification_time: 1970-01-01 03:00:00 +data_paths: ['/var/lib/clickhouse/store/81b/81b1c20a-b7c6-4116-a2ce-7583fb6b6736/'] +metadata_path: /var/lib/clickhouse/store/461/461cf698-fd0b-406d-8c01-5d8fd5748a91/t1.sql +metadata_modification_time: 2021-01-25 19:14:32 dependencies_database: [] dependencies_table: [] -create_table_query: -engine_full: -partition_key: -sorting_key: -primary_key: -sampling_key: -storage_policy: -total_rows: ᴺᵁᴸᴸ -total_bytes: ᴺᵁᴸᴸ +create_table_query: CREATE TABLE base.t1 (`n` UInt64) ENGINE = MergeTree ORDER BY n SETTINGS index_granularity = 8192 +engine_full: MergeTree ORDER BY n SETTINGS index_granularity = 8192 +partition_key: +sorting_key: n +primary_key: n +sampling_key: +storage_policy: default +total_rows: 1 +total_bytes: 99 +lifetime_rows: ᴺᵁᴸᴸ +lifetime_bytes: ᴺᵁᴸᴸ +comment: Row 2: ────── -database: system -name: asynchronous_metrics +database: default +name: 53r93yleapyears uuid: 00000000-0000-0000-0000-000000000000 -engine: SystemAsynchronousMetrics +engine: MergeTree is_temporary: 0 -data_paths: [] -metadata_path: /var/lib/clickhouse/metadata/system/asynchronous_metrics.sql -metadata_modification_time: 1970-01-01 03:00:00 +data_paths: ['/var/lib/clickhouse/data/default/53r93yleapyears/'] +metadata_path: /var/lib/clickhouse/metadata/default/53r93yleapyears.sql +metadata_modification_time: 2020-09-23 09:05:36 dependencies_database: [] dependencies_table: [] -create_table_query: -engine_full: -partition_key: -sorting_key: -primary_key: -sampling_key: -storage_policy: -total_rows: ᴺᵁᴸᴸ -total_bytes: ᴺᵁᴸᴸ +create_table_query: CREATE TABLE default.`53r93yleapyears` (`id` Int8, `febdays` Int8) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192 +engine_full: MergeTree ORDER BY id SETTINGS index_granularity = 8192 +partition_key: +sorting_key: id +primary_key: id +sampling_key: +storage_policy: default +total_rows: 2 +total_bytes: 155 +lifetime_rows: ᴺᵁᴸᴸ +lifetime_bytes: ᴺᵁᴸᴸ +comment: ``` diff --git a/docs/ru/sql-reference/aggregate-functions/reference/rankCorr.md b/docs/ru/sql-reference/aggregate-functions/reference/rankCorr.md index c98e7b88bcf..73d0552fc6f 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/rankCorr.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/rankCorr.md @@ -1,4 +1,8 @@ -## rankCorr {#agg_function-rankcorr} +--- +toc_priority: 145 +--- + +# rankCorr {#agg_function-rankcorr} Вычисляет коэффициент ранговой корреляции. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/sumcount.md b/docs/ru/sql-reference/aggregate-functions/reference/sumcount.md new file mode 100644 index 00000000000..0606b06fba0 --- /dev/null +++ b/docs/ru/sql-reference/aggregate-functions/reference/sumcount.md @@ -0,0 +1,46 @@ +--- +toc_priority: 144 +--- + +# sumCount {#agg_function-sumCount} + +Вычисляет сумму чисел и одновременно подсчитывает количество строк. + +**Синтаксис** + +``` sql +sumCount(x) +``` + +**Аргументы** + +- `x` — Входное значение типа [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md), или [Decimal](../../../sql-reference/data-types/decimal.md). + +**Возвращаемое значение** + +- Кортеж из элементов `(sum, count)`, где `sum` — это сумма чисел и `count` — количество строк со значениями, отличными от `NULL`. + +Тип: [Tuple](../../../sql-reference/data-types/tuple.md). + +**Пример** + +Запрос: + +``` sql +CREATE TABLE s_table (x Nullable(Int8)) Engine = Log; +INSERT INTO s_table SELECT number FROM numbers(0, 20); +INSERT INTO s_table VALUES (NULL); +SELECT sumCount(x) from s_table; +``` + +Результат: + +``` text +┌─sumCount(x)─┐ +│ (190,20) │ +└─────────────┘ +``` + +**Смотрите также** + +- Настройка [optimize_fuse_sum_count_avg](../../../operations/settings/settings.md#optimize_fuse_sum_count_avg) diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index f4fa21ba46a..23e840a7898 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -153,8 +153,60 @@ Result: ## unhex(str) {#unhexstr} -Accepts a string containing any number of hexadecimal digits, and returns a string containing the corresponding bytes. Supports both uppercase and lowercase letters A-F. The number of hexadecimal digits does not have to be even. If it is odd, the last digit is interpreted as the least significant half of the 00-0F byte. If the argument string contains anything other than hexadecimal digits, some implementation-defined result is returned (an exception isn’t thrown). -If you want to convert the result to a number, you can use the ‘reverse’ and ‘reinterpretAsType’ functions. +Выполняет операцию, обратную [hex](#hex). Функция интерпретирует каждую пару шестнадцатеричных цифр аргумента как число и преобразует его в символ. Возвращаемое значение представляет собой двоичную строку (BLOB). + +Если вы хотите преобразовать результат в число, вы можете использовать функции [reverse](../../sql-reference/functions/string-functions.md#reverse) и [reinterpretAs](../../sql-reference/functions/type-conversion-functions.md#type-conversion-functions). + +!!! note "Примечание" + Если `unhex` вызывается из `clickhouse-client`, двоичные строки отображаются с использованием UTF-8. + +Синоним: `UNHEX`. + +**Синтаксис** + +``` sql +unhex(arg) +``` + +**Аргументы** + +- `arg` — Строка, содержащая любое количество шестнадцатеричных цифр. Тип: [String](../../sql-reference/data-types/string.md). + +Поддерживаются как прописные, так и строчные буквы `A-F`. Количество шестнадцатеричных цифр не обязательно должно быть четным. Если оно нечетное, последняя цифра интерпретируется как наименее значимая половина байта `00-0F`. Если строка аргумента содержит что-либо, кроме шестнадцатеричных цифр, возвращается некоторый результат, определенный реализацией (исключение не создается). + +**Возвращаемое значение** + +- Бинарная строка (BLOB). + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: +``` sql +SELECT unhex('303132'), UNHEX('4D7953514C'); +``` + +Результат: +``` text +┌─unhex('303132')─┬─unhex('4D7953514C')─┐ +│ 012 │ MySQL │ +└─────────────────┴─────────────────────┘ +``` + +Запрос: + +``` sql +SELECT reinterpretAsUInt64(reverse(unhex('FFF'))) AS num; +``` + +Результат: + +``` text +┌──num─┐ +│ 4095 │ +└──────┘ +``` ## UUIDStringToNum(str) {#uuidstringtonumstr} @@ -171,4 +223,3 @@ If you want to convert the result to a number, you can use the ‘reverse’ and ## bitmaskToArray(num) {#bitmasktoarraynum} Принимает целое число. Возвращает массив чисел типа UInt64, содержащий степени двойки, в сумме дающих исходное число; числа в массиве идут по возрастанию. - diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md index be91145659e..612477dc806 100644 --- a/docs/ru/sql-reference/functions/ext-dict-functions.md +++ b/docs/ru/sql-reference/functions/ext-dict-functions.md @@ -3,6 +3,9 @@ toc_priority: 58 toc_title: "Функции для работы с внешними словарями" --- +!!! attention "Внимание" + Для словарей, созданных с помощью [DDL-запросов](../../sql-reference/statements/create/dictionary.md), в параметре `dict_name` указывается полное имя словаря вместе с базой данных, например: `.`. Если база данных не указана, используется текущая. + # Функции для работы с внешними словарями {#ext_dict_functions} Информацию о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). @@ -241,7 +244,7 @@ dictHas('dict_name', id) - 0, если ключа нет. - 1, если ключ есть. -Тип — `UInt8`. +Тип: [UInt8](../../sql-reference/data-types/int-uint.md). ## dictGetHierarchy {#dictgethierarchy} @@ -262,7 +265,7 @@ dictGetHierarchy('dict_name', key) - Цепочка предков заданного ключа. -Type: [Array(UInt64)](../../sql-reference/functions/ext-dict-functions.md). +Type: [Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md)). ## dictIsIn {#dictisin} @@ -281,7 +284,120 @@ Type: [Array(UInt64)](../../sql-reference/functions/ext-dict-functions.md). - 0, если `child_id_expr` — не дочерний элемент `ancestor_id_expr`. - 1, если `child_id_expr` — дочерний элемент `ancestor_id_expr` или если `child_id_expr` и есть `ancestor_id_expr`. -Тип — `UInt8`. +Тип: [UInt8](../../sql-reference/data-types/int-uint.md). + +## dictGetChildren {#dictgetchildren} + +Возвращает потомков первого уровня в виде массива индексов. Это обратное преобразование для [dictGetHierarchy](#dictgethierarchy). + +**Синтаксис** + +``` sql +dictGetChildren(dict_name, key) +``` + +**Аргументы** + +- `dict_name` — имя словаря. [String literal](../../sql-reference/syntax.md#syntax-string-literal). +- `key` — значение ключа. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md). + +**Возвращаемые значения** + +- Потомки первого уровня для ключа. + +Тип: [Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md)). + +**Пример** + +Рассмотрим иерархический словарь: + +``` text +┌─id─┬─parent_id─┐ +│ 1 │ 0 │ +│ 2 │ 1 │ +│ 3 │ 1 │ +│ 4 │ 2 │ +└────┴───────────┘ +``` + +Потомки первого уровня: + +``` sql +SELECT dictGetChildren('hierarchy_flat_dictionary', number) FROM system.numbers LIMIT 4; +``` + +``` text +┌─dictGetChildren('hierarchy_flat_dictionary', number)─┐ +│ [1] │ +│ [2,3] │ +│ [4] │ +│ [] │ +└──────────────────────────────────────────────────────┘ +``` + +## dictGetDescendant {#dictgetdescendant} + +Возвращает всех потомков, как если бы функция [dictGetChildren](#dictgetchildren) была выполнена `level` раз рекурсивно. + +**Синтаксис** + +``` sql +dictGetDescendants(dict_name, key, level) +``` + +**Аргументы** + +- `dict_name` — имя словаря. [String literal](../../sql-reference/syntax.md#syntax-string-literal). +- `key` — значение ключа. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md). +- `level` — уровень иерархии. Если `level = 0`, возвращаются все потомки. [UInt8](../../sql-reference/data-types/int-uint.md). + +**Возвращаемые значения** + +- Потомки для ключа. + +Тип: [Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md)). + +**Пример** + +Рассмотрим иерархический словарь: + +``` text +┌─id─┬─parent_id─┐ +│ 1 │ 0 │ +│ 2 │ 1 │ +│ 3 │ 1 │ +│ 4 │ 2 │ +└────┴───────────┘ +``` +Все потомки: + +``` sql +SELECT dictGetDescendants('hierarchy_flat_dictionary', number) FROM system.numbers LIMIT 4; +``` + +``` text +┌─dictGetDescendants('hierarchy_flat_dictionary', number)─┐ +│ [1,2,3,4] │ +│ [2,3,4] │ +│ [4] │ +│ [] │ +└─────────────────────────────────────────────────────────┘ +``` + +Потомки первого уровня: + +``` sql +SELECT dictGetDescendants('hierarchy_flat_dictionary', number, 1) FROM system.numbers LIMIT 4; +``` + +``` text +┌─dictGetDescendants('hierarchy_flat_dictionary', number, 1)─┐ +│ [1] │ +│ [2,3] │ +│ [4] │ +│ [] │ +└────────────────────────────────────────────────────────────┘ +``` ## Прочие функции {#ext_dict_functions-other} diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 2226c90525d..8707642eb59 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -3,7 +3,7 @@ toc_priority: 38 toc_title: "Функции преобразования типов" --- -# Функции преобразования типов {#funktsii-preobrazovaniia-tipov} +# Функции преобразования типов {#type-conversion-functions} ## Общие проблемы преобразования чисел {#numeric-conversion-issues} diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index 1ccd0a600f3..1d65d82b24c 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -346,4 +346,39 @@ SELECT * FROM base.t1; └───┘ ``` +## Секция COMMENT {#comment-table} + +Вы можете добавить комментарий к таблице при ее создании. + +!!!note "Замечание" + Комментарий поддерживается для всех движков таблиц, кроме [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) и [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md). + +**Синтаксис** + +``` sql +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ... +) +ENGINE = engine +COMMENT 'Comment' +``` + +**Пример** + +Запрос: + +``` sql +CREATE TABLE t1 (x String) ENGINE = Memory COMMENT 'The temporary table'; +SELECT name, comment FROM system.tables WHERE name = 't1'; +``` + +Результат: + +```text +┌─name─┬─comment─────────────┐ +│ t1 │ The temporary table │ +└──────┴─────────────────────┘ +``` + diff --git a/docs/ru/sql-reference/statements/explain.md b/docs/ru/sql-reference/statements/explain.md index c2a35f1b925..c925e7030a7 100644 --- a/docs/ru/sql-reference/statements/explain.md +++ b/docs/ru/sql-reference/statements/explain.md @@ -111,9 +111,11 @@ CROSS JOIN system.numbers AS c Настройки: -- `header` — выводит выходной заголовок для шага. По умолчанию: 0. -- `description` — выводит описание шага. По умолчанию: 1. -- `actions` — выводит подробную информацию о действиях, выполняемых на данном шаге. По умолчанию: 0. +- `header` — выводит выходной заголовок для шага. По умолчанию: 0. +- `description` — выводит описание шага. По умолчанию: 1. +- `indexes` — показывает используемые индексы, количество отфильтрованных кусков и гранул для каждого примененного индекса. По умолчанию: 0. Поддерживается для таблиц семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). +- `actions` — выводит подробную информацию о действиях, выполняемых на данном шаге. По умолчанию: 0. +- `json` — выводит шаги выполнения запроса в виде строки в формате [JSON](../../interfaces/formats.md#json). По умолчанию: 0. Чтобы избежать ненужного экранирования, рекомендуется использовать формат [TSVRaw](../../interfaces/formats.md#tabseparatedraw). Пример: @@ -133,6 +135,225 @@ Union !!! note "Примечание" Оценка стоимости выполнения шага и запроса не поддерживается. + +При `json = 1` шаги выполнения запроса выводятся в формате JSON. Каждый узел — это словарь, в котором всегда есть ключи `Node Type` и `Plans`. `Node Type` — это строка с именем шага. `Plans` — это массив с описаниями дочерних шагов. Другие дополнительные ключи могут быть добавлены в зависимости от типа узла и настроек. + +Пример: + +```sql +EXPLAIN json = 1, description = 0 SELECT 1 UNION ALL SELECT 2 FORMAT TSVRaw; +``` + +```json +[ + { + "Plan": { + "Node Type": "Union", + "Plans": [ + { + "Node Type": "Expression", + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Plans": [ + { + "Node Type": "ReadFromStorage" + } + ] + } + ] + }, + { + "Node Type": "Expression", + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Plans": [ + { + "Node Type": "ReadFromStorage" + } + ] + } + ] + } + ] + } + } +] +``` + +При `description` = 1 к шагу добавляется ключ `Description`: + +```json +{ + "Node Type": "ReadFromStorage", + "Description": "SystemOne" +} +``` + +При `header` = 1 к шагу добавляется ключ `Header` в виде массива столбцов. + +Пример: + +```sql +EXPLAIN json = 1, description = 0, header = 1 SELECT 1, 2 + dummy; +``` + +```json +[ + { + "Plan": { + "Node Type": "Expression", + "Header": [ + { + "Name": "1", + "Type": "UInt8" + }, + { + "Name": "plus(2, dummy)", + "Type": "UInt16" + } + ], + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Header": [ + { + "Name": "dummy", + "Type": "UInt8" + } + ], + "Plans": [ + { + "Node Type": "ReadFromStorage", + "Header": [ + { + "Name": "dummy", + "Type": "UInt8" + } + ] + } + ] + } + ] + } + } +] +``` + +При `indexes` = 1 добавляется ключ `Indexes`. Он содержит массив используемых индексов. Каждый индекс описывается как строка в формате JSON с ключом `Type` (`MinMax`, `Partition`, `PrimaryKey` или `Skip`) и дополнительные ключи: + +- `Name` — имя индекса (на данный момент используется только для индекса `Skip`). +- `Keys` — массив столбцов, используемых индексом. +- `Condition` — строка с используемым условием. +- `Description` — индекс (на данный момент используется только для индекса `Skip`). +- `Initial Parts` — количество кусков до применения индекса. +- `Selected Parts` — количество кусков после применения индекса. +- `Initial Granules` — количество гранул до применения индекса. +- `Selected Granulesis` — количество гранул после применения индекса. + +Пример: + +```json +"Node Type": "ReadFromMergeTree", +"Indexes": [ + { + "Type": "MinMax", + "Keys": ["y"], + "Condition": "(y in [1, +inf))", + "Initial Parts": 5, + "Selected Parts": 4, + "Initial Granules": 12, + "Selected Granules": 11 + }, + { + "Type": "Partition", + "Keys": ["y", "bitAnd(z, 3)"], + "Condition": "and((bitAnd(z, 3) not in [1, 1]), and((y in [1, +inf)), (bitAnd(z, 3) not in [1, 1])))", + "Initial Parts": 4, + "Selected Parts": 3, + "Initial Granules": 11, + "Selected Granules": 10 + }, + { + "Type": "PrimaryKey", + "Keys": ["x", "y"], + "Condition": "and((x in [11, +inf)), (y in [1, +inf)))", + "Initial Parts": 3, + "Selected Parts": 2, + "Initial Granules": 10, + "Selected Granules": 6 + }, + { + "Type": "Skip", + "Name": "t_minmax", + "Description": "minmax GRANULARITY 2", + "Initial Parts": 2, + "Selected Parts": 1, + "Initial Granules": 6, + "Selected Granules": 2 + }, + { + "Type": "Skip", + "Name": "t_set", + "Description": "set GRANULARITY 2", + "Initial Parts": 1, + "Selected Parts": 1, + "Initial Granules": 2, + "Selected Granules": 1 + } +] +``` + +При `actions` = 1 добавляются ключи, зависящие от типа шага. + +Пример: + +```sql +EXPLAIN json = 1, actions = 1, description = 0 SELECT 1 FORMAT TSVRaw; +``` + +```json +[ + { + "Plan": { + "Node Type": "Expression", + "Expression": { + "Inputs": [], + "Actions": [ + { + "Node Type": "Column", + "Result Type": "UInt8", + "Result Type": "Column", + "Column": "Const(UInt8)", + "Arguments": [], + "Removed Arguments": [], + "Result": 0 + } + ], + "Outputs": [ + { + "Name": "1", + "Type": "UInt8" + } + ], + "Positions": [0], + "Project Input": true + }, + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Plans": [ + { + "Node Type": "ReadFromStorage" + } + ] + } + ] + } + } +] +``` ### EXPLAIN PIPELINE {#explain-pipeline} @@ -141,7 +362,6 @@ Union - `header` — выводит заголовок для каждого выходного порта. По умолчанию: 0. - `graph` — выводит граф, описанный на языке [DOT](https://ru.wikipedia.org/wiki/DOT_(язык)). По умолчанию: 0. - `compact` — выводит граф в компактном режиме, если включена настройка `graph`. По умолчанию: 1. -- `indexes` — показывает используемые индексы, количество отфильтрованных кусков и гранул для каждого примененного индекса. По умолчанию: 0. Поддерживается для таблиц семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). Пример: diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index bbd330962cf..328f1023624 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -107,6 +107,8 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... Для табличной функции [input()](../table-functions/input.md) после секции `SELECT` должна следовать секция `FORMAT`. +Чтобы вставить значение по умолчанию вместо `NULL` в столбец, который не позволяет хранить `NULL`, включите настройку [insert_null_as_default](../../operations/settings/settings.md#insert_null_as_default). + ### Замечания о производительности {#zamechaniia-o-proizvoditelnosti} `INSERT` сортирует входящие данные по первичному ключу и разбивает их на партиции по ключу партиционирования. Если вы вставляете данные в несколько партиций одновременно, то это может значительно снизить производительность запроса `INSERT`. Чтобы избежать этого: diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index e1a9d613537..70503ec4de9 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -32,10 +32,22 @@ ClickHouse не оповещает клиента. Чтобы включить Список столбцов для дедупликации должен включать все столбцы, указанные в условиях сортировки (первичный ключ и ключ сортировки), а также в условиях партиционирования (ключ партиционирования). - !!! note "Примечание" - Обратите внимание, что символ подстановки `*` обрабатывается так же, как и в запросах `SELECT`: столбцы `MATERIALIZED` и `ALIAS` не включаются в результат. - Если указать пустой список или выражение, которое возвращает пустой список, или дедуплицировать столбец по псевдониму (`ALIAS`), то сервер вернет ошибку. +!!! note "Примечание" + Обратите внимание, что символ подстановки `*` обрабатывается так же, как и в запросах `SELECT`: столбцы [MATERIALIZED](../../sql-reference/statements/create/table.md#materialized) и [ALIAS](../../sql-reference/statements/create/table.md#alias) не включаются в результат. + Если указать пустой список или выражение, которое возвращает пустой список, то сервер вернет ошибку. Запрос вида `DEDUPLICATE BY aliased_value` также вернет ошибку. +**Синтаксис** + +``` sql +OPTIMIZE TABLE table DEDUPLICATE; -- по всем столбцам +OPTIMIZE TABLE table DEDUPLICATE BY *; -- исключаются столбцы MATERIALIZED и ALIAS +OPTIMIZE TABLE table DEDUPLICATE BY colX,colY,colZ; +OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT colX; +OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT (colX, colY); +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex'); +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT colX; +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (colX, colY); +``` **Примеры** @@ -50,38 +62,131 @@ CREATE TABLE example ( materialized_value UInt32 MATERIALIZED 12345, aliased_value UInt32 ALIAS 2, PRIMARY KEY primary_key -) ENGINE=MergeTree -PARTITION BY partition_key; +) ENGINE=MergeTree　 +PARTITION BY partition_key +ORDER BY (primary_key, secondary_key); +``` +``` sql +INSERT INTO example (primary_key, secondary_key, value, partition_key) +VALUES (0, 0, 0, 0), (0, 0, 0, 0), (1, 1, 2, 2), (1, 1, 2, 3), (1, 1, 3, 3); +``` +``` sql +SELECT * FROM example; +``` +Результат: ``` -Прежний способ дедупликации, когда учитываются все столбцы. Строка удаляется только в том случае, если все значения во всех столбцах равны соответствующим значениям в предыдущей строке. +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` +Если в запросе не указаны столбцы, по которым нужно дедуплицировать, то учитываются все столбцы таблицы. Строка удаляется только в том случае, если все значения во всех столбцах равны соответствующим значениям в другой строке. ``` sql OPTIMIZE TABLE example FINAL DEDUPLICATE; ``` - -Дедупликация по всем столбцам, кроме `ALIAS` и `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key` и `materialized_value`. - - ``` sql -OPTIMIZE TABLE example FINAL DEDUPLICATE BY *; +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ ``` -Дедупликация по всем столбцам, кроме `ALIAS`, `MATERIALIZED` и `materialized_value`: столбцы `primary_key`, `secondary_key`, `value` и `partition_key`. - - +Если столбцы в запросе указаны через `*`, то дедупликация пройдет по всем столбцам, кроме `ALIAS` и `MATERIALIZED`. Для таблицы `example` будут учтены: `primary_key`, `secondary_key`, `value` и `partition_key`. +```sql +OPTIMIZE TABLE example FINAL DEDUPLICATE BY *; +``` ``` sql -OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT materialized_value; +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +Дедупликация по всем столбцам, кроме `ALIAS` и `MATERIALIZED` (`BY *`), и с исключением столбца `value`: `primary_key`, `secondary_key` и `partition_key`. +``` sql +OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT value; +``` +``` sql +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ ``` Дедупликация по столбцам `primary_key`, `secondary_key` и `partition_key`. - -``` sql +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY primary_key, secondary_key, partition_key; ``` - -Дедупликация по любому столбцу, соответствующему регулярному выражению: столбцам `primary_key`, `secondary_key` и `partition_key`. - ``` sql +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +Дедупликация по любому столбцу, который соответствует регулярному выражению `.*_key`: `primary_key`, `secondary_key` и `partition_key`. +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY COLUMNS('.*_key'); ``` +``` sql +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/select/union.md b/docs/ru/sql-reference/statements/select/union.md index de8a9b0e4ea..a1e31a0be7f 100644 --- a/docs/ru/sql-reference/statements/select/union.md +++ b/docs/ru/sql-reference/statements/select/union.md @@ -78,3 +78,7 @@ SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; Запросы, которые являются частью `UNION/UNION ALL/UNION DISTINCT`, выполняются параллельно, и их результаты могут быть смешаны вместе. +**Смотрите также** + +- Настройка [insert_null_as_default](../../../operations/settings/settings.md#insert_null_as_default). +- Настройка [union_default_mode](../../../operations/settings/settings.md#union-default-mode). diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index f0f9b77b5ba..8c82cacdc43 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -196,7 +196,7 @@ SYSTEM STOP MOVES [[db.]merge_tree_family_table_name] Возвращает `Ok.` даже если указана несуществующая таблица или таблица имеет тип отличный от MergeTree. Возвращает ошибку если указана не существующая база данных: ``` sql -SYSTEM STOP MOVES [[db.]merge_tree_family_table_name] +SYSTEM START MOVES [[db.]merge_tree_family_table_name] ``` ## Managing ReplicatedMergeTree Tables {#query-language-system-replicated} diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 85f9dc2a9dd..9bb4f57e9e2 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -27,12 +27,12 @@ pymdown-extensions==8.0 python-slugify==4.0.1 PyYAML==5.4.1 repackage==0.7.3 -requests==2.24.0 +requests==2.25.1 singledispatch==3.4.0.3 six==1.15.0 soupsieve==2.0.1 termcolor==1.1.0 tornado==6.1 Unidecode==1.1.1 -urllib3==1.25.10 +urllib3>=1.26.5 Pygments>=2.7.4 diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 1d2b579db3a..2e48c5d20c5 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -36,7 +35,9 @@ #include #include #include +#include +namespace fs = std::filesystem; /** A tool for evaluating ClickHouse performance. * The tool emulates a case with fixed amount of simultaneously executing queries. @@ -119,8 +120,8 @@ public: int main(const std::vector &) override { - if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results - Poco::File(json_path).remove(); + if (!json_path.empty() && fs::exists(json_path)) /// Clear file with previous results + fs::remove(json_path); readQueries(); runBenchmark(); @@ -159,7 +160,7 @@ private: bool print_stacktrace; const Settings & settings; SharedContextHolder shared_context; - ContextPtr global_context; + ContextMutablePtr global_context; QueryProcessingStage::Enum query_processing_stage; /// Don't execute new queries after timelimit or SIGINT or exception diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 098f7e689c5..f268d2b5cdc 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -87,6 +86,8 @@ #include #include #include +#include +#include #if !defined(ARCADIA_BUILD) # include @@ -96,6 +97,7 @@ #pragma GCC optimize("-fno-var-tracking-assignments") #endif +namespace fs = std::filesystem; namespace DB { @@ -181,7 +183,7 @@ private: bool has_vertical_output_suffix = false; /// Is \G present at the end of the query string? SharedContextHolder shared_context = Context::createShared(); - ContextPtr context = Context::createGlobal(shared_context.get()); + ContextMutablePtr context = Context::createGlobal(shared_context.get()); /// Buffer that reads from stdin in batch mode. ReadBufferFromFileDescriptor std_in{STDIN_FILENO}; @@ -276,7 +278,7 @@ private: /// Set path for format schema files if (config().has("format_schema_path")) - context->setFormatSchemaPath(Poco::Path(config().getString("format_schema_path")).toString()); + context->setFormatSchemaPath(fs::weakly_canonical(config().getString("format_schema_path"))); /// Initialize query_id_formats if any if (config().has("query_id_formats")) @@ -633,8 +635,8 @@ private: history_file = home_path + "/.clickhouse-client-history"; } - if (!history_file.empty() && !Poco::File(history_file).exists()) - Poco::File(history_file).createFile(); + if (!history_file.empty() && !fs::exists(history_file)) + FS::createFile(history_file); LineReader::Patterns query_extenders = {"\\"}; LineReader::Patterns query_delimiters = {";", "\\G"}; diff --git a/programs/copier/ClusterCopier.h b/programs/copier/ClusterCopier.h index e875ca7df2e..085fa2ece06 100644 --- a/programs/copier/ClusterCopier.h +++ b/programs/copier/ClusterCopier.h @@ -12,14 +12,14 @@ namespace DB { -class ClusterCopier : WithContext +class ClusterCopier : WithMutableContext { public: ClusterCopier(const String & task_path_, const String & host_id_, const String & proxy_database_name_, - ContextPtr context_) - : WithContext(context_), + ContextMutablePtr context_) + : WithMutableContext(context_), task_zookeeper_path(task_path_), host_id(host_id_), working_database_name(proxy_database_name_), diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index d3fff616b65..8925ab63f99 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -5,7 +5,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -26,7 +28,7 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) copy_fault_probability = std::max(std::min(config().getDouble("copy-fault-probability"), 1.0), 0.0); if (config().has("move-fault-probability")) move_fault_probability = std::max(std::min(config().getDouble("move-fault-probability"), 1.0), 0.0); - base_dir = (config().has("base-dir")) ? config().getString("base-dir") : Poco::Path::current(); + base_dir = (config().has("base-dir")) ? config().getString("base-dir") : fs::current_path().string(); if (config().has("experimental-use-sample-offset")) @@ -38,18 +40,18 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) process_id = std::to_string(DateLUT::instance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid); host_id = escapeForFileName(getFQDNOrHostName()) + '#' + process_id; - process_path = Poco::Path(base_dir + "/clickhouse-copier_" + process_id).absolute().toString(); - Poco::File(process_path).createDirectories(); + process_path = fs::weakly_canonical(fs::path(base_dir) / ("clickhouse-copier_" + process_id)); + fs::create_directories(process_path); /// Override variables for BaseDaemon if (config().has("log-level")) config().setString("logger.level", config().getString("log-level")); if (config().has("base-dir") || !config().has("logger.log")) - config().setString("logger.log", process_path + "/log.log"); + config().setString("logger.log", fs::path(process_path) / "log.log"); if (config().has("base-dir") || !config().has("logger.errorlog")) - config().setString("logger.errorlog", process_path + "/log.err.log"); + config().setString("logger.errorlog", fs::path(process_path) / "log.err.log"); Base::initialize(self); } diff --git a/programs/copier/Internals.h b/programs/copier/Internals.h index 7e45c0ea2ee..9e40d7ebd7b 100644 --- a/programs/copier/Internals.h +++ b/programs/copier/Internals.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 8c187978106..a7f566a78b8 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -288,7 +288,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv) bool is_symlink = fs::is_symlink(symlink_path); fs::path points_to; if (is_symlink) - points_to = fs::absolute(fs::read_symlink(symlink_path)); + points_to = fs::weakly_canonical(fs::read_symlink(symlink_path)); if (is_symlink && points_to == main_bin_path) { diff --git a/programs/keeper/Keeper.h b/programs/keeper/Keeper.h index e80fe10b61c..f5b97dacf7d 100644 --- a/programs/keeper/Keeper.h +++ b/programs/keeper/Keeper.h @@ -32,7 +32,7 @@ public: return BaseDaemon::logger(); } - ContextPtr context() const override + ContextMutablePtr context() const override { return global_context; } @@ -58,7 +58,7 @@ protected: std::string getDefaultConfigFileName() const override; private: - ContextPtr global_context; + ContextMutablePtr global_context; Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) const; diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 043cc596e2b..f48e8d4d0a0 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -42,9 +42,9 @@ #include #include #include - #include +namespace fs = std::filesystem; namespace DB { @@ -72,11 +72,11 @@ void LocalServer::initialize(Poco::Util::Application & self) Poco::Util::Application::initialize(self); /// Load config files if exists - if (config().has("config-file") || Poco::File("config.xml").exists()) + if (config().has("config-file") || fs::exists("config.xml")) { const auto config_path = config().getString("config-file", "config.xml"); ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); + config_processor.setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", ".")); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); @@ -100,7 +100,7 @@ void LocalServer::initialize(Poco::Util::Application & self) } } -void LocalServer::applyCmdSettings(ContextPtr context) +void LocalServer::applyCmdSettings(ContextMutablePtr context) { context->applySettingsChanges(cmd_settings.changes()); } @@ -287,8 +287,8 @@ try status.emplace(path + "status", StatusFile::write_full_info); LOG_DEBUG(log, "Loading metadata from {}", path); - Poco::File(path + "data/").createDirectories(); - Poco::File(path + "metadata/").createDirectories(); + fs::create_directories(fs::path(path) / "data/"); + fs::create_directories(fs::path(path) / "metadata/"); loadMetadataSystem(global_context); attachSystemTables(global_context); loadMetadata(global_context); @@ -479,7 +479,7 @@ void LocalServer::setupUsers() { ConfigurationPtr users_config; - if (config().has("users_config") || config().has("config-file") || Poco::File("config.xml").exists()) + if (config().has("users_config") || config().has("config-file") || fs::exists("config.xml")) { const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml")); ConfigProcessor config_processor(users_config_path); @@ -645,7 +645,7 @@ void LocalServer::init(int argc, char ** argv) argsToConfig(arguments, config(), 100); } -void LocalServer::applyCmdOptions(ContextPtr context) +void LocalServer::applyCmdOptions(ContextMutablePtr context) { context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV"))); applyCmdSettings(context); diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index c5e9d5716dd..cf8886d9652 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -36,8 +36,8 @@ private: std::string getInitialCreateTableQuery(); void tryInitPath(); - void applyCmdOptions(ContextPtr context); - void applyCmdSettings(ContextPtr context); + void applyCmdOptions(ContextMutablePtr context); + void applyCmdSettings(ContextMutablePtr context); void processQueries(); void setupUsers(); void cleanup(); @@ -45,7 +45,7 @@ private: protected: SharedContextHolder shared_context; - ContextPtr global_context; + ContextMutablePtr global_context; /// Settings specified via command line args Settings cmd_settings; diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index fb6817fbf80..f68b255158c 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -1133,7 +1133,7 @@ try } SharedContextHolder shared_context = Context::createShared(); - ContextPtr context = Context::createGlobal(shared_context.get()); + auto context = Context::createGlobal(shared_context.get()); context->makeGlobalContext(); ReadBufferFromFileDescriptor file_in(STDIN_FILENO); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 6b3136dc200..f169d793ee9 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -74,6 +74,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) @@ -117,6 +118,8 @@ namespace CurrentMetrics extern const Metric MaxDDLEntryID; } +namespace fs = std::filesystem; + #if USE_JEMALLOC static bool jemallocOptionEnabled(const char *name) { @@ -183,19 +186,19 @@ void setupTmpPath(Poco::Logger * log, const std::string & path) { LOG_DEBUG(log, "Setting up {} to store temporary data in it", path); - Poco::File(path).createDirectories(); + fs::create_directories(path); /// Clearing old temporary files. - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(path); it != dir_end; ++it) + fs::directory_iterator dir_end; + for (fs::directory_iterator it(path); it != dir_end; ++it) { - if (it->isFile() && startsWith(it.name(), "tmp")) + if (it->is_regular_file() && startsWith(it->path().filename(), "tmp")) { - LOG_DEBUG(log, "Removing old temporary file {}", it->path()); - it->remove(); + LOG_DEBUG(log, "Removing old temporary file {}", it->path().string()); + fs::remove(it->path()); } else - LOG_DEBUG(log, "Skipped file in temporary path {}", it->path()); + LOG_DEBUG(log, "Skipped file in temporary path {}", it->path().string()); } } @@ -678,37 +681,38 @@ int Server::main(const std::vector & /*args*/) * Examples: do repair of local data; clone all replicated tables from replica. */ { - Poco::File(path + "flags/").createDirectories(); - global_context->setFlagsPath(path + "flags/"); + auto flags_path = fs::path(path) / "flags/"; + fs::create_directories(flags_path); + global_context->setFlagsPath(flags_path); } /** Directory with user provided files that are usable by 'file' table function. */ { - std::string user_files_path = config().getString("user_files_path", path + "user_files/"); + std::string user_files_path = config().getString("user_files_path", fs::path(path) / "user_files/"); global_context->setUserFilesPath(user_files_path); - Poco::File(user_files_path).createDirectories(); + fs::create_directories(user_files_path); } { - std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", path + "dictionaries_lib/"); + std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", fs::path(path) / "dictionaries_lib/"); global_context->setDictionariesLibPath(dictionaries_lib_path); - Poco::File(dictionaries_lib_path).createDirectories(); + fs::create_directories(dictionaries_lib_path); } /// top_level_domains_lists { - const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/") + "/"; - TLDListsHolder::getInstance().parseConfig(top_level_domains_path, config()); + const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/"); + TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config()); } { - Poco::File(path + "data/").createDirectories(); - Poco::File(path + "metadata/").createDirectories(); + fs::create_directories(fs::path(path) / "data/"); + fs::create_directories(fs::path(path) / "metadata/"); /// Directory with metadata of tables, which was marked as dropped by Atomic database - Poco::File(path + "metadata_dropped/").createDirectories(); + fs::create_directories(fs::path(path) / "metadata_dropped/"); } if (config().has("interserver_http_port") && config().has("interserver_https_port")) @@ -891,9 +895,9 @@ int Server::main(const std::vector & /*args*/) #endif /// Set path for format schema files - auto format_schema_path = Poco::File(config().getString("format_schema_path", path + "format_schemas/")); - global_context->setFormatSchemaPath(format_schema_path.path()); - format_schema_path.createDirectories(); + fs::path format_schema_path(config().getString("format_schema_path", fs::path(path) / "format_schemas/")); + global_context->setFormatSchemaPath(format_schema_path); + fs::create_directories(format_schema_path); /// Check sanity of MergeTreeSettings on server startup global_context->getMergeTreeSettings().sanityCheck(settings); diff --git a/programs/server/Server.h b/programs/server/Server.h index c698108767c..45e5fccd51d 100644 --- a/programs/server/Server.h +++ b/programs/server/Server.h @@ -40,7 +40,7 @@ public: return BaseDaemon::logger(); } - ContextPtr context() const override + ContextMutablePtr context() const override { return global_context; } @@ -64,7 +64,7 @@ protected: std::string getDefaultCorePath() const override; private: - ContextPtr global_context; + ContextMutablePtr global_context; Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) const; using CreateServerFunc = std::function; diff --git a/src/Access/AllowedClientHosts.h b/src/Access/AllowedClientHosts.h index a6895b120e0..7b21fd7e236 100644 --- a/src/Access/AllowedClientHosts.h +++ b/src/Access/AllowedClientHosts.h @@ -7,7 +7,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -198,9 +200,9 @@ inline String AllowedClientHosts::IPSubnet::toString() const if (isMaskAllBitsOne()) return prefix.toString(); else if (IPAddress{prefix_length, mask.family()} == mask) - return prefix.toString() + "/" + std::to_string(prefix_length); + return fs::path(prefix.toString()) / std::to_string(prefix_length); else - return prefix.toString() + "/" + mask.toString(); + return fs::path(prefix.toString()) / mask.toString(); } inline bool AllowedClientHosts::IPSubnet::isMaskAllBitsOne() const diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index 92439342168..512ccff1d1b 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -51,7 +51,7 @@ private: void setAllNoLock(const std::vector> & all_entities, Notifications & notifications); void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const; - mutable std::mutex mutex; + mutable std::recursive_mutex mutex; std::unordered_map entries_by_id; /// We want to search entries both by ID and by the pair of name and type. std::unordered_map entries_by_name_and_type[static_cast(EntityType::MAX)]; mutable std::list handlers_by_type[static_cast(EntityType::MAX)]; diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index c84a04cc872..bdc057817c9 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -89,6 +89,17 @@ AggregateFunctionPtr AggregateFunctionFactory::get( AggregateFunctionPtr nested_function = getImpl( name, nested_types, nested_parameters, out_properties, has_null_arguments); + + // Pure window functions are not real aggregate functions. Applying + // combinators doesn't make sense for them, they must handle the + // nullability themselves. Another special case is functions from Nothing + // that are rewritten to AggregateFunctionNothing, in this case + // nested_function is nullptr. + if (nested_function && nested_function->asWindowFunction()) + { + return nested_function; + } + return combinator->transformAggregateFunction(nested_function, out_properties, type_without_low_cardinality, parameters); } @@ -122,7 +133,7 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( is_case_insensitive = true; } - ContextPtr query_context; + ContextConstPtr query_context; if (CurrentThread::isInitialized()) query_context = CurrentThread::get().getQueryContext(); diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp index 0b6764960fb..e7f79c276ed 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp @@ -148,7 +148,7 @@ void LinearModelData::predict( const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, - ContextPtr context) const + ContextConstPtr context) const { gradient_computer->predict(container, arguments, offset, limit, weights, bias, context); } @@ -455,7 +455,7 @@ void LogisticRegression::predict( size_t limit, const std::vector & weights, Float64 bias, - ContextPtr /*context*/) const + ContextConstPtr /*context*/) const { size_t rows_num = arguments.front().column->size(); @@ -523,7 +523,7 @@ void LinearRegression::predict( size_t limit, const std::vector & weights, Float64 bias, - ContextPtr /*context*/) const + ContextConstPtr /*context*/) const { if (weights.size() + 1 != arguments.size()) { diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.h b/src/AggregateFunctions/AggregateFunctionMLMethod.h index 91302269759..a97b565eed3 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.h +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.h @@ -46,7 +46,7 @@ public: size_t limit, const std::vector & weights, Float64 bias, - ContextPtr context) const = 0; + ContextConstPtr context) const = 0; }; @@ -71,7 +71,7 @@ public: size_t limit, const std::vector & weights, Float64 bias, - ContextPtr context) const override; + ContextConstPtr context) const override; }; @@ -96,7 +96,7 @@ public: size_t limit, const std::vector & weights, Float64 bias, - ContextPtr context) const override; + ContextConstPtr context) const override; }; @@ -266,7 +266,7 @@ public: const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, - ContextPtr context) const; + ContextConstPtr context) const; void returnWeights(IColumn & to) const; private: @@ -367,7 +367,7 @@ public: const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, - ContextPtr context) const override + ContextConstPtr context) const override { if (arguments.size() != param_num + 1) throw Exception( diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index a05f71a61c8..2d47a1e7fe0 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -123,7 +123,7 @@ public: const ColumnsWithTypeAndName & /*arguments*/, size_t /*offset*/, size_t /*limit*/, - ContextPtr /*context*/) const + ContextConstPtr /*context*/) const { throw Exception("Method predictValues is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Bridge/IBridgeHelper.cpp b/src/Bridge/IBridgeHelper.cpp index 16886bae4fe..b6f3446d0a6 100644 --- a/src/Bridge/IBridgeHelper.cpp +++ b/src/Bridge/IBridgeHelper.cpp @@ -3,9 +3,10 @@ #include #include #include -#include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -87,10 +88,10 @@ std::unique_ptr IBridgeHelper::startBridgeCommand() const const auto & config = getConfig(); /// Path to executable folder - Poco::Path path{config.getString("application.dir", "/usr/bin")}; + fs::path path(config.getString("application.dir", "/usr/bin")); std::vector cmd_args; - path.setFileName(serviceFileName()); + path /= serviceFileName(); cmd_args.push_back("--http-port"); cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", getDefaultPort()))); @@ -126,7 +127,7 @@ std::unique_ptr IBridgeHelper::startBridgeCommand() const LOG_TRACE(getLog(), "Starting {}", serviceAlias()); - return ShellCommand::executeDirect(path.toString(), cmd_args, ShellCommandDestructorStrategy(true)); + return ShellCommand::executeDirect(path.string(), cmd_args, ShellCommandDestructorStrategy(true)); } } diff --git a/src/Bridge/IBridgeHelper.h b/src/Bridge/IBridgeHelper.h index caaf031b7d8..34be69a7db7 100644 --- a/src/Bridge/IBridgeHelper.h +++ b/src/Bridge/IBridgeHelper.h @@ -14,7 +14,7 @@ namespace DB /// Common base class for XDBC and Library bridge helpers. /// Contains helper methods to check/start bridge sync. -class IBridgeHelper: protected WithContext +class IBridgeHelper: protected WithConstContext { public: @@ -27,7 +27,7 @@ public: static const inline std::string PING_METHOD = Poco::Net::HTTPRequest::HTTP_GET; static const inline std::string MAIN_METHOD = Poco::Net::HTTPRequest::HTTP_POST; - explicit IBridgeHelper(ContextPtr context_) : WithContext(context_) {} + explicit IBridgeHelper(ContextConstPtr context_) : WithConstContext(context_) {} virtual ~IBridgeHelper() = default; void startBridgeSync() const; diff --git a/src/Bridge/LibraryBridgeHelper.cpp b/src/Bridge/LibraryBridgeHelper.cpp index c589d0ce09e..8967b191b73 100644 --- a/src/Bridge/LibraryBridgeHelper.cpp +++ b/src/Bridge/LibraryBridgeHelper.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -21,7 +20,7 @@ namespace DB { LibraryBridgeHelper::LibraryBridgeHelper( - ContextPtr context_, + ContextConstPtr context_, const Block & sample_block_, const Field & dictionary_id_) : IBridgeHelper(context_->getGlobalContext()) diff --git a/src/Bridge/LibraryBridgeHelper.h b/src/Bridge/LibraryBridgeHelper.h index 12fe0c33363..c3c42e85f93 100644 --- a/src/Bridge/LibraryBridgeHelper.h +++ b/src/Bridge/LibraryBridgeHelper.h @@ -17,7 +17,7 @@ class LibraryBridgeHelper : public IBridgeHelper public: static constexpr inline size_t DEFAULT_PORT = 9012; - LibraryBridgeHelper(ContextPtr context_, const Block & sample_block, const Field & dictionary_id_); + LibraryBridgeHelper(ContextConstPtr context_, const Block & sample_block, const Field & dictionary_id_); bool initLibrary(const std::string & library_path, std::string library_settings, std::string attributes_names); diff --git a/src/Bridge/XDBCBridgeHelper.h b/src/Bridge/XDBCBridgeHelper.h index 299df6ff888..97a6251a9a0 100644 --- a/src/Bridge/XDBCBridgeHelper.h +++ b/src/Bridge/XDBCBridgeHelper.h @@ -5,10 +5,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -62,7 +60,7 @@ public: static constexpr inline auto SCHEMA_ALLOWED_HANDLER = "/schema_allowed"; XDBCBridgeHelper( - ContextPtr context_, + ContextConstPtr context_, Poco::Timespan http_timeout_, const std::string & connection_string_) : IXDBCBridgeHelper(context_->getGlobalContext()) diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 8178802f3bd..a5bfcae05e4 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -162,7 +162,7 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr colum return res; } -MutableColumnPtr ColumnAggregateFunction::predictValues(const ColumnsWithTypeAndName & arguments, ContextPtr context) const +MutableColumnPtr ColumnAggregateFunction::predictValues(const ColumnsWithTypeAndName & arguments, ContextConstPtr context) const { MutableColumnPtr res = func->getReturnTypeToPredict()->createColumn(); res->reserve(data.size()); diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 8eb1a04b174..e75af280ed5 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -119,7 +119,7 @@ public: const char * getFamilyName() const override { return "AggregateFunction"; } TypeIndex getDataType() const override { return TypeIndex::AggregateFunction; } - MutableColumnPtr predictValues(const ColumnsWithTypeAndName & arguments, ContextPtr context) const; + MutableColumnPtr predictValues(const ColumnsWithTypeAndName & arguments, ContextConstPtr context) const; size_t size() const override { diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index fa9e9b72087..7bbb4798d6f 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -62,7 +62,7 @@ static std::string numberFromHost(const std::string & s) bool ConfigProcessor::isPreprocessedFile(const std::string & path) { - return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX); + return endsWith(fs::path(path).stem(), PREPROCESSED_SUFFIX); } @@ -416,34 +416,32 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string & { Files files; - Poco::Path merge_dir_path(config_path); + fs::path merge_dir_path(config_path); std::set merge_dirs; /// Add path_to_config/config_name.d dir - merge_dir_path.setExtension("d"); - merge_dirs.insert(merge_dir_path.toString()); + merge_dir_path.replace_extension("d"); + merge_dirs.insert(merge_dir_path); /// Add path_to_config/conf.d dir - merge_dir_path.setBaseName("conf"); - merge_dirs.insert(merge_dir_path.toString()); + merge_dir_path.replace_filename("conf.d"); + merge_dirs.insert(merge_dir_path); for (const std::string & merge_dir_name : merge_dirs) { - Poco::File merge_dir(merge_dir_name); - if (!merge_dir.exists() || !merge_dir.isDirectory()) + if (!fs::exists(merge_dir_name) || !fs::is_directory(merge_dir_name)) continue; - for (Poco::DirectoryIterator it(merge_dir_name); it != Poco::DirectoryIterator(); ++it) + for (fs::directory_iterator it(merge_dir_name); it != fs::directory_iterator(); ++it) { - Poco::File & file = *it; - Poco::Path path(file.path()); - std::string extension = path.getExtension(); - std::string base_name = path.getBaseName(); + fs::path path(it->path()); + std::string extension = path.extension(); + std::string base_name = path.stem(); // Skip non-config and temporary files - if (file.isFile() && (extension == "xml" || extension == "conf" || extension == "yaml" || extension == "yml") && !startsWith(base_name, ".")) - { - files.push_back(file.path()); - } + if (fs::is_regular_file(path) + && (extension == ".xml" || extension == ".conf" || extension == ".yaml" || extension == ".yml") + && !startsWith(base_name, ".")) + files.push_back(it->path()); } } @@ -548,7 +546,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( else { std::string default_path = "/etc/metrika.xml"; - if (Poco::File(default_path).exists()) + if (fs::exists(default_path)) include_from_path = default_path; } if (!include_from_path.empty()) @@ -660,11 +658,11 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, if (!loaded_config.configuration->has("path")) { // Will use current directory - auto parent_path = Poco::Path(loaded_config.config_path).makeParent(); - preprocessed_dir = parent_path.toString(); - Poco::Path poco_new_path(new_path); - poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX); - new_path = poco_new_path.toString(); + fs::path parent_path = fs::path(loaded_config.config_path).parent_path(); + preprocessed_dir = parent_path.string(); + fs::path fs_new_path(new_path); + fs_new_path.replace_filename(fs_new_path.stem().string() + PREPROCESSED_SUFFIX + fs_new_path.extension().string()); + new_path = fs_new_path.string(); } else { @@ -679,9 +677,9 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, } preprocessed_path = (fs::path(preprocessed_dir) / fs::path(new_path)).string(); - auto preprocessed_path_parent = Poco::Path(preprocessed_path).makeParent(); - if (!preprocessed_path_parent.toString().empty()) - Poco::File(preprocessed_path_parent).createDirectories(); + auto preprocessed_path_parent = fs::path(preprocessed_path).parent_path(); + if (!preprocessed_path_parent.empty()) + fs::create_directories(preprocessed_path_parent); } DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml); LOG_DEBUG(log, "Saved preprocessed configuration to '{}'.", preprocessed_path); diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h index 5b16bc0cb1b..e13e66547a3 100644 --- a/src/Common/Config/ConfigProcessor.h +++ b/src/Common/Config/ConfigProcessor.h @@ -15,12 +15,9 @@ #include #include #include -#include -#include #include #include #include - #include diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index afff08e82bb..01bb4f556d6 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -1,12 +1,15 @@ #include "ConfigReloader.h" #include -#include #include #include #include "ConfigProcessor.h" +#include +#include +namespace fs = std::filesystem; + namespace DB { @@ -167,8 +170,8 @@ struct ConfigReloader::FileWithTimestamp void ConfigReloader::FilesChangesTracker::addIfExists(const std::string & path_to_add) { - if (!path_to_add.empty() && Poco::File(path_to_add).exists()) - files.emplace(path_to_add, Poco::File(path_to_add).getLastModified().epochTime()); + if (!path_to_add.empty() && fs::exists(path_to_add)) + files.emplace(path_to_add, FS::getModificationTime(path_to_add)); } bool ConfigReloader::FilesChangesTracker::isDifferOrNewerThan(const FilesChangesTracker & rhs) diff --git a/src/Common/Config/configReadClient.cpp b/src/Common/Config/configReadClient.cpp index 61d8e507c05..8ce2a8b03e9 100644 --- a/src/Common/Config/configReadClient.cpp +++ b/src/Common/Config/configReadClient.cpp @@ -1,8 +1,10 @@ #include "configReadClient.h" #include -#include #include "ConfigProcessor.h" +#include + +namespace fs = std::filesystem; namespace DB { @@ -11,11 +13,11 @@ bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::stri std::string config_path; if (config.has("config-file")) config_path = config.getString("config-file"); - else if (Poco::File("./clickhouse-client.xml").exists()) + else if (fs::exists("./clickhouse-client.xml")) config_path = "./clickhouse-client.xml"; - else if (!home_path.empty() && Poco::File(home_path + "/.clickhouse-client/config.xml").exists()) + else if (!home_path.empty() && fs::exists(home_path + "/.clickhouse-client/config.xml")) config_path = home_path + "/.clickhouse-client/config.xml"; - else if (Poco::File("/etc/clickhouse-client/config.xml").exists()) + else if (fs::exists("/etc/clickhouse-client/config.xml")) config_path = "/etc/clickhouse-client/config.xml"; if (!config_path.empty()) diff --git a/src/Common/CounterInFile.h b/src/Common/CounterInFile.h index 8cd4534d413..6326261234d 100644 --- a/src/Common/CounterInFile.h +++ b/src/Common/CounterInFile.h @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -59,7 +58,7 @@ public: Int64 res = -1; - bool file_doesnt_exists = !Poco::File(path).exists(); + bool file_doesnt_exists = !fs::exists(path); if (file_doesnt_exists && !create_if_need) { throw Poco::Exception("File " + path + " does not exist. " @@ -138,7 +137,7 @@ public: // Not thread-safe and not synchronized between processes. void fixIfBroken(UInt64 value) { - bool file_exists = Poco::File(path).exists(); + bool file_exists = fs::exists(path); int fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); if (-1 == fd) diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index 069f9cf2af7..c6ac7e9933a 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -87,7 +87,7 @@ public: /// Initializes query with current thread as master thread in constructor, and detaches it in destructor struct QueryScope { - explicit QueryScope(ContextPtr query_context); + explicit QueryScope(ContextMutablePtr query_context); ~QueryScope(); void logPeakMemoryUsage(); diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 330a193212b..d840830bf28 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -553,6 +553,7 @@ M(583, ILLEGAL_PROJECTION) \ M(584, PROJECTION_NOT_USED) \ M(585, CANNOT_PARSE_YAML) \ + M(586, CANNOT_CREATE_FILE) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index dca19eea7f2..e98cd3c3046 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -21,6 +21,8 @@ # include #endif +namespace fs = std::filesystem; + namespace DB { @@ -138,20 +140,8 @@ void throwFromErrnoWithPath(const std::string & s, const std::string & path, int throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno, path); } -void tryLogCurrentException(const char * log_name, const std::string & start_of_message) +static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message) { - tryLogCurrentException(&Poco::Logger::get(log_name), start_of_message); -} - -void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message) -{ - /// Under high memory pressure, any new allocation will definitelly lead - /// to MEMORY_LIMIT_EXCEEDED exception. - /// - /// And in this case the exception will not be logged, so let's block the - /// MemoryTracker until the exception will be logged. - MemoryTracker::LockExceptionInThread lock_memory_tracker(VariableContext::Global); - try { if (start_of_message.empty()) @@ -164,7 +154,32 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_ } } -static void getNoSpaceLeftInfoMessage(std::filesystem::path path, std::string & msg) +void tryLogCurrentException(const char * log_name, const std::string & start_of_message) +{ + /// Under high memory pressure, any new allocation will definitelly lead + /// to MEMORY_LIMIT_EXCEEDED exception. + /// + /// And in this case the exception will not be logged, so let's block the + /// MemoryTracker until the exception will be logged. + MemoryTracker::LockExceptionInThread lock_memory_tracker(VariableContext::Global); + + /// Poco::Logger::get can allocate memory too + tryLogCurrentExceptionImpl(&Poco::Logger::get(log_name), start_of_message); +} + +void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_message) +{ + /// Under high memory pressure, any new allocation will definitelly lead + /// to MEMORY_LIMIT_EXCEEDED exception. + /// + /// And in this case the exception will not be logged, so let's block the + /// MemoryTracker until the exception will be logged. + MemoryTracker::LockExceptionInThread lock_memory_tracker(VariableContext::Global); + + tryLogCurrentExceptionImpl(logger, start_of_message); +} + +static void getNoSpaceLeftInfoMessage(std::filesystem::path path, String & msg) { path = std::filesystem::absolute(path); /// It's possible to get ENOSPC for non existent file (e.g. if there are no free inodes and creat() fails) @@ -251,22 +266,12 @@ static std::string getExtraExceptionInfo(const std::exception & e) String msg; try { - if (const auto * file_exception = dynamic_cast(&e)) + if (const auto * file_exception = dynamic_cast(&e)) { - if (file_exception->code() == ENOSPC) - { - /// See Poco::FileImpl::handleLastErrorImpl(...) - constexpr const char * expected_error_message = "no space left on device: "; - if (startsWith(file_exception->message(), expected_error_message)) - { - String path = file_exception->message().substr(strlen(expected_error_message)); - getNoSpaceLeftInfoMessage(path, msg); - } - else - { - msg += "\nCannot print extra info for Poco::Exception"; - } - } + if (file_exception->code() == std::errc::no_space_on_device) + getNoSpaceLeftInfoMessage(file_exception->path1(), msg); + else + msg += "\nCannot print extra info for Poco::Exception"; } else if (const auto * errno_exception = dynamic_cast(&e)) { diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp index e7fcc8cadb7..173c4bd8a3a 100644 --- a/src/Common/FileChecker.cpp +++ b/src/Common/FileChecker.cpp @@ -21,7 +21,15 @@ namespace ErrorCodes FileChecker::FileChecker(DiskPtr disk_, const String & file_info_path_) : disk(std::move(disk_)) { setPath(file_info_path_); - load(); + try + { + load(); + } + catch (DB::Exception & e) + { + e.addMessage("Error loading file {}", files_info_path); + throw; + } } void FileChecker::setPath(const String & file_info_path_) diff --git a/src/Common/FileUpdatesTracker.h b/src/Common/FileUpdatesTracker.h index 0914c75693c..9fe37c2a75d 100644 --- a/src/Common/FileUpdatesTracker.h +++ b/src/Common/FileUpdatesTracker.h @@ -1,10 +1,11 @@ #pragma once -#include #include - #include +#include +#include +namespace fs = std::filesystem; class FileUpdatesTracker { @@ -31,6 +32,6 @@ public: private: Poco::Timestamp getLastModificationTime() const { - return Poco::File(path).getLastModified(); + return FS::getModificationTimestamp(path); } }; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index e9ad40075e6..a05fa3b5ad5 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -173,7 +173,7 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded) } #ifdef MEMORY_TRACKER_DEBUG_CHECKS - if (unlikely(_memory_tracker_always_throw_logical_error_on_allocation) && throw_if_memory_exceeded) + if (unlikely(_memory_tracker_always_throw_logical_error_on_allocation)) { _memory_tracker_always_throw_logical_error_on_allocation = false; throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Memory tracker: allocations not allowed."); diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index b21454c9ed8..ceedf518608 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -14,7 +13,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -45,7 +46,7 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_) : path(std::move(path_)), fill(std::move(fill_)) { /// If file already exists. NOTE Minor race condition. - if (Poco::File(path).exists()) + if (fs::exists(path)) { std::string contents; { diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index b9bd6c75708..e6ccf405e9f 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -81,7 +81,7 @@ template template ReturnType ThreadPoolImpl::scheduleImpl(Job job, int priority, std::optional wait_microseconds) { - auto on_error = [&] + auto on_error = [&](const std::string & reason) { if constexpr (std::is_same_v) { @@ -91,7 +91,9 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, int priority, std::opti std::swap(exception, first_exception); std::rethrow_exception(exception); } - throw DB::Exception("Cannot schedule a task", DB::ErrorCodes::CANNOT_SCHEDULE_TASK); + throw DB::Exception(DB::ErrorCodes::CANNOT_SCHEDULE_TASK, + "Cannot schedule a task: {} (threads={}, jobs={})", reason, + threads.size(), scheduled_jobs); } else return false; @@ -105,13 +107,13 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, int priority, std::opti if (wait_microseconds) /// Check for optional. Condition is true if the optional is set and the value is zero. { if (!job_finished.wait_for(lock, std::chrono::microseconds(*wait_microseconds), pred)) - return on_error(); + return on_error(fmt::format("no free thread (timeout={})", *wait_microseconds)); } else job_finished.wait(lock, pred); if (shutdown) - return on_error(); + return on_error("shutdown"); /// We must not to allocate any memory after we emplaced a job in a queue. /// Because if an exception would be thrown, we won't notify a thread about job occurrence. @@ -126,7 +128,7 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, int priority, std::opti catch (...) { /// Most likely this is a std::bad_alloc exception - return on_error(); + return on_error("cannot allocate thread slot"); } try @@ -136,7 +138,7 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, int priority, std::opti catch (...) { threads.pop_front(); - return on_error(); + return on_error("cannot allocate thread"); } } diff --git a/src/Common/ZooKeeper/IKeeper.cpp b/src/Common/ZooKeeper/IKeeper.cpp index 94fd291bd12..4f0c5efe680 100644 --- a/src/Common/ZooKeeper/IKeeper.cpp +++ b/src/Common/ZooKeeper/IKeeper.cpp @@ -142,6 +142,8 @@ void GetRequest::addRootPath(const String & root_path) { Coordination::addRootPa void SetRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void ListRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void CheckRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } +void SetACLRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } +void GetACLRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void MultiRequest::addRootPath(const String & root_path) { diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index 2d947bb402c..30d816aad15 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -148,6 +148,40 @@ struct WatchResponse : virtual Response using WatchCallback = std::function; +struct SetACLRequest : virtual Request +{ + String path; + ACLs acls; + int32_t version = -1; + + void addRootPath(const String & root_path) override; + String getPath() const override { return path; } + size_t bytesSize() const override { return path.size() + sizeof(version) + acls.size() * sizeof(ACL); } +}; + +struct SetACLResponse : virtual Response +{ + Stat stat; + + size_t bytesSize() const override { return sizeof(Stat); } +}; + +struct GetACLRequest : virtual Request +{ + String path; + + void addRootPath(const String & root_path) override; + String getPath() const override { return path; } + size_t bytesSize() const override { return path.size(); } +}; + +struct GetACLResponse : virtual Response +{ + ACLs acl; + Stat stat; + size_t bytesSize() const override { return sizeof(Stat) + acl.size() * sizeof(ACL); } +}; + struct CreateRequest : virtual Request { String path; @@ -377,7 +411,7 @@ public: * - whenever you receive exception with ZSESSIONEXPIRED code or method isExpired returns true, * the ZooKeeper instance is no longer usable - you may only destroy it and probably create another. * - whenever session is expired or ZooKeeper instance is destroying, all callbacks are notified with special event. - * - data for callbacks must be alive when ZooKeeper instance is alive. + * - data for callbacks must be alive when ZooKeeper instance is alive, so try to avoid capturing references in callbacks, it's error-prone. */ class IKeeper { @@ -394,6 +428,9 @@ public: /// /// After the method is executed successfully, you must wait for callbacks /// (don't destroy callback data before it will be called). + /// TODO: The above line is the description of an error-prone interface. It's better + /// to replace callbacks with std::future results, so the caller shouldn't think about + /// lifetime of the callback data. /// /// All callbacks are executed sequentially (the execution of callbacks is serialized). /// diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 9f59da233fc..1ee70b0cc3f 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -4,6 +4,7 @@ #include "TestKeeper.h" #include +#include #include #include @@ -17,6 +18,7 @@ #define ZOOKEEPER_CONNECTION_TIMEOUT_MS 1000 +namespace fs = std::filesystem; namespace DB { @@ -238,24 +240,25 @@ Coordination::Error ZooKeeper::getChildrenImpl(const std::string & path, Strings Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - Coordination::Error code = Coordination::Error::ZOK; - Poco::Event event; + auto future_result = asyncTryGetChildrenNoThrow(path, watch_callback); - auto callback = [&](const Coordination::ListResponse & response) + if (future_result.wait_for(std::chrono::milliseconds(operation_timeout_ms)) != std::future_status::ready) { - SCOPE_EXIT(event.set()); - code = response.error; + impl->finalize(); + return Coordination::Error::ZOPERATIONTIMEOUT; + } + else + { + auto response = future_result.get(); + Coordination::Error code = response.error; if (code == Coordination::Error::ZOK) { res = response.names; if (stat) *stat = response.stat; } - }; - - impl->list(path, callback, watch_callback); - event.wait(); - return code; + return code; + } } Strings ZooKeeper::getChildren( @@ -298,20 +301,21 @@ Coordination::Error ZooKeeper::tryGetChildrenWatch(const std::string & path, Str Coordination::Error ZooKeeper::createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created) { - Coordination::Error code = Coordination::Error::ZOK; - Poco::Event event; + auto future_result = asyncTryCreateNoThrow(path, data, mode); - auto callback = [&](const Coordination::CreateResponse & response) + if (future_result.wait_for(std::chrono::milliseconds(operation_timeout_ms)) != std::future_status::ready) { - SCOPE_EXIT(event.set()); - code = response.error; + impl->finalize(); + return Coordination::Error::ZOPERATIONTIMEOUT; + } + else + { + auto response = future_result.get(); + Coordination::Error code = response.error; if (code == Coordination::Error::ZOK) path_created = response.path_created; - }; - - impl->create(path, data, mode & 1, mode & 2, {}, callback); /// TODO better mode - event.wait(); - return code; + return code; + } } std::string ZooKeeper::create(const std::string & path, const std::string & data, int32_t mode) @@ -366,19 +370,19 @@ void ZooKeeper::createAncestors(const std::string & path) Coordination::Error ZooKeeper::removeImpl(const std::string & path, int32_t version) { - Coordination::Error code = Coordination::Error::ZOK; - Poco::Event event; + auto future_result = asyncTryRemoveNoThrow(path, version); - auto callback = [&](const Coordination::RemoveResponse & response) + + if (future_result.wait_for(std::chrono::milliseconds(operation_timeout_ms)) != std::future_status::ready) { - SCOPE_EXIT(event.set()); - if (response.error != Coordination::Error::ZOK) - code = response.error; - }; - - impl->remove(path, version, callback); - event.wait(); - return code; + impl->finalize(); + return Coordination::Error::ZOPERATIONTIMEOUT; + } + else + { + auto response = future_result.get(); + return response.error; + } } void ZooKeeper::remove(const std::string & path, int32_t version) @@ -399,20 +403,22 @@ Coordination::Error ZooKeeper::tryRemove(const std::string & path, int32_t versi Coordination::Error ZooKeeper::existsImpl(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - Coordination::Error code = Coordination::Error::ZOK; - Poco::Event event; + auto future_result = asyncTryExistsNoThrow(path, watch_callback); - auto callback = [&](const Coordination::ExistsResponse & response) + if (future_result.wait_for(std::chrono::milliseconds(operation_timeout_ms)) != std::future_status::ready) { - SCOPE_EXIT(event.set()); - code = response.error; + impl->finalize(); + return Coordination::Error::ZOPERATIONTIMEOUT; + } + else + { + auto response = future_result.get(); + Coordination::Error code = response.error; if (code == Coordination::Error::ZOK && stat) *stat = response.stat; - }; - impl->exists(path, callback, watch_callback); - event.wait(); - return code; + return code; + } } bool ZooKeeper::exists(const std::string & path, Coordination::Stat * stat, const EventPtr & watch) @@ -431,24 +437,25 @@ bool ZooKeeper::existsWatch(const std::string & path, Coordination::Stat * stat, Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - Coordination::Error code = Coordination::Error::ZOK; - Poco::Event event; + auto future_result = asyncTryGetNoThrow(path, watch_callback); - auto callback = [&](const Coordination::GetResponse & response) + if (future_result.wait_for(std::chrono::milliseconds(operation_timeout_ms)) != std::future_status::ready) { - SCOPE_EXIT(event.set()); - code = response.error; + impl->finalize(); + return Coordination::Error::ZOPERATIONTIMEOUT; + } + else + { + auto response = future_result.get(); + Coordination::Error code = response.error; if (code == Coordination::Error::ZOK) { res = response.data; if (stat) *stat = response.stat; } - }; - - impl->get(path, callback, watch_callback); - event.wait(); - return code; + return code; + } } @@ -503,20 +510,22 @@ bool ZooKeeper::tryGetWatch( Coordination::Error ZooKeeper::setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat) { - Coordination::Error code = Coordination::Error::ZOK; - Poco::Event event; + auto future_result = asyncTrySetNoThrow(path, data, version); - auto callback = [&](const Coordination::SetResponse & response) + if (future_result.wait_for(std::chrono::milliseconds(operation_timeout_ms)) != std::future_status::ready) { - SCOPE_EXIT(event.set()); - code = response.error; + impl->finalize(); + return Coordination::Error::ZOPERATIONTIMEOUT; + } + else + { + auto response = future_result.get(); + Coordination::Error code = response.error; if (code == Coordination::Error::ZOK && stat) *stat = response.stat; - }; - impl->set(path, data, version, callback); - event.wait(); - return code; + return code; + } } void ZooKeeper::set(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat) @@ -553,19 +562,20 @@ Coordination::Error ZooKeeper::multiImpl(const Coordination::Requests & requests if (requests.empty()) return Coordination::Error::ZOK; - Coordination::Error code = Coordination::Error::ZOK; - Poco::Event event; + auto future_result = asyncTryMultiNoThrow(requests); - auto callback = [&](const Coordination::MultiResponse & response) + if (future_result.wait_for(std::chrono::milliseconds(operation_timeout_ms)) != std::future_status::ready) { - SCOPE_EXIT(event.set()); - code = response.error; + impl->finalize(); + return Coordination::Error::ZOPERATIONTIMEOUT; + } + else + { + auto response = future_result.get(); + Coordination::Error code = response.error; responses = response.responses; - }; - - impl->multi(requests, callback); - event.wait(); - return code; + return code; + } } Coordination::Responses ZooKeeper::multi(const Coordination::Requests & requests) @@ -593,7 +603,7 @@ void ZooKeeper::removeChildren(const std::string & path) Coordination::Requests ops; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1)); + ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); children.pop_back(); } multi(ops); @@ -609,9 +619,9 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path, const String & Coordination::Requests ops; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - removeChildrenRecursive(path + "/" + children.back()); + removeChildrenRecursive(fs::path(path) / children.back()); if (likely(keep_child_node.empty() || keep_child_node != children.back())) - ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1)); + ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); children.pop_back(); } multi(ops); @@ -629,7 +639,7 @@ void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, const Strin Strings batch; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - String child_path = path + "/" + children.back(); + String child_path = fs::path(path) / children.back(); tryRemoveChildrenRecursive(child_path); if (likely(keep_child_node.empty() || keep_child_node != children.back())) { @@ -700,9 +710,7 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & /// Use getData insteand of exists to avoid watch leak. impl->get(path, callback, watch); - if (!condition) - state->event.wait(); - else if (!state->event.tryWait(1000)) + if (!state->event.tryWait(1000)) continue; if (state->code == int32_t(Coordination::Error::ZNONODE)) @@ -752,8 +760,21 @@ std::future ZooKeeper::asyncCreate(const std::stri return future; } +std::future ZooKeeper::asyncTryCreateNoThrow(const std::string & path, const std::string & data, int32_t mode) +{ + auto promise = std::make_shared>(); + auto future = promise->get_future(); -std::future ZooKeeper::asyncGet(const std::string & path) + auto callback = [promise](const Coordination::CreateResponse & response) mutable + { + promise->set_value(response); + }; + + impl->create(path, data, mode & 1, mode & 2, {}, std::move(callback)); + return future; +} + +std::future ZooKeeper::asyncGet(const std::string & path, Coordination::WatchCallback watch_callback) { auto promise = std::make_shared>(); auto future = promise->get_future(); @@ -766,7 +787,21 @@ std::future ZooKeeper::asyncGet(const std::string & p promise->set_value(response); }; - impl->get(path, std::move(callback), {}); + impl->get(path, std::move(callback), watch_callback); + return future; +} + +std::future ZooKeeper::asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallback watch_callback) +{ + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto callback = [promise](const Coordination::GetResponse & response) mutable + { + promise->set_value(response); + }; + + impl->get(path, std::move(callback), watch_callback); return future; } @@ -788,7 +823,7 @@ std::future ZooKeeper::asyncTryGet(const std::string return future; } -std::future ZooKeeper::asyncExists(const std::string & path) +std::future ZooKeeper::asyncExists(const std::string & path, Coordination::WatchCallback watch_callback) { auto promise = std::make_shared>(); auto future = promise->get_future(); @@ -801,7 +836,21 @@ std::future ZooKeeper::asyncExists(const std::stri promise->set_value(response); }; - impl->exists(path, std::move(callback), {}); + impl->exists(path, std::move(callback), watch_callback); + return future; +} + +std::future ZooKeeper::asyncTryExistsNoThrow(const std::string & path, Coordination::WatchCallback watch_callback) +{ + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto callback = [promise](const Coordination::ExistsResponse & response) mutable + { + promise->set_value(response); + }; + + impl->exists(path, std::move(callback), watch_callback); return future; } @@ -822,7 +871,22 @@ std::future ZooKeeper::asyncSet(const std::string & p return future; } -std::future ZooKeeper::asyncGetChildren(const std::string & path) + +std::future ZooKeeper::asyncTrySetNoThrow(const std::string & path, const std::string & data, int32_t version) +{ + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto callback = [promise](const Coordination::SetResponse & response) mutable + { + promise->set_value(response); + }; + + impl->set(path, data, version, std::move(callback)); + return future; +} + +std::future ZooKeeper::asyncGetChildren(const std::string & path, Coordination::WatchCallback watch_callback) { auto promise = std::make_shared>(); auto future = promise->get_future(); @@ -835,7 +899,21 @@ std::future ZooKeeper::asyncGetChildren(const std::s promise->set_value(response); }; - impl->list(path, std::move(callback), {}); + impl->list(path, std::move(callback), watch_callback); + return future; +} + +std::future ZooKeeper::asyncTryGetChildrenNoThrow(const std::string & path, Coordination::WatchCallback watch_callback) +{ + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto callback = [promise](const Coordination::ListResponse & response) mutable + { + promise->set_value(response); + }; + + impl->list(path, std::move(callback), watch_callback); return future; } @@ -878,7 +956,21 @@ std::future ZooKeeper::asyncTryRemove(const std::s return future; } -std::future ZooKeeper::tryAsyncMulti(const Coordination::Requests & ops) +std::future ZooKeeper::asyncTryRemoveNoThrow(const std::string & path, int32_t version) +{ + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto callback = [promise](const Coordination::RemoveResponse & response) mutable + { + promise->set_value(response); + }; + + impl->remove(path, version, std::move(callback)); + return future; +} + +std::future ZooKeeper::asyncTryMultiNoThrow(const Coordination::Requests & ops) { auto promise = std::make_shared>(); auto future = promise->get_future(); diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 4a65ff070f7..7aafee52bf0 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -39,9 +39,6 @@ constexpr size_t MULTI_BATCH_SIZE = 100; /// watch notification. /// Callback-based watch interface is also provided. /// -/// Read-only methods retry retry_num times if recoverable errors like OperationTimeout -/// or ConnectionLoss are encountered. -/// /// Modifying methods do not retry, because it leads to problems of the double-delete type. /// /// Methods with names not starting at try- raise KeeperException on any error. @@ -220,39 +217,55 @@ public: /// auto result1 = future1.get(); /// auto result2 = future2.get(); /// - /// Future should not be destroyed before the result is gotten. + /// NoThrow versions never throw any exception on future.get(), even on SessionExpired error. using FutureCreate = std::future; FutureCreate asyncCreate(const std::string & path, const std::string & data, int32_t mode); + /// Like the previous one but don't throw any exceptions on future.get() + FutureCreate asyncTryCreateNoThrow(const std::string & path, const std::string & data, int32_t mode); using FutureGet = std::future; - FutureGet asyncGet(const std::string & path); - - FutureGet asyncTryGet(const std::string & path); + FutureGet asyncGet(const std::string & path, Coordination::WatchCallback watch_callback = {}); + /// Like the previous one but don't throw any exceptions on future.get() + FutureGet asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallback watch_callback = {}); using FutureExists = std::future; - FutureExists asyncExists(const std::string & path); + FutureExists asyncExists(const std::string & path, Coordination::WatchCallback watch_callback = {}); + /// Like the previous one but don't throw any exceptions on future.get() + FutureExists asyncTryExistsNoThrow(const std::string & path, Coordination::WatchCallback watch_callback = {}); using FutureGetChildren = std::future; - FutureGetChildren asyncGetChildren(const std::string & path); + FutureGetChildren asyncGetChildren(const std::string & path, Coordination::WatchCallback watch_callback = {}); + /// Like the previous one but don't throw any exceptions on future.get() + FutureGetChildren asyncTryGetChildrenNoThrow(const std::string & path, Coordination::WatchCallback watch_callback = {}); using FutureSet = std::future; FutureSet asyncSet(const std::string & path, const std::string & data, int32_t version = -1); + /// Like the previous one but don't throw any exceptions on future.get() + FutureSet asyncTrySetNoThrow(const std::string & path, const std::string & data, int32_t version = -1); using FutureRemove = std::future; FutureRemove asyncRemove(const std::string & path, int32_t version = -1); + /// Like the previous one but don't throw any exceptions on future.get() + FutureRemove asyncTryRemoveNoThrow(const std::string & path, int32_t version = -1); + using FutureMulti = std::future; + FutureMulti asyncMulti(const Coordination::Requests & ops); + /// Like the previous one but don't throw any exceptions on future.get() + FutureMulti asyncTryMultiNoThrow(const Coordination::Requests & ops); + + /// Very specific methods introduced without following general style. Implements + /// some custom throw/no throw logic on future.get(). + /// /// Doesn't throw in the following cases: /// * The node doesn't exist /// * The versions do not match /// * The node has children FutureRemove asyncTryRemove(const std::string & path, int32_t version = -1); - using FutureMulti = std::future; - FutureMulti asyncMulti(const Coordination::Requests & ops); - - /// Like the previous one but don't throw any exceptions on future.get() - FutureMulti tryAsyncMulti(const Coordination::Requests & ops); + /// Doesn't throw in the following cases: + /// * The node doesn't exist + FutureGet asyncTryGet(const std::string & path); void finalize(); @@ -262,7 +275,7 @@ private: void init(const std::string & implementation_, const Strings & hosts_, const std::string & identity_, int32_t session_timeout_ms_, int32_t operation_timeout_ms_, const std::string & chroot_); - /// The following methods don't throw exceptions but return error codes. + /// The following methods don't any throw exceptions but return error codes. Coordination::Error createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created); Coordination::Error removeImpl(const std::string & path, int32_t version); Coordination::Error getImpl( diff --git a/src/Common/examples/lru_hash_map_perf.cpp b/src/Common/examples/lru_hash_map_perf.cpp index c40548bea49..7793f4262ff 100644 --- a/src/Common/examples/lru_hash_map_perf.cpp +++ b/src/Common/examples/lru_hash_map_perf.cpp @@ -19,7 +19,7 @@ public: using list_type = std::list>; using map_type = std::unordered_map; - LRUHashMapBasic(size_t max_size_, bool preallocated = false) + explicit LRUHashMapBasic(size_t max_size_, bool preallocated = false) : hash_map(preallocated ? max_size_ : 32) , max_size(max_size_) { diff --git a/src/Common/examples/parallel_aggregation2.cpp b/src/Common/examples/parallel_aggregation2.cpp index e2ad36232de..496331e203d 100644 --- a/src/Common/examples/parallel_aggregation2.cpp +++ b/src/Common/examples/parallel_aggregation2.cpp @@ -94,7 +94,7 @@ struct AggregateIndependentWithSequentialKeysOptimization if (it != begin && *it == prev_key) { assert(place != nullptr); - updater(place->getMapped()); + updater(place->getMapped()); // NOLINT continue; } prev_key = *it; diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 1750fa586b7..4855500b776 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -6,10 +6,15 @@ # include #endif #include -#include -#include #include +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -21,6 +26,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int CANNOT_STATVFS; extern const int PATH_ACCESS_DENIED; + extern const int CANNOT_CREATE_FILE; } @@ -39,17 +45,13 @@ struct statvfs getStatVFS(const String & path) bool enoughSpaceInDirectory(const std::string & path [[maybe_unused]], size_t data_size [[maybe_unused]]) { -#if POCO_VERSION >= 0x01090000 - auto free_space = Poco::File(path).freeSpace(); + auto free_space = fs::space(path).free; return data_size <= free_space; -#else - return true; -#endif } std::unique_ptr createTemporaryFile(const std::string & path) { - Poco::File(path).createDirectories(); + fs::create_directories(path); /// NOTE: std::make_shared cannot use protected constructors return std::make_unique(path); @@ -129,3 +131,73 @@ bool pathStartsWith(const String & path, const String & prefix_path) } } + + +/// Copied from Poco::File +namespace FS +{ + +bool createFile(const std::string & path) +{ + int n = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + if (n != -1) + { + close(n); + return true; + } + DB::throwFromErrnoWithPath("Cannot create file: " + path, path, DB::ErrorCodes::CANNOT_CREATE_FILE); +} + +bool canRead(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + { + if (st.st_uid == geteuid()) + return (st.st_mode & S_IRUSR) != 0; + else if (st.st_gid == getegid()) + return (st.st_mode & S_IRGRP) != 0; + else + return (st.st_mode & S_IROTH) != 0 || geteuid() == 0; + } + DB::throwFromErrnoWithPath("Cannot check read access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + + +bool canWrite(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + { + if (st.st_uid == geteuid()) + return (st.st_mode & S_IWUSR) != 0; + else if (st.st_gid == getegid()) + return (st.st_mode & S_IWGRP) != 0; + else + return (st.st_mode & S_IWOTH) != 0 || geteuid() == 0; + } + DB::throwFromErrnoWithPath("Cannot check write access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + +time_t getModificationTime(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + return st.st_mtime; + DB::throwFromErrnoWithPath("Cannot check modification time for file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + +Poco::Timestamp getModificationTimestamp(const std::string & path) +{ + return Poco::Timestamp::fromEpochTime(getModificationTime(path)); +} + +void setModificationTime(const std::string & path, time_t time) +{ + struct utimbuf tb; + tb.actime = time; + tb.modtime = time; + if (utime(path.c_str(), &tb) != 0) + DB::throwFromErrnoWithPath("Cannot set modification time for file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} +} diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index 0ffbf19643d..b7525a64fae 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -36,3 +36,15 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p bool pathStartsWith(const String & path, const String & prefix_path); } + +namespace FS +{ +bool createFile(const std::string & path); + +bool canRead(const std::string & path); +bool canWrite(const std::string & path); + +time_t getModificationTime(const std::string & path); +Poco::Timestamp getModificationTimestamp(const std::string & path); +void setModificationTime(const std::string & path, time_t time); +} diff --git a/src/Common/remapExecutable.cpp b/src/Common/remapExecutable.cpp index 5418290b24f..a774b1028d5 100644 --- a/src/Common/remapExecutable.cpp +++ b/src/Common/remapExecutable.cpp @@ -26,6 +26,7 @@ namespace ErrorCodes namespace { +/// NOLINTNEXTLINE(cert-dcl50-cpp) __attribute__((__noinline__)) int64_t our_syscall(...) { __asm__ __volatile__ (R"( diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index a735a9d72d4..26d90427889 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #if defined(linux) || defined(__linux) || defined(__linux__) #include @@ -10,6 +10,8 @@ #include #endif +namespace fs = std::filesystem; + namespace DB { @@ -93,9 +95,9 @@ static bool renameat2(const std::string &, const std::string &, int) static void renameNoReplaceFallback(const std::string & old_path, const std::string & new_path) { /// NOTE it's unsafe - if (Poco::File{new_path}.exists()) + if (fs::exists(new_path)) throw Exception("File " + new_path + " exists", ErrorCodes::FILE_ALREADY_EXISTS); - Poco::File{old_path}.renameTo(new_path); + fs::rename(old_path, new_path); } /// Do not use [[noreturn]] to avoid warnings like "code will never be executed" in other places diff --git a/src/Common/tests/gtest_global_context.h b/src/Common/tests/gtest_global_context.h index 30ebf1dbca9..9bd7c2490d6 100644 --- a/src/Common/tests/gtest_global_context.h +++ b/src/Common/tests/gtest_global_context.h @@ -5,7 +5,7 @@ struct ContextHolder { DB::SharedContextHolder shared_context; - DB::ContextPtr context; + DB::ContextMutablePtr context; ContextHolder() : shared_context(DB::Context::createShared()) diff --git a/src/Coordination/ACLMap.cpp b/src/Coordination/ACLMap.cpp new file mode 100644 index 00000000000..863dfdec281 --- /dev/null +++ b/src/Coordination/ACLMap.cpp @@ -0,0 +1,95 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +size_t ACLMap::ACLsHash::operator()(const Coordination::ACLs & acls) const +{ + SipHash hash; + for (const auto & acl : acls) + { + hash.update(acl.permissions); + hash.update(acl.scheme); + hash.update(acl.id); + } + return hash.get64(); +} + +bool ACLMap::ACLsComparator::operator()(const Coordination::ACLs & left, const Coordination::ACLs & right) const +{ + if (left.size() != right.size()) + return false; + + for (size_t i = 0; i < left.size(); ++i) + { + if (left[i].permissions != right[i].permissions) + return false; + + if (left[i].scheme != right[i].scheme) + return false; + + if (left[i].id != right[i].id) + return false; + } + return true; +} + +uint64_t ACLMap::convertACLs(const Coordination::ACLs & acls) +{ + if (acl_to_num.count(acls)) + return acl_to_num[acls]; + + /// Start from one + auto index = acl_to_num.size() + 1; + + acl_to_num[acls] = index; + num_to_acl[index] = acls; + + return index; +} + +Coordination::ACLs ACLMap::convertNumber(uint64_t acls_id) const +{ + if (acls_id == 0) + return Coordination::ACLs{}; + + if (!num_to_acl.count(acls_id)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown ACL id {}. It's a bug", acls_id); + + return num_to_acl.at(acls_id); +} + +void ACLMap::addMapping(uint64_t acls_id, const Coordination::ACLs & acls) +{ + num_to_acl[acls_id] = acls; + acl_to_num[acls] = acls_id; +} + +void ACLMap::addUsage(uint64_t acl_id) +{ + usage_counter[acl_id]++; +} + +void ACLMap::removeUsage(uint64_t acl_id) +{ + if (usage_counter.count(acl_id) == 0) + return; + + usage_counter[acl_id]--; + + if (usage_counter[acl_id] == 0) + { + auto acls = num_to_acl[acl_id]; + num_to_acl.erase(acl_id); + acl_to_num.erase(acls); + usage_counter.erase(acl_id); + } +} + +} diff --git a/src/Coordination/ACLMap.h b/src/Coordination/ACLMap.h new file mode 100644 index 00000000000..2313b3e7cd3 --- /dev/null +++ b/src/Coordination/ACLMap.h @@ -0,0 +1,54 @@ +#pragma once +#include +#include +#include + + +namespace DB +{ + +/// Simple mapping of different ACLs to sequentially growing numbers +/// Allows to store single number instead of vector of ACLs on disk and in memory. +class ACLMap +{ +private: + struct ACLsHash + { + size_t operator()(const Coordination::ACLs & acls) const; + }; + + struct ACLsComparator + { + bool operator()(const Coordination::ACLs & left, const Coordination::ACLs & right) const; + }; + + using ACLToNumMap = std::unordered_map; + + using NumToACLMap = std::unordered_map; + + using UsageCounter = std::unordered_map; + + ACLToNumMap acl_to_num; + NumToACLMap num_to_acl; + UsageCounter usage_counter; +public: + + /// Convert ACL to number. If it's new ACL than adds it to map + /// with new id. + uint64_t convertACLs(const Coordination::ACLs & acls); + + /// Convert number to ACL vector. If number is unknown for map + /// than throws LOGICAL ERROR + Coordination::ACLs convertNumber(uint64_t acls_id) const; + /// Mapping from numbers to ACLs vectors. Used during serialization. + const NumToACLMap & getMapping() const { return num_to_acl; } + + /// Add mapping to ACLMap. Used during deserialization. + void addMapping(uint64_t acls_id, const Coordination::ACLs & acls); + + /// Add/remove usage of some id. Used to remove unused ACLs. + void addUsage(uint64_t acl_id); + void removeUsage(uint64_t acl_id); +}; + +} diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index ba1664b23da..6ec9b17d0a7 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -296,7 +296,7 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin break; } else if (changelog_description.from_log_index > start_to_read_from) - LOG_WARNING(log, "Don't have required amount of reserved log records. Need to read from {}, smalled available log index on disk {}.", start_to_read_from, changelog_description.from_log_index); + LOG_WARNING(log, "Don't have required amount of reserved log records. Need to read from {}, smallest available log index on disk {}.", start_to_read_from, changelog_description.from_log_index); } started = true; diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index ba904a535d0..b6b0ab8cb72 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB { @@ -26,6 +27,7 @@ namespace ErrorCodes extern const int NO_ELEMENTS_IN_CONFIG; extern const int SUPPORT_IS_DISABLED; extern const int LOGICAL_ERROR; + extern const int INVALID_CONFIG_PARAMETER; } namespace @@ -75,6 +77,20 @@ std::string getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/snapshots"; } +std::string checkAndGetSuperdigest(const Poco::Util::AbstractConfiguration & config) +{ + if (!config.has("keeper_server.superdigest")) + return ""; + + auto user_and_digest = config.getString("keeper_server.superdigest"); + std::vector scheme_and_id; + boost::split(scheme_and_id, user_and_digest, [](char c) { return c == ':'; }); + if (scheme_and_id.size() != 2 || scheme_and_id[0] != "super") + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Incorrect superdigest in keeper_server config. Must be 'super:base64string'"); + + return user_and_digest; +} + } KeeperServer::KeeperServer( @@ -89,7 +105,8 @@ KeeperServer::KeeperServer( , state_machine(nuraft::cs_new( responses_queue_, snapshots_queue_, getSnapshotsPathFromConfig(config, standalone_keeper), - coordination_settings)) + coordination_settings, + checkAndGetSuperdigest(config))) , state_manager(nuraft::cs_new(server_id, "keeper_server", config, coordination_settings, standalone_keeper)) , log(&Poco::Logger::get("KeeperServer")) { diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h index 421be331537..282a7b48dfb 100644 --- a/src/Coordination/KeeperServer.h +++ b/src/Coordination/KeeperServer.h @@ -48,7 +48,6 @@ private: void shutdownRaftServer(); - public: KeeperServer( int server_id_, diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index d223f2fe7d3..7520f9b3ba2 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -56,14 +56,7 @@ namespace writeBinary(node.data, out); /// Serialize ACL - writeBinary(node.acls.size(), out); - for (const auto & acl : node.acls) - { - writeBinary(acl.permissions, out); - writeBinary(acl.scheme, out); - writeBinary(acl.id, out); - } - + writeBinary(node.acl_id, out); writeBinary(node.is_sequental, out); /// Serialize stat writeBinary(node.stat.czxid, out); @@ -81,21 +74,33 @@ namespace writeBinary(node.seq_num, out); } - void readNode(KeeperStorage::Node & node, ReadBuffer & in) + void readNode(KeeperStorage::Node & node, ReadBuffer & in, SnapshotVersion version, ACLMap & acl_map) { readBinary(node.data, in); - /// Deserialize ACL - size_t acls_size; - readBinary(acls_size, in); - for (size_t i = 0; i < acls_size; ++i) + if (version >= SnapshotVersion::V1) { - Coordination::ACL acl; - readBinary(acl.permissions, in); - readBinary(acl.scheme, in); - readBinary(acl.id, in); - node.acls.push_back(acl); + readBinary(node.acl_id, in); } + else if (version == SnapshotVersion::V0) + { + /// Deserialize ACL + size_t acls_size; + readBinary(acls_size, in); + Coordination::ACLs acls; + for (size_t i = 0; i < acls_size; ++i) + { + Coordination::ACL acl; + readBinary(acl.permissions, in); + readBinary(acl.scheme, in); + readBinary(acl.id, in); + acls.push_back(acl); + } + node.acl_id = acl_map.convertACLs(acls); + } + + acl_map.addUsage(node.acl_id); + readBinary(node.is_sequental, in); /// Deserialize stat @@ -137,6 +142,22 @@ void KeeperStorageSnapshot::serialize(const KeeperStorageSnapshot & snapshot, Wr writeBinary(static_cast(snapshot.version), out); serializeSnapshotMetadata(snapshot.snapshot_meta, out); writeBinary(snapshot.session_id, out); + + /// Serialize ACLs MAP + writeBinary(snapshot.acl_map.size(), out); + for (const auto & [acl_id, acls] : snapshot.acl_map) + { + writeBinary(acl_id, out); + writeBinary(acls.size(), out); + for (const auto & acl : acls) + { + writeBinary(acl.permissions, out); + writeBinary(acl.scheme, out); + writeBinary(acl.id, out); + } + } + + /// Serialize data tree writeBinary(snapshot.snapshot_container_size, out); size_t counter = 0; for (auto it = snapshot.begin; counter < snapshot.snapshot_container_size; ++counter) @@ -157,12 +178,24 @@ void KeeperStorageSnapshot::serialize(const KeeperStorageSnapshot & snapshot, Wr ++it; } + /// Serialize sessions size_t size = snapshot.session_and_timeout.size(); writeBinary(size, out); for (const auto & [session_id, timeout] : snapshot.session_and_timeout) { writeBinary(session_id, out); writeBinary(timeout, out); + + KeeperStorage::AuthIDs ids; + if (snapshot.session_and_auth.count(session_id)) + ids = snapshot.session_and_auth.at(session_id); + + writeBinary(ids.size(), out); + for (const auto & [scheme, id] : ids) + { + writeBinary(scheme, out); + writeBinary(id, out); + } } } @@ -170,15 +203,42 @@ SnapshotMetadataPtr KeeperStorageSnapshot::deserialize(KeeperStorage & storage, { uint8_t version; readBinary(version, in); - if (static_cast(version) > SnapshotVersion::V0) + SnapshotVersion current_version = static_cast(version); + if (current_version > SnapshotVersion::V1) throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unsupported snapshot version {}", version); SnapshotMetadataPtr result = deserializeSnapshotMetadata(in); int64_t session_id; readBinary(session_id, in); + storage.zxid = result->get_last_log_idx(); storage.session_id_counter = session_id; + if (current_version >= SnapshotVersion::V1) + { + size_t acls_map_size; + readBinary(acls_map_size, in); + size_t current_map_size = 0; + while (current_map_size < acls_map_size) + { + uint64_t acl_id; + readBinary(acl_id, in); + size_t acls_size; + readBinary(acls_size, in); + Coordination::ACLs acls; + for (size_t i = 0; i < acls_size; ++i) + { + Coordination::ACL acl; + readBinary(acl.permissions, in); + readBinary(acl.scheme, in); + readBinary(acl.id, in); + acls.push_back(acl); + } + storage.acl_map.addMapping(acl_id, acls); + current_map_size++; + } + } + size_t snapshot_container_size; readBinary(snapshot_container_size, in); @@ -187,8 +247,8 @@ SnapshotMetadataPtr KeeperStorageSnapshot::deserialize(KeeperStorage & storage, { std::string path; readBinary(path, in); - KeeperStorage::Node node; - readNode(node, in); + KeeperStorage::Node node{}; + readNode(node, in, current_version, storage.acl_map); storage.container.insertOrReplace(path, node); if (node.stat.ephemeralOwner != 0) storage.ephemerals[node.stat.ephemeralOwner].insert(path); @@ -215,6 +275,26 @@ SnapshotMetadataPtr KeeperStorageSnapshot::deserialize(KeeperStorage & storage, readBinary(active_session_id, in); readBinary(timeout, in); storage.addSessionID(active_session_id, timeout); + + if (current_version >= SnapshotVersion::V1) + { + size_t session_auths_size; + readBinary(session_auths_size, in); + + KeeperStorage::AuthIDs ids; + size_t session_auth_counter = 0; + while (session_auth_counter < session_auths_size) + { + String scheme, id; + readBinary(scheme, in); + readBinary(id, in); + ids.emplace_back(KeeperStorage::AuthID{scheme, id}); + + session_auth_counter++; + } + if (!ids.empty()) + storage.session_and_auth[active_session_id] = ids; + } current_session_size++; } @@ -230,6 +310,8 @@ KeeperStorageSnapshot::KeeperStorageSnapshot(KeeperStorage * storage_, uint64_t snapshot_container_size = storage->container.snapshotSize(); begin = storage->getSnapshotIteratorBegin(); session_and_timeout = storage->getActiveSessions(); + acl_map = storage->acl_map.getMapping(); + session_and_auth = storage->session_and_auth; } KeeperStorageSnapshot::KeeperStorageSnapshot(KeeperStorage * storage_, const SnapshotMetadataPtr & snapshot_meta_) @@ -241,6 +323,8 @@ KeeperStorageSnapshot::KeeperStorageSnapshot(KeeperStorage * storage_, const Sna snapshot_container_size = storage->container.snapshotSize(); begin = storage->getSnapshotIteratorBegin(); session_and_timeout = storage->getActiveSessions(); + acl_map = storage->acl_map.getMapping(); + session_and_auth = storage->session_and_auth; } KeeperStorageSnapshot::~KeeperStorageSnapshot() @@ -248,9 +332,10 @@ KeeperStorageSnapshot::~KeeperStorageSnapshot() storage->disableSnapshotMode(); } -KeeperSnapshotManager::KeeperSnapshotManager(const std::string & snapshots_path_, size_t snapshots_to_keep_, size_t storage_tick_time_) +KeeperSnapshotManager::KeeperSnapshotManager(const std::string & snapshots_path_, size_t snapshots_to_keep_, const std::string & superdigest_, size_t storage_tick_time_) : snapshots_path(snapshots_path_) , snapshots_to_keep(snapshots_to_keep_) + , superdigest(superdigest_) , storage_tick_time(storage_tick_time_) { namespace fs = std::filesystem; @@ -337,7 +422,7 @@ SnapshotMetaAndStorage KeeperSnapshotManager::deserializeSnapshotFromBuffer(nura { ReadBufferFromNuraftBuffer reader(buffer); CompressedReadBuffer compressed_reader(reader); - auto storage = std::make_unique(storage_tick_time); + auto storage = std::make_unique(storage_tick_time, superdigest); auto snapshot_metadata = KeeperStorageSnapshot::deserialize(*storage, compressed_reader); return std::make_pair(snapshot_metadata, std::move(storage)); } diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h index 95d1ce831d4..3dbd7c9328e 100644 --- a/src/Coordination/KeeperSnapshotManager.h +++ b/src/Coordination/KeeperSnapshotManager.h @@ -13,6 +13,7 @@ using SnapshotMetadataPtr = std::shared_ptr; enum SnapshotVersion : uint8_t { V0 = 0, + V1 = 1, /// with ACL map }; struct KeeperStorageSnapshot @@ -29,12 +30,14 @@ public: KeeperStorage * storage; - SnapshotVersion version = SnapshotVersion::V0; + SnapshotVersion version = SnapshotVersion::V1; SnapshotMetadataPtr snapshot_meta; int64_t session_id; size_t snapshot_container_size; KeeperStorage::Container::const_iterator begin; SessionAndTimeout session_and_timeout; + KeeperStorage::SessionAndAuth session_and_auth; + std::unordered_map acl_map; }; using KeeperStorageSnapshotPtr = std::shared_ptr; @@ -46,7 +49,7 @@ using SnapshotMetaAndStorage = std::pair; class KeeperSnapshotManager { public: - KeeperSnapshotManager(const std::string & snapshots_path_, size_t snapshots_to_keep_, size_t storage_tick_time_ = 500); + KeeperSnapshotManager(const std::string & snapshots_path_, size_t snapshots_to_keep_, const std::string & superdigest_ = "", size_t storage_tick_time_ = 500); SnapshotMetaAndStorage restoreFromLatestSnapshot(); @@ -77,6 +80,7 @@ private: const std::string snapshots_path; const size_t snapshots_to_keep; std::map existing_snapshots; + const std::string superdigest; size_t storage_tick_time; }; diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index b5b393523bf..a76b86a8171 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -36,13 +36,19 @@ KeeperStorage::RequestForSession parseRequest(nuraft::buffer & data) return request_for_session; } -KeeperStateMachine::KeeperStateMachine(ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, const std::string & snapshots_path_, const CoordinationSettingsPtr & coordination_settings_) + KeeperStateMachine::KeeperStateMachine( + ResponsesQueue & responses_queue_, + SnapshotsQueue & snapshots_queue_, + const std::string & snapshots_path_, + const CoordinationSettingsPtr & coordination_settings_, + const std::string & superdigest_) : coordination_settings(coordination_settings_) - , snapshot_manager(snapshots_path_, coordination_settings->snapshots_to_keep, coordination_settings->dead_session_check_period_ms.totalMicroseconds()) + , snapshot_manager(snapshots_path_, coordination_settings->snapshots_to_keep, superdigest_, coordination_settings->dead_session_check_period_ms.totalMicroseconds()) , responses_queue(responses_queue_) , snapshots_queue(snapshots_queue_) , last_committed_idx(0) , log(&Poco::Logger::get("KeeperStateMachine")) + , superdigest(superdigest_) { } @@ -85,7 +91,7 @@ void KeeperStateMachine::init() } if (!storage) - storage = std::make_unique(coordination_settings->dead_session_check_period_ms.totalMilliseconds()); + storage = std::make_unique(coordination_settings->dead_session_check_period_ms.totalMilliseconds(), superdigest); } nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, nuraft::buffer & data) diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index 8712adaf4b1..fb46f507baf 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -16,7 +16,10 @@ using SnapshotsQueue = ConcurrentBoundedQueue; class KeeperStateMachine : public nuraft::state_machine { public: - KeeperStateMachine(ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, const std::string & snapshots_path_, const CoordinationSettingsPtr & coordination_settings_); + KeeperStateMachine( + ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, + const std::string & snapshots_path_, const CoordinationSettingsPtr & coordination_settings_, + const std::string & superdigest_ = ""); void init(); @@ -84,6 +87,8 @@ private: /// Last committed Raft log number. std::atomic last_committed_idx; Poco::Logger * log; + + const std::string superdigest; }; } diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 9e8d2a124e9..3ae29edb77a 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include namespace DB { @@ -31,6 +34,100 @@ static std::string getBaseName(const String & path) return std::string{&path[basename_start + 1], path.length() - basename_start - 1}; } +static String base64Encode(const String & decoded) +{ + std::ostringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + ostr.exceptions(std::ios::failbit); + Poco::Base64Encoder encoder(ostr); + encoder.rdbuf()->setLineLength(0); + encoder << decoded; + encoder.close(); + return ostr.str(); +} + +static String getSHA1(const String & userdata) +{ + Poco::SHA1Engine engine; + engine.update(userdata); + const auto & digest_id = engine.digest(); + return String{digest_id.begin(), digest_id.end()}; +} + +static String generateDigest(const String & userdata) +{ + std::vector user_password; + boost::split(user_password, userdata, [](char c) { return c == ':'; }); + return user_password[0] + ":" + base64Encode(getSHA1(user_password[1])); +} + +static bool checkACL(int32_t permission, const Coordination::ACLs & node_acls, const std::vector & session_auths) +{ + if (node_acls.empty()) + return true; + + for (const auto & session_auth : session_auths) + if (session_auth.scheme == "super") + return true; + + for (const auto & node_acl : node_acls) + { + if (node_acl.permissions & permission) + { + if (node_acl.scheme == "world" && node_acl.id == "anyone") + return true; + + for (const auto & session_auth : session_auths) + if (node_acl.scheme == session_auth.scheme && node_acl.id == session_auth.id) + return true; + } + } + + return false; +} + +static bool fixupACL( + const std::vector & request_acls, + const std::vector & current_ids, + std::vector & result_acls) +{ + if (request_acls.empty()) + return true; + + bool valid_found = false; + for (const auto & request_acl : request_acls) + { + if (request_acl.scheme == "auth") + { + for (const auto & current_id : current_ids) + { + valid_found = true; + Coordination::ACL new_acl = request_acl; + new_acl.scheme = current_id.scheme; + new_acl.id = current_id.id; + result_acls.push_back(new_acl); + } + } + else if (request_acl.scheme == "world" && request_acl.id == "anyone") + { + /// We don't need to save default ACLs + valid_found = true; + } + else if (request_acl.scheme == "digest") + { + Coordination::ACL new_acl = request_acl; + + /// Bad auth + if (std::count(new_acl.id.begin(), new_acl.id.end(), ':') != 1) + return false; + + valid_found = true; + new_acl.id = generateDigest(new_acl.id); + result_acls.push_back(new_acl); + } + } + return valid_found; +} + static KeeperStorage::ResponsesForSessions processWatchesImpl(const String & path, KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches, Coordination::Event event_type) { KeeperStorage::ResponsesForSessions result; @@ -67,8 +164,9 @@ static KeeperStorage::ResponsesForSessions processWatchesImpl(const String & pat return result; } -KeeperStorage::KeeperStorage(int64_t tick_time_ms) +KeeperStorage::KeeperStorage(int64_t tick_time_ms, const String & superdigest_) : session_expiry_queue(tick_time_ms) + , superdigest(superdigest_) { container.insert("/", Node()); } @@ -82,8 +180,9 @@ struct KeeperStorageRequest explicit KeeperStorageRequest(const Coordination::ZooKeeperRequestPtr & zk_request_) : zk_request(zk_request_) {} - virtual std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & ephemerals, int64_t zxid, int64_t session_id) const = 0; + virtual std::pair process(KeeperStorage & storage, int64_t zxid, int64_t session_id) const = 0; virtual KeeperStorage::ResponsesForSessions processWatches(KeeperStorage::Watches & /*watches*/, KeeperStorage::Watches & /*list_watches*/) const { return {}; } + virtual bool checkAuth(KeeperStorage & /*storage*/, int64_t /*session_id*/) const { return true; } virtual ~KeeperStorageRequest() = default; }; @@ -91,7 +190,7 @@ struct KeeperStorageRequest struct KeeperStorageHeartbeatRequest final : public KeeperStorageRequest { using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container & /* container */, KeeperStorage::Ephemerals & /* ephemerals */, int64_t /* zxid */, int64_t /* session_id */) const override + std::pair process(KeeperStorage & /* storage */, int64_t /* zxid */, int64_t /* session_id */) const override { return {zk_request->makeResponse(), {}}; } @@ -100,7 +199,7 @@ struct KeeperStorageHeartbeatRequest final : public KeeperStorageRequest struct KeeperStorageSyncRequest final : public KeeperStorageRequest { using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container & /* container */, KeeperStorage::Ephemerals & /* ephemerals */, int64_t /* zxid */, int64_t /* session_id */) const override + std::pair process(KeeperStorage & /* storage */, int64_t /* zxid */, int64_t /* session_id */) const override { auto response = zk_request->makeResponse(); dynamic_cast(*response).path @@ -118,8 +217,28 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest return processWatchesImpl(zk_request->getPath(), watches, list_watches, Coordination::Event::CREATED); } - std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & ephemerals, int64_t zxid, int64_t session_id) const override + bool checkAuth(KeeperStorage & storage, int64_t session_id) const override { + auto & container = storage.container; + auto parent_path = parentPath(zk_request->getPath()); + + auto it = container.find(parent_path); + if (it == container.end()) + return true; + + const auto & node_acls = storage.acl_map.convertNumber(it->value.acl_id); + if (node_acls.empty()) + return true; + + const auto & session_auths = storage.session_and_auth[session_id]; + return checkACL(Coordination::ACL::Create, node_acls, session_auths); + } + + std::pair process(KeeperStorage & storage, int64_t zxid, int64_t session_id) const override + { + auto & container = storage.container; + auto & ephemerals = storage.ephemerals; + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Undo undo; Coordination::ZooKeeperCreateResponse & response = dynamic_cast(*response_ptr); @@ -144,7 +263,21 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest } else { + auto & session_auth_ids = storage.session_and_auth[session_id]; + KeeperStorage::Node created_node; + + Coordination::ACLs node_acls; + if (!fixupACL(request.acls, session_auth_ids, node_acls)) + { + response.error = Coordination::Error::ZINVALIDACL; + return {response_ptr, {}}; + } + + uint64_t acl_id = storage.acl_map.convertACLs(node_acls); + storage.acl_map.addUsage(acl_id); + + created_node.acl_id = acl_id; created_node.stat.czxid = zxid; created_node.stat.mzxid = zxid; created_node.stat.ctime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); @@ -154,6 +287,7 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest created_node.stat.ephemeralOwner = request.is_ephemeral ? session_id : 0; created_node.data = request.data; created_node.is_sequental = request.is_sequential; + std::string path_created = request.path; if (request.is_sequential) @@ -183,13 +317,15 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest if (request.is_ephemeral) ephemerals[session_id].emplace(path_created); - undo = [&container, &ephemerals, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path] + undo = [&storage, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id] { - container.erase(path_created); - if (is_ephemeral) - ephemerals[session_id].erase(path_created); + storage.container.erase(path_created); + storage.acl_map.removeUsage(acl_id); - container.updateValue(parent_path, [child_path] (KeeperStorage::Node & undo_parent) + if (is_ephemeral) + storage.ephemerals[session_id].erase(path_created); + + storage.container.updateValue(parent_path, [child_path] (KeeperStorage::Node & undo_parent) { --undo_parent.stat.cversion; --undo_parent.stat.numChildren; @@ -208,9 +344,26 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest struct KeeperStorageGetRequest final : public KeeperStorageRequest { - using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & /* ephemerals */, int64_t /* zxid */, int64_t /* session_id */) const override + + bool checkAuth(KeeperStorage & storage, int64_t session_id) const override { + auto & container = storage.container; + auto it = container.find(zk_request->getPath()); + if (it == container.end()) + return true; + + const auto & node_acls = storage.acl_map.convertNumber(it->value.acl_id); + if (node_acls.empty()) + return true; + + const auto & session_auths = storage.session_and_auth[session_id]; + return checkACL(Coordination::ACL::Read, node_acls, session_auths); + } + + using KeeperStorageRequest::KeeperStorageRequest; + std::pair process(KeeperStorage & storage, int64_t /* zxid */, int64_t /* session_id */) const override + { + auto & container = storage.container; Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperGetResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperGetRequest & request = dynamic_cast(*zk_request); @@ -233,9 +386,27 @@ struct KeeperStorageGetRequest final : public KeeperStorageRequest struct KeeperStorageRemoveRequest final : public KeeperStorageRequest { - using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & ephemerals, int64_t /*zxid*/, int64_t /*session_id*/) const override + bool checkAuth(KeeperStorage & storage, int64_t session_id) const override { + auto & container = storage.container; + auto it = container.find(parentPath(zk_request->getPath())); + if (it == container.end()) + return true; + + const auto & node_acls = storage.acl_map.convertNumber(it->value.acl_id); + if (node_acls.empty()) + return true; + + const auto & session_auths = storage.session_and_auth[session_id]; + return checkACL(Coordination::ACL::Delete, node_acls, session_auths); + } + + using KeeperStorageRequest::KeeperStorageRequest; + std::pair process(KeeperStorage & storage, int64_t /*zxid*/, int64_t /*session_id*/) const override + { + auto & container = storage.container; + auto & ephemerals = storage.ephemerals; + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperRemoveResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperRemoveRequest & request = dynamic_cast(*zk_request); @@ -265,6 +436,8 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest ephemerals.erase(ephemerals_it); } + storage.acl_map.removeUsage(prev_node.acl_id); + auto child_basename = getBaseName(it->key); container.updateValue(parentPath(request.path), [&child_basename] (KeeperStorage::Node & parent) { @@ -277,13 +450,15 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest container.erase(request.path); - undo = [prev_node, &container, &ephemerals, path = request.path, child_basename] + undo = [prev_node, &storage, path = request.path, child_basename] { if (prev_node.stat.ephemeralOwner != 0) - ephemerals[prev_node.stat.ephemeralOwner].emplace(path); + storage.ephemerals[prev_node.stat.ephemeralOwner].emplace(path); - container.insert(path, prev_node); - container.updateValue(parentPath(path), [&child_basename] (KeeperStorage::Node & parent) + storage.acl_map.addUsage(prev_node.acl_id); + + storage.container.insert(path, prev_node); + storage.container.updateValue(parentPath(path), [&child_basename] (KeeperStorage::Node & parent) { ++parent.stat.numChildren; --parent.stat.cversion; @@ -304,8 +479,10 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest struct KeeperStorageExistsRequest final : public KeeperStorageRequest { using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & /* ephemerals */, int64_t /*zxid*/, int64_t /* session_id */) const override + std::pair process(KeeperStorage & storage, int64_t /*zxid*/, int64_t /* session_id */) const override { + auto & container = storage.container; + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperExistsResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperExistsRequest & request = dynamic_cast(*zk_request); @@ -327,9 +504,26 @@ struct KeeperStorageExistsRequest final : public KeeperStorageRequest struct KeeperStorageSetRequest final : public KeeperStorageRequest { - using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & /* ephemerals */, int64_t zxid, int64_t /* session_id */) const override + bool checkAuth(KeeperStorage & storage, int64_t session_id) const override { + auto & container = storage.container; + auto it = container.find(zk_request->getPath()); + if (it == container.end()) + return true; + + const auto & node_acls = storage.acl_map.convertNumber(it->value.acl_id); + if (node_acls.empty()) + return true; + + const auto & session_auths = storage.session_and_auth[session_id]; + return checkACL(Coordination::ACL::Write, node_acls, session_auths); + } + + using KeeperStorageRequest::KeeperStorageRequest; + std::pair process(KeeperStorage & storage, int64_t zxid, int64_t /* session_id */) const override + { + auto & container = storage.container; + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperSetResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperSetRequest & request = dynamic_cast(*zk_request); @@ -387,9 +581,25 @@ struct KeeperStorageSetRequest final : public KeeperStorageRequest struct KeeperStorageListRequest final : public KeeperStorageRequest { - using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & /* ephemerals */, int64_t /*zxid*/, int64_t /*session_id*/) const override + bool checkAuth(KeeperStorage & storage, int64_t session_id) const override { + auto & container = storage.container; + auto it = container.find(zk_request->getPath()); + if (it == container.end()) + return true; + + const auto & node_acls = storage.acl_map.convertNumber(it->value.acl_id); + if (node_acls.empty()) + return true; + + const auto & session_auths = storage.session_and_auth[session_id]; + return checkACL(Coordination::ACL::Read, node_acls, session_auths); + } + + using KeeperStorageRequest::KeeperStorageRequest; + std::pair process(KeeperStorage & storage, int64_t /*zxid*/, int64_t /*session_id*/) const override + { + auto & container = storage.container; Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperListResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperListRequest & request = dynamic_cast(*zk_request); @@ -416,9 +626,26 @@ struct KeeperStorageListRequest final : public KeeperStorageRequest struct KeeperStorageCheckRequest final : public KeeperStorageRequest { - using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & /* ephemerals */, int64_t /*zxid*/, int64_t /*session_id*/) const override + bool checkAuth(KeeperStorage & storage, int64_t session_id) const override { + auto & container = storage.container; + auto it = container.find(zk_request->getPath()); + if (it == container.end()) + return true; + + const auto & node_acls = storage.acl_map.convertNumber(it->value.acl_id); + if (node_acls.empty()) + return true; + + const auto & session_auths = storage.session_and_auth[session_id]; + return checkACL(Coordination::ACL::Read, node_acls, session_auths); + } + + using KeeperStorageRequest::KeeperStorageRequest; + std::pair process(KeeperStorage & storage, int64_t /*zxid*/, int64_t /*session_id*/) const override + { + auto & container = storage.container; + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperCheckResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperCheckRequest & request = dynamic_cast(*zk_request); @@ -442,6 +669,14 @@ struct KeeperStorageCheckRequest final : public KeeperStorageRequest struct KeeperStorageMultiRequest final : public KeeperStorageRequest { + bool checkAuth(KeeperStorage & storage, int64_t session_id) const override + { + for (const auto & concrete_request : concrete_requests) + if (!concrete_request->checkAuth(storage, session_id)) + return false; + return true; + } + std::vector concrete_requests; explicit KeeperStorageMultiRequest(const Coordination::ZooKeeperRequestPtr & zk_request_) : KeeperStorageRequest(zk_request_) @@ -473,7 +708,7 @@ struct KeeperStorageMultiRequest final : public KeeperStorageRequest } } - std::pair process(KeeperStorage::Container & container, KeeperStorage::Ephemerals & ephemerals, int64_t zxid, int64_t session_id) const override + std::pair process(KeeperStorage & storage, int64_t zxid, int64_t session_id) const override { Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperMultiResponse & response = dynamic_cast(*response_ptr); @@ -484,7 +719,7 @@ struct KeeperStorageMultiRequest final : public KeeperStorageRequest size_t i = 0; for (const auto & concrete_request : concrete_requests) { - auto [ cur_response, undo_action ] = concrete_request->process(container, ephemerals, zxid, session_id); + auto [ cur_response, undo_action ] = concrete_request->process(storage, zxid, session_id); response.responses[i] = cur_response; if (cur_response->error != Coordination::Error::ZOK) @@ -541,19 +776,44 @@ struct KeeperStorageMultiRequest final : public KeeperStorageRequest struct KeeperStorageCloseRequest final : public KeeperStorageRequest { using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container &, KeeperStorage::Ephemerals &, int64_t, int64_t) const override + std::pair process(KeeperStorage &, int64_t, int64_t) const override { throw DB::Exception("Called process on close request", ErrorCodes::LOGICAL_ERROR); } }; -/// Dummy implementation TODO: implement simple ACL struct KeeperStorageAuthRequest final : public KeeperStorageRequest { using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage::Container &, KeeperStorage::Ephemerals &, int64_t, int64_t) const override + std::pair process(KeeperStorage & storage, int64_t /*zxid*/, int64_t session_id) const override { + Coordination::ZooKeeperAuthRequest & auth_request = dynamic_cast(*zk_request); Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + Coordination::ZooKeeperAuthResponse & auth_response = dynamic_cast(*response_ptr); + auto & sessions_and_auth = storage.session_and_auth; + + if (auth_request.scheme != "digest" || std::count(auth_request.data.begin(), auth_request.data.end(), ':') != 1) + { + auth_response.error = Coordination::Error::ZAUTHFAILED; + } + else + { + auto digest = generateDigest(auth_request.data); + if (digest == storage.superdigest) + { + KeeperStorage::AuthID auth{"super", ""}; + sessions_and_auth[session_id].emplace_back(auth); + } + else + { + KeeperStorage::AuthID auth{auth_request.scheme, digest}; + auto & session_ids = sessions_and_auth[session_id]; + if (std::find(session_ids.begin(), session_ids.end(), auth) == session_ids.end()) + sessions_and_auth[session_id].emplace_back(auth); + } + + } + return { response_ptr, {} }; } }; @@ -668,6 +928,9 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(const Coordina ephemerals.erase(it); } clearDeadWatches(session_id); + auto auth_it = session_and_auth.find(session_id); + if (auth_it != session_and_auth.end()) + session_and_auth.erase(auth_it); /// Finish connection auto response = std::make_shared(); @@ -680,7 +943,7 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(const Coordina else if (zk_request->getOpNum() == Coordination::OpNum::Heartbeat) { KeeperStorageRequestPtr storage_request = KeeperWrapperFactory::instance().get(zk_request); - auto [response, _] = storage_request->process(container, ephemerals, zxid, session_id); + auto [response, _] = storage_request->process(*this, zxid, session_id); response->xid = zk_request->xid; response->zxid = getZXID(); @@ -689,7 +952,18 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(const Coordina else { KeeperStorageRequestPtr storage_request = KeeperWrapperFactory::instance().get(zk_request); - auto [response, _] = storage_request->process(container, ephemerals, zxid, session_id); + Coordination::ZooKeeperResponsePtr response; + + if (!storage_request->checkAuth(*this, session_id)) + { + response = zk_request->makeResponse(); + /// Original ZooKeeper always throws no auth, even when user provided some credentials + response->error = Coordination::Error::ZNOAUTH; + } + else + { + std::tie(response, std::ignore) = storage_request->process(*this, zxid, session_id); + } if (zk_request->has_watch) { diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index 585426a7441..7c90a9bd661 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,7 @@ public: struct Node { String data; - Coordination::ACLs acls{}; + uint64_t acl_id = 0; /// 0 -- no ACL by default bool is_sequental = false; Coordination::Stat stat{}; int32_t seq_num = 0; @@ -51,6 +52,17 @@ public: Coordination::ZooKeeperRequestPtr request; }; + struct AuthID + { + std::string scheme; + std::string id; + + bool operator==(const AuthID & other) const + { + return scheme == other.scheme && id == other.id; + } + }; + using RequestsForSessions = std::vector; using Container = SnapshotableHashTable; @@ -58,6 +70,11 @@ public: using SessionAndWatcher = std::unordered_map>; using SessionIDs = std::vector; + /// Just vector of SHA1 from user:password + using AuthIDs = std::vector; + using SessionAndAuth = std::unordered_map; + SessionAndAuth session_and_auth; + using Watches = std::map; Container container; @@ -65,6 +82,7 @@ public: SessionAndWatcher sessions_and_watchers; SessionExpiryQueue session_expiry_queue; SessionAndTimeout session_and_timeout; + ACLMap acl_map; int64_t zxid{0}; bool finalized{false}; @@ -79,8 +97,10 @@ public: return zxid; } + const String superdigest; + public: - KeeperStorage(int64_t tick_time_ms); + KeeperStorage(int64_t tick_time_ms, const String & superdigest_); int64_t getSessionID(int64_t session_timeout_ms) { diff --git a/src/Coordination/tests/gtest_for_build.cpp b/src/Coordination/tests/gtest_for_build.cpp index a0313651c21..9a744d2bbed 100644 --- a/src/Coordination/tests/gtest_for_build.cpp +++ b/src/Coordination/tests/gtest_for_build.cpp @@ -911,7 +911,7 @@ TEST(CoordinationTest, TestStorageSnapshotSimple) ChangelogDirTest test("./snapshots"); DB::KeeperSnapshotManager manager("./snapshots", 3); - DB::KeeperStorage storage(500); + DB::KeeperStorage storage(500, ""); addNode(storage, "/hello", "world", 1); addNode(storage, "/hello/somepath", "somedata", 3); storage.session_id_counter = 5; @@ -958,7 +958,7 @@ TEST(CoordinationTest, TestStorageSnapshotMoreWrites) ChangelogDirTest test("./snapshots"); DB::KeeperSnapshotManager manager("./snapshots", 3); - DB::KeeperStorage storage(500); + DB::KeeperStorage storage(500, ""); storage.getSessionID(130); for (size_t i = 0; i < 50; ++i) @@ -998,7 +998,7 @@ TEST(CoordinationTest, TestStorageSnapshotManySnapshots) ChangelogDirTest test("./snapshots"); DB::KeeperSnapshotManager manager("./snapshots", 3); - DB::KeeperStorage storage(500); + DB::KeeperStorage storage(500, ""); storage.getSessionID(130); for (size_t j = 1; j <= 5; ++j) @@ -1035,7 +1035,7 @@ TEST(CoordinationTest, TestStorageSnapshotMode) { ChangelogDirTest test("./snapshots"); DB::KeeperSnapshotManager manager("./snapshots", 3); - DB::KeeperStorage storage(500); + DB::KeeperStorage storage(500, ""); for (size_t i = 0; i < 50; ++i) { addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i)); @@ -1086,7 +1086,7 @@ TEST(CoordinationTest, TestStorageSnapshotBroken) { ChangelogDirTest test("./snapshots"); DB::KeeperSnapshotManager manager("./snapshots", 3); - DB::KeeperStorage storage(500); + DB::KeeperStorage storage(500, ""); for (size_t i = 0; i < 50; ++i) { addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i)); diff --git a/src/Core/ExternalTable.h b/src/Core/ExternalTable.h index fcefa3d7fe3..f682bf9b27f 100644 --- a/src/Core/ExternalTable.h +++ b/src/Core/ExternalTable.h @@ -80,10 +80,10 @@ public: /// Parsing of external table used when sending tables via http /// The `handlePart` function will be called for each table passed, /// so it's also necessary to call `clean` at the end of the `handlePart`. -class ExternalTablesHandler : public HTMLForm::PartHandler, BaseExternalTable, WithContext +class ExternalTablesHandler : public HTMLForm::PartHandler, BaseExternalTable, WithMutableContext { public: - ExternalTablesHandler(ContextPtr context_, const Poco::Net::NameValueCollection & params_) : WithContext(context_), params(params_) {} + ExternalTablesHandler(ContextMutablePtr context_, const Poco::Net::NameValueCollection & params_) : WithMutableContext(context_), params(params_) {} void handlePart(const Poco::Net::MessageHeader & header, ReadBuffer & stream) override; diff --git a/src/Core/MySQL/Authentication.cpp b/src/Core/MySQL/Authentication.cpp index b8dff9972c7..0258f9f94af 100644 --- a/src/Core/MySQL/Authentication.cpp +++ b/src/Core/MySQL/Authentication.cpp @@ -73,7 +73,7 @@ Native41::Native41(const String & password, const String & auth_plugin_data) } void Native41::authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool, const Poco::Net::SocketAddress & address) { if (!auth_response) @@ -136,7 +136,7 @@ Sha256Password::Sha256Password(RSA & public_key_, RSA & private_key_, Poco::Logg } void Sha256Password::authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) { if (!auth_response) diff --git a/src/Core/MySQL/Authentication.h b/src/Core/MySQL/Authentication.h index 5358e2da737..acbda2bdb58 100644 --- a/src/Core/MySQL/Authentication.h +++ b/src/Core/MySQL/Authentication.h @@ -32,7 +32,7 @@ public: virtual String getAuthPluginData() = 0; virtual void authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) = 0; }; @@ -49,7 +49,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool /* is_secure_connection */, const Poco::Net::SocketAddress & address) override; private: @@ -69,7 +69,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) override; private: diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 9e1afcd187c..7c5800f5a8f 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -802,7 +802,7 @@ protected: static void setPassword( const String & user_name, const String & password, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { @@ -821,7 +821,7 @@ protected: public: virtual void authenticate( const String & user_name, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) = 0; @@ -835,7 +835,7 @@ class NoPasswordAuth : public AuthenticationMethod public: void authenticate( const String & user_name, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { @@ -853,7 +853,7 @@ class CleartextPasswordAuth : public AuthenticationMethod public: void authenticate( const String & user_name, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { @@ -896,7 +896,7 @@ public: void authenticate( const String & user_name, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 321957e2c90..bc308b56599 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -415,7 +415,6 @@ class IColumn; M(Bool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \ M(String, function_implementation, "", "Choose function implementation for specific target or variant (experimental). If empty enable all of them.", 0) \ M(Bool, allow_experimental_geo_types, false, "Allow geo data types such as Point, Ring, Polygon, MultiPolygon", 0) \ - M(Bool, allow_experimental_bigint_types, false, "Allow Int128, Int256, UInt256 and Decimal256 types", 0) \ M(Bool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ M(Bool, cast_keep_nullable, false, "CAST operator keep Nullable for result data type", 0) \ M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ @@ -458,13 +457,6 @@ class IColumn; M(UInt64, distributed_ddl_entry_format_version, 1, "Version of DDL entry to write into ZooKeeper", 0) \ M(UInt64, external_storage_max_read_rows, 0, "Limit maximum number of rows when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializeMySQL. If equal to 0, this setting is disabled", 0) \ M(UInt64, external_storage_max_read_bytes, 0, "Limit maximum number of bytes when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializeMySQL. If equal to 0, this setting is disabled", 0) \ - \ - /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ - \ - M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ - M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing. Will be removed after 2021-03-31", 0) \ - M(Bool, enable_debug_queries, false, "Enabled debug queries, but now is obsolete", 0) \ - M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing. Will be removed after 2021-02-12", 0) \ M(UnionMode, union_default_mode, UnionMode::Unspecified, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) \ M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ @@ -473,11 +465,20 @@ class IColumn; M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \ M(Bool, query_plan_filter_push_down, true, "Allow to push down filter by predicate query plan step", 0) \ \ - M(Bool, database_replicated_ddl_output, true, "Obsolete setting, does nothing. Will be removed after 2021-09-08", 0) \ - M(HandleKafkaErrorMode, handle_kafka_error_mode, HandleKafkaErrorMode::DEFAULT, "How to handle errors for Kafka engine. Passible values: default, stream.", 0) \ M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ - M(Bool, allow_experimental_funnel_functions, true, "Enable experimental functions for funnel analysis.", 0) \ + \ + \ + /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ + M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete setting, does nothing.", 0) \ + M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing.", 0) \ + M(Bool, enable_debug_queries, false, "Obsolete setting, does nothing.", 0) \ + M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing.", 0) \ + M(Bool, allow_experimental_funnel_functions, true, "Obsolete setting, does nothing.", 0) \ + M(Bool, allow_experimental_bigint_types, true, "Obsolete setting, does nothing.", 0) \ + M(HandleKafkaErrorMode, handle_kafka_error_mode, HandleKafkaErrorMode::DEFAULT, "Obsolete setting, does nothing.", 0) \ + M(Bool, database_replicated_ddl_output, true, "Obsolete setting, does nothing.", 0) \ + /** The section above is for obsolete settings. Do not add anything there. */ // End of COMMON_SETTINGS diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.h b/src/DataStreams/PushingToViewsBlockOutputStream.h index 2ae941efc2e..552a0a3452a 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -54,8 +54,8 @@ private: }; std::vector views; - ContextPtr select_context; - ContextPtr insert_context; + ContextMutablePtr select_context; + ContextMutablePtr insert_context; void process(const Block & block, ViewInfo & view); }; diff --git a/src/DataTypes/DataTypeDecimalBase.cpp b/src/DataTypes/DataTypeDecimalBase.cpp index 683710b8880..830b22b5943 100644 --- a/src/DataTypes/DataTypeDecimalBase.cpp +++ b/src/DataTypes/DataTypeDecimalBase.cpp @@ -19,11 +19,11 @@ namespace ErrorCodes { } -bool decimalCheckComparisonOverflow(ContextPtr context) +bool decimalCheckComparisonOverflow(ContextConstPtr context) { return context->getSettingsRef().decimal_check_overflow; } -bool decimalCheckArithmeticOverflow(ContextPtr context) +bool decimalCheckArithmeticOverflow(ContextConstPtr context) { return context->getSettingsRef().decimal_check_overflow; } diff --git a/src/DataTypes/DataTypeDecimalBase.h b/src/DataTypes/DataTypeDecimalBase.h index aea837f890f..17b73a8e3d6 100644 --- a/src/DataTypes/DataTypeDecimalBase.h +++ b/src/DataTypes/DataTypeDecimalBase.h @@ -18,8 +18,8 @@ namespace ErrorCodes extern const int ARGUMENT_OUT_OF_BOUND; } -bool decimalCheckComparisonOverflow(ContextPtr context); -bool decimalCheckArithmeticOverflow(ContextPtr context); +bool decimalCheckComparisonOverflow(ContextConstPtr context); +bool decimalCheckArithmeticOverflow(ContextConstPtr context); inline UInt32 leastDecimalPrecisionFor(TypeIndex int_type) { diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index b30efb163ab..9ebc061d6f9 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -307,26 +307,30 @@ SerializationPtr DataTypeTuple::getSubcolumnSerialization( SerializationPtr DataTypeTuple::doGetDefaultSerialization() const { SerializationTuple::ElementSerializations serializations(elems.size()); + bool use_explicit_names = have_explicit_names && serialize_names; for (size_t i = 0; i < elems.size(); ++i) { + String elem_name = use_explicit_names ? names[i] : toString(i + 1); auto serialization = elems[i]->getDefaultSerialization(); - serializations[i] = std::make_shared(serialization, names[i]); + serializations[i] = std::make_shared(serialization, elem_name); } - return std::make_shared(std::move(serializations), have_explicit_names); + return std::make_shared(std::move(serializations), use_explicit_names); } SerializationPtr DataTypeTuple::getSerialization(const String & column_name, const StreamExistenceCallback & callback) const { SerializationTuple::ElementSerializations serializations(elems.size()); + bool use_explicit_names = have_explicit_names && serialize_names; for (size_t i = 0; i < elems.size(); ++i) { - auto subcolumn_name = Nested::concatenateName(column_name, names[i]); + String elem_name = use_explicit_names ? names[i] : toString(i + 1); + auto subcolumn_name = Nested::concatenateName(column_name, elem_name); auto serializaion = elems[i]->getSerialization(subcolumn_name, callback); - serializations[i] = std::make_shared(serializaion, names[i]); + serializations[i] = std::make_shared(serializaion, elem_name); } - return std::make_shared(std::move(serializations), have_explicit_names); + return std::make_shared(std::move(serializations), use_explicit_names); } static DataTypePtr create(const ASTPtr & arguments) diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 85526cd98de..e0f5d3302df 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -341,8 +341,6 @@ struct WhichDataType constexpr bool isNullable() const { return idx == TypeIndex::Nullable; } constexpr bool isFunction() const { return idx == TypeIndex::Function; } constexpr bool isAggregateFunction() const { return idx == TypeIndex::AggregateFunction; } - - constexpr bool IsBigIntOrDeimal() const { return isInt128() || isUInt128() || isInt256() || isUInt256() || isDecimal256(); } }; /// IDataType helpers (alternative for IDataType virtual methods with single point of truth) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index e939f76f372..6b8c470861d 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -1,7 +1,5 @@ #include #include -#include -#include #include #include #include @@ -13,6 +11,8 @@ #include #include +namespace fs = std::filesystem; + namespace DB { namespace ErrorCodes @@ -37,12 +37,12 @@ public: DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, const String & logger_name, ContextPtr context_) : DatabaseOrdinary(name_, std::move(metadata_path_), "store/", logger_name, context_) - , path_to_table_symlinks(getContext()->getPath() + "data/" + escapeForFileName(name_) + "/") - , path_to_metadata_symlink(getContext()->getPath() + "metadata/" + escapeForFileName(name_)) + , path_to_table_symlinks(fs::path(getContext()->getPath()) / "data" / escapeForFileName(name_) / "") + , path_to_metadata_symlink(fs::path(getContext()->getPath()) / "metadata" / escapeForFileName(name_)) , db_uuid(uuid) { assert(db_uuid != UUIDHelpers::Nil); - Poco::File(path_to_table_symlinks).createDirectories(); + fs::create_directories(path_to_table_symlinks); tryCreateMetadataSymlink(); } @@ -73,14 +73,14 @@ void DatabaseAtomic::drop(ContextPtr) assert(tables.empty()); try { - Poco::File(path_to_metadata_symlink).remove(); - Poco::File(path_to_table_symlinks).remove(true); + fs::remove(path_to_metadata_symlink); + fs::remove_all(path_to_table_symlinks); } catch (...) { LOG_WARNING(log, getCurrentExceptionMessage(true)); } - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } void DatabaseAtomic::attachTable(const String & name, const StoragePtr & table, const String & relative_table_path) @@ -132,8 +132,8 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na /// (it's more likely to lost connection, than to fail before applying local changes). /// TODO better detection and recovery - Poco::File(table_metadata_path).renameTo(table_metadata_path_drop); /// Mark table as dropped - DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw + fs::rename(table_metadata_path, table_metadata_path_drop); /// Mark table as dropped + DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw table_name_to_path.erase(table_name); } @@ -316,7 +316,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); if (locked_uuid) DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; @@ -416,11 +416,11 @@ UUID DatabaseAtomic::tryGetTableUUID(const String & table_name) const return UUIDHelpers::Nil; } -void DatabaseAtomic::loadStoredObjects(ContextPtr local_context, bool has_force_restore_data_flag, bool force_attach) +void DatabaseAtomic::loadStoredObjects(ContextMutablePtr local_context, bool has_force_restore_data_flag, bool force_attach) { /// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken if (has_force_restore_data_flag) - Poco::File(path_to_table_symlinks).remove(true); + fs::remove_all(path_to_table_symlinks); DatabaseOrdinary::loadStoredObjects(local_context, has_force_restore_data_flag, force_attach); @@ -432,7 +432,7 @@ void DatabaseAtomic::loadStoredObjects(ContextPtr local_context, bool has_force_ table_names = table_name_to_path; } - Poco::File(path_to_table_symlinks).createDirectories(); + fs::create_directories(path_to_table_symlinks); for (const auto & table : table_names) tryCreateSymlink(table.first, table.second, true); } @@ -443,9 +443,9 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String & try { String link = path_to_table_symlinks + escapeForFileName(table_name); - Poco::File data = Poco::Path(getContext()->getPath()).makeAbsolute().toString() + actual_data_path; - if (!if_data_path_exist || data.exists()) - data.linkTo(link, Poco::File::LINK_SYMBOLIC); + fs::path data = fs::canonical(getContext()->getPath()) / actual_data_path; + if (!if_data_path_exist || fs::exists(data)) + fs::create_directory_symlink(data, link); } catch (...) { @@ -458,7 +458,7 @@ void DatabaseAtomic::tryRemoveSymlink(const String & table_name) try { String path = path_to_table_symlinks + escapeForFileName(table_name); - Poco::File{path}.remove(); + fs::remove(path); } catch (...) { @@ -471,17 +471,17 @@ void DatabaseAtomic::tryCreateMetadataSymlink() /// Symlinks in data/db_name/ directory and metadata/db_name/ are not used by ClickHouse, /// it's needed only for convenient introspection. assert(path_to_metadata_symlink != metadata_path); - Poco::File metadata_symlink(path_to_metadata_symlink); - if (metadata_symlink.exists()) + fs::path metadata_symlink(path_to_metadata_symlink); + if (fs::exists(metadata_symlink)) { - if (!metadata_symlink.isLink()) + if (!fs::is_symlink(metadata_symlink)) throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Directory {} exists", path_to_metadata_symlink); } else { try { - Poco::File{metadata_path}.linkTo(path_to_metadata_symlink, Poco::File::LINK_SYMBOLIC); + fs::create_directory_symlink(metadata_path, path_to_metadata_symlink); } catch (...) { @@ -495,7 +495,7 @@ void DatabaseAtomic::renameDatabase(const String & new_name) /// CREATE, ATTACH, DROP, DETACH and RENAME DATABASE must hold DDLGuard try { - Poco::File(path_to_metadata_symlink).remove(); + fs::remove(path_to_metadata_symlink); } catch (...) { @@ -526,7 +526,7 @@ void DatabaseAtomic::renameDatabase(const String & new_name) path_to_table_symlinks = getContext()->getPath() + "data/" + new_name_escaped + "/"; } - Poco::File(old_path_to_table_symlinks).renameTo(path_to_table_symlinks); + fs::rename(old_path_to_table_symlinks, path_to_table_symlinks); tryCreateMetadataSymlink(); } diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h index a6acd10b656..c5dedf00d23 100644 --- a/src/Databases/DatabaseAtomic.h +++ b/src/Databases/DatabaseAtomic.h @@ -47,7 +47,7 @@ public: DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) override; - void loadStoredObjects(ContextPtr context, bool has_force_restore_data_flag, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr context, bool has_force_restore_data_flag, bool force_attach) override; /// Atomic database cannot be detached if there is detached table which still in use void assertCanBeDetached(bool cleanup) override; diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 4626d4f47b6..e09860e79b6 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -11,10 +11,9 @@ #include #include #include -#include -#include #include #include +#include #if !defined(ARCADIA_BUILD) # include "config_core.h" @@ -40,6 +39,8 @@ #include #endif +namespace fs = std::filesystem; + namespace DB { @@ -58,11 +59,12 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m try { /// Creates store/xxx/ for Atomic - Poco::File(Poco::Path(metadata_path).makeParent()).createDirectories(); + fs::create_directories(fs::path(metadata_path).parent_path()); + /// Before 20.7 it's possible that .sql metadata file does not exist for some old database. /// In this case Ordinary database is created on server startup if the corresponding metadata directory exists. /// So we should remove metadata directory if database creation failed. - created = Poco::File(metadata_path).createDirectory(); + created = fs::create_directory(metadata_path); DatabasePtr impl = getImpl(create, metadata_path, context); @@ -74,11 +76,8 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m } catch (...) { - Poco::File metadata_dir(metadata_path); - - if (created && metadata_dir.exists()) - metadata_dir.remove(true); - + if (created && fs::exists(metadata_path)) + fs::remove_all(metadata_path); throw; } } diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 80179de104a..22afaa971a1 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -12,8 +12,9 @@ #include #include #include -#include +#include +namespace fs = std::filesystem; namespace DB { @@ -35,7 +36,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, void DatabaseLazy::loadStoredObjects( - ContextPtr local_context, + ContextMutablePtr local_context, bool /* has_force_restore_data_flag */, bool /*force_attach*/) { @@ -43,8 +44,8 @@ void DatabaseLazy::loadStoredObjects( { const std::string table_name = file_name.substr(0, file_name.size() - 4); - auto detached_permanently_flag = Poco::File(getMetadataPath() + "/" + file_name + detached_suffix); - if (detached_permanently_flag.exists()) + fs::path detached_permanently_flag = fs::path(getMetadataPath()) / (file_name + detached_suffix); + if (fs::exists(detached_permanently_flag)) { LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); return; @@ -228,7 +229,7 @@ StoragePtr DatabaseLazy::loadTable(const String & table_name) const LOG_DEBUG(log, "Load table {} to cache.", backQuote(table_name)); - const String table_metadata_path = getMetadataPath() + "/" + escapeForFileName(table_name) + ".sql"; + const String table_metadata_path = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + ".sql"); try { diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 99a71b342fa..949869f4509 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -27,7 +27,7 @@ public: bool canContainDistributedTables() const override { return false; } void loadStoredObjects( - ContextPtr context, + ContextMutablePtr context, bool has_force_restore_data_flag, bool force_attach) override; void createTable( diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index f21a145df55..c0af027e027 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -4,9 +4,9 @@ #include #include #include -#include #include +namespace fs = std::filesystem; namespace DB { @@ -42,9 +42,9 @@ void DatabaseMemory::dropTable( try { table->drop(); - Poco::File table_data_dir{getTableDataPath(table_name)}; - if (table_data_dir.exists()) - table_data_dir.remove(true); + fs::path table_data_dir{getTableDataPath(table_name)}; + if (fs::exists(table_data_dir)) + fs::remove_all(table_data_dir); } catch (...) { diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 3db5314bbd3..620e560b64c 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -14,14 +14,14 @@ #include #include #include - #include -#include - #include #include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -45,7 +45,7 @@ std::pair createTableFromAST( ASTCreateQuery ast_create_query, const String & database_name, const String & table_data_path_relative, - ContextPtr context, + ContextMutablePtr context, bool has_force_restore_data_flag) { ast_create_query.attach = true; @@ -201,8 +201,8 @@ DatabaseOnDisk::DatabaseOnDisk( , metadata_path(metadata_path_) , data_path(data_path_) { - Poco::File(local_context->getPath() + data_path).createDirectories(); - Poco::File(metadata_path).createDirectories(); + fs::create_directories(local_context->getPath() + data_path); + fs::create_directories(metadata_path); } @@ -245,7 +245,7 @@ void DatabaseOnDisk::createTable( if (!create.attach) checkMetadataFilenameAvailability(table_name); - if (create.attach && Poco::File(table_metadata_path).exists()) + if (create.attach && fs::exists(table_metadata_path)) { ASTPtr ast_detached = parseQueryFromMetadata(log, local_context, table_metadata_path); auto & create_detached = ast_detached->as(); @@ -285,10 +285,10 @@ void DatabaseOnDisk::removeDetachedPermanentlyFlag(ContextPtr, const String & ta { try { - auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix); + fs::path detached_permanently_flag(table_metadata_path + detached_suffix); - if (detached_permanently_flag.exists()) - detached_permanently_flag.remove(); + if (fs::exists(detached_permanently_flag)) + fs::remove(detached_permanently_flag); } catch (Exception & e) { @@ -308,11 +308,11 @@ void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const Stora /// If it was ATTACH query and file with table metadata already exist /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one. - Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); + fs::rename(table_metadata_tmp_path, table_metadata_path); } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); throw; } } @@ -321,10 +321,10 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr, const String & table_nam { auto table = detachTable(table_name); - Poco::File detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix); + fs::path detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix); try { - detached_permanently_flag.createFile(); + FS::createFile(detached_permanently_flag); } catch (Exception & e) { @@ -350,25 +350,25 @@ void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_na bool renamed = false; try { - Poco::File(table_metadata_path).renameTo(table_metadata_path_drop); + fs::rename(table_metadata_path, table_metadata_path_drop); renamed = true; table->drop(); table->is_dropped = true; - Poco::File table_data_dir{local_context->getPath() + table_data_path_relative}; - if (table_data_dir.exists()) - table_data_dir.remove(true); + fs::path table_data_dir(local_context->getPath() + table_data_path_relative); + if (fs::exists(table_data_dir)) + fs::remove_all(table_data_dir); } catch (...) { LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); attachTable(table_name, table, table_data_path_relative); if (renamed) - Poco::File(table_metadata_path_drop).renameTo(table_metadata_path); + fs::rename(table_metadata_path_drop, table_metadata_path); throw; } - Poco::File(table_metadata_path_drop).remove(); + fs::remove(table_metadata_path_drop); } void DatabaseOnDisk::checkMetadataFilenameAvailability(const String & to_table_name) const @@ -381,11 +381,11 @@ void DatabaseOnDisk::checkMetadataFilenameAvailabilityUnlocked(const String & to { String table_metadata_path = getObjectMetadataPath(to_table_name); - if (Poco::File(table_metadata_path).exists()) + if (fs::exists(table_metadata_path)) { - auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix); + fs::path detached_permanently_flag(table_metadata_path + detached_suffix); - if (detached_permanently_flag.exists()) + if (fs::exists(detached_permanently_flag)) throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached permanently)", backQuote(database_name), backQuote(to_table_name)); else throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached)", backQuote(database_name), backQuote(to_table_name)); @@ -463,7 +463,7 @@ void DatabaseOnDisk::renameTable( /// Now table data are moved to new database, so we must add metadata and attach table to new database to_database.createTable(local_context, to_table_name, table, attach_query); - Poco::File(table_metadata_path).remove(); + fs::remove(table_metadata_path); if (from_atomic_to_ordinary) { @@ -528,8 +528,8 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const void DatabaseOnDisk::drop(ContextPtr local_context) { assert(tables.empty()); - Poco::File(local_context->getPath() + getDataPath()).remove(false); - Poco::File(getMetadataPath()).remove(false); + fs::remove(local_context->getPath() + getDataPath()); + fs::remove(getMetadataPath()); } String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const @@ -540,10 +540,9 @@ String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_name) const { String table_metadata_path = getObjectMetadataPath(object_name); - Poco::File meta_file(table_metadata_path); - if (meta_file.exists()) - return meta_file.getLastModified().epochTime(); + if (fs::exists(table_metadata_path)) + return FS::getModificationTime(table_metadata_path); else return static_cast(0); } @@ -555,56 +554,57 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat assert(getUUID() == UUIDHelpers::Nil); static const char * tmp_drop_ext = ".sql.tmp_drop"; const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext)); - if (Poco::File(local_context->getPath() + getDataPath() + '/' + object_name).exists()) + + if (fs::exists(local_context->getPath() + getDataPath() + '/' + object_name)) { - Poco::File(getMetadataPath() + file_name).renameTo(getMetadataPath() + object_name + ".sql"); + fs::rename(getMetadataPath() + file_name, getMetadataPath() + object_name + ".sql"); LOG_WARNING(log, "Object {} was not dropped previously and will be restored", backQuote(object_name)); process_metadata_file(object_name + ".sql"); } else { LOG_INFO(log, "Removing file {}", getMetadataPath() + file_name); - Poco::File(getMetadataPath() + file_name).remove(); + fs::remove(getMetadataPath() + file_name); } }; /// Metadata files to load: name and flag for .tmp_drop files std::set> metadata_files; - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) + fs::directory_iterator dir_end; + for (fs::directory_iterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) { + String file_name = dir_it->path().filename(); /// For '.svn', '.gitignore' directory and similar. - if (dir_it.name().at(0) == '.') + if (file_name.at(0) == '.') continue; /// There are .sql.bak files - skip them. - if (endsWith(dir_it.name(), ".sql.bak")) + if (endsWith(file_name, ".sql.bak")) continue; /// Permanently detached table flag - if (endsWith(dir_it.name(), ".sql.detached")) + if (endsWith(file_name, ".sql.detached")) continue; - if (endsWith(dir_it.name(), ".sql.tmp_drop")) + if (endsWith(file_name, ".sql.tmp_drop")) { /// There are files that we tried to delete previously - metadata_files.emplace(dir_it.name(), false); + metadata_files.emplace(file_name, false); } - else if (endsWith(dir_it.name(), ".sql.tmp")) + else if (endsWith(file_name, ".sql.tmp")) { /// There are files .sql.tmp - delete - LOG_INFO(log, "Removing file {}", dir_it->path()); - Poco::File(dir_it->path()).remove(); + LOG_INFO(log, "Removing file {}", dir_it->path().string()); + fs::remove(dir_it->path()); } - else if (endsWith(dir_it.name(), ".sql")) + else if (endsWith(file_name, ".sql")) { /// The required files have names like `table_name.sql` - metadata_files.emplace(dir_it.name(), true); + metadata_files.emplace(file_name, true); } else - throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(), - ErrorCodes::INCORRECT_FILE_NAME); + throw Exception(ErrorCodes::INCORRECT_FILE_NAME, "Incorrect file extension: {} in metadata directory {}", file_name, getMetadataPath()); } /// Read and parse metadata in parallel @@ -651,7 +651,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata( { if (logger) LOG_ERROR(logger, "File {} is empty. Removing.", metadata_file_path); - Poco::File(metadata_file_path).remove(); + fs::remove(metadata_file_path); return nullptr; } @@ -670,8 +670,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata( auto & create = ast->as(); if (!create.table.empty() && create.uuid != UUIDHelpers::Nil) { - String table_name = Poco::Path(metadata_file_path).makeFile().getBaseName(); - table_name = unescapeForFileName(table_name); + String table_name = unescapeForFileName(fs::path(metadata_file_path).stem()); if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER && logger) LOG_WARNING( diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index 677465e306e..73b69dd43a5 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -16,7 +16,7 @@ std::pair createTableFromAST( ASTCreateQuery ast_create_query, const String & database_name, const String & table_data_path_relative, - ContextPtr context, + ContextMutablePtr context, bool has_force_restore_data_flag); /** Get the string with the table definition based on the CREATE query. diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 4a150972573..7758841afc2 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -35,7 +34,7 @@ static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; namespace { void tryAttachTable( - ContextPtr context, + ContextMutablePtr context, const ASTCreateQuery & query, DatabaseOrdinary & database, const String & database_name, @@ -84,7 +83,7 @@ DatabaseOrdinary::DatabaseOrdinary( { } -void DatabaseOrdinary::loadStoredObjects(ContextPtr local_context, bool has_force_restore_data_flag, bool /*force_attach*/) +void DatabaseOrdinary::loadStoredObjects(ContextMutablePtr local_context, bool has_force_restore_data_flag, bool /*force_attach*/) { /** Tables load faster if they are loaded in sorted (by name) order. * Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order, @@ -111,8 +110,7 @@ void DatabaseOrdinary::loadStoredObjects(ContextPtr local_context, bool has_forc auto * create_query = ast->as(); create_query->database = database_name; - auto detached_permanently_flag = Poco::File(full_path.string() + detached_suffix); - if (detached_permanently_flag.exists()) + if (fs::exists(full_path.string() + detached_suffix)) { /// FIXME: even if we don't load the table we can still mark the uuid of it as taken. /// if (create_query->uuid != UUIDHelpers::Nil) @@ -281,11 +279,11 @@ void DatabaseOrdinary::commitAlterTable(const StorageID &, const String & table_ try { /// rename atomically replaces the old file with the new one. - Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); + fs::rename(table_metadata_tmp_path, table_metadata_path); } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); throw; } } diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 8a32bb10991..4d68890cc2b 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -20,7 +20,7 @@ public: String getEngineName() const override { return "Ordinary"; } - void loadStoredObjects(ContextPtr context, bool has_force_restore_data_flag, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr context, bool has_force_restore_data_flag, bool force_attach) override; void alterTable( ContextPtr context, diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index d36fe45f748..b3e5fc67151 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -212,7 +213,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach) createDatabaseNodesInZooKeeper(current_zookeeper); } - replica_path = zookeeper_path + "/replicas/" + getFullReplicaName(); + replica_path = fs::path(zookeeper_path) / "replicas" / getFullReplicaName(); String replica_host_id; if (current_zookeeper->tryGet(replica_path, replica_host_id)) @@ -273,19 +274,11 @@ bool DatabaseReplicated::createDatabaseNodesInZooKeeper(const zkutil::ZooKeeperP __builtin_unreachable(); } -void DatabaseReplicated::createEmptyLogEntry(Coordination::Requests & ops, const ZooKeeperPtr & current_zookeeper) +void DatabaseReplicated::createEmptyLogEntry(const ZooKeeperPtr & current_zookeeper) { /// On replica creation add empty entry to log. Can be used to trigger some actions on other replicas (e.g. update cluster info). DDLLogEntry entry{}; - - String query_path_prefix = zookeeper_path + "/log/query-"; - String counter_prefix = zookeeper_path + "/counter/cnt-"; - String counter_path = current_zookeeper->create(counter_prefix, "", zkutil::CreateMode::EphemeralSequential); - String query_path = query_path_prefix + counter_path.substr(counter_prefix.size()); - - ops.emplace_back(zkutil::makeCreateRequest(query_path, entry.toString(), zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(query_path + "/committed", getFullReplicaName(), zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeRemoveRequest(counter_path, -1)); + DatabaseReplicatedDDLWorker::enqueueQueryImpl(current_zookeeper, entry, this, true); } void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPtr & current_zookeeper) @@ -296,11 +289,11 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt Coordination::Requests ops; ops.emplace_back(zkutil::makeCreateRequest(replica_path, host_id, zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/log_ptr", "0", zkutil::CreateMode::Persistent)); - createEmptyLogEntry(ops, current_zookeeper); current_zookeeper->multi(ops); + createEmptyLogEntry(current_zookeeper); } -void DatabaseReplicated::loadStoredObjects(ContextPtr local_context, bool has_force_restore_data_flag, bool force_attach) +void DatabaseReplicated::loadStoredObjects(ContextMutablePtr local_context, bool has_force_restore_data_flag, bool force_attach) { tryConnectToZooKeeperAndInitDatabase(force_attach); @@ -325,10 +318,25 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_ if (!replicated_table || !create->storage->engine->arguments) return; - ASTs & args = create->storage->engine->arguments->children; + ASTs & args_ref = create->storage->engine->arguments->children; + ASTs args = args_ref; if (args.size() < 2) return; + /// It can be a constant expression. Try to evaluate it, ignore exception if we cannot. + bool has_expression_argument = args_ref[0]->as() || args_ref[1]->as(); + if (has_expression_argument) + { + try + { + args[0] = evaluateConstantExpressionAsLiteral(args_ref[0]->clone(), query_context); + args[1] = evaluateConstantExpressionAsLiteral(args_ref[1]->clone(), query_context); + } + catch (...) + { + } + } + ASTLiteral * arg1 = args[0]->as(); ASTLiteral * arg2 = args[1]->as(); if (!arg1 || !arg2 || arg1->value.getType() != Field::Types::String || arg2->value.getType() != Field::Types::String) @@ -356,12 +364,12 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_ if (maybe_shard_macros && maybe_replica_macros) return; - if (enable_functional_tests_helper) + if (enable_functional_tests_helper && !has_expression_argument) { if (maybe_path.empty() || maybe_path.back() != '/') maybe_path += '/'; - arg1->value = maybe_path + "auto_{shard}"; - arg2->value = maybe_replica + "auto_{replica}"; + args_ref[0]->as()->value = maybe_path + "auto_{shard}"; + args_ref[1]->as()->value = maybe_replica + "auto_{replica}"; return; } @@ -659,10 +667,8 @@ ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node void DatabaseReplicated::drop(ContextPtr context_) { auto current_zookeeper = getZooKeeper(); - Coordination::Requests ops; - ops.emplace_back(zkutil::makeSetRequest(replica_path, DROPPED_MARK, -1)); - createEmptyLogEntry(ops, current_zookeeper); - current_zookeeper->multi(ops); + current_zookeeper->set(replica_path, DROPPED_MARK, -1); + createEmptyLogEntry(current_zookeeper); DatabaseAtomic::drop(context_); diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index b930d27c19b..41b1bf13e5f 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -57,7 +57,7 @@ public: void drop(ContextPtr /*context*/) override; - void loadStoredObjects(ContextPtr context, bool has_force_restore_data_flag, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr context, bool has_force_restore_data_flag, bool force_attach) override; void shutdown() override; friend struct DatabaseReplicatedTask; @@ -78,7 +78,7 @@ private: ClusterPtr getClusterImpl() const; void setCluster(ClusterPtr && new_cluster); - void createEmptyLogEntry(Coordination::Requests & ops, const ZooKeeperPtr & current_zookeeper); + void createEmptyLogEntry(const ZooKeeperPtr & current_zookeeper); String zookeeper_path; String shard_name; diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 9ae4d026bf0..eb7e65e1b70 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -1,6 +1,10 @@ #include #include #include +#include +#include + +namespace fs = std::filesystem; namespace DB { @@ -69,25 +73,63 @@ void DatabaseReplicatedDDLWorker::initializeReplication() String DatabaseReplicatedDDLWorker::enqueueQuery(DDLLogEntry & entry) { auto zookeeper = getAndSetZooKeeper(); - const String query_path_prefix = queue_dir + "/query-"; + return enqueueQueryImpl(zookeeper, entry, database); +} + +String DatabaseReplicatedDDLWorker::enqueueQueryImpl(const ZooKeeperPtr & zookeeper, DDLLogEntry & entry, + DatabaseReplicated * const database, bool committed) +{ + const String query_path_prefix = database->zookeeper_path + "/log/query-"; /// We cannot create sequential node and it's ephemeral child in a single transaction, so allocate sequential number another way String counter_prefix = database->zookeeper_path + "/counter/cnt-"; - String counter_path = zookeeper->create(counter_prefix, "", zkutil::CreateMode::EphemeralSequential); + String counter_lock_path = database->zookeeper_path + "/counter_lock"; + + String counter_path; + size_t iters = 1000; + while (--iters) + { + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCreateRequest(counter_lock_path, database->getFullReplicaName(), zkutil::CreateMode::Ephemeral)); + ops.emplace_back(zkutil::makeCreateRequest(counter_prefix, "", zkutil::CreateMode::EphemeralSequential)); + Coordination::Responses res; + + Coordination::Error code = zookeeper->tryMulti(ops, res); + if (code == Coordination::Error::ZOK) + { + counter_path = dynamic_cast(*res.back()).path_created; + break; + } + else if (code != Coordination::Error::ZNODEEXISTS) + zkutil::KeeperMultiException::check(code, ops, res); + } + + if (iters == 0) + throw Exception(ErrorCodes::UNFINISHED, + "Cannot enqueue query, because some replica are trying to enqueue another query. " + "It may happen on high queries rate or, in rare cases, after connection loss. Client should retry."); + String node_path = query_path_prefix + counter_path.substr(counter_prefix.size()); + /// Now create task in queue Coordination::Requests ops; /// Query is not committed yet, but we have to write it into log to avoid reordering ops.emplace_back(zkutil::makeCreateRequest(node_path, entry.toString(), zkutil::CreateMode::Persistent)); /// '/try' will be replaced with '/committed' or will be removed due to expired session or other error - ops.emplace_back(zkutil::makeCreateRequest(node_path + "/try", database->getFullReplicaName(), zkutil::CreateMode::Ephemeral)); + if (committed) + ops.emplace_back(zkutil::makeCreateRequest(node_path + "/committed", database->getFullReplicaName(), zkutil::CreateMode::Persistent)); + else + ops.emplace_back(zkutil::makeCreateRequest(node_path + "/try", database->getFullReplicaName(), zkutil::CreateMode::Ephemeral)); /// We don't need it anymore ops.emplace_back(zkutil::makeRemoveRequest(counter_path, -1)); + /// Unlock counters + ops.emplace_back(zkutil::makeRemoveRequest(counter_lock_path, -1)); /// Create status dirs ops.emplace_back(zkutil::makeCreateRequest(node_path + "/active", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(node_path + "/finished", "", zkutil::CreateMode::Persistent)); zookeeper->multi(ops); + return node_path; } @@ -156,7 +198,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na } } - UInt32 our_log_ptr = parse(current_zookeeper->get(database->replica_path + "/log_ptr")); + UInt32 our_log_ptr = parse(current_zookeeper->get(fs::path(database->replica_path) / "log_ptr")); UInt32 entry_num = DatabaseReplicatedTask::getLogEntryNumber(entry_name); if (entry_num <= our_log_ptr) @@ -165,13 +207,13 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na return {}; } - String entry_path = queue_dir + "/" + entry_name; + String entry_path = fs::path(queue_dir) / entry_name; auto task = std::make_unique(entry_name, entry_path, database); String initiator_name; zkutil::EventPtr wait_committed_or_failed = std::make_shared(); - String try_node_path = entry_path + "/try"; + String try_node_path = fs::path(entry_path) / "try"; if (zookeeper->tryGet(try_node_path, initiator_name, nullptr, wait_committed_or_failed)) { task->is_initial_query = initiator_name == task->host_id_str; @@ -203,7 +245,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) throw Coordination::Exception(code, try_node_path); - if (!zookeeper->exists(entry_path + "/committed")) + if (!zookeeper->exists(fs::path(entry_path) / "committed")) { out_reason = fmt::format("Entry {} was forcefully cancelled due to timeout", entry_name); return {}; @@ -212,7 +254,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na } } - if (!zookeeper->exists(entry_path + "/committed")) + if (!zookeeper->exists(fs::path(entry_path) / "committed")) { out_reason = fmt::format("Entry {} hasn't been committed", entry_name); return {}; @@ -220,8 +262,8 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na if (task->is_initial_query) { - assert(!zookeeper->exists(entry_path + "/try")); - assert(zookeeper->exists(entry_path + "/committed") == (zookeeper->get(task->getFinishedNodePath()) == ExecutionStatus(0).serializeText())); + assert(!zookeeper->exists(fs::path(entry_path) / "try")); + assert(zookeeper->exists(fs::path(entry_path) / "committed") == (zookeeper->get(task->getFinishedNodePath()) == ExecutionStatus(0).serializeText())); out_reason = fmt::format("Entry {} has been executed as initial query", entry_name); return {}; } @@ -257,7 +299,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na bool DatabaseReplicatedDDLWorker::canRemoveQueueEntry(const String & entry_name, const Coordination::Stat &) { UInt32 entry_number = DDLTaskBase::getLogEntryNumber(entry_name); - UInt32 max_log_ptr = parse(getAndSetZooKeeper()->get(database->zookeeper_path + "/max_log_ptr")); + UInt32 max_log_ptr = parse(getAndSetZooKeeper()->get(fs::path(database->zookeeper_path) / "max_log_ptr")); return entry_number + logs_to_keep < max_log_ptr; } diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h index 16ad100b81a..4020906f9b2 100644 --- a/src/Databases/DatabaseReplicatedWorker.h +++ b/src/Databases/DatabaseReplicatedWorker.h @@ -29,6 +29,9 @@ public: void shutdown() override; + static String enqueueQueryImpl(const ZooKeeperPtr & zookeeper, DDLLogEntry & entry, + DatabaseReplicated * const database, bool committed = false); + private: bool initializeMainThread() override; void initializeReplication(); diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index ea07e51bffd..ba5fa974d5c 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -122,7 +122,7 @@ public: /// Load a set of existing tables. /// You can call only once, right after the object is created. - virtual void loadStoredObjects(ContextPtr /*context*/, bool /*has_force_restore_data_flag*/, bool /*force_attach*/ = false) {} + virtual void loadStoredObjects(ContextMutablePtr /*context*/, bool /*has_force_restore_data_flag*/, bool /*force_attach*/ = false) {} /// Check the existence of the table. virtual bool isTableExist(const String & name, ContextPtr context) const = 0; diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp index 9b71fe537ec..866b03e3bd6 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp @@ -24,10 +24,10 @@ # include # include # include +# include +# include -# include -# include - +namespace fs = std::filesystem; namespace DB { @@ -326,7 +326,7 @@ void DatabaseConnectionMySQL::shutdown() void DatabaseConnectionMySQL::drop(ContextPtr /*context*/) { - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } void DatabaseConnectionMySQL::cleanOutdatedTables() @@ -372,10 +372,10 @@ void DatabaseConnectionMySQL::attachTable(const String & table_name, const Stora local_tables_cache[table_name].second = storage; remove_or_detach_tables.erase(table_name); - Poco::File remove_flag(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); + fs::path remove_flag = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); - if (remove_flag.exists()) - remove_flag.remove(); + if (fs::exists(remove_flag)) + fs::remove(remove_flag); } StoragePtr DatabaseConnectionMySQL::detachTable(const String & table_name) @@ -399,17 +399,17 @@ String DatabaseConnectionMySQL::getMetadataPath() const return metadata_path; } -void DatabaseConnectionMySQL::loadStoredObjects(ContextPtr, bool, bool /*force_attach*/) +void DatabaseConnectionMySQL::loadStoredObjects(ContextMutablePtr, bool, bool /*force_attach*/) { std::lock_guard lock{mutex}; - Poco::DirectoryIterator iterator(getMetadataPath()); + fs::directory_iterator iter(getMetadataPath()); - for (Poco::DirectoryIterator end; iterator != end; ++iterator) + for (fs::directory_iterator end; iter != end; ++iter) { - if (iterator->isFile() && endsWith(iterator.name(), suffix)) + if (fs::is_regular_file(iter->path()) && endsWith(iter->path().filename(), suffix)) { - const auto & filename = iterator.name(); + const auto & filename = iter->path().filename().string(); const auto & table_name = unescapeForFileName(filename.substr(0, filename.size() - strlen(suffix))); remove_or_detach_tables.emplace(table_name); } @@ -420,27 +420,25 @@ void DatabaseConnectionMySQL::detachTablePermanently(ContextPtr, const String & { std::lock_guard lock{mutex}; - Poco::File remove_flag(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); + fs::path remove_flag = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); if (remove_or_detach_tables.count(table_name)) - throw Exception("Table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " is dropped", - ErrorCodes::TABLE_IS_DROPPED); + throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); - if (remove_flag.exists()) - throw Exception("The remove flag file already exists but the " + backQuoteIfNeed(database_name) + - "." + backQuoteIfNeed(table_name) + " does not exists remove tables, it is bug.", ErrorCodes::LOGICAL_ERROR); + if (fs::exists(remove_flag)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The remove flag file already exists but the {}.{} does not exists remove tables, it is bug.", + backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); auto table_iter = local_tables_cache.find(table_name); if (table_iter == local_tables_cache.end()) - throw Exception("Table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " doesn't exist.", - ErrorCodes::UNKNOWN_TABLE); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); remove_or_detach_tables.emplace(table_name); try { table_iter->second.second->drop(); - remove_flag.createFile(); + FS::createFile(remove_flag); } catch (...) { diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.h b/src/Databases/MySQL/DatabaseConnectionMySQL.h index c1936e51999..32741185cfa 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.h +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.h @@ -74,7 +74,7 @@ public: void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override; - void loadStoredObjects(ContextPtr, bool, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr, bool, bool force_attach) override; StoragePtr detachTable(const String & table_name) override; diff --git a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp index 62a66b22c93..bf9d6cdfbfa 100644 --- a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp @@ -13,9 +13,11 @@ # include # include # include -# include # include # include +# include + +namespace fs = std::filesystem; namespace DB { @@ -92,7 +94,7 @@ void DatabaseMaterializeMySQL::setException(const std::exception_ptr & exc } template -void DatabaseMaterializeMySQL::loadStoredObjects(ContextPtr context_, bool has_force_restore_data_flag, bool force_attach) +void DatabaseMaterializeMySQL::loadStoredObjects(ContextMutablePtr context_, bool has_force_restore_data_flag, bool force_attach) { Base::loadStoredObjects(context_, has_force_restore_data_flag, force_attach); if (!force_attach) @@ -158,10 +160,10 @@ template void DatabaseMaterializeMySQL::drop(ContextPtr context_) { /// Remove metadata info - Poco::File metadata(Base::getMetadataPath() + "/.metadata"); + fs::path metadata(Base::getMetadataPath() + "/.metadata"); - if (metadata.exists()) - metadata.remove(false); + if (fs::exists(metadata)) + fs::remove(metadata); Base::drop(context_); } diff --git a/src/Databases/MySQL/DatabaseMaterializeMySQL.h b/src/Databases/MySQL/DatabaseMaterializeMySQL.h index cd9fe640239..74a3c06e6f0 100644 --- a/src/Databases/MySQL/DatabaseMaterializeMySQL.h +++ b/src/Databases/MySQL/DatabaseMaterializeMySQL.h @@ -43,7 +43,7 @@ protected: public: String getEngineName() const override { return "MaterializeMySQL"; } - void loadStoredObjects(ContextPtr context_, bool has_force_restore_data_flag, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr context_, bool has_force_restore_data_flag, bool force_attach) override; void createTable(ContextPtr context_, const String & name, const StoragePtr & table, const ASTPtr & query) override; diff --git a/src/Databases/MySQL/MaterializeMetadata.cpp b/src/Databases/MySQL/MaterializeMetadata.cpp index c389ab5a1b0..4fd99ca3438 100644 --- a/src/Databases/MySQL/MaterializeMetadata.cpp +++ b/src/Databases/MySQL/MaterializeMetadata.cpp @@ -8,11 +8,13 @@ #include #include #include -#include #include #include #include #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -193,12 +195,11 @@ void commitMetadata(const std::function & function, const String & persi try { function(); - - Poco::File(persistent_tmp_path).renameTo(persistent_path); + fs::rename(persistent_tmp_path, persistent_path); } catch (...) { - Poco::File(persistent_tmp_path).remove(); + fs::remove(persistent_tmp_path); throw; } } @@ -231,7 +232,7 @@ void MaterializeMetadata::transaction(const MySQLReplication::Position & positio MaterializeMetadata::MaterializeMetadata(const String & path_, const Settings & settings_) : persistent_path(path_), settings(settings_) { - if (Poco::File(persistent_path).exists()) + if (fs::exists(persistent_path)) { ReadBufferFromFile in(persistent_path, DBMS_DEFAULT_BUFFER_SIZE); assertString("Version:\t" + toString(meta_version), in); diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp index a7ebcb0d3f2..e5783be4624 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp @@ -41,7 +41,7 @@ namespace ErrorCodes static constexpr auto MYSQL_BACKGROUND_THREAD_NAME = "MySQLDBSync"; -static ContextPtr createQueryContext(ContextPtr context) +static ContextMutablePtr createQueryContext(ContextPtr context) { Settings new_query_settings = context->getSettings(); new_query_settings.insert_allow_materialized_columns = true; @@ -59,7 +59,7 @@ static ContextPtr createQueryContext(ContextPtr context) return query_context; } -static BlockIO tryToExecuteQuery(const String & query_to_execute, ContextPtr query_context, const String & database, const String & comment) +static BlockIO tryToExecuteQuery(const String & query_to_execute, ContextMutablePtr query_context, const String & database, const String & comment) { try { @@ -281,7 +281,7 @@ static inline void cleanOutdatedTables(const String & database_name, ContextPtr } static inline BlockOutputStreamPtr -getTableOutput(const String & database_name, const String & table_name, ContextPtr query_context, bool insert_materialized = false) +getTableOutput(const String & database_name, const String & table_name, ContextMutablePtr query_context, bool insert_materialized = false) { const StoragePtr & storage = DatabaseCatalog::instance().getTable(StorageID(database_name, table_name), query_context); diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 55813e519e4..243ab9166be 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -12,11 +12,12 @@ #include #include #include -#include -#include #include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -209,9 +210,9 @@ void DatabasePostgreSQL::attachTable(const String & table_name, const StoragePtr detached_or_dropped.erase(table_name); - Poco::File table_marked_as_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - if (table_marked_as_removed.exists()) - table_marked_as_removed.remove(); + fs::path table_marked_as_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + if (fs::exists(table_marked_as_removed)) + fs::remove(table_marked_as_removed); } @@ -256,16 +257,8 @@ void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool / if (detached_or_dropped.count(table_name)) throw Exception(fmt::format("Table {}.{} is already dropped/detached", database_name, table_name), ErrorCodes::TABLE_IS_DROPPED); - Poco::File mark_table_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - - try - { - mark_table_removed.createFile(); - } - catch (...) - { - throw; - } + fs::path mark_table_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + FS::createFile(mark_table_removed); if (cache_tables) cached_tables.erase(table_name); @@ -276,22 +269,22 @@ void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool / void DatabasePostgreSQL::drop(ContextPtr /*context*/) { - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } -void DatabasePostgreSQL::loadStoredObjects(ContextPtr /* context */, bool, bool /*force_attach*/) +void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, bool, bool /*force_attach*/) { { std::lock_guard lock{mutex}; - Poco::DirectoryIterator iterator(getMetadataPath()); + fs::directory_iterator iter(getMetadataPath()); /// Check for previously dropped tables - for (Poco::DirectoryIterator end; iterator != end; ++iterator) + for (fs::directory_iterator end; iter != end; ++iter) { - if (iterator->isFile() && endsWith(iterator.name(), suffix)) + if (fs::is_regular_file(iter->path()) && endsWith(iter->path().filename(), suffix)) { - const auto & file_name = iterator.name(); + const auto & file_name = iter->path().filename().string(); const auto & table_name = unescapeForFileName(file_name.substr(0, file_name.size() - strlen(suffix))); detached_or_dropped.emplace(table_name); } @@ -325,9 +318,9 @@ void DatabasePostgreSQL::removeOutdatedTables() { auto table_name = *iter; iter = detached_or_dropped.erase(iter); - Poco::File table_marked_as_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - if (table_marked_as_removed.exists()) - table_marked_as_removed.remove(); + fs::path table_marked_as_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + if (fs::exists(table_marked_as_removed)) + fs::remove(table_marked_as_removed); } else ++iter; diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index c5054883f2c..3c3a67ec9d3 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -47,7 +47,7 @@ public: bool empty() const override; - void loadStoredObjects(ContextPtr, bool, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr, bool, bool force_attach) override; DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) override; diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index 7605b86ef90..b91ce6c82c9 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -17,7 +17,7 @@ void registerDictionarySourceCassandra(DictionarySourceFactory & factory) [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, [[maybe_unused]] const std::string & config_prefix, [[maybe_unused]] Block & sample_block, - ContextPtr /* context */, + ContextConstPtr /* context */, const std::string & /* default_database */, bool /*created_from_ddl*/) -> DictionarySourcePtr { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 776e74672b4..9e44a7b442d 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -27,7 +27,7 @@ namespace { constexpr size_t MAX_CONNECTIONS = 16; - inline UInt16 getPortFromContext(ContextPtr context, bool secure) + inline UInt16 getPortFromContext(ContextConstPtr context, bool secure) { return secure ? context->getTCPPortSecure().value_or(0) : context->getTCPPort(); } @@ -60,7 +60,7 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( const DictionaryStructure & dict_struct_, const Configuration & configuration_, const Block & sample_block_, - ContextPtr context_) + ContextConstPtr context_) : update_time{std::chrono::system_clock::from_time_t(0)} , dict_struct{dict_struct_} , configuration{configuration_} @@ -209,7 +209,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & default_database [[maybe_unused]], bool /* created_from_ddl */) -> DictionarySourcePtr { diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index d96330f9bdb..cf954a9620d 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -37,7 +37,7 @@ public: const DictionaryStructure & dict_struct_, const Configuration & configuration_, const Block & sample_block_, - ContextPtr context); + ContextConstPtr context); /// copy-constructor is provided in order to support cloneability ClickHouseDictionarySource(const ClickHouseDictionarySource & other); @@ -79,7 +79,7 @@ private: mutable std::string invalidate_query_response; ExternalQueryBuilder query_builder; Block sample_block; - ContextPtr context; + ContextMutablePtr context; ConnectionPoolWithFailoverPtr pool; const std::string load_all_query; Poco::Logger * log = &Poco::Logger::get("ClickHouseDictionarySource"); diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index 62b28ed7d14..1f23200a745 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -31,7 +31,7 @@ DictionaryPtr DictionaryFactory::create( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, - ContextPtr context, + ContextConstPtr context, bool created_from_ddl) const { Poco::Util::AbstractConfiguration::Keys keys; @@ -68,7 +68,7 @@ DictionaryPtr DictionaryFactory::create( layout_type); } -DictionaryPtr DictionaryFactory::create(const std::string & name, const ASTCreateQuery & ast, ContextPtr context) const +DictionaryPtr DictionaryFactory::create(const std::string & name, const ASTCreateQuery & ast, ContextConstPtr context) const { auto configuration = getDictionaryConfigurationFromAST(ast, context); return DictionaryFactory::create(name, *configuration, "dictionary", context, true); diff --git a/src/Dictionaries/DictionaryFactory.h b/src/Dictionaries/DictionaryFactory.h index b717009aa8a..a261c377cad 100644 --- a/src/Dictionaries/DictionaryFactory.h +++ b/src/Dictionaries/DictionaryFactory.h @@ -36,13 +36,13 @@ public: const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, - ContextPtr context, + ContextConstPtr context, bool created_from_ddl) const; /// Create dictionary from DDL-query DictionaryPtr create(const std::string & name, const ASTCreateQuery & ast, - ContextPtr context) const; + ContextConstPtr context) const; using LayoutCreateFunction = std::function; bool isComplex(const std::string & layout_type) const; diff --git a/src/Dictionaries/DictionarySourceFactory.cpp b/src/Dictionaries/DictionarySourceFactory.cpp index 50ba6405074..507d11eba8e 100644 --- a/src/Dictionaries/DictionarySourceFactory.cpp +++ b/src/Dictionaries/DictionarySourceFactory.cpp @@ -79,7 +79,7 @@ DictionarySourcePtr DictionarySourceFactory::create( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const DictionaryStructure & dict_struct, - ContextPtr context, + ContextConstPtr context, const std::string & default_database, bool check_config) const { diff --git a/src/Dictionaries/DictionarySourceFactory.h b/src/Dictionaries/DictionarySourceFactory.h index bb583927ac4..9a0235a5eb0 100644 --- a/src/Dictionaries/DictionarySourceFactory.h +++ b/src/Dictionaries/DictionarySourceFactory.h @@ -35,7 +35,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & default_database, bool check_config)>; @@ -48,7 +48,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const DictionaryStructure & dict_struct, - ContextPtr context, + ContextConstPtr context, const std::string & default_database, bool check_config) const; diff --git a/src/Dictionaries/DictionarySourceHelpers.cpp b/src/Dictionaries/DictionarySourceHelpers.cpp index e175c6c3eae..723a0c338fd 100644 --- a/src/Dictionaries/DictionarySourceHelpers.cpp +++ b/src/Dictionaries/DictionarySourceHelpers.cpp @@ -67,9 +67,9 @@ Block blockForKeys( return block; } -ContextPtr copyContextAndApplySettings( +ContextMutablePtr copyContextAndApplySettings( const std::string & config_prefix, - ContextPtr context, + ContextConstPtr context, const Poco::Util::AbstractConfiguration & config) { auto local_context = Context::createCopy(context); diff --git a/src/Dictionaries/DictionarySourceHelpers.h b/src/Dictionaries/DictionarySourceHelpers.h index 1febf921e07..1066574f8e9 100644 --- a/src/Dictionaries/DictionarySourceHelpers.h +++ b/src/Dictionaries/DictionarySourceHelpers.h @@ -3,14 +3,13 @@ #include #include - -#include #include #include #include #include #include + namespace DB { @@ -37,14 +36,9 @@ Block blockForKeys( const std::vector & requested_rows); /// Used for applying settings to copied context in some register[...]Source functions -ContextPtr copyContextAndApplySettings( +ContextMutablePtr copyContextAndApplySettings( const std::string & config_prefix, - ContextPtr context, - const Poco::Util::AbstractConfiguration & config); - -void applySettingsToContext( - const std::string & config_prefix, - ContextPtr context, + ContextConstPtr context, const Poco::Util::AbstractConfiguration & config); /** A stream, adds additional columns to each block that it will read from inner stream. diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 0508a0d70ad..2cc1d6caf5f 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -306,7 +306,7 @@ namespace const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextConstPtr /* context */, bool /* created_from_ddl */) { const auto * layout_name = dictionary_key_type == DictionaryKeyType::simple ? "direct" : "complex_key_direct"; diff --git a/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp b/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp index 3b8848ab19b..210459da0be 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp +++ b/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp @@ -5,7 +5,9 @@ #include #include #include "HierarchyFormatReader.h" +#include +namespace fs = std::filesystem; bool RegionsHierarchyDataSource::isModified() const { @@ -27,14 +29,13 @@ RegionsHierarchiesDataProvider::RegionsHierarchiesDataProvider(const std::string void RegionsHierarchiesDataProvider::discoverFilesWithCustomHierarchies() { - std::string basename = Poco::Path(path).getBaseName(); + std::string basename = fs::path(path).stem(); + fs::path dir_path = fs::canonical(path).parent_path(); - Poco::Path dir_path = Poco::Path(path).absolute().parent(); - - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(dir_path); dir_it != dir_end; ++dir_it) + fs::directory_iterator dir_end; + for (fs::directory_iterator dir_it(dir_path); dir_it != dir_end; ++dir_it) { - std::string candidate_basename = dir_it.path().getBaseName(); + std::string candidate_basename = dir_it->path().stem(); if (candidate_basename.starts_with(basename) && (candidate_basename.size() > basename.size() + 1) diff --git a/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp b/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp index f3e49545481..5f79fda070f 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp +++ b/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp @@ -2,7 +2,9 @@ #include #include "NamesFormatReader.h" +#include +namespace fs = std::filesystem; bool LanguageRegionsNamesDataSource::isModified() const { @@ -11,7 +13,7 @@ bool LanguageRegionsNamesDataSource::isModified() const size_t LanguageRegionsNamesDataSource::estimateTotalSize() const { - return Poco::File(path).getSize(); + return fs::file_size(path); } ILanguageRegionsNamesReaderPtr LanguageRegionsNamesDataSource::createReader() @@ -39,7 +41,7 @@ RegionsNamesDataProvider::RegionsNamesDataProvider(const std::string & directory ILanguageRegionsNamesDataSourcePtr RegionsNamesDataProvider::getLanguageRegionsNamesSource(const std::string & language) const { const auto data_file = getDataFilePath(language); - if (Poco::File(data_file).exists()) + if (fs::exists(data_file)) return std::make_unique(data_file, language); else return {}; diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index 55d06984df4..2fb1bcf89ac 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -63,7 +63,7 @@ ExecutableDictionarySource::ExecutableDictionarySource( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block_, - ContextPtr context_) + ContextConstPtr context_) : log(&Poco::Logger::get("ExecutableDictionarySource")) , dict_struct{dict_struct_} , implicit_key{config.getBool(config_prefix + ".implicit_key", false)} @@ -140,7 +140,7 @@ namespace { public: BlockInputStreamWithBackgroundThread( - ContextPtr context, + ContextConstPtr context, const std::string & format, const Block & sample_block, const std::string & command_str, @@ -265,7 +265,7 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { diff --git a/src/Dictionaries/ExecutableDictionarySource.h b/src/Dictionaries/ExecutableDictionarySource.h index 878cb086873..4b2326b2a58 100644 --- a/src/Dictionaries/ExecutableDictionarySource.h +++ b/src/Dictionaries/ExecutableDictionarySource.h @@ -20,7 +20,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block_, - ContextPtr context_); + ContextConstPtr context_); ExecutableDictionarySource(const ExecutableDictionarySource & other); ExecutableDictionarySource & operator=(const ExecutableDictionarySource &) = delete; @@ -58,7 +58,7 @@ private: const std::string update_field; const std::string format; Block sample_block; - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.cpp b/src/Dictionaries/ExecutablePoolDictionarySource.cpp index 2e01c75234c..21818baa658 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp +++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp @@ -32,7 +32,7 @@ ExecutablePoolDictionarySource::ExecutablePoolDictionarySource( const DictionaryStructure & dict_struct_, const Configuration & configuration_, Block & sample_block_, - ContextPtr context_) + ContextConstPtr context_) : log(&Poco::Logger::get("ExecutablePoolDictionarySource")) , dict_struct{dict_struct_} , configuration{configuration_} @@ -273,7 +273,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.h b/src/Dictionaries/ExecutablePoolDictionarySource.h index 7a0b8681a21..9bf6e89c1f0 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.h +++ b/src/Dictionaries/ExecutablePoolDictionarySource.h @@ -42,7 +42,7 @@ public: const DictionaryStructure & dict_struct_, const Configuration & configuration_, Block & sample_block_, - ContextPtr context_); + ContextConstPtr context_); ExecutablePoolDictionarySource(const ExecutablePoolDictionarySource & other); ExecutablePoolDictionarySource & operator=(const ExecutablePoolDictionarySource &) = delete; @@ -78,7 +78,7 @@ private: const Configuration configuration; Block sample_block; - ContextPtr context; + ContextConstPtr context; std::shared_ptr process_pool; }; diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index a32bd081dd7..49e87e24543 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -1,9 +1,4 @@ #include "FileDictionarySource.h" - -#include - -#include - #include #include #include @@ -15,6 +10,7 @@ #include "registerDictionaries.h" #include "DictionarySourceHelpers.h" + namespace DB { static const UInt64 max_block_size = 8192; @@ -28,7 +24,7 @@ namespace ErrorCodes FileDictionarySource::FileDictionarySource( const std::string & filepath_, const std::string & format_, - Block & sample_block_, ContextPtr context_, bool created_from_ddl) + Block & sample_block_, ContextConstPtr context_, bool created_from_ddl) : filepath{filepath_} , format{format_} , sample_block{sample_block_} @@ -68,16 +64,17 @@ std::string FileDictionarySource::toString() const Poco::Timestamp FileDictionarySource::getLastModification() const { - return Poco::File{filepath}.getLastModified(); + return FS::getModificationTimestamp(filepath); } + void registerDictionarySourceFile(DictionarySourceFactory & factory) { auto create_table_source = [=](const DictionaryStructure & dict_struct, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { diff --git a/src/Dictionaries/FileDictionarySource.h b/src/Dictionaries/FileDictionarySource.h index ffc29374f4f..d31a6dedb7f 100644 --- a/src/Dictionaries/FileDictionarySource.h +++ b/src/Dictionaries/FileDictionarySource.h @@ -17,7 +17,7 @@ class FileDictionarySource final : public IDictionarySource { public: FileDictionarySource(const std::string & filepath_, const std::string & format_, - Block & sample_block_, ContextPtr context_, bool created_from_ddl); + Block & sample_block_, ContextConstPtr context_, bool created_from_ddl); FileDictionarySource(const FileDictionarySource & other); @@ -61,7 +61,7 @@ private: const std::string filepath; const std::string format; Block sample_block; - ContextPtr context; + ContextConstPtr context; Poco::Timestamp last_modification; }; diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index 1c7fd076e6e..f58384a3b8e 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -506,7 +506,7 @@ void registerDictionaryFlat(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextConstPtr /* context */, bool /* created_from_ddl */) -> DictionaryPtr { if (dict_struct.key) diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 7fc1778545c..b3527b06117 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -31,7 +31,7 @@ HTTPDictionarySource::HTTPDictionarySource( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block_, - ContextPtr context_, + ContextConstPtr context_, bool created_from_ddl) : log(&Poco::Logger::get("HTTPDictionarySource")) , update_time{std::chrono::system_clock::from_time_t(0)} @@ -231,7 +231,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { if (dict_struct.has_expressions) diff --git a/src/Dictionaries/HTTPDictionarySource.h b/src/Dictionaries/HTTPDictionarySource.h index e348075edc3..9b93856a6bc 100644 --- a/src/Dictionaries/HTTPDictionarySource.h +++ b/src/Dictionaries/HTTPDictionarySource.h @@ -27,7 +27,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block_, - ContextPtr context_, + ContextConstPtr context_, bool created_from_ddl); HTTPDictionarySource(const HTTPDictionarySource & other); @@ -69,7 +69,7 @@ private: std::string update_field; const std::string format; Block sample_block; - ContextPtr context; + ContextConstPtr context; ConnectionTimeouts timeouts; }; diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index 24450b87274..0de28f18006 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -751,13 +751,13 @@ void registerDictionaryHashed(DictionaryFactory & factory) using namespace std::placeholders; factory.registerLayout("hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::simple, /* sparse = */ false); }, false); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextConstPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::simple, /* sparse = */ false); }, false); factory.registerLayout("sparse_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::simple, /* sparse = */ true); }, false); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextConstPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::simple, /* sparse = */ true); }, false); factory.registerLayout("complex_key_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::complex, /* sparse = */ false); }, true); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextConstPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::complex, /* sparse = */ false); }, true); factory.registerLayout("complex_key_sparse_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::complex, /* sparse = */ true); }, true); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextConstPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::complex, /* sparse = */ true); }, true); } diff --git a/src/Dictionaries/IPAddressDictionary.cpp b/src/Dictionaries/IPAddressDictionary.cpp index 648e756e9a7..a0630ab324c 100644 --- a/src/Dictionaries/IPAddressDictionary.cpp +++ b/src/Dictionaries/IPAddressDictionary.cpp @@ -935,7 +935,7 @@ void registerDictionaryTrie(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextConstPtr /* context */, bool /*created_from_ddl*/) -> DictionaryPtr { if (!dict_struct.key || dict_struct.key->size() != 1) diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index f51c4908942..f52d774f279 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -1,19 +1,19 @@ #include "LibraryDictionarySource.h" -#include - +#include +#include #include #include #include #include -#include -#include +#include #include #include #include #include +namespace fs = std::filesystem; namespace DB { @@ -31,7 +31,7 @@ LibraryDictionarySource::LibraryDictionarySource( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix_, Block & sample_block_, - ContextPtr context_, + ContextConstPtr context_, bool created_from_ddl) : log(&Poco::Logger::get("LibraryDictionarySource")) , dict_struct{dict_struct_} @@ -44,8 +44,8 @@ LibraryDictionarySource::LibraryDictionarySource( if (created_from_ddl && !pathStartsWith(path, context->getDictionariesLibPath())) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, context->getDictionariesLibPath()); - if (!Poco::File(path).exists()) - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", Poco::File(path).path()); + if (!fs::exists(path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", path); description.init(sample_block); bridge_helper = std::make_shared(context, description.sample_block, dictionary_id); @@ -172,7 +172,7 @@ void registerDictionarySourceLibrary(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { diff --git a/src/Dictionaries/LibraryDictionarySource.h b/src/Dictionaries/LibraryDictionarySource.h index 88e133666e6..eb346d09fab 100644 --- a/src/Dictionaries/LibraryDictionarySource.h +++ b/src/Dictionaries/LibraryDictionarySource.h @@ -39,7 +39,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix_, Block & sample_block_, - ContextPtr context_, + ContextConstPtr context_, bool created_from_ddl); LibraryDictionarySource(const LibraryDictionarySource & other); @@ -86,7 +86,7 @@ private: const Field dictionary_id; Block sample_block; - ContextPtr context; + ContextConstPtr context; LibraryBridgeHelperPtr bridge_helper; ExternalResultDescription description; diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 2a809f52817..c0780616191 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -13,7 +13,7 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & root_config_prefix, Block & sample_block, - ContextPtr, + ContextConstPtr, const std::string & /* default_database */, bool /* created_from_ddl */) { diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index afda8241b38..40398a7ba42 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -24,7 +24,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, [[maybe_unused]] const std::string & config_prefix, [[maybe_unused]] Block & sample_block, - [[maybe_unused]] ContextPtr context, + [[maybe_unused]] ContextConstPtr context, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { #if USE_MYSQL diff --git a/src/Dictionaries/PolygonDictionaryImplementations.cpp b/src/Dictionaries/PolygonDictionaryImplementations.cpp index 7c3eb421a4a..fd3b9efd532 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.cpp +++ b/src/Dictionaries/PolygonDictionaryImplementations.cpp @@ -167,7 +167,7 @@ DictionaryPtr createLayout(const std::string & , const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextConstPtr /* context */, bool /*created_from_ddl*/) { const String database = config.getString(config_prefix + ".database", ""); diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index d23c9a7e725..c563f9aab45 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -188,7 +188,7 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & root_config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 4049bd3a027..b771bca068f 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -637,7 +637,7 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextConstPtr /* context */, bool /*created_from_ddl*/) -> DictionaryPtr { if (dict_struct.key) diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index 4e79a75c1d4..ce979462a36 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -12,7 +12,7 @@ void registerDictionarySourceRedis(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const String & config_prefix, Block & sample_block, - ContextPtr /* context */, + ContextConstPtr /* context */, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { return std::make_unique(dict_struct, config, config_prefix + ".redis", sample_block); diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 0ebd15ff515..fa7138348c1 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -37,7 +37,7 @@ namespace const Poco::URI & uri, std::function callback, const Block & sample_block, - ContextPtr context, + ContextConstPtr context, UInt64 max_block_size, const ConnectionTimeouts & timeouts, const String name_) @@ -102,7 +102,7 @@ XDBCDictionarySource::XDBCDictionarySource( const Poco::Util::AbstractConfiguration & config_, const std::string & config_prefix_, const Block & sample_block_, - ContextPtr context_, + ContextConstPtr context_, const BridgeHelperPtr bridge_) : WithContext(context_->getGlobalContext()) , log(&Poco::Logger::get(bridge_->getName() + "DictionarySource")) @@ -278,7 +278,7 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextConstPtr context, const std::string & /* default_database */, bool /* check_config */) -> DictionarySourcePtr { #if USE_ODBC @@ -305,7 +305,7 @@ void registerDictionarySourceJDBC(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & /* config */, const std::string & /* config_prefix */, Block & /* sample_block */, - ContextPtr /* context */, + ContextConstPtr /* context */, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, diff --git a/src/Dictionaries/XDBCDictionarySource.h b/src/Dictionaries/XDBCDictionarySource.h index bd473e0db8a..4a6f226102b 100644 --- a/src/Dictionaries/XDBCDictionarySource.h +++ b/src/Dictionaries/XDBCDictionarySource.h @@ -31,7 +31,7 @@ public: const Poco::Util::AbstractConfiguration & config_, const std::string & config_prefix_, const Block & sample_block_, - ContextPtr context_, + ContextConstPtr context_, BridgeHelperPtr bridge); /// copy-constructor is provided in order to support cloneability diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index 2063ebcbf79..4816c6fd6b8 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -397,7 +397,7 @@ void buildConfigurationFromFunctionWithKeyValueArguments( AutoPtr doc, AutoPtr root, const ASTExpressionList * ast_expr_list, - ContextPtr context) + ContextConstPtr context) { const auto & children = ast_expr_list->children; for (size_t i = 0; i != children.size(); ++i) @@ -464,7 +464,7 @@ void buildSourceConfiguration( AutoPtr root, const ASTFunctionWithKeyValueArguments * source, const ASTDictionarySettings * settings, - ContextPtr context) + ContextConstPtr context) { AutoPtr outer_element(doc->createElement("source")); root->appendChild(outer_element); @@ -525,7 +525,7 @@ void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs DictionaryConfigurationPtr -getDictionaryConfigurationFromAST(const ASTCreateQuery & query, ContextPtr context, const std::string & database_) +getDictionaryConfigurationFromAST(const ASTCreateQuery & query, ContextConstPtr context, const std::string & database_) { checkAST(query); diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.h b/src/Dictionaries/getDictionaryConfigurationFromAST.h index de8659e4d7b..01f13ab4030 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.h +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.h @@ -12,5 +12,5 @@ using DictionaryConfigurationPtr = Poco::AutoPtr DictionaryPtr { return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr)); @@ -278,7 +278,7 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextConstPtr /* context */, bool /* created_from_ddl */) -> DictionaryPtr { return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr)); @@ -293,7 +293,7 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextConstPtr context, bool created_from_ddl) -> DictionaryPtr { return createSSDCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), context, created_from_ddl); @@ -306,7 +306,7 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextConstPtr context, bool created_from_ddl) -> DictionaryPtr { return createSSDCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), context, created_from_ddl); }; diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index d0cf6a00344..768d1c2354a 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -6,9 +6,9 @@ #include #include #include - #include +#include #include @@ -60,27 +60,28 @@ class DiskLocalDirectoryIterator : public IDiskDirectoryIterator { public: explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_) - : dir_path(dir_path_), iter(disk_path_ + dir_path_) + : dir_path(dir_path_), entry(fs::path(disk_path_) / dir_path_) { } - void next() override { ++iter; } + void next() override { ++entry; } - bool isValid() const override { return iter != Poco::DirectoryIterator(); } + bool isValid() const override { return entry != fs::directory_iterator(); } String path() const override { - if (iter->isDirectory()) - return dir_path + iter.name() + '/'; + if (entry->is_directory()) + return dir_path / entry->path().filename() / ""; else - return dir_path + iter.name(); + return dir_path / entry->path().filename(); } - String name() const override { return iter.name(); } + + String name() const override { return entry->path().filename(); } private: - String dir_path; - Poco::DirectoryIterator iter; + fs::path dir_path; + fs::directory_iterator entry; }; @@ -118,7 +119,7 @@ UInt64 DiskLocal::getTotalSpace() const { struct statvfs fs; if (name == "default") /// for default disk we get space from path/data/ - fs = getStatVFS(disk_path + "data/"); + fs = getStatVFS((fs::path(disk_path) / "data/").string()); else fs = getStatVFS(disk_path); UInt64 total_size = fs.f_blocks * fs.f_bsize; @@ -133,7 +134,7 @@ UInt64 DiskLocal::getAvailableSpace() const /// available for superuser only and for system purposes struct statvfs fs; if (name == "default") /// for default disk we get space from path/data/ - fs = getStatVFS(disk_path + "data/"); + fs = getStatVFS((fs::path(disk_path) / "data/").string()); else fs = getStatVFS(disk_path); UInt64 total_size = fs.f_bavail * fs.f_bsize; @@ -152,45 +153,43 @@ UInt64 DiskLocal::getUnreservedSpace() const bool DiskLocal::exists(const String & path) const { - return Poco::File(disk_path + path).exists(); + return fs::exists(fs::path(disk_path) / path); } bool DiskLocal::isFile(const String & path) const { - return Poco::File(disk_path + path).isFile(); + return fs::is_regular_file(fs::path(disk_path) / path); } bool DiskLocal::isDirectory(const String & path) const { - return Poco::File(disk_path + path).isDirectory(); + return fs::is_directory(fs::path(disk_path) / path); } size_t DiskLocal::getFileSize(const String & path) const { - return Poco::File(disk_path + path).getSize(); + return fs::file_size(fs::path(disk_path) / path); } void DiskLocal::createDirectory(const String & path) { - Poco::File(disk_path + path).createDirectory(); + fs::create_directory(fs::path(disk_path) / path); } void DiskLocal::createDirectories(const String & path) { - Poco::File(disk_path + path).createDirectories(); + fs::create_directories(fs::path(disk_path) / path); } void DiskLocal::clearDirectory(const String & path) { - std::vector files; - Poco::File(disk_path + path).list(files); - for (auto & file : files) - file.remove(); + for (const auto & entry : fs::directory_iterator(fs::path(disk_path) / path)) + fs::remove(entry.path()); } void DiskLocal::moveDirectory(const String & from_path, const String & to_path) { - Poco::File(disk_path + from_path).renameTo(disk_path + to_path); + fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path); } DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) @@ -200,99 +199,95 @@ DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) void DiskLocal::moveFile(const String & from_path, const String & to_path) { - Poco::File(disk_path + from_path).renameTo(disk_path + to_path); + fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path); } void DiskLocal::replaceFile(const String & from_path, const String & to_path) { - Poco::File from_file(disk_path + from_path); - Poco::File to_file(disk_path + to_path); - if (to_file.exists()) - { - Poco::File tmp_file(disk_path + to_path + ".old"); - to_file.renameTo(tmp_file.path()); - from_file.renameTo(disk_path + to_path); - tmp_file.remove(); - } - else - from_file.renameTo(to_file.path()); + fs::path from_file = fs::path(disk_path) / from_path; + fs::path to_file = fs::path(disk_path) / to_path; + fs::rename(from_file, to_file); } std::unique_ptr DiskLocal::readFile( const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const { - return createReadBufferFromFileBase(disk_path + path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size); + return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size); } std::unique_ptr DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode) { int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1; - return std::make_unique(disk_path + path, buf_size, flags); + return std::make_unique(fs::path(disk_path) / path, buf_size, flags); } void DiskLocal::removeFile(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != unlink(fs_path.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK); + throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK); } void DiskLocal::removeFileIfExists(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != unlink(fs_path.c_str()) && errno != ENOENT) - throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK); + throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK); } void DiskLocal::removeDirectory(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != rmdir(fs_path.c_str())) - throwFromErrnoWithPath("Cannot rmdir " + fs_path, fs_path, ErrorCodes::CANNOT_RMDIR); + throwFromErrnoWithPath("Cannot rmdir " + fs_path.string(), fs_path, ErrorCodes::CANNOT_RMDIR); } void DiskLocal::removeRecursive(const String & path) { - Poco::File(disk_path + path).remove(true); + fs::remove_all(fs::path(disk_path) / path); } void DiskLocal::listFiles(const String & path, std::vector & file_names) { - Poco::File(disk_path + path).list(file_names); + file_names.clear(); + for (const auto & entry : fs::directory_iterator(fs::path(disk_path) / path)) + file_names.emplace_back(entry.path().filename()); } void DiskLocal::setLastModified(const String & path, const Poco::Timestamp & timestamp) { - Poco::File(disk_path + path).setLastModified(timestamp); + FS::setModificationTime(fs::path(disk_path) / path, timestamp.epochTime()); } Poco::Timestamp DiskLocal::getLastModified(const String & path) { - return Poco::File(disk_path + path).getLastModified(); + return FS::getModificationTimestamp(fs::path(disk_path) / path); } void DiskLocal::createHardLink(const String & src_path, const String & dst_path) { - DB::createHardLink(disk_path + src_path, disk_path + dst_path); + DB::createHardLink(fs::path(disk_path) / src_path, fs::path(disk_path) / dst_path); } void DiskLocal::truncateFile(const String & path, size_t size) { - int res = truncate((disk_path + path).c_str(), size); + int res = truncate((fs::path(disk_path) / path).string().data(), size); if (-1 == res) throwFromErrnoWithPath("Cannot truncate file " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE); } void DiskLocal::createFile(const String & path) { - Poco::File(disk_path + path).createFile(); + FS::createFile(fs::path(disk_path) / path); } void DiskLocal::setReadOnly(const String & path) { - Poco::File(disk_path + path).setReadOnly(true); + fs::permissions(fs::path(disk_path) / path, + fs::perms::owner_write | fs::perms::group_write | fs::perms::others_write, + fs::perm_options::remove); } bool inline isSameDiskType(const IDisk & one, const IDisk & another) @@ -303,14 +298,23 @@ bool inline isSameDiskType(const IDisk & one, const IDisk & another) void DiskLocal::copy(const String & from_path, const std::shared_ptr & to_disk, const String & to_path) { if (isSameDiskType(*this, *to_disk)) - Poco::File(disk_path + from_path).copyTo(to_disk->getPath() + to_path); /// Use more optimal way. + { + fs::path to = fs::path(to_disk->getPath()) / to_path; + fs::path from = fs::path(disk_path) / from_path; + if (from_path.ends_with('/')) + from = from.parent_path(); + if (fs::is_directory(from)) + to /= from.filename(); + + fs::copy(from, to, fs::copy_options::recursive | fs::copy_options::overwrite_existing); /// Use more optimal way. + } else IDisk::copy(from_path, to_disk, to_path); /// Copy files through buffers. } SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const { - return std::make_unique(disk_path + path); + return std::make_unique(fs::path(disk_path) / path); } DiskPtr DiskLocalReservation::getDisk(size_t i) const @@ -381,10 +385,8 @@ void registerDiskLocal(DiskFactory & factory) throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); } - if (Poco::File disk{path}; !disk.canRead() || !disk.canWrite()) - { + if (!FS::canRead(path) || !FS::canWrite(path)) throw Exception("There is no RW access to the disk " + name + " (" + path + ")", ErrorCodes::PATH_ACCESS_DENIED); - } bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio"); diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 567ca24eb50..47482ad8d67 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -6,8 +6,6 @@ #include #include -#include -#include namespace DB { @@ -27,7 +25,7 @@ public: : name(name_), disk_path(path_), keep_free_space_bytes(keep_free_space_bytes_) { if (disk_path.back() != '/') - throw Exception("Disk path must ends with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Disk path must end with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR); } const String & getName() const override { return name; } diff --git a/src/Disks/DiskMemory.cpp b/src/Disks/DiskMemory.cpp index 68257ec4948..423dba6bed6 100644 --- a/src/Disks/DiskMemory.cpp +++ b/src/Disks/DiskMemory.cpp @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB @@ -24,7 +23,7 @@ namespace ErrorCodes class DiskMemoryDirectoryIterator final : public IDiskDirectoryIterator { public: - explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) + explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) : dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin()) { } @@ -33,13 +32,13 @@ public: bool isValid() const override { return iter != dir_file_paths.end(); } - String path() const override { return (*iter).toString(); } + String path() const override { return iter->string(); } - String name() const override { return (*iter).getFileName(); } + String name() const override { return iter->filename(); } private: - std::vector dir_file_paths; - std::vector::iterator iter; + std::vector dir_file_paths; + std::vector::iterator iter; }; @@ -268,7 +267,7 @@ DiskDirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path) if (!path.empty() && files.find(path) == files.end()) throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); - std::vector dir_file_paths; + std::vector dir_file_paths; for (const auto & file : files) if (parentPath(file.first) == path) dir_file_paths.emplace_back(file.first); diff --git a/src/Disks/HDFS/DiskHDFS.cpp b/src/Disks/HDFS/DiskHDFS.cpp index 0648fd9f08c..da6ccb024c7 100644 --- a/src/Disks/HDFS/DiskHDFS.cpp +++ b/src/Disks/HDFS/DiskHDFS.cpp @@ -172,8 +172,8 @@ void registerDiskHDFS(DiskFactory & factory) const String & config_prefix, ContextConstPtr context_) -> DiskPtr { - Poco::File disk{context_->getPath() + "disks/" + name}; - disk.createDirectories(); + fs::path disk = fs::path(context_->getPath()) / "disks" / name; + fs::create_directories(disk); String uri{config.getString(config_prefix + ".endpoint")}; diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index ee7f57af771..82705b5dcc8 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -22,7 +22,8 @@ bool IDisk::isDirectoryEmpty(const String & path) void copyFile(IDisk & from_disk, const String & from_path, IDisk & to_disk, const String & to_path) { - LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} {} to {} {}.", from_disk.getName(), from_path, to_disk.getName(), to_path); + LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} (path: {}) {} to {} (path: {}) {}.", + from_disk.getName(), from_disk.getPath(), from_path, to_disk.getName(), to_disk.getPath(), to_path); auto in = from_disk.readFile(from_path); auto out = to_disk.writeFile(to_path); @@ -41,16 +42,15 @@ void asyncCopy(IDisk & from_disk, String from_path, IDisk & to_disk, String to_p [&from_disk, from_path, &to_disk, to_path]() { setThreadName("DiskCopier"); - DB::copyFile(from_disk, from_path, to_disk, to_path + fileName(from_path)); + DB::copyFile(from_disk, from_path, to_disk, fs::path(to_path) / fileName(from_path)); }); results.push_back(std::move(result)); } else { - Poco::Path path(from_path); - const String & dir_name = path.directory(path.depth() - 1); - const String dest = to_path + dir_name + "/"; + fs::path dir_name = fs::path(from_path).parent_path().filename(); + fs::path dest(fs::path(to_path) / dir_name); to_disk.createDirectories(dest); for (auto it = from_disk.iterateDirectory(from_path); it->isValid(); it->next()) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index a42a60959c5..cb718605dd6 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -7,16 +7,16 @@ #include #include #include -#include "Disks/Executor.h" #include #include #include #include -#include #include +#include #include "Poco/Util/AbstractConfiguration.h" +namespace fs = std::filesystem; namespace CurrentMetrics { @@ -212,10 +212,10 @@ public: virtual DiskType::Type getType() const = 0; /// Invoked when Global Context is shutdown. - virtual void shutdown() { } + virtual void shutdown() {} /// Performs action on disk startup. - virtual void startup() { } + virtual void startup() {} /// Return some uniq string for file, overrode for S3 /// Required for distinguish different copies of the same part on S3 @@ -233,7 +233,7 @@ public: virtual SyncGuardPtr getDirectorySyncGuard(const String & path) const; /// Applies new settings for disk in runtime. - virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextConstPtr) { } + virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextConstPtr) {} protected: friend class DiskDecorator; @@ -294,25 +294,27 @@ public: /// Return full path to a file on disk. inline String fullPath(const DiskPtr & disk, const String & path) { - return disk->getPath() + path; + return fs::path(disk->getPath()) / path; } /// Return parent path for the specified path. inline String parentPath(const String & path) { - return Poco::Path(path).parent().toString(); + if (path.ends_with('/')) + return fs::path(path).parent_path().parent_path() / ""; + return fs::path(path).parent_path() / ""; } /// Return file name for the specified path. inline String fileName(const String & path) { - return Poco::Path(path).getFileName(); + return fs::path(path).filename(); } /// Return directory path for the specified path. inline String directoryPath(const String & path) { - return Poco::Path(path).setFileName("").toString(); + return fs::path(path).parent_path() / ""; } } diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index bcb399f5d07..b30e9613ed8 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -6,12 +6,12 @@ #include #include #include -#include #include #include #include #include #include +#include namespace DB @@ -179,9 +179,9 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths { LOG_DEBUG(log, "Remove file by path: {}", backQuote(metadata_path + path)); - Poco::File file(metadata_path + path); + fs::path file(metadata_path + path); - if (!file.isFile()) + if (!fs::is_regular_file(file)) throw Exception(ErrorCodes::CANNOT_DELETE_DIRECTORY, "Path '{}' is a directory", path); try @@ -191,7 +191,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths /// If there is no references - delete content from remote FS. if (metadata.ref_count == 0) { - file.remove(); + fs::remove(file); for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects) fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path); } @@ -199,7 +199,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths { --metadata.ref_count; metadata.save(); - file.remove(); + fs::remove(file); } } catch (const Exception & e) @@ -210,7 +210,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths LOG_WARNING(log, "Metadata file {} can't be read by reason: {}. Removing it forcibly.", backQuote(path), e.nested() ? e.nested()->message() : e.message()); - file.remove(); + fs::remove(file); } else throw; @@ -222,8 +222,8 @@ void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. - Poco::File file(metadata_path + path); - if (file.isFile()) + fs::path file = fs::path(metadata_path) / path; + if (fs::is_regular_file(file)) { removeMeta(path, fs_paths_keeper); } @@ -231,7 +231,7 @@ void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr { for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) removeMetaRecursive(it->path(), fs_paths_keeper); - file.remove(); + fs::remove(file); } } @@ -296,13 +296,13 @@ IDiskRemote::IDiskRemote( bool IDiskRemote::exists(const String & path) const { - return Poco::File(metadata_path + path).exists(); + return fs::exists(fs::path(metadata_path) / path); } bool IDiskRemote::isFile(const String & path) const { - return Poco::File(metadata_path + path).isFile(); + return fs::is_regular_file(fs::path(metadata_path) / path); } @@ -326,7 +326,7 @@ void IDiskRemote::moveFile(const String & from_path, const String & to_path) if (exists(to_path)) throw Exception("File already exists: " + to_path, ErrorCodes::FILE_ALREADY_EXISTS); - Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path); + fs::rename(fs::path(metadata_path) / from_path, fs::path(metadata_path) / to_path); } @@ -347,7 +347,7 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path) void IDiskRemote::removeFileIfExists(const String & path) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); - if (Poco::File(metadata_path + path).exists()) + if (fs::exists(fs::path(metadata_path) / path)) { removeMeta(path, fs_paths_keeper); removeFromRemoteFS(fs_paths_keeper); @@ -385,19 +385,19 @@ void IDiskRemote::setReadOnly(const String & path) bool IDiskRemote::isDirectory(const String & path) const { - return Poco::File(metadata_path + path).isDirectory(); + return fs::is_directory(fs::path(metadata_path) / path); } void IDiskRemote::createDirectory(const String & path) { - Poco::File(metadata_path + path).createDirectory(); + fs::create_directory(fs::path(metadata_path) / path); } void IDiskRemote::createDirectories(const String & path) { - Poco::File(metadata_path + path).createDirectories(); + fs::create_directories(fs::path(metadata_path) / path); } @@ -411,7 +411,7 @@ void IDiskRemote::clearDirectory(const String & path) void IDiskRemote::removeDirectory(const String & path) { - Poco::File(metadata_path + path).remove(); + fs::remove(fs::path(metadata_path) / path); } @@ -430,13 +430,13 @@ void IDiskRemote::listFiles(const String & path, std::vector & file_name void IDiskRemote::setLastModified(const String & path, const Poco::Timestamp & timestamp) { - Poco::File(metadata_path + path).setLastModified(timestamp); + FS::setModificationTime(fs::path(metadata_path) / path, timestamp.epochTime()); } Poco::Timestamp IDiskRemote::getLastModified(const String & path) { - return Poco::File(metadata_path + path).getLastModified(); + return FS::getModificationTimestamp(fs::path(metadata_path) / path); } diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h index b32258331a7..a90621443e4 100644 --- a/src/Disks/IDiskRemote.h +++ b/src/Disks/IDiskRemote.h @@ -4,11 +4,12 @@ #include #include "Disks/DiskFactory.h" #include "Disks/Executor.h" -#include #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -193,21 +194,21 @@ public: void next() override { ++iter; } - bool isValid() const override { return iter != Poco::DirectoryIterator(); } + bool isValid() const override { return iter != fs::directory_iterator(); } String path() const override { - if (iter->isDirectory()) - return folder_path + iter.name() + '/'; + if (fs::is_directory(iter->path())) + return folder_path / iter->path().filename().string() / ""; else - return folder_path + iter.name(); + return folder_path / iter->path().filename().string(); } - String name() const override { return iter.name(); } + String name() const override { return iter->path().filename(); } private: - Poco::DirectoryIterator iter; - String folder_path; + fs::directory_iterator iter; + fs::path folder_path; }; diff --git a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp index 955986e5259..6d4764e4392 100644 --- a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp @@ -21,6 +21,7 @@ ReadIndirectBufferFromRemoteFS::ReadIndirectBufferFromRemoteFS( { } + template off_t ReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence) { diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index a3f5fe89870..89c2d20db9f 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -215,7 +214,7 @@ void DiskS3::moveFile(const String & from_path, const String & to_path, bool sen createFileOperationObject("rename", revision, object_metadata); } - Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path); + fs::rename(fs::path(metadata_path) / from_path, fs::path(metadata_path) / to_path); } std::unique_ptr DiskS3::readFile(const String & path, size_t buf_size, size_t, size_t, size_t, MMappedFileCache *) const @@ -675,8 +674,8 @@ void DiskS3::restore() restoreFiles(information); restoreFileOperations(information); - Poco::File restore_file(metadata_path + RESTORE_FILE_NAME); - restore_file.remove(); + fs::path restore_file = fs::path(metadata_path) / RESTORE_FILE_NAME; + fs::remove(restore_file); saveSchemaVersion(RESTORABLE_SCHEMA_VERSION); @@ -863,8 +862,9 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio continue; /// Skip not finished parts. They shouldn't be in 'detached' directory, because CH wouldn't be able to finish processing them. - Poco::Path directory_path (path); - auto directory_name = directory_path.directory(directory_path.depth() - 1); + fs::path directory_path(path); + auto directory_name = directory_path.parent_path().filename().string(); + auto predicate = [&directory_name](String & prefix) { return directory_name.starts_with(prefix); }; if (std::any_of(not_finished_prefixes.begin(), not_finished_prefixes.end(), predicate)) continue; @@ -873,7 +873,14 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio LOG_DEBUG(log, "Move directory to 'detached' {} -> {}", path, detached_path); - Poco::File(metadata_path + path).moveTo(metadata_path + detached_path); + fs::path from_path = fs::path(metadata_path) / path; + fs::path to_path = fs::path(metadata_path) / detached_path; + if (path.ends_with('/')) + to_path /= from_path.parent_path().filename(); + else + to_path /= from_path.filename(); + fs::copy(from_path, to_path, fs::copy_options::recursive | fs::copy_options::overwrite_existing); + fs::remove_all(from_path); } } @@ -905,7 +912,9 @@ String DiskS3::revisionToString(UInt64 revision) String DiskS3::pathToDetached(const String & source_path) { - return Poco::Path(source_path).parent().append(Poco::Path("detached")).toString() + '/'; + if (source_path.ends_with('/')) + return fs::path(source_path).parent_path().parent_path() / "detached/"; + return fs::path(source_path).parent_path() / "detached/"; } void DiskS3::onFreeze(const String & path) diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index e02f413c65e..21eff5f3d31 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -174,7 +174,7 @@ void registerDiskS3(DiskFactory & factory) throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS); String metadata_path = config.getString(config_prefix + ".metadata_path", context->getPath() + "disks/" + name + "/"); - Poco::File (metadata_path).createDirectories(); + fs::create_directories(metadata_path); std::shared_ptr s3disk = std::make_shared( name, diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index cff2685ca24..efedff9929f 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -8,8 +8,6 @@ #include -#include - namespace { diff --git a/src/Disks/WriteIndirectBufferFromRemoteFS.cpp b/src/Disks/WriteIndirectBufferFromRemoteFS.cpp index adc711608d7..6951b9fa92e 100644 --- a/src/Disks/WriteIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/WriteIndirectBufferFromRemoteFS.cpp @@ -8,7 +8,6 @@ namespace DB { -/// Stores data in S3/HDFS and adds the object key (S3 path) and object size to metadata file on local FS. template WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( std::unique_ptr impl_, diff --git a/src/Disks/WriteIndirectBufferFromRemoteFS.h b/src/Disks/WriteIndirectBufferFromRemoteFS.h index cda7523e19e..ece7b9d5871 100644 --- a/src/Disks/WriteIndirectBufferFromRemoteFS.h +++ b/src/Disks/WriteIndirectBufferFromRemoteFS.h @@ -10,7 +10,7 @@ namespace DB { -/// Stores data in S3/HDFS and adds the object key (S3 path) and object size to metadata file on local FS. +/// Stores data in S3/HDFS and adds the object path and object size to metadata file on local FS. template class WriteIndirectBufferFromRemoteFS final : public WriteBufferFromFileDecorator { diff --git a/src/Disks/tests/gtest_disk.cpp b/src/Disks/tests/gtest_disk.cpp index 714abf485ee..36f91249391 100644 --- a/src/Disks/tests/gtest_disk.cpp +++ b/src/Disks/tests/gtest_disk.cpp @@ -2,6 +2,9 @@ #include #include #include "gtest_disk.h" +#include + +namespace fs = std::filesystem; #if !defined(__clang__) @@ -22,7 +25,7 @@ DB::DiskPtr createDisk() template <> DB::DiskPtr createDisk() { - Poco::File("tmp/").createDirectory(); + fs::create_directory("tmp/"); return std::make_shared("local_disk", "tmp/", 0); } @@ -43,7 +46,7 @@ template <> void destroyDisk(DB::DiskPtr & disk) { disk.reset(); - Poco::File("tmp/").remove(true); + fs::remove_all("tmp/"); } diff --git a/src/Formats/FormatSchemaInfo.cpp b/src/Formats/FormatSchemaInfo.cpp index 707f9babe8d..2605c0bdf04 100644 --- a/src/Formats/FormatSchemaInfo.cpp +++ b/src/Formats/FormatSchemaInfo.cpp @@ -1,7 +1,7 @@ #include -#include #include #include +#include namespace DB @@ -11,6 +11,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +namespace fs = std::filesystem; namespace { @@ -34,55 +35,66 @@ FormatSchemaInfo::FormatSchemaInfo(const String & format_schema, const String & String default_file_extension = getFormatSchemaDefaultFileExtension(format); - Poco::Path path; + fs::path path; if (require_message) { size_t colon_pos = format_schema.find(':'); - if ((colon_pos == String::npos) || (colon_pos == 0) || (colon_pos == format_schema.length() - 1) - || path.assign(format_schema.substr(0, colon_pos)).makeFile().getFileName().empty()) + if ((colon_pos == String::npos) || (colon_pos == 0) || (colon_pos == format_schema.length() - 1)) { throw Exception( "Format schema requires the 'format_schema' setting to have the 'schema_file:message_name' format" + (default_file_extension.empty() ? "" : ", e.g. 'schema." + default_file_extension + ":Message'") + - ". Got '" + format_schema - + "'", - ErrorCodes::BAD_ARGUMENTS); + ". Got '" + format_schema + "'", ErrorCodes::BAD_ARGUMENTS); + } + else + { + path = fs::path(format_schema.substr(0, colon_pos)); + String filename = path.has_filename() ? path.filename() : path.parent_path().filename(); + if (filename.empty()) + throw Exception( + "Format schema requires the 'format_schema' setting to have the 'schema_file:message_name' format" + + (default_file_extension.empty() ? "" : ", e.g. 'schema." + default_file_extension + ":Message'") + + ". Got '" + format_schema + "'", ErrorCodes::BAD_ARGUMENTS); } - message_name = format_schema.substr(colon_pos + 1); } else - path.assign(format_schema).makeFile().getFileName(); + { + path = fs::path(format_schema); + if (!path.has_filename()) + path = path.parent_path() / ""; + } auto default_schema_directory = [&format_schema_path]() { - static const String str = Poco::Path(format_schema_path).makeAbsolute().makeDirectory().toString(); + static const String str = fs::canonical(format_schema_path) / ""; return str; }; - if (path.getExtension().empty() && !default_file_extension.empty()) - path.setExtension(default_file_extension); + if (!path.has_extension() && !default_file_extension.empty()) + path = path.parent_path() / (path.stem().string() + '.' + default_file_extension); - if (path.isAbsolute()) + fs::path default_schema_directory_path(default_schema_directory()); + if (path.is_absolute()) { if (is_server) - throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.toString(), ErrorCodes::BAD_ARGUMENTS); - schema_path = path.getFileName(); - schema_directory = path.makeParent().toString(); + throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.string(), ErrorCodes::BAD_ARGUMENTS); + schema_path = path.filename(); + schema_directory = path.parent_path() / ""; } - else if (path.depth() >= 1 && path.directory(0) == "..") + else if (path.has_parent_path() && !fs::weakly_canonical(default_schema_directory_path / path).string().starts_with(fs::weakly_canonical(default_schema_directory_path).string())) { if (is_server) - throw Exception( - "Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: " + path.toString(), - ErrorCodes::BAD_ARGUMENTS); - path = Poco::Path(default_schema_directory()).resolve(path).toString(); - schema_path = path.getFileName(); - schema_directory = path.makeParent().toString(); + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: {} ({} not in {})", + path.string()); + path = default_schema_directory_path / path; + schema_path = path.filename(); + schema_directory = path.parent_path() / ""; } else { - schema_path = path.toString(); + schema_path = path; schema_directory = default_schema_directory(); } } diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index 29aa5913b83..6a8e3d2d94a 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -61,7 +61,7 @@ class FunctionBase64Conversion : public IFunction public: static constexpr auto name = Func::name; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 927b870891f..4ad876fc461 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -514,7 +514,7 @@ class FunctionBinaryArithmetic : public IFunction static constexpr const bool is_multiply = IsOperation::multiply; static constexpr const bool is_division = IsOperation::division; - ContextPtr context; + ContextConstPtr context; bool check_decimal_overflow = true; template @@ -597,7 +597,7 @@ class FunctionBinaryArithmetic : public IFunction } static FunctionOverloadResolverPtr - getFunctionForIntervalArithmetic(const DataTypePtr & type0, const DataTypePtr & type1, ContextPtr context) + getFunctionForIntervalArithmetic(const DataTypePtr & type0, const DataTypePtr & type1, ContextConstPtr context) { bool first_is_date_or_datetime = isDateOrDateTime(type0); bool second_is_date_or_datetime = isDateOrDateTime(type1); @@ -945,9 +945,9 @@ class FunctionBinaryArithmetic : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionBinaryArithmetic(ContextPtr context_) + explicit FunctionBinaryArithmetic(ContextConstPtr context_) : context(context_), check_decimal_overflow(decimalCheckArithmeticOverflow(context)) {} @@ -961,7 +961,7 @@ public: return getReturnTypeImplStatic(arguments, context); } - static DataTypePtr getReturnTypeImplStatic(const DataTypes & arguments, ContextPtr context) + static DataTypePtr getReturnTypeImplStatic(const DataTypes & arguments, ContextConstPtr context) { /// Special case when multiply aggregate function state if (isAggregateMultiply(arguments[0], arguments[1])) @@ -1373,7 +1373,7 @@ public: const ColumnWithTypeAndName & left_, const ColumnWithTypeAndName & right_, const DataTypePtr & return_type_, - ContextPtr context) + ContextConstPtr context) { return std::make_shared(left_, right_, return_type_, context); } @@ -1382,7 +1382,7 @@ public: const ColumnWithTypeAndName & left_, const ColumnWithTypeAndName & right_, const DataTypePtr & return_type_, - ContextPtr context_) + ContextConstPtr context_) : Base(context_), left(left_), right(right_), return_type(return_type_) { } @@ -1536,12 +1536,12 @@ class BinaryArithmeticOverloadResolver : public IFunctionOverloadResolver { public: static constexpr auto name = Name::name; - static FunctionOverloadResolverPtr create(ContextPtr context) + static FunctionOverloadResolverPtr create(ContextConstPtr context) { return std::make_unique(context); } - explicit BinaryArithmeticOverloadResolver(ContextPtr context_) : context(context_) {} + explicit BinaryArithmeticOverloadResolver(ContextConstPtr context_) : context(context_) {} String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } @@ -1577,6 +1577,6 @@ public: } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/FunctionBitTestMany.h b/src/Functions/FunctionBitTestMany.h index 567be37cda8..030919e8836 100644 --- a/src/Functions/FunctionBitTestMany.h +++ b/src/Functions/FunctionBitTestMany.h @@ -24,7 +24,7 @@ struct FunctionBitTestMany : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 84ecd4d7a5a..d30ce7b5c86 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -24,7 +24,7 @@ class FunctionCustomWeekToSomething : public IFunction { public: static constexpr auto name = Transform::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 14ab9f00452..4418f6e4b4b 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -352,7 +352,7 @@ class FunctionDateOrDateTimeAddInterval : public IFunction { public: static constexpr auto name = Transform::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index 8b8f1b483c8..bdf3f425a0b 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -24,7 +24,7 @@ class FunctionDateOrDateTimeToSomething : public IFunction { public: static constexpr auto name = Transform::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/FunctionFQDN.cpp b/src/Functions/FunctionFQDN.cpp index 304aad17d22..36fcdfacc99 100644 --- a/src/Functions/FunctionFQDN.cpp +++ b/src/Functions/FunctionFQDN.cpp @@ -12,7 +12,7 @@ class FunctionFQDN : public IFunction { public: static constexpr auto name = "FQDN"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/FunctionFactory.cpp b/src/Functions/FunctionFactory.cpp index b1437d58c09..1bb78ead872 100644 --- a/src/Functions/FunctionFactory.cpp +++ b/src/Functions/FunctionFactory.cpp @@ -52,7 +52,7 @@ void FunctionFactory::registerFunction(const FunctionOverloadResolverPtr FunctionFactory::getImpl( const std::string & name, - ContextPtr context) const + ContextConstPtr context) const { auto res = tryGetImpl(name, context); if (!res) @@ -82,14 +82,14 @@ std::vector FunctionFactory::getAllNames() const FunctionOverloadResolverPtr FunctionFactory::get( const std::string & name, - ContextPtr context) const + ContextConstPtr context) const { return getImpl(name, context); } FunctionOverloadResolverPtr FunctionFactory::tryGetImpl( const std::string & name_param, - ContextPtr context) const + ContextConstPtr context) const { String name = getAliasToOrName(name_param); FunctionOverloadResolverPtr res; @@ -120,7 +120,7 @@ FunctionOverloadResolverPtr FunctionFactory::tryGetImpl( FunctionOverloadResolverPtr FunctionFactory::tryGet( const std::string & name, - ContextPtr context) const + ContextConstPtr context) const { auto impl = tryGetImpl(name, context); return impl ? std::move(impl) : nullptr; diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h index bac55dade16..bf380d51312 100644 --- a/src/Functions/FunctionFactory.h +++ b/src/Functions/FunctionFactory.h @@ -19,7 +19,7 @@ namespace DB * some dictionaries from Context. */ class FunctionFactory : private boost::noncopyable, - public IFactoryWithAliases> + public IFactoryWithAliases> { public: static FunctionFactory & instance(); @@ -44,14 +44,14 @@ public: std::vector getAllNames() const; /// Throws an exception if not found. - FunctionOverloadResolverPtr get(const std::string & name, ContextPtr context) const; + FunctionOverloadResolverPtr get(const std::string & name, ContextConstPtr context) const; /// Returns nullptr if not found. - FunctionOverloadResolverPtr tryGet(const std::string & name, ContextPtr context) const; + FunctionOverloadResolverPtr tryGet(const std::string & name, ContextConstPtr context) const; /// The same methods to get developer interface implementation. - FunctionOverloadResolverPtr getImpl(const std::string & name, ContextPtr context) const; - FunctionOverloadResolverPtr tryGetImpl(const std::string & name, ContextPtr context) const; + FunctionOverloadResolverPtr getImpl(const std::string & name, ContextConstPtr context) const; + FunctionOverloadResolverPtr tryGetImpl(const std::string & name, ContextConstPtr context) const; /// Register a function by its name. /// No locking, you must register all functions before usage of get. @@ -67,7 +67,7 @@ private: Functions case_insensitive_functions; template - static FunctionOverloadResolverPtr adaptFunctionToOverloadResolver(ContextPtr context) + static FunctionOverloadResolverPtr adaptFunctionToOverloadResolver(ContextConstPtr context) { return std::make_unique(Function::create(context)); } diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index 9247152367b..cd81f2b4e0b 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -3,10 +3,11 @@ #include #include #include -#include -#include #include #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -21,12 +22,12 @@ namespace ErrorCodes } /// A function to read file as a string. -class FunctionFile : public IFunction, WithContext +class FunctionFile : public IFunction, WithConstContext { public: static constexpr auto name = "file"; - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } - explicit FunctionFile(ContextPtr context_) : WithContext(context_) {} + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } + explicit FunctionFile(ContextConstPtr context_) : WithConstContext(context_) {} String getName() const override { return name; } @@ -68,21 +69,19 @@ public: { const char * filename = reinterpret_cast(&chars[source_offset]); - const String user_files_path = getContext()->getUserFilesPath(); - String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); - Poco::Path poco_filepath = Poco::Path(filename); - if (poco_filepath.isRelative()) - poco_filepath = Poco::Path(user_files_absolute_path, poco_filepath); - const String file_absolute_path = poco_filepath.absolute().toString(); - checkReadIsAllowedOrThrow(user_files_absolute_path, file_absolute_path); + fs::path user_files_absolute_path = fs::canonical(fs::path(getContext()->getUserFilesPath())); + fs::path file_path(filename); + if (file_path.is_relative()) + file_path = user_files_absolute_path / file_path; + fs::path file_absolute_path = fs::canonical(file_path); + checkReadIsAllowedOrThrow(user_files_absolute_path.string(), file_absolute_path); - checked_filenames[row] = file_absolute_path; - auto file = Poco::File(file_absolute_path); + checked_filenames[row] = file_absolute_path.string(); - if (!file.exists()) - throw Exception(fmt::format("File {} doesn't exist.", file_absolute_path), ErrorCodes::FILE_DOESNT_EXIST); + if (!fs::exists(file_absolute_path)) + throw Exception(fmt::format("File {} doesn't exist.", file_absolute_path.string()), ErrorCodes::FILE_DOESNT_EXIST); - const auto current_file_size = Poco::File(file_absolute_path).getSize(); + const auto current_file_size = fs::file_size(file_absolute_path); result_offset += current_file_size + 1; res_offsets[row] = result_offset; @@ -117,8 +116,8 @@ private: if (file_absolute_path.find(user_files_absolute_path) != 0) throw Exception("File is not inside " + user_files_absolute_path, ErrorCodes::DATABASE_ACCESS_DENIED); - Poco::File path_poco_file = Poco::File(file_absolute_path); - if (path_poco_file.exists() && path_poco_file.isDirectory()) + fs::path fs_path(file_absolute_path); + if (fs::exists(fs_path) && fs::is_directory(fs_path)) throw Exception("File can't be a directory", ErrorCodes::INCORRECT_FILE_NAME); } }; diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 81545143839..3a17d049099 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -36,7 +36,7 @@ ExecutableFunctionPtr FunctionJoinGet::prepare(const ColumnsWithTypeAnd } static std::pair, String> -getJoin(const ColumnsWithTypeAndName & arguments, ContextPtr context) +getJoin(const ColumnsWithTypeAndName & arguments, ContextConstPtr context) { String join_name; if (const auto * name_col = checkAndGetColumnConst(arguments[0].column.get())) @@ -63,7 +63,7 @@ getJoin(const ColumnsWithTypeAndName & arguments, ContextPtr context) String table_name = join_name.substr(dot); if (table_name.empty()) throw Exception("joinGet does not allow empty table name", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - auto table = DatabaseCatalog::instance().getTable({database_name, table_name}, context); + auto table = DatabaseCatalog::instance().getTable({database_name, table_name}, std::const_pointer_cast(context)); auto storage_join = std::dynamic_pointer_cast(table); if (!storage_join) throw Exception("Table " + join_name + " should have engine StorageJoin", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h index 2250fa3ccf0..0328d1a16e1 100644 --- a/src/Functions/FunctionJoinGet.h +++ b/src/Functions/FunctionJoinGet.h @@ -74,13 +74,13 @@ private: }; template -class JoinGetOverloadResolver final : public IFunctionOverloadResolver, WithContext +class JoinGetOverloadResolver final : public IFunctionOverloadResolver, WithConstContext { public: static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet"; - static FunctionOverloadResolverPtr create(ContextPtr context_) { return std::make_unique(context_); } + static FunctionOverloadResolverPtr create(ContextConstPtr context_) { return std::make_unique(context_); } - explicit JoinGetOverloadResolver(ContextPtr context_) : WithContext(context_) {} + explicit JoinGetOverloadResolver(ContextConstPtr context_) : WithConstContext(context_) {} String getName() const override { return name; } diff --git a/src/Functions/FunctionMathBinaryFloat64.h b/src/Functions/FunctionMathBinaryFloat64.h index 8cc012d3ab2..340470fa0b8 100644 --- a/src/Functions/FunctionMathBinaryFloat64.h +++ b/src/Functions/FunctionMathBinaryFloat64.h @@ -28,7 +28,7 @@ class FunctionMathBinaryFloat64 : public IFunction { public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } static_assert(Impl::rows_per_iteration > 0, "Impl must process at least one row per iteration"); bool useDefaultImplementationForConstants() const override { return true; } diff --git a/src/Functions/FunctionMathConstFloat64.h b/src/Functions/FunctionMathConstFloat64.h index ab7d401e99e..b85b65fc43d 100644 --- a/src/Functions/FunctionMathConstFloat64.h +++ b/src/Functions/FunctionMathConstFloat64.h @@ -13,7 +13,7 @@ class FunctionMathConstFloat64 : public IFunction { public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: String getName() const override { return name; } diff --git a/src/Functions/FunctionMathUnary.h b/src/Functions/FunctionMathUnary.h index a637bbe3bd8..d7804af4f7a 100644 --- a/src/Functions/FunctionMathUnary.h +++ b/src/Functions/FunctionMathUnary.h @@ -35,7 +35,7 @@ class FunctionMathUnary : public IFunction { public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: String getName() const override { return name; } diff --git a/src/Functions/FunctionNumericPredicate.h b/src/Functions/FunctionNumericPredicate.h index d5fa9f03aba..006f74f531a 100644 --- a/src/Functions/FunctionNumericPredicate.h +++ b/src/Functions/FunctionNumericPredicate.h @@ -23,7 +23,7 @@ class FunctionNumericPredicate : public IFunction { public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h index 65dbf393290..cf3a406498a 100644 --- a/src/Functions/FunctionStartsEndsWith.h +++ b/src/Functions/FunctionStartsEndsWith.h @@ -142,7 +142,7 @@ template class FunctionStartsEndsWith : public TargetSpecific::Default::FunctionStartsEndsWith { public: - explicit FunctionStartsEndsWith(ContextPtr context) : selector(context) + explicit FunctionStartsEndsWith(ContextConstPtr context) : selector(context) { selector.registerImplementation>(); @@ -164,7 +164,7 @@ public: return selector.selectAndExecute(arguments, result_type, input_rows_count); } - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared>(context); } diff --git a/src/Functions/FunctionStringOrArrayToT.h b/src/Functions/FunctionStringOrArrayToT.h index 158179fffe9..ceb58a5f7f6 100644 --- a/src/Functions/FunctionStringOrArrayToT.h +++ b/src/Functions/FunctionStringOrArrayToT.h @@ -25,7 +25,7 @@ class FunctionStringOrArrayToT : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/FunctionStringReplace.h b/src/Functions/FunctionStringReplace.h index 2e99f58531a..2ff09f79f9e 100644 --- a/src/Functions/FunctionStringReplace.h +++ b/src/Functions/FunctionStringReplace.h @@ -23,7 +23,7 @@ class FunctionStringReplace : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionStringToString.h b/src/Functions/FunctionStringToString.h index 26480a83995..9b63eec4a60 100644 --- a/src/Functions/FunctionStringToString.h +++ b/src/Functions/FunctionStringToString.h @@ -23,7 +23,7 @@ class FunctionStringToString : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 68fcfadfb84..4b5fdf86078 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -111,7 +111,7 @@ class FunctionUnaryArithmetic : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index b76b454fd77..4dfe5cdea91 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -141,7 +141,7 @@ class FunctionEncrypt : public IFunction public: static constexpr OpenSSLDetails::CompatibilityMode compatibility_mode = Impl::compatibility_mode; static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: using CipherMode = OpenSSLDetails::CipherMode; @@ -416,7 +416,7 @@ class FunctionDecrypt : public IFunction public: static constexpr OpenSSLDetails::CompatibilityMode compatibility_mode = Impl::compatibility_mode; static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: using CipherMode = OpenSSLDetails::CipherMode; diff --git a/src/Functions/FunctionsBitmap.h b/src/Functions/FunctionsBitmap.h index bbdc53c3006..368e5bccbd1 100644 --- a/src/Functions/FunctionsBitmap.h +++ b/src/Functions/FunctionsBitmap.h @@ -93,7 +93,7 @@ class FunctionBitmapBuildImpl : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -221,7 +221,7 @@ class FunctionBitmapToArrayImpl : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -311,7 +311,7 @@ class FunctionBitmapSubset : public IFunction public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } String getName() const override { return name; } @@ -469,7 +469,7 @@ class FunctionBitmapTransform : public IFunction public: static constexpr auto name = "bitmapTransform"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -635,7 +635,7 @@ class FunctionBitmapSelfCardinalityImpl : public IFunction public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } String getName() const override { return name; } @@ -807,7 +807,7 @@ class FunctionBitmapContains : public IFunction public: static constexpr auto name = "bitmapContains"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -911,7 +911,7 @@ class FunctionBitmapCardinality : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -1054,7 +1054,7 @@ class FunctionBitmap : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 01602cf1175..6c9cd6d333b 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -69,7 +69,7 @@ class FunctionIPv6NumToString : public IFunction { public: static constexpr auto name = "IPv6NumToString"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -139,7 +139,7 @@ class FunctionCutIPv6 : public IFunction { public: static constexpr auto name = "cutIPv6"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -262,7 +262,7 @@ class FunctionIPv6StringToNum : public IFunction { public: static constexpr auto name = "IPv6StringToNum"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } static inline bool tryParseIPv4(const char * pos) { @@ -340,7 +340,7 @@ class FunctionIPv4NumToString : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } String getName() const override { @@ -401,7 +401,7 @@ class FunctionIPv4StringToNum : public IFunction { public: static constexpr auto name = "IPv4StringToNum"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -464,7 +464,7 @@ class FunctionIPv4ToIPv6 : public IFunction { public: static constexpr auto name = "IPv4ToIPv6"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -519,7 +519,7 @@ class FunctionToIPv4 : public FunctionIPv4StringToNum { public: static constexpr auto name = "toIPv4"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -542,7 +542,7 @@ class FunctionToIPv6 : public FunctionIPv6StringToNum { public: static constexpr auto name = "toIPv6"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -560,7 +560,7 @@ class FunctionMACNumToString : public IFunction { public: static constexpr auto name = "MACNumToString"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -690,7 +690,7 @@ class FunctionMACStringTo : public IFunction { public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } String getName() const override { @@ -753,7 +753,7 @@ class FunctionUUIDNumToString : public IFunction public: static constexpr auto name = "UUIDNumToString"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -851,7 +851,7 @@ private: public: static constexpr auto name = "UUIDStringToNum"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -955,7 +955,7 @@ class FunctionHex : public IFunction { public: static constexpr auto name = "hex"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -1238,7 +1238,7 @@ class FunctionUnhex : public IFunction { public: static constexpr auto name = "unhex"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -1327,7 +1327,7 @@ class FunctionChar : public IFunction { public: static constexpr auto name = "char"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -1422,7 +1422,7 @@ class FunctionBitmaskToArray : public IFunction { public: static constexpr auto name = "bitmaskToArray"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -1508,7 +1508,7 @@ class FunctionToStringCutToZero : public IFunction { public: static constexpr auto name = "toStringCutToZero"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -1673,7 +1673,7 @@ private: public: static constexpr auto name = "IPv6CIDRToRange"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } @@ -1787,7 +1787,7 @@ private: public: static constexpr auto name = "IPv4CIDRToRange"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } @@ -1868,7 +1868,7 @@ class FunctionIsIPv4String : public FunctionIPv4StringToNum public: static constexpr auto name = "isIPv4String"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -1914,7 +1914,7 @@ class FunctionIsIPv6String : public FunctionIPv6StringToNum public: static constexpr auto name = "isIPv6String"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 9ffb0cd0fc3..0696b0fd335 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -557,13 +557,13 @@ class FunctionComparison : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionComparison(ContextPtr context_) + explicit FunctionComparison(ContextConstPtr context_) : context(context_), check_decimal_overflow(decimalCheckComparisonOverflow(context)) {} private: - ContextPtr context; + ContextConstPtr context; bool check_decimal_overflow = true; template diff --git a/src/Functions/FunctionsConsistentHashing.h b/src/Functions/FunctionsConsistentHashing.h index 4c393f6ee01..69163917430 100644 --- a/src/Functions/FunctionsConsistentHashing.h +++ b/src/Functions/FunctionsConsistentHashing.h @@ -24,7 +24,7 @@ class FunctionConsistentHashImpl : public IFunction public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 589bd4cc16e..5af86d9ff8b 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -1277,7 +1277,7 @@ public: static constexpr bool to_string_or_fixed_string = std::is_same_v || std::is_same_v; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } static FunctionPtr create() { return std::make_shared(); } String getName() const override @@ -1592,7 +1592,7 @@ public: static constexpr bool to_datetime64 = std::is_same_v; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } static FunctionPtr create() { return std::make_shared(); } String getName() const override @@ -3196,7 +3196,7 @@ public: ? accurate_cast_name : (cast_type == CastType::accurateOrNull ? accurate_cast_or_null_name : cast_name); - static FunctionOverloadResolverPtr create(ContextPtr context) + static FunctionOverloadResolverPtr create(ContextConstPtr context) { return createImpl(context->getSettingsRef().cast_keep_nullable); } diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index 884f53125eb..a19c6226f91 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -483,7 +483,7 @@ struct NameRegionIn { static constexpr auto name = "regionIn" struct FunctionRegionToCity : public FunctionTransformWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -492,7 +492,7 @@ struct FunctionRegionToCity : struct FunctionRegionToArea : public FunctionTransformWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -501,7 +501,7 @@ struct FunctionRegionToArea : struct FunctionRegionToDistrict : public FunctionTransformWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -510,7 +510,7 @@ struct FunctionRegionToDistrict : struct FunctionRegionToCountry : public FunctionTransformWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -519,7 +519,7 @@ struct FunctionRegionToCountry : struct FunctionRegionToContinent : public FunctionTransformWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -528,7 +528,7 @@ struct FunctionRegionToContinent : struct FunctionRegionToTopContinent : public FunctionTransformWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -537,7 +537,7 @@ struct FunctionRegionToTopContinent : struct FunctionRegionToPopulation : public FunctionTransformWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -546,7 +546,7 @@ struct FunctionRegionToPopulation : struct FunctionRegionIn : public FunctionIsInWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -555,7 +555,7 @@ struct FunctionRegionIn : struct FunctionRegionHierarchy : public FunctionHierarchyWithDictionary { - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsHierarchies()); } @@ -567,7 +567,7 @@ class FunctionRegionToName : public IFunction { public: static constexpr auto name = "regionToName"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getEmbeddedDictionaries().getRegionsNames()); } diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 6d9e4ce6768..89c76f93061 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -62,10 +62,10 @@ namespace ErrorCodes */ -class FunctionDictHelper : WithContext +class FunctionDictHelper : WithConstContext { public: - explicit FunctionDictHelper(ContextPtr context_) : WithContext(context_) {} + explicit FunctionDictHelper(ContextConstPtr context_) : WithConstContext(context_) {} std::shared_ptr getDictionary(const String & dictionary_name) { @@ -132,12 +132,12 @@ class FunctionDictHas final : public IFunction public: static constexpr auto name = "dictHas"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionDictHas(ContextPtr context_) : helper(context_) {} + explicit FunctionDictHas(ContextConstPtr context_) : helper(context_) {} String getName() const override { return name; } @@ -270,12 +270,12 @@ class FunctionDictGetNoType final : public IFunction public: static constexpr auto name = dictionary_get_function_type == DictionaryGetFunctionType::get ? "dictGet" : "dictGetOrDefault"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionDictGetNoType(ContextPtr context_) : helper(context_) {} + explicit FunctionDictGetNoType(ContextConstPtr context_) : helper(context_) {} String getName() const override { return name; } @@ -604,12 +604,12 @@ class FunctionDictGetImpl final : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionDictGetImpl(ContextPtr context_) : impl(context_) {} + explicit FunctionDictGetImpl(ContextConstPtr context_) : impl(context_) {} String getName() const override { return name; } @@ -743,12 +743,12 @@ class FunctionDictGetOrNull final : public IFunction public: static constexpr auto name = "dictGetOrNull"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionDictGetOrNull(ContextPtr context_) + explicit FunctionDictGetOrNull(ContextConstPtr context_) : dictionary_get_func_impl(context_) , dictionary_has_func_impl(context_) {} @@ -906,12 +906,12 @@ class FunctionDictGetHierarchy final : public IFunction public: static constexpr auto name = "dictGetHierarchy"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionDictGetHierarchy(ContextPtr context_) : helper(context_) {} + explicit FunctionDictGetHierarchy(ContextConstPtr context_) : helper(context_) {} String getName() const override { return name; } @@ -966,12 +966,12 @@ class FunctionDictIsIn final : public IFunction public: static constexpr auto name = "dictIsIn"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionDictIsIn(ContextPtr context_) + explicit FunctionDictIsIn(ContextConstPtr context_) : helper(context_) {} String getName() const override { return name; } @@ -1032,12 +1032,12 @@ class FunctionDictGetChildren final : public IFunction public: static constexpr auto name = "dictGetChildren"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionDictGetChildren(ContextPtr context_) + explicit FunctionDictGetChildren(ContextConstPtr context_) : helper(context_) {} String getName() const override { return name; } @@ -1091,12 +1091,12 @@ class FunctionDictGetDescendants final : public IFunction public: static constexpr auto name = "dictGetDescendants"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionDictGetDescendants(ContextPtr context_) + explicit FunctionDictGetDescendants(ContextConstPtr context_) : helper(context_) {} String getName() const override { return name; } diff --git a/src/Functions/FunctionsExternalModels.cpp b/src/Functions/FunctionsExternalModels.cpp index da46f7fcb38..6bcaf13cd69 100644 --- a/src/Functions/FunctionsExternalModels.cpp +++ b/src/Functions/FunctionsExternalModels.cpp @@ -18,7 +18,7 @@ namespace DB { -FunctionPtr FunctionModelEvaluate::create(ContextPtr context) +FunctionPtr FunctionModelEvaluate::create(ContextConstPtr context) { return std::make_shared(context->getExternalModelsLoader()); } diff --git a/src/Functions/FunctionsExternalModels.h b/src/Functions/FunctionsExternalModels.h index 8f8b0e0c860..b9d611934e6 100644 --- a/src/Functions/FunctionsExternalModels.h +++ b/src/Functions/FunctionsExternalModels.h @@ -17,7 +17,7 @@ class FunctionModelEvaluate final : public IFunction public: static constexpr auto name = "modelEvaluate"; - static FunctionPtr create(ContextPtr context); + static FunctionPtr create(ContextConstPtr context); explicit FunctionModelEvaluate(const ExternalModelsLoader & models_loader_) : models_loader(models_loader_) {} diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 690991759a3..25f57e06f2e 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -535,7 +535,7 @@ class FunctionStringHashFixedString : public IFunction { public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -700,7 +700,7 @@ template class FunctionIntHash : public TargetSpecific::Default::FunctionIntHash { public: - explicit FunctionIntHash(ContextPtr context) : selector(context) + explicit FunctionIntHash(ContextConstPtr context) : selector(context) { selector.registerImplementation>(); @@ -718,7 +718,7 @@ public: return selector.selectAndExecute(arguments, result_type, input_rows_count); } - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } @@ -1077,7 +1077,7 @@ template class FunctionAnyHash : public TargetSpecific::Default::FunctionAnyHash { public: - explicit FunctionAnyHash(ContextPtr context) : selector(context) + explicit FunctionAnyHash(ContextConstPtr context) : selector(context) { selector.registerImplementation>(); @@ -1095,7 +1095,7 @@ public: return selector.selectAndExecute(arguments, result_type, input_rows_count); } - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } @@ -1182,7 +1182,7 @@ class FunctionURLHash : public IFunction { public: static constexpr auto name = "URLHash"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 2b49d01de2c..6024658dd6f 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -270,11 +270,11 @@ private: template typename Impl> -class FunctionJSON : public IFunction, WithContext +class FunctionJSON : public IFunction, WithConstContext { public: - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } - FunctionJSON(ContextPtr context_) : WithContext(context_) {} + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } + FunctionJSON(ContextConstPtr context_) : WithConstContext(context_) {} static constexpr auto name = Name::name; String getName() const override { return Name::name; } diff --git a/src/Functions/FunctionsLogical.h b/src/Functions/FunctionsLogical.h index 264eeeef0fe..339cc25510d 100644 --- a/src/Functions/FunctionsLogical.h +++ b/src/Functions/FunctionsLogical.h @@ -139,7 +139,7 @@ class FunctionAnyArityLogical : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } public: String getName() const override @@ -206,7 +206,7 @@ class FunctionUnaryLogical : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } public: String getName() const override diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 209efb0fc2f..3c6e67c1546 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -34,7 +34,7 @@ class FunctionsMultiStringFuzzySearch : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { if (Impl::is_using_hyperscan && !context->getSettingsRef().allow_hyperscan) throw Exception( diff --git a/src/Functions/FunctionsMultiStringPosition.h b/src/Functions/FunctionsMultiStringPosition.h index f36f7639ccd..68c658c0aa7 100644 --- a/src/Functions/FunctionsMultiStringPosition.h +++ b/src/Functions/FunctionsMultiStringPosition.h @@ -51,7 +51,7 @@ class FunctionsMultiStringPosition : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 08b4668940e..4d391124de9 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -47,7 +47,7 @@ class FunctionsMultiStringSearch : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { if (Impl::is_using_hyperscan && !context->getSettingsRef().allow_hyperscan) throw Exception( diff --git a/src/Functions/FunctionsRandom.h b/src/Functions/FunctionsRandom.h index 75037d02a2d..0eae9ef183f 100644 --- a/src/Functions/FunctionsRandom.h +++ b/src/Functions/FunctionsRandom.h @@ -91,7 +91,7 @@ template class FunctionRandom : public FunctionRandomImpl { public: - explicit FunctionRandom(ContextPtr context) : selector(context) + explicit FunctionRandom(ContextConstPtr context) : selector(context) { selector.registerImplementation>(); @@ -107,7 +107,7 @@ public: return selector.selectAndExecute(arguments, result_type, input_rows_count); } - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared>(context); } diff --git a/src/Functions/FunctionsRound.h b/src/Functions/FunctionsRound.h index f56b92d6db5..98f35e52a4c 100644 --- a/src/Functions/FunctionsRound.h +++ b/src/Functions/FunctionsRound.h @@ -520,7 +520,7 @@ class FunctionRounding : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -629,7 +629,7 @@ class FunctionRoundDown : public IFunction { public: static constexpr auto name = "roundDown"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionsStringArray.h b/src/Functions/FunctionsStringArray.h index 27f10797651..2141adbc72f 100644 --- a/src/Functions/FunctionsStringArray.h +++ b/src/Functions/FunctionsStringArray.h @@ -427,7 +427,7 @@ class FunctionTokens : public IFunction { public: static constexpr auto name = Generator::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -590,7 +590,7 @@ private: public: static constexpr auto name = "arrayStringConcat"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/FunctionsStringHash.h b/src/Functions/FunctionsStringHash.h index 37fa7d618b9..8336a4ee85f 100644 --- a/src/Functions/FunctionsStringHash.h +++ b/src/Functions/FunctionsStringHash.h @@ -35,7 +35,7 @@ public: static constexpr size_t max_shingle_size = 25; static constexpr size_t max_num_hashes = 25; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h index 0789247e2d4..524fe43f42f 100644 --- a/src/Functions/FunctionsStringSearch.h +++ b/src/Functions/FunctionsStringSearch.h @@ -51,7 +51,7 @@ class FunctionsStringSearch : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionsStringSearchToString.h b/src/Functions/FunctionsStringSearchToString.h index af91a9511e1..3b3383cf162 100644 --- a/src/Functions/FunctionsStringSearchToString.h +++ b/src/Functions/FunctionsStringSearchToString.h @@ -36,7 +36,7 @@ class FunctionsStringSearchToString : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/FunctionsStringSimilarity.h b/src/Functions/FunctionsStringSimilarity.h index 6efb373aace..49fc8a21129 100644 --- a/src/Functions/FunctionsStringSimilarity.h +++ b/src/Functions/FunctionsStringSimilarity.h @@ -33,7 +33,7 @@ class FunctionsStringSimilarity : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/LeastGreatestGeneric.h b/src/Functions/LeastGreatestGeneric.h index 51564ad7cdc..f33bda801cb 100644 --- a/src/Functions/LeastGreatestGeneric.h +++ b/src/Functions/LeastGreatestGeneric.h @@ -30,7 +30,7 @@ class FunctionLeastGreatestGeneric : public IFunction { public: static constexpr auto name = kind == LeastGreatest::Least ? "least" : "greatest"; - static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } private: String getName() const override { return name; } @@ -92,12 +92,12 @@ class LeastGreatestOverloadResolver : public IFunctionOverloadResolver public: static constexpr auto name = kind == LeastGreatest::Least ? "least" : "greatest"; - static FunctionOverloadResolverPtr create(ContextPtr context) + static FunctionOverloadResolverPtr create(ContextConstPtr context) { return std::make_unique>(context); } - explicit LeastGreatestOverloadResolver(ContextPtr context_) : context(context_) {} + explicit LeastGreatestOverloadResolver(ContextConstPtr context_) : context(context_) {} String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -127,7 +127,7 @@ public: } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h index 9ef6454d085..04bf19bbd88 100644 --- a/src/Functions/PerformanceAdaptors.h +++ b/src/Functions/PerformanceAdaptors.h @@ -172,7 +172,7 @@ namespace detail * /// default implementation. * class MyFunction : public MyDefaultImpl * { - * MyFunction(ContextPtr context) : selector(context) { + * MyFunction(ContextConstPtr context) : selector(context) { * /// Register all implementations in constructor. * /// There could be as many implementation for every target as you want. * selector.registerImplementation(); @@ -185,7 +185,7 @@ namespace detail * selector.selectAndExecute(...); * } * - * static FunctionPtr create(ContextPtr context) { + * static FunctionPtr create(ContextConstPtr context) { * return std::make_shared(context); * } * private: @@ -193,12 +193,12 @@ namespace detail * }; */ template -class ImplementationSelector : WithContext +class ImplementationSelector : WithConstContext { public: using ImplementationPtr = std::shared_ptr; - ImplementationSelector(ContextPtr context_) : WithContext(context_) {} + ImplementationSelector(ContextConstPtr context_) : WithConstContext(context_) {} /* Select the best implementation based on previous runs. * If FunctionInterface is IFunction, then "executeImpl" method of the implementation will be called diff --git a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h index 4670d610725..fa985f194f3 100644 --- a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h +++ b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h @@ -36,7 +36,7 @@ class FunctionCutToFirstSignificantSubdomainCustomImpl : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp index 179a2be4471..6f277de45a5 100644 --- a/src/Functions/URL/port.cpp +++ b/src/Functions/URL/port.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes struct FunctionPort : public IFunction { static constexpr auto name = "port"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/abtesting.cpp b/src/Functions/abtesting.cpp index 871357fe450..fa0e5e8a6bd 100644 --- a/src/Functions/abtesting.cpp +++ b/src/Functions/abtesting.cpp @@ -168,7 +168,7 @@ class FunctionBayesAB : public IFunction public: static constexpr auto name = "bayesAB"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/addressToLine.cpp b/src/Functions/addressToLine.cpp index aabf388b428..1ae8d9126bb 100644 --- a/src/Functions/addressToLine.cpp +++ b/src/Functions/addressToLine.cpp @@ -36,7 +36,7 @@ class FunctionAddressToLine : public IFunction { public: static constexpr auto name = "addressToLine"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { context->checkAccess(AccessType::addressToLine); return std::make_shared(); diff --git a/src/Functions/addressToSymbol.cpp b/src/Functions/addressToSymbol.cpp index 7ffdc6d4260..1c744a11755 100644 --- a/src/Functions/addressToSymbol.cpp +++ b/src/Functions/addressToSymbol.cpp @@ -28,7 +28,7 @@ class FunctionAddressToSymbol : public IFunction { public: static constexpr auto name = "addressToSymbol"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { context->checkAccess(AccessType::addressToSymbol); return std::make_shared(); diff --git a/src/Functions/appendTrailingCharIfAbsent.cpp b/src/Functions/appendTrailingCharIfAbsent.cpp index 266b2a44f4a..89d7f12f780 100644 --- a/src/Functions/appendTrailingCharIfAbsent.cpp +++ b/src/Functions/appendTrailingCharIfAbsent.cpp @@ -24,7 +24,7 @@ class FunctionAppendTrailingCharIfAbsent : public IFunction { public: static constexpr auto name = "appendTrailingCharIfAbsent"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 3609398bc3f..9b51000144f 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -44,7 +44,7 @@ class FunctionArrayMapped : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 1f513defc6f..62ae0b4f2d8 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -14,7 +14,7 @@ class FunctionArray : public IFunction { public: static constexpr auto name = "array"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/array/arrayConcat.cpp b/src/Functions/array/arrayConcat.cpp index 57a783a3244..23193e28816 100644 --- a/src/Functions/array/arrayConcat.cpp +++ b/src/Functions/array/arrayConcat.cpp @@ -26,7 +26,7 @@ class FunctionArrayConcat : public IFunction { public: static constexpr auto name = "arrayConcat"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/array/arrayDistinct.cpp b/src/Functions/array/arrayDistinct.cpp index 916af560c8f..164e3b19c83 100644 --- a/src/Functions/array/arrayDistinct.cpp +++ b/src/Functions/array/arrayDistinct.cpp @@ -26,7 +26,7 @@ class FunctionArrayDistinct : public IFunction public: static constexpr auto name = "arrayDistinct"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index f3d3f558d7b..5ec5c4d4213 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -41,7 +41,7 @@ class FunctionArrayElement : public IFunction { public: static constexpr auto name = "arrayElement"; - static FunctionPtr create(ContextPtr context); + static FunctionPtr create(ContextConstPtr context); String getName() const override; @@ -449,7 +449,7 @@ struct ArrayElementGenericImpl } -FunctionPtr FunctionArrayElement::create(ContextPtr) +FunctionPtr FunctionArrayElement::create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/array/arrayEnumerate.cpp b/src/Functions/array/arrayEnumerate.cpp index 6d59a2c3805..a80e4f9e016 100644 --- a/src/Functions/array/arrayEnumerate.cpp +++ b/src/Functions/array/arrayEnumerate.cpp @@ -22,7 +22,7 @@ class FunctionArrayEnumerate : public IFunction public: static constexpr auto name = "arrayEnumerate"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/array/arrayEnumerateExtended.h b/src/Functions/array/arrayEnumerateExtended.h index 7e4fe24873a..921a0f0c75d 100644 --- a/src/Functions/array/arrayEnumerateExtended.h +++ b/src/Functions/array/arrayEnumerateExtended.h @@ -30,7 +30,7 @@ template class FunctionArrayEnumerateExtended : public IFunction { public: - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return Derived::name; } diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h index 2c999415f33..3dbb2d2d0c9 100644 --- a/src/Functions/array/arrayEnumerateRanked.h +++ b/src/Functions/array/arrayEnumerateRanked.h @@ -90,7 +90,7 @@ template class FunctionArrayEnumerateRankedExtended : public IFunction { public: - static FunctionPtr create(ContextPtr /* context */) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr /* context */) { return std::make_shared(); } String getName() const override { return Derived::name; } diff --git a/src/Functions/array/arrayFlatten.cpp b/src/Functions/array/arrayFlatten.cpp index 3d286aa0bb4..dfd1811d91d 100644 --- a/src/Functions/array/arrayFlatten.cpp +++ b/src/Functions/array/arrayFlatten.cpp @@ -20,7 +20,7 @@ class ArrayFlatten : public IFunction public: static constexpr auto name = "arrayFlatten"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } size_t getNumberOfArguments() const override { return 1; } bool useDefaultImplementationForConstants() const override { return true; } diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index f3b279faaef..fbd3501298e 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -355,7 +355,7 @@ class FunctionArrayIndex : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } /// Get function name. String getName() const override { return name; } diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index 93c1ee06403..1d3bcdd6fcc 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -38,8 +38,8 @@ class FunctionArrayIntersect : public IFunction { public: static constexpr auto name = "arrayIntersect"; - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionArrayIntersect(ContextPtr context_) : context(context_) {} + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } + explicit FunctionArrayIntersect(ContextConstPtr context_) : context(context_) {} String getName() const override { return name; } @@ -53,7 +53,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } private: - ContextPtr context; + ContextConstPtr context; /// Initially allocate a piece of memory for 64 elements. NOTE: This is just a guess. static constexpr size_t INITIAL_SIZE_DEGREE = 6; @@ -281,7 +281,7 @@ FunctionArrayIntersect::CastArgumentsResult FunctionArrayIntersect::castColumns( return {.initial = initial_columns, .casted = casted_columns}; } -static ColumnPtr callFunctionNotEquals(ColumnWithTypeAndName first, ColumnWithTypeAndName second, ContextPtr context) +static ColumnPtr callFunctionNotEquals(ColumnWithTypeAndName first, ColumnWithTypeAndName second, ContextConstPtr context) { ColumnsWithTypeAndName args{first, second}; auto eq_func = FunctionFactory::instance().get("notEquals", context)->build(args); diff --git a/src/Functions/array/arrayJoin.cpp b/src/Functions/array/arrayJoin.cpp index da8c4e6e80b..953c29ae67d 100644 --- a/src/Functions/array/arrayJoin.cpp +++ b/src/Functions/array/arrayJoin.cpp @@ -20,7 +20,7 @@ class FunctionArrayJoin : public IFunction { public: static constexpr auto name = "arrayJoin"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/array/arrayPopBack.cpp b/src/Functions/array/arrayPopBack.cpp index 0b532c22661..dea16c228fb 100644 --- a/src/Functions/array/arrayPopBack.cpp +++ b/src/Functions/array/arrayPopBack.cpp @@ -9,7 +9,7 @@ class FunctionArrayPopBack : public FunctionArrayPop { public: static constexpr auto name = "arrayPopBack"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } FunctionArrayPopBack() : FunctionArrayPop(false, name) {} }; diff --git a/src/Functions/array/arrayPopFront.cpp b/src/Functions/array/arrayPopFront.cpp index 57eedf9b44a..14a2d5f583e 100644 --- a/src/Functions/array/arrayPopFront.cpp +++ b/src/Functions/array/arrayPopFront.cpp @@ -9,7 +9,7 @@ class FunctionArrayPopFront : public FunctionArrayPop { public: static constexpr auto name = "arrayPopFront"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } FunctionArrayPopFront() : FunctionArrayPop(true, name) {} }; diff --git a/src/Functions/array/arrayPushBack.cpp b/src/Functions/array/arrayPushBack.cpp index 3171bca9d2c..1e12bf72d87 100644 --- a/src/Functions/array/arrayPushBack.cpp +++ b/src/Functions/array/arrayPushBack.cpp @@ -9,7 +9,7 @@ class FunctionArrayPushBack : public FunctionArrayPush { public: static constexpr auto name = "arrayPushBack"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } FunctionArrayPushBack() : FunctionArrayPush(false, name) {} }; diff --git a/src/Functions/array/arrayPushFront.cpp b/src/Functions/array/arrayPushFront.cpp index d12dfe02af1..08b4efa207e 100644 --- a/src/Functions/array/arrayPushFront.cpp +++ b/src/Functions/array/arrayPushFront.cpp @@ -10,7 +10,7 @@ class FunctionArrayPushFront : public FunctionArrayPush { public: static constexpr auto name = "arrayPushFront"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } FunctionArrayPushFront() : FunctionArrayPush(true, name) {} }; diff --git a/src/Functions/array/arrayReduce.cpp b/src/Functions/array/arrayReduce.cpp index bbdd142c062..3126554c127 100644 --- a/src/Functions/array/arrayReduce.cpp +++ b/src/Functions/array/arrayReduce.cpp @@ -37,7 +37,7 @@ class FunctionArrayReduce : public IFunction { public: static constexpr auto name = "arrayReduce"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/array/arrayReduceInRanges.cpp b/src/Functions/array/arrayReduceInRanges.cpp index bcb34385d17..dfeb79027f2 100644 --- a/src/Functions/array/arrayReduceInRanges.cpp +++ b/src/Functions/array/arrayReduceInRanges.cpp @@ -40,7 +40,7 @@ class FunctionArrayReduceInRanges : public IFunction public: static const size_t minimum_step = 64; static constexpr auto name = "arrayReduceInRanges"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/array/arrayResize.cpp b/src/Functions/array/arrayResize.cpp index f8eea06335b..03b0430d6ec 100644 --- a/src/Functions/array/arrayResize.cpp +++ b/src/Functions/array/arrayResize.cpp @@ -25,7 +25,7 @@ class FunctionArrayResize : public IFunction { public: static constexpr auto name = "arrayResize"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/array/arrayReverse.cpp b/src/Functions/array/arrayReverse.cpp index fb4a559b37f..2a2e24828ac 100644 --- a/src/Functions/array/arrayReverse.cpp +++ b/src/Functions/array/arrayReverse.cpp @@ -24,7 +24,7 @@ class FunctionArrayReverse : public IFunction { public: static constexpr auto name = "arrayReverse"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/array/arrayScalarProduct.h b/src/Functions/array/arrayScalarProduct.h index 6a23d6a45d8..0d35157d133 100644 --- a/src/Functions/array/arrayScalarProduct.h +++ b/src/Functions/array/arrayScalarProduct.h @@ -25,7 +25,7 @@ class FunctionArrayScalarProduct : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: using ResultColumnType = ColumnVector; diff --git a/src/Functions/array/arraySlice.cpp b/src/Functions/array/arraySlice.cpp index 567135de884..4fc32bfa981 100644 --- a/src/Functions/array/arraySlice.cpp +++ b/src/Functions/array/arraySlice.cpp @@ -34,7 +34,7 @@ class FunctionArraySlice : public IFunction { public: static constexpr auto name = "arraySlice"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 62de746f136..6d461206f38 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -31,7 +31,7 @@ class FunctionArrayUniq : public IFunction public: static constexpr auto name = "arrayUniq"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/array/arrayWithConstant.cpp b/src/Functions/array/arrayWithConstant.cpp index 578e8bf2296..7d00c2227ee 100644 --- a/src/Functions/array/arrayWithConstant.cpp +++ b/src/Functions/array/arrayWithConstant.cpp @@ -30,7 +30,7 @@ class FunctionArrayWithConstant : public IFunction public: static constexpr auto name = "arrayWithConstant"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } diff --git a/src/Functions/array/arrayZip.cpp b/src/Functions/array/arrayZip.cpp index f77b4f6dfe9..8e4d41ffca2 100644 --- a/src/Functions/array/arrayZip.cpp +++ b/src/Functions/array/arrayZip.cpp @@ -23,7 +23,7 @@ class FunctionArrayZip : public IFunction { public: static constexpr auto name = "arrayZip"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/array/emptyArray.cpp b/src/Functions/array/emptyArray.cpp index 370c51f3e8d..38b68043628 100644 --- a/src/Functions/array/emptyArray.cpp +++ b/src/Functions/array/emptyArray.cpp @@ -22,7 +22,7 @@ class FunctionEmptyArray : public IFunction { public: static String getNameImpl() { return "emptyArray" + DataType().getName(); } - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: String getName() const override diff --git a/src/Functions/array/emptyArrayToSingle.cpp b/src/Functions/array/emptyArrayToSingle.cpp index be387620e60..a116adca4e7 100644 --- a/src/Functions/array/emptyArrayToSingle.cpp +++ b/src/Functions/array/emptyArrayToSingle.cpp @@ -27,7 +27,7 @@ class FunctionEmptyArrayToSingle : public IFunction { public: static constexpr auto name = "emptyArrayToSingle"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/array/hasAll.cpp b/src/Functions/array/hasAll.cpp index c1959515e9f..7b5b83334b9 100644 --- a/src/Functions/array/hasAll.cpp +++ b/src/Functions/array/hasAll.cpp @@ -10,7 +10,7 @@ class FunctionArrayHasAll : public FunctionArrayHasAllAny { public: static constexpr auto name = "hasAll"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } FunctionArrayHasAll() : FunctionArrayHasAllAny(GatherUtils::ArraySearchType::All, name) {} }; diff --git a/src/Functions/array/hasAny.cpp b/src/Functions/array/hasAny.cpp index fac810c7ba1..94d429d6c0e 100644 --- a/src/Functions/array/hasAny.cpp +++ b/src/Functions/array/hasAny.cpp @@ -10,7 +10,7 @@ class FunctionArrayHasAny : public FunctionArrayHasAllAny { public: static constexpr auto name = "hasAny"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } FunctionArrayHasAny() : FunctionArrayHasAllAny(GatherUtils::ArraySearchType::Any, name) {} }; diff --git a/src/Functions/array/hasSubstr.cpp b/src/Functions/array/hasSubstr.cpp index 886f82f577b..f304e8bc3ba 100644 --- a/src/Functions/array/hasSubstr.cpp +++ b/src/Functions/array/hasSubstr.cpp @@ -10,7 +10,7 @@ class FunctionArrayHasSubstr : public FunctionArrayHasAllAny { public: static constexpr auto name = "hasSubstr"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } FunctionArrayHasSubstr() : FunctionArrayHasAllAny(GatherUtils::ArraySearchType::Substr, name) {} }; diff --git a/src/Functions/array/mapOp.cpp b/src/Functions/array/mapOp.cpp index 1a19ee41d2f..0709c1f2664 100644 --- a/src/Functions/array/mapOp.cpp +++ b/src/Functions/array/mapOp.cpp @@ -43,7 +43,7 @@ class FunctionMapOp : public IFunction { public: static constexpr auto name = (op_type == OpTypes::ADD) ? "mapAdd" : "mapSubtract"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: String getName() const override { return name; } diff --git a/src/Functions/array/mapPopulateSeries.cpp b/src/Functions/array/mapPopulateSeries.cpp index eb2f6192346..7e05eb81150 100644 --- a/src/Functions/array/mapPopulateSeries.cpp +++ b/src/Functions/array/mapPopulateSeries.cpp @@ -23,7 +23,7 @@ class FunctionMapPopulateSeries : public IFunction { public: static constexpr auto name = "mapPopulateSeries"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: String getName() const override { return name; } diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index 5b9886580dc..9f4c9fb0676 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -32,7 +32,7 @@ class FunctionRange : public IFunction public: static constexpr auto name = "range"; static constexpr size_t max_elements = 100'000'000; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: String getName() const override { return name; } diff --git a/src/Functions/assumeNotNull.cpp b/src/Functions/assumeNotNull.cpp index 019c637e50c..2b8743b1507 100644 --- a/src/Functions/assumeNotNull.cpp +++ b/src/Functions/assumeNotNull.cpp @@ -19,7 +19,7 @@ class FunctionAssumeNotNull : public IFunction public: static constexpr auto name = "assumeNotNull"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/bar.cpp b/src/Functions/bar.cpp index 2e26f7af0bb..49f3be46d60 100644 --- a/src/Functions/bar.cpp +++ b/src/Functions/bar.cpp @@ -28,7 +28,7 @@ class FunctionBar : public IFunction { public: static constexpr auto name = "bar"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/bitHammingDistance.cpp b/src/Functions/bitHammingDistance.cpp index 25b902b8bd5..a89ffa16010 100644 --- a/src/Functions/bitHammingDistance.cpp +++ b/src/Functions/bitHammingDistance.cpp @@ -74,7 +74,7 @@ class FunctionBitHammingDistance : public IFunction public: static constexpr auto name = "bitHammingDistance"; using ResultType = UInt8; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/bitmaskToList.cpp b/src/Functions/bitmaskToList.cpp index 8c3105724ac..0f3362aa031 100644 --- a/src/Functions/bitmaskToList.cpp +++ b/src/Functions/bitmaskToList.cpp @@ -30,7 +30,7 @@ class FunctionBitmaskToList : public IFunction { public: static constexpr auto name = "bitmaskToList"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/blockNumber.cpp b/src/Functions/blockNumber.cpp index 9a57f8a96b0..6998c02b3b6 100644 --- a/src/Functions/blockNumber.cpp +++ b/src/Functions/blockNumber.cpp @@ -18,7 +18,7 @@ private: public: static constexpr auto name = "blockNumber"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/blockSerializedSize.cpp b/src/Functions/blockSerializedSize.cpp index 30f77bbf627..bd65f6dd084 100644 --- a/src/Functions/blockSerializedSize.cpp +++ b/src/Functions/blockSerializedSize.cpp @@ -15,7 +15,7 @@ class FunctionBlockSerializedSize : public IFunction public: static constexpr auto name = "blockSerializedSize"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/blockSize.cpp b/src/Functions/blockSize.cpp index 294686054f0..481a89b1b82 100644 --- a/src/Functions/blockSize.cpp +++ b/src/Functions/blockSize.cpp @@ -15,7 +15,7 @@ class FunctionBlockSize : public IFunction { public: static constexpr auto name = "blockSize"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/buildId.cpp b/src/Functions/buildId.cpp index cc0c21350ca..4f452ac38a2 100644 --- a/src/Functions/buildId.cpp +++ b/src/Functions/buildId.cpp @@ -18,7 +18,7 @@ class FunctionBuildId : public IFunction { public: static constexpr auto name = "buildId"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/byteSize.cpp b/src/Functions/byteSize.cpp index 54a7da59b9c..5d43ddb2929 100644 --- a/src/Functions/byteSize.cpp +++ b/src/Functions/byteSize.cpp @@ -17,7 +17,7 @@ class FunctionByteSize : public IFunction { public: static constexpr auto name = "byteSize"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/caseWithExpression.cpp b/src/Functions/caseWithExpression.cpp index 24b5855cc5b..95c06ee683a 100644 --- a/src/Functions/caseWithExpression.cpp +++ b/src/Functions/caseWithExpression.cpp @@ -20,9 +20,9 @@ class FunctionCaseWithExpression : public IFunction { public: static constexpr auto name = "caseWithExpression"; - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } - explicit FunctionCaseWithExpression(ContextPtr context_) : context(context_) {} + explicit FunctionCaseWithExpression(ContextConstPtr context_) : context(context_) {} bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } String getName() const override { return name; } @@ -98,7 +98,7 @@ public: } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/coalesce.cpp b/src/Functions/coalesce.cpp index 791b9d993b4..69b69b7d3f1 100644 --- a/src/Functions/coalesce.cpp +++ b/src/Functions/coalesce.cpp @@ -24,12 +24,12 @@ class FunctionCoalesce : public IFunction public: static constexpr auto name = "coalesce"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionCoalesce(ContextPtr context_) : context(context_) {} + explicit FunctionCoalesce(ContextConstPtr context_) : context(context_) {} std::string getName() const override { @@ -160,7 +160,7 @@ public: } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index c1b9f8e7967..cc5636ffb5f 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -33,8 +33,8 @@ class ConcatImpl : public IFunction { public: static constexpr auto name = Name::name; - explicit ConcatImpl(ContextPtr context_) : context(context_) {} - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + explicit ConcatImpl(ContextConstPtr context_) : context(context_) {} + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } String getName() const override { return name; } @@ -85,7 +85,7 @@ public: } private: - ContextWeakPtr context; + ContextWeakConstPtr context; ColumnPtr executeBinary(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { @@ -191,9 +191,9 @@ class ConcatOverloadResolver : public IFunctionOverloadResolver { public: static constexpr auto name = "concat"; - static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } + static FunctionOverloadResolverPtr create(ContextConstPtr context) { return std::make_unique(context); } - explicit ConcatOverloadResolver(ContextPtr context_) : context(context_) {} + explicit ConcatOverloadResolver(ContextConstPtr context_) : context(context_) {} String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -223,7 +223,7 @@ public: } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/connectionId.cpp b/src/Functions/connectionId.cpp index ae04cfd1af6..4d22bb6ffad 100644 --- a/src/Functions/connectionId.cpp +++ b/src/Functions/connectionId.cpp @@ -8,14 +8,14 @@ namespace DB { /// Get the connection Id. It's used for MySQL handler only. -class FunctionConnectionId : public IFunction, WithContext +class FunctionConnectionId : public IFunction, WithConstContext { public: static constexpr auto name = "connectionId"; - explicit FunctionConnectionId(ContextPtr context_) : WithContext(context_) {} + explicit FunctionConnectionId(ContextConstPtr context_) : WithConstContext(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } String getName() const override { return name; } diff --git a/src/Functions/convertCharset.cpp b/src/Functions/convertCharset.cpp index b204875b4ff..e7a43e73a80 100644 --- a/src/Functions/convertCharset.cpp +++ b/src/Functions/convertCharset.cpp @@ -162,7 +162,7 @@ private: public: static constexpr auto name = "convertCharset"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/countDigits.cpp b/src/Functions/countDigits.cpp index 597a2c625b9..76038a88d41 100644 --- a/src/Functions/countDigits.cpp +++ b/src/Functions/countDigits.cpp @@ -28,7 +28,7 @@ class FunctionCountDigits : public IFunction public: static constexpr auto name = "countDigits"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/countMatches.h b/src/Functions/countMatches.h index 6ae69520cb9..0c6a95a94ea 100644 --- a/src/Functions/countMatches.h +++ b/src/Functions/countMatches.h @@ -27,7 +27,7 @@ class FunctionCountMatches : public IFunction { public: static constexpr auto name = CountMatchesBase::name; - static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } diff --git a/src/Functions/currentDatabase.cpp b/src/Functions/currentDatabase.cpp index 16eff20cfd5..efb98260e02 100644 --- a/src/Functions/currentDatabase.cpp +++ b/src/Functions/currentDatabase.cpp @@ -16,7 +16,7 @@ class FunctionCurrentDatabase : public IFunction public: static constexpr auto name = "currentDatabase"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getCurrentDatabase()); } diff --git a/src/Functions/currentUser.cpp b/src/Functions/currentUser.cpp index 22ad49fb29d..ea9943b3647 100644 --- a/src/Functions/currentUser.cpp +++ b/src/Functions/currentUser.cpp @@ -16,7 +16,7 @@ class FunctionCurrentUser : public IFunction public: static constexpr auto name = "currentUser"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getClientInfo().initial_user); } diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index cbad9dc1ce4..9d8b98aab41 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -47,7 +47,7 @@ class FunctionDateDiff : public IFunction using ColumnDateTime64 = ColumnDecimal; public: static constexpr auto name = "dateDiff"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp index 7952eda4f2b..d90586d6703 100644 --- a/src/Functions/date_trunc.cpp +++ b/src/Functions/date_trunc.cpp @@ -25,9 +25,9 @@ class FunctionDateTrunc : public IFunction public: static constexpr auto name = "date_trunc"; - explicit FunctionDateTrunc(ContextPtr context_) : context(context_) {} + explicit FunctionDateTrunc(ContextConstPtr context_) : context(context_) {} - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } String getName() const override { return name; } @@ -146,7 +146,7 @@ public: } private: - ContextPtr context; + ContextConstPtr context; mutable IntervalKind::Kind datepart_kind = IntervalKind::Kind::Second; }; diff --git a/src/Functions/defaultValueOfArgumentType.cpp b/src/Functions/defaultValueOfArgumentType.cpp index c0abe1b63d9..d7f6be1b55f 100644 --- a/src/Functions/defaultValueOfArgumentType.cpp +++ b/src/Functions/defaultValueOfArgumentType.cpp @@ -13,7 +13,7 @@ class FunctionDefaultValueOfArgumentType : public IFunction { public: static constexpr auto name = "defaultValueOfArgumentType"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/defaultValueOfTypeName.cpp b/src/Functions/defaultValueOfTypeName.cpp index 1bf978ab17b..fec72067069 100644 --- a/src/Functions/defaultValueOfTypeName.cpp +++ b/src/Functions/defaultValueOfTypeName.cpp @@ -21,7 +21,7 @@ class FunctionDefaultValueOfTypeName : public IFunction { public: static constexpr auto name = "defaultValueOfTypeName"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/demange.cpp b/src/Functions/demange.cpp index 755a50dccbb..b2df742b30a 100644 --- a/src/Functions/demange.cpp +++ b/src/Functions/demange.cpp @@ -25,7 +25,7 @@ class FunctionDemangle : public IFunction { public: static constexpr auto name = "demangle"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { context->checkAccess(AccessType::demangle); return std::make_shared(); diff --git a/src/Functions/dumpColumnStructure.cpp b/src/Functions/dumpColumnStructure.cpp index 3189e343beb..28e1b874f3e 100644 --- a/src/Functions/dumpColumnStructure.cpp +++ b/src/Functions/dumpColumnStructure.cpp @@ -14,7 +14,7 @@ class FunctionDumpColumnStructure : public IFunction { public: static constexpr auto name = "dumpColumnStructure"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/errorCodeToName.cpp b/src/Functions/errorCodeToName.cpp index d8e8e0f0d29..617731ba78c 100644 --- a/src/Functions/errorCodeToName.cpp +++ b/src/Functions/errorCodeToName.cpp @@ -19,7 +19,7 @@ class FunctionErrorCodeToName : public IFunction { public: static constexpr auto name = "errorCodeToName"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/evalMLMethod.cpp b/src/Functions/evalMLMethod.cpp index a69a6f0f550..1ad5d10df92 100644 --- a/src/Functions/evalMLMethod.cpp +++ b/src/Functions/evalMLMethod.cpp @@ -28,11 +28,11 @@ class FunctionEvalMLMethod : public IFunction { public: static constexpr auto name = "evalMLMethod"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionEvalMLMethod(ContextPtr context_) : context(context_) + explicit FunctionEvalMLMethod(ContextConstPtr context_) : context(context_) {} String getName() const override @@ -81,7 +81,7 @@ public: return agg_function->predictValues(arguments, context); } - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/extractAllGroups.h b/src/Functions/extractAllGroups.h index 864a788cf18..866a900f6a7 100644 --- a/src/Functions/extractAllGroups.h +++ b/src/Functions/extractAllGroups.h @@ -51,7 +51,7 @@ public: static constexpr auto Kind = Impl::Kind; static constexpr auto name = Impl::Name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/extractGroups.cpp b/src/Functions/extractGroups.cpp index f1d728ee3f6..981382828bd 100644 --- a/src/Functions/extractGroups.cpp +++ b/src/Functions/extractGroups.cpp @@ -31,7 +31,7 @@ class FunctionExtractGroups : public IFunction { public: static constexpr auto name = "extractGroups"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/extractTextFromHTML.cpp b/src/Functions/extractTextFromHTML.cpp index 6a7bdbeaba9..aa7c208904e 100644 --- a/src/Functions/extractTextFromHTML.cpp +++ b/src/Functions/extractTextFromHTML.cpp @@ -296,7 +296,7 @@ class FunctionExtractTextFromHTML : public IFunction public: static constexpr auto name = "extractTextFromHTML"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } bool useDefaultImplementationForConstants() const override { return true; } diff --git a/src/Functions/filesystem.cpp b/src/Functions/filesystem.cpp index d264c972656..d60b2133813 100644 --- a/src/Functions/filesystem.cpp +++ b/src/Functions/filesystem.cpp @@ -34,7 +34,7 @@ class FilesystemImpl : public IFunction public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared>(std::filesystem::space(context->getConfigRef().getString("path"))); } diff --git a/src/Functions/finalizeAggregation.cpp b/src/Functions/finalizeAggregation.cpp index b3dfbc0aa15..c8379e6a66f 100644 --- a/src/Functions/finalizeAggregation.cpp +++ b/src/Functions/finalizeAggregation.cpp @@ -24,7 +24,7 @@ class FunctionFinalizeAggregation : public IFunction { public: static constexpr auto name = "finalizeAggregation"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index d16985b5b34..c64c182baf6 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -281,7 +281,7 @@ private: public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/formatReadable.h b/src/Functions/formatReadable.h index 6cf4fadbf05..fc8f5afe764 100644 --- a/src/Functions/formatReadable.h +++ b/src/Functions/formatReadable.h @@ -28,7 +28,7 @@ class FunctionFormatReadable : public IFunction { public: static constexpr auto name = Impl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } String getName() const override { diff --git a/src/Functions/formatReadableTimeDelta.cpp b/src/Functions/formatReadableTimeDelta.cpp index 69dbaa71041..1be6fdf36c5 100644 --- a/src/Functions/formatReadableTimeDelta.cpp +++ b/src/Functions/formatReadableTimeDelta.cpp @@ -38,7 +38,7 @@ class FunctionFormatReadableTimeDelta : public IFunction { public: static constexpr auto name = "formatReadableTimeDelta"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/formatRow.cpp b/src/Functions/formatRow.cpp index 2d7739a02d6..60ed6be1351 100644 --- a/src/Functions/formatRow.cpp +++ b/src/Functions/formatRow.cpp @@ -35,7 +35,7 @@ class FunctionFormatRow : public IFunction public: static constexpr auto name = no_newline ? "formatRowNoNewline" : "formatRow"; - FunctionFormatRow(const String & format_name_, ContextPtr context_) : format_name(format_name_), context(context_) + FunctionFormatRow(const String & format_name_, ContextConstPtr context_) : format_name(format_name_), context(context_) { if (!FormatFactory::instance().getAllFormats().count(format_name)) throw Exception("Unknown format " + format_name, ErrorCodes::UNKNOWN_FORMAT); @@ -76,7 +76,7 @@ public: private: String format_name; - ContextPtr context; + ContextConstPtr context; }; template @@ -84,8 +84,8 @@ class FormatRowOverloadResolver : public IFunctionOverloadResolver { public: static constexpr auto name = no_newline ? "formatRowNoNewline" : "formatRow"; - static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } - explicit FormatRowOverloadResolver(ContextPtr context_) : context(context_) { } + static FunctionOverloadResolverPtr create(ContextConstPtr context) { return std::make_unique(context); } + explicit FormatRowOverloadResolver(ContextConstPtr context_) : context(context_) { } String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -111,7 +111,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared(); } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/formatString.cpp b/src/Functions/formatString.cpp index 4a3b04aaa05..9c222d085c2 100644 --- a/src/Functions/formatString.cpp +++ b/src/Functions/formatString.cpp @@ -31,7 +31,7 @@ class FormatFunction : public IFunction public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/fromModifiedJulianDay.cpp b/src/Functions/fromModifiedJulianDay.cpp index bdd8b9b6bd4..277d876e42f 100644 --- a/src/Functions/fromModifiedJulianDay.cpp +++ b/src/Functions/fromModifiedJulianDay.cpp @@ -151,7 +151,7 @@ namespace DB public: static constexpr auto name = Name::name; - static FunctionOverloadResolverPtr create(ContextPtr) + static FunctionOverloadResolverPtr create(ContextConstPtr) { return std::make_unique>(); } diff --git a/src/Functions/fromUnixTimestamp64Micro.cpp b/src/Functions/fromUnixTimestamp64Micro.cpp index 1fd98e6e673..7eb9e41474b 100644 --- a/src/Functions/fromUnixTimestamp64Micro.cpp +++ b/src/Functions/fromUnixTimestamp64Micro.cpp @@ -7,7 +7,7 @@ namespace DB void registerFromUnixTimestamp64Micro(FunctionFactory & factory) { factory.registerFunction("fromUnixTimestamp64Micro", - [](ContextPtr){ return std::make_unique( + [](ContextConstPtr){ return std::make_unique( std::make_shared(6, "fromUnixTimestamp64Micro")); }); } diff --git a/src/Functions/fromUnixTimestamp64Milli.cpp b/src/Functions/fromUnixTimestamp64Milli.cpp index 90f18699e5d..68ee1ed4078 100644 --- a/src/Functions/fromUnixTimestamp64Milli.cpp +++ b/src/Functions/fromUnixTimestamp64Milli.cpp @@ -7,7 +7,7 @@ namespace DB void registerFromUnixTimestamp64Milli(FunctionFactory & factory) { factory.registerFunction("fromUnixTimestamp64Milli", - [](ContextPtr){ return std::make_unique( + [](ContextConstPtr){ return std::make_unique( std::make_shared(3, "fromUnixTimestamp64Milli")); }); } diff --git a/src/Functions/fromUnixTimestamp64Nano.cpp b/src/Functions/fromUnixTimestamp64Nano.cpp index c489b48fb6e..19fe8524b78 100644 --- a/src/Functions/fromUnixTimestamp64Nano.cpp +++ b/src/Functions/fromUnixTimestamp64Nano.cpp @@ -7,7 +7,7 @@ namespace DB void registerFromUnixTimestamp64Nano(FunctionFactory & factory) { factory.registerFunction("fromUnixTimestamp64Nano", - [](ContextPtr){ return std::make_unique( + [](ContextConstPtr){ return std::make_unique( std::make_shared(9, "fromUnixTimestamp64Nano")); }); } diff --git a/src/Functions/fuzzBits.cpp b/src/Functions/fuzzBits.cpp index 0884f586082..b4110eae32a 100644 --- a/src/Functions/fuzzBits.cpp +++ b/src/Functions/fuzzBits.cpp @@ -53,7 +53,7 @@ class FunctionFuzzBits : public IFunction public: static constexpr auto name = "fuzzBits"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/generateUUIDv4.cpp b/src/Functions/generateUUIDv4.cpp index 7ca127857df..7d3f80d9096 100644 --- a/src/Functions/generateUUIDv4.cpp +++ b/src/Functions/generateUUIDv4.cpp @@ -60,7 +60,7 @@ public: class FunctionGenerateUUIDv4 : public TargetSpecific::Default::FunctionGenerateUUIDv4 { public: - explicit FunctionGenerateUUIDv4(ContextPtr context) : selector(context) + explicit FunctionGenerateUUIDv4(ContextConstPtr context) : selector(context) { selector.registerImplementation(); @@ -76,7 +76,7 @@ public: return selector.selectAndExecute(arguments, result_type, input_rows_count); } - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 6030cbeea65..45e6e06d114 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -33,7 +33,7 @@ class FunctionGeoToH3 : public IFunction public: static constexpr auto name = "geoToH3"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/geohashDecode.cpp b/src/Functions/geohashDecode.cpp index bf774905868..89e8d40402a 100644 --- a/src/Functions/geohashDecode.cpp +++ b/src/Functions/geohashDecode.cpp @@ -27,7 +27,7 @@ class FunctionGeohashDecode : public IFunction { public: static constexpr auto name = "geohashDecode"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/geohashEncode.cpp b/src/Functions/geohashEncode.cpp index 15abe96e3e5..fbee2047ed2 100644 --- a/src/Functions/geohashEncode.cpp +++ b/src/Functions/geohashEncode.cpp @@ -28,7 +28,7 @@ class FunctionGeohashEncode : public IFunction { public: static constexpr auto name = "geohashEncode"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/geohashesInBox.cpp b/src/Functions/geohashesInBox.cpp index c0629ab9655..fdccc97879f 100644 --- a/src/Functions/geohashesInBox.cpp +++ b/src/Functions/geohashesInBox.cpp @@ -29,7 +29,7 @@ class FunctionGeohashesInBox : public IFunction { public: static constexpr auto name = "geohashesInBox"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/getMacro.cpp b/src/Functions/getMacro.cpp index c869685af42..a2fda60623f 100644 --- a/src/Functions/getMacro.cpp +++ b/src/Functions/getMacro.cpp @@ -30,7 +30,7 @@ private: public: static constexpr auto name = "getMacro"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getMacros()); } diff --git a/src/Functions/getScalar.cpp b/src/Functions/getScalar.cpp index a29abd257e7..d74d881473f 100644 --- a/src/Functions/getScalar.cpp +++ b/src/Functions/getScalar.cpp @@ -20,16 +20,16 @@ namespace /** Get scalar value of sub queries from query context via IAST::Hash. */ -class FunctionGetScalar : public IFunction, WithContext +class FunctionGetScalar : public IFunction, WithConstContext { public: static constexpr auto name = "__getScalar"; - static FunctionPtr create(ContextPtr context_) + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } - explicit FunctionGetScalar(ContextPtr context_) : WithContext(context_) {} + explicit FunctionGetScalar(ContextConstPtr context_) : WithConstContext(context_) {} String getName() const override { @@ -46,7 +46,7 @@ public: if (arguments.size() != 1 || !isString(arguments[0].type) || !arguments[0].column || !isColumnConst(*arguments[0].column)) throw Exception("Function " + getName() + " accepts one const string argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto scalar_name = assert_cast(*arguments[0].column).getValue(); - ContextPtr query_context = getContext()->hasQueryContext() ? getContext()->getQueryContext() : getContext(); + ContextConstPtr query_context = getContext()->hasQueryContext() ? getContext()->getQueryContext() : getContext(); scalar = query_context->getScalar(scalar_name).getByPosition(0); return scalar.type; } diff --git a/src/Functions/getSetting.cpp b/src/Functions/getSetting.cpp index 0206de33125..7c6592faf57 100644 --- a/src/Functions/getSetting.cpp +++ b/src/Functions/getSetting.cpp @@ -19,13 +19,13 @@ namespace { /// Get the value of a setting. -class FunctionGetSetting : public IFunction, WithContext +class FunctionGetSetting : public IFunction, WithConstContext { public: static constexpr auto name = "getSetting"; - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } - explicit FunctionGetSetting(ContextPtr context_) : WithContext(context_) {} + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } + explicit FunctionGetSetting(ContextConstPtr context_) : WithConstContext(context_) {} String getName() const override { return name; } bool isDeterministic() const override { return false; } diff --git a/src/Functions/getSizeOfEnumType.cpp b/src/Functions/getSizeOfEnumType.cpp index 504c088a737..61f4d7efced 100644 --- a/src/Functions/getSizeOfEnumType.cpp +++ b/src/Functions/getSizeOfEnumType.cpp @@ -20,7 +20,7 @@ class FunctionGetSizeOfEnumType : public IFunction { public: static constexpr auto name = "getSizeOfEnumType"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/globalVariable.cpp b/src/Functions/globalVariable.cpp index c141ceb2692..be2d4488daa 100644 --- a/src/Functions/globalVariable.cpp +++ b/src/Functions/globalVariable.cpp @@ -30,7 +30,7 @@ class FunctionGlobalVariable : public IFunction { public: static constexpr auto name = "globalVariable"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index 756b76859ae..e160b72740e 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -287,7 +287,7 @@ template class FunctionGeoDistance : public TargetSpecific::Default::FunctionGeoDistance { public: - explicit FunctionGeoDistance(ContextPtr context) : selector(context) + explicit FunctionGeoDistance(ContextConstPtr context) : selector(context) { selector.registerImplementation>(); @@ -307,7 +307,7 @@ public: return selector.selectAndExecute(arguments, result_type, input_rows_count); } - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared>(context); } diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index ad8d1ee441f..4a0f69ab139 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -32,7 +32,7 @@ class FunctionH3EdgeAngle : public IFunction public: static constexpr auto name = "h3EdgeAngle"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 85d307e1da3..523c4bde061 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -37,7 +37,7 @@ class FunctionH3EdgeLengthM : public IFunction public: static constexpr auto name = "h3EdgeLengthM"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3GetBaseCell.cpp b/src/Functions/h3GetBaseCell.cpp index fcad82098e6..83d3064a023 100644 --- a/src/Functions/h3GetBaseCell.cpp +++ b/src/Functions/h3GetBaseCell.cpp @@ -29,7 +29,7 @@ class FunctionH3GetBaseCell : public IFunction public: static constexpr auto name = "h3GetBaseCell"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3GetResolution.cpp b/src/Functions/h3GetResolution.cpp index d1bfd899150..197b0516da9 100644 --- a/src/Functions/h3GetResolution.cpp +++ b/src/Functions/h3GetResolution.cpp @@ -29,7 +29,7 @@ class FunctionH3GetResolution : public IFunction public: static constexpr auto name = "h3GetResolution"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index e44bf660333..c7eca3396bc 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -32,7 +32,7 @@ class FunctionH3HexAreaM2 : public IFunction public: static constexpr auto name = "h3HexAreaM2"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3IndexesAreNeighbors.cpp b/src/Functions/h3IndexesAreNeighbors.cpp index 56c69ed0a8b..d2873f954e8 100644 --- a/src/Functions/h3IndexesAreNeighbors.cpp +++ b/src/Functions/h3IndexesAreNeighbors.cpp @@ -29,7 +29,7 @@ class FunctionH3IndexesAreNeighbors : public IFunction public: static constexpr auto name = "h3IndexesAreNeighbors"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3IsValid.cpp b/src/Functions/h3IsValid.cpp index 1e4840b3537..930b1d3c4a7 100644 --- a/src/Functions/h3IsValid.cpp +++ b/src/Functions/h3IsValid.cpp @@ -29,7 +29,7 @@ class FunctionH3IsValid : public IFunction public: static constexpr auto name = "h3IsValid"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index 9730aefe3d4..3b09b5a1866 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -38,7 +38,7 @@ class FunctionH3ToChildren : public IFunction public: static constexpr auto name = "h3ToChildren"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index 4baf9527301..b5c49d4c253 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -32,7 +32,7 @@ class FunctionH3ToParent : public IFunction public: static constexpr auto name = "h3ToParent"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3ToString.cpp b/src/Functions/h3ToString.cpp index dcd0951f67f..f2a777580d0 100644 --- a/src/Functions/h3ToString.cpp +++ b/src/Functions/h3ToString.cpp @@ -29,7 +29,7 @@ class FunctionH3ToString : public IFunction public: static constexpr auto name = "h3ToString"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index 40e73f9dea2..e616c253c58 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -35,7 +35,7 @@ class FunctionH3KRing : public IFunction public: static constexpr auto name = "h3kRing"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp index da7e59d69e3..941ffa48e74 100644 --- a/src/Functions/hasColumnInTable.cpp +++ b/src/Functions/hasColumnInTable.cpp @@ -25,16 +25,16 @@ namespace /** Usage: * hasColumnInTable(['hostname'[, 'username'[, 'password']],] 'database', 'table', 'column') */ -class FunctionHasColumnInTable : public IFunction, WithContext +class FunctionHasColumnInTable : public IFunction, WithConstContext { public: static constexpr auto name = "hasColumnInTable"; - static FunctionPtr create(ContextPtr context_) + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_->getGlobalContext()); } - explicit FunctionHasColumnInTable(ContextPtr global_context_) : WithContext(global_context_) + explicit FunctionHasColumnInTable(ContextConstPtr global_context_) : WithConstContext(global_context_) { } @@ -114,7 +114,12 @@ ColumnPtr FunctionHasColumnInTable::executeImpl(const ColumnsWithTypeAndName & a bool has_column; if (host_name.empty()) { - const StoragePtr & table = DatabaseCatalog::instance().getTable({database_name, table_name}, getContext()); + // FIXME this (probably) needs a non-constant access to query context, + // because it might initialized a storage. Ideally, the tables required + // by the query should be initialized at an earlier stage. + const StoragePtr & table = DatabaseCatalog::instance().getTable( + {database_name, table_name}, + const_pointer_cast(getContext())); auto table_metadata = table->getInMemoryMetadataPtr(); has_column = table_metadata->getColumns().hasPhysical(column_name); } @@ -130,7 +135,13 @@ ColumnPtr FunctionHasColumnInTable::executeImpl(const ColumnsWithTypeAndName & a getContext()->getTCPPort(), false); - auto remote_columns = getStructureOfRemoteTable(*cluster, {database_name, table_name}, getContext()); + // FIXME this (probably) needs a non-constant access to query context, + // because it might initialized a storage. Ideally, the tables required + // by the query should be initialized at an earlier stage. + auto remote_columns = getStructureOfRemoteTable(*cluster, + {database_name, table_name}, + const_pointer_cast(getContext())); + has_column = remote_columns.hasPhysical(column_name); } diff --git a/src/Functions/hasThreadFuzzer.cpp b/src/Functions/hasThreadFuzzer.cpp index e9b48aa70f9..fb7014a49b1 100644 --- a/src/Functions/hasThreadFuzzer.cpp +++ b/src/Functions/hasThreadFuzzer.cpp @@ -16,7 +16,7 @@ class FunctionHasThreadFuzzer : public IFunction { public: static constexpr auto name = "hasThreadFuzzer"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/hostName.cpp b/src/Functions/hostName.cpp index 0aba155bb36..1fdc039be96 100644 --- a/src/Functions/hostName.cpp +++ b/src/Functions/hostName.cpp @@ -15,7 +15,7 @@ class FunctionHostName : public IFunction { public: static constexpr auto name = "hostName"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/identity.cpp b/src/Functions/identity.cpp index d15d9e1f710..7889a17f05a 100644 --- a/src/Functions/identity.cpp +++ b/src/Functions/identity.cpp @@ -11,7 +11,7 @@ class FunctionIdentity : public IFunction { public: static constexpr auto name = "identity"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index ec3447ffb81..dbe36d4c728 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -173,7 +173,7 @@ class FunctionIf : public FunctionIfBase { public: static constexpr auto name = "if"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: template diff --git a/src/Functions/ifNotFinite.cpp b/src/Functions/ifNotFinite.cpp index a5e3131117e..be975f28ee1 100644 --- a/src/Functions/ifNotFinite.cpp +++ b/src/Functions/ifNotFinite.cpp @@ -17,9 +17,9 @@ class FunctionIfNotFinite : public IFunction public: static constexpr auto name = "ifNotFinite"; - explicit FunctionIfNotFinite(ContextPtr context_) : context(context_) {} + explicit FunctionIfNotFinite(ContextConstPtr context_) : context(context_) {} - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } @@ -59,7 +59,7 @@ public: } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/ifNull.cpp b/src/Functions/ifNull.cpp index 82b63f4dbda..69c7e9346f2 100644 --- a/src/Functions/ifNull.cpp +++ b/src/Functions/ifNull.cpp @@ -21,9 +21,9 @@ class FunctionIfNull : public IFunction public: static constexpr auto name = "ifNull"; - explicit FunctionIfNull(ContextPtr context_) : context(context_) {} + explicit FunctionIfNull(ContextConstPtr context_) : context(context_) {} - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } @@ -83,7 +83,7 @@ public: } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/ignore.cpp b/src/Functions/ignore.cpp index 176bc9afee2..819cd9a0b52 100644 --- a/src/Functions/ignore.cpp +++ b/src/Functions/ignore.cpp @@ -14,7 +14,7 @@ class FunctionIgnore : public IFunction { public: static constexpr auto name = "ignore"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 827e0212396..b1686e2a15a 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -52,7 +52,7 @@ public: /// It is needed to perform type analysis without creation of set. static constexpr auto name = FunctionInName::name; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/indexHint.cpp b/src/Functions/indexHint.cpp index f3c856c38ce..6fd41a910d3 100644 --- a/src/Functions/indexHint.cpp +++ b/src/Functions/indexHint.cpp @@ -23,7 +23,7 @@ class FunctionIndexHint : public IFunction { public: static constexpr auto name = "indexHint"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/initializeAggregation.cpp b/src/Functions/initializeAggregation.cpp index aea97a48037..e3d79c41f5b 100644 --- a/src/Functions/initializeAggregation.cpp +++ b/src/Functions/initializeAggregation.cpp @@ -29,7 +29,7 @@ class FunctionInitializeAggregation : public IFunction { public: static constexpr auto name = "initializeAggregation"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/isConstant.cpp b/src/Functions/isConstant.cpp index 3069ec79ae1..474fb46eed3 100644 --- a/src/Functions/isConstant.cpp +++ b/src/Functions/isConstant.cpp @@ -15,7 +15,7 @@ class FunctionIsConstant : public IFunction { public: static constexpr auto name = "isConstant"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/isDecimalOverflow.cpp b/src/Functions/isDecimalOverflow.cpp index d409afaf234..c82845b7cb8 100644 --- a/src/Functions/isDecimalOverflow.cpp +++ b/src/Functions/isDecimalOverflow.cpp @@ -28,7 +28,7 @@ class FunctionIsDecimalOverflow : public IFunction public: static constexpr auto name = "isDecimalOverflow"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/isIPAddressContainedIn.cpp b/src/Functions/isIPAddressContainedIn.cpp index 0886ef55e7b..439bf0a893e 100644 --- a/src/Functions/isIPAddressContainedIn.cpp +++ b/src/Functions/isIPAddressContainedIn.cpp @@ -125,7 +125,7 @@ namespace DB public: static constexpr auto name = "isIPAddressInRange"; String getName() const override { return name; } - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /* return_type */, size_t input_rows_count) const override { diff --git a/src/Functions/isNotNull.cpp b/src/Functions/isNotNull.cpp index 81c870a6303..a0119bb48c2 100644 --- a/src/Functions/isNotNull.cpp +++ b/src/Functions/isNotNull.cpp @@ -19,7 +19,7 @@ class FunctionIsNotNull : public IFunction public: static constexpr auto name = "isNotNull"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/isNull.cpp b/src/Functions/isNull.cpp index de840dab2bf..36da85dcdd3 100644 --- a/src/Functions/isNull.cpp +++ b/src/Functions/isNull.cpp @@ -18,7 +18,7 @@ class FunctionIsNull : public IFunction public: static constexpr auto name = "isNull"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/isZeroOrNull.cpp b/src/Functions/isZeroOrNull.cpp index a2a06af7569..4b863871629 100644 --- a/src/Functions/isZeroOrNull.cpp +++ b/src/Functions/isZeroOrNull.cpp @@ -26,7 +26,7 @@ class FunctionIsZeroOrNull : public IFunction public: static constexpr auto name = "isZeroOrNull"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/logTrace.cpp b/src/Functions/logTrace.cpp index 2ccc4d2ffd2..a291a59504f 100644 --- a/src/Functions/logTrace.cpp +++ b/src/Functions/logTrace.cpp @@ -21,7 +21,7 @@ namespace { public: static constexpr auto name = "logTrace"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/lowCardinalityIndices.cpp b/src/Functions/lowCardinalityIndices.cpp index 580e7381955..ac7fd45e7b6 100644 --- a/src/Functions/lowCardinalityIndices.cpp +++ b/src/Functions/lowCardinalityIndices.cpp @@ -21,7 +21,7 @@ class FunctionLowCardinalityIndices: public IFunction { public: static constexpr auto name = "lowCardinalityIndices"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/lowCardinalityKeys.cpp b/src/Functions/lowCardinalityKeys.cpp index 46128267871..2781248d15a 100644 --- a/src/Functions/lowCardinalityKeys.cpp +++ b/src/Functions/lowCardinalityKeys.cpp @@ -19,7 +19,7 @@ class FunctionLowCardinalityKeys: public IFunction { public: static constexpr auto name = "lowCardinalityKeys"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index f1b08dc75b7..76b57d596cc 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -34,7 +34,7 @@ class FunctionMap : public IFunction public: static constexpr auto name = "map"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } @@ -146,7 +146,7 @@ class FunctionMapContains : public IFunction { public: static constexpr auto name = NameMapContains::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -209,7 +209,7 @@ class FunctionMapKeys : public IFunction { public: static constexpr auto name = "mapKeys"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { @@ -256,7 +256,7 @@ class FunctionMapValues : public IFunction { public: static constexpr auto name = "mapValues"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/materialize.h b/src/Functions/materialize.h index 4676b263f39..99b556dfff5 100644 --- a/src/Functions/materialize.h +++ b/src/Functions/materialize.h @@ -11,7 +11,7 @@ class FunctionMaterialize : public IFunction { public: static constexpr auto name = "materialize"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 92f05df472b..56399a7f056 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -35,7 +35,7 @@ class FunctionMultiIf final : public FunctionIfBase { public: static constexpr auto name = "multiIf"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/neighbor.cpp b/src/Functions/neighbor.cpp index 785c5817176..53c47d3972d 100644 --- a/src/Functions/neighbor.cpp +++ b/src/Functions/neighbor.cpp @@ -31,7 +31,7 @@ class FunctionNeighbor : public IFunction { public: static constexpr auto name = "neighbor"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } /// Get the name of the function. String getName() const override { return name; } diff --git a/src/Functions/normalizedQueryHash.cpp b/src/Functions/normalizedQueryHash.cpp index e08680518c1..5cfd5ff4904 100644 --- a/src/Functions/normalizedQueryHash.cpp +++ b/src/Functions/normalizedQueryHash.cpp @@ -51,7 +51,7 @@ class FunctionNormalizedQueryHash : public IFunction { public: static constexpr auto name = keep_names ? "normalizedQueryHashKeepNames" : "normalizedQueryHash"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/now.cpp b/src/Functions/now.cpp index 673484b2ad2..9d6569fd5df 100644 --- a/src/Functions/now.cpp +++ b/src/Functions/now.cpp @@ -83,7 +83,7 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } - static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique(); } + static FunctionOverloadResolverPtr create(ContextConstPtr) { return std::make_unique(); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index cccfb9de6b6..b31a78bf0a7 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -107,7 +107,7 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } - static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique(); } + static FunctionOverloadResolverPtr create(ContextConstPtr) { return std::make_unique(); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { diff --git a/src/Functions/nullIf.cpp b/src/Functions/nullIf.cpp index 14f2f72fe61..8f69ef4f550 100644 --- a/src/Functions/nullIf.cpp +++ b/src/Functions/nullIf.cpp @@ -18,16 +18,16 @@ namespace class FunctionNullIf : public IFunction { private: - ContextPtr context; + ContextConstPtr context; public: static constexpr auto name = "nullIf"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - explicit FunctionNullIf(ContextPtr context_) : context(context_) {} + explicit FunctionNullIf(ContextConstPtr context_) : context(context_) {} std::string getName() const override { diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp index cf679452da1..f33467e6752 100644 --- a/src/Functions/partitionId.cpp +++ b/src/Functions/partitionId.cpp @@ -23,7 +23,7 @@ class FunctionPartitionId : public IFunction public: static constexpr auto name = "partitionId"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/pointInEllipses.cpp b/src/Functions/pointInEllipses.cpp index f7bfc24559c..06eae802864 100644 --- a/src/Functions/pointInEllipses.cpp +++ b/src/Functions/pointInEllipses.cpp @@ -39,7 +39,7 @@ class FunctionPointInEllipses : public IFunction { public: static constexpr auto name = "pointInEllipses"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } private: diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 842a5010190..82e69c468cc 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -60,7 +60,7 @@ public: explicit FunctionPointInPolygon(bool validate_) : validate(validate_) {} - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared>( context->getSettingsRef().validate_polygons); diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 9a36b71d5b1..07e0f922ae0 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -34,7 +34,7 @@ public: explicit FunctionPolygonArea() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 5bce3c5bbdd..b84e327ac81 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -33,7 +33,7 @@ public: explicit FunctionPolygonConvexHull() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 5a6e293630e..11bb5488fd9 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -33,7 +33,7 @@ public: explicit FunctionPolygonPerimeter() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index d6c4ff36e4d..d92a1931f2b 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -35,7 +35,7 @@ public: explicit FunctionPolygonsDistance() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 9e2902d0528..c14f9a18980 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -34,7 +34,7 @@ public: explicit FunctionPolygonsEquals() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 699488abab1..5676323a08a 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -34,7 +34,7 @@ public: explicit FunctionPolygonsIntersection() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 129e553f060..991d66a2802 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -34,7 +34,7 @@ public: explicit FunctionPolygonsSymDifference() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index e7281c299d0..9a9bfc9acfe 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -34,7 +34,7 @@ public: explicit FunctionPolygonsUnion() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 2fd5b7e2225..45fc06d4fbe 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -36,7 +36,7 @@ public: explicit FunctionPolygonsWithin() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/randConstant.cpp b/src/Functions/randConstant.cpp index 77d8e1f2a33..13a07050717 100644 --- a/src/Functions/randConstant.cpp +++ b/src/Functions/randConstant.cpp @@ -79,7 +79,7 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } - static FunctionOverloadResolverPtr create(ContextPtr) + static FunctionOverloadResolverPtr create(ContextConstPtr) { return std::make_unique>(); } diff --git a/src/Functions/randomFixedString.cpp b/src/Functions/randomFixedString.cpp index 13996eee677..2a60f8dd88d 100644 --- a/src/Functions/randomFixedString.cpp +++ b/src/Functions/randomFixedString.cpp @@ -77,7 +77,7 @@ public: class FunctionRandomFixedString : public FunctionRandomFixedStringImpl { public: - explicit FunctionRandomFixedString(ContextPtr context) : selector(context) + explicit FunctionRandomFixedString(ContextConstPtr context) : selector(context) { selector.registerImplementation>(); @@ -93,7 +93,7 @@ public: return selector.selectAndExecute(arguments, result_type, input_rows_count); } - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } diff --git a/src/Functions/randomPrintableASCII.cpp b/src/Functions/randomPrintableASCII.cpp index ba8c16aa689..d27772e28fb 100644 --- a/src/Functions/randomPrintableASCII.cpp +++ b/src/Functions/randomPrintableASCII.cpp @@ -27,7 +27,7 @@ class FunctionRandomPrintableASCII : public IFunction { public: static constexpr auto name = "randomPrintableASCII"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/randomString.cpp b/src/Functions/randomString.cpp index 10795626326..680b01b9d08 100644 --- a/src/Functions/randomString.cpp +++ b/src/Functions/randomString.cpp @@ -97,7 +97,7 @@ public: class FunctionRandomString : public FunctionRandomStringImpl { public: - explicit FunctionRandomString(ContextPtr context) : selector(context) + explicit FunctionRandomString(ContextConstPtr context) : selector(context) { selector.registerImplementation>(); @@ -113,7 +113,7 @@ public: return selector.selectAndExecute(arguments, result_type, input_rows_count); } - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp index 018c591a1fe..48c828d2444 100644 --- a/src/Functions/randomStringUTF8.cpp +++ b/src/Functions/randomStringUTF8.cpp @@ -30,7 +30,7 @@ class FunctionRandomStringUTF8 : public IFunction public: static constexpr auto name = "randomStringUTF8"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index 2ce4becb01c..26bb13a2e93 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -68,7 +68,7 @@ public: return true; } - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } diff --git a/src/Functions/regexpQuoteMeta.cpp b/src/Functions/regexpQuoteMeta.cpp index 0f1ec476a3b..cae2fdc1323 100644 --- a/src/Functions/regexpQuoteMeta.cpp +++ b/src/Functions/regexpQuoteMeta.cpp @@ -21,7 +21,7 @@ class FunctionRegexpQuoteMeta : public IFunction public: static constexpr auto name = "regexpQuoteMeta"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/reinterpretAs.cpp b/src/Functions/reinterpretAs.cpp index 94c9eec5868..d007e479c04 100644 --- a/src/Functions/reinterpretAs.cpp +++ b/src/Functions/reinterpretAs.cpp @@ -47,7 +47,7 @@ class FunctionReinterpret : public IFunction public: static constexpr auto name = "reinterpret"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -349,7 +349,7 @@ class FunctionReinterpretAs : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp index 423ed53c53f..e87a51b807d 100644 --- a/src/Functions/repeat.cpp +++ b/src/Functions/repeat.cpp @@ -173,7 +173,7 @@ class FunctionRepeat : public IFunction public: static constexpr auto name = "repeat"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/replicate.cpp b/src/Functions/replicate.cpp index 796d5645a2e..5f6f187031f 100644 --- a/src/Functions/replicate.cpp +++ b/src/Functions/replicate.cpp @@ -55,7 +55,7 @@ ColumnPtr FunctionReplicate::executeImpl(const ColumnsWithTypeAndName & argument offsets = array_column->getOffsetsPtr(); } - const auto & offsets_data = assert_cast(*offsets).getData(); + const auto & offsets_data = assert_cast(*offsets).getData(); /// NOLINT return ColumnArray::create(first_column->replicate(offsets_data)->convertToFullColumnIfConst(), offsets); } diff --git a/src/Functions/replicate.h b/src/Functions/replicate.h index 71b39e9e60e..4493d132002 100644 --- a/src/Functions/replicate.h +++ b/src/Functions/replicate.h @@ -13,7 +13,7 @@ class FunctionReplicate : public IFunction public: static constexpr auto name = "replicate"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/reverse.cpp b/src/Functions/reverse.cpp index f9701af75da..49ce3a214e3 100644 --- a/src/Functions/reverse.cpp +++ b/src/Functions/reverse.cpp @@ -57,7 +57,7 @@ class FunctionReverse : public IFunction { public: static constexpr auto name = "reverse"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } @@ -117,9 +117,9 @@ class ReverseOverloadResolver : public IFunctionOverloadResolver { public: static constexpr auto name = "reverse"; - static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } + static FunctionOverloadResolverPtr create(ContextConstPtr context) { return std::make_unique(context); } - explicit ReverseOverloadResolver(ContextPtr context_) : context(context_) {} + explicit ReverseOverloadResolver(ContextConstPtr context_) : context(context_) {} String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } @@ -141,7 +141,7 @@ public: } private: - ContextPtr context; + ContextConstPtr context; }; } diff --git a/src/Functions/rowNumberInAllBlocks.cpp b/src/Functions/rowNumberInAllBlocks.cpp index 9c358aec8f0..85211d9e740 100644 --- a/src/Functions/rowNumberInAllBlocks.cpp +++ b/src/Functions/rowNumberInAllBlocks.cpp @@ -18,7 +18,7 @@ private: public: static constexpr auto name = "rowNumberInAllBlocks"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/rowNumberInBlock.cpp b/src/Functions/rowNumberInBlock.cpp index 48fa472e1dd..e924207a6d1 100644 --- a/src/Functions/rowNumberInBlock.cpp +++ b/src/Functions/rowNumberInBlock.cpp @@ -13,7 +13,7 @@ class FunctionRowNumberInBlock : public IFunction { public: static constexpr auto name = "rowNumberInBlock"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/runningAccumulate.cpp b/src/Functions/runningAccumulate.cpp index 1eab573b046..b17faf9c031 100644 --- a/src/Functions/runningAccumulate.cpp +++ b/src/Functions/runningAccumulate.cpp @@ -33,7 +33,7 @@ class FunctionRunningAccumulate : public IFunction { public: static constexpr auto name = "runningAccumulate"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/runningConcurrency.cpp b/src/Functions/runningConcurrency.cpp index 022e2be5f6c..6f842bcc342 100644 --- a/src/Functions/runningConcurrency.cpp +++ b/src/Functions/runningConcurrency.cpp @@ -154,7 +154,7 @@ namespace DB public: static constexpr auto name = Name::name; - static FunctionOverloadResolverPtr create(ContextPtr) + static FunctionOverloadResolverPtr create(ContextConstPtr) { return std::make_unique>(); } diff --git a/src/Functions/runningDifference.h b/src/Functions/runningDifference.h index f87c57af043..41a418b9b58 100644 --- a/src/Functions/runningDifference.h +++ b/src/Functions/runningDifference.h @@ -124,7 +124,7 @@ private: public: static constexpr auto name = FunctionRunningDifferenceName::name; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } diff --git a/src/Functions/sleep.h b/src/Functions/sleep.h index c0aad0b3820..16b3973cdc2 100644 --- a/src/Functions/sleep.h +++ b/src/Functions/sleep.h @@ -36,7 +36,7 @@ class FunctionSleep : public IFunction { public: static constexpr auto name = variant == FunctionSleepVariant::PerBlock ? "sleep" : "sleepEachRow"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared>(); } diff --git a/src/Functions/stringToH3.cpp b/src/Functions/stringToH3.cpp index 9b02711db50..97f9ba3f2ab 100644 --- a/src/Functions/stringToH3.cpp +++ b/src/Functions/stringToH3.cpp @@ -34,7 +34,7 @@ class FunctionStringToH3 : public IFunction public: static constexpr auto name = "stringToH3"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } std::string getName() const override { return name; } diff --git a/src/Functions/substring.cpp b/src/Functions/substring.cpp index 5ce75035475..13f0e296602 100644 --- a/src/Functions/substring.cpp +++ b/src/Functions/substring.cpp @@ -36,7 +36,7 @@ class FunctionSubstring : public IFunction { public: static constexpr auto name = is_utf8 ? "substringUTF8" : "substring"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 39473816c8c..4d0edaf004b 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -24,7 +24,7 @@ public: explicit FunctionSvg() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/tcpPort.cpp b/src/Functions/tcpPort.cpp index 484843ced3f..baacdb0a335 100644 --- a/src/Functions/tcpPort.cpp +++ b/src/Functions/tcpPort.cpp @@ -14,7 +14,7 @@ class FunctionTcpPort : public IFunction public: static constexpr auto name = "tcpPort"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getTCPPort()); } diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp index 1b56cc0d188..a3285f64c73 100644 --- a/src/Functions/throwIf.cpp +++ b/src/Functions/throwIf.cpp @@ -26,7 +26,7 @@ class FunctionThrowIf : public IFunction { public: static constexpr auto name = "throwIf"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/tid.cpp b/src/Functions/tid.cpp index 404eff862b3..6f2eb3cc595 100644 --- a/src/Functions/tid.cpp +++ b/src/Functions/tid.cpp @@ -13,7 +13,7 @@ namespace { public: static constexpr auto name = "tid"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index b64d2687b05..4f8882a83ad 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -109,7 +109,7 @@ class FunctionTimeSlots : public IFunction public: static constexpr auto name = "timeSlots"; static constexpr UInt32 TIME_SLOT_SIZE = 1800; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/timezone.cpp b/src/Functions/timezone.cpp index 67f7462fc95..c0d136ab7d4 100644 --- a/src/Functions/timezone.cpp +++ b/src/Functions/timezone.cpp @@ -16,7 +16,7 @@ class FunctionTimezone : public IFunction { public: static constexpr auto name = "timezone"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/timezoneOf.cpp b/src/Functions/timezoneOf.cpp index 3dc72424a4f..f768722a284 100644 --- a/src/Functions/timezoneOf.cpp +++ b/src/Functions/timezoneOf.cpp @@ -28,7 +28,7 @@ class FunctionTimezoneOf : public IFunction public: static constexpr auto name = "timezoneOf"; String getName() const override { return name; } - static FunctionPtr create(ContextPtr) { return std::make_unique(); } + static FunctionPtr create(ContextConstPtr) { return std::make_unique(); } size_t getNumberOfArguments() const override { return 1; } diff --git a/src/Functions/toColumnTypeName.cpp b/src/Functions/toColumnTypeName.cpp index d64fa12604e..2d3635f4ed1 100644 --- a/src/Functions/toColumnTypeName.cpp +++ b/src/Functions/toColumnTypeName.cpp @@ -14,7 +14,7 @@ class FunctionToColumnTypeName : public IFunction { public: static constexpr auto name = "toColumnTypeName"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/toFixedString.h b/src/Functions/toFixedString.h index fe437a24303..303d51f41ad 100644 --- a/src/Functions/toFixedString.h +++ b/src/Functions/toFixedString.h @@ -32,7 +32,7 @@ class FunctionToFixedString : public IFunction { public: static constexpr auto name = "toFixedString"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } static FunctionPtr create() { return std::make_shared(); } String getName() const override diff --git a/src/Functions/toLowCardinality.cpp b/src/Functions/toLowCardinality.cpp index 983e66d1007..8f102d5db9c 100644 --- a/src/Functions/toLowCardinality.cpp +++ b/src/Functions/toLowCardinality.cpp @@ -14,7 +14,7 @@ class FunctionToLowCardinality: public IFunction { public: static constexpr auto name = "toLowCardinality"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } diff --git a/src/Functions/toModifiedJulianDay.cpp b/src/Functions/toModifiedJulianDay.cpp index a44979b52ff..c76b158baa8 100644 --- a/src/Functions/toModifiedJulianDay.cpp +++ b/src/Functions/toModifiedJulianDay.cpp @@ -172,7 +172,7 @@ namespace DB public: static constexpr auto name = Name::name; - static FunctionOverloadResolverPtr create(ContextPtr) + static FunctionOverloadResolverPtr create(ContextConstPtr) { return std::make_unique>(); } diff --git a/src/Functions/toNullable.cpp b/src/Functions/toNullable.cpp index 5e383893476..12d23e59f49 100644 --- a/src/Functions/toNullable.cpp +++ b/src/Functions/toNullable.cpp @@ -16,7 +16,7 @@ class FunctionToNullable : public IFunction public: static constexpr auto name = "toNullable"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index ee592fbb1e3..97d2d65b67a 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -184,7 +184,7 @@ namespace class FunctionToStartOfInterval : public IFunction { public: - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } static constexpr auto name = function_name; String getName() const override { return name; } diff --git a/src/Functions/toTimezone.cpp b/src/Functions/toTimezone.cpp index 551e07a8354..ef80c2f0849 100644 --- a/src/Functions/toTimezone.cpp +++ b/src/Functions/toTimezone.cpp @@ -25,7 +25,7 @@ class FunctionToTimezone : public IFunction { public: static constexpr auto name = "toTimezone"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index 3c733fb3ea7..f369ff1ccef 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -18,7 +18,7 @@ public: static constexpr auto name = "toTypeName"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/toUnixTimestamp64Micro.cpp b/src/Functions/toUnixTimestamp64Micro.cpp index c5b841a1a81..bd13f753802 100644 --- a/src/Functions/toUnixTimestamp64Micro.cpp +++ b/src/Functions/toUnixTimestamp64Micro.cpp @@ -7,7 +7,7 @@ namespace DB void registerToUnixTimestamp64Micro(FunctionFactory & factory) { factory.registerFunction("toUnixTimestamp64Micro", - [](ContextPtr){ return std::make_unique( + [](ContextConstPtr){ return std::make_unique( std::make_shared(6, "toUnixTimestamp64Micro")); }); } diff --git a/src/Functions/toUnixTimestamp64Milli.cpp b/src/Functions/toUnixTimestamp64Milli.cpp index bfceb3708d3..088cd2011c5 100644 --- a/src/Functions/toUnixTimestamp64Milli.cpp +++ b/src/Functions/toUnixTimestamp64Milli.cpp @@ -7,7 +7,7 @@ namespace DB void registerToUnixTimestamp64Milli(FunctionFactory & factory) { factory.registerFunction("toUnixTimestamp64Milli", - [](ContextPtr){ return std::make_unique( + [](ContextConstPtr){ return std::make_unique( std::make_shared(3, "toUnixTimestamp64Milli")); }); } diff --git a/src/Functions/toUnixTimestamp64Nano.cpp b/src/Functions/toUnixTimestamp64Nano.cpp index 2256dc369b9..3961b9630ba 100644 --- a/src/Functions/toUnixTimestamp64Nano.cpp +++ b/src/Functions/toUnixTimestamp64Nano.cpp @@ -7,7 +7,7 @@ namespace DB void registerToUnixTimestamp64Nano(FunctionFactory & factory) { factory.registerFunction("toUnixTimestamp64Nano", - [](ContextPtr){ return std::make_unique( + [](ContextConstPtr){ return std::make_unique( std::make_shared(9, "toUnixTimestamp64Nano")); }); } diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp index 43be4c4582a..9d04bd3a84d 100644 --- a/src/Functions/today.cpp +++ b/src/Functions/today.cpp @@ -71,7 +71,7 @@ public: size_t getNumberOfArguments() const override { return 0; } - static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique(); } + static FunctionOverloadResolverPtr create(ContextConstPtr) { return std::make_unique(); } DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared(); } diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index 1debc2cb6a0..e9305304ac0 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -58,7 +58,7 @@ class FunctionTransform : public IFunction { public: static constexpr auto name = "transform"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { diff --git a/src/Functions/trap.cpp b/src/Functions/trap.cpp index f946a7fabfb..c7f935bac3e 100644 --- a/src/Functions/trap.cpp +++ b/src/Functions/trap.cpp @@ -32,16 +32,16 @@ namespace ErrorCodes class FunctionTrap : public IFunction { private: - ContextPtr context; + ContextConstPtr context; public: static constexpr auto name = "trap"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } - FunctionTrap(ContextPtr context_) : context(context_) {} + FunctionTrap(ContextConstPtr context_) : context(context_) {} String getName() const override { diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index dda034ee911..8c1ad90070e 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -24,7 +24,7 @@ class FunctionTuple : public IFunction public: static constexpr auto name = "tuple"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/tupleElement.cpp b/src/Functions/tupleElement.cpp index 97e2f70fccf..7f71e9d93e9 100644 --- a/src/Functions/tupleElement.cpp +++ b/src/Functions/tupleElement.cpp @@ -30,7 +30,7 @@ class FunctionTupleElement : public IFunction { public: static constexpr auto name = "tupleElement"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/tupleHammingDistance.cpp b/src/Functions/tupleHammingDistance.cpp index 9d660e388cb..a4ad7e9a5ba 100644 --- a/src/Functions/tupleHammingDistance.cpp +++ b/src/Functions/tupleHammingDistance.cpp @@ -19,14 +19,14 @@ namespace ErrorCodes class FunctionTupleHammingDistance : public IFunction { private: - ContextPtr context; + ContextConstPtr context; public: static constexpr auto name = "tupleHammingDistance"; using ResultType = UInt8; - explicit FunctionTupleHammingDistance(ContextPtr context_) : context(context_) {} - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + explicit FunctionTupleHammingDistance(ContextConstPtr context_) : context(context_) {} + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context); } String getName() const override { return name; } diff --git a/src/Functions/uptime.cpp b/src/Functions/uptime.cpp index 02454df4de5..7e2165318a3 100644 --- a/src/Functions/uptime.cpp +++ b/src/Functions/uptime.cpp @@ -13,7 +13,7 @@ class FunctionUptime : public IFunction { public: static constexpr auto name = "uptime"; - static FunctionPtr create(ContextPtr context) + static FunctionPtr create(ContextConstPtr context) { return std::make_shared(context->getUptimeSeconds()); } diff --git a/src/Functions/validateNestedArraySizes.cpp b/src/Functions/validateNestedArraySizes.cpp index 1d96f988690..f3327479844 100644 --- a/src/Functions/validateNestedArraySizes.cpp +++ b/src/Functions/validateNestedArraySizes.cpp @@ -24,7 +24,7 @@ class FunctionValidateNestedArraySizes : public IFunction { public: static constexpr auto name = "validateNestedArraySizes"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/version.cpp b/src/Functions/version.cpp index 4e0ddf60975..6181999e2f5 100644 --- a/src/Functions/version.cpp +++ b/src/Functions/version.cpp @@ -16,7 +16,7 @@ class FunctionVersion : public IFunction { public: static constexpr auto name = "version"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/visibleWidth.cpp b/src/Functions/visibleWidth.cpp index 6e96a4844ce..f6a79a82df6 100644 --- a/src/Functions/visibleWidth.cpp +++ b/src/Functions/visibleWidth.cpp @@ -19,7 +19,7 @@ class FunctionVisibleWidth : public IFunction { public: static constexpr auto name = "visibleWidth"; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index b5567a19e89..888f6a4ea5f 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -16,7 +16,7 @@ public: explicit FunctionWkt() = default; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } diff --git a/src/Functions/yesterday.cpp b/src/Functions/yesterday.cpp index 737552e192e..0b3ac6d2a8e 100644 --- a/src/Functions/yesterday.cpp +++ b/src/Functions/yesterday.cpp @@ -69,7 +69,7 @@ public: size_t getNumberOfArguments() const override { return 0; } - static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique(); } + static FunctionOverloadResolverPtr create(ContextConstPtr) { return std::make_unique(); } DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared(); } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index 42b800f15d0..160566c0f8d 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -111,7 +111,7 @@ struct ConnectionTimeouts static ConnectionTimeouts getTCPTimeoutsWithoutFailover(const Settings & settings); /// Timeouts for the case when we will try many addresses in a loop. static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); - static ConnectionTimeouts getHTTPTimeouts(ContextPtr context); + static ConnectionTimeouts getHTTPTimeouts(ContextConstPtr context); }; } diff --git a/src/IO/ConnectionTimeoutsContext.h b/src/IO/ConnectionTimeoutsContext.h index 8d6f96b82a3..909ccff8ef0 100644 --- a/src/IO/ConnectionTimeoutsContext.h +++ b/src/IO/ConnectionTimeoutsContext.h @@ -27,7 +27,7 @@ inline ConnectionTimeouts ConnectionTimeouts::getTCPTimeoutsWithFailover(const S settings.receive_data_timeout_ms); } -inline ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(ContextPtr context) +inline ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(ContextConstPtr context) { const auto & settings = context->getSettingsRef(); const auto & config = context->getConfigRef(); diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 9003ec2d9ed..a772d4ccd69 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -270,16 +270,37 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) } const size_t initial_pos = buf.count(); + bool has_sign = false; + bool has_number = false; while (!buf.eof()) { switch (*buf.position()) { case '+': { + if (has_sign || has_number) + { + if constexpr (throw_exception) + throw ParsingException( + "Cannot parse number with multiple sign (+/-) characters or intermediate sign character", + ErrorCodes::CANNOT_PARSE_NUMBER); + else + return ReturnType(false); + } + has_sign = true; break; } case '-': { + if (has_sign || has_number) + { + if constexpr (throw_exception) + throw ParsingException( + "Cannot parse number with multiple sign (+/-) characters or intermediate sign character", + ErrorCodes::CANNOT_PARSE_NUMBER); + else + return ReturnType(false); + } if constexpr (is_signed_v) negative = true; else @@ -289,6 +310,7 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) else return ReturnType(false); } + has_sign = true; break; } case '0': [[fallthrough]]; @@ -302,6 +324,7 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) case '8': [[fallthrough]]; case '9': { + has_number = true; if constexpr (check_overflow == ReadIntTextCheckOverflow::CHECK_OVERFLOW && !is_big_int_v) { /// Perform relativelly slow overflow check only when @@ -330,6 +353,14 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) } end: + if (has_sign && !has_number) + { + if constexpr (throw_exception) + throw ParsingException( + "Cannot parse number with a sign character but without any numeric character", ErrorCodes::CANNOT_PARSE_NUMBER); + else + return ReturnType(false); + } x = res; if constexpr (is_signed_v) { diff --git a/src/IO/examples/parse_int_perf.cpp b/src/IO/examples/parse_int_perf.cpp index e35a3d8a857..5a8f8b01f50 100644 --- a/src/IO/examples/parse_int_perf.cpp +++ b/src/IO/examples/parse_int_perf.cpp @@ -81,8 +81,8 @@ int main(int argc, char ** argv) << "Written " << n << " numbers (" << wb.count() / 1000000.0 << " MB) in " << watch.elapsedSeconds() << " sec., " << n / watch.elapsedSeconds() << " num/s., " << wb.count() / watch.elapsedSeconds() / 1000000 << " MB/s., " - << watch.elapsed() / n << " ns/num., " - << tsc / n << " ticks/num., " + << watch.elapsed() / n << " ns/num., " // NOLINT + << tsc / n << " ticks/num., " // NOLINT << watch.elapsed() / wb.count() << " ns/byte., " << tsc / wb.count() << " ticks/byte." << std::endl; diff --git a/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp b/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp index 53248467418..656e40b2ad7 100644 --- a/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp +++ b/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include #include @@ -9,7 +8,9 @@ #include #include #include +#include +namespace fs = std::filesystem; using namespace DB; @@ -235,7 +236,7 @@ try buf.reset(); reread_buf.reset(); - ASSERT_TRUE(!Poco::File(tmp_filename).exists()); + ASSERT_TRUE(!fs::exists(tmp_filename)); } } catch (...) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 5284756a4ff..2af65e0fd87 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -24,7 +24,7 @@ namespace ErrorCodes namespace ClusterProxy { -ContextPtr updateSettingsForCluster(const Cluster & cluster, ContextPtr context, const Settings & settings, Poco::Logger * log) +ContextMutablePtr updateSettingsForCluster(const Cluster & cluster, ContextPtr context, const Settings & settings, Poco::Logger * log) { Settings new_settings = settings; new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time); diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index 46525335803..c9efedfc422 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -31,7 +31,7 @@ class IStreamFactory; /// - optimize_skip_unused_shards_nesting /// /// @return new Context with adjusted settings -ContextPtr updateSettingsForCluster(const Cluster & cluster, ContextPtr context, const Settings & settings, Poco::Logger * log = nullptr); +ContextMutablePtr updateSettingsForCluster(const Cluster & cluster, ContextPtr context, const Settings & settings, Poco::Logger * log = nullptr); /// Execute a distributed query, creating a vector of BlockInputStreams, from which the result can be read. /// `stream_factory` object encapsulates the logic of creating streams for a different type of query diff --git a/src/Interpreters/ColumnAliasesVisitor.h b/src/Interpreters/ColumnAliasesVisitor.h index a1f8e79f64c..6b89c1a086f 100644 --- a/src/Interpreters/ColumnAliasesVisitor.h +++ b/src/Interpreters/ColumnAliasesVisitor.h @@ -56,12 +56,12 @@ public: /// Do not analyze joined columns. /// They may have aliases and come to description as is. const NameSet & forbidden_columns; - ContextPtr context; + ContextConstPtr context; /// private_aliases are from lambda, so these are local names. NameSet private_aliases; - Data(const ColumnsDescription & columns_, const NameSet & forbidden_columns_, ContextPtr context_) + Data(const ColumnsDescription & columns_, const NameSet & forbidden_columns_, ContextConstPtr context_) : columns(columns_) , forbidden_columns(forbidden_columns_) , context(context_) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7d9c2c27ee2..a920a26e4c4 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -74,8 +74,11 @@ #include #include #include +#include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event ContextLock; @@ -140,7 +143,7 @@ public: /// Find existing session or create a new. std::shared_ptr acquireSession( const String & session_id, - ContextPtr context, + ContextMutablePtr context, std::chrono::steady_clock::duration timeout, bool throw_if_not_found) { @@ -513,7 +516,7 @@ SharedContextHolder::SharedContextHolder(std::unique_ptr shar void SharedContextHolder::reset() { shared.reset(); } -ContextPtr Context::createGlobal(ContextSharedPart * shared) +ContextMutablePtr Context::createGlobal(ContextSharedPart * shared) { auto res = std::shared_ptr(new Context); res->shared = shared; @@ -530,19 +533,19 @@ SharedContextHolder Context::createShared() return SharedContextHolder(std::make_unique()); } -ContextPtr Context::createCopy(const ContextConstPtr & other) +ContextMutablePtr Context::createCopy(const ContextPtr & other) { return std::shared_ptr(new Context(*other)); } -ContextPtr Context::createCopy(const ContextWeakConstPtr & other) +ContextMutablePtr Context::createCopy(const ContextWeakConstPtr & other) { auto ptr = other.lock(); if (!ptr) throw Exception("Can't copy an expired context", ErrorCodes::LOGICAL_ERROR); return createCopy(ptr); } -ContextPtr Context::createCopy(const ContextPtr & other) +ContextMutablePtr Context::createCopy(const ContextMutablePtr & other) { return createCopy(std::const_pointer_cast(other)); } @@ -1079,7 +1082,7 @@ void Context::addViewSource(const StoragePtr & storage) } -StoragePtr Context::getViewSource() +StoragePtr Context::getViewSource() const { return view_source; } @@ -1313,7 +1316,7 @@ void Context::setMacros(std::unique_ptr && macros) shared->macros.set(std::move(macros)); } -ContextPtr Context::getQueryContext() const +ContextMutablePtr Context::getQueryContext() const { auto ptr = query_context.lock(); if (!ptr) throw Exception("There is no query or query context has expired", ErrorCodes::THERE_IS_NO_QUERY); @@ -1326,21 +1329,21 @@ bool Context::isInternalSubquery() const return ptr && ptr.get() != this; } -ContextPtr Context::getSessionContext() const +ContextMutablePtr Context::getSessionContext() const { auto ptr = session_context.lock(); if (!ptr) throw Exception("There is no session or session context has expired", ErrorCodes::THERE_IS_NO_SESSION); return ptr; } -ContextPtr Context::getGlobalContext() const +ContextMutablePtr Context::getGlobalContext() const { auto ptr = global_context.lock(); if (!ptr) throw Exception("There is no global context or global context has expired", ErrorCodes::LOGICAL_ERROR); return ptr; } -ContextPtr Context::getBufferContext() const +ContextMutablePtr Context::getBufferContext() const { if (!buffer_context) throw Exception("There is no buffer context", ErrorCodes::LOGICAL_ERROR); return buffer_context; @@ -1871,7 +1874,7 @@ std::shared_ptr Context::tryGetCluster(const std::string & cluster_name } -void Context::reloadClusterConfig() +void Context::reloadClusterConfig() const { while (true) { @@ -1958,7 +1961,7 @@ bool Context::hasTraceCollector() const } -std::shared_ptr Context::getQueryLog() +std::shared_ptr Context::getQueryLog() const { auto lock = getLock(); @@ -1969,7 +1972,7 @@ std::shared_ptr Context::getQueryLog() } -std::shared_ptr Context::getQueryThreadLog() +std::shared_ptr Context::getQueryThreadLog() const { auto lock = getLock(); @@ -1980,7 +1983,7 @@ std::shared_ptr Context::getQueryThreadLog() } -std::shared_ptr Context::getPartLog(const String & part_database) +std::shared_ptr Context::getPartLog(const String & part_database) const { auto lock = getLock(); @@ -1998,7 +2001,7 @@ std::shared_ptr Context::getPartLog(const String & part_database) } -std::shared_ptr Context::getTraceLog() +std::shared_ptr Context::getTraceLog() const { auto lock = getLock(); @@ -2009,7 +2012,7 @@ std::shared_ptr Context::getTraceLog() } -std::shared_ptr Context::getTextLog() +std::shared_ptr Context::getTextLog() const { auto lock = getLock(); @@ -2020,7 +2023,7 @@ std::shared_ptr Context::getTextLog() } -std::shared_ptr Context::getMetricLog() +std::shared_ptr Context::getMetricLog() const { auto lock = getLock(); @@ -2042,7 +2045,7 @@ std::shared_ptr Context::getAsynchronousMetricLog() const } -std::shared_ptr Context::getOpenTelemetrySpanLog() +std::shared_ptr Context::getOpenTelemetrySpanLog() const { auto lock = getLock(); @@ -2212,14 +2215,14 @@ void Context::checkCanBeDropped(const String & database, const String & table, c if (!max_size_to_drop || size <= max_size_to_drop) return; - Poco::File force_file(getFlagsPath() + "force_drop_table"); - bool force_file_exists = force_file.exists(); + fs::path force_file(getFlagsPath() + "force_drop_table"); + bool force_file_exists = fs::exists(force_file); if (force_file_exists) { try { - force_file.remove(); + fs::remove(force_file); return; } catch (...) @@ -2241,9 +2244,9 @@ void Context::checkCanBeDropped(const String & database, const String & table, c "Example:\nsudo touch '{}' && sudo chmod 666 '{}'", backQuoteIfNeed(database), backQuoteIfNeed(table), size_str, max_size_to_drop_str, - force_file.path(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist", - force_file.path(), - force_file.path(), force_file.path()); + force_file.string(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist", + force_file.string(), + force_file.string(), force_file.string()); } @@ -2640,11 +2643,14 @@ ZooKeeperMetadataTransactionPtr Context::getZooKeeperMetadataTransaction() const return metadata_transaction; } -PartUUIDsPtr Context::getPartUUIDs() +PartUUIDsPtr Context::getPartUUIDs() const { auto lock = getLock(); if (!part_uuids) - part_uuids = std::make_shared(); + /// For context itself, only this initialization is not const. + /// We could have done in constructor. + /// TODO: probably, remove this from Context. + const_cast(part_uuids) = std::make_shared(); return part_uuids; } @@ -2663,11 +2669,11 @@ void Context::setReadTaskCallback(ReadTaskCallback && callback) next_task_callback = callback; } -PartUUIDsPtr Context::getIgnoredPartUUIDs() +PartUUIDsPtr Context::getIgnoredPartUUIDs() const { auto lock = getLock(); if (!ignored_part_uuids) - ignored_part_uuids = std::make_shared(); + const_cast(ignored_part_uuids) = std::make_shared(); return ignored_part_uuids; } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 5089d2c0288..97cb1b980f1 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -252,12 +252,12 @@ private: StoragePtr view_source; /// Temporary StorageValues used to generate alias columns for materialized views Tables table_function_results; /// Temporary tables obtained by execution of table functions. Keyed by AST tree id. - ContextWeakPtr query_context; - ContextWeakPtr session_context; /// Session context or nullptr. Could be equal to this. - ContextWeakPtr global_context; /// Global context. Could be equal to this. + ContextWeakMutablePtr query_context; + ContextWeakMutablePtr session_context; /// Session context or nullptr. Could be equal to this. + ContextWeakMutablePtr global_context; /// Global context. Could be equal to this. /// XXX: move this stuff to shared part instead. - ContextPtr buffer_context; /// Buffer context. Could be equal to this. + ContextMutablePtr buffer_context; /// Buffer context. Could be equal to this. public: // Top-level OpenTelemetry trace context for the query. Makes sense only for a query context. @@ -293,10 +293,10 @@ private: public: /// Create initial Context with ContextShared and etc. - static ContextPtr createGlobal(ContextSharedPart * shared); - static ContextPtr createCopy(const ContextWeakConstPtr & other); - static ContextPtr createCopy(const ContextConstPtr & other); - static ContextPtr createCopy(const ContextPtr & other); + static ContextMutablePtr createGlobal(ContextSharedPart * shared); + static ContextMutablePtr createCopy(const ContextWeakConstPtr & other); + static ContextMutablePtr createCopy(const ContextMutablePtr & other); + static ContextMutablePtr createCopy(const ContextPtr & other); static SharedContextHolder createShared(); void copyFrom(const ContextPtr & other); @@ -459,7 +459,7 @@ public: StoragePtr executeTableFunction(const ASTPtr & table_expression); void addViewSource(const StoragePtr & storage); - StoragePtr getViewSource(); + StoragePtr getViewSource() const; String getCurrentDatabase() const; String getCurrentQueryId() const { return client_info.current_query_id; } @@ -555,14 +555,14 @@ public: /// For methods below you may need to acquire the context lock by yourself. - ContextPtr getQueryContext() const; + ContextMutablePtr getQueryContext() const; bool hasQueryContext() const { return !query_context.expired(); } bool isInternalSubquery() const; - ContextPtr getSessionContext() const; + ContextMutablePtr getSessionContext() const; bool hasSessionContext() const { return !session_context.expired(); } - ContextPtr getGlobalContext() const; + ContextMutablePtr getGlobalContext() const; bool hasGlobalContext() const { return !global_context.expired(); } bool isGlobalContext() const { @@ -570,10 +570,10 @@ public: return ptr && ptr.get() == this; } - ContextPtr getBufferContext() const; + ContextMutablePtr getBufferContext() const; - void setQueryContext(ContextPtr context_) { query_context = context_; } - void setSessionContext(ContextPtr context_) { session_context = context_; } + void setQueryContext(ContextMutablePtr context_) { query_context = context_; } + void setSessionContext(ContextMutablePtr context_) { session_context = context_; } void makeQueryContext() { query_context = shared_from_this(); } void makeSessionContext() { session_context = shared_from_this(); } @@ -668,7 +668,7 @@ public: void setClustersConfig(const ConfigurationPtr & config, const String & config_name = "remote_servers"); /// Sets custom cluster, but doesn't update configuration void setCluster(const String & cluster_name, const std::shared_ptr & cluster); - void reloadClusterConfig(); + void reloadClusterConfig() const; Compiler & getCompiler(); @@ -681,17 +681,17 @@ public: bool hasTraceCollector() const; /// Nullptr if the query log is not ready for this moment. - std::shared_ptr getQueryLog(); - std::shared_ptr getQueryThreadLog(); - std::shared_ptr getTraceLog(); - std::shared_ptr getTextLog(); - std::shared_ptr getMetricLog(); + std::shared_ptr getQueryLog() const; + std::shared_ptr getQueryThreadLog() const; + std::shared_ptr getTraceLog() const; + std::shared_ptr getTextLog() const; + std::shared_ptr getMetricLog() const; std::shared_ptr getAsynchronousMetricLog() const; - std::shared_ptr getOpenTelemetrySpanLog(); + std::shared_ptr getOpenTelemetrySpanLog() const; /// Returns an object used to log operations with parts if it possible. /// Provide table name to make required checks. - std::shared_ptr getPartLog(const String & part_database); + std::shared_ptr getPartLog(const String & part_database) const; const MergeTreeSettings & getMergeTreeSettings() const; const MergeTreeSettings & getReplicatedMergeTreeSettings() const; @@ -778,8 +778,8 @@ public: MySQLWireContext mysql; - PartUUIDsPtr getPartUUIDs(); - PartUUIDsPtr getIgnoredPartUUIDs(); + PartUUIDsPtr getPartUUIDs() const; + PartUUIDsPtr getIgnoredPartUUIDs() const; ReadTaskCallback getReadTaskCallback() const; void setReadTaskCallback(ReadTaskCallback && callback); @@ -820,7 +820,7 @@ struct NamedSession { NamedSessionKey key; UInt64 close_cycle = 0; - ContextPtr context; + ContextMutablePtr context; std::chrono::steady_clock::duration timeout; NamedSessions & parent; diff --git a/src/Interpreters/Context_fwd.h b/src/Interpreters/Context_fwd.h index d7232717bb2..af18e5a7fdd 100644 --- a/src/Interpreters/Context_fwd.h +++ b/src/Interpreters/Context_fwd.h @@ -22,10 +22,12 @@ class Context; /// Most used types have shorter names /// TODO: in the first part of refactoring all the context pointers are non-const. -using ContextPtr = std::shared_ptr; +using ContextPtr = std::shared_ptr; using ContextConstPtr = std::shared_ptr; -using ContextWeakPtr = std::weak_ptr; +using ContextMutablePtr = std::shared_ptr; +using ContextWeakPtr = std::weak_ptr; using ContextWeakConstPtr = std::weak_ptr; +using ContextWeakMutablePtr = std::weak_ptr; template struct WithContextImpl @@ -50,5 +52,6 @@ protected: using WithContext = WithContextImpl<>; using WithConstContext = WithContextImpl; +using WithMutableContext = WithContextImpl; } diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 087e5504672..4fb44738d8d 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -145,7 +145,7 @@ void DDLTaskBase::parseQueryFromEntry(ContextPtr context) query = parseQuery(parser_query, begin, end, description, 0, context->getSettingsRef().max_parser_depth); } -ContextPtr DDLTaskBase::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & /*zookeeper*/) +ContextMutablePtr DDLTaskBase::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & /*zookeeper*/) { auto query_context = Context::createCopy(from_context); query_context->makeQueryContext(); @@ -355,7 +355,7 @@ void DatabaseReplicatedTask::parseQueryFromEntry(ContextPtr context) } } -ContextPtr DatabaseReplicatedTask::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper) +ContextMutablePtr DatabaseReplicatedTask::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper) { auto query_context = DDLTaskBase::makeQueryContext(from_context, zookeeper); query_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index 5fafb42bfd2..703d691a358 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace Poco { @@ -14,6 +15,8 @@ namespace zkutil class ZooKeeper; } +namespace fs = std::filesystem; + namespace DB { @@ -98,11 +101,11 @@ struct DDLTaskBase virtual String getShardID() const = 0; - virtual ContextPtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper); + virtual ContextMutablePtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper); - inline String getActiveNodePath() const { return entry_path + "/active/" + host_id_str; } - inline String getFinishedNodePath() const { return entry_path + "/finished/" + host_id_str; } - inline String getShardNodePath() const { return entry_path + "/shards/" + getShardID(); } + inline String getActiveNodePath() const { return fs::path(entry_path) / "active" / host_id_str; } + inline String getFinishedNodePath() const { return fs::path(entry_path) / "finished" / host_id_str; } + inline String getShardNodePath() const { return fs::path(entry_path) / "shards" / getShardID(); } static String getLogEntryName(UInt32 log_entry_number); static UInt32 getLogEntryNumber(const String & log_entry_name); @@ -136,7 +139,7 @@ struct DatabaseReplicatedTask : public DDLTaskBase String getShardID() const override; void parseQueryFromEntry(ContextPtr context) override; - ContextPtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper) override; + ContextMutablePtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper) override; DatabaseReplicated * database; }; diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index e92644f0f99..45218226fee 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -110,7 +110,7 @@ protected: void runMainThread(); void runCleanupThread(); - ContextPtr context; + ContextMutablePtr context; Poco::Logger * log; std::string host_fqdn; /// current host domain name diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 09166103155..a70cdb2e054 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,8 @@ #include #include #include +#include +#include #if !defined(ARCADIA_BUILD) # include "config_core.h" @@ -27,8 +28,7 @@ # include #endif -#include - +namespace fs = std::filesystem; namespace CurrentMetrics { @@ -354,10 +354,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, bool d db->drop(getContext()); /// Old ClickHouse versions did not store database.sql files - Poco::File database_metadata_file( - getContext()->getPath() + "metadata/" + escapeForFileName(database_name) + ".sql"); - if (database_metadata_file.exists()) - database_metadata_file.remove(false); + fs::path database_metadata_file = fs::path(getContext()->getPath()) / "metadata" / (escapeForFileName(database_name) + ".sql"); + if (fs::exists(database_metadata_file)) + fs::remove(database_metadata_file); } return db; @@ -528,13 +527,13 @@ void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, std::unique_ptr DatabaseCatalog::database_catalog; -DatabaseCatalog::DatabaseCatalog(ContextPtr global_context_) - : WithContext(global_context_), log(&Poco::Logger::get("DatabaseCatalog")) +DatabaseCatalog::DatabaseCatalog(ContextMutablePtr global_context_) + : WithMutableContext(global_context_), log(&Poco::Logger::get("DatabaseCatalog")) { TemporaryLiveViewCleaner::init(global_context_); } -DatabaseCatalog & DatabaseCatalog::init(ContextPtr global_context_) +DatabaseCatalog & DatabaseCatalog::init(ContextMutablePtr global_context_) { if (database_catalog) { @@ -783,7 +782,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr } addUUIDMapping(table_id.uuid); - drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); + drop_time = FS::getModificationTime(dropped_metadata_path); } std::lock_guard lock(tables_marked_dropped_mutex); @@ -892,16 +891,15 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) /// Even if table is not loaded, try remove its data from disk. /// TODO remove data from all volumes - String data_path = getContext()->getPath() + "store/" + getPathForUUID(table.table_id.uuid); - Poco::File table_data_dir{data_path}; - if (table_data_dir.exists()) + fs::path data_path = fs::path(getContext()->getPath()) / "store" / getPathForUUID(table.table_id.uuid); + if (fs::exists(data_path)) { - LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path, table.table_id.getNameForLogs()); - table_data_dir.remove(true); + LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path.string(), table.table_id.getNameForLogs()); + fs::remove_all(data_path); } LOG_INFO(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); - Poco::File(table.metadata_path).remove(); + fs::remove(fs::path(table.metadata_path)); removeUUIDMappingFinally(table.table_id.uuid); CurrentMetrics::sub(CurrentMetrics::TablesToDropQueueSize, 1); diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 783c511fa08..e20259fb973 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -117,13 +117,13 @@ using TemporaryTablesMapping = std::map database_catalog; - explicit DatabaseCatalog(ContextPtr global_context_); + explicit DatabaseCatalog(ContextMutablePtr global_context_); void assertDatabaseExistsUnlocked(const String & database_name) const; void assertDatabaseDoesntExistUnlocked(const String & database_name) const; diff --git a/src/Interpreters/DuplicateOrderByVisitor.h b/src/Interpreters/DuplicateOrderByVisitor.h index 4231b2600af..bd8652ff298 100644 --- a/src/Interpreters/DuplicateOrderByVisitor.h +++ b/src/Interpreters/DuplicateOrderByVisitor.h @@ -21,7 +21,7 @@ class ASTFunctionStatefulData public: using TypeToVisit = ASTFunction; - ContextPtr context; + ContextConstPtr context; bool & is_stateful; void visit(ASTFunction & ast_function, ASTPtr &) { @@ -80,7 +80,7 @@ class DuplicateOrderByData public: using TypeToVisit = ASTSelectQuery; - ContextPtr context; + ContextConstPtr context; void visit(ASTSelectQuery & select_query, ASTPtr &) { diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 97ad191289c..1c947e62e69 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -25,7 +25,6 @@ namespace DB namespace ErrorCodes { extern const int INCORRECT_RESULT_OF_SCALAR_SUBQUERY; - extern const int TOO_MANY_ROWS; } @@ -116,36 +115,26 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr { auto io = interpreter.execute(); - try + PullingAsyncPipelineExecutor executor(io.pipeline); + while (block.rows() == 0 && executor.pull(block)); + + if (block.rows() == 0) { - PullingAsyncPipelineExecutor executor(io.pipeline); - while (block.rows() == 0 && executor.pull(block)); - - if (block.rows() == 0) - { - /// Interpret subquery with empty result as Null literal - auto ast_new = std::make_unique(Null()); - ast_new->setAlias(ast->tryGetAlias()); - ast = std::move(ast_new); - return; - } - - if (block.rows() != 1) - throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); - - Block tmp_block; - while (tmp_block.rows() == 0 && executor.pull(tmp_block)); - - if (tmp_block.rows() != 0) - throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); - } - catch (const Exception & e) - { - if (e.code() == ErrorCodes::TOO_MANY_ROWS) - throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); - else - throw; + /// Interpret subquery with empty result as Null literal + auto ast_new = std::make_unique(Null()); + ast_new->setAlias(ast->tryGetAlias()); + ast = std::move(ast_new); + return; } + + if (block.rows() != 1) + throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); + + Block tmp_block; + while (tmp_block.rows() == 0 && executor.pull(tmp_block)); + + if (tmp_block.rows() != 0) + throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); } block = materializeBlock(block); diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.h b/src/Interpreters/ExecuteScalarSubqueriesVisitor.h index c230f346779..aa404f2c622 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.h +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.h @@ -33,7 +33,7 @@ class ExecuteScalarSubqueriesMatcher public: using Visitor = InDepthNodeVisitor; - struct Data : public WithContext + struct Data : public WithConstContext { size_t subquery_depth; Scalars & scalars; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index fc8a7ccbabd..36d369e88e1 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -474,10 +474,58 @@ bool ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions) return !aggregates().empty(); } -void makeWindowDescriptionFromAST(WindowDescription & desc, const IAST * ast) +void makeWindowDescriptionFromAST(const WindowDescriptions & existing_descriptions, + WindowDescription & desc, const IAST * ast) { const auto & definition = ast->as(); + if (!definition.parent_window_name.empty()) + { + auto it = existing_descriptions.find(definition.parent_window_name); + if (it == existing_descriptions.end()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Window definition '{}' references an unknown window '{}'", + definition.formatForErrorMessage(), + definition.parent_window_name); + } + + const auto & parent = it->second; + desc.partition_by = parent.partition_by; + desc.order_by = parent.order_by; + desc.frame = parent.frame; + + // If an existing_window_name is specified it must refer to an earlier + // entry in the WINDOW list; the new window copies its partitioning clause + // from that entry, as well as its ordering clause if any. In this case + // the new window cannot specify its own PARTITION BY clause, and it can + // specify ORDER BY only if the copied window does not have one. The new + // window always uses its own frame clause; the copied window must not + // specify a frame clause. + // -- https://www.postgresql.org/docs/current/sql-select.html + if (definition.partition_by) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Derived window definition '{}' is not allowed to override PARTITION BY", + definition.formatForErrorMessage()); + } + + if (definition.order_by && !parent.order_by.empty()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Derived window definition '{}' is not allowed to override a non-empty ORDER BY", + definition.formatForErrorMessage()); + } + + if (!parent.frame.is_default) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Parent window '{}' is not allowed to define a frame: while processing derived window definition '{}'", + definition.parent_window_name, + definition.formatForErrorMessage()); + } + } + if (definition.partition_by) { for (const auto & column_ast : definition.partition_by->children) @@ -557,7 +605,7 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) const auto & elem = ptr->as(); WindowDescription desc; desc.window_name = elem.name; - makeWindowDescriptionFromAST(desc, elem.definition.get()); + makeWindowDescriptionFromAST(window_descriptions, desc, elem.definition.get()); auto [it, inserted] = window_descriptions.insert( {desc.window_name, desc}); @@ -642,7 +690,7 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) const ASTWindowDefinition &>(); WindowDescription desc; desc.window_name = definition.getDefaultWindowName(); - makeWindowDescriptionFromAST(desc, &definition); + makeWindowDescriptionFromAST(window_descriptions, desc, &definition); auto [it, inserted] = window_descriptions.insert( {desc.window_name, desc}); diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 21326947213..b9fb1ae89d8 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -286,15 +286,13 @@ static FunctionBasePtr compile( return nullptr; } - LOG_TRACE(getLogger(), "Try to compile expression {}", dag.dump()); - auto llvm_function = std::make_shared(dag); if (auto * compilation_cache = CompiledExpressionCacheFactory::instance().tryGetCache()) { - - auto [compiled_function_cache_entry, was_inserted] = compilation_cache->getOrSet(hash_key, [&] () + auto [compiled_function_cache_entry, _] = compilation_cache->getOrSet(hash_key, [&] () { + LOG_TRACE(getLogger(), "Compile expression {}", llvm_function->getName()); CHJIT::CompiledModuleInfo compiled_module_info = compileFunction(getJITInstance(), *llvm_function); auto * compiled_jit_function = getJITInstance().findCompiledFunction(compiled_module_info, llvm_function->getName()); auto compiled_function = std::make_shared(compiled_jit_function, compiled_module_info); @@ -302,27 +300,17 @@ static FunctionBasePtr compile( return std::make_shared(std::move(compiled_function), compiled_module_info.size); }); - if (was_inserted) - LOG_TRACE(getLogger(), - "Put compiled expression {} in cache used cache size {} total cache size {}", - llvm_function->getName(), - compilation_cache->weight(), - compilation_cache->maxSize()); - else - LOG_TRACE(getLogger(), "Get compiled expression {} from cache", llvm_function->getName()); - llvm_function->setCompiledFunction(compiled_function_cache_entry->getCompiledFunction()); } else { + LOG_TRACE(getLogger(), "Compile expression {}", llvm_function->getName()); CHJIT::CompiledModuleInfo compiled_module_info = compileFunction(getJITInstance(), *llvm_function); auto * compiled_jit_function = getJITInstance().findCompiledFunction(compiled_module_info, llvm_function->getName()); auto compiled_function = std::make_shared(compiled_jit_function, compiled_module_info); llvm_function->setCompiledFunction(compiled_function); } - LOG_TRACE(getLogger(), "Use compiled expression {}", llvm_function->getName()); - return llvm_function; } diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 5c6da033396..e6414474817 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -23,9 +23,9 @@ namespace ErrorCodes } /// Must not acquire Context lock in constructor to avoid possibility of deadlocks. -ExternalDictionariesLoader::ExternalDictionariesLoader(ContextPtr global_context_) +ExternalDictionariesLoader::ExternalDictionariesLoader(ContextConstPtr global_context_) : ExternalLoader("external dictionary", &Poco::Logger::get("ExternalDictionariesLoader")) - , WithContext(global_context_) + , WithConstContext(global_context_) { setConfigSettings({"dictionary", "name", "database", "uuid"}); enableAsyncLoading(true); @@ -42,26 +42,26 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( return DictionaryFactory::instance().create(name, config, key_in_config, getContext(), created_from_ddl); } -ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(const std::string & dictionary_name, ContextPtr local_context) const +ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(const std::string & dictionary_name, ContextConstPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); return std::static_pointer_cast(load(resolved_dictionary_name)); } -ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::tryGetDictionary(const std::string & dictionary_name, ContextPtr local_context) const +ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::tryGetDictionary(const std::string & dictionary_name, ContextConstPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); return std::static_pointer_cast(tryLoad(resolved_dictionary_name)); } -void ExternalDictionariesLoader::reloadDictionary(const std::string & dictionary_name, ContextPtr local_context) const +void ExternalDictionariesLoader::reloadDictionary(const std::string & dictionary_name, ContextConstPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); loadOrReload(resolved_dictionary_name); } -DictionaryStructure ExternalDictionariesLoader::getDictionaryStructure(const std::string & dictionary_name, ContextPtr query_context) const +DictionaryStructure ExternalDictionariesLoader::getDictionaryStructure(const std::string & dictionary_name, ContextConstPtr query_context) const { std::string resolved_name = resolveDictionaryName(dictionary_name, query_context->getCurrentDatabase()); @@ -109,7 +109,10 @@ std::string ExternalDictionariesLoader::resolveDictionaryNameFromDatabaseCatalog std::string maybe_database_name = name.substr(0, pos); std::string maybe_table_name = name.substr(pos + 1); - auto [db, table] = DatabaseCatalog::instance().tryGetDatabaseAndTable({maybe_database_name, maybe_table_name}, getContext()); + auto [db, table] = DatabaseCatalog::instance().tryGetDatabaseAndTable( + {maybe_database_name, maybe_table_name}, + const_pointer_cast(getContext())); + if (!db) return name; assert(table); diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h index 06f64ef30c5..1182e96573c 100644 --- a/src/Interpreters/ExternalDictionariesLoader.h +++ b/src/Interpreters/ExternalDictionariesLoader.h @@ -13,21 +13,21 @@ namespace DB class IExternalLoaderConfigRepository; /// Manages user-defined dictionaries. -class ExternalDictionariesLoader : public ExternalLoader, WithContext +class ExternalDictionariesLoader : public ExternalLoader, WithConstContext { public: using DictPtr = std::shared_ptr; /// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. - explicit ExternalDictionariesLoader(ContextPtr global_context_); + explicit ExternalDictionariesLoader(ContextConstPtr global_context_); - DictPtr getDictionary(const std::string & dictionary_name, ContextPtr context) const; + DictPtr getDictionary(const std::string & dictionary_name, ContextConstPtr context) const; - DictPtr tryGetDictionary(const std::string & dictionary_name, ContextPtr context) const; + DictPtr tryGetDictionary(const std::string & dictionary_name, ContextConstPtr context) const; - void reloadDictionary(const std::string & dictionary_name, ContextPtr context) const; + void reloadDictionary(const std::string & dictionary_name, ContextConstPtr context) const; - DictionaryStructure getDictionaryStructure(const std::string & dictionary_name, ContextPtr context) const; + DictionaryStructure getDictionaryStructure(const std::string & dictionary_name, ContextConstPtr context) const; static DictionaryStructure getDictionaryStructure(const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config = "dictionary"); diff --git a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp index 63755ee1839..00c31110ea4 100644 --- a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp @@ -3,12 +3,13 @@ #include #include #include - #include -#include -#include +#include +#include +namespace fs = std::filesystem; + namespace DB { ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository( @@ -19,7 +20,7 @@ ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository( Poco::Timestamp ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name) { - return Poco::File(definition_entity_name).getLastModified(); + return FS::getModificationTimestamp(definition_entity_name); } std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefinitionNames() @@ -36,8 +37,8 @@ std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti if (pattern[0] != '/') { const auto app_config_path = main_config.getString("config-file", "config.xml"); - const auto config_dir = Poco::Path{app_config_path}.parent().toString(); - const auto absolute_path = config_dir + pattern; + const String config_dir = fs::path(app_config_path).parent_path(); + const String absolute_path = fs::path(config_dir) / pattern; Poco::Glob::glob(absolute_path, files, 0); if (!files.empty()) continue; @@ -59,7 +60,7 @@ std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti bool ExternalLoaderXMLConfigRepository::exists(const std::string & definition_entity_name) { - return Poco::File(definition_entity_name).exists(); + return fs::exists(fs::path(definition_entity_name)); } Poco::AutoPtr ExternalLoaderXMLConfigRepository::load( diff --git a/src/Interpreters/ExtractExpressionInfoVisitor.cpp b/src/Interpreters/ExtractExpressionInfoVisitor.cpp index 2d46fe08e95..34632a96bda 100644 --- a/src/Interpreters/ExtractExpressionInfoVisitor.cpp +++ b/src/Interpreters/ExtractExpressionInfoVisitor.cpp @@ -82,12 +82,12 @@ bool ExpressionInfoMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &) return !node->as(); } -bool hasNonRewritableFunction(const ASTPtr & node, ContextPtr context) +bool hasNonRewritableFunction(const ASTPtr & node, ContextConstPtr context) { for (const auto & select_expression : node->children) { TablesWithColumns tables; - ExpressionInfoVisitor::Data expression_info{WithContext{context}, tables}; + ExpressionInfoVisitor::Data expression_info{WithConstContext{context}, tables}; ExpressionInfoVisitor(expression_info).visit(select_expression); if (expression_info.is_stateful_function diff --git a/src/Interpreters/ExtractExpressionInfoVisitor.h b/src/Interpreters/ExtractExpressionInfoVisitor.h index c84e243ce2e..bfb414e8601 100644 --- a/src/Interpreters/ExtractExpressionInfoVisitor.h +++ b/src/Interpreters/ExtractExpressionInfoVisitor.h @@ -13,7 +13,7 @@ namespace DB struct ExpressionInfoMatcher { - struct Data : public WithContext + struct Data : public WithConstContext { const TablesWithColumns & tables; @@ -36,6 +36,6 @@ struct ExpressionInfoMatcher using ExpressionInfoVisitor = ConstInDepthNodeVisitor; -bool hasNonRewritableFunction(const ASTPtr & node, ContextPtr context); +bool hasNonRewritableFunction(const ASTPtr & node, ContextConstPtr context); } diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 51373ea49d4..09c4c538347 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -680,7 +680,19 @@ namespace class AddedColumns { public: - using TypeAndNames = std::vector>; + + struct TypeAndName + { + DataTypePtr type; + String name; + String qualified_name; + + TypeAndName(DataTypePtr type_, const String & name_, const String & qualified_name_) + : type(type_) + , name(name_) + , qualified_name(qualified_name_) + {} + }; AddedColumns(const Block & block_with_columns_to_add, const Block & block, @@ -705,27 +717,30 @@ public: for (const auto & src_column : block_with_columns_to_add) { + /// Column names `src_column.name` and `qualified_name` can differ for StorageJoin, + /// because it uses not qualified right block column names + auto qualified_name = join.getTableJoin().renamedRightColumnName(src_column.name); /// Don't insert column if it's in left block - if (!block.has(src_column.name)) - addColumn(src_column); + if (!block.has(qualified_name)) + addColumn(src_column, qualified_name); } if (is_asof_join) { const ColumnWithTypeAndName & right_asof_column = join.rightAsofKeyColumn(); - addColumn(right_asof_column); + addColumn(right_asof_column, right_asof_column.name); left_asof_key = key_columns.back(); } for (auto & tn : type_name) - right_indexes.push_back(saved_block_sample.getPositionByName(tn.second)); + right_indexes.push_back(saved_block_sample.getPositionByName(tn.name)); } size_t size() const { return columns.size(); } ColumnWithTypeAndName moveColumn(size_t i) { - return ColumnWithTypeAndName(std::move(columns[i]), type_name[i].first, type_name[i].second); + return ColumnWithTypeAndName(std::move(columns[i]), type_name[i].type, type_name[i].qualified_name); } template @@ -748,7 +763,7 @@ public: if (lazy_defaults_count) { for (size_t j = 0, size = right_indexes.size(); j < size; ++j) - JoinCommon::addDefaultValues(*columns[j], type_name[j].first, lazy_defaults_count); + JoinCommon::addDefaultValues(*columns[j], type_name[j].type, lazy_defaults_count); lazy_defaults_count = 0; } } @@ -764,7 +779,7 @@ public: bool need_filter = false; private: - TypeAndNames type_name; + std::vector type_name; MutableColumns columns; std::vector right_indexes; size_t lazy_defaults_count = 0; @@ -773,11 +788,11 @@ private: ASOF::Inequality asof_inequality; const IColumn * left_asof_key = nullptr; - void addColumn(const ColumnWithTypeAndName & src_column) + void addColumn(const ColumnWithTypeAndName & src_column, const std::string & qualified_name) { columns.push_back(src_column.column->cloneEmpty()); columns.back()->reserve(src_column.column->size()); - type_name.emplace_back(src_column.type, src_column.name); + type_name.emplace_back(src_column.type, src_column.name, qualified_name); } }; @@ -1081,7 +1096,8 @@ void HashJoin::joinBlockImpl( const auto & col = block.getByName(left_name); bool is_nullable = nullable_right_side || right_key.type->isNullable(); - block.insert(correctNullability({col.column, col.type, right_key.name}, is_nullable)); + auto right_col_name = getTableJoin().renamedRightColumnName(right_key.name); + block.insert(correctNullability({col.column, col.type, right_col_name}, is_nullable)); } } else if (has_required_right_keys) @@ -1106,7 +1122,8 @@ void HashJoin::joinBlockImpl( bool is_nullable = nullable_right_side || right_key.type->isNullable(); ColumnPtr thin_column = filterWithBlanks(col.column, filter); - block.insert(correctNullability({thin_column, col.type, right_key.name}, is_nullable, null_map_filter)); + auto right_col_name = getTableJoin().renamedRightColumnName(right_key.name); + block.insert(correctNullability({thin_column, col.type, right_col_name}, is_nullable, null_map_filter)); if constexpr (need_replication) right_keys_to_replicate.push_back(block.getPositionByName(right_key.name)); @@ -1311,7 +1328,12 @@ void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) void HashJoin::joinTotals(Block & block) const { - JoinCommon::joinTotals(totals, sample_block_with_columns_to_add, *table_join, block); + Block sample_right_block = sample_block_with_columns_to_add.cloneEmpty(); + /// For StorageJoin column names isn't qualified in sample_block_with_columns_to_add + for (auto & col : sample_right_block) + col.name = getTableJoin().renamedRightColumnName(col.name); + + JoinCommon::joinTotals(totals, sample_right_block, *table_join, block); } diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.h b/src/Interpreters/IInterpreterUnionOrSelectQuery.h index 72723d68161..0b07f27e14a 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.h +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.h @@ -32,7 +32,7 @@ public: protected: ASTPtr query_ptr; - ContextPtr context; + ContextMutablePtr context; Block result_header; SelectQueryOptions options; size_t max_streams = 1; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 6cb823aae3c..dc3fc0458b9 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -89,8 +89,8 @@ namespace ErrorCodes namespace fs = std::filesystem; -InterpreterCreateQuery::InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_) +InterpreterCreateQuery::InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) { } @@ -645,22 +645,6 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat } } - if (!create.attach && !settings.allow_experimental_bigint_types) - { - for (const auto & name_and_type_pair : properties.columns.getAllPhysical()) - { - WhichDataType which(*name_and_type_pair.type); - if (which.IsBigIntOrDeimal()) - { - const auto & type_name = name_and_type_pair.type->getName(); - String message = "Cannot create table with column '" + name_and_type_pair.name + "' which type is '" - + type_name + "' because experimental bigint types are not allowed. " - + "Set 'allow_experimental_bigint_types' setting to enable."; - throw Exception(message, ErrorCodes::ILLEGAL_COLUMN); - } - } - } - if (!create.attach && !settings.allow_experimental_map_type) { for (const auto & name_and_type_pair : properties.columns.getAllPhysical()) diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 990b87f02be..45f0bbd7cf8 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -22,10 +22,10 @@ using DatabasePtr = std::shared_ptr; /** Allows to create new table or database, * or create an object for existing table or database. */ -class InterpreterCreateQuery : public IInterpreter, WithContext +class InterpreterCreateQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateQuotaQuery.h b/src/Interpreters/InterpreterCreateQuotaQuery.h index d8edd24b2d9..7fb60cde1f6 100644 --- a/src/Interpreters/InterpreterCreateQuotaQuery.h +++ b/src/Interpreters/InterpreterCreateQuotaQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateQuotaQuery; struct Quota; -class InterpreterCreateQuotaQuery : public IInterpreter, WithContext +class InterpreterCreateQuotaQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateQuotaQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateQuotaQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateRoleQuery.h b/src/Interpreters/InterpreterCreateRoleQuery.h index 18b3f946837..ee8ff8ff7fb 100644 --- a/src/Interpreters/InterpreterCreateRoleQuery.h +++ b/src/Interpreters/InterpreterCreateRoleQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateRoleQuery; struct Role; -class InterpreterCreateRoleQuery : public IInterpreter, WithContext +class InterpreterCreateRoleQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateRoleQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateRoleQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.h b/src/Interpreters/InterpreterCreateRowPolicyQuery.h index 10167bac669..8adfe6b0855 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.h +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateRowPolicyQuery; struct RowPolicy; -class InterpreterCreateRowPolicyQuery : public IInterpreter, WithContext +class InterpreterCreateRowPolicyQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateRowPolicyQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateRowPolicyQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateSettingsProfileQuery.h b/src/Interpreters/InterpreterCreateSettingsProfileQuery.h index 9ef1f0354a9..aa9264fbf22 100644 --- a/src/Interpreters/InterpreterCreateSettingsProfileQuery.h +++ b/src/Interpreters/InterpreterCreateSettingsProfileQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateSettingsProfileQuery; struct SettingsProfile; -class InterpreterCreateSettingsProfileQuery : public IInterpreter, WithContext +class InterpreterCreateSettingsProfileQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateSettingsProfileQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateSettingsProfileQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateUserQuery.h b/src/Interpreters/InterpreterCreateUserQuery.h index e9f4e82e767..7d357924d35 100644 --- a/src/Interpreters/InterpreterCreateUserQuery.h +++ b/src/Interpreters/InterpreterCreateUserQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateUserQuery; struct User; -class InterpreterCreateUserQuery : public IInterpreter, WithContext +class InterpreterCreateUserQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateUserQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateUserQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.h b/src/Interpreters/InterpreterDropAccessEntityQuery.h index 7f0f6348610..0ee478e904e 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.h +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.h @@ -9,10 +9,10 @@ namespace DB class AccessRightsElements; -class InterpreterDropAccessEntityQuery : public IInterpreter, WithContext +class InterpreterDropAccessEntityQuery : public IInterpreter, WithMutableContext { public: - InterpreterDropAccessEntityQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterDropAccessEntityQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 24cff90caea..3c515d58e0c 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -1,5 +1,3 @@ -#include - #include #include #include @@ -42,7 +40,7 @@ static DatabasePtr tryGetDatabase(const String & database_name, bool if_exists) } -InterpreterDropQuery::InterpreterDropQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) +InterpreterDropQuery::InterpreterDropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) { } diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 8e8d577deec..16a08e95a3b 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -16,10 +16,10 @@ class AccessRightsElements; * or remove information about table (just forget) from server (DETACH), * or just clear all data in table (TRUNCATE). */ -class InterpreterDropQuery : public IInterpreter, WithContext +class InterpreterDropQuery : public IInterpreter, WithMutableContext { public: - InterpreterDropQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterDropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); /// Drop table or database. BlockIO execute() override; diff --git a/src/Interpreters/InterpreterExternalDDLQuery.cpp b/src/Interpreters/InterpreterExternalDDLQuery.cpp index 8f9f0cf9ddb..e91b95a988d 100644 --- a/src/Interpreters/InterpreterExternalDDLQuery.cpp +++ b/src/Interpreters/InterpreterExternalDDLQuery.cpp @@ -26,8 +26,8 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -InterpreterExternalDDLQuery::InterpreterExternalDDLQuery(const ASTPtr & query_, ContextPtr context_) - : WithContext(context_), query(query_) +InterpreterExternalDDLQuery::InterpreterExternalDDLQuery(const ASTPtr & query_, ContextMutablePtr context_) + : WithMutableContext(context_), query(query_) { } diff --git a/src/Interpreters/InterpreterExternalDDLQuery.h b/src/Interpreters/InterpreterExternalDDLQuery.h index 15a842a2611..d084e68d7c8 100644 --- a/src/Interpreters/InterpreterExternalDDLQuery.h +++ b/src/Interpreters/InterpreterExternalDDLQuery.h @@ -6,10 +6,10 @@ namespace DB { -class InterpreterExternalDDLQuery : public IInterpreter, WithContext +class InterpreterExternalDDLQuery : public IInterpreter, WithMutableContext { public: - InterpreterExternalDDLQuery(const ASTPtr & query_, ContextPtr context_); + InterpreterExternalDDLQuery(const ASTPtr & query_, ContextMutablePtr context_); BlockIO execute() override; diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 4af8b6ffa7d..79cda364c42 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -92,7 +92,7 @@ namespace ErrorCodes } -std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextPtr context, const SelectQueryOptions & options) +std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMutablePtr context, const SelectQueryOptions & options) { OpenTelemetrySpanHolder span("InterpreterFactory::get()"); diff --git a/src/Interpreters/InterpreterFactory.h b/src/Interpreters/InterpreterFactory.h index c122fe11b7d..774cbd1cb0f 100644 --- a/src/Interpreters/InterpreterFactory.h +++ b/src/Interpreters/InterpreterFactory.h @@ -16,7 +16,7 @@ class InterpreterFactory public: static std::unique_ptr get( ASTPtr & query, - ContextPtr context, + ContextMutablePtr context, const SelectQueryOptions & options = {}); }; diff --git a/src/Interpreters/InterpreterGrantQuery.h b/src/Interpreters/InterpreterGrantQuery.h index f5939ff3cb7..abaddcc599b 100644 --- a/src/Interpreters/InterpreterGrantQuery.h +++ b/src/Interpreters/InterpreterGrantQuery.h @@ -12,10 +12,10 @@ class ASTGrantQuery; struct User; struct Role; -class InterpreterGrantQuery : public IInterpreter, WithContext +class InterpreterGrantQuery : public IInterpreter, WithMutableContext { public: - InterpreterGrantQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterGrantQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterKillQueryQuery.h b/src/Interpreters/InterpreterKillQueryQuery.h index 5ffd9a525a2..9284d777ca7 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.h +++ b/src/Interpreters/InterpreterKillQueryQuery.h @@ -10,10 +10,10 @@ namespace DB class AccessRightsElements; -class InterpreterKillQueryQuery final : public IInterpreter, WithContext +class InterpreterKillQueryQuery final : public IInterpreter, WithMutableContext { public: - InterpreterKillQueryQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) { } + InterpreterKillQueryQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) { } BlockIO execute() override; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c230ec4aafc..014f3158ce1 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1296,9 +1296,12 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu */ if (from_aggregation_stage) - executeMergeSorted(query_plan, "for ORDER BY"); - else if (!expressions.first_stage && !expressions.need_aggregate && !(query.group_by_with_totals && !aggregate_final)) - executeMergeSorted(query_plan, "for ORDER BY"); + executeMergeSorted(query_plan, "after aggregation stage for ORDER BY"); + else if (!expressions.first_stage + && !expressions.need_aggregate + && !expressions.has_window + && !(query.group_by_with_totals && !aggregate_final)) + executeMergeSorted(query_plan, "for ORDER BY, without aggregation"); else /// Otherwise, just sort. executeOrder( query_plan, diff --git a/src/Interpreters/InterpreterSetQuery.h b/src/Interpreters/InterpreterSetQuery.h index 31519be6f29..9bd49708421 100644 --- a/src/Interpreters/InterpreterSetQuery.h +++ b/src/Interpreters/InterpreterSetQuery.h @@ -11,10 +11,10 @@ class ASTSetQuery; /** Change one or several settings for the session or just for the current context. */ -class InterpreterSetQuery : public IInterpreter, WithContext +class InterpreterSetQuery : public IInterpreter, WithMutableContext { public: - InterpreterSetQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterSetQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} /** Usual SET query. Set setting for the session. */ diff --git a/src/Interpreters/InterpreterSetRoleQuery.h b/src/Interpreters/InterpreterSetRoleQuery.h index 70ba3c381ab..0a489f45fcd 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.h +++ b/src/Interpreters/InterpreterSetRoleQuery.h @@ -11,10 +11,10 @@ class ASTSetRoleQuery; struct RolesOrUsersSet; struct User; -class InterpreterSetRoleQuery : public IInterpreter, WithContext +class InterpreterSetRoleQuery : public IInterpreter, WithMutableContext { public: - InterpreterSetRoleQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterSetRoleQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp index c2c2305f976..41b986e43a2 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -17,8 +17,8 @@ namespace ErrorCodes using EntityType = IAccessEntity::Type; -InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_) +InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) { } diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h index 7224f0d593b..35511a38d8f 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h @@ -7,10 +7,10 @@ namespace DB { -class InterpreterShowAccessEntitiesQuery : public IInterpreter, WithContext +class InterpreterShowAccessEntitiesQuery : public IInterpreter, WithMutableContext { public: - InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; diff --git a/src/Interpreters/InterpreterShowPrivilegesQuery.cpp b/src/Interpreters/InterpreterShowPrivilegesQuery.cpp index c566d31e2fc..201c1cfece8 100644 --- a/src/Interpreters/InterpreterShowPrivilegesQuery.cpp +++ b/src/Interpreters/InterpreterShowPrivilegesQuery.cpp @@ -4,7 +4,7 @@ namespace DB { -InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, ContextPtr context_) +InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : query_ptr(query_ptr_), context(context_) { } diff --git a/src/Interpreters/InterpreterShowPrivilegesQuery.h b/src/Interpreters/InterpreterShowPrivilegesQuery.h index 75989263405..852d5173eb1 100644 --- a/src/Interpreters/InterpreterShowPrivilegesQuery.h +++ b/src/Interpreters/InterpreterShowPrivilegesQuery.h @@ -11,7 +11,7 @@ class Context; class InterpreterShowPrivilegesQuery : public IInterpreter { public: - InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; @@ -20,7 +20,7 @@ public: private: ASTPtr query_ptr; - ContextPtr context; + ContextMutablePtr context; }; } diff --git a/src/Interpreters/InterpreterShowProcesslistQuery.h b/src/Interpreters/InterpreterShowProcesslistQuery.h index 5eedb67595e..31454882a89 100644 --- a/src/Interpreters/InterpreterShowProcesslistQuery.h +++ b/src/Interpreters/InterpreterShowProcesslistQuery.h @@ -9,11 +9,11 @@ namespace DB /** Return list of currently executing queries. */ -class InterpreterShowProcesslistQuery : public IInterpreter, WithContext +class InterpreterShowProcesslistQuery : public IInterpreter, WithMutableContext { public: - InterpreterShowProcesslistQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterShowProcesslistQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 901999f004f..609df1404ca 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -18,8 +18,8 @@ namespace ErrorCodes } -InterpreterShowTablesQuery::InterpreterShowTablesQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_) +InterpreterShowTablesQuery::InterpreterShowTablesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) { } diff --git a/src/Interpreters/InterpreterShowTablesQuery.h b/src/Interpreters/InterpreterShowTablesQuery.h index b61be568e35..16fc9ef2cf4 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.h +++ b/src/Interpreters/InterpreterShowTablesQuery.h @@ -13,10 +13,10 @@ class Context; /** Return a list of tables or databases meets specified conditions. * Interprets a query through replacing it to SELECT query from system.tables or system.databases. */ -class InterpreterShowTablesQuery : public IInterpreter, WithContext +class InterpreterShowTablesQuery : public IInterpreter, WithMutableContext { public: - InterpreterShowTablesQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterShowTablesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index e7babb9b83f..f97001883bd 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -192,8 +192,8 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type, } -InterpreterSystemQuery::InterpreterSystemQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_->clone()), log(&Poco::Logger::get("InterpreterSystemQuery")) +InterpreterSystemQuery::InterpreterSystemQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_->clone()), log(&Poco::Logger::get("InterpreterSystemQuery")) { } @@ -424,7 +424,7 @@ BlockIO InterpreterSystemQuery::execute() } -StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextPtr system_context, bool need_ddl_guard) +StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context, bool need_ddl_guard) { getContext()->checkAccess(AccessType::SYSTEM_RESTART_REPLICA, replica); @@ -469,7 +469,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, return table; } -void InterpreterSystemQuery::restartReplicas(ContextPtr system_context) +void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context) { std::vector replica_names; auto & catalog = DatabaseCatalog::instance(); @@ -542,7 +542,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) else if (!query.replica_zk_path.empty()) { getContext()->checkAccess(AccessType::SYSTEM_DROP_REPLICA); - auto remote_replica_path = query.replica_zk_path + "/replicas/" + query.replica; + String remote_replica_path = fs::path(query.replica_zk_path) / "replicas" / query.replica; /// This check is actually redundant, but it may prevent from some user mistakes for (auto & elem : DatabaseCatalog::instance().getDatabases()) diff --git a/src/Interpreters/InterpreterSystemQuery.h b/src/Interpreters/InterpreterSystemQuery.h index 341611e0af1..297f7225a92 100644 --- a/src/Interpreters/InterpreterSystemQuery.h +++ b/src/Interpreters/InterpreterSystemQuery.h @@ -30,10 +30,10 @@ class ASTSystemQuery; * - start/stop actions for all existing tables. * Note that the actions for tables that will be created after this query will not be affected. */ -class InterpreterSystemQuery : public IInterpreter, WithContext +class InterpreterSystemQuery : public IInterpreter, WithMutableContext { public: - InterpreterSystemQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterSystemQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; @@ -45,9 +45,9 @@ private: /// Tries to get a replicated table and restart it /// Returns pointer to a newly created table if the restart was successful - StoragePtr tryRestartReplica(const StorageID & replica, ContextPtr context, bool need_ddl_guard = true); + StoragePtr tryRestartReplica(const StorageID & replica, ContextMutablePtr context, bool need_ddl_guard = true); - void restartReplicas(ContextPtr system_context); + void restartReplicas(ContextMutablePtr system_context); void syncReplica(ASTSystemQuery & query); void dropReplica(ASTSystemQuery & query); bool dropReplicaImpl(ASTSystemQuery & query, const StoragePtr & table); diff --git a/src/Interpreters/MonotonicityCheckVisitor.h b/src/Interpreters/MonotonicityCheckVisitor.h index 350318047c7..f0698b27426 100644 --- a/src/Interpreters/MonotonicityCheckVisitor.h +++ b/src/Interpreters/MonotonicityCheckVisitor.h @@ -26,7 +26,7 @@ public: struct Data { const TablesWithColumns & tables; - ContextPtr context; + ContextConstPtr context; const std::unordered_set & group_by_function_hashes; Monotonicity monotonicity{true, true, true}; ASTIdentifier * identifier = nullptr; diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 9210cfd48d3..a14b4c5f854 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -179,7 +179,7 @@ bool isStorageTouchedByMutations( const StoragePtr & storage, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, - ContextPtr context_copy) + ContextMutablePtr context_copy) { if (commands.empty()) return false; diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 7e2f910466b..0d91da5613c 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -23,7 +23,7 @@ bool isStorageTouchedByMutations( const StoragePtr & storage, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, - ContextPtr context_copy + ContextMutablePtr context_copy ); ASTPtr getPartitionAndPredicateExpressionForMutationCommand( diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h index 3202612ac94..1efbe8ee744 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h @@ -55,12 +55,12 @@ namespace MySQLInterpreter }; template -class InterpreterMySQLDDLQuery : public IInterpreter, WithContext +class InterpreterMySQLDDLQuery : public IInterpreter, WithMutableContext { public: InterpreterMySQLDDLQuery( - const ASTPtr & query_ptr_, ContextPtr context_, const String & mapped_to_database_, const String & mysql_database_) - : WithContext(context_), query_ptr(query_ptr_), mapped_to_database(mapped_to_database_), mysql_database(mysql_database_) + const ASTPtr & query_ptr_, ContextMutablePtr context_, const String & mapped_to_database_, const String & mysql_database_) + : WithMutableContext(context_), query_ptr(query_ptr_), mapped_to_database(mapped_to_database_), mysql_database(mysql_database_) { } diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index f2e55441fb6..b3ef2e42e4a 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -19,8 +19,8 @@ namespace ErrorCodes } PredicateExpressionsOptimizer::PredicateExpressionsOptimizer( - ContextPtr context_, const TablesWithColumns & tables_with_columns_, const Settings & settings) - : WithContext(context_) + ContextConstPtr context_, const TablesWithColumns & tables_with_columns_, const Settings & settings) + : WithConstContext(context_) , enable_optimize_predicate_expression(settings.enable_optimize_predicate_expression) , enable_optimize_predicate_expression_to_final_subquery(settings.enable_optimize_predicate_expression_to_final_subquery) , allow_push_predicate_when_subquery_contains_with(settings.allow_push_predicate_when_subquery_contains_with) @@ -87,7 +87,7 @@ std::vector PredicateExpressionsOptimizer::extractTablesPredicates(const A for (const auto & predicate_expression : splitConjunctionPredicate({where, prewhere})) { - ExpressionInfoVisitor::Data expression_info{WithContext{getContext()}, tables_with_columns}; + ExpressionInfoVisitor::Data expression_info{WithConstContext{getContext()}, tables_with_columns}; ExpressionInfoVisitor(expression_info).visit(predicate_expression); if (expression_info.is_stateful_function @@ -188,7 +188,7 @@ bool PredicateExpressionsOptimizer::tryMovePredicatesFromHavingToWhere(ASTSelect for (const auto & moving_predicate: splitConjunctionPredicate({select_query.having()})) { TablesWithColumns tables; - ExpressionInfoVisitor::Data expression_info{WithContext{getContext()}, tables}; + ExpressionInfoVisitor::Data expression_info{WithConstContext{getContext()}, tables}; ExpressionInfoVisitor(expression_info).visit(moving_predicate); /// TODO: If there is no group by, where, and prewhere expression, we can push down the stateful function diff --git a/src/Interpreters/PredicateExpressionsOptimizer.h b/src/Interpreters/PredicateExpressionsOptimizer.h index a31b9907da6..db580032f73 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.h +++ b/src/Interpreters/PredicateExpressionsOptimizer.h @@ -15,10 +15,10 @@ struct Settings; * - Move predicates from having to where * - Push the predicate down from the current query to the having of the subquery */ -class PredicateExpressionsOptimizer : WithContext +class PredicateExpressionsOptimizer : WithConstContext { public: - PredicateExpressionsOptimizer(ContextPtr context_, const TablesWithColumns & tables_with_columns_, const Settings & settings_); + PredicateExpressionsOptimizer(ContextConstPtr context_, const TablesWithColumns & tables_with_columns_, const Settings & settings_); bool optimize(ASTSelectQuery & select_query); diff --git a/src/Interpreters/PredicateRewriteVisitor.cpp b/src/Interpreters/PredicateRewriteVisitor.cpp index 092d37d78dd..3dda066c4ac 100644 --- a/src/Interpreters/PredicateRewriteVisitor.cpp +++ b/src/Interpreters/PredicateRewriteVisitor.cpp @@ -17,12 +17,12 @@ namespace DB { PredicateRewriteVisitorData::PredicateRewriteVisitorData( - ContextPtr context_, + ContextConstPtr context_, const ASTs & predicates_, const TableWithColumnNamesAndTypes & table_columns_, bool optimize_final_, bool optimize_with_) - : WithContext(context_) + : WithConstContext(context_) , predicates(predicates_) , table_columns(table_columns_) , optimize_final(optimize_final_) @@ -72,7 +72,9 @@ void PredicateRewriteVisitorData::visitOtherInternalSelect(ASTSelectQuery & sele } const Names & internal_columns = InterpreterSelectQuery( - temp_internal_select, getContext(), SelectQueryOptions().analyze()).getSampleBlock().getNames(); + temp_internal_select, + const_pointer_cast(getContext()), + SelectQueryOptions().analyze()).getSampleBlock().getNames(); if (rewriteSubquery(*temp_select_query, internal_columns)) { diff --git a/src/Interpreters/PredicateRewriteVisitor.h b/src/Interpreters/PredicateRewriteVisitor.h index fc076464925..b4ff063c954 100644 --- a/src/Interpreters/PredicateRewriteVisitor.h +++ b/src/Interpreters/PredicateRewriteVisitor.h @@ -10,7 +10,7 @@ namespace DB { -class PredicateRewriteVisitorData : WithContext +class PredicateRewriteVisitorData : WithConstContext { public: bool is_rewrite = false; @@ -24,7 +24,7 @@ public: } PredicateRewriteVisitorData( - ContextPtr context_, + ContextConstPtr context_, const ASTs & predicates_, const TableWithColumnNamesAndTypes & table_columns_, bool optimize_final_, diff --git a/src/Interpreters/RedundantFunctionsInOrderByVisitor.h b/src/Interpreters/RedundantFunctionsInOrderByVisitor.h index f807849fb86..04ac1607b60 100644 --- a/src/Interpreters/RedundantFunctionsInOrderByVisitor.h +++ b/src/Interpreters/RedundantFunctionsInOrderByVisitor.h @@ -16,7 +16,7 @@ public: struct Data { std::unordered_set & keys; - ContextPtr context; + ContextConstPtr context; bool redundant = true; bool done = false; diff --git a/src/Interpreters/RemoveInjectiveFunctionsVisitor.cpp b/src/Interpreters/RemoveInjectiveFunctionsVisitor.cpp index 8d030379909..e242eab919a 100644 --- a/src/Interpreters/RemoveInjectiveFunctionsVisitor.cpp +++ b/src/Interpreters/RemoveInjectiveFunctionsVisitor.cpp @@ -17,7 +17,7 @@ static bool isUniq(const ASTFunction & func) } /// Remove injective functions of one argument: replace with a child -static bool removeInjectiveFunction(ASTPtr & ast, ContextPtr context, const FunctionFactory & function_factory) +static bool removeInjectiveFunction(ASTPtr & ast, ContextConstPtr context, const FunctionFactory & function_factory) { const ASTFunction * func = ast->as(); if (!func) diff --git a/src/Interpreters/RemoveInjectiveFunctionsVisitor.h b/src/Interpreters/RemoveInjectiveFunctionsVisitor.h index a3bbd562407..29fd9bb0af5 100644 --- a/src/Interpreters/RemoveInjectiveFunctionsVisitor.h +++ b/src/Interpreters/RemoveInjectiveFunctionsVisitor.h @@ -13,9 +13,9 @@ class ASTFunction; class RemoveInjectiveFunctionsMatcher { public: - struct Data : public WithContext + struct Data : public WithConstContext { - explicit Data(ContextPtr context_) : WithContext(context_) {} + explicit Data(ContextConstPtr context_) : WithConstContext(context_) {} }; static void visit(ASTPtr & ast, const Data & data); diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index f547e011a73..122e2cd6479 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -156,9 +156,12 @@ NameSet TableJoin::requiredRightKeys() const { NameSet required; for (const auto & name : key_names_right) + { + auto rename = renamedRightColumnName(name); for (const auto & column : columns_added_by_join) - if (name == column.name) + if (rename == column.name) required.insert(name); + } return required; } @@ -464,4 +467,11 @@ ActionsDAGPtr TableJoin::applyKeyConvertToTable( return dag; } +String TableJoin::renamedRightColumnName(const String & name) const +{ + if (const auto it = renames.find(name); it != renames.end()) + return it->second; + return name; +} + } diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index b75ef848f13..dc8ebeb7413 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -203,6 +203,8 @@ public: /// Split key and other columns by keys name list void splitAdditionalColumns(const Block & sample_block, Block & block_keys, Block & block_others) const; Block getRequiredRightKeys(const Block & right_table_keys, std::vector & keys_sources) const; + + String renamedRightColumnName(const String & name) const; }; } diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index cff6fb9cbcd..8590b3c94f3 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -508,7 +508,7 @@ void CurrentThread::detachQueryIfNotDetached() } -CurrentThread::QueryScope::QueryScope(ContextPtr query_context) +CurrentThread::QueryScope::QueryScope(ContextMutablePtr query_context) { CurrentThread::initializeQuery(); CurrentThread::attachQueryContext(query_context); diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 5b06c00435a..a2725f2506e 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -81,7 +81,7 @@ void appendUnusedGroupByColumn(ASTSelectQuery * select_query, const NameSet & so } /// Eliminates injective function calls and constant expressions from group by statement. -void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_columns, ContextPtr context) +void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_columns, ContextConstPtr context) { const FunctionFactory & function_factory = FunctionFactory::instance(); @@ -270,7 +270,7 @@ void optimizeDuplicatesInOrderBy(const ASTSelectQuery * select_query) } /// Optimize duplicate ORDER BY -void optimizeDuplicateOrderBy(ASTPtr & query, ContextPtr context) +void optimizeDuplicateOrderBy(ASTPtr & query, ContextConstPtr context) { DuplicateOrderByVisitor::Data order_by_data{context}; DuplicateOrderByVisitor(order_by_data).visit(query); @@ -396,7 +396,7 @@ void optimizeDuplicateDistinct(ASTSelectQuery & select) /// Replace monotonous functions in ORDER BY if they don't participate in GROUP BY expression, /// has a single argument and not an aggregate functions. -void optimizeMonotonousFunctionsInOrderBy(ASTSelectQuery * select_query, ContextPtr context, +void optimizeMonotonousFunctionsInOrderBy(ASTSelectQuery * select_query, ContextConstPtr context, const TablesWithColumns & tables_with_columns, const Names & sorting_key_columns) { @@ -448,7 +448,7 @@ void optimizeMonotonousFunctionsInOrderBy(ASTSelectQuery * select_query, Context /// Optimize ORDER BY x, y, f(x), g(x, y), f(h(x)), t(f(x), g(x)) into ORDER BY x, y /// in case if f(), g(), h(), t() are deterministic (in scope of query). /// Don't optimize ORDER BY f(x), g(x), x even if f(x) is bijection for x or g(x). -void optimizeRedundantFunctionsInOrderBy(const ASTSelectQuery * select_query, ContextPtr context) +void optimizeRedundantFunctionsInOrderBy(const ASTSelectQuery * select_query, ContextConstPtr context) { const auto & order_by = select_query->orderBy(); if (!order_by) @@ -561,7 +561,7 @@ void optimizeCountConstantAndSumOne(ASTPtr & query) } -void optimizeInjectiveFunctionsInsideUniq(ASTPtr & query, ContextPtr context) +void optimizeInjectiveFunctionsInsideUniq(ASTPtr & query, ContextConstPtr context) { RemoveInjectiveFunctionsVisitor::Data data(context); RemoveInjectiveFunctionsVisitor(data).visit(query); @@ -592,7 +592,7 @@ void TreeOptimizer::optimizeIf(ASTPtr & query, Aliases & aliases, bool if_chain_ void TreeOptimizer::apply(ASTPtr & query, Aliases & aliases, const NameSet & source_columns_set, const std::vector & tables_with_columns, - ContextPtr context, const StorageMetadataPtr & metadata_snapshot, + ContextConstPtr context, const StorageMetadataPtr & metadata_snapshot, bool & rewrite_subqueries) { const auto & settings = context->getSettingsRef(); diff --git a/src/Interpreters/TreeOptimizer.h b/src/Interpreters/TreeOptimizer.h index b268b230f4e..706f030e620 100644 --- a/src/Interpreters/TreeOptimizer.h +++ b/src/Interpreters/TreeOptimizer.h @@ -21,7 +21,7 @@ public: Aliases & aliases, const NameSet & source_columns_set, const std::vector & tables_with_columns, - ContextPtr context, + ContextConstPtr context, const StorageMetadataPtr & metadata_snapshot, bool & rewrite_subqueries); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 92cfba1bcb1..5b4a869d44b 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -413,10 +413,10 @@ void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query, } /// Replacing scalar subqueries with constant values. -void executeScalarSubqueries(ASTPtr & query, ContextPtr context, size_t subquery_depth, Scalars & scalars, bool only_analyze) +void executeScalarSubqueries(ASTPtr & query, ContextConstPtr context, size_t subquery_depth, Scalars & scalars, bool only_analyze) { LogAST log; - ExecuteScalarSubqueriesVisitor::Data visitor_data{WithContext{context}, subquery_depth, scalars, only_analyze}; + ExecuteScalarSubqueriesVisitor::Data visitor_data{WithConstContext{context}, subquery_depth, scalars, only_analyze}; ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(query); } diff --git a/src/Interpreters/TreeRewriter.h b/src/Interpreters/TreeRewriter.h index 32826bcc61d..1f535325666 100644 --- a/src/Interpreters/TreeRewriter.h +++ b/src/Interpreters/TreeRewriter.h @@ -92,10 +92,10 @@ using TreeRewriterResultPtr = std::shared_ptr; /// * scalar subqueries are executed replaced with constants /// * unneeded columns are removed from SELECT clause /// * duplicated columns are removed from ORDER BY, LIMIT BY, USING(...). -class TreeRewriter : WithContext +class TreeRewriter : WithConstContext { public: - explicit TreeRewriter(ContextPtr context_) : WithContext(context_) {} + explicit TreeRewriter(ContextConstPtr context_) : WithConstContext(context_) {} /// Analyze and rewrite not select query TreeRewriterResultPtr analyze( diff --git a/src/Interpreters/addTypeConversionToAST.cpp b/src/Interpreters/addTypeConversionToAST.cpp index 73c95bd9a8c..86fd7926e78 100644 --- a/src/Interpreters/addTypeConversionToAST.cpp +++ b/src/Interpreters/addTypeConversionToAST.cpp @@ -32,10 +32,12 @@ ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name) return func; } -ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name, const NamesAndTypesList & all_columns, ContextPtr context) +ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name, const NamesAndTypesList & all_columns, ContextConstPtr context) { auto syntax_analyzer_result = TreeRewriter(context).analyze(ast, all_columns); - const auto actions = ExpressionAnalyzer(ast, syntax_analyzer_result, context).getActions(true); + const auto actions = ExpressionAnalyzer(ast, + syntax_analyzer_result, + const_pointer_cast(context)).getActions(true); for (const auto & action : actions->getActions()) if (action.node->type == ActionsDAG::ActionType::ARRAY_JOIN) diff --git a/src/Interpreters/addTypeConversionToAST.h b/src/Interpreters/addTypeConversionToAST.h index eb391b2c749..7a4d879dc61 100644 --- a/src/Interpreters/addTypeConversionToAST.h +++ b/src/Interpreters/addTypeConversionToAST.h @@ -14,6 +14,6 @@ class NamesAndTypesList; ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name); // If same type, then ignore the wrapper of CAST function -ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name, const NamesAndTypesList & all_columns, ContextPtr context); +ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name, const NamesAndTypesList & all_columns, ContextConstPtr context); } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 542ed7ca0f9..a78c810d5d4 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -335,7 +335,7 @@ static void onExceptionBeforeStart(const String & query_for_logging, ContextPtr } } -static void setQuerySpecificSettings(ASTPtr & ast, ContextPtr context) +static void setQuerySpecificSettings(ASTPtr & ast, ContextMutablePtr context) { if (auto * ast_insert_into = dynamic_cast(ast.get())) { @@ -347,7 +347,7 @@ static void setQuerySpecificSettings(ASTPtr & ast, ContextPtr context) static std::tuple executeQueryImpl( const char * begin, const char * end, - ContextPtr context, + ContextMutablePtr context, bool internal, QueryProcessingStage::Enum stage, bool has_query_tail, @@ -910,7 +910,7 @@ static std::tuple executeQueryImpl( BlockIO executeQuery( const String & query, - ContextPtr context, + ContextMutablePtr context, bool internal, QueryProcessingStage::Enum stage, bool may_have_embedded_data) @@ -935,7 +935,7 @@ BlockIO executeQuery( BlockIO executeQuery( const String & query, - ContextPtr context, + ContextMutablePtr context, bool internal, QueryProcessingStage::Enum stage, bool may_have_embedded_data, @@ -954,7 +954,7 @@ void executeQuery( ReadBuffer & istr, WriteBuffer & ostr, bool allow_into_outfile, - ContextPtr context, + ContextMutablePtr context, std::function set_result_details) { PODArray parse_buf; diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index bdb1f877ce3..6448b26a652 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -16,7 +16,7 @@ void executeQuery( ReadBuffer & istr, /// Where to read query from (and data for INSERT, if present). WriteBuffer & ostr, /// Where to write query output to. bool allow_into_outfile, /// If true and the query contains INTO OUTFILE section, redirect output to that file. - ContextPtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... + ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... std::function set_result_details /// If a non-empty callback is passed, it will be called with the query id, the content-type, the format, and the timezone. ); @@ -37,7 +37,7 @@ void executeQuery( /// must be done separately. BlockIO executeQuery( const String & query, /// Query text without INSERT data. The latter must be written to BlockIO::out. - ContextPtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... + ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... bool internal = false, /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, /// To which stage the query must be executed. bool may_have_embedded_data = false /// If insert query may have embedded data @@ -46,7 +46,7 @@ BlockIO executeQuery( /// Old interface with allow_processors flag. For compatibility. BlockIO executeQuery( const String & query, - ContextPtr context, + ContextMutablePtr context, bool internal, QueryProcessingStage::Enum stage, bool may_have_embedded_data, diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 79076e57328..43f9727c355 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -18,14 +18,16 @@ #include #include +#include +namespace fs = std::filesystem; namespace DB { static void executeCreateQuery( const String & query, - ContextPtr context, + ContextMutablePtr context, const String & database, const String & file_name, bool has_force_restore_data_flag) @@ -46,7 +48,7 @@ static void executeCreateQuery( static void loadDatabase( - ContextPtr context, + ContextMutablePtr context, const String & database, const String & database_path, bool force_restore_data) @@ -54,13 +56,13 @@ static void loadDatabase( String database_attach_query; String database_metadata_file = database_path + ".sql"; - if (Poco::File(database_metadata_file).exists()) + if (fs::exists(fs::path(database_metadata_file))) { /// There is .sql file with database creation statement. ReadBufferFromFile in(database_metadata_file, 1024); readStringUntilEOF(database_attach_query, in); } - else if (Poco::File(database_path).exists()) + else if (fs::exists(fs::path(database_path))) { /// Database exists, but .sql file is absent. It's old-style Ordinary database (e.g. system or default) database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database) + " ENGINE = Ordinary"; @@ -84,7 +86,7 @@ static void loadDatabase( } -void loadMetadata(ContextPtr context, const String & default_database_name) +void loadMetadata(ContextMutablePtr context, const String & default_database_name) { Poco::Logger * log = &Poco::Logger::get("loadMetadata"); @@ -95,34 +97,35 @@ void loadMetadata(ContextPtr context, const String & default_database_name) * This file is deleted after successful loading of tables. * (flag is "one-shot") */ - Poco::File force_restore_data_flag_file(context->getFlagsPath() + "force_restore_data"); - bool has_force_restore_data_flag = force_restore_data_flag_file.exists(); + auto force_restore_data_flag_file = fs::path(context->getFlagsPath()) / "force_restore_data"; + bool has_force_restore_data_flag = fs::exists(force_restore_data_flag_file); /// Loop over databases. std::map databases; - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(path); it != dir_end; ++it) + fs::directory_iterator dir_end; + for (fs::directory_iterator it(path); it != dir_end; ++it) { - if (it->isLink()) + if (it->is_symlink()) continue; - if (!it->isDirectory()) + const auto current_file = it->path().filename().string(); + if (!it->is_directory()) { /// TODO: DETACH DATABASE PERMANENTLY ? - if (endsWith(it.name(), ".sql")) + if (fs::path(current_file).extension() == ".sql") { - String db_name = it.name().substr(0, it.name().size() - 4); + String db_name = fs::path(current_file).stem(); if (db_name != DatabaseCatalog::SYSTEM_DATABASE) - databases.emplace(unescapeForFileName(db_name), path + "/" + db_name); + databases.emplace(unescapeForFileName(db_name), fs::path(path) / db_name); } /// Temporary fails may be left from previous server runs. - if (endsWith(it.name(), ".tmp")) + if (fs::path(current_file).extension() == ".tmp") { - LOG_WARNING(log, "Removing temporary file {}", it->path()); + LOG_WARNING(log, "Removing temporary file {}", it->path().string()); try { - it->remove(); + fs::remove(it->path()); } catch (...) { @@ -135,13 +138,13 @@ void loadMetadata(ContextPtr context, const String & default_database_name) } /// For '.svn', '.gitignore' directory and similar. - if (it.name().at(0) == '.') + if (current_file.at(0) == '.') continue; - if (it.name() == DatabaseCatalog::SYSTEM_DATABASE) + if (current_file == DatabaseCatalog::SYSTEM_DATABASE) continue; - databases.emplace(unescapeForFileName(it.name()), it.path().toString()); + databases.emplace(unescapeForFileName(current_file), it->path().string()); } /// clickhouse-local creates DatabaseMemory as default database by itself @@ -158,7 +161,7 @@ void loadMetadata(ContextPtr context, const String & default_database_name) { try { - force_restore_data_flag_file.remove(); + fs::remove(force_restore_data_flag_file); } catch (...) { @@ -168,11 +171,11 @@ void loadMetadata(ContextPtr context, const String & default_database_name) } -void loadMetadataSystem(ContextPtr context) +void loadMetadataSystem(ContextMutablePtr context) { String path = context->getPath() + "metadata/" + DatabaseCatalog::SYSTEM_DATABASE; String metadata_file = path + ".sql"; - if (Poco::File(path).exists() || Poco::File(metadata_file).exists()) + if (fs::exists(fs::path(path)) || fs::exists(fs::path(metadata_file))) { /// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted. loadDatabase(context, DatabaseCatalog::SYSTEM_DATABASE, path, true); diff --git a/src/Interpreters/loadMetadata.h b/src/Interpreters/loadMetadata.h index 047def84bba..cf038a42855 100644 --- a/src/Interpreters/loadMetadata.h +++ b/src/Interpreters/loadMetadata.h @@ -8,9 +8,9 @@ namespace DB /// Load tables from system database. Only real tables like query_log, part_log. /// You should first load system database, then attach system tables that you need into it, then load other databases. -void loadMetadataSystem(ContextPtr context); +void loadMetadataSystem(ContextMutablePtr context); /// Load tables from databases and add them to context. Database 'system' is ignored. Use separate function to load system tables. -void loadMetadata(ContextPtr context, const String & default_database_name = {}); +void loadMetadata(ContextMutablePtr context, const String & default_database_name = {}); } diff --git a/src/Interpreters/replaceAliasColumnsInQuery.cpp b/src/Interpreters/replaceAliasColumnsInQuery.cpp index 4c8367b269a..d85c1b23c2f 100644 --- a/src/Interpreters/replaceAliasColumnsInQuery.cpp +++ b/src/Interpreters/replaceAliasColumnsInQuery.cpp @@ -6,11 +6,11 @@ namespace DB { -void replaceAliasColumnsInQuery(ASTPtr & ast, const ColumnsDescription & columns, const NameSet & forbidden_columns, ContextPtr context) +void replaceAliasColumnsInQuery(ASTPtr & ast, const ColumnsDescription & columns, const NameSet & forbidden_columns, ContextConstPtr context) { - ColumnAliasesVisitor::Data aliase_column_data(columns, forbidden_columns, context); - ColumnAliasesVisitor aliase_column_visitor(aliase_column_data); - aliase_column_visitor.visit(ast); + ColumnAliasesVisitor::Data aliases_column_data(columns, forbidden_columns, context); + ColumnAliasesVisitor aliases_column_visitor(aliases_column_data); + aliases_column_visitor.visit(ast); } } diff --git a/src/Interpreters/replaceAliasColumnsInQuery.h b/src/Interpreters/replaceAliasColumnsInQuery.h index 92d2686b45b..90963ea167b 100644 --- a/src/Interpreters/replaceAliasColumnsInQuery.h +++ b/src/Interpreters/replaceAliasColumnsInQuery.h @@ -10,6 +10,6 @@ namespace DB class ColumnsDescription; -void replaceAliasColumnsInQuery(ASTPtr & ast, const ColumnsDescription & columns, const NameSet & forbidden_columns, ContextPtr context); +void replaceAliasColumnsInQuery(ASTPtr & ast, const ColumnsDescription & columns, const NameSet & forbidden_columns, ContextConstPtr context); } diff --git a/src/Parsers/ASTProjectionSelectQuery.cpp b/src/Parsers/ASTProjectionSelectQuery.cpp index 58943ed0430..cc0d387b933 100644 --- a/src/Parsers/ASTProjectionSelectQuery.cpp +++ b/src/Parsers/ASTProjectionSelectQuery.cpp @@ -134,8 +134,6 @@ ASTPtr ASTProjectionSelectQuery::cloneToASTSelect() const if (groupBy()) select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, groupBy()->clone()); // Get rid of orderBy. It's used for projection definition only - if (orderBy()) - select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, orderBy()->clone()); return node; } diff --git a/src/Parsers/ASTWindowDefinition.cpp b/src/Parsers/ASTWindowDefinition.cpp index 37b27b22c7d..376df5bc80c 100644 --- a/src/Parsers/ASTWindowDefinition.cpp +++ b/src/Parsers/ASTWindowDefinition.cpp @@ -12,6 +12,8 @@ ASTPtr ASTWindowDefinition::clone() const { auto result = std::make_shared(); + result->parent_window_name = parent_window_name; + if (partition_by) { result->partition_by = partition_by->clone(); @@ -38,31 +40,48 @@ void ASTWindowDefinition::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked format_frame) const { format_frame.expression_list_prepend_whitespace = false; + bool need_space = false; + + if (!parent_window_name.empty()) + { + settings.ostr << backQuoteIfNeed(parent_window_name); + + need_space = true; + } if (partition_by) { + if (need_space) + { + settings.ostr << " "; + } + settings.ostr << "PARTITION BY "; partition_by->formatImpl(settings, state, format_frame); - } - if (partition_by && order_by) - { - settings.ostr << " "; + need_space = true; } if (order_by) { + if (need_space) + { + settings.ostr << " "; + } + settings.ostr << "ORDER BY "; order_by->formatImpl(settings, state, format_frame); - } - if ((partition_by || order_by) && !frame.is_default) - { - settings.ostr << " "; + need_space = true; } if (!frame.is_default) { + if (need_space) + { + settings.ostr << " "; + } + settings.ostr << WindowFrame::toString(frame.type) << " BETWEEN "; if (frame.begin_type == WindowFrame::BoundaryType::Current) { diff --git a/src/Parsers/ASTWindowDefinition.h b/src/Parsers/ASTWindowDefinition.h index b57c1094e42..7547f1527f2 100644 --- a/src/Parsers/ASTWindowDefinition.h +++ b/src/Parsers/ASTWindowDefinition.h @@ -10,6 +10,8 @@ namespace DB struct ASTWindowDefinition : public IAST { + std::string parent_window_name; + ASTPtr partition_by; ASTPtr order_by; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index bd2dae58306..8e67c00d4a2 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -511,9 +511,6 @@ bool ParserWindowReference::parseImpl(Pos & pos, ASTPtr & node, Expected & expec // Variant 1: // function_name ( * ) OVER window_name - // FIXME doesn't work anyway for now -- never used anywhere, window names - // can't be defined, and TreeRewriter thinks the window name is a column so - // the query fails. if (pos->type != TokenType::OpeningRoundBracket) { ASTPtr window_name_ast; @@ -671,16 +668,10 @@ static bool tryParseFrameDefinition(ASTWindowDefinition * node, IParser::Pos & p return true; } -bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +// All except parent window name. +static bool parseWindowDefinitionParts(IParser::Pos & pos, + ASTWindowDefinition & node, Expected & expected) { - auto result = std::make_shared(); - - ParserToken parser_openging_bracket(TokenType::OpeningRoundBracket); - if (!parser_openging_bracket.ignore(pos, expected)) - { - return false; - } - ParserKeyword keyword_partition_by("PARTITION BY"); ParserNotEmptyExpressionList columns_partition_by( false /* we don't allow declaring aliases here*/); @@ -692,8 +683,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ASTPtr partition_by_ast; if (columns_partition_by.parse(pos, partition_by_ast, expected)) { - result->children.push_back(partition_by_ast); - result->partition_by = partition_by_ast; + node.children.push_back(partition_by_ast); + node.partition_by = partition_by_ast; } else { @@ -706,8 +697,8 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ASTPtr order_by_ast; if (columns_order_by.parse(pos, order_by_ast, expected)) { - result->children.push_back(order_by_ast); - result->order_by = order_by_ast; + node.children.push_back(order_by_ast); + node.order_by = order_by_ast; } else { @@ -715,9 +706,45 @@ bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expe } } - if (!tryParseFrameDefinition(result.get(), pos, expected)) + return tryParseFrameDefinition(&node, pos, expected); +} + +bool ParserWindowDefinition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto result = std::make_shared(); + + ParserToken parser_openging_bracket(TokenType::OpeningRoundBracket); + if (!parser_openging_bracket.ignore(pos, expected)) + { + return false; + } + + // We can have a parent window name specified before all other things. No + // easy way to distinguish identifier from keywords, so just try to parse it + // both ways. + if (parseWindowDefinitionParts(pos, *result, expected)) + { + // Successfully parsed without parent window specifier. It can be empty, + // so check that it is followed by the closing bracket. + ParserToken parser_closing_bracket(TokenType::ClosingRoundBracket); + if (parser_closing_bracket.ignore(pos, expected)) + { + node = result; + return true; + } + } + + // Try to parse with parent window specifier. + ParserIdentifier parser_parent_window; + ASTPtr window_name_identifier; + if (!parser_parent_window.parse(pos, window_name_identifier, expected)) + { + return false; + } + result->parent_window_name = window_name_identifier->as().name(); + + if (!parseWindowDefinitionParts(pos, *result, expected)) { - /* Broken frame definition. */ return false; } diff --git a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp index 36b57e242d7..2d820a5985f 100644 --- a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp @@ -28,7 +28,7 @@ bool BinaryRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) void BinaryRowInputFormat::readPrefix() { - /// NOTE The header is completely ignored. This can be easily improved. + /// NOTE: The header is completely ignored. This can be easily improved. UInt64 columns = 0; String tmp; diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h index f69c2562d7f..dc3be3289fc 100644 --- a/src/Processors/Pipe.h +++ b/src/Processors/Pipe.h @@ -110,7 +110,7 @@ public: /// Do not allow to change the table while the processors of pipe are alive. void addTableLock(TableLockHolder lock) { holder.table_locks.emplace_back(std::move(lock)); } /// This methods are from QueryPipeline. Needed to make conversion from pipeline to pipe possible. - void addInterpreterContext(std::shared_ptr context) { holder.interpreter_context.emplace_back(std::move(context)); } + void addInterpreterContext(std::shared_ptr context) { holder.interpreter_context.emplace_back(std::move(context)); } void addStorageHolder(StoragePtr storage) { holder.storage_holders.emplace_back(std::move(storage)); } void addQueryIdHolder(std::shared_ptr query_id_holder) { holder.query_id_holder = std::move(query_id_holder); } /// For queries with nested interpreters (i.e. StorageDistributed) @@ -129,7 +129,7 @@ private: /// Some processors may implicitly use Context or temporary Storage created by Interpreter. /// But lifetime of Streams is not nested in lifetime of Interpreters, so we have to store it here, /// because QueryPipeline is alive until query is finished. - std::vector> interpreter_context; + std::vector> interpreter_context; std::vector storage_holders; std::vector table_locks; std::vector> query_plans; diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 90c7e370880..1585f2532ff 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -119,7 +119,7 @@ public: const Block & getHeader() const { return pipe.getHeader(); } void addTableLock(TableLockHolder lock) { pipe.addTableLock(std::move(lock)); } - void addInterpreterContext(std::shared_ptr context) { pipe.addInterpreterContext(std::move(context)); } + void addInterpreterContext(std::shared_ptr context) { pipe.addInterpreterContext(std::move(context)); } void addStorageHolder(StoragePtr storage) { pipe.addStorageHolder(std::move(storage)); } void addQueryPlan(std::unique_ptr plan) { pipe.addQueryPlan(std::move(plan)); } void setLimits(const StreamLocalLimits & limits) { pipe.setLimits(limits); } diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp deleted file mode 100644 index c564a28d377..00000000000 --- a/src/Processors/QueryPlan/MergingFinal.cpp +++ /dev/null @@ -1,170 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -static ITransformingStep::Traits getTraits() -{ - return ITransformingStep::Traits - { - { - .preserves_distinct_columns = true, - .returns_single_stream = false, - .preserves_number_of_streams = false, - .preserves_sorting = false, - }, - { - .preserves_number_of_rows = true, - } - }; -} - -MergingFinal::MergingFinal( - const DataStream & input_stream, - size_t num_output_streams_, - SortDescription sort_description_, - MergeTreeData::MergingParams params_, - Names partition_key_columns_, - size_t max_block_size_) - : ITransformingStep(input_stream, input_stream.header, getTraits()) - , num_output_streams(num_output_streams_) - , sort_description(std::move(sort_description_)) - , merging_params(std::move(params_)) - , partition_key_columns(std::move(partition_key_columns_)) - , max_block_size(max_block_size_) -{ - /// TODO: check input_stream is partially sorted (each port) by the same description. -// output_stream->sort_description = sort_description; -// output_stream->sort_mode = DataStream::SortMode::Stream; -} - -void MergingFinal::transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) -{ - const auto & header = pipeline.getHeader(); - size_t num_outputs = pipeline.getNumStreams(); - - auto get_merging_processor = [&]() -> MergingTransformPtr - { - switch (merging_params.mode) - { - case MergeTreeData::MergingParams::Ordinary: - { - return std::make_shared(header, num_outputs, - sort_description, max_block_size); - } - - case MergeTreeData::MergingParams::Collapsing: - return std::make_shared(header, num_outputs, - sort_description, merging_params.sign_column, true, max_block_size); - - case MergeTreeData::MergingParams::Summing: - return std::make_shared(header, num_outputs, - sort_description, merging_params.columns_to_sum, partition_key_columns, max_block_size); - - case MergeTreeData::MergingParams::Aggregating: - return std::make_shared(header, num_outputs, - sort_description, max_block_size); - - case MergeTreeData::MergingParams::Replacing: - return std::make_shared(header, num_outputs, - sort_description, merging_params.version_column, max_block_size); - - case MergeTreeData::MergingParams::VersionedCollapsing: - return std::make_shared(header, num_outputs, - sort_description, merging_params.sign_column, max_block_size); - - case MergeTreeData::MergingParams::Graphite: - throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); - } - - __builtin_unreachable(); - }; - - if (num_output_streams <= 1 || sort_description.empty()) - { - pipeline.addTransform(get_merging_processor()); - return; - } - - ColumnNumbers key_columns; - key_columns.reserve(sort_description.size()); - - for (auto & desc : sort_description) - { - if (!desc.column_name.empty()) - key_columns.push_back(header.getPositionByName(desc.column_name)); - else - key_columns.emplace_back(desc.column_number); - } - - pipeline.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, num_output_streams, key_columns); - }); - - pipeline.transform([&](OutputPortRawPtrs ports) - { - Processors transforms; - std::vector output_ports; - transforms.reserve(ports.size() + num_output_streams); - output_ports.reserve(ports.size()); - - for (auto & port : ports) - { - auto copier = std::make_shared(header, num_output_streams); - connect(*port, copier->getInputPort()); - output_ports.emplace_back(copier->getOutputs().begin()); - transforms.emplace_back(std::move(copier)); - } - - for (size_t i = 0; i < num_output_streams; ++i) - { - auto merge = get_merging_processor(); - merge->setSelectorPosition(i); - auto input = merge->getInputs().begin(); - - /// Connect i-th merge with i-th input port of every copier. - for (size_t j = 0; j < ports.size(); ++j) - { - connect(*output_ports[j], *input); - ++output_ports[j]; - ++input; - } - - transforms.emplace_back(std::move(merge)); - } - - return transforms; - }); -} - -void MergingFinal::describeActions(FormatSettings & settings) const -{ - String prefix(settings.offset, ' '); - settings.out << prefix << "Sort description: "; - dumpSortDescription(sort_description, input_streams.front().header, settings.out); - settings.out << '\n'; -} - -void MergingFinal::describeActions(JSONBuilder::JSONMap & map) const -{ - map.add("Sort Description", explainSortDescription(sort_description, input_streams.front().header)); -} - -} diff --git a/src/Processors/QueryPlan/MergingFinal.h b/src/Processors/QueryPlan/MergingFinal.h deleted file mode 100644 index ed0394a62f4..00000000000 --- a/src/Processors/QueryPlan/MergingFinal.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once -#include -#include -#include - -namespace DB -{ - -/// Merge streams of data into single sorted stream. -class MergingFinal : public ITransformingStep -{ -public: - explicit MergingFinal( - const DataStream & input_stream, - size_t num_output_streams_, - SortDescription sort_description_, - MergeTreeData::MergingParams params_, - Names partition_key_columns_, - size_t max_block_size_); - - String getName() const override { return "MergingFinal"; } - - void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; - - void describeActions(JSONBuilder::JSONMap & map) const override; - void describeActions(FormatSettings & settings) const override; - -private: - size_t num_output_streams; - SortDescription sort_description; - MergeTreeData::MergingParams merging_params; - Names partition_key_columns; - size_t max_block_size; -}; - -} diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index d7a78e6cc6b..fd5de98b4c0 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -2,82 +2,162 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include +#include +#include +#include #include #include +namespace ProfileEvents +{ + extern const Event SelectedParts; + extern const Event SelectedRanges; + extern const Event SelectedMarks; +} + namespace DB { -ReadFromMergeTree::ReadFromMergeTree( - const MergeTreeData & storage_, - StorageMetadataPtr metadata_snapshot_, - String query_id_, - Names required_columns_, - RangesInDataParts parts_, - IndexStatPtr index_stats_, - PrewhereInfoPtr prewhere_info_, - Names virt_column_names_, - Settings settings_, - size_t num_streams_, - ReadType read_type_) - : ISourceStep(DataStream{.header = MergeTreeBaseSelectProcessor::transformHeader( - metadata_snapshot_->getSampleBlockForColumns(required_columns_, storage_.getVirtuals(), storage_.getStorageID()), - prewhere_info_, - storage_.getPartitionValueType(), - virt_column_names_)}) - , storage(storage_) - , metadata_snapshot(std::move(metadata_snapshot_)) - , query_id(std::move(query_id_)) - , required_columns(std::move(required_columns_)) - , parts(std::move(parts_)) - , index_stats(std::move(index_stats_)) - , prewhere_info(std::move(prewhere_info_)) - , virt_column_names(std::move(virt_column_names_)) - , settings(std::move(settings_)) - , num_streams(num_streams_) - , read_type(read_type_) +namespace ErrorCodes { + extern const int INDEX_NOT_USED; + extern const int LOGICAL_ERROR; } -Pipe ReadFromMergeTree::readFromPool() +struct ReadFromMergeTree::AnalysisResult +{ + RangesInDataParts parts_with_ranges; + MergeTreeDataSelectSamplingData sampling; + IndexStats index_stats; + Names column_names_to_read; + ReadFromMergeTree::ReadType read_type = ReadFromMergeTree::ReadType::Default; +}; + +static MergeTreeReaderSettings getMergeTreeReaderSettings(const ContextPtr & context) +{ + const auto & settings = context->getSettingsRef(); + return { + .min_bytes_to_use_direct_io = settings.min_bytes_to_use_direct_io, + .min_bytes_to_use_mmap_io = settings.min_bytes_to_use_mmap_io, + .mmap_cache = context->getMMappedFileCache(), + .max_read_buffer_size = settings.max_read_buffer_size, + .save_marks_in_cache = true, + .checksum_on_read = settings.checksum_on_read, + }; +} + +static const PrewhereInfoPtr & getPrewhereInfo(const SelectQueryInfo & query_info) +{ + return query_info.projection ? query_info.projection->prewhere_info + : query_info.prewhere_info; +} + +ReadFromMergeTree::ReadFromMergeTree( + MergeTreeData::DataPartsVector parts_, + Names real_column_names_, + Names virt_column_names_, + const MergeTreeData & data_, + const SelectQueryInfo & query_info_, + StorageMetadataPtr metadata_snapshot_, + StorageMetadataPtr metadata_snapshot_base_, + ContextPtr context_, + size_t max_block_size_, + size_t num_streams_, + bool sample_factor_column_queried_, + std::shared_ptr max_block_numbers_to_read_, + Poco::Logger * log_) + : ISourceStep(DataStream{.header = MergeTreeBaseSelectProcessor::transformHeader( + metadata_snapshot_->getSampleBlockForColumns(real_column_names_, data_.getVirtuals(), data_.getStorageID()), + getPrewhereInfo(query_info_), + data_.getPartitionValueType(), + virt_column_names_)}) + , reader_settings(getMergeTreeReaderSettings(context_)) + , prepared_parts(std::move(parts_)) + , real_column_names(std::move(real_column_names_)) + , virt_column_names(std::move(virt_column_names_)) + , data(data_) + , query_info(query_info_) + , prewhere_info(getPrewhereInfo(query_info)) + , metadata_snapshot(std::move(metadata_snapshot_)) + , metadata_snapshot_base(std::move(metadata_snapshot_base_)) + , context(std::move(context_)) + , max_block_size(max_block_size_) + , requested_num_streams(num_streams_) + , preferred_block_size_bytes(context->getSettingsRef().preferred_block_size_bytes) + , preferred_max_column_in_block_size_bytes(context->getSettingsRef().preferred_max_column_in_block_size_bytes) + , sample_factor_column_queried(sample_factor_column_queried_) + , max_block_numbers_to_read(std::move(max_block_numbers_to_read_)) + , log(log_) +{ + if (sample_factor_column_queried) + { + /// Only _sample_factor virtual column is added by ReadFromMergeTree + /// Other virtual columns are added by MergeTreeBaseSelectProcessor. + auto type = std::make_shared(); + output_stream->header.insert({type->createColumn(), type, "_sample_factor"}); + } +} + +Pipe ReadFromMergeTree::readFromPool( + RangesInDataParts parts_with_range, + Names required_columns, + size_t max_streams, + size_t min_marks_for_concurrent_read, + bool use_uncompressed_cache) { Pipes pipes; size_t sum_marks = 0; size_t total_rows = 0; - for (const auto & part : parts) + for (const auto & part : parts_with_range) { sum_marks += part.getMarksCount(); total_rows += part.getRowsCount(); } + const auto & settings = context->getSettingsRef(); + MergeTreeReadPool::BackoffSettings backoff_settings(settings); + auto pool = std::make_shared( - num_streams, + max_streams, sum_marks, - settings.min_marks_for_concurrent_read, - std::move(parts), - storage, + min_marks_for_concurrent_read, + std::move(parts_with_range), + data, metadata_snapshot, prewhere_info, true, required_columns, - settings.backoff_settings, + backoff_settings, settings.preferred_block_size_bytes, false); - auto * logger = &Poco::Logger::get(storage.getLogName() + " (SelectExecutor)"); - LOG_DEBUG(logger, "Reading approx. {} rows with {} streams", total_rows, num_streams); + auto * logger = &Poco::Logger::get(data.getLogName() + " (SelectExecutor)"); + LOG_DEBUG(logger, "Reading approx. {} rows with {} streams", total_rows, max_streams); - for (size_t i = 0; i < num_streams; ++i) + for (size_t i = 0; i < max_streams; ++i) { auto source = std::make_shared( - i, pool, settings.min_marks_for_concurrent_read, settings.max_block_size, + i, pool, min_marks_for_concurrent_read, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, - storage, metadata_snapshot, settings.use_uncompressed_cache, - prewhere_info, settings.reader_settings, virt_column_names); + data, metadata_snapshot, use_uncompressed_cache, + prewhere_info, reader_settings, virt_column_names); if (i == 0) { @@ -92,22 +172,29 @@ Pipe ReadFromMergeTree::readFromPool() } template -ProcessorPtr ReadFromMergeTree::createSource(const RangesInDataPart & part) +ProcessorPtr ReadFromMergeTree::createSource( + const RangesInDataPart & part, + const Names & required_columns, + bool use_uncompressed_cache) { return std::make_shared( - storage, metadata_snapshot, part.data_part, settings.max_block_size, settings.preferred_block_size_bytes, - settings.preferred_max_column_in_block_size_bytes, required_columns, part.ranges, settings.use_uncompressed_cache, - prewhere_info, true, settings.reader_settings, virt_column_names, part.part_index_in_query); + data, metadata_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, + preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, + prewhere_info, true, reader_settings, virt_column_names, part.part_index_in_query); } -Pipe ReadFromMergeTree::readInOrder() +Pipe ReadFromMergeTree::readInOrder( + RangesInDataParts parts_with_range, + Names required_columns, + ReadType read_type, + bool use_uncompressed_cache) { Pipes pipes; - for (const auto & part : parts) + for (const auto & part : parts_with_range) { auto source = read_type == ReadType::InReverseOrder - ? createSource(part) - : createSource(part); + ? createSource(part, required_columns, use_uncompressed_cache) + : createSource(part, required_columns, use_uncompressed_cache); pipes.emplace_back(std::move(source)); } @@ -125,12 +212,15 @@ Pipe ReadFromMergeTree::readInOrder() return pipe; } -Pipe ReadFromMergeTree::read() +Pipe ReadFromMergeTree::read( + RangesInDataParts parts_with_range, Names required_columns, ReadType read_type, + size_t max_streams, size_t min_marks_for_concurrent_read, bool use_uncompressed_cache) { - if (read_type == ReadType::Default && num_streams > 1) - return readFromPool(); + if (read_type == ReadType::Default && max_streams > 1) + return readFromPool(parts_with_range, required_columns, max_streams, + min_marks_for_concurrent_read, use_uncompressed_cache); - auto pipe = readInOrder(); + auto pipe = readInOrder(parts_with_range, required_columns, read_type, use_uncompressed_cache); /// Use ConcatProcessor to concat sources together. /// It is needed to read in parts order (and so in PK order) if single thread is used. @@ -140,16 +230,796 @@ Pipe ReadFromMergeTree::read() return pipe; } +namespace +{ + +struct PartRangesReadInfo +{ + std::vector sum_marks_in_parts; + + size_t sum_marks = 0; + size_t total_rows = 0; + size_t adaptive_parts = 0; + size_t index_granularity_bytes = 0; + size_t max_marks_to_use_cache = 0; + size_t min_marks_for_concurrent_read = 0; + + bool use_uncompressed_cache = false; + + PartRangesReadInfo( + const RangesInDataParts & parts, + const Settings & settings, + const MergeTreeSettings & data_settings) + { + /// Count marks for each part. + sum_marks_in_parts.resize(parts.size()); + for (size_t i = 0; i < parts.size(); ++i) + { + total_rows += parts[i].getRowsCount(); + sum_marks_in_parts[i] = parts[i].getMarksCount(); + sum_marks += sum_marks_in_parts[i]; + + if (parts[i].data_part->index_granularity_info.is_adaptive) + ++adaptive_parts; + } + + if (adaptive_parts > parts.size() / 2) + index_granularity_bytes = data_settings.index_granularity_bytes; + + max_marks_to_use_cache = MergeTreeDataSelectExecutor::roundRowsOrBytesToMarks( + settings.merge_tree_max_rows_to_use_cache, + settings.merge_tree_max_bytes_to_use_cache, + data_settings.index_granularity, + index_granularity_bytes); + + min_marks_for_concurrent_read = MergeTreeDataSelectExecutor::minMarksForConcurrentRead( + settings.merge_tree_min_rows_for_concurrent_read, + settings.merge_tree_min_bytes_for_concurrent_read, + data_settings.index_granularity, + index_granularity_bytes, + sum_marks); + + use_uncompressed_cache = settings.use_uncompressed_cache; + if (sum_marks > max_marks_to_use_cache) + use_uncompressed_cache = false; + } +}; + +} + +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( + RangesInDataParts && parts_with_ranges, + const Names & column_names) +{ + const auto & settings = context->getSettingsRef(); + const auto data_settings = data.getSettings(); + + PartRangesReadInfo info(parts_with_ranges, settings, *data_settings); + + if (0 == info.sum_marks) + return {}; + + size_t num_streams = requested_num_streams; + if (num_streams > 1) + { + /// Reduce the number of num_streams if the data is small. + if (info.sum_marks < num_streams * info.min_marks_for_concurrent_read && parts_with_ranges.size() < num_streams) + num_streams = std::max((info.sum_marks + info.min_marks_for_concurrent_read - 1) / info.min_marks_for_concurrent_read, parts_with_ranges.size()); + } + + return read(std::move(parts_with_ranges), column_names, ReadType::Default, + num_streams, info.min_marks_for_concurrent_read, info.use_uncompressed_cache); +} + +static ActionsDAGPtr createProjection(const Block & header) +{ + auto projection = std::make_shared(header.getNamesAndTypesList()); + projection->removeUnusedActions(header.getNames()); + projection->projectInput(); + return projection; +} + +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( + RangesInDataParts && parts_with_ranges, + const Names & column_names, + const ActionsDAGPtr & sorting_key_prefix_expr, + ActionsDAGPtr & out_projection, + const InputOrderInfoPtr & input_order_info) +{ + const auto & settings = context->getSettingsRef(); + const auto data_settings = data.getSettings(); + + PartRangesReadInfo info(parts_with_ranges, settings, *data_settings); + + Pipes res; + + if (info.sum_marks == 0) + return {}; + + /// Let's split ranges to avoid reading much data. + auto split_ranges = [rows_granularity = data_settings->index_granularity, max_block_size = max_block_size] + (const auto & ranges, int direction) + { + MarkRanges new_ranges; + const size_t max_marks_in_range = (max_block_size + rows_granularity - 1) / rows_granularity; + size_t marks_in_range = 1; + + if (direction == 1) + { + /// Split first few ranges to avoid reading much data. + bool split = false; + for (auto range : ranges) + { + while (!split && range.begin + marks_in_range < range.end) + { + new_ranges.emplace_back(range.begin, range.begin + marks_in_range); + range.begin += marks_in_range; + marks_in_range *= 2; + + if (marks_in_range > max_marks_in_range) + split = true; + } + new_ranges.emplace_back(range.begin, range.end); + } + } + else + { + /// Split all ranges to avoid reading much data, because we have to + /// store whole range in memory to reverse it. + for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) + { + auto range = *it; + while (range.begin + marks_in_range < range.end) + { + new_ranges.emplace_front(range.end - marks_in_range, range.end); + range.end -= marks_in_range; + marks_in_range = std::min(marks_in_range * 2, max_marks_in_range); + } + new_ranges.emplace_front(range.begin, range.end); + } + } + + return new_ranges; + }; + + const size_t min_marks_per_stream = (info.sum_marks - 1) / requested_num_streams + 1; + bool need_preliminary_merge = (parts_with_ranges.size() > settings.read_in_order_two_level_merge_threshold); + + Pipes pipes; + + for (size_t i = 0; i < requested_num_streams && !parts_with_ranges.empty(); ++i) + { + size_t need_marks = min_marks_per_stream; + RangesInDataParts new_parts; + + /// Loop over parts. + /// We will iteratively take part or some subrange of a part from the back + /// and assign a stream to read from it. + while (need_marks > 0 && !parts_with_ranges.empty()) + { + RangesInDataPart part = parts_with_ranges.back(); + parts_with_ranges.pop_back(); + + size_t & marks_in_part = info.sum_marks_in_parts.back(); + + /// We will not take too few rows from a part. + if (marks_in_part >= info.min_marks_for_concurrent_read && + need_marks < info.min_marks_for_concurrent_read) + need_marks = info.min_marks_for_concurrent_read; + + /// Do not leave too few rows in the part. + if (marks_in_part > need_marks && + marks_in_part - need_marks < info.min_marks_for_concurrent_read) + need_marks = marks_in_part; + + MarkRanges ranges_to_get_from_part; + + /// We take the whole part if it is small enough. + if (marks_in_part <= need_marks) + { + ranges_to_get_from_part = part.ranges; + + need_marks -= marks_in_part; + info.sum_marks_in_parts.pop_back(); + } + else + { + /// Loop through ranges in part. Take enough ranges to cover "need_marks". + while (need_marks > 0) + { + if (part.ranges.empty()) + throw Exception("Unexpected end of ranges while spreading marks among streams", + ErrorCodes::LOGICAL_ERROR); + + MarkRange & range = part.ranges.front(); + + const size_t marks_in_range = range.end - range.begin; + const size_t marks_to_get_from_range = std::min(marks_in_range, need_marks); + + ranges_to_get_from_part.emplace_back(range.begin, range.begin + marks_to_get_from_range); + range.begin += marks_to_get_from_range; + marks_in_part -= marks_to_get_from_range; + need_marks -= marks_to_get_from_range; + if (range.begin == range.end) + part.ranges.pop_front(); + } + parts_with_ranges.emplace_back(part); + } + ranges_to_get_from_part = split_ranges(ranges_to_get_from_part, input_order_info->direction); + new_parts.emplace_back(part.data_part, part.part_index_in_query, std::move(ranges_to_get_from_part)); + } + + auto read_type = input_order_info->direction == 1 + ? ReadFromMergeTree::ReadType::InOrder + : ReadFromMergeTree::ReadType::InReverseOrder; + + pipes.emplace_back(read(std::move(new_parts), column_names, read_type, + requested_num_streams, info.min_marks_for_concurrent_read, info.use_uncompressed_cache)); + } + + if (need_preliminary_merge) + { + SortDescription sort_description; + for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j) + sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j], + input_order_info->direction, 1); + + auto sorting_key_expr = std::make_shared(sorting_key_prefix_expr); + + for (auto & pipe : pipes) + { + /// Drop temporary columns, added by 'sorting_key_prefix_expr' + out_projection = createProjection(pipe.getHeader()); + + pipe.addSimpleTransform([sorting_key_expr](const Block & header) + { + return std::make_shared(header, sorting_key_expr); + }); + + if (pipe.numOutputPorts() > 1) + { + auto transform = std::make_shared( + pipe.getHeader(), + pipe.numOutputPorts(), + sort_description, + max_block_size); + + pipe.addTransform(std::move(transform)); + } + } + } + + return Pipe::unitePipes(std::move(pipes)); +} + +static void addMergingFinal( + Pipe & pipe, + size_t num_output_streams, + const SortDescription & sort_description, + MergeTreeData::MergingParams merging_params, + Names partition_key_columns, + size_t max_block_size) +{ + const auto & header = pipe.getHeader(); + size_t num_outputs = pipe.numOutputPorts(); + + auto get_merging_processor = [&]() -> MergingTransformPtr + { + switch (merging_params.mode) + { + case MergeTreeData::MergingParams::Ordinary: + { + return std::make_shared(header, num_outputs, + sort_description, max_block_size); + } + + case MergeTreeData::MergingParams::Collapsing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.sign_column, true, max_block_size); + + case MergeTreeData::MergingParams::Summing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.columns_to_sum, partition_key_columns, max_block_size); + + case MergeTreeData::MergingParams::Aggregating: + return std::make_shared(header, num_outputs, + sort_description, max_block_size); + + case MergeTreeData::MergingParams::Replacing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.version_column, max_block_size); + + case MergeTreeData::MergingParams::VersionedCollapsing: + return std::make_shared(header, num_outputs, + sort_description, merging_params.sign_column, max_block_size); + + case MergeTreeData::MergingParams::Graphite: + throw Exception("GraphiteMergeTree doesn't support FINAL", ErrorCodes::LOGICAL_ERROR); + } + + __builtin_unreachable(); + }; + + if (num_output_streams <= 1 || sort_description.empty()) + { + pipe.addTransform(get_merging_processor()); + return; + } + + ColumnNumbers key_columns; + key_columns.reserve(sort_description.size()); + + for (const auto & desc : sort_description) + { + if (!desc.column_name.empty()) + key_columns.push_back(header.getPositionByName(desc.column_name)); + else + key_columns.emplace_back(desc.column_number); + } + + pipe.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, num_output_streams, key_columns); + }); + + pipe.transform([&](OutputPortRawPtrs ports) + { + Processors transforms; + std::vector output_ports; + transforms.reserve(ports.size() + num_output_streams); + output_ports.reserve(ports.size()); + + for (auto & port : ports) + { + auto copier = std::make_shared(header, num_output_streams); + connect(*port, copier->getInputPort()); + output_ports.emplace_back(copier->getOutputs().begin()); + transforms.emplace_back(std::move(copier)); + } + + for (size_t i = 0; i < num_output_streams; ++i) + { + auto merge = get_merging_processor(); + merge->setSelectorPosition(i); + auto input = merge->getInputs().begin(); + + /// Connect i-th merge with i-th input port of every copier. + for (size_t j = 0; j < ports.size(); ++j) + { + connect(*output_ports[j], *input); + ++output_ports[j]; + ++input; + } + + transforms.emplace_back(std::move(merge)); + } + + return transforms; + }); +} + + +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( + RangesInDataParts && parts_with_ranges, + const Names & column_names, + ActionsDAGPtr & out_projection) +{ + const auto & settings = context->getSettingsRef(); + const auto data_settings = data.getSettings(); + + PartRangesReadInfo info(parts_with_ranges, settings, *data_settings); + + size_t num_streams = requested_num_streams; + if (num_streams > settings.max_final_threads) + num_streams = settings.max_final_threads; + + /// If setting do_not_merge_across_partitions_select_final is true than we won't merge parts from different partitions. + /// We have all parts in parts vector, where parts with same partition are nearby. + /// So we will store iterators pointed to the beginning of each partition range (and parts.end()), + /// then we will create a pipe for each partition that will run selecting processor and merging processor + /// for the parts with this partition. In the end we will unite all the pipes. + std::vector parts_to_merge_ranges; + auto it = parts_with_ranges.begin(); + parts_to_merge_ranges.push_back(it); + + if (settings.do_not_merge_across_partitions_select_final) + { + while (it != parts_with_ranges.end()) + { + it = std::find_if( + it, parts_with_ranges.end(), [&it](auto & part) { return it->data_part->info.partition_id != part.data_part->info.partition_id; }); + parts_to_merge_ranges.push_back(it); + } + /// We divide threads for each partition equally. But we will create at least the number of partitions threads. + /// (So, the total number of threads could be more than initial num_streams. + num_streams /= (parts_to_merge_ranges.size() - 1); + } + else + { + /// If do_not_merge_across_partitions_select_final is false we just merge all the parts. + parts_to_merge_ranges.push_back(parts_with_ranges.end()); + } + + Pipes partition_pipes; + + /// If do_not_merge_across_partitions_select_final is true and num_streams > 1 + /// we will store lonely parts with level > 0 to use parallel select on them. + std::vector lonely_parts; + size_t total_rows_in_lonely_parts = 0; + size_t sum_marks_in_lonely_parts = 0; + + for (size_t range_index = 0; range_index < parts_to_merge_ranges.size() - 1; ++range_index) + { + Pipe pipe; + + { + RangesInDataParts new_parts; + + /// If do_not_merge_across_partitions_select_final is true and there is only one part in partition + /// with level > 0 then we won't postprocess this part and if num_streams > 1 we + /// can use parallel select on such parts. We save such parts in one vector and then use + /// MergeTreeReadPool and MergeTreeThreadSelectBlockInputProcessor for parallel select. + if (num_streams > 1 && settings.do_not_merge_across_partitions_select_final && + std::distance(parts_to_merge_ranges[range_index], parts_to_merge_ranges[range_index + 1]) == 1 && + parts_to_merge_ranges[range_index]->data_part->info.level > 0) + { + total_rows_in_lonely_parts += parts_to_merge_ranges[range_index]->getRowsCount(); + sum_marks_in_lonely_parts += parts_to_merge_ranges[range_index]->getMarksCount(); + lonely_parts.push_back(std::move(*parts_to_merge_ranges[range_index])); + continue; + } + else + { + for (auto part_it = parts_to_merge_ranges[range_index]; part_it != parts_to_merge_ranges[range_index + 1]; ++part_it) + { + new_parts.emplace_back(part_it->data_part, part_it->part_index_in_query, part_it->ranges); + } + } + + if (new_parts.empty()) + continue; + + pipe = read(std::move(new_parts), column_names, ReadFromMergeTree::ReadType::InOrder, + num_streams, 0, info.use_uncompressed_cache); + + /// Drop temporary columns, added by 'sorting_key_expr' + if (!out_projection) + out_projection = createProjection(pipe.getHeader()); + } + + auto sorting_expr = std::make_shared( + metadata_snapshot->getSortingKey().expression->getActionsDAG().clone()); + + pipe.addSimpleTransform([sorting_expr](const Block & header) + { + return std::make_shared(header, sorting_expr); + }); + + /// If do_not_merge_across_partitions_select_final is true and there is only one part in partition + /// with level > 0 then we won't postprocess this part + if (settings.do_not_merge_across_partitions_select_final && + std::distance(parts_to_merge_ranges[range_index], parts_to_merge_ranges[range_index + 1]) == 1 && + parts_to_merge_ranges[range_index]->data_part->info.level > 0) + { + partition_pipes.emplace_back(std::move(pipe)); + continue; + } + + Names sort_columns = metadata_snapshot->getSortingKeyColumns(); + SortDescription sort_description; + size_t sort_columns_size = sort_columns.size(); + sort_description.reserve(sort_columns_size); + + Names partition_key_columns = metadata_snapshot->getPartitionKey().column_names; + + const auto & header = pipe.getHeader(); + for (size_t i = 0; i < sort_columns_size; ++i) + sort_description.emplace_back(header.getPositionByName(sort_columns[i]), 1, 1); + + addMergingFinal( + pipe, + std::min(num_streams, settings.max_final_threads), + sort_description, data.merging_params, partition_key_columns, max_block_size); + + partition_pipes.emplace_back(std::move(pipe)); + } + + if (!lonely_parts.empty()) + { + RangesInDataParts new_parts; + + size_t num_streams_for_lonely_parts = num_streams * lonely_parts.size(); + + + const size_t min_marks_for_concurrent_read = MergeTreeDataSelectExecutor::minMarksForConcurrentRead( + settings.merge_tree_min_rows_for_concurrent_read, + settings.merge_tree_min_bytes_for_concurrent_read, + data_settings->index_granularity, + info.index_granularity_bytes, + sum_marks_in_lonely_parts); + + /// Reduce the number of num_streams_for_lonely_parts if the data is small. + if (sum_marks_in_lonely_parts < num_streams_for_lonely_parts * min_marks_for_concurrent_read && lonely_parts.size() < num_streams_for_lonely_parts) + num_streams_for_lonely_parts = std::max((sum_marks_in_lonely_parts + min_marks_for_concurrent_read - 1) / min_marks_for_concurrent_read, lonely_parts.size()); + + auto pipe = read(std::move(lonely_parts), column_names, ReadFromMergeTree::ReadType::Default, + num_streams_for_lonely_parts, min_marks_for_concurrent_read, info.use_uncompressed_cache); + + /// Drop temporary columns, added by 'sorting_key_expr' + if (!out_projection) + out_projection = createProjection(pipe.getHeader()); + + auto sorting_expr = std::make_shared( + metadata_snapshot->getSortingKey().expression->getActionsDAG().clone()); + + pipe.addSimpleTransform([sorting_expr](const Block & header) + { + return std::make_shared(header, sorting_expr); + }); + + partition_pipes.emplace_back(std::move(pipe)); + } + + return Pipe::unitePipes(std::move(partition_pipes)); +} + +ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTreeData::DataPartsVector parts) const +{ + AnalysisResult result; + const auto & settings = context->getSettingsRef(); + + size_t total_parts = parts.size(); + + auto part_values = MergeTreeDataSelectExecutor::filterPartsByVirtualColumns(data, parts, query_info.query, context); + if (part_values && part_values->empty()) + return result; + + result.column_names_to_read = real_column_names; + + /// If there are only virtual columns in the query, you must request at least one non-virtual one. + if (result.column_names_to_read.empty()) + { + NamesAndTypesList available_real_columns = metadata_snapshot->getColumns().getAllPhysical(); + result.column_names_to_read.push_back(ExpressionActions::getSmallestColumn(available_real_columns)); + } + + metadata_snapshot->check(result.column_names_to_read, data.getVirtuals(), data.getStorageID()); + + // Build and check if primary key is used when necessary + const auto & primary_key = metadata_snapshot->getPrimaryKey(); + Names primary_key_columns = primary_key.column_names; + KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expression); + + if (settings.force_primary_key && key_condition.alwaysUnknownOrTrue()) + { + throw Exception( + ErrorCodes::INDEX_NOT_USED, + "Primary key ({}) is not used and setting 'force_primary_key' is set.", + fmt::join(primary_key_columns, ", ")); + } + LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); + + const auto & select = query_info.query->as(); + + MergeTreeDataSelectExecutor::filterPartsByPartition( + parts, part_values, metadata_snapshot_base, data, query_info, context, + max_block_numbers_to_read.get(), log, result.index_stats); + + result.sampling = MergeTreeDataSelectExecutor::getSampling( + select, metadata_snapshot->getColumns().getAllPhysical(), parts, key_condition, + data, metadata_snapshot, context, sample_factor_column_queried, log); + + if (result.sampling.read_nothing) + return result; + + size_t total_marks_pk = 0; + for (const auto & part : parts) + total_marks_pk += part->index_granularity.getMarksCountWithoutFinal(); + + size_t parts_before_pk = parts.size(); + + result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( + std::move(parts), + metadata_snapshot, + query_info, + context, + key_condition, + reader_settings, + log, + requested_num_streams, + result.index_stats, + true); + + size_t sum_marks_pk = total_marks_pk; + for (const auto & stat : result.index_stats) + if (stat.type == IndexType::PrimaryKey) + sum_marks_pk = stat.num_granules_after; + + size_t sum_marks = 0; + size_t sum_ranges = 0; + + for (const auto & part : result.parts_with_ranges) + { + sum_ranges += part.ranges.size(); + sum_marks += part.getMarksCount(); + } + + LOG_DEBUG( + log, + "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", + parts_before_pk, + total_parts, + result.parts_with_ranges.size(), + sum_marks_pk, + total_marks_pk, + sum_marks, + sum_ranges); + + ProfileEvents::increment(ProfileEvents::SelectedParts, result.parts_with_ranges.size()); + ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges); + ProfileEvents::increment(ProfileEvents::SelectedMarks, sum_marks); + + const auto & input_order_info = query_info.input_order_info + ? query_info.input_order_info + : (query_info.projection ? query_info.projection->input_order_info : nullptr); + + if ((settings.optimize_read_in_order || settings.optimize_aggregation_in_order) && input_order_info) + result.read_type = (input_order_info->direction > 0) ? ReadType::InOrder + : ReadType::InReverseOrder; + + return result; +} + void ReadFromMergeTree::initializePipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) { - Pipe pipe = read(); + auto result = selectRangesToRead(prepared_parts); + auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result.parts_with_ranges, context); + + if (result.parts_with_ranges.empty()) + { + pipeline.init(Pipe(std::make_shared(getOutputStream().header))); + return; + } + + /// Projection, that needed to drop columns, which have appeared by execution + /// of some extra expressions, and to allow execute the same expressions later. + /// NOTE: It may lead to double computation of expressions. + ActionsDAGPtr result_projection; + + Names column_names_to_read = std::move(result.column_names_to_read); + const auto & select = query_info.query->as(); + if (!select.final() && result.sampling.use_sampling) + { + /// Add columns needed for `sample_by_ast` to `column_names_to_read`. + /// Skip this if final was used, because such columns were already added from PK. + std::vector add_columns = result.sampling.filter_expression->getRequiredColumns().getNames(); + column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); + std::sort(column_names_to_read.begin(), column_names_to_read.end()); + column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), + column_names_to_read.end()); + } + + const auto & input_order_info = query_info.input_order_info + ? query_info.input_order_info + : (query_info.projection ? query_info.projection->input_order_info : nullptr); + + Pipe pipe; + + const auto & settings = context->getSettingsRef(); + + if (select.final()) + { + /// Add columns needed to calculate the sorting expression and the sign. + std::vector add_columns = metadata_snapshot->getColumnsRequiredForSortingKey(); + column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); + + if (!data.merging_params.sign_column.empty()) + column_names_to_read.push_back(data.merging_params.sign_column); + if (!data.merging_params.version_column.empty()) + column_names_to_read.push_back(data.merging_params.version_column); + + std::sort(column_names_to_read.begin(), column_names_to_read.end()); + column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); + + pipe = spreadMarkRangesAmongStreamsFinal( + std::move(result.parts_with_ranges), + column_names_to_read, + result_projection); + } + else if ((settings.optimize_read_in_order || settings.optimize_aggregation_in_order) && input_order_info) + { + size_t prefix_size = input_order_info->order_key_prefix_descr.size(); + auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone(); + order_key_prefix_ast->children.resize(prefix_size); + + auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); + auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false); + + pipe = spreadMarkRangesAmongStreamsWithOrder( + std::move(result.parts_with_ranges), + column_names_to_read, + sorting_key_prefix_expr, + result_projection, + input_order_info); + } + else + { + pipe = spreadMarkRangesAmongStreams( + std::move(result.parts_with_ranges), + column_names_to_read); + } + + if (pipe.empty()) + { + pipeline.init(Pipe(std::make_shared(getOutputStream().header))); + return; + } + + if (result.sampling.use_sampling) + { + auto sampling_actions = std::make_shared(result.sampling.filter_expression); + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, + sampling_actions, + result.sampling.filter_function->getColumnName(), + false); + }); + } + + Block cur_header = result_projection ? result_projection->getResultColumns() + : pipe.getHeader(); + + auto append_actions = [&result_projection, &cur_header](ActionsDAGPtr actions) + { + if (!result_projection) + result_projection = std::move(actions); + else + result_projection = ActionsDAG::merge(std::move(*result_projection), std::move(*actions)); + + cur_header = result_projection->getResultColumns(); + }; + + /// By the way, if a distributed query or query to a Merge table is made, then the `_sample_factor` column can have different values. + if (sample_factor_column_queried) + { + ColumnWithTypeAndName column; + column.name = "_sample_factor"; + column.type = std::make_shared(); + column.column = column.type->createColumnConst(0, Field(result.sampling.used_sample_factor)); + + auto adding_column = ActionsDAG::makeAddingColumnActions(std::move(column)); + append_actions(std::move(adding_column)); + } + + /// Extra columns may be returned (for example, if sampling is used). + /// Convert pipe to step header structure. + if (!isCompatibleHeader(cur_header, getOutputStream().header)) + { + auto converting = ActionsDAG::makeConvertingActions( + cur_header.getColumnsWithTypeAndName(), + getOutputStream().header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + + append_actions(std::move(converting)); + } + + if (result_projection) + { + auto projection_actions = std::make_shared(result_projection); + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, projection_actions); + }); + } for (const auto & processor : pipe.getProcessors()) processors.emplace_back(processor); // Attach QueryIdHolder if needed - if (!query_id.empty()) - pipe.addQueryIdHolder(std::make_shared(query_id, storage)); + if (query_id_holder) + pipe.addQueryIdHolder(std::move(query_id_holder)); pipeline.init(std::move(pipe)); } @@ -190,41 +1060,46 @@ static const char * readTypeToString(ReadFromMergeTree::ReadType type) void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const { + auto result = selectRangesToRead(prepared_parts); std::string prefix(format_settings.offset, format_settings.indent_char); - format_settings.out << prefix << "ReadType: " << readTypeToString(read_type) << '\n'; + format_settings.out << prefix << "ReadType: " << readTypeToString(result.read_type) << '\n'; - if (index_stats && !index_stats->empty()) + if (!result.index_stats.empty()) { - format_settings.out << prefix << "Parts: " << index_stats->back().num_parts_after << '\n'; - format_settings.out << prefix << "Granules: " << index_stats->back().num_granules_after << '\n'; + format_settings.out << prefix << "Parts: " << result.index_stats.back().num_parts_after << '\n'; + format_settings.out << prefix << "Granules: " << result.index_stats.back().num_granules_after << '\n'; } } void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const { - map.add("Read Type", readTypeToString(read_type)); - if (index_stats && !index_stats->empty()) + auto result = selectRangesToRead(prepared_parts); + map.add("Read Type", readTypeToString(result.read_type)); + if (!result.index_stats.empty()) { - map.add("Parts", index_stats->back().num_parts_after); - map.add("Granules", index_stats->back().num_granules_after); + map.add("Parts", result.index_stats.back().num_parts_after); + map.add("Granules", result.index_stats.back().num_granules_after); } } void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const { + auto result = selectRangesToRead(prepared_parts); + auto index_stats = std::move(result.index_stats); + std::string prefix(format_settings.offset, format_settings.indent_char); - if (index_stats && !index_stats->empty()) + if (!index_stats.empty()) { /// Do not print anything if no indexes is applied. - if (index_stats->size() == 1 && index_stats->front().type == IndexType::None) + if (index_stats.size() == 1 && index_stats.front().type == IndexType::None) return; std::string indent(format_settings.indent, format_settings.indent_char); format_settings.out << prefix << "Indexes:\n"; - for (size_t i = 0; i < index_stats->size(); ++i) + for (size_t i = 0; i < index_stats.size(); ++i) { - const auto & stat = (*index_stats)[i]; + const auto & stat = index_stats[i]; if (stat.type == IndexType::None) continue; @@ -248,12 +1123,12 @@ void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const format_settings.out << prefix << indent << indent << "Parts: " << stat.num_parts_after; if (i) - format_settings.out << '/' << (*index_stats)[i - 1].num_parts_after; + format_settings.out << '/' << index_stats[i - 1].num_parts_after; format_settings.out << '\n'; format_settings.out << prefix << indent << indent << "Granules: " << stat.num_granules_after; if (i) - format_settings.out << '/' << (*index_stats)[i - 1].num_granules_after; + format_settings.out << '/' << index_stats[i - 1].num_granules_after; format_settings.out << '\n'; } } @@ -261,17 +1136,20 @@ void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const void ReadFromMergeTree::describeIndexes(JSONBuilder::JSONMap & map) const { - if (index_stats && !index_stats->empty()) + auto result = selectRangesToRead(prepared_parts); + auto index_stats = std::move(result.index_stats); + + if (!index_stats.empty()) { /// Do not print anything if no indexes is applied. - if (index_stats->size() == 1 && index_stats->front().type == IndexType::None) + if (index_stats.size() == 1 && index_stats.front().type == IndexType::None) return; auto indexes_array = std::make_unique(); - for (size_t i = 0; i < index_stats->size(); ++i) + for (size_t i = 0; i < index_stats.size(); ++i) { - const auto & stat = (*index_stats)[i]; + const auto & stat = index_stats[i]; if (stat.type == IndexType::None) continue; @@ -299,11 +1177,11 @@ void ReadFromMergeTree::describeIndexes(JSONBuilder::JSONMap & map) const index_map->add("Condition", stat.condition); if (i) - index_map->add("Initial Parts", (*index_stats)[i - 1].num_parts_after); + index_map->add("Initial Parts", index_stats[i - 1].num_parts_after); index_map->add("Selected Parts", stat.num_parts_after); if (i) - index_map->add("Initial Granules", (*index_stats)[i - 1].num_granules_after); + index_map->add("Initial Granules", index_stats[i - 1].num_granules_after); index_map->add("Selected Granules", stat.num_granules_after); indexes_array->add(std::move(index_map)); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 479610b3edc..6e1efffdb02 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -1,12 +1,14 @@ #pragma once #include -#include #include -#include namespace DB { +using PartitionIdToMaxBlock = std::unordered_map; + +class Pipe; + /// This step is created to read from MergeTree* table. /// For now, it takes a list of parts and creates source from it. class ReadFromMergeTree final : public ISourceStep @@ -36,20 +38,6 @@ public: }; using IndexStats = std::vector; - using IndexStatPtr = std::unique_ptr; - - /// Part of settings which are needed for reading. - struct Settings - { - UInt64 max_block_size; - size_t preferred_block_size_bytes; - size_t preferred_max_column_in_block_size_bytes; - size_t min_marks_for_concurrent_read; - bool use_uncompressed_cache; - - MergeTreeReaderSettings reader_settings; - MergeTreeReadPool::BackoffSettings backoff_settings; - }; enum class ReadType { @@ -67,17 +55,19 @@ public: }; ReadFromMergeTree( - const MergeTreeData & storage_, - StorageMetadataPtr metadata_snapshot_, - String query_id_, - Names required_columns_, - RangesInDataParts parts_, - IndexStatPtr index_stats_, - PrewhereInfoPtr prewhere_info_, + MergeTreeData::DataPartsVector parts_, + Names real_column_names_, Names virt_column_names_, - Settings settings_, + const MergeTreeData & data_, + const SelectQueryInfo & query_info_, + StorageMetadataPtr metadata_snapshot_, + StorageMetadataPtr metadata_snapshot_base_, + ContextPtr context_, + size_t max_block_size_, size_t num_streams_, - ReadType read_type_ + bool sample_factor_column_queried_, + std::shared_ptr max_block_numbers_to_read_, + Poco::Logger * log_ ); String getName() const override { return "ReadFromMergeTree"; } @@ -91,26 +81,56 @@ public: void describeIndexes(JSONBuilder::JSONMap & map) const override; private: - const MergeTreeData & storage; - StorageMetadataPtr metadata_snapshot; - String query_id; + const MergeTreeReaderSettings reader_settings; - Names required_columns; - RangesInDataParts parts; - IndexStatPtr index_stats; - PrewhereInfoPtr prewhere_info; + MergeTreeData::DataPartsVector prepared_parts; + Names real_column_names; Names virt_column_names; - Settings settings; - size_t num_streams; - ReadType read_type; + const MergeTreeData & data; + SelectQueryInfo query_info; + PrewhereInfoPtr prewhere_info; - Pipe read(); - Pipe readFromPool(); - Pipe readInOrder(); + StorageMetadataPtr metadata_snapshot; + StorageMetadataPtr metadata_snapshot_base; + + ContextPtr context; + + const size_t max_block_size; + const size_t requested_num_streams; + const size_t preferred_block_size_bytes; + const size_t preferred_max_column_in_block_size_bytes; + const bool sample_factor_column_queried; + + std::shared_ptr max_block_numbers_to_read; + + Poco::Logger * log; + + Pipe read(RangesInDataParts parts_with_range, Names required_columns, ReadType read_type, size_t max_streams, size_t min_marks_for_concurrent_read, bool use_uncompressed_cache); + Pipe readFromPool(RangesInDataParts parts_with_ranges, Names required_columns, size_t max_streams, size_t min_marks_for_concurrent_read, bool use_uncompressed_cache); + Pipe readInOrder(RangesInDataParts parts_with_range, Names required_columns, ReadType read_type, bool use_uncompressed_cache); template - ProcessorPtr createSource(const RangesInDataPart & part); + ProcessorPtr createSource(const RangesInDataPart & part, const Names & required_columns, bool use_uncompressed_cache); + + Pipe spreadMarkRangesAmongStreams( + RangesInDataParts && parts_with_ranges, + const Names & column_names); + + Pipe spreadMarkRangesAmongStreamsWithOrder( + RangesInDataParts && parts_with_ranges, + const Names & column_names, + const ActionsDAGPtr & sorting_key_prefix_expr, + ActionsDAGPtr & out_projection, + const InputOrderInfoPtr & input_order_info); + + Pipe spreadMarkRangesAmongStreamsFinal( + RangesInDataParts && parts, + const Names & column_names, + ActionsDAGPtr & out_projection); + + struct AnalysisResult; + AnalysisResult selectRangesToRead(MergeTreeData::DataPartsVector parts) const; }; } diff --git a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp index d7711abb3e1..0d1a0fdc619 100644 --- a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp +++ b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp @@ -4,7 +4,7 @@ namespace DB { -ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_) +ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_) : ISourceStep(DataStream{.header = pipe_.getHeader()}) , pipe(std::move(pipe_)) , context(std::move(context_)) diff --git a/src/Processors/QueryPlan/ReadFromPreparedSource.h b/src/Processors/QueryPlan/ReadFromPreparedSource.h index 69297c3c9ae..0d4e9653806 100644 --- a/src/Processors/QueryPlan/ReadFromPreparedSource.h +++ b/src/Processors/QueryPlan/ReadFromPreparedSource.h @@ -9,7 +9,7 @@ namespace DB class ReadFromPreparedSource : public ISourceStep { public: - explicit ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_ = nullptr); + explicit ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_ = nullptr); String getName() const override { return "ReadFromPreparedSource"; } @@ -17,7 +17,7 @@ public: private: Pipe pipe; - std::shared_ptr context; + std::shared_ptr context; }; class ReadFromStorageStep : public ReadFromPreparedSource diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 5ab9c79511f..86a40685d1f 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -112,7 +112,6 @@ SRCS( QueryPlan/LimitStep.cpp QueryPlan/MergeSortingStep.cpp QueryPlan/MergingAggregatedStep.cpp - QueryPlan/MergingFinal.cpp QueryPlan/MergingSortedStep.cpp QueryPlan/OffsetStep.cpp QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 6f0f2d30123..2d6de423c5b 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -521,7 +521,7 @@ namespace Poco::Logger * log = nullptr; std::shared_ptr session; - ContextPtr query_context; + ContextMutablePtr query_context; std::optional query_scope; String query_text; ASTPtr ast; @@ -932,8 +932,8 @@ namespace String format = external_table.format(); if (format.empty()) format = "TabSeparated"; - ContextPtr external_table_context = query_context; - ContextPtr temp_context; + ContextMutablePtr external_table_context = query_context; + ContextMutablePtr temp_context; if (!external_table.settings().empty()) { temp_context = Context::createCopy(query_context); diff --git a/src/Server/HTTP/HTMLForm.cpp b/src/Server/HTTP/HTMLForm.cpp index 7a87f484b5c..9e0f74dcc33 100644 --- a/src/Server/HTTP/HTMLForm.cpp +++ b/src/Server/HTTP/HTMLForm.cpp @@ -244,7 +244,7 @@ bool HTMLForm::MultipartReadBuffer::skipToNextBoundary() while (!in.eof()) { - auto line = readLine(); + auto line = readLine(true); if (startsWith(line, boundary)) { set(in.position(), 0); @@ -256,29 +256,36 @@ bool HTMLForm::MultipartReadBuffer::skipToNextBoundary() throw Poco::Net::HTMLFormException("No boundary line found"); } -std::string HTMLForm::MultipartReadBuffer::readLine(bool strict) +std::string HTMLForm::MultipartReadBuffer::readLine(bool append_crlf) { std::string line; char ch = 0; // silence "uninitialized" warning from gcc-* - while (in.read(ch) && ch != '\r' && ch != '\n') + /// If we don't append CRLF, it means that we may have to prepend CRLF from previous content line, which wasn't the boundary. + if (in.read(ch)) line += ch; - - if (in.eof()) - { - if (strict) - throw Poco::Net::HTMLFormException("Unexpected end of message"); + if (in.read(ch)) + line += ch; + if (append_crlf && line == "\r\n") return line; - } - line += ch; - - if (ch == '\r') + while (!in.eof()) { - if (!in.read(ch) || ch != '\n') - throw Poco::Net::HTMLFormException("No CRLF found"); - else + while (in.read(ch) && ch != '\r') line += ch; + + if (in.eof()) break; + + assert(ch == '\r'); + + if (in.peek(ch) && ch == '\n') + { + in.ignore(); + if (append_crlf) line += "\r\n"; + break; + } + + line += ch; } return line; @@ -300,19 +307,12 @@ bool HTMLForm::MultipartReadBuffer::nextImpl() /// FIXME: there is an extra copy because we cannot traverse PeekableBuffer from checkpoint to position() /// since it may store different data parts in different sub-buffers, /// anyway calling makeContinuousMemoryFromCheckpointToPos() will also make an extra copy. - std::string line = readLine(false); - /// According to RFC2046 the preceding CRLF is a part of boundary line. - if (line == "\r\n") - { - line = readLine(false); - boundary_hit = startsWith(line, boundary); - if (!boundary_hit) line = "\r\n"; - } - else - boundary_hit = startsWith(line, boundary); + std::string line = readLine(false); + boundary_hit = startsWith(line, "\r\n" + boundary); + bool has_next = !boundary_hit && !line.empty(); - if (!line.empty()) + if (has_next) /// If we don't make sure that memory is contiguous then situation may happen, when part of the line is inside internal memory /// and other part is inside sub-buffer, thus we'll be unable to setup our working buffer properly. in.makeContinuousMemoryFromCheckpointToPos(); @@ -323,7 +323,7 @@ bool HTMLForm::MultipartReadBuffer::nextImpl() /// Limit readable data to a single line. BufferBase::set(in.position(), line.size(), 0); - return !boundary_hit && !line.empty(); + return has_next; } } diff --git a/src/Server/HTTP/HTMLForm.h b/src/Server/HTTP/HTMLForm.h index 8d8fb0d1719..ca6bb9048f1 100644 --- a/src/Server/HTTP/HTMLForm.h +++ b/src/Server/HTTP/HTMLForm.h @@ -118,7 +118,7 @@ private: const std::string boundary; bool boundary_hit = true; - std::string readLine(bool strict = true); + std::string readLine(bool append_crlf); bool nextImpl() override; }; diff --git a/src/Server/HTTP/HTTPServerResponse.cpp b/src/Server/HTTP/HTTPServerResponse.cpp index db5cfb132e3..25e7604a515 100644 --- a/src/Server/HTTP/HTTPServerResponse.cpp +++ b/src/Server/HTTP/HTTPServerResponse.cpp @@ -1,11 +1,8 @@ #include - #include - #include #include #include -#include #include #include #include @@ -13,6 +10,7 @@ #include #include + namespace DB { diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 8aed5d20f74..6eab6cdda5e 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -39,7 +39,6 @@ #include #include -#include #include #include #include @@ -277,7 +276,7 @@ HTTPHandler::~HTTPHandler() bool HTTPHandler::authenticateUser( - ContextPtr context, + ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response) @@ -441,7 +440,7 @@ bool HTTPHandler::authenticateUser( void HTTPHandler::processQuery( - ContextPtr context, + ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response, @@ -937,7 +936,7 @@ DynamicQueryHandler::DynamicQueryHandler(IServer & server_, const std::string & { } -bool DynamicQueryHandler::customizeQueryParam(ContextPtr context, const std::string & key, const std::string & value) +bool DynamicQueryHandler::customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) { if (key == param_name) return true; /// do nothing @@ -953,7 +952,7 @@ bool DynamicQueryHandler::customizeQueryParam(ContextPtr context, const std::str return false; } -std::string DynamicQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) +std::string DynamicQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) { if (likely(!startsWith(request.getContentType(), "multipart/form-data"))) { @@ -992,7 +991,7 @@ PredefinedQueryHandler::PredefinedQueryHandler( { } -bool PredefinedQueryHandler::customizeQueryParam(ContextPtr context, const std::string & key, const std::string & value) +bool PredefinedQueryHandler::customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) { if (receive_params.count(key)) { @@ -1003,7 +1002,7 @@ bool PredefinedQueryHandler::customizeQueryParam(ContextPtr context, const std:: return false; } -void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, ContextPtr context) +void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, ContextMutablePtr context) { /// If in the configuration file, the handler's header is regex and contains named capture group /// We will extract regex named capture groups as query parameters @@ -1039,7 +1038,7 @@ void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, Conte } } -std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) +std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) { if (unlikely(startsWith(request.getContentType(), "multipart/form-data"))) { diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index 4715949cb87..2149a7ca55c 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -33,11 +33,11 @@ public: void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; /// This method is called right before the query execution. - virtual void customizeContext(HTTPServerRequest & /* request */, ContextPtr /* context */) {} + virtual void customizeContext(HTTPServerRequest & /* request */, ContextMutablePtr /* context */) {} - virtual bool customizeQueryParam(ContextPtr context, const std::string & key, const std::string & value) = 0; + virtual bool customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) = 0; - virtual std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) = 0; + virtual std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) = 0; private: struct Output @@ -73,7 +73,7 @@ private: // The request_context and the request_credentials instances may outlive a single request/response loop. // This happens only when the authentication mechanism requires more than a single request/response exchange (e.g., SPNEGO). - ContextPtr request_context; + ContextMutablePtr request_context; std::unique_ptr request_credentials; // Returns true when the user successfully authenticated, @@ -82,14 +82,14 @@ private: // the request_context and request_credentials instances are preserved. // Throws an exception if authentication failed. bool authenticateUser( - ContextPtr context, + ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response); /// Also initializes 'used_output'. void processQuery( - ContextPtr context, + ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response, @@ -113,9 +113,9 @@ private: public: explicit DynamicQueryHandler(IServer & server_, const std::string & param_name_ = "query"); - std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) override; + std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) override; - bool customizeQueryParam(ContextPtr context, const std::string &key, const std::string &value) override; + bool customizeQueryParam(ContextMutablePtr context, const std::string &key, const std::string &value) override; }; class PredefinedQueryHandler : public HTTPHandler @@ -130,11 +130,11 @@ public: IServer & server_, const NameSet & receive_params_, const std::string & predefined_query_ , const CompiledRegexPtr & url_regex_, const std::unordered_map & header_name_with_regex_); - virtual void customizeContext(HTTPServerRequest & request, ContextPtr context) override; + virtual void customizeContext(HTTPServerRequest & request, ContextMutablePtr context) override; - std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) override; + std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) override; - bool customizeQueryParam(ContextPtr context, const std::string & key, const std::string & value) override; + bool customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) override; }; } diff --git a/src/Server/IServer.h b/src/Server/IServer.h index 80736fda3ea..c55b045d2a5 100644 --- a/src/Server/IServer.h +++ b/src/Server/IServer.h @@ -28,7 +28,7 @@ public: virtual Poco::Logger & logger() const = 0; /// Returns global application's context. - virtual ContextPtr context() const = 0; + virtual ContextMutablePtr context() const = 0; /// Returns true if shutdown signaled. virtual bool isCancelled() const = 0; diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index f5fb82b5bef..e681ad2e6f6 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -56,7 +56,7 @@ private: protected: Poco::Logger * log; - ContextPtr connection_context; + ContextMutablePtr connection_context; std::shared_ptr packet_endpoint; diff --git a/src/Server/PostgreSQLHandler.h b/src/Server/PostgreSQLHandler.h index 0f114d388fb..9aaad1d7aa7 100644 --- a/src/Server/PostgreSQLHandler.h +++ b/src/Server/PostgreSQLHandler.h @@ -37,7 +37,7 @@ private: Poco::Logger * log = &Poco::Logger::get("PostgreSQLHandler"); IServer & server; - ContextPtr connection_context; + ContextMutablePtr connection_context; bool ssl_enabled = false; Int32 connection_id = 0; Int32 secret_key = 0; diff --git a/src/Server/StaticRequestHandler.cpp b/src/Server/StaticRequestHandler.cpp index 169d6859b43..19b91ae9c42 100644 --- a/src/Server/StaticRequestHandler.cpp +++ b/src/Server/StaticRequestHandler.cpp @@ -14,14 +14,15 @@ #include -#include -#include #include #include #include #include +#include +namespace fs = std::filesystem; + namespace DB { @@ -137,11 +138,14 @@ void StaticRequestHandler::writeResponse(WriteBuffer & out) if (startsWith(response_expression, file_prefix)) { - const auto & user_files_absolute_path = Poco::Path(server.context()->getUserFilesPath()).makeAbsolute().makeDirectory().toString(); - const auto & file_name = response_expression.substr(file_prefix.size(), response_expression.size() - file_prefix.size()); + auto file_name = response_expression.substr(file_prefix.size(), response_expression.size() - file_prefix.size()); + if (file_name.starts_with('/')) + file_name = file_name.substr(1); - const auto & file_path = Poco::Path(user_files_absolute_path, file_name).makeAbsolute().toString(); - if (!Poco::File(file_path).exists()) + fs::path user_files_absolute_path = fs::canonical(fs::path(server.context()->getUserFilesPath())); + String file_path = fs::weakly_canonical(user_files_absolute_path / file_name); + + if (!fs::exists(file_path)) throw Exception("Invalid file name " + file_path + " for static HTTPHandler. ", ErrorCodes::INCORRECT_FILE_NAME); ReadBufferFromFile in(file_path); diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 8387ca5f254..086c1f7d5e5 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -119,7 +119,7 @@ public: void run() override; /// This method is called right before the query execution. - virtual void customizeContext(ContextPtr /*context*/) {} + virtual void customizeContext(ContextMutablePtr /*context*/) {} private: IServer & server; @@ -132,8 +132,8 @@ private: UInt64 client_version_patch = 0; UInt64 client_tcp_protocol_version = 0; - ContextPtr connection_context; - ContextPtr query_context; + ContextMutablePtr connection_context; + ContextMutablePtr query_context; size_t unknown_packet_in_send_data = 0; diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index f78002c977a..d0d995ba3a6 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -542,7 +542,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) else if (type == DROP_PROJECTION) { if (!partition && !clear) - metadata.projections.remove(projection_name); + metadata.projections.remove(projection_name, if_exists); } else if (type == MODIFY_TTL) { diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 55ee884646e..e8835132f8f 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -25,11 +25,9 @@ #include #include #include - #include #include - -#include +#include namespace CurrentMetrics @@ -39,6 +37,8 @@ namespace CurrentMetrics extern const Metric BrokenDistributedFilesToInsert; } +namespace fs = std::filesystem; + namespace DB { @@ -293,7 +293,7 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( , pool(std::move(pool_)) , disk(disk_) , relative_path(relative_path_) - , path(disk->getPath() + relative_path + '/') + , path(fs::path(disk->getPath()) / relative_path / "") , should_batch_inserts(storage.getContext()->getSettingsRef().distributed_directory_monitor_batch_inserts) , dir_fsync(storage.getDistributedSettingsRef().fsync_directories) , min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows) @@ -347,7 +347,7 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData() } auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - Poco::File(path).remove(true); + fs::remove_all(path); } @@ -490,16 +490,14 @@ std::map StorageDistributedDirectoryMonitor::getFiles() std::map files; size_t new_bytes_count = 0; - Poco::DirectoryIterator end; - for (Poco::DirectoryIterator it{path}; it != end; ++it) + fs::directory_iterator end; + for (fs::directory_iterator it{path}; it != end; ++it) { const auto & file_path_str = it->path(); - Poco::Path file_path{file_path_str}; - - if (!it->isDirectory() && startsWith(file_path.getExtension(), "bin")) + if (!it->is_directory() && startsWith(fs::path(file_path_str).extension(), ".bin")) { - files[parse(file_path.getBaseName())] = file_path_str; - new_bytes_count += Poco::File(file_path).getSize(); + files[parse(fs::path(file_path_str).stem())] = file_path_str; + new_bytes_count += fs::file_size(fs::path(file_path_str)); } } @@ -663,8 +661,7 @@ struct StorageDistributedDirectoryMonitor::Batch String tmp_file{parent.current_batch_file_path + ".tmp"}; auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path); - - if (Poco::File{tmp_file}.exists()) + if (fs::exists(tmp_file)) LOG_ERROR(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); { @@ -676,7 +673,7 @@ struct StorageDistributedDirectoryMonitor::Batch out.sync(); } - Poco::File{tmp_file}.renameTo(parent.current_batch_file_path); + fs::rename(tmp_file, parent.current_batch_file_path); } auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(parent.storage.getContext()->getSettingsRef()); auto connection = parent.pool->get(timeouts); @@ -757,7 +754,7 @@ struct StorageDistributedDirectoryMonitor::Batch total_bytes = 0; recovered = false; - Poco::File{parent.current_batch_file_path}.setSize(0); + fs::resize_file(parent.current_batch_file_path, 0); } void writeText(WriteBuffer & out) @@ -850,7 +847,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map { std::unordered_set file_indices_to_skip; - if (Poco::File{current_batch_file_path}.exists()) + if (fs::exists(current_batch_file_path)) { /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch. Batch batch(*this, files); @@ -951,8 +948,8 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map /// current_batch.txt will not exist if there was no send /// (this is the case when all batches that was pending has been marked as pending) - if (Poco::File{current_batch_file_path}.exists()) - Poco::File{current_batch_file_path}.remove(); + if (fs::exists(current_batch_file_path)) + fs::remove(current_batch_file_path); } } @@ -961,20 +958,18 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p const auto last_path_separator_pos = file_path.rfind('/'); const auto & base_path = file_path.substr(0, last_path_separator_pos + 1); const auto & file_name = file_path.substr(last_path_separator_pos + 1); - const auto & broken_path = base_path + "broken/"; - const auto & broken_file_path = broken_path + file_name; + const String & broken_path = fs::path(base_path) / "broken/"; + const String & broken_file_path = fs::path(broken_path) / file_name; - Poco::File{broken_path}.createDirectory(); + fs::create_directory(broken_path); auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path + "/broken/"); - - Poco::File file(file_path); + auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, fs::path(relative_path) / "broken/"); { std::lock_guard status_lock(status_mutex); - size_t file_size = file.getSize(); + size_t file_size = fs::file_size(file_path); --status.files_count; status.bytes_count -= file_size; @@ -985,15 +980,13 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p metric_broken_files.add(); } - file.renameTo(broken_file_path); - + fs::rename(file_path, broken_file_path); LOG_ERROR(log, "Renamed `{}` to `{}`", file_path, broken_file_path); } + void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_path) { - Poco::File file(file_path); - - size_t file_size = file.getSize(); + size_t file_size = fs::file_size(file_path); { std::lock_guard status_lock(status_mutex); @@ -1002,7 +995,7 @@ void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_pat status.bytes_count -= file_size; } - file.remove(); + fs::remove(file_path); } bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) @@ -1030,7 +1023,7 @@ void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_rela { std::lock_guard status_lock(status_mutex); relative_path = new_relative_path; - path = disk->getPath() + relative_path + '/'; + path = fs::path(disk->getPath()) / relative_path / ""; } current_batch_file_path = path + "current_batch.txt"; diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index a4aa2779771..ea694fb0cfe 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -33,11 +33,10 @@ #include #include -#include - #include #include #include +#include namespace CurrentMetrics @@ -50,10 +49,11 @@ namespace ProfileEvents extern const Event DistributedSyncInsertionTimeoutExceeded; } +namespace fs = std::filesystem; + namespace DB { - namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -660,10 +660,10 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: /// hardlinking to ensure the inode is not freed until we're done { const std::string path(disk_path + data_path + *it); - Poco::File(path).createDirectory(); - const std::string tmp_path(path + "/tmp/"); - Poco::File(tmp_path).createDirectory(); + + fs::create_directory(path); + fs::create_directory(tmp_path); const std::string file_name(toString(storage.file_names_increment.get()) + ".bin"); @@ -717,7 +717,7 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: } // Create hardlink here to reuse increment number - const std::string block_file_path(path + '/' + file_name); + const std::string block_file_path(fs::path(path) / file_name); createHardLink(first_file_tmp_path, block_file_path); auto dir_sync_guard = make_directory_sync_guard(*it); } @@ -726,18 +726,18 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: /// Make hardlinks for (; it != dir_names.end(); ++it) { - const std::string path(disk_path + data_path + *it); - Poco::File(path).createDirectory(); + const std::string path(fs::path(disk_path) / (data_path + *it)); + fs::create_directory(path); - const std::string block_file_path(path + '/' + toString(storage.file_names_increment.get()) + ".bin"); + const std::string block_file_path(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin")); createHardLink(first_file_tmp_path, block_file_path); auto dir_sync_guard = make_directory_sync_guard(*it); } - auto file_size = Poco::File(first_file_tmp_path).getSize(); + auto file_size = fs::file_size(first_file_tmp_path); /// remove the temporary file, enabling the OS to reclaim inode after all threads /// have removed their corresponding files - Poco::File(first_file_tmp_path).remove(); + fs::remove(first_file_tmp_path); /// Notify auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms; diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.h b/src/Storages/Distributed/DistributedBlockOutputStream.h index f574702f35f..a17e7d6565b 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.h +++ b/src/Storages/Distributed/DistributedBlockOutputStream.h @@ -84,7 +84,7 @@ private: /// Returns the number of blocks was written for each cluster node. Uses during exception handling. std::string getCurrentStateDescription(); - ContextPtr context; + ContextMutablePtr context; StorageDistributed & storage; StorageMetadataPtr metadata_snapshot; ASTPtr query_ast; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 1428e80f9af..e3fd287bad8 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -18,7 +18,6 @@ #include #include #include - #include #include #include @@ -26,7 +25,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -257,7 +258,7 @@ Strings LSWithRegexpMatching(const String & path_for_ls, const HDFSFSPtr & fs, c { if (re2::RE2::FullMatch(file_name, matcher)) { - Strings result_part = LSWithRegexpMatching(full_path + "/", fs, suffix_with_globs.substr(next_slash)); + Strings result_part = LSWithRegexpMatching(fs::path(full_path) / "", fs, suffix_with_globs.substr(next_slash)); /// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check. std::move(result_part.begin(), result_part.end(), std::back_inserter(result)); } diff --git a/src/Storages/Kafka/KafkaBlockOutputStream.h b/src/Storages/Kafka/KafkaBlockOutputStream.h index 715ed39b8d6..9f413ae527f 100644 --- a/src/Storages/Kafka/KafkaBlockOutputStream.h +++ b/src/Storages/Kafka/KafkaBlockOutputStream.h @@ -12,7 +12,7 @@ public: explicit KafkaBlockOutputStream( StorageKafka & storage_, const StorageMetadataPtr & metadata_snapshot_, - const std::shared_ptr & context_); + const std::shared_ptr & context_); Block getHeader() const override; @@ -25,7 +25,7 @@ public: private: StorageKafka & storage; StorageMetadataPtr metadata_snapshot; - const std::shared_ptr context; + const std::shared_ptr context; ProducerBufferPtr buffer; BlockOutputStreamPtr child; }; diff --git a/src/Storages/LiveView/StorageLiveView.h b/src/Storages/LiveView/StorageLiveView.h index df09316f333..e787b7bf939 100644 --- a/src/Storages/LiveView/StorageLiveView.h +++ b/src/Storages/LiveView/StorageLiveView.h @@ -191,7 +191,7 @@ private: ASTPtr inner_query; /// stored query : SELECT * FROM ( SELECT a FROM A) ASTPtr inner_subquery; /// stored query's innermost subquery if any ASTPtr inner_blocks_query; /// query over the mergeable blocks to produce final result - ContextPtr live_view_context; + ContextMutablePtr live_view_context; Poco::Logger * log; diff --git a/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp b/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp index 7294b82f10d..69369cbc1a3 100644 --- a/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp +++ b/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes namespace { - void executeDropQuery(const StorageID & storage_id, ContextPtr context) + void executeDropQuery(const StorageID & storage_id, ContextMutablePtr context) { if (!DatabaseCatalog::instance().isTableExist(storage_id, context)) return; @@ -42,7 +42,7 @@ namespace std::unique_ptr TemporaryLiveViewCleaner::the_instance; -void TemporaryLiveViewCleaner::init(ContextPtr global_context_) +void TemporaryLiveViewCleaner::init(ContextMutablePtr global_context_) { if (the_instance) throw Exception("TemporaryLiveViewCleaner already initialized", ErrorCodes::LOGICAL_ERROR); @@ -63,7 +63,7 @@ void TemporaryLiveViewCleaner::shutdown() the_instance.reset(); } -TemporaryLiveViewCleaner::TemporaryLiveViewCleaner(ContextPtr global_context_) : WithContext(global_context_) +TemporaryLiveViewCleaner::TemporaryLiveViewCleaner(ContextMutablePtr global_context_) : WithMutableContext(global_context_) { } diff --git a/src/Storages/LiveView/TemporaryLiveViewCleaner.h b/src/Storages/LiveView/TemporaryLiveViewCleaner.h index 9b31bf9c999..3fe0079a46f 100644 --- a/src/Storages/LiveView/TemporaryLiveViewCleaner.h +++ b/src/Storages/LiveView/TemporaryLiveViewCleaner.h @@ -14,7 +14,7 @@ struct StorageID; /// This class removes temporary live views in the background thread when it's possible. /// There should only a single instance of this class. -class TemporaryLiveViewCleaner : WithContext +class TemporaryLiveViewCleaner : WithMutableContext { public: static TemporaryLiveViewCleaner & instance() { return *the_instance; } @@ -23,7 +23,7 @@ public: void addView(const std::shared_ptr & view); /// Should be called once. - static void init(ContextPtr global_context_); + static void init(ContextMutablePtr global_context_); static void shutdown(); void startup(); @@ -31,7 +31,7 @@ public: private: friend std::unique_ptr::deleter_type; - TemporaryLiveViewCleaner(ContextPtr global_context_); + TemporaryLiveViewCleaner(ContextMutablePtr global_context_); ~TemporaryLiveViewCleaner(); void backgroundThreadFunc(); diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 5ccb2ca8c27..529fc9f54b8 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -13,11 +13,11 @@ #include #include #include - -#include #include +namespace fs = std::filesystem; + namespace CurrentMetrics { extern const Metric ReplicatedSend; @@ -289,7 +289,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( { String file_name = it.first; - String path = part->getFullRelativePath() + file_name; + String path = fs::path(part->getFullRelativePath()) / file_name; UInt64 size = disk->getFileSize(path); @@ -339,15 +339,15 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB { String file_name = it.first; - String metadata_file = disk->getPath() + part->getFullRelativePath() + file_name; + String metadata_file = fs::path(disk->getPath()) / part->getFullRelativePath() / file_name; - Poco::File metadata(metadata_file); + fs::path metadata(metadata_file); - if (!metadata.exists()) + if (!fs::exists(metadata)) throw Exception("S3 metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA); - if (!metadata.isFile()) + if (!fs::is_regular_file(metadata)) throw Exception("S3 metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA); - UInt64 file_size = metadata.getSize(); + UInt64 file_size = fs::file_size(metadata); writeStringBinary(it.first, out); writeBinary(file_size, out); @@ -570,7 +570,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( readUUIDText(part_uuid, in); auto storage_id = data.getStorageID(); - String new_part_path = part_type == "InMemory" ? "memory" : data.getFullPathOnDisk(reservation->getDisk()) + part_name + "/"; + String new_part_path = part_type == "InMemory" ? "memory" : fs::path(data.getFullPathOnDisk(reservation->getDisk())) / part_name / ""; auto entry = data.getContext()->getReplicatedFetchList().insert( storage_id.getDatabaseName(), storage_id.getTableName(), part_info.partition_id, part_name, new_part_path, @@ -681,13 +681,13 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( /// File must be inside "absolute_part_path" directory. /// Otherwise malicious ClickHouse replica may force us to write to arbitrary path. - String absolute_file_path = Poco::Path(part_download_path + file_name).absolute().toString(); - if (!startsWith(absolute_file_path, Poco::Path(part_download_path).absolute().toString())) + String absolute_file_path = fs::weakly_canonical(fs::path(part_download_path) / file_name); + if (!startsWith(absolute_file_path, fs::weakly_canonical(part_download_path).string())) throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + part_download_path + ")." " This may happen if we are trying to download part from malicious replica or logical error.", ErrorCodes::INSECURE_PATH); - auto file_out = disk->writeFile(part_download_path + file_name); + auto file_out = disk->writeFile(fs::path(part_download_path) / file_name); HashingWriteBuffer hashing_out(*file_out); copyData(in, hashing_out, file_size, blocker.getCounter()); @@ -704,7 +704,7 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( readPODBinary(expected_hash, in); if (expected_hash != hashing_out.getHash()) - throw Exception("Checksum mismatch for file " + fullPath(disk, part_download_path + file_name) + " transferred from " + replica_path, + throw Exception("Checksum mismatch for file " + fullPath(disk, (fs::path(part_download_path) / file_name).string()) + " transferred from " + replica_path, ErrorCodes::CHECKSUM_DOESNT_MATCH); if (file_name != "checksums.txt" && @@ -811,7 +811,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; String part_relative_path = String(to_detached ? "detached/" : "") + tmp_prefix + part_name; - String part_download_path = data.getRelativeDataPath() + part_relative_path + "/"; + String part_download_path = fs::path(data.getRelativeDataPath()) / part_relative_path / ""; if (disk->exists(part_download_path)) throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); @@ -833,7 +833,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( readStringBinary(file_name, in); readBinary(file_size, in); - String data_path = part_download_path + file_name; + String data_path = fs::path(part_download_path) / file_name; String metadata_file = fullPath(disk, data_path); { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 8a35ce3dcb8..703cf32f743 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -69,7 +69,7 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Dis hyperrectangle.reserve(minmax_idx_size); for (size_t i = 0; i < minmax_idx_size; ++i) { - String file_name = part_path + "minmax_" + escapeForFileName(minmax_column_names[i]) + ".idx"; + String file_name = fs::path(part_path) / ("minmax_" + escapeForFileName(minmax_column_names[i]) + ".idx"); auto file = openForReading(disk_, file_name); auto serialization = minmax_column_types[i]->getDefaultSerialization(); @@ -111,7 +111,7 @@ void IMergeTreeDataPart::MinMaxIndex::store( String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; auto serialization = data_types.at(i)->getDefaultSerialization(); - auto out = disk_->writeFile(part_path + file_name); + auto out = disk_->writeFile(fs::path(part_path) / file_name); HashingWriteBuffer out_hashing(*out); serialization->serializeBinary(hyperrectangle[i].left, out_hashing); serialization->serializeBinary(hyperrectangle[i].right, out_hashing); @@ -560,7 +560,7 @@ String IMergeTreeDataPart::getFullPath() const if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return storage.getFullPathOnDisk(volume->getDisk()) + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + return fs::path(storage.getFullPathOnDisk(volume->getDisk())) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; } String IMergeTreeDataPart::getFullRelativePath() const @@ -568,7 +568,7 @@ String IMergeTreeDataPart::getFullRelativePath() const if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return storage.relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + return fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; } void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) @@ -643,7 +643,7 @@ void IMergeTreeDataPart::loadIndex() loaded_index[i]->reserve(index_granularity.getMarksCount()); } - String index_path = getFullRelativePath() + "primary.idx"; + String index_path = fs::path(getFullRelativePath()) / "primary.idx"; auto index_file = openForReading(volume->getDisk(), index_path); size_t marks_count = index_granularity.getMarksCount(); @@ -678,7 +678,7 @@ NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const return {}; NameSet result = {"checksums.txt", "columns.txt"}; - String default_codec_path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME; + String default_codec_path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; if (volume->getDisk()->exists(default_codec_path)) result.emplace(DEFAULT_COMPRESSION_CODEC_FILE_NAME); @@ -695,7 +695,7 @@ void IMergeTreeDataPart::loadDefaultCompressionCodec() return; } - String path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME; + String path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; if (!volume->getDisk()->exists(path)) { default_codec = detectDefaultCompressionCodec(); @@ -756,7 +756,7 @@ CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const { if (path_to_data_file.empty()) { - String candidate_path = getFullRelativePath() + ISerialization::getFileNameForStream(part_column, substream_path) + ".bin"; + String candidate_path = fs::path(getFullRelativePath()) / (ISerialization::getFileNameForStream(part_column, substream_path) + ".bin"); /// We can have existing, but empty .bin files. Example: LowCardinality(Nullable(...)) columns and column_name.dict.null.bin file. if (volume->getDisk()->exists(candidate_path) && volume->getDisk()->getFileSize(candidate_path) != 0) @@ -822,7 +822,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() void IMergeTreeDataPart::loadChecksums(bool require) { - const String path = getFullRelativePath() + "checksums.txt"; + const String path = fs::path(getFullRelativePath()) / "checksums.txt"; if (volume->getDisk()->exists(path)) { @@ -847,11 +847,11 @@ void IMergeTreeDataPart::loadChecksums(bool require) checksums = checkDataPart(shared_from_this(), false); { - auto out = volume->getDisk()->writeFile(getFullRelativePath() + "checksums.txt.tmp", 4096); + auto out = volume->getDisk()->writeFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", 4096); checksums.write(*out); } - volume->getDisk()->moveFile(getFullRelativePath() + "checksums.txt.tmp", getFullRelativePath() + "checksums.txt"); + volume->getDisk()->moveFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", fs::path(getFullRelativePath()) / "checksums.txt"); bytes_on_disk = checksums.getTotalSizeOnDisk(); } @@ -859,7 +859,7 @@ void IMergeTreeDataPart::loadChecksums(bool require) void IMergeTreeDataPart::loadRowsCount() { - String path = getFullRelativePath() + "count.txt"; + String path = fs::path(getFullRelativePath()) / "count.txt"; if (index_granularity.empty()) { rows_count = 0; @@ -960,7 +960,7 @@ void IMergeTreeDataPart::loadRowsCount() void IMergeTreeDataPart::loadTTLInfos() { - String path = getFullRelativePath() + "ttl.txt"; + String path = fs::path(getFullRelativePath()) / "ttl.txt"; if (volume->getDisk()->exists(path)) { auto in = openForReading(volume->getDisk(), path); @@ -987,7 +987,7 @@ void IMergeTreeDataPart::loadTTLInfos() void IMergeTreeDataPart::loadUUID() { - String path = getFullRelativePath() + UUID_FILE_NAME; + String path = fs::path(getFullRelativePath()) / UUID_FILE_NAME; if (volume->getDisk()->exists(path)) { @@ -1000,7 +1000,7 @@ void IMergeTreeDataPart::loadUUID() void IMergeTreeDataPart::loadColumns(bool require) { - String path = getFullRelativePath() + "columns.txt"; + String path = fs::path(getFullRelativePath()) / "columns.txt"; auto metadata_snapshot = storage.getInMemoryMetadataPtr(); if (parent_part) metadata_snapshot = metadata_snapshot->projections.get(name).metadata; @@ -1015,7 +1015,7 @@ void IMergeTreeDataPart::loadColumns(bool require) /// If there is no file with a list of columns, write it down. for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAllPhysical()) - if (volume->getDisk()->exists(getFullRelativePath() + getFileNameForColumn(column) + ".bin")) + if (volume->getDisk()->exists(fs::path(getFullRelativePath()) / (getFileNameForColumn(column) + ".bin"))) loaded_columns.push_back(column); if (columns.empty()) @@ -1053,7 +1053,7 @@ UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DiskPtr & disk_, const disk_->listFiles(from, files); UInt64 res = 0; for (const auto & file : files) - res += calculateTotalSizeOnDisk(disk_, from + "/" + file); + res += calculateTotalSizeOnDisk(disk_, fs::path(from) / file); return res; } @@ -1063,7 +1063,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ assertOnDisk(); String from = getFullRelativePath(); - String to = storage.relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + new_relative_path + "/"; + String to = fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / new_relative_path / ""; if (!volume->getDisk()->exists(from)) throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); @@ -1124,8 +1124,8 @@ void IMergeTreeDataPart::remove(bool keep_s3) const * And a race condition can happen that will lead to "File not found" error here. */ - String from = storage.relative_data_path + relative_path; - String to = storage.relative_data_path + "delete_tmp_" + name; + fs::path from = fs::path(storage.relative_data_path) / relative_path; + fs::path to = fs::path(storage.relative_data_path) / ("delete_tmp_" + name); // TODO directory delete_tmp_ is never removed if server crashes before returning from this function auto disk = volume->getDisk(); @@ -1134,7 +1134,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_WARNING(storage.log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it.", fullPath(disk, to)); try { - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } catch (...) { @@ -1147,11 +1147,14 @@ void IMergeTreeDataPart::remove(bool keep_s3) const { disk->moveDirectory(from, to); } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - LOG_ERROR(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(disk, to)); - - return; + if (e.code() == std::errc::no_such_file_or_directory) + { + LOG_ERROR(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(disk, to)); + return; + } + throw; } // Record existing projection directories so we don't remove them twice @@ -1166,7 +1169,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const if (checksums.empty()) { /// If the part is not completely written, we cannot use fast path by listing files. - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } else { @@ -1181,17 +1184,17 @@ void IMergeTreeDataPart::remove(bool keep_s3) const for (const auto & [file, _] : checksums.files) { if (projection_directories.find(file) == projection_directories.end()) - disk->removeSharedFile(to + "/" + file, keep_s3); + disk->removeSharedFile(fs::path(to) / file, keep_s3); } #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(to + "/" + file, keep_s3); + disk->removeSharedFile(fs::path(to) / file, keep_s3); - disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); - disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); + disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); + disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); disk->removeDirectory(to); } @@ -1201,7 +1204,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false)); - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } } } @@ -1268,7 +1271,7 @@ String IMergeTreeDataPart::getRelativePathForPrefix(const String & prefix) const { res = (prefix.empty() ? "" : prefix + "_") + name + (try_no ? "_try" + DB::toString(try_no) : ""); - if (!volume->getDisk()->exists(getFullRelativePath() + res)) + if (!volume->getDisk()->exists(fs::path(getFullRelativePath()) / res)) return res; LOG_WARNING(storage.log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res); @@ -1291,11 +1294,11 @@ void IMergeTreeDataPart::renameToDetached(const String & prefix) const void IMergeTreeDataPart::makeCloneInDetached(const String & prefix, const StorageMetadataPtr & /*metadata_snapshot*/) const { - String destination_path = storage.relative_data_path + getRelativePathForDetachedPart(prefix); + String destination_path = fs::path(storage.relative_data_path) / getRelativePathForDetachedPart(prefix); /// Backup is not recursive (max_level is 0), so do not copy inner directories localBackup(volume->getDisk(), getFullRelativePath(), destination_path, 0); - volume->getDisk()->removeFileIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME); + volume->getDisk()->removeFileIfExists(fs::path(destination_path) / DELETE_ON_DESTROY_MARKER_FILE_NAME); } void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & directory_name) const @@ -1307,16 +1310,16 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di if (directory_name.empty()) throw Exception("Can not clone data part " + name + " to empty directory.", ErrorCodes::LOGICAL_ERROR); - String path_to_clone = storage.relative_data_path + directory_name + '/'; + String path_to_clone = fs::path(storage.relative_data_path) / directory_name / ""; - if (disk->exists(path_to_clone + relative_path)) + if (disk->exists(fs::path(path_to_clone) / relative_path)) { LOG_WARNING(storage.log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); - disk->removeRecursive(path_to_clone + relative_path + '/'); + disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone); - volume->getDisk()->removeFileIfExists(path_to_clone + '/' + DELETE_ON_DESTROY_MARKER_FILE_NAME); + volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / DELETE_ON_DESTROY_MARKER_FILE_NAME); } void IMergeTreeDataPart::checkConsistencyBase() const @@ -1370,19 +1373,19 @@ void IMergeTreeDataPart::checkConsistencyBase() const /// Check that the primary key index is not empty. if (!pk.column_names.empty()) - check_file_not_empty(volume->getDisk(), path + "primary.idx"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - check_file_not_empty(volume->getDisk(), path + "count.txt"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "count.txt"); if (metadata_snapshot->hasPartitionKey()) - check_file_not_empty(volume->getDisk(), path + "partition.dat"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "partition.dat"); if (!parent_part) { for (const String & col_name : storage.getMinMaxColumnsNames(partition_key)) - check_file_not_empty(volume->getDisk(), path + "minmax_" + escapeForFileName(col_name) + ".idx"); + check_file_not_empty(volume->getDisk(), fs::path(path) / ("minmax_" + escapeForFileName(col_name) + ".idx")); } } } @@ -1477,7 +1480,7 @@ String IMergeTreeDataPart::getUniqueId() const auto disk = volume->getDisk(); if (disk->getType() == DB::DiskType::Type::S3) - id = disk->getUniqueId(getFullRelativePath() + "checksums.txt"); + id = disk->getUniqueId(fs::path(getFullRelativePath()) / "checksums.txt"); if (id.empty()) throw Exception("Can't get unique S3 object", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index a8a49680dd7..53640b41507 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -17,8 +17,6 @@ #include #include -#include - #include namespace zkutil diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 52d3e7ca9ab..14187564536 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index 7cdfd8e566d..2810385e9e5 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -83,7 +83,7 @@ private: void createNode() { shutdown_called = false; - node = EphemeralNodeHolder::createSequential(path + "/leader_election-", zookeeper, identifier); + node = EphemeralNodeHolder::createSequential(fs::path(path) / "leader_election-", zookeeper, identifier); std::string node_path = node->getPath(); node_name = node_path.substr(node_path.find_last_of('/') + 1); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0ed3f89b2c3..acf362e5b1f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -52,8 +52,6 @@ #include #include -#include - #include #include @@ -67,8 +65,11 @@ #include #include #include +#include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event RejectedInserts; @@ -136,7 +137,7 @@ MergeTreeData::MergeTreeData( const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr storage_settings_, @@ -144,7 +145,7 @@ MergeTreeData::MergeTreeData( bool attach, BrokenPartCallback broken_part_callback_) : IStorage(table_id_) - , WithContext(context_->getGlobalContext()) + , WithMutableContext(context_->getGlobalContext()) , merging_params(merging_params_) , require_part_metadata(require_part_metadata_) , relative_data_path(relative_data_path_) @@ -210,8 +211,8 @@ MergeTreeData::MergeTreeData( for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { disk->createDirectories(path); - disk->createDirectories(path + MergeTreeData::DETACHED_DIR_NAME); - auto current_version_file_path = path + MergeTreeData::FORMAT_VERSION_FILE_NAME; + disk->createDirectories(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); + String current_version_file_path = fs::path(path) / MergeTreeData::FORMAT_VERSION_FILE_NAME; if (disk->exists(current_version_file_path)) { if (!version_file.first.empty()) @@ -225,7 +226,7 @@ MergeTreeData::MergeTreeData( /// If not choose any if (version_file.first.empty()) - version_file = {relative_data_path + MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()}; + version_file = {fs::path(relative_data_path) / MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()}; bool version_file_exists = version_file.second->exists(version_file.first); @@ -927,8 +928,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) auto part = createPart(part_name, part_info, single_disk_volume, part_name); bool broken = false; - String part_path = relative_data_path + "/" + part_name; - String marker_path = part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; + String part_path = fs::path(relative_data_path) / part_name; + String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; if (part_disk_ptr->exists(marker_path)) { LOG_WARNING(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name); @@ -1016,7 +1017,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) else has_adaptive_parts.store(true, std::memory_order_relaxed); - part->modification_time = part_disk_ptr->getLastModified(relative_data_path + part_name).epochTime(); + part->modification_time = part_disk_ptr->getLastModified(fs::path(relative_data_path) / part_name).epochTime(); /// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later part->setState(DataPartState::Committed); @@ -1160,9 +1161,14 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life disk->removeRecursive(it->path()); } } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - /// If the file is already deleted, do nothing. + if (e.code() == std::errc::no_such_file_or_directory) + { + /// If the file is already deleted, do nothing. + } + else + throw; } } } @@ -1387,11 +1393,7 @@ void MergeTreeData::clearEmptyParts() for (const auto & part : parts) { if (part->rows_count == 0) - { - ASTPtr literal = std::make_shared(part->name); - /// If another replica has already started drop, it's ok, no need to throw. - dropPartition(literal, /* detach = */ false, /*drop_part = */ true, getContext(), /* throw_if_noop = */ false); - } + dropPartNoWaitNoThrow(part->name); } } @@ -1448,10 +1450,15 @@ void MergeTreeData::dropAllData() { disk->removeRecursive(path); } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - /// If the file is already deleted, log the error message and do nothing. - tryLogCurrentException(__PRETTY_FUNCTION__); + if (e.code() == std::errc::no_such_file_or_directory) + { + /// If the file is already deleted, log the error message and do nothing. + tryLogCurrentException(__PRETTY_FUNCTION__); + } + else + throw; } } @@ -1474,8 +1481,8 @@ void MergeTreeData::dropIfEmpty() for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { /// Non recursive, exception is thrown if there are more files. - disk->removeFileIfExists(path + MergeTreeData::FORMAT_VERSION_FILE_NAME); - disk->removeDirectory(path + MergeTreeData::DETACHED_DIR_NAME); + disk->removeFileIfExists(fs::path(path) / MergeTreeData::FORMAT_VERSION_FILE_NAME); + disk->removeDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); disk->removeDirectory(path); } } @@ -1925,7 +1932,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part) const { MergeTreeDataPartType type; - auto full_path = relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + auto full_path = fs::path(relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(volume->getDisk(), full_path); if (mrk_ext) @@ -1978,7 +1985,7 @@ void MergeTreeData::changeSettings( { auto disk = new_storage_policy->getDiskByName(disk_name); disk->createDirectories(relative_data_path); - disk->createDirectories(relative_data_path + MergeTreeData::DETACHED_DIR_NAME); + disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME); } /// FIXME how would that be done while reloading configuration??? @@ -2007,7 +2014,7 @@ void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const old_and_new_names.push_back({old_name, new_name}); for (const auto & [path, disk] : storage.getRelativeDataPathsWithDisks()) { - for (auto it = disk->iterateDirectory(path + source_dir); it->isValid(); it->next()) + for (auto it = disk->iterateDirectory(fs::path(path) / source_dir); it->isValid(); it->next()) { if (it->name() == old_name) { @@ -2029,8 +2036,8 @@ void MergeTreeData::PartsTemporaryRename::tryRenameAll() if (old_name.empty() || new_name.empty()) throw DB::Exception("Empty part name. Most likely it's a bug.", ErrorCodes::INCORRECT_FILE_NAME); const auto & [path, disk] = old_part_name_to_path_and_disk[old_name]; - const auto full_path = path + source_dir; /// for old_name - disk->moveFile(full_path + old_name, full_path + new_name); + const auto full_path = fs::path(path) / source_dir; /// for old_name + disk->moveFile(fs::path(full_path) / old_name, fs::path(full_path) / new_name); } catch (...) { @@ -2054,8 +2061,8 @@ MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename() try { const auto & [path, disk] = old_part_name_to_path_and_disk[old_name]; - const auto full_path = path + source_dir; /// for old_name - disk->moveFile(full_path + new_name, full_path + old_name); + const String full_path = fs::path(path) / source_dir; /// for old_name + disk->moveFile(fs::path(full_path) / new_name, fs::path(full_path) / old_name); } catch (...) { @@ -2752,7 +2759,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) addPartContributionToDataVolume(part_copy); auto disk = original_active_part->volume->getDisk(); - String marker_path = original_active_part->getFullRelativePath() + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; + String marker_path = fs::path(original_active_part->getFullRelativePath()) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; try { disk->createFile(marker_path); @@ -2904,9 +2911,8 @@ void MergeTreeData::checkPartitionCanBeDropped(const ASTPtr & partition) getContext()->checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, partition_size); } -void MergeTreeData::checkPartCanBeDropped(const ASTPtr & part_ast) +void MergeTreeData::checkPartCanBeDropped(const String & part_name) { - String part_name = part_ast->as().value.safeGet(); auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Committed}); if (!part) throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in committed state", part_name); @@ -3041,12 +3047,20 @@ Pipe MergeTreeData::alterPartition( switch (command.type) { case PartitionCommand::DROP_PARTITION: + { if (command.part) - checkPartCanBeDropped(command.partition); + { + auto part_name = command.partition->as().value.safeGet(); + checkPartCanBeDropped(part_name); + dropPart(part_name, command.detach, query_context); + } else + { checkPartitionCanBeDropped(command.partition); - dropPartition(command.partition, command.detach, command.part, query_context); - break; + dropPartition(command.partition, command.detach, query_context); + } + } + break; case PartitionCommand::DROP_DETACHED_PARTITION: dropDetached(command.partition, command.part, query_context); @@ -3315,7 +3329,7 @@ MergeTreeData::getDetachedParts() const for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { - for (auto it = disk->iterateDirectory(path + MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next()) + for (auto it = disk->iterateDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next()) { res.emplace_back(); auto & part = res.back(); @@ -3370,7 +3384,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr for (auto & [old_name, new_name] : renamed_parts.old_and_new_names) { const auto & [path, disk] = renamed_parts.old_part_name_to_path_and_disk[old_name]; - disk->removeRecursive(path + "detached/" + new_name + "/"); + disk->removeRecursive(fs::path(path) / "detached" / new_name / ""); LOG_DEBUG(log, "Dropped detached part {}", old_name); old_name.clear(); } @@ -3431,8 +3445,8 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const if (!containing_part.empty() && containing_part != name) // TODO maybe use PartsTemporaryRename here? - disk->moveDirectory(relative_data_path + source_dir + name, - relative_data_path + source_dir + "inactive_" + name); + disk->moveDirectory(fs::path(relative_data_path) / source_dir / name, + fs::path(relative_data_path) / source_dir / ("inactive_" + name)); else renamed_parts.addPart(name, "attaching_" + name); } @@ -3834,7 +3848,7 @@ static void selectBestProjection( const Names & required_columns, ProjectionCandidate & candidate, ContextPtr query_context, - const PartitionIdToMaxBlock * max_added_blocks, + std::shared_ptr max_added_blocks, const Settings & settings, const MergeTreeData::DataPartsVector & parts, ProjectionCandidate *& selected_candidate, @@ -3855,21 +3869,16 @@ static void selectBestProjection( if (projection_parts.empty()) return; - candidate.merge_tree_data_select_base_cache = std::make_unique(); - candidate.merge_tree_data_select_projection_cache = std::make_unique(); - reader.readFromParts( + auto sum_marks = reader.estimateNumMarksToRead( projection_parts, candidate.required_columns, metadata_snapshot, candidate.desc->metadata, query_info, // TODO syntax_analysis_result set in index query_context, - 0, // max_block_size is unused when getting cache settings.max_threads, - max_added_blocks, - candidate.merge_tree_data_select_projection_cache.get()); + max_added_blocks); - size_t sum_marks = candidate.merge_tree_data_select_projection_cache->sum_marks; if (normal_parts.empty()) { // All parts are projection parts which allows us to use in_order_optimization. @@ -3878,18 +3887,15 @@ static void selectBestProjection( } else { - reader.readFromParts( + sum_marks += reader.estimateNumMarksToRead( normal_parts, required_columns, metadata_snapshot, metadata_snapshot, query_info, // TODO syntax_analysis_result set in index query_context, - 0, // max_block_size is unused when getting cache settings.max_threads, - max_added_blocks, - candidate.merge_tree_data_select_base_cache.get()); - sum_marks += candidate.merge_tree_data_select_base_cache->sum_marks; + max_added_blocks); } // We choose the projection with least sum_marks to read. @@ -4117,32 +4123,21 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( if (!candidates.empty()) { // First build a MergeTreeDataSelectCache to check if a projection is indeed better than base - query_info.merge_tree_data_select_cache = std::make_unique(); + // query_info.merge_tree_data_select_cache = std::make_unique(); - std::unique_ptr max_added_blocks; + std::shared_ptr max_added_blocks; if (settings.select_sequential_consistency) { if (const StorageReplicatedMergeTree * replicated = dynamic_cast(this)) - max_added_blocks = std::make_unique(replicated->getMaxAddedBlocks()); + max_added_blocks = std::make_shared(replicated->getMaxAddedBlocks()); } auto parts = getDataPartsVector(); MergeTreeDataSelectExecutor reader(*this); - reader.readFromParts( - parts, - analysis_result.required_columns, - metadata_snapshot, - metadata_snapshot, - query_info, // TODO syntax_analysis_result set in index - query_context, - 0, // max_block_size is unused when getting cache - settings.max_threads, - max_added_blocks.get(), - query_info.merge_tree_data_select_cache.get()); - // Add 1 to base sum_marks so that we prefer projections even when they have equal number of marks to read. - size_t min_sum_marks = query_info.merge_tree_data_select_cache->sum_marks + 1; ProjectionCandidate * selected_candidate = nullptr; + size_t min_sum_marks = std::numeric_limits::max(); + bool has_ordinary_projection = false; /// Favor aggregate projections for (auto & candidate : candidates) { @@ -4155,17 +4150,33 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( analysis_result.required_columns, candidate, query_context, - max_added_blocks.get(), + max_added_blocks, settings, parts, selected_candidate, min_sum_marks); } + else + has_ordinary_projection = true; } /// Select the best normal projection if no aggregate projection is available - if (!selected_candidate) + if (!selected_candidate && has_ordinary_projection) { + min_sum_marks = reader.estimateNumMarksToRead( + parts, + analysis_result.required_columns, + metadata_snapshot, + metadata_snapshot, + query_info, // TODO syntax_analysis_result set in index + query_context, + settings.max_threads, + max_added_blocks); + + // Add 1 to base sum_marks so that we prefer projections even when they have equal number of marks to read. + // NOTE: It is not clear if we need it. E.g. projections do not support skip index for now. + min_sum_marks += 1; + for (auto & candidate : candidates) { if (candidate.desc->type == ProjectionDescription::Type::Normal) @@ -4177,7 +4188,7 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( analysis_result.required_columns, candidate, query_context, - max_added_blocks.get(), + max_added_blocks, settings, parts, selected_candidate, @@ -4294,12 +4305,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( const auto & src_relative_data_path = src_part_in_memory->storage.relative_data_path; auto flushed_part_path = src_part_in_memory->getRelativePathForPrefix(tmp_part_prefix); src_part_in_memory->flushToDisk(src_relative_data_path, flushed_part_path, metadata_snapshot); - src_part_path = src_relative_data_path + flushed_part_path + "/"; + src_part_path = fs::path(src_relative_data_path) / flushed_part_path / ""; } LOG_DEBUG(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path)); localBackup(disk, src_part_path, dst_part_path); - disk->removeFileIfExists(dst_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + disk->removeFileIfExists(fs::path(dst_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); auto dst_data_part = createPart(dst_part_name, dst_part_info, single_disk_volume, tmp_dst_part_name); @@ -4411,10 +4422,10 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( const String & with_name, ContextPtr local_context) { - String clickhouse_path = Poco::Path(local_context->getPath()).makeAbsolute().toString(); - String default_shadow_path = clickhouse_path + "shadow/"; - Poco::File(default_shadow_path).createDirectories(); - auto increment = Increment(default_shadow_path + "increment.txt").get(true); + String clickhouse_path = fs::canonical(local_context->getPath()); + String default_shadow_path = fs::path(clickhouse_path) / "shadow/"; + fs::create_directories(default_shadow_path); + auto increment = Increment(fs::path(default_shadow_path) / "increment.txt").get(true); const String shadow_path = "shadow/"; @@ -4422,7 +4433,7 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( const auto data_parts = getDataParts(); String backup_name = (!with_name.empty() ? escapeForFileName(with_name) : toString(increment)); - String backup_path = shadow_path + backup_name + "/"; + String backup_path = fs::path(shadow_path) / backup_name / ""; for (const auto & disk : getStoragePolicy()->getDisks()) disk->onFreeze(backup_path); @@ -4439,20 +4450,20 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( part->volume->getDisk()->createDirectories(backup_path); - String backup_part_path = backup_path + relative_data_path + part->relative_path; + String backup_part_path = fs::path(backup_path) / relative_data_path / part->relative_path; if (auto part_in_memory = asInMemoryPart(part)) - part_in_memory->flushToDisk(backup_path + relative_data_path, part->relative_path, metadata_snapshot); + part_in_memory->flushToDisk(fs::path(backup_path) / relative_data_path, part->relative_path, metadata_snapshot); else localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path); - part->volume->getDisk()->removeFileIfExists(backup_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + part->volume->getDisk()->removeFileIfExists(fs::path(backup_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); part->is_frozen.store(true, std::memory_order_relaxed); result.push_back(PartitionCommandResultInfo{ .partition_id = part->info.partition_id, .part_name = part->name, - .backup_path = part->volume->getDisk()->getPath() + backup_path, - .part_backup_path = part->volume->getDisk()->getPath() + backup_part_path, + .backup_path = fs::path(part->volume->getDisk()->getPath()) / backup_path, + .part_backup_path = fs::path(part->volume->getDisk()->getPath()) / backup_part_path, .backup_name = backup_name, }); ++parts_processed; @@ -4481,7 +4492,7 @@ PartitionCommandsResultInfo MergeTreeData::unfreezeAll( PartitionCommandsResultInfo MergeTreeData::unfreezePartitionsByMatcher(MatcherFn matcher, const String & backup_name, ContextPtr) { - auto backup_path = std::filesystem::path("shadow") / escapeForFileName(backup_name) / relative_data_path; + auto backup_path = fs::path("shadow") / escapeForFileName(backup_name) / relative_data_path; LOG_DEBUG(log, "Unfreezing parts by path {}", backup_path.generic_string()); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index a80d7eae139..4f33aa30bdc 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -113,7 +113,7 @@ namespace ErrorCodes /// - MergeTreeDataWriter /// - MergeTreeDataMergerMutator -class MergeTreeData : public IStorage, public WithContext +class MergeTreeData : public IStorage, public WithMutableContext { public: /// Function to call if the part is suspected to contain corrupt data. @@ -353,7 +353,7 @@ public: MergeTreeData(const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, @@ -613,7 +613,7 @@ public: void checkPartitionCanBeDropped(const ASTPtr & partition) override; - void checkPartCanBeDropped(const ASTPtr & part); + void checkPartCanBeDropped(const String & part_name); Pipe alterPartition( const StorageMetadataPtr & metadata_snapshot, @@ -993,7 +993,11 @@ protected: // Partition helpers bool canReplacePartition(const DataPartPtr & src_part) const; - virtual void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr context, bool throw_if_noop = true) = 0; + /// Tries to drop part in background without any waits or throwing exceptions in case of errors. + virtual void dropPartNoWaitNoThrow(const String & part_name) = 0; + + virtual void dropPart(const String & part_name, bool detach, ContextPtr context) = 0; + virtual void dropPartition(const ASTPtr & partition, bool detach, ContextPtr context) = 0; virtual PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr context) = 0; virtual void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr context) = 0; virtual void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr context) = 0; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index f5edde01478..846ad7b026d 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -2268,7 +2268,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( if (need_remove_expired_values) { /// Write a file with ttl infos in json format. - auto out_ttl = disk->writeFile(new_data_part->getFullRelativePath() + "ttl.txt", 4096); + auto out_ttl = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "ttl.txt", 4096); HashingWriteBuffer out_hashing(*out_ttl); new_data_part->ttl_infos.write(out_hashing); new_data_part->checksums.files["ttl.txt"].file_size = out_hashing.count(); @@ -2277,7 +2277,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( { /// Write file with checksums. - auto out_checksums = disk->writeFile(new_data_part->getFullRelativePath() + "checksums.txt", 4096); + auto out_checksums = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "checksums.txt", 4096); new_data_part->checksums.write(*out_checksums); } /// close fd @@ -2288,7 +2288,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( { /// Write a file with a description of columns. - auto out_columns = disk->writeFile(new_data_part->getFullRelativePath() + "columns.txt", 4096); + auto out_columns = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "columns.txt", 4096); new_data_part->getColumns().writeText(*out_columns); } /// close fd diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 8f5c4b6a848..1df97dc9241 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -7,7 +7,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 5c1a3b01804..7b1641a0537 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 473f2e598a9..e929bfc6862 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 84378faa513..4eebd729dd8 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -1,5 +1,4 @@ #include "MergeTreeDataPartWide.h" -#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index b1d2f33c659..894e57d41e5 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -3,13 +3,8 @@ #include #include -#include - #include -#include -#include #include -#include #include #include #include @@ -28,9 +23,7 @@ #include #include #include -#include #include -#include #include #include @@ -43,11 +36,8 @@ #include -#include -#include #include #include -#include #include namespace ProfileEvents @@ -86,7 +76,8 @@ size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead( const MergeTreeData::DataPartsVector & parts, const StorageMetadataPtr & metadata_snapshot, const KeyCondition & key_condition, - const Settings & settings) const + const Settings & settings, + Poco::Logger * log) { size_t rows_count = 0; @@ -139,17 +130,14 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( const UInt64 max_block_size, const unsigned num_streams, QueryProcessingStage::Enum processed_stage, - const PartitionIdToMaxBlock * max_block_numbers_to_read) const + std::shared_ptr max_block_numbers_to_read) const { const auto & settings = context->getSettingsRef(); + auto parts = data.getDataPartsVector(); if (!query_info.projection) { - if (settings.allow_experimental_projection_optimization && settings.force_optimize_projection - && !metadata_snapshot->projections.empty()) - throw Exception("No projection is used when allow_experimental_projection_optimization = 1", ErrorCodes::PROJECTION_NOT_USED); - - return readFromParts( - data.getDataPartsVector(), + auto plan = readFromParts( + parts, column_names_to_return, metadata_snapshot, metadata_snapshot, @@ -157,8 +145,15 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( context, max_block_size, num_streams, - max_block_numbers_to_read, - query_info.merge_tree_data_select_cache.get()); + max_block_numbers_to_read); + + if (plan->isInitialized() && settings.allow_experimental_projection_optimization && settings.force_optimize_projection + && !metadata_snapshot->projections.empty()) + throw Exception( + "No projection is used when allow_experimental_projection_optimization = 1 and force_optimize_projection = 1", + ErrorCodes::PROJECTION_NOT_USED); + + return plan; } LOG_DEBUG( @@ -167,21 +162,28 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( ProjectionDescription::typeToString(query_info.projection->desc->type), query_info.projection->desc->name); - if (query_info.projection->merge_tree_data_select_base_cache->sum_marks - + query_info.projection->merge_tree_data_select_projection_cache->sum_marks - == 0) - return std::make_unique(); + MergeTreeData::DataPartsVector projection_parts; + MergeTreeData::DataPartsVector normal_parts; + for (const auto & part : parts) + { + const auto & projections = part->getProjectionParts(); + auto it = projections.find(query_info.projection->desc->name); + if (it != projections.end()) + projection_parts.push_back(it->second); + else + normal_parts.push_back(part); + } Pipes pipes; Pipe projection_pipe; Pipe ordinary_pipe; const auto & given_select = query_info.query->as(); - if (query_info.projection->merge_tree_data_select_projection_cache->sum_marks > 0) + if (!projection_parts.empty()) { LOG_DEBUG(log, "projection required columns: {}", fmt::join(query_info.projection->required_columns, ", ")); auto plan = readFromParts( - {}, + projection_parts, query_info.projection->required_columns, metadata_snapshot, query_info.projection->desc->metadata, @@ -189,8 +191,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( context, max_block_size, num_streams, - max_block_numbers_to_read, - query_info.projection->merge_tree_data_select_projection_cache.get()); + max_block_numbers_to_read); if (plan) { @@ -224,9 +225,9 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( } } - if (query_info.projection->merge_tree_data_select_base_cache->sum_marks > 0) + if (!normal_parts.empty()) { - auto storage_from_base_parts_of_projection = StorageFromBasePartsOfProjection::create(data, metadata_snapshot); + auto storage_from_base_parts_of_projection = StorageFromMergeTreeDataPart::create(std::move(normal_parts)); auto ast = query_info.projection->desc->query_ast->clone(); auto & select = ast->as(); if (given_select.where()) @@ -369,31 +370,673 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( return plan; } -QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( - MergeTreeData::DataPartsVector parts, - const Names & column_names_to_return, - const StorageMetadataPtr & metadata_snapshot_base, +MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling( + const ASTSelectQuery & select, + NamesAndTypesList available_real_columns, + const MergeTreeData::DataPartsVector & parts, + KeyCondition & key_condition, + const MergeTreeData & data, const StorageMetadataPtr & metadata_snapshot, - const SelectQueryInfo & query_info, ContextPtr context, - const UInt64 max_block_size, - const unsigned num_streams, - const PartitionIdToMaxBlock * max_block_numbers_to_read, - MergeTreeDataSelectCache * cache) const + bool sample_factor_column_queried, + Poco::Logger * log) { - bool use_cache = cache && cache->use_cache; + const Settings & settings = context->getSettingsRef(); + /// Sampling. + MergeTreeDataSelectSamplingData sampling; - /// If query contains restrictions on the virtual column `_part` or `_part_index`, select only parts suitable for it. - /// The virtual column `_sample_factor` (which is equal to 1 / used sample rate) can be requested in the query. - Names virt_column_names; - Names real_column_names; + RelativeSize relative_sample_size = 0; + RelativeSize relative_sample_offset = 0; - size_t total_parts = parts.size(); - if (!use_cache && total_parts == 0) - return std::make_unique(); + auto select_sample_size = select.sampleSize(); + auto select_sample_offset = select.sampleOffset(); - bool sample_factor_column_queried = false; - Float64 used_sample_factor = 1; + if (select_sample_size) + { + relative_sample_size.assign( + select_sample_size->as().ratio.numerator, + select_sample_size->as().ratio.denominator); + + if (relative_sample_size < 0) + throw Exception("Negative sample size", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + relative_sample_offset = 0; + if (select_sample_offset) + relative_sample_offset.assign( + select_sample_offset->as().ratio.numerator, + select_sample_offset->as().ratio.denominator); + + if (relative_sample_offset < 0) + throw Exception("Negative sample offset", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + /// Convert absolute value of the sampling (in form `SAMPLE 1000000` - how many rows to + /// read) into the relative `SAMPLE 0.1` (how much data to read). + size_t approx_total_rows = 0; + if (relative_sample_size > 1 || relative_sample_offset > 1) + approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, key_condition, settings, log); + + if (relative_sample_size > 1) + { + relative_sample_size = convertAbsoluteSampleSizeToRelative(select_sample_size, approx_total_rows); + LOG_DEBUG(log, "Selected relative sample size: {}", toString(relative_sample_size)); + } + + /// SAMPLE 1 is the same as the absence of SAMPLE. + if (relative_sample_size == RelativeSize(1)) + relative_sample_size = 0; + + if (relative_sample_offset > 0 && RelativeSize(0) == relative_sample_size) + throw Exception("Sampling offset is incorrect because no sampling", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + if (relative_sample_offset > 1) + { + relative_sample_offset = convertAbsoluteSampleSizeToRelative(select_sample_offset, approx_total_rows); + LOG_DEBUG(log, "Selected relative sample offset: {}", toString(relative_sample_offset)); + } + } + + /** Which range of sampling key values do I need to read? + * First, in the whole range ("universe") we select the interval + * of relative `relative_sample_size` size, offset from the beginning by `relative_sample_offset`. + * + * Example: SAMPLE 0.4 OFFSET 0.3 + * + * [------********------] + * ^ - offset + * <------> - size + * + * If the interval passes through the end of the universe, then cut its right side. + * + * Example: SAMPLE 0.4 OFFSET 0.8 + * + * [----------------****] + * ^ - offset + * <------> - size + * + * Next, if the `parallel_replicas_count`, `parallel_replica_offset` settings are set, + * then it is necessary to break the received interval into pieces of the number `parallel_replicas_count`, + * and select a piece with the number `parallel_replica_offset` (from zero). + * + * Example: SAMPLE 0.4 OFFSET 0.3, parallel_replicas_count = 2, parallel_replica_offset = 1 + * + * [----------****------] + * ^ - offset + * <------> - size + * <--><--> - pieces for different `parallel_replica_offset`, select the second one. + * + * It is very important that the intervals for different `parallel_replica_offset` cover the entire range without gaps and overlaps. + * It is also important that the entire universe can be covered using SAMPLE 0.1 OFFSET 0, ... OFFSET 0.9 and similar decimals. + */ + + /// Parallel replicas has been requested but there is no way to sample data. + /// Select all data from first replica and no data from other replicas. + if (settings.parallel_replicas_count > 1 && !data.supportsSampling() && settings.parallel_replica_offset > 0) + { + LOG_DEBUG(log, "Will use no data on this replica because parallel replicas processing has been requested" + " (the setting 'max_parallel_replicas') but the table does not support sampling and this replica is not the first."); + sampling.read_nothing = true; + return sampling; + } + + sampling.use_sampling = relative_sample_size > 0 || (settings.parallel_replicas_count > 1 && data.supportsSampling()); + bool no_data = false; /// There is nothing left after sampling. + + if (sampling.use_sampling) + { + if (sample_factor_column_queried && relative_sample_size != RelativeSize(0)) + sampling.used_sample_factor = 1.0 / boost::rational_cast(relative_sample_size); + + RelativeSize size_of_universum = 0; + const auto & sampling_key = metadata_snapshot->getSamplingKey(); + DataTypePtr sampling_column_type = sampling_key.data_types[0]; + + if (sampling_key.data_types.size() == 1) + { + if (typeid_cast(sampling_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(sampling_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(sampling_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(sampling_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + } + + if (size_of_universum == RelativeSize(0)) + throw Exception( + "Invalid sampling column type in storage parameters: " + sampling_column_type->getName() + + ". Must be one unsigned integer type", + ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER); + + if (settings.parallel_replicas_count > 1) + { + if (relative_sample_size == RelativeSize(0)) + relative_sample_size = 1; + + relative_sample_size /= settings.parallel_replicas_count.value; + relative_sample_offset += relative_sample_size * RelativeSize(settings.parallel_replica_offset.value); + } + + if (relative_sample_offset >= RelativeSize(1)) + no_data = true; + + /// Calculate the half-interval of `[lower, upper)` column values. + bool has_lower_limit = false; + bool has_upper_limit = false; + + RelativeSize lower_limit_rational = relative_sample_offset * size_of_universum; + RelativeSize upper_limit_rational = (relative_sample_offset + relative_sample_size) * size_of_universum; + + UInt64 lower = boost::rational_cast(lower_limit_rational); + UInt64 upper = boost::rational_cast(upper_limit_rational); + + if (lower > 0) + has_lower_limit = true; + + if (upper_limit_rational < size_of_universum) + has_upper_limit = true; + + /*std::cerr << std::fixed << std::setprecision(100) + << "relative_sample_size: " << relative_sample_size << "\n" + << "relative_sample_offset: " << relative_sample_offset << "\n" + << "lower_limit_float: " << lower_limit_rational << "\n" + << "upper_limit_float: " << upper_limit_rational << "\n" + << "lower: " << lower << "\n" + << "upper: " << upper << "\n";*/ + + if ((has_upper_limit && upper == 0) + || (has_lower_limit && has_upper_limit && lower == upper)) + no_data = true; + + if (no_data || (!has_lower_limit && !has_upper_limit)) + { + sampling.use_sampling = false; + } + else + { + /// Let's add the conditions to cut off something else when the index is scanned again and when the request is processed. + + std::shared_ptr lower_function; + std::shared_ptr upper_function; + + /// If sample and final are used together no need to calculate sampling expression twice. + /// The first time it was calculated for final, because sample key is a part of the PK. + /// So, assume that we already have calculated column. + ASTPtr sampling_key_ast = metadata_snapshot->getSamplingKeyAST(); + + if (select.final()) + { + sampling_key_ast = std::make_shared(sampling_key.column_names[0]); + /// We do spoil available_real_columns here, but it is not used later. + available_real_columns.emplace_back(sampling_key.column_names[0], std::move(sampling_column_type)); + } + + if (has_lower_limit) + { + if (!key_condition.addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true))) + throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); + + ASTPtr args = std::make_shared(); + args->children.push_back(sampling_key_ast); + args->children.push_back(std::make_shared(lower)); + + lower_function = std::make_shared(); + lower_function->name = "greaterOrEquals"; + lower_function->arguments = args; + lower_function->children.push_back(lower_function->arguments); + + sampling.filter_function = lower_function; + } + + if (has_upper_limit) + { + if (!key_condition.addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false))) + throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); + + ASTPtr args = std::make_shared(); + args->children.push_back(sampling_key_ast); + args->children.push_back(std::make_shared(upper)); + + upper_function = std::make_shared(); + upper_function->name = "less"; + upper_function->arguments = args; + upper_function->children.push_back(upper_function->arguments); + + sampling.filter_function = upper_function; + } + + if (has_lower_limit && has_upper_limit) + { + ASTPtr args = std::make_shared(); + args->children.push_back(lower_function); + args->children.push_back(upper_function); + + sampling.filter_function = std::make_shared(); + sampling.filter_function->name = "and"; + sampling.filter_function->arguments = args; + sampling.filter_function->children.push_back(sampling.filter_function->arguments); + } + + ASTPtr query = sampling.filter_function; + auto syntax_result = TreeRewriter(context).analyze(query, available_real_columns); + sampling.filter_expression = ExpressionAnalyzer(sampling.filter_function, syntax_result, context).getActionsDAG(false); + } + } + + if (no_data) + { + LOG_DEBUG(log, "Sampling yields no data."); + sampling.read_nothing = true; + } + + return sampling; +} + +std::optional> MergeTreeDataSelectExecutor::filterPartsByVirtualColumns( + const MergeTreeData & data, + const MergeTreeData::DataPartsVector & parts, + const ASTPtr & query, + ContextPtr context) +{ + std::unordered_set part_values; + ASTPtr expression_ast; + auto virtual_columns_block = data.getBlockWithVirtualPartColumns(parts, true /* one_part */); + + // Generate valid expressions for filtering + VirtualColumnUtils::prepareFilterBlockWithQuery(query, context, virtual_columns_block, expression_ast); + + // If there is still something left, fill the virtual block and do the filtering. + if (expression_ast) + { + virtual_columns_block = data.getBlockWithVirtualPartColumns(parts, false /* one_part */); + VirtualColumnUtils::filterBlockWithQuery(query, virtual_columns_block, context, expression_ast); + return VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); + } + + return {}; +} + +void MergeTreeDataSelectExecutor::filterPartsByPartition( + MergeTreeData::DataPartsVector & parts, + const std::optional> & part_values, + const StorageMetadataPtr & metadata_snapshot, + const MergeTreeData & data, + const SelectQueryInfo & query_info, + const ContextPtr & context, + const PartitionIdToMaxBlock * max_block_numbers_to_read, + Poco::Logger * log, + ReadFromMergeTree::IndexStats & index_stats) +{ + const Settings & settings = context->getSettingsRef(); + std::optional partition_pruner; + std::optional minmax_idx_condition; + DataTypes minmax_columns_types; + if (metadata_snapshot->hasPartitionKey()) + { + const auto & partition_key = metadata_snapshot->getPartitionKey(); + auto minmax_columns_names = data.getMinMaxColumnsNames(partition_key); + minmax_columns_types = data.getMinMaxColumnsTypes(partition_key); + + minmax_idx_condition.emplace( + query_info, context, minmax_columns_names, data.getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(context))); + partition_pruner.emplace(metadata_snapshot, query_info, context, false /* strict */); + + if (settings.force_index_by_date && (minmax_idx_condition->alwaysUnknownOrTrue() && partition_pruner->isUseless())) + { + String msg = "Neither MinMax index by columns ("; + bool first = true; + for (const String & col : minmax_columns_names) + { + if (first) + first = false; + else + msg += ", "; + msg += col; + } + msg += ") nor partition expr is used and setting 'force_index_by_date' is set"; + + throw Exception(msg, ErrorCodes::INDEX_NOT_USED); + } + } + + auto query_context = context->hasQueryContext() ? context->getQueryContext() : context; + PartFilterCounters part_filter_counters; + if (query_context->getSettingsRef().allow_experimental_query_deduplication) + selectPartsToReadWithUUIDFilter( + parts, + part_values, + data.getPinnedPartUUIDs(), + minmax_idx_condition, + minmax_columns_types, + partition_pruner, + max_block_numbers_to_read, + query_context, + part_filter_counters, + log); + else + selectPartsToRead( + parts, + part_values, + minmax_idx_condition, + minmax_columns_types, + partition_pruner, + max_block_numbers_to_read, + part_filter_counters); + + index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + .type = ReadFromMergeTree::IndexType::None, + .num_parts_after = part_filter_counters.num_initial_selected_parts, + .num_granules_after = part_filter_counters.num_initial_selected_granules}); + + if (minmax_idx_condition) + { + auto description = minmax_idx_condition->getDescription(); + index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + .type = ReadFromMergeTree::IndexType::MinMax, + .condition = std::move(description.condition), + .used_keys = std::move(description.used_keys), + .num_parts_after = part_filter_counters.num_parts_after_minmax, + .num_granules_after = part_filter_counters.num_granules_after_minmax}); + LOG_DEBUG(log, "MinMax index condition: {}", minmax_idx_condition->toString()); + } + + if (partition_pruner) + { + auto description = partition_pruner->getKeyCondition().getDescription(); + index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + .type = ReadFromMergeTree::IndexType::Partition, + .condition = std::move(description.condition), + .used_keys = std::move(description.used_keys), + .num_parts_after = part_filter_counters.num_parts_after_partition_pruner, + .num_granules_after = part_filter_counters.num_granules_after_partition_pruner}); + } +} + +RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( + MergeTreeData::DataPartsVector && parts, + StorageMetadataPtr metadata_snapshot, + const SelectQueryInfo & query_info, + const ContextPtr & context, + const KeyCondition & key_condition, + const MergeTreeReaderSettings & reader_settings, + Poco::Logger * log, + size_t num_streams, + ReadFromMergeTree::IndexStats & index_stats, + bool use_skip_indexes) +{ + RangesInDataParts parts_with_ranges(parts.size()); + const Settings & settings = context->getSettingsRef(); + + /// Let's start analyzing all useful indices + + struct DataSkippingIndexAndCondition + { + MergeTreeIndexPtr index; + MergeTreeIndexConditionPtr condition; + std::atomic total_granules{0}; + std::atomic granules_dropped{0}; + std::atomic total_parts{0}; + std::atomic parts_dropped{0}; + + DataSkippingIndexAndCondition(MergeTreeIndexPtr index_, MergeTreeIndexConditionPtr condition_) + : index(index_), condition(condition_) + { + } + }; + std::list useful_indices; + + if (use_skip_indexes) + { + for (const auto & index : metadata_snapshot->getSecondaryIndices()) + { + auto index_helper = MergeTreeIndexFactory::instance().get(index); + auto condition = index_helper->createIndexCondition(query_info, context); + if (!condition->alwaysUnknownOrTrue()) + useful_indices.emplace_back(index_helper, condition); + } + } + + if (use_skip_indexes && settings.force_data_skipping_indices.changed) + { + const auto & indices = settings.force_data_skipping_indices.toString(); + + Strings forced_indices; + { + Tokens tokens(&indices[0], &indices[indices.size()], settings.max_query_size); + IParser::Pos pos(tokens, settings.max_parser_depth); + Expected expected; + if (!parseIdentifiersOrStringLiterals(pos, expected, forced_indices)) + throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "Cannot parse force_data_skipping_indices ('{}')", indices); + } + + if (forced_indices.empty()) + throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "No indices parsed from force_data_skipping_indices ('{}')", indices); + + std::unordered_set useful_indices_names; + for (const auto & useful_index : useful_indices) + useful_indices_names.insert(useful_index.index->index.name); + + for (const auto & index_name : forced_indices) + { + if (!useful_indices_names.count(index_name)) + { + throw Exception( + ErrorCodes::INDEX_NOT_USED, + "Index {} is not used and setting 'force_data_skipping_indices' contains it", + backQuote(index_name)); + } + } + } + + std::atomic sum_marks_pk = 0; + std::atomic sum_parts_pk = 0; + + /// Let's find what range to read from each part. + { + std::atomic total_rows{0}; + + SizeLimits limits; + if (settings.read_overflow_mode == OverflowMode::THROW && settings.max_rows_to_read) + limits = SizeLimits(settings.max_rows_to_read, 0, settings.read_overflow_mode); + + SizeLimits leaf_limits; + if (settings.read_overflow_mode_leaf == OverflowMode::THROW && settings.max_rows_to_read_leaf) + leaf_limits = SizeLimits(settings.max_rows_to_read_leaf, 0, settings.read_overflow_mode_leaf); + + auto process_part = [&](size_t part_index) + { + auto & part = parts[part_index]; + + RangesInDataPart ranges(part, part_index); + + size_t total_marks_count = part->index_granularity.getMarksCountWithoutFinal(); + + if (metadata_snapshot->hasPrimaryKey()) + ranges.ranges = markRangesFromPKRange(part, metadata_snapshot, key_condition, settings, log); + else if (total_marks_count) + ranges.ranges = MarkRanges{MarkRange{0, total_marks_count}}; + + sum_marks_pk.fetch_add(ranges.getMarksCount(), std::memory_order_relaxed); + + if (!ranges.ranges.empty()) + sum_parts_pk.fetch_add(1, std::memory_order_relaxed); + + for (auto & index_and_condition : useful_indices) + { + if (ranges.ranges.empty()) + break; + + index_and_condition.total_parts.fetch_add(1, std::memory_order_relaxed); + + size_t total_granules = 0; + size_t granules_dropped = 0; + ranges.ranges = filterMarksUsingIndex( + index_and_condition.index, + index_and_condition.condition, + part, + ranges.ranges, + settings, + reader_settings, + total_granules, + granules_dropped, + log); + + index_and_condition.total_granules.fetch_add(total_granules, std::memory_order_relaxed); + index_and_condition.granules_dropped.fetch_add(granules_dropped, std::memory_order_relaxed); + + if (ranges.ranges.empty()) + index_and_condition.parts_dropped.fetch_add(1, std::memory_order_relaxed); + } + + if (!ranges.ranges.empty()) + { + if (limits.max_rows || leaf_limits.max_rows) + { + /// Fail fast if estimated number of rows to read exceeds the limit + auto current_rows_estimate = ranges.getRowsCount(); + size_t prev_total_rows_estimate = total_rows.fetch_add(current_rows_estimate); + size_t total_rows_estimate = current_rows_estimate + prev_total_rows_estimate; + limits.check(total_rows_estimate, 0, "rows (controlled by 'max_rows_to_read' setting)", ErrorCodes::TOO_MANY_ROWS); + leaf_limits.check( + total_rows_estimate, 0, "rows (controlled by 'max_rows_to_read_leaf' setting)", ErrorCodes::TOO_MANY_ROWS); + } + + parts_with_ranges[part_index] = std::move(ranges); + } + }; + + size_t num_threads = std::min(size_t(num_streams), parts.size()); + + if (num_threads <= 1) + { + for (size_t part_index = 0; part_index < parts.size(); ++part_index) + process_part(part_index); + } + else + { + /// Parallel loading of data parts. + ThreadPool pool(num_threads); + + for (size_t part_index = 0; part_index < parts.size(); ++part_index) + pool.scheduleOrThrowOnError([&, part_index, thread_group = CurrentThread::getGroup()] + { + SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachQueryIfNotDetached();); + if (thread_group) + CurrentThread::attachTo(thread_group); + + process_part(part_index); + }); + + pool.wait(); + } + + /// Skip empty ranges. + size_t next_part = 0; + for (size_t part_index = 0; part_index < parts.size(); ++part_index) + { + auto & part = parts_with_ranges[part_index]; + if (!part.data_part) + continue; + + if (next_part != part_index) + std::swap(parts_with_ranges[next_part], part); + + ++next_part; + } + + parts_with_ranges.resize(next_part); + } + + if (metadata_snapshot->hasPrimaryKey()) + { + auto description = key_condition.getDescription(); + + index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + .type = ReadFromMergeTree::IndexType::PrimaryKey, + .condition = std::move(description.condition), + .used_keys = std::move(description.used_keys), + .num_parts_after = sum_parts_pk.load(std::memory_order_relaxed), + .num_granules_after = sum_marks_pk.load(std::memory_order_relaxed)}); + } + + for (const auto & index_and_condition : useful_indices) + { + const auto & index_name = index_and_condition.index->index.name; + LOG_DEBUG( + log, + "Index {} has dropped {}/{} granules.", + backQuote(index_name), + index_and_condition.granules_dropped, + index_and_condition.total_granules); + + std::string description + = index_and_condition.index->index.type + " GRANULARITY " + std::to_string(index_and_condition.index->index.granularity); + + index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + .type = ReadFromMergeTree::IndexType::Skip, + .name = index_name, + .description = std::move(description), + .num_parts_after = index_and_condition.total_parts - index_and_condition.parts_dropped, + .num_granules_after = index_and_condition.total_granules - index_and_condition.granules_dropped}); + } + + return parts_with_ranges; +} + +std::shared_ptr MergeTreeDataSelectExecutor::checkLimits( + const MergeTreeData & data, + const RangesInDataParts & parts_with_ranges, + const ContextPtr & context) +{ + const auto & settings = context->getSettingsRef(); + // Check limitations. query_id is used as the quota RAII's resource key. + String query_id; + { + const auto data_settings = data.getSettings(); + auto max_partitions_to_read + = settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data_settings->max_partitions_to_read; + if (max_partitions_to_read > 0) + { + std::set partitions; + for (const auto & part_with_ranges : parts_with_ranges) + partitions.insert(part_with_ranges.data_part->info.partition_id); + if (partitions.size() > size_t(max_partitions_to_read)) + throw Exception( + ErrorCodes::TOO_MANY_PARTITIONS, + "Too many partitions to read. Current {}, max {}", + partitions.size(), + max_partitions_to_read); + } + + if (data_settings->max_concurrent_queries > 0 && data_settings->min_marks_to_honor_max_concurrent_queries > 0) + { + size_t sum_marks = 0; + for (const auto & part : parts_with_ranges) + sum_marks += part.getMarksCount(); + + if (sum_marks >= data_settings->min_marks_to_honor_max_concurrent_queries) + { + query_id = context->getCurrentQueryId(); + if (!query_id.empty()) + data.insertQueryIdOrThrow(query_id, data_settings->max_concurrent_queries); + } + } + } + + if (!query_id.empty()) + return std::make_shared(query_id, data); + + return nullptr; +} + +static void selectColumnNames( + const Names & column_names_to_return, + const MergeTreeData & data, + Names & real_column_names, + Names & virt_column_names, + bool & sample_factor_column_queried) +{ + sample_factor_column_queried = false; for (const String & name : column_names_to_return) { @@ -435,851 +1078,136 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( real_column_names.push_back(name); } } +} - // Filter parts by virtual columns. - std::unordered_set part_values; - if (!use_cache) - { - ASTPtr expression_ast; - auto virtual_columns_block = data.getBlockWithVirtualPartColumns(parts, true /* one_part */); +size_t MergeTreeDataSelectExecutor::estimateNumMarksToRead( + MergeTreeData::DataPartsVector parts, + const Names & column_names_to_return, + const StorageMetadataPtr & metadata_snapshot_base, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + unsigned num_streams, + std::shared_ptr max_block_numbers_to_read) const +{ + size_t total_parts = parts.size(); + if (total_parts == 0) + return 0; - // Generate valid expressions for filtering - VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, context, virtual_columns_block, expression_ast); + Names real_column_names; + Names virt_column_names; + /// If query contains restrictions on the virtual column `_part` or `_part_index`, select only parts suitable for it. + /// The virtual column `_sample_factor` (which is equal to 1 / used sample rate) can be requested in the query. + bool sample_factor_column_queried = false; - // If there is still something left, fill the virtual block and do the filtering. - if (expression_ast) - { - virtual_columns_block = data.getBlockWithVirtualPartColumns(parts, false /* one_part */); - VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, context, expression_ast); - part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); - if (part_values.empty()) - return std::make_unique(); - } - } - // At this point, empty `part_values` means all parts. + selectColumnNames(column_names_to_return, data, real_column_names, virt_column_names, sample_factor_column_queried); - const Settings & settings = context->getSettingsRef(); - NamesAndTypesList available_real_columns = metadata_snapshot->getColumns().getAllPhysical(); + auto part_values = filterPartsByVirtualColumns(data, parts, query_info.query, context); + if (part_values && part_values->empty()) + return 0; /// If there are only virtual columns in the query, you must request at least one non-virtual one. if (real_column_names.empty()) + { + NamesAndTypesList available_real_columns = metadata_snapshot->getColumns().getAllPhysical(); real_column_names.push_back(ExpressionActions::getSmallestColumn(available_real_columns)); + } metadata_snapshot->check(real_column_names, data.getVirtuals(), data.getStorageID()); - // Build and check if primary key is used when necessary - std::optional key_condition; - if (!use_cache) - { - const auto & primary_key = metadata_snapshot->getPrimaryKey(); - Names primary_key_columns = primary_key.column_names; - key_condition.emplace(query_info, context, primary_key_columns, primary_key.expression); + const auto & primary_key = metadata_snapshot->getPrimaryKey(); + Names primary_key_columns = primary_key.column_names; + KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expression); - if (settings.force_primary_key && key_condition->alwaysUnknownOrTrue()) - { - throw Exception( - ErrorCodes::INDEX_NOT_USED, - "Primary key ({}) is not used and setting 'force_primary_key' is set.", - fmt::join(primary_key_columns, ", ")); - } - LOG_DEBUG(log, "Key condition: {}", key_condition->toString()); + if (key_condition.alwaysUnknownOrTrue()) + { + size_t total_marks = 0; + for (const auto & part : parts) + total_marks += part->index_granularity.getMarksCountWithoutFinal(); + + return total_marks; } const auto & select = query_info.query->as(); - auto query_context = context->hasQueryContext() ? context->getQueryContext() : context; - auto index_stats = use_cache ? std::move(cache->index_stats) : std::make_unique(); - - // Select parts to read and do partition pruning via partition value and minmax indices - if (!use_cache) - { - std::optional partition_pruner; - std::optional minmax_idx_condition; - DataTypes minmax_columns_types; - if (metadata_snapshot_base->hasPartitionKey()) - { - const auto & partition_key = metadata_snapshot_base->getPartitionKey(); - auto minmax_columns_names = data.getMinMaxColumnsNames(partition_key); - minmax_columns_types = data.getMinMaxColumnsTypes(partition_key); - - minmax_idx_condition.emplace( - query_info, context, minmax_columns_names, data.getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(context))); - partition_pruner.emplace(metadata_snapshot_base, query_info, context, false /* strict */); - - if (settings.force_index_by_date && (minmax_idx_condition->alwaysUnknownOrTrue() && partition_pruner->isUseless())) - { - String msg = "Neither MinMax index by columns ("; - bool first = true; - for (const String & col : minmax_columns_names) - { - if (first) - first = false; - else - msg += ", "; - msg += col; - } - msg += ") nor partition expr is used and setting 'force_index_by_date' is set"; - - throw Exception(msg, ErrorCodes::INDEX_NOT_USED); - } - } - - PartFilterCounters part_filter_counters; - if (query_context->getSettingsRef().allow_experimental_query_deduplication) - selectPartsToReadWithUUIDFilter( - parts, - part_values, - data.getPinnedPartUUIDs(), - minmax_idx_condition, - minmax_columns_types, - partition_pruner, - max_block_numbers_to_read, - query_context, - part_filter_counters); - else - selectPartsToRead( - parts, - part_values, - minmax_idx_condition, - minmax_columns_types, - partition_pruner, - max_block_numbers_to_read, - part_filter_counters); - - index_stats->emplace_back(ReadFromMergeTree::IndexStat{ - .type = ReadFromMergeTree::IndexType::None, - .num_parts_after = part_filter_counters.num_initial_selected_parts, - .num_granules_after = part_filter_counters.num_initial_selected_granules}); - - if (minmax_idx_condition) - { - auto description = minmax_idx_condition->getDescription(); - index_stats->emplace_back(ReadFromMergeTree::IndexStat{ - .type = ReadFromMergeTree::IndexType::MinMax, - .condition = std::move(description.condition), - .used_keys = std::move(description.used_keys), - .num_parts_after = part_filter_counters.num_parts_after_minmax, - .num_granules_after = part_filter_counters.num_granules_after_minmax}); - LOG_DEBUG(log, "MinMax index condition: {}", minmax_idx_condition->toString()); - } - - if (partition_pruner) - { - auto description = partition_pruner->getKeyCondition().getDescription(); - index_stats->emplace_back(ReadFromMergeTree::IndexStat{ - .type = ReadFromMergeTree::IndexType::Partition, - .condition = std::move(description.condition), - .used_keys = std::move(description.used_keys), - .num_parts_after = part_filter_counters.num_parts_after_partition_pruner, - .num_granules_after = part_filter_counters.num_granules_after_partition_pruner}); - } - } - - /// Sampling. - MergeTreeDataSelectSamplingData sampling = use_cache ? std::move(cache->sampling) : MergeTreeDataSelectSamplingData{}; - if (!use_cache) - { - assert(key_condition.has_value()); - - RelativeSize relative_sample_size = 0; - RelativeSize relative_sample_offset = 0; - - auto select_sample_size = select.sampleSize(); - auto select_sample_offset = select.sampleOffset(); - - if (select_sample_size) - { - relative_sample_size.assign( - select_sample_size->as().ratio.numerator, - select_sample_size->as().ratio.denominator); - - if (relative_sample_size < 0) - throw Exception("Negative sample size", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - - relative_sample_offset = 0; - if (select_sample_offset) - relative_sample_offset.assign( - select_sample_offset->as().ratio.numerator, - select_sample_offset->as().ratio.denominator); - - if (relative_sample_offset < 0) - throw Exception("Negative sample offset", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - - /// Convert absolute value of the sampling (in form `SAMPLE 1000000` - how many rows to - /// read) into the relative `SAMPLE 0.1` (how much data to read). - size_t approx_total_rows = 0; - if (relative_sample_size > 1 || relative_sample_offset > 1) - approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, *key_condition, settings); //-V1007 - - if (relative_sample_size > 1) - { - relative_sample_size = convertAbsoluteSampleSizeToRelative(select_sample_size, approx_total_rows); - LOG_DEBUG(log, "Selected relative sample size: {}", toString(relative_sample_size)); - } - - /// SAMPLE 1 is the same as the absence of SAMPLE. - if (relative_sample_size == RelativeSize(1)) - relative_sample_size = 0; - - if (relative_sample_offset > 0 && RelativeSize(0) == relative_sample_size) - throw Exception("Sampling offset is incorrect because no sampling", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - - if (relative_sample_offset > 1) - { - relative_sample_offset = convertAbsoluteSampleSizeToRelative(select_sample_offset, approx_total_rows); - LOG_DEBUG(log, "Selected relative sample offset: {}", toString(relative_sample_offset)); - } - } - - /** Which range of sampling key values do I need to read? - * First, in the whole range ("universe") we select the interval - * of relative `relative_sample_size` size, offset from the beginning by `relative_sample_offset`. - * - * Example: SAMPLE 0.4 OFFSET 0.3 - * - * [------********------] - * ^ - offset - * <------> - size - * - * If the interval passes through the end of the universe, then cut its right side. - * - * Example: SAMPLE 0.4 OFFSET 0.8 - * - * [----------------****] - * ^ - offset - * <------> - size - * - * Next, if the `parallel_replicas_count`, `parallel_replica_offset` settings are set, - * then it is necessary to break the received interval into pieces of the number `parallel_replicas_count`, - * and select a piece with the number `parallel_replica_offset` (from zero). - * - * Example: SAMPLE 0.4 OFFSET 0.3, parallel_replicas_count = 2, parallel_replica_offset = 1 - * - * [----------****------] - * ^ - offset - * <------> - size - * <--><--> - pieces for different `parallel_replica_offset`, select the second one. - * - * It is very important that the intervals for different `parallel_replica_offset` cover the entire range without gaps and overlaps. - * It is also important that the entire universe can be covered using SAMPLE 0.1 OFFSET 0, ... OFFSET 0.9 and similar decimals. - */ - - /// Parallel replicas has been requested but there is no way to sample data. - /// Select all data from first replica and no data from other replicas. - if (settings.parallel_replicas_count > 1 && !data.supportsSampling() && settings.parallel_replica_offset > 0) - { - LOG_DEBUG(log, "Will use no data on this replica because parallel replicas processing has been requested" - " (the setting 'max_parallel_replicas') but the table does not support sampling and this replica is not the first."); - return std::make_unique(); - } - - sampling.use_sampling = relative_sample_size > 0 || (settings.parallel_replicas_count > 1 && data.supportsSampling()); - bool no_data = false; /// There is nothing left after sampling. - - if (sampling.use_sampling) - { - if (sample_factor_column_queried && relative_sample_size != RelativeSize(0)) - used_sample_factor = 1.0 / boost::rational_cast(relative_sample_size); - - RelativeSize size_of_universum = 0; - const auto & sampling_key = metadata_snapshot->getSamplingKey(); - DataTypePtr sampling_column_type = sampling_key.data_types[0]; - - if (sampling_key.data_types.size() == 1) - { - if (typeid_cast(sampling_column_type.get())) - size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); - else if (typeid_cast(sampling_column_type.get())) - size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); - else if (typeid_cast(sampling_column_type.get())) - size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); - else if (typeid_cast(sampling_column_type.get())) - size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); - } - - if (size_of_universum == RelativeSize(0)) - throw Exception( - "Invalid sampling column type in storage parameters: " + sampling_column_type->getName() - + ". Must be one unsigned integer type", - ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER); - - if (settings.parallel_replicas_count > 1) - { - if (relative_sample_size == RelativeSize(0)) - relative_sample_size = 1; - - relative_sample_size /= settings.parallel_replicas_count.value; - relative_sample_offset += relative_sample_size * RelativeSize(settings.parallel_replica_offset.value); - } - - if (relative_sample_offset >= RelativeSize(1)) - no_data = true; - - /// Calculate the half-interval of `[lower, upper)` column values. - bool has_lower_limit = false; - bool has_upper_limit = false; - - RelativeSize lower_limit_rational = relative_sample_offset * size_of_universum; - RelativeSize upper_limit_rational = (relative_sample_offset + relative_sample_size) * size_of_universum; - - UInt64 lower = boost::rational_cast(lower_limit_rational); - UInt64 upper = boost::rational_cast(upper_limit_rational); - - if (lower > 0) - has_lower_limit = true; - - if (upper_limit_rational < size_of_universum) - has_upper_limit = true; - - /*std::cerr << std::fixed << std::setprecision(100) - << "relative_sample_size: " << relative_sample_size << "\n" - << "relative_sample_offset: " << relative_sample_offset << "\n" - << "lower_limit_float: " << lower_limit_rational << "\n" - << "upper_limit_float: " << upper_limit_rational << "\n" - << "lower: " << lower << "\n" - << "upper: " << upper << "\n";*/ - - if ((has_upper_limit && upper == 0) - || (has_lower_limit && has_upper_limit && lower == upper)) - no_data = true; - - if (no_data || (!has_lower_limit && !has_upper_limit)) - { - sampling.use_sampling = false; - } - else - { - /// Let's add the conditions to cut off something else when the index is scanned again and when the request is processed. - - std::shared_ptr lower_function; - std::shared_ptr upper_function; - - /// If sample and final are used together no need to calculate sampling expression twice. - /// The first time it was calculated for final, because sample key is a part of the PK. - /// So, assume that we already have calculated column. - ASTPtr sampling_key_ast = metadata_snapshot->getSamplingKeyAST(); - - if (select.final()) - { - sampling_key_ast = std::make_shared(sampling_key.column_names[0]); - /// We do spoil available_real_columns here, but it is not used later. - available_real_columns.emplace_back(sampling_key.column_names[0], std::move(sampling_column_type)); - } - - if (has_lower_limit) - { - if (!key_condition->addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true))) //-V1007 - throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); - - ASTPtr args = std::make_shared(); - args->children.push_back(sampling_key_ast); - args->children.push_back(std::make_shared(lower)); - - lower_function = std::make_shared(); - lower_function->name = "greaterOrEquals"; - lower_function->arguments = args; - lower_function->children.push_back(lower_function->arguments); - - sampling.filter_function = lower_function; - } - - if (has_upper_limit) - { - if (!key_condition->addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false))) //-V1007 - throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); - - ASTPtr args = std::make_shared(); - args->children.push_back(sampling_key_ast); - args->children.push_back(std::make_shared(upper)); - - upper_function = std::make_shared(); - upper_function->name = "less"; - upper_function->arguments = args; - upper_function->children.push_back(upper_function->arguments); - - sampling.filter_function = upper_function; - } - - if (has_lower_limit && has_upper_limit) - { - ASTPtr args = std::make_shared(); - args->children.push_back(lower_function); - args->children.push_back(upper_function); - - sampling.filter_function = std::make_shared(); - sampling.filter_function->name = "and"; - sampling.filter_function->arguments = args; - sampling.filter_function->children.push_back(sampling.filter_function->arguments); - } - - ASTPtr query = sampling.filter_function; - auto syntax_result = TreeRewriter(context).analyze(query, available_real_columns); - sampling.filter_expression = ExpressionAnalyzer(sampling.filter_function, syntax_result, context).getActionsDAG(false); - } - } - - if (no_data) - { - LOG_DEBUG(log, "Sampling yields no data."); - return std::make_unique(); - } - } - - MergeTreeReaderSettings reader_settings = - { - .min_bytes_to_use_direct_io = settings.min_bytes_to_use_direct_io, - .min_bytes_to_use_mmap_io = settings.min_bytes_to_use_mmap_io, - .mmap_cache = context->getMMappedFileCache(), - .max_read_buffer_size = settings.max_read_buffer_size, - .save_marks_in_cache = true, - .checksum_on_read = settings.checksum_on_read, - }; - - RangesInDataParts parts_with_ranges(parts.size()); - size_t sum_marks = 0; - size_t sum_ranges = 0; - - /// Let's start analyzing all useful indices - if (!use_cache) - { - struct DataSkippingIndexAndCondition - { - MergeTreeIndexPtr index; - MergeTreeIndexConditionPtr condition; - std::atomic total_granules{0}; - std::atomic granules_dropped{0}; - std::atomic total_parts{0}; - std::atomic parts_dropped{0}; - - DataSkippingIndexAndCondition(MergeTreeIndexPtr index_, MergeTreeIndexConditionPtr condition_) - : index(index_), condition(condition_) - { - } - }; - std::list useful_indices; - - for (const auto & index : metadata_snapshot->getSecondaryIndices()) - { - auto index_helper = MergeTreeIndexFactory::instance().get(index); - auto condition = index_helper->createIndexCondition(query_info, context); - if (!condition->alwaysUnknownOrTrue()) - useful_indices.emplace_back(index_helper, condition); - } - - if (settings.force_data_skipping_indices.changed) - { - const auto & indices = settings.force_data_skipping_indices.toString(); - - Strings forced_indices; - { - Tokens tokens(&indices[0], &indices[indices.size()], settings.max_query_size); - IParser::Pos pos(tokens, settings.max_parser_depth); - Expected expected; - if (!parseIdentifiersOrStringLiterals(pos, expected, forced_indices)) - throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "Cannot parse force_data_skipping_indices ('{}')", indices); - } - - if (forced_indices.empty()) - throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "No indices parsed from force_data_skipping_indices ('{}')", indices); - - std::unordered_set useful_indices_names; - for (const auto & useful_index : useful_indices) - useful_indices_names.insert(useful_index.index->index.name); - - for (const auto & index_name : forced_indices) - { - if (!useful_indices_names.count(index_name)) - { - throw Exception( - ErrorCodes::INDEX_NOT_USED, - "Index {} is not used and setting 'force_data_skipping_indices' contains it", - backQuote(index_name)); - } - } - } - - std::atomic sum_marks_pk = 0; - std::atomic sum_parts_pk = 0; - std::atomic total_marks_pk = 0; - - /// Let's find what range to read from each part. - { - std::atomic total_rows{0}; - - SizeLimits limits; - if (settings.read_overflow_mode == OverflowMode::THROW && settings.max_rows_to_read) - limits = SizeLimits(settings.max_rows_to_read, 0, settings.read_overflow_mode); - - SizeLimits leaf_limits; - if (settings.read_overflow_mode_leaf == OverflowMode::THROW && settings.max_rows_to_read_leaf) - leaf_limits = SizeLimits(settings.max_rows_to_read_leaf, 0, settings.read_overflow_mode_leaf); - - auto process_part = [&](size_t part_index) - { - auto & part = parts[part_index]; - - RangesInDataPart ranges(part, part_index); - - size_t total_marks_count = part->getMarksCount(); - if (total_marks_count && part->index_granularity.hasFinalMark()) - --total_marks_count; - - total_marks_pk.fetch_add(total_marks_count, std::memory_order_relaxed); - - if (metadata_snapshot->hasPrimaryKey()) - ranges.ranges = markRangesFromPKRange(part, metadata_snapshot, *key_condition, settings, log); - else if (total_marks_count) - ranges.ranges = MarkRanges{MarkRange{0, total_marks_count}}; - - sum_marks_pk.fetch_add(ranges.getMarksCount(), std::memory_order_relaxed); - - if (!ranges.ranges.empty()) - sum_parts_pk.fetch_add(1, std::memory_order_relaxed); - - for (auto & index_and_condition : useful_indices) - { - if (ranges.ranges.empty()) - break; - - index_and_condition.total_parts.fetch_add(1, std::memory_order_relaxed); - - size_t total_granules = 0; - size_t granules_dropped = 0; - ranges.ranges = filterMarksUsingIndex( - index_and_condition.index, - index_and_condition.condition, - part, - ranges.ranges, - settings, - reader_settings, - total_granules, - granules_dropped, - log); - - index_and_condition.total_granules.fetch_add(total_granules, std::memory_order_relaxed); - index_and_condition.granules_dropped.fetch_add(granules_dropped, std::memory_order_relaxed); - - if (ranges.ranges.empty()) - index_and_condition.parts_dropped.fetch_add(1, std::memory_order_relaxed); - } - - if (!ranges.ranges.empty()) - { - if (limits.max_rows || leaf_limits.max_rows) - { - /// Fail fast if estimated number of rows to read exceeds the limit - auto current_rows_estimate = ranges.getRowsCount(); - size_t prev_total_rows_estimate = total_rows.fetch_add(current_rows_estimate); - size_t total_rows_estimate = current_rows_estimate + prev_total_rows_estimate; - limits.check(total_rows_estimate, 0, "rows (controlled by 'max_rows_to_read' setting)", ErrorCodes::TOO_MANY_ROWS); - leaf_limits.check( - total_rows_estimate, 0, "rows (controlled by 'max_rows_to_read_leaf' setting)", ErrorCodes::TOO_MANY_ROWS); - } - - parts_with_ranges[part_index] = std::move(ranges); - } - }; - - size_t num_threads = std::min(size_t(num_streams), parts.size()); - - if (num_threads <= 1) - { - for (size_t part_index = 0; part_index < parts.size(); ++part_index) - process_part(part_index); - } - else - { - /// Parallel loading of data parts. - ThreadPool pool(num_threads); - - for (size_t part_index = 0; part_index < parts.size(); ++part_index) - pool.scheduleOrThrowOnError([&, part_index, thread_group = CurrentThread::getGroup()] - { - SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachQueryIfNotDetached();); - if (thread_group) - CurrentThread::attachTo(thread_group); - - process_part(part_index); - }); - - pool.wait(); - } - - /// Skip empty ranges. - size_t next_part = 0; - for (size_t part_index = 0; part_index < parts.size(); ++part_index) - { - auto & part = parts_with_ranges[part_index]; - if (!part.data_part) - continue; - - sum_ranges += part.ranges.size(); - sum_marks += part.getMarksCount(); - - if (next_part != part_index) - std::swap(parts_with_ranges[next_part], part); - - ++next_part; - } - - parts_with_ranges.resize(next_part); - } - - if (metadata_snapshot->hasPrimaryKey()) - { - auto description = key_condition->getDescription(); - - index_stats->emplace_back(ReadFromMergeTree::IndexStat{ - .type = ReadFromMergeTree::IndexType::PrimaryKey, - .condition = std::move(description.condition), - .used_keys = std::move(description.used_keys), - .num_parts_after = sum_parts_pk.load(std::memory_order_relaxed), - .num_granules_after = sum_marks_pk.load(std::memory_order_relaxed)}); - } - - for (const auto & index_and_condition : useful_indices) - { - const auto & index_name = index_and_condition.index->index.name; - LOG_DEBUG( - log, - "Index {} has dropped {}/{} granules.", - backQuote(index_name), - index_and_condition.granules_dropped, - index_and_condition.total_granules); - - std::string description - = index_and_condition.index->index.type + " GRANULARITY " + std::to_string(index_and_condition.index->index.granularity); - - index_stats->emplace_back(ReadFromMergeTree::IndexStat{ - .type = ReadFromMergeTree::IndexType::Skip, - .name = index_name, - .description = std::move(description), - .num_parts_after = index_and_condition.total_parts - index_and_condition.parts_dropped, - .num_granules_after = index_and_condition.total_granules - index_and_condition.granules_dropped}); - } - - LOG_DEBUG( - log, - "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", - parts.size(), - total_parts, - parts_with_ranges.size(), - sum_marks_pk.load(std::memory_order_relaxed), - total_marks_pk.load(std::memory_order_relaxed), - sum_marks, - sum_ranges); - } - - if (cache) - { - if (cache->use_cache) - { - parts_with_ranges = std::move(cache->parts_with_ranges); - sum_marks = cache->sum_marks; - sum_ranges = cache->sum_ranges; - } - else - { - // We are asking for ranges_to_read. Return immediately without further planning. - cache->parts_with_ranges = std::move(parts_with_ranges); - cache->sampling = std::move(sampling); - cache->index_stats = std::move(index_stats); - cache->sum_marks = sum_marks; - cache->sum_ranges = sum_ranges; - cache->use_cache = true; - return std::make_unique(); - } - } - - if (parts_with_ranges.empty()) + ReadFromMergeTree::IndexStats index_stats; + + filterPartsByPartition( + parts, part_values, metadata_snapshot_base, data, query_info, + context, max_block_numbers_to_read.get(), log, index_stats); + + auto sampling = MergeTreeDataSelectExecutor::getSampling( + select, metadata_snapshot->getColumns().getAllPhysical(), parts, key_condition, + data, metadata_snapshot, context, sample_factor_column_queried, log); + + if (sampling.read_nothing) + return 0; + + /// Do not init. It is not used (cause skip index is ignored) + MergeTreeReaderSettings reader_settings; + + auto parts_with_ranges = filterPartsByPrimaryKeyAndSkipIndexes( + std::move(parts), + metadata_snapshot, + query_info, + context, + key_condition, + reader_settings, + log, + num_streams, + index_stats, + false); + + return index_stats.back().num_granules_after; +} + +QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( + MergeTreeData::DataPartsVector parts, + const Names & column_names_to_return, + const StorageMetadataPtr & metadata_snapshot_base, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + const UInt64 max_block_size, + const unsigned num_streams, + std::shared_ptr max_block_numbers_to_read) const +{ + size_t total_parts = parts.size(); + if (total_parts == 0) return std::make_unique(); - // Check limitations. query_id is used as the quota RAII's resource key. - String query_id; - { - const auto data_settings = data.getSettings(); - auto max_partitions_to_read - = settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data_settings->max_partitions_to_read; - if (max_partitions_to_read > 0) - { - std::set partitions; - for (auto & part_with_ranges : parts_with_ranges) - partitions.insert(part_with_ranges.data_part->info.partition_id); - if (partitions.size() > size_t(max_partitions_to_read)) - throw Exception( - ErrorCodes::TOO_MANY_PARTITIONS, - "Too many partitions to read. Current {}, max {}", - partitions.size(), - max_partitions_to_read); - } + Names real_column_names; + Names virt_column_names; + /// If query contains restrictions on the virtual column `_part` or `_part_index`, select only parts suitable for it. + /// The virtual column `_sample_factor` (which is equal to 1 / used sample rate) can be requested in the query. + bool sample_factor_column_queried = false; - if (data_settings->max_concurrent_queries > 0) - { - if (data_settings->min_marks_to_honor_max_concurrent_queries > 0 - && sum_marks >= data_settings->min_marks_to_honor_max_concurrent_queries) - { - query_id = context->getCurrentQueryId(); - if (!query_id.empty()) - data.insertQueryIdOrThrow(query_id, data_settings->max_concurrent_queries); - } - } - } + selectColumnNames(column_names_to_return, data, real_column_names, virt_column_names, sample_factor_column_queried); - ProfileEvents::increment(ProfileEvents::SelectedParts, parts_with_ranges.size()); - ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges); - ProfileEvents::increment(ProfileEvents::SelectedMarks, sum_marks); - - QueryPlanPtr plan; - - /// Projection, that needed to drop columns, which have appeared by execution - /// of some extra expressions, and to allow execute the same expressions later. - /// NOTE: It may lead to double computation of expressions. - ActionsDAGPtr result_projection; - - Names column_names_to_read = real_column_names; - if (!select.final() && sampling.use_sampling) - { - /// Add columns needed for `sample_by_ast` to `column_names_to_read`. - /// Skip this if final was used, because such columns were already added from PK. - std::vector add_columns = sampling.filter_expression->getRequiredColumns().getNames(); - column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); - std::sort(column_names_to_read.begin(), column_names_to_read.end()); - column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), - column_names_to_read.end()); - } - - const auto & input_order_info = query_info.input_order_info - ? query_info.input_order_info - : (query_info.projection ? query_info.projection->input_order_info : nullptr); - - if (select.final()) - { - /// Add columns needed to calculate the sorting expression and the sign. - std::vector add_columns = metadata_snapshot->getColumnsRequiredForSortingKey(); - column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); - - if (!data.merging_params.sign_column.empty()) - column_names_to_read.push_back(data.merging_params.sign_column); - if (!data.merging_params.version_column.empty()) - column_names_to_read.push_back(data.merging_params.version_column); - - std::sort(column_names_to_read.begin(), column_names_to_read.end()); - column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); - - plan = spreadMarkRangesAmongStreamsFinal( - std::move(parts_with_ranges), - std::move(index_stats), - num_streams, - column_names_to_read, - metadata_snapshot, - max_block_size, - settings.use_uncompressed_cache, - query_info, - virt_column_names, - settings, - reader_settings, - result_projection, - query_id); - } - else if ((settings.optimize_read_in_order || settings.optimize_aggregation_in_order) && input_order_info) - { - size_t prefix_size = input_order_info->order_key_prefix_descr.size(); - auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone(); - order_key_prefix_ast->children.resize(prefix_size); - - auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); - auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false); - - plan = spreadMarkRangesAmongStreamsWithOrder( - std::move(parts_with_ranges), - std::move(index_stats), - num_streams, - column_names_to_read, - metadata_snapshot, - max_block_size, - settings.use_uncompressed_cache, - query_info, - sorting_key_prefix_expr, - virt_column_names, - settings, - reader_settings, - result_projection, - query_id, - input_order_info); - } - else - { - plan = spreadMarkRangesAmongStreams( - std::move(parts_with_ranges), - std::move(index_stats), - num_streams, - column_names_to_read, - metadata_snapshot, - max_block_size, - settings.use_uncompressed_cache, - query_info, - virt_column_names, - settings, - reader_settings, - query_id); - } - - if (!plan) - return std::make_unique(); - - if (sampling.use_sampling) - { - auto sampling_step = std::make_unique( - plan->getCurrentDataStream(), - sampling.filter_expression, - sampling.filter_function->getColumnName(), - false); - - sampling_step->setStepDescription("Sampling"); - plan->addStep(std::move(sampling_step)); - } - - if (result_projection) - { - auto projection_step = std::make_unique(plan->getCurrentDataStream(), result_projection); - projection_step->setStepDescription("Remove unused columns after reading from storage"); - plan->addStep(std::move(projection_step)); - } - - /// By the way, if a distributed query or query to a Merge table is made, then the `_sample_factor` column can have different values. - if (sample_factor_column_queried) - { - ColumnWithTypeAndName column; - column.name = "_sample_factor"; - column.type = std::make_shared(); - column.column = column.type->createColumnConst(0, Field(used_sample_factor)); - - auto adding_column_action = ActionsDAG::makeAddingColumnActions(std::move(column)); - - auto adding_column = std::make_unique(plan->getCurrentDataStream(), std::move(adding_column_action)); - adding_column->setStepDescription("Add _sample_factor column"); - plan->addStep(std::move(adding_column)); - } - - // TODO There seems to be no place initializing remove_columns_actions - if (query_info.prewhere_info && query_info.prewhere_info->remove_columns_actions) - { - auto expression_step = std::make_unique( - plan->getCurrentDataStream(), - query_info.prewhere_info->remove_columns_actions->getActionsDAG().clone()); - - expression_step->setStepDescription("Remove unused columns after PREWHERE"); - plan->addStep(std::move(expression_step)); - } + auto read_from_merge_tree = std::make_unique( + parts, + real_column_names, + virt_column_names, + data, + query_info, + metadata_snapshot, + metadata_snapshot_base, + context, + max_block_size, + num_streams, + sample_factor_column_queried, + max_block_numbers_to_read, + log + ); + QueryPlanPtr plan = std::make_unique(); + plan->addStep(std::move(read_from_merge_tree)); return plan; } -namespace -{ /// Marks are placed whenever threshold on rows or bytes is met. /// So we have to return the number of marks on whatever estimate is higher - by rows or by bytes. -size_t roundRowsOrBytesToMarks( +size_t MergeTreeDataSelectExecutor::roundRowsOrBytesToMarks( size_t rows_setting, size_t bytes_setting, size_t rows_granularity, @@ -1292,8 +1220,9 @@ size_t roundRowsOrBytesToMarks( else return std::max(res, (bytes_setting + bytes_granularity - 1) / bytes_granularity); } + /// Same as roundRowsOrBytesToMarks() but do not return more then max_marks -size_t minMarksForConcurrentRead( +size_t MergeTreeDataSelectExecutor::minMarksForConcurrentRead( size_t rows_setting, size_t bytes_setting, size_t rows_granularity, @@ -1321,609 +1250,6 @@ size_t minMarksForConcurrentRead( } } -} - -QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( - RangesInDataParts && parts, - ReadFromMergeTree::IndexStatPtr index_stats, - size_t num_streams, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - UInt64 max_block_size, - bool use_uncompressed_cache, - const SelectQueryInfo & query_info, - const Names & virt_columns, - const Settings & settings, - const MergeTreeReaderSettings & reader_settings, - const String & query_id) const -{ - /// Count marks for each part. - std::vector sum_marks_in_parts(parts.size()); - size_t sum_marks = 0; - size_t total_rows = 0; - - const auto data_settings = data.getSettings(); - size_t adaptive_parts = 0; - for (size_t i = 0; i < parts.size(); ++i) - { - total_rows += parts[i].getRowsCount(); - sum_marks_in_parts[i] = parts[i].getMarksCount(); - sum_marks += sum_marks_in_parts[i]; - - if (parts[i].data_part->index_granularity_info.is_adaptive) - ++adaptive_parts; - } - - size_t index_granularity_bytes = 0; - if (adaptive_parts > parts.size() / 2) - index_granularity_bytes = data_settings->index_granularity_bytes; - - const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks( - settings.merge_tree_max_rows_to_use_cache, - settings.merge_tree_max_bytes_to_use_cache, - data_settings->index_granularity, - index_granularity_bytes); - - const size_t min_marks_for_concurrent_read = minMarksForConcurrentRead( - settings.merge_tree_min_rows_for_concurrent_read, - settings.merge_tree_min_bytes_for_concurrent_read, - data_settings->index_granularity, - index_granularity_bytes, - sum_marks); - - if (sum_marks > max_marks_to_use_cache) - use_uncompressed_cache = false; - - if (0 == sum_marks) - return {}; - - ReadFromMergeTree::Settings step_settings - { - .max_block_size = max_block_size, - .preferred_block_size_bytes = settings.preferred_block_size_bytes, - .preferred_max_column_in_block_size_bytes = settings.preferred_max_column_in_block_size_bytes, - .min_marks_for_concurrent_read = min_marks_for_concurrent_read, - .use_uncompressed_cache = use_uncompressed_cache, - .reader_settings = reader_settings, - .backoff_settings = MergeTreeReadPool::BackoffSettings(settings), - }; - - if (num_streams > 1) - { - /// Reduce the number of num_streams if the data is small. - if (sum_marks < num_streams * min_marks_for_concurrent_read && parts.size() < num_streams) - num_streams = std::max((sum_marks + min_marks_for_concurrent_read - 1) / min_marks_for_concurrent_read, parts.size()); - } - - auto plan = std::make_unique(); - auto step = std::make_unique( - data, - metadata_snapshot, - query_id, - column_names, - std::move(parts), - std::move(index_stats), - query_info.projection ? query_info.projection->prewhere_info : query_info.prewhere_info, - virt_columns, - step_settings, - num_streams, - ReadFromMergeTree::ReadType::Default); - - plan->addStep(std::move(step)); - return plan; -} - -static ActionsDAGPtr createProjection(const Block & header) -{ - auto projection = std::make_shared(header.getNamesAndTypesList()); - projection->removeUnusedActions(header.getNames()); - projection->projectInput(); - return projection; -} - -QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( - RangesInDataParts && parts, - ReadFromMergeTree::IndexStatPtr index_stats, - size_t num_streams, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - UInt64 max_block_size, - bool use_uncompressed_cache, - const SelectQueryInfo & query_info, - const ActionsDAGPtr & sorting_key_prefix_expr, - const Names & virt_columns, - const Settings & settings, - const MergeTreeReaderSettings & reader_settings, - ActionsDAGPtr & out_projection, - const String & query_id, - const InputOrderInfoPtr & input_order_info) const -{ - size_t sum_marks = 0; - size_t adaptive_parts = 0; - std::vector sum_marks_in_parts(parts.size()); - const auto data_settings = data.getSettings(); - - for (size_t i = 0; i < parts.size(); ++i) - { - sum_marks_in_parts[i] = parts[i].getMarksCount(); - sum_marks += sum_marks_in_parts[i]; - - if (parts[i].data_part->index_granularity_info.is_adaptive) - ++adaptive_parts; - } - - size_t index_granularity_bytes = 0; - if (adaptive_parts > parts.size() / 2) - index_granularity_bytes = data_settings->index_granularity_bytes; - - const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks( - settings.merge_tree_max_rows_to_use_cache, - settings.merge_tree_max_bytes_to_use_cache, - data_settings->index_granularity, - index_granularity_bytes); - - const size_t min_marks_for_concurrent_read = minMarksForConcurrentRead( - settings.merge_tree_min_rows_for_concurrent_read, - settings.merge_tree_min_bytes_for_concurrent_read, - data_settings->index_granularity, - index_granularity_bytes, - sum_marks); - - if (sum_marks > max_marks_to_use_cache) - use_uncompressed_cache = false; - - Pipes res; - - if (sum_marks == 0) - return {}; - - /// Let's split ranges to avoid reading much data. - auto split_ranges = [rows_granularity = data_settings->index_granularity, max_block_size](const auto & ranges, int direction) - { - MarkRanges new_ranges; - const size_t max_marks_in_range = (max_block_size + rows_granularity - 1) / rows_granularity; - size_t marks_in_range = 1; - - if (direction == 1) - { - /// Split first few ranges to avoid reading much data. - bool split = false; - for (auto range : ranges) - { - while (!split && range.begin + marks_in_range < range.end) - { - new_ranges.emplace_back(range.begin, range.begin + marks_in_range); - range.begin += marks_in_range; - marks_in_range *= 2; - - if (marks_in_range > max_marks_in_range) - split = true; - } - new_ranges.emplace_back(range.begin, range.end); - } - } - else - { - /// Split all ranges to avoid reading much data, because we have to - /// store whole range in memory to reverse it. - for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) - { - auto range = *it; - while (range.begin + marks_in_range < range.end) - { - new_ranges.emplace_front(range.end - marks_in_range, range.end); - range.end -= marks_in_range; - marks_in_range = std::min(marks_in_range * 2, max_marks_in_range); - } - new_ranges.emplace_front(range.begin, range.end); - } - } - - return new_ranges; - }; - - const size_t min_marks_per_stream = (sum_marks - 1) / num_streams + 1; - bool need_preliminary_merge = (parts.size() > settings.read_in_order_two_level_merge_threshold); - - std::vector plans; - - for (size_t i = 0; i < num_streams && !parts.empty(); ++i) - { - size_t need_marks = min_marks_per_stream; - RangesInDataParts new_parts; - - /// Loop over parts. - /// We will iteratively take part or some subrange of a part from the back - /// and assign a stream to read from it. - while (need_marks > 0 && !parts.empty()) - { - RangesInDataPart part = parts.back(); - parts.pop_back(); - - size_t & marks_in_part = sum_marks_in_parts.back(); - - /// We will not take too few rows from a part. - if (marks_in_part >= min_marks_for_concurrent_read && - need_marks < min_marks_for_concurrent_read) - need_marks = min_marks_for_concurrent_read; - - /// Do not leave too few rows in the part. - if (marks_in_part > need_marks && - marks_in_part - need_marks < min_marks_for_concurrent_read) - need_marks = marks_in_part; - - MarkRanges ranges_to_get_from_part; - - /// We take the whole part if it is small enough. - if (marks_in_part <= need_marks) - { - ranges_to_get_from_part = part.ranges; - - need_marks -= marks_in_part; - sum_marks_in_parts.pop_back(); - } - else - { - /// Loop through ranges in part. Take enough ranges to cover "need_marks". - while (need_marks > 0) - { - if (part.ranges.empty()) - throw Exception("Unexpected end of ranges while spreading marks among streams", ErrorCodes::LOGICAL_ERROR); - - MarkRange & range = part.ranges.front(); - - const size_t marks_in_range = range.end - range.begin; - const size_t marks_to_get_from_range = std::min(marks_in_range, need_marks); - - ranges_to_get_from_part.emplace_back(range.begin, range.begin + marks_to_get_from_range); - range.begin += marks_to_get_from_range; - marks_in_part -= marks_to_get_from_range; - need_marks -= marks_to_get_from_range; - if (range.begin == range.end) - part.ranges.pop_front(); - } - parts.emplace_back(part); - } - ranges_to_get_from_part = split_ranges(ranges_to_get_from_part, input_order_info->direction); - new_parts.emplace_back(part.data_part, part.part_index_in_query, std::move(ranges_to_get_from_part)); - } - - ReadFromMergeTree::Settings step_settings - { - .max_block_size = max_block_size, - .preferred_block_size_bytes = settings.preferred_block_size_bytes, - .preferred_max_column_in_block_size_bytes = settings.preferred_max_column_in_block_size_bytes, - .min_marks_for_concurrent_read = min_marks_for_concurrent_read, - .use_uncompressed_cache = use_uncompressed_cache, - .reader_settings = reader_settings, - .backoff_settings = MergeTreeReadPool::BackoffSettings(settings), - }; - - auto read_type = input_order_info->direction == 1 - ? ReadFromMergeTree::ReadType::InOrder - : ReadFromMergeTree::ReadType::InReverseOrder; - - auto plan = std::make_unique(); - auto step = std::make_unique( - data, - metadata_snapshot, - query_id, - column_names, - std::move(new_parts), - std::move(index_stats), - query_info.projection ? query_info.projection->prewhere_info : query_info.prewhere_info, - virt_columns, - step_settings, - num_streams, - read_type); - - plan->addStep(std::move(step)); - plans.emplace_back(std::move(plan)); - } - - if (need_preliminary_merge) - { - SortDescription sort_description; - for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j) - sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j], - input_order_info->direction, 1); - - for (auto & plan : plans) - { - /// Drop temporary columns, added by 'sorting_key_prefix_expr' - out_projection = createProjection(plan->getCurrentDataStream().header); - - auto expression_step = std::make_unique( - plan->getCurrentDataStream(), - sorting_key_prefix_expr); - - expression_step->setStepDescription("Calculate sorting key prefix"); - plan->addStep(std::move(expression_step)); - - auto merging_sorted = std::make_unique( - plan->getCurrentDataStream(), - sort_description, - max_block_size); - - merging_sorted->setStepDescription("Merge sorting mark ranges"); - plan->addStep(std::move(merging_sorted)); - } - } - - if (plans.size() == 1) - return std::move(plans.front()); - - DataStreams input_streams; - for (const auto & plan : plans) - input_streams.emplace_back(plan->getCurrentDataStream()); - - auto union_step = std::make_unique(std::move(input_streams)); - - auto plan = std::make_unique(); - plan->unitePlans(std::move(union_step), std::move(plans)); - - return plan; -} - - -QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( - RangesInDataParts && parts, - ReadFromMergeTree::IndexStatPtr index_stats, - size_t num_streams, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - UInt64 max_block_size, - bool use_uncompressed_cache, - const SelectQueryInfo & query_info, - const Names & virt_columns, - const Settings & settings, - const MergeTreeReaderSettings & reader_settings, - ActionsDAGPtr & out_projection, - const String & query_id) const -{ - const auto data_settings = data.getSettings(); - size_t sum_marks = 0; - size_t adaptive_parts = 0; - for (const auto & part : parts) - { - for (const auto & range : part.ranges) - sum_marks += range.end - range.begin; - - if (part.data_part->index_granularity_info.is_adaptive) - ++adaptive_parts; - } - - size_t index_granularity_bytes = 0; - if (adaptive_parts >= parts.size() / 2) - index_granularity_bytes = data_settings->index_granularity_bytes; - - const size_t max_marks_to_use_cache = roundRowsOrBytesToMarks( - settings.merge_tree_max_rows_to_use_cache, - settings.merge_tree_max_bytes_to_use_cache, - data_settings->index_granularity, - index_granularity_bytes); - - if (sum_marks > max_marks_to_use_cache) - use_uncompressed_cache = false; - - if (num_streams > settings.max_final_threads) - num_streams = settings.max_final_threads; - - /// If setting do_not_merge_across_partitions_select_final is true than we won't merge parts from different partitions. - /// We have all parts in parts vector, where parts with same partition are nearby. - /// So we will store iterators pointed to the beginning of each partition range (and parts.end()), - /// then we will create a pipe for each partition that will run selecting processor and merging processor - /// for the parts with this partition. In the end we will unite all the pipes. - std::vector parts_to_merge_ranges; - auto it = parts.begin(); - parts_to_merge_ranges.push_back(it); - - if (settings.do_not_merge_across_partitions_select_final) - { - while (it != parts.end()) - { - it = std::find_if( - it, parts.end(), [&it](auto & part) { return it->data_part->info.partition_id != part.data_part->info.partition_id; }); - parts_to_merge_ranges.push_back(it); - } - /// We divide threads for each partition equally. But we will create at least the number of partitions threads. - /// (So, the total number of threads could be more than initial num_streams. - num_streams /= (parts_to_merge_ranges.size() - 1); - } - else - { - /// If do_not_merge_across_partitions_select_final is false we just merge all the parts. - parts_to_merge_ranges.push_back(parts.end()); - } - - std::vector partition_plans; - - /// If do_not_merge_across_partitions_select_final is true and num_streams > 1 - /// we will store lonely parts with level > 0 to use parallel select on them. - std::vector lonely_parts; - size_t total_rows_in_lonely_parts = 0; - size_t sum_marks_in_lonely_parts = 0; - - for (size_t range_index = 0; range_index < parts_to_merge_ranges.size() - 1; ++range_index) - { - QueryPlanPtr plan; - - { - RangesInDataParts new_parts; - - /// If do_not_merge_across_partitions_select_final is true and there is only one part in partition - /// with level > 0 then we won't postprocess this part and if num_streams > 1 we - /// can use parallel select on such parts. We save such parts in one vector and then use - /// MergeTreeReadPool and MergeTreeThreadSelectBlockInputProcessor for parallel select. - if (num_streams > 1 && settings.do_not_merge_across_partitions_select_final && - std::distance(parts_to_merge_ranges[range_index], parts_to_merge_ranges[range_index + 1]) == 1 && - parts_to_merge_ranges[range_index]->data_part->info.level > 0) - { - total_rows_in_lonely_parts += parts_to_merge_ranges[range_index]->getRowsCount(); - sum_marks_in_lonely_parts += parts_to_merge_ranges[range_index]->getMarksCount(); - lonely_parts.push_back(std::move(*parts_to_merge_ranges[range_index])); - continue; - } - else - { - for (auto part_it = parts_to_merge_ranges[range_index]; part_it != parts_to_merge_ranges[range_index + 1]; ++part_it) - { - new_parts.emplace_back(part_it->data_part, part_it->part_index_in_query, part_it->ranges); - } - } - - if (new_parts.empty()) - continue; - - ReadFromMergeTree::Settings step_settings - { - .max_block_size = max_block_size, - .preferred_block_size_bytes = settings.preferred_block_size_bytes, - .preferred_max_column_in_block_size_bytes = settings.preferred_max_column_in_block_size_bytes, - .min_marks_for_concurrent_read = 0, /// this setting is not used for reading in order - .use_uncompressed_cache = use_uncompressed_cache, - .reader_settings = reader_settings, - .backoff_settings = MergeTreeReadPool::BackoffSettings(settings), - }; - - plan = std::make_unique(); - auto step = std::make_unique( - data, - metadata_snapshot, - query_id, - column_names, - std::move(new_parts), - std::move(index_stats), - query_info.projection ? query_info.projection->prewhere_info : query_info.prewhere_info, - virt_columns, - step_settings, - num_streams, - ReadFromMergeTree::ReadType::InOrder); - - plan->addStep(std::move(step)); - - /// Drop temporary columns, added by 'sorting_key_expr' - if (!out_projection) - out_projection = createProjection(plan->getCurrentDataStream().header); - } - - auto expression_step = std::make_unique( - plan->getCurrentDataStream(), - metadata_snapshot->getSortingKey().expression->getActionsDAG().clone()); - - expression_step->setStepDescription("Calculate sorting key expression"); - plan->addStep(std::move(expression_step)); - - /// If do_not_merge_across_partitions_select_final is true and there is only one part in partition - /// with level > 0 then we won't postprocess this part - if (settings.do_not_merge_across_partitions_select_final && - std::distance(parts_to_merge_ranges[range_index], parts_to_merge_ranges[range_index + 1]) == 1 && - parts_to_merge_ranges[range_index]->data_part->info.level > 0) - { - partition_plans.emplace_back(std::move(plan)); - continue; - } - - Names sort_columns = metadata_snapshot->getSortingKeyColumns(); - SortDescription sort_description; - size_t sort_columns_size = sort_columns.size(); - sort_description.reserve(sort_columns_size); - - Names partition_key_columns = metadata_snapshot->getPartitionKey().column_names; - - const auto & header = plan->getCurrentDataStream().header; - for (size_t i = 0; i < sort_columns_size; ++i) - sort_description.emplace_back(header.getPositionByName(sort_columns[i]), 1, 1); - - auto final_step = std::make_unique( - plan->getCurrentDataStream(), - std::min(num_streams, settings.max_final_threads), - sort_description, - data.merging_params, - partition_key_columns, - max_block_size); - - final_step->setStepDescription("Merge rows for FINAL"); - plan->addStep(std::move(final_step)); - - partition_plans.emplace_back(std::move(plan)); - } - - if (!lonely_parts.empty()) - { - RangesInDataParts new_parts; - - size_t num_streams_for_lonely_parts = num_streams * lonely_parts.size(); - - const size_t min_marks_for_concurrent_read = minMarksForConcurrentRead( - settings.merge_tree_min_rows_for_concurrent_read, - settings.merge_tree_min_bytes_for_concurrent_read, - data_settings->index_granularity, - index_granularity_bytes, - sum_marks_in_lonely_parts); - - /// Reduce the number of num_streams_for_lonely_parts if the data is small. - if (sum_marks_in_lonely_parts < num_streams_for_lonely_parts * min_marks_for_concurrent_read && lonely_parts.size() < num_streams_for_lonely_parts) - num_streams_for_lonely_parts = std::max((sum_marks_in_lonely_parts + min_marks_for_concurrent_read - 1) / min_marks_for_concurrent_read, lonely_parts.size()); - - ReadFromMergeTree::Settings step_settings - { - .max_block_size = max_block_size, - .preferred_block_size_bytes = settings.preferred_block_size_bytes, - .preferred_max_column_in_block_size_bytes = settings.preferred_max_column_in_block_size_bytes, - .min_marks_for_concurrent_read = min_marks_for_concurrent_read, - .use_uncompressed_cache = use_uncompressed_cache, - .reader_settings = reader_settings, - .backoff_settings = MergeTreeReadPool::BackoffSettings(settings), - }; - - auto plan = std::make_unique(); - auto step = std::make_unique( - data, - metadata_snapshot, - query_id, - column_names, - std::move(lonely_parts), - std::move(index_stats), - query_info.projection ? query_info.projection->prewhere_info : query_info.prewhere_info, - virt_columns, - step_settings, - num_streams_for_lonely_parts, - ReadFromMergeTree::ReadType::Default); - - plan->addStep(std::move(step)); - - /// Drop temporary columns, added by 'sorting_key_expr' - if (!out_projection) - out_projection = createProjection(plan->getCurrentDataStream().header); - - auto expression_step = std::make_unique( - plan->getCurrentDataStream(), - metadata_snapshot->getSortingKey().expression->getActionsDAG().clone()); - - expression_step->setStepDescription("Calculate sorting key expression"); - plan->addStep(std::move(expression_step)); - - partition_plans.emplace_back(std::move(plan)); - } - - if (partition_plans.empty()) - return {}; - - if (partition_plans.size() == 1) - return std::move(partition_plans.front()); - - auto result_header = partition_plans.front()->getCurrentDataStream().header; - DataStreams input_streams; - for (const auto & partition_plan : partition_plans) - input_streams.push_back(partition_plan->getCurrentDataStream()); - - auto union_step = std::make_unique(std::move(input_streams), result_header); - union_step->setStepDescription("Unite sources after FINAL"); - QueryPlanPtr plan = std::make_unique(); - plan->unitePlans(std::move(union_step), std::move(partition_plans)); - return plan; -} /// Calculates a set of mark ranges, that could possibly contain keys, required by condition. /// In other words, it removes subranges from whole range, that definitely could not contain required keys. @@ -2192,7 +1518,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( void MergeTreeDataSelectExecutor::selectPartsToRead( MergeTreeData::DataPartsVector & parts, - const std::unordered_set & part_values, + const std::optional> & part_values, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, std::optional & partition_pruner, @@ -2204,7 +1530,7 @@ void MergeTreeDataSelectExecutor::selectPartsToRead( for (const auto & part_or_projection : prev_parts) { const auto * part = part_or_projection->isProjectionPart() ? part_or_projection->getParentPart() : part_or_projection.get(); - if (!part_values.empty() && part_values.find(part->name) == part_values.end()) + if (part_values && part_values->find(part->name) == part_values->end()) continue; if (part->isEmpty()) @@ -2246,14 +1572,15 @@ void MergeTreeDataSelectExecutor::selectPartsToRead( void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( MergeTreeData::DataPartsVector & parts, - const std::unordered_set & part_values, + const std::optional> & part_values, MergeTreeData::PinnedPartUUIDsPtr pinned_part_uuids, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, std::optional & partition_pruner, const PartitionIdToMaxBlock * max_block_numbers_to_read, ContextPtr query_context, - PartFilterCounters & counters) const + PartFilterCounters & counters, + Poco::Logger * log) { const Settings & settings = query_context->getSettings(); @@ -2269,7 +1596,7 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( for (const auto & part_or_projection : prev_parts) { const auto * part = part_or_projection->isProjectionPart() ? part_or_projection->getParentPart() : part_or_projection.get(); - if (!part_values.empty() && part_values.find(part->name) == part_values.end()) + if (part_values && part_values->find(part->name) == part_values->end()) continue; if (part->isEmpty()) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index c058c7e936b..bd2a79f0aee 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -16,19 +16,13 @@ class KeyCondition; struct MergeTreeDataSelectSamplingData { bool use_sampling = false; + bool read_nothing = false; + Float64 used_sample_factor = 1.0; std::shared_ptr filter_function; ActionsDAGPtr filter_expression; }; -struct MergeTreeDataSelectCache -{ - RangesInDataParts parts_with_ranges; - MergeTreeDataSelectSamplingData sampling; - std::unique_ptr index_stats; - size_t sum_marks = 0; - size_t sum_ranges = 0; - bool use_cache = false; -}; +using PartitionIdToMaxBlock = std::unordered_map; /** Executes SELECT queries on data from the merge tree. */ @@ -40,7 +34,6 @@ public: /** When reading, selects a set of parts that covers the desired range of the index. * max_blocks_number_to_read - if not nullptr, do not read all the parts whose right border is greater than max_block in partition. */ - using PartitionIdToMaxBlock = std::unordered_map; QueryPlanPtr read( const Names & column_names, @@ -50,8 +43,9 @@ public: UInt64 max_block_size, unsigned num_streams, QueryProcessingStage::Enum processed_stage, - const PartitionIdToMaxBlock * max_block_numbers_to_read = nullptr) const; + std::shared_ptr max_block_numbers_to_read = nullptr) const; + /// The same as read, but with specified set of parts. QueryPlanPtr readFromParts( MergeTreeData::DataPartsVector parts, const Names & column_names, @@ -61,67 +55,32 @@ public: ContextPtr context, UInt64 max_block_size, unsigned num_streams, - const PartitionIdToMaxBlock * max_block_numbers_to_read = nullptr, - MergeTreeDataSelectCache * cache = nullptr) const; + std::shared_ptr max_block_numbers_to_read = nullptr) const; + + /// Get an estimation for the number of marks we are going to read. + /// Reads nothing. Secondary indexes are not used. + /// This method is used to select best projection for table. + size_t estimateNumMarksToRead( + MergeTreeData::DataPartsVector parts, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot_base, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + unsigned num_streams, + std::shared_ptr max_block_numbers_to_read = nullptr) const; private: const MergeTreeData & data; - Poco::Logger * log; - QueryPlanPtr spreadMarkRangesAmongStreams( - RangesInDataParts && parts, - ReadFromMergeTree::IndexStatPtr index_stats, - size_t num_streams, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - UInt64 max_block_size, - bool use_uncompressed_cache, - const SelectQueryInfo & query_info, - const Names & virt_columns, - const Settings & settings, - const MergeTreeReaderSettings & reader_settings, - const String & query_id) const; - - /// out_projection - save projection only with columns, requested to read - QueryPlanPtr spreadMarkRangesAmongStreamsWithOrder( - RangesInDataParts && parts, - ReadFromMergeTree::IndexStatPtr index_stats, - size_t num_streams, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - UInt64 max_block_size, - bool use_uncompressed_cache, - const SelectQueryInfo & query_info, - const ActionsDAGPtr & sorting_key_prefix_expr, - const Names & virt_columns, - const Settings & settings, - const MergeTreeReaderSettings & reader_settings, - ActionsDAGPtr & out_projection, - const String & query_id, - const InputOrderInfoPtr & input_order_info) const; - - QueryPlanPtr spreadMarkRangesAmongStreamsFinal( - RangesInDataParts && parts, - ReadFromMergeTree::IndexStatPtr index_stats, - size_t num_streams, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - UInt64 max_block_size, - bool use_uncompressed_cache, - const SelectQueryInfo & query_info, - const Names & virt_columns, - const Settings & settings, - const MergeTreeReaderSettings & reader_settings, - ActionsDAGPtr & out_projection, - const String & query_id) const; - /// Get the approximate value (bottom estimate - only by full marks) of the number of rows falling under the index. - size_t getApproximateTotalRowsToRead( + static size_t getApproximateTotalRowsToRead( const MergeTreeData::DataPartsVector & parts, const StorageMetadataPtr & metadata_snapshot, const KeyCondition & key_condition, - const Settings & settings) const; + const Settings & settings, + Poco::Logger * log); static MarkRanges markRangesFromPKRange( const MergeTreeData::DataPartPtr & part, @@ -155,7 +114,7 @@ private: /// as well as `max_block_number_to_read`. static void selectPartsToRead( MergeTreeData::DataPartsVector & parts, - const std::unordered_set & part_values, + const std::optional> & part_values, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, std::optional & partition_pruner, @@ -163,16 +122,90 @@ private: PartFilterCounters & counters); /// Same as previous but also skip parts uuids if any to the query context, or skip parts which uuids marked as excluded. - void selectPartsToReadWithUUIDFilter( + static void selectPartsToReadWithUUIDFilter( MergeTreeData::DataPartsVector & parts, - const std::unordered_set & part_values, + const std::optional> & part_values, MergeTreeData::PinnedPartUUIDsPtr pinned_part_uuids, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, std::optional & partition_pruner, const PartitionIdToMaxBlock * max_block_numbers_to_read, ContextPtr query_context, - PartFilterCounters & counters) const; + PartFilterCounters & counters, + Poco::Logger * log); + +public: + /// For given number rows and bytes, get the number of marks to read. + /// It is a minimal number of marks which contain so many rows and bytes. + static size_t roundRowsOrBytesToMarks( + size_t rows_setting, + size_t bytes_setting, + size_t rows_granularity, + size_t bytes_granularity); + + /// The same as roundRowsOrBytesToMarks, but return no more than max_marks. + static size_t minMarksForConcurrentRead( + size_t rows_setting, + size_t bytes_setting, + size_t rows_granularity, + size_t bytes_granularity, + size_t max_marks); + + /// If possible, filter using expression on virtual columns. + /// Example: SELECT count() FROM table WHERE _part = 'part_name' + /// If expression found, return a set with allowed part names (std::nullopt otherwise). + static std::optional> filterPartsByVirtualColumns( + const MergeTreeData & data, + const MergeTreeData::DataPartsVector & parts, + const ASTPtr & query, + ContextPtr context); + + /// Filter parts using minmax index and partition key. + static void filterPartsByPartition( + MergeTreeData::DataPartsVector & parts, + const std::optional> & part_values, + const StorageMetadataPtr & metadata_snapshot, + const MergeTreeData & data, + const SelectQueryInfo & query_info, + const ContextPtr & context, + const PartitionIdToMaxBlock * max_block_numbers_to_read, + Poco::Logger * log, + ReadFromMergeTree::IndexStats & index_stats); + + /// Filter parts using primary key and secondary indexes. + /// For every part, select mark ranges to read. + static RangesInDataParts filterPartsByPrimaryKeyAndSkipIndexes( + MergeTreeData::DataPartsVector && parts, + StorageMetadataPtr metadata_snapshot, + const SelectQueryInfo & query_info, + const ContextPtr & context, + const KeyCondition & key_condition, + const MergeTreeReaderSettings & reader_settings, + Poco::Logger * log, + size_t num_streams, + ReadFromMergeTree::IndexStats & index_stats, + bool use_skip_indexes); + + /// Create expression for sampling. + /// Also, calculate _sample_factor if needed. + /// Also, update key condition with selected sampling range. + static MergeTreeDataSelectSamplingData getSampling( + const ASTSelectQuery & select, + NamesAndTypesList available_real_columns, + const MergeTreeData::DataPartsVector & parts, + KeyCondition & key_condition, + const MergeTreeData & data, + const StorageMetadataPtr & metadata_snapshot, + ContextPtr context, + bool sample_factor_column_queried, + Poco::Logger * log); + + /// Check query limits: max_partitions_to_read, max_concurrent_queries. + /// Also, return QueryIdHolder. If not null, we should keep it until query finishes. + static std::shared_ptr checkLimits( + const MergeTreeData & data, + const RangesInDataParts & parts_with_ranges, + const ContextPtr & context); }; } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 3bf33d45802..0b05650b42c 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index ed0e7e55fc8..6da0a822f7f 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -1,6 +1,8 @@ #include #include -#include + + +namespace fs = std::filesystem; namespace DB { @@ -17,8 +19,7 @@ std::optional MergeTreeIndexGranularityInfo::getMarksExtensionFromF { for (DiskDirectoryIteratorPtr it = disk->iterateDirectory(path_to_part); it->isValid(); it->next()) { - Poco::Path path(it->path()); - const auto & ext = "." + path.getExtension(); + const auto & ext = fs::path(it->path()).extension(); if (ext == getNonAdaptiveMrkExtension() || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE) || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT)) diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index c5a99b128e9..717179e5f26 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp index 49c4e93eb1d..2aefb3df2be 100644 --- a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp @@ -4,9 +4,6 @@ #include #include -#include -#include - #include diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index f9e3883d5e2..5b77ac9ec4a 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -206,18 +206,18 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt /// Try to fetch part from S3 without copy and fallback to default copy /// if it's not possible moving_part.part->assertOnDisk(); - String path_to_clone = data->getRelativeDataPath() + directory_to_move + "/"; + String path_to_clone = fs::path(data->getRelativeDataPath()) / directory_to_move / ""; String relative_path = part->relative_path; if (disk->exists(path_to_clone + relative_path)) { LOG_WARNING(log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); - disk->removeRecursive(path_to_clone + relative_path + "/"); + disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); - bool is_fetched = data->tryToFetchIfShared(*part, disk, path_to_clone + "/" + part->name); + bool is_fetched = data->tryToFetchIfShared(*part, disk, fs::path(path_to_clone) / part->name); if (!is_fetched) - part->volume->getDisk()->copy(data->getRelativeDataPath() + relative_path + "/", disk, path_to_clone); - part->volume->getDisk()->removeFileIfExists(path_to_clone + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + part->volume->getDisk()->copy(fs::path(data->getRelativeDataPath()) / relative_path / "", disk, path_to_clone); + part->volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); } else { @@ -226,7 +226,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt auto single_disk_volume = std::make_shared("volume_" + part->name, moving_part.reserved_space->getDisk(), 0); MergeTreeData::MutableDataPartPtr cloned_part = - data->createPart(part->name, single_disk_volume, directory_to_move + '/' + part->name); + data->createPart(part->name, single_disk_volume, fs::path(directory_to_move) / part->name); LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index da28f75b57f..783a37cce60 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp index 5ee4aa555e6..cf5c850daa1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp @@ -3,7 +3,6 @@ #include #include #include -#include namespace DB { diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 3655743d41a..6de01caa833 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -1,6 +1,5 @@ #include #include -#include #include @@ -123,7 +122,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { if (new_part->uuid != UUIDHelpers::Nil) { - auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::UUID_FILE_NAME, 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / IMergeTreeDataPart::UUID_FILE_NAME, 4096); HashingWriteBuffer out_hashing(*out); writeUUIDText(new_part->uuid, out_hashing); checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count(); @@ -142,7 +141,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); - auto count_out = volume->getDisk()->writeFile(part_path + "count.txt", 4096); + auto count_out = volume->getDisk()->writeFile(fs::path(part_path) / "count.txt", 4096); HashingWriteBuffer count_out_hashing(*count_out); writeIntText(rows_count, count_out_hashing); count_out_hashing.next(); @@ -157,7 +156,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( if (!new_part->ttl_infos.empty()) { /// Write a file with ttl infos in json format. - auto out = volume->getDisk()->writeFile(part_path + "ttl.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "ttl.txt", 4096); HashingWriteBuffer out_hashing(*out); new_part->ttl_infos.write(out_hashing); checksums.files["ttl.txt"].file_size = out_hashing.count(); @@ -171,7 +170,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { /// Write a file with a description of columns. - auto out = volume->getDisk()->writeFile(part_path + "columns.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "columns.txt", 4096); part_columns.writeText(*out); out->finalize(); if (sync) @@ -192,7 +191,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { /// Write file with checksums. - auto out = volume->getDisk()->writeFile(part_path + "checksums.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "checksums.txt", 4096); checksums.write(*out); out->finalize(); if (sync) diff --git a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp index 7a4c3d530a7..4c187109ac6 100644 --- a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp +++ b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp @@ -347,7 +347,7 @@ void PartMovesBetweenShardsOrchestrator::stepEntry(const Entry & entry, zkutil:: { { ReplicatedMergeTreeLogEntry log_entry; - if (storage.dropPart(zk, entry.part_name, log_entry,false, false)) + if (storage.dropPartImpl(zk, entry.part_name, log_entry, false, false)) storage.waitForAllReplicasToProcessLogEntry(log_entry, true); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index d3496d99cef..10e2d77eb27 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -291,7 +291,7 @@ void ReplicatedMergeTreeCleanupThread::markLostReplicas(const std::unordered_map std::vector futures; for (size_t i = 0; i < candidate_lost_replicas.size(); ++i) - futures.emplace_back(zookeeper->tryAsyncMulti(requests[i])); + futures.emplace_back(zookeeper->asyncTryMultiNoThrow(requests[i])); for (size_t i = 0; i < candidate_lost_replicas.size(); ++i) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 30569e53f64..439d4970aa8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -62,7 +62,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr & bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { - auto queue_path = replica_path + "/queue"; + String queue_path = fs::path(replica_path) / "queue"; LOG_DEBUG(log, "Loading queue from {}", queue_path); bool updated = false; @@ -74,7 +74,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) /// Reset batch size on initialization to recover from possible errors of too large batch size. current_multi_batch_size = 1; - String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); std::unordered_set already_loaded_paths; @@ -101,7 +101,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) futures.reserve(children.size()); for (const String & child : children) - futures.emplace_back(child, zookeeper->asyncGet(queue_path + "/" + child)); + futures.emplace_back(child, zookeeper->asyncGet(fs::path(queue_path) / child)); for (auto & future : futures) { @@ -116,7 +116,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) updated = true; } - zookeeper->tryGet(replica_path + "/mutation_pointer", mutation_pointer); + zookeeper->tryGet(fs::path(replica_path) / "mutation_pointer", mutation_pointer); } updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, {}); @@ -399,7 +399,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep if (!need_remove_from_zk) return; - auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); + auto code = zookeeper->tryRemove(fs::path(replica_path) / "queue" / entry->znode_name); if (code != Coordination::Error::ZOK) LOG_ERROR(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, Coordination::errorMessage(code)); @@ -455,7 +455,7 @@ bool ReplicatedMergeTreeQueue::remove(zkutil::ZooKeeperPtr zookeeper, const Stri notifySubscribers(queue_size); - zookeeper->tryRemove(replica_path + "/queue/" + found->znode_name); + zookeeper->tryRemove(fs::path(replica_path) / "queue" / found->znode_name); updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); return true; @@ -474,14 +474,14 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper if (pull_log_blocker.isCancelled()) throw Exception("Log pulling is cancelled", ErrorCodes::ABORTED); - String index_str = zookeeper->get(replica_path + "/log_pointer"); + String index_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); UInt64 index; /// The version of "/log" is modified when new entries to merge/mutate/drop appear. Coordination::Stat stat; - zookeeper->get(zookeeper_path + "/log", &stat); + zookeeper->get(fs::path(zookeeper_path) / "log", &stat); - Strings log_entries = zookeeper->getChildrenWatch(zookeeper_path + "/log", nullptr, watch_callback); + Strings log_entries = zookeeper->getChildrenWatch(fs::path(zookeeper_path) / "log", nullptr, watch_callback); /// We update mutations after we have loaded the list of log entries, but before we insert them /// in the queue. @@ -494,7 +494,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper /// If we do not already have a pointer to the log, put a pointer to the first entry in it. index = log_entries.empty() ? 0 : parse(std::min_element(log_entries.begin(), log_entries.end())->substr(strlen("log-"))); - zookeeper->set(replica_path + "/log_pointer", toString(index)); + zookeeper->set(fs::path(replica_path) / "log_pointer", toString(index)); } else { @@ -541,7 +541,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper futures.reserve(end - begin); for (auto it = begin; it != end; ++it) - futures.emplace_back(*it, zookeeper->asyncGet(zookeeper_path + "/log/" + *it)); + futures.emplace_back(*it, zookeeper->asyncGet(fs::path(zookeeper_path) / "log" / *it)); /// Simultaneously add all new entries to the queue and move the pointer to the log. @@ -558,7 +558,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper copied_entries.emplace_back(LogEntry::parse(res.data, res.stat)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", res.data, zkutil::CreateMode::PersistentSequential)); + fs::path(replica_path) / "queue/queue-", res.data, zkutil::CreateMode::PersistentSequential)); const auto & entry = *copied_entries.back(); if (entry.type == LogEntry::GET_PART || entry.type == LogEntry::ATTACH_PART) @@ -573,11 +573,11 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper } ops.emplace_back(zkutil::makeSetRequest( - replica_path + "/log_pointer", toString(last_entry_index + 1), -1)); + fs::path(replica_path) / "log_pointer", toString(last_entry_index + 1), -1)); if (min_unprocessed_insert_time_changed) ops.emplace_back(zkutil::makeSetRequest( - replica_path + "/min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1)); + fs::path(replica_path) / "min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1)); auto responses = zookeeper->multi(ops); @@ -655,7 +655,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C { std::lock_guard lock(update_mutations_mutex); - Strings entries_in_zk = zookeeper->getChildrenWatch(zookeeper_path + "/mutations", nullptr, watch_callback); + Strings entries_in_zk = zookeeper->getChildrenWatch(fs::path(zookeeper_path) / "mutations", nullptr, watch_callback); StringSet entries_in_zk_set(entries_in_zk.begin(), entries_in_zk.end()); /// Compare with the local state, delete obsolete entries and determine which new entries to load. @@ -712,7 +712,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C std::vector> futures; for (const String & entry : entries_to_load) - futures.emplace_back(zookeeper->asyncGet(zookeeper_path + "/mutations/" + entry)); + futures.emplace_back(zookeeper->asyncGet(fs::path(zookeeper_path) / "mutations" / entry)); std::vector new_mutations; for (size_t i = 0; i < entries_to_load.size(); ++i) @@ -796,7 +796,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( { std::lock_guard lock(update_mutations_mutex); - auto rc = zookeeper->tryRemove(zookeeper_path + "/mutations/" + mutation_id); + auto rc = zookeeper->tryRemove(fs::path(zookeeper_path) / "mutations" / mutation_id); if (rc == Coordination::Error::ZOK) LOG_DEBUG(log, "Removed mutation {} from ZooKeeper.", mutation_id); @@ -940,12 +940,12 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( { if ((*it)->currently_executing) to_wait.push_back(*it); - auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name); + auto code = zookeeper->tryRemove(fs::path(replica_path) / "queue" / (*it)->znode_name); /// FIXME it's probably unsafe to remove entries non-atomically /// when this method called directly from alter query (not from replication queue task), /// because entries will be lost if ALTER fails. if (code != Coordination::Error::ZOK) - LOG_INFO(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, Coordination::errorMessage(code)); + LOG_INFO(log, "Couldn't remove {}: {}", (fs::path(replica_path) / "queue" / (*it)->znode_name).string(), Coordination::errorMessage(code)); updateStateOnQueueEntryRemoval( *it, /* is_successful = */ false, @@ -1554,7 +1554,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep if (!finished.empty()) { - zookeeper->set(replica_path + "/mutation_pointer", finished.back()->znode_name); + zookeeper->set(fs::path(replica_path) / "mutation_pointer", finished.back()->znode_name); std::lock_guard lock(state_mutex); @@ -1750,22 +1750,22 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( } /// Load current quorum status. - auto quorum_status_future = zookeeper->asyncTryGet(queue.zookeeper_path + "/quorum/status"); + auto quorum_status_future = zookeeper->asyncTryGet(fs::path(queue.zookeeper_path) / "quorum" / "status"); /// Load current inserts std::unordered_set lock_holder_paths; - for (const String & entry : zookeeper->getChildren(queue.zookeeper_path + "/temp")) + for (const String & entry : zookeeper->getChildren(fs::path(queue.zookeeper_path) / "temp")) { if (startsWith(entry, "abandonable_lock-")) - lock_holder_paths.insert(queue.zookeeper_path + "/temp/" + entry); + lock_holder_paths.insert(fs::path(queue.zookeeper_path) / "temp" / entry); } if (!lock_holder_paths.empty()) { - Strings partitions = zookeeper->getChildren(queue.zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(queue.zookeeper_path) / "block_numbers"); std::vector> lock_futures; for (const String & partition : partitions) - lock_futures.push_back(zookeeper->asyncGetChildren(queue.zookeeper_path + "/block_numbers/" + partition)); + lock_futures.push_back(zookeeper->asyncGetChildren(fs::path(queue.zookeeper_path) / "block_numbers" / partition)); struct BlockInfoInZooKeeper { @@ -1786,7 +1786,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( if (startsWith(entry, "block-")) { Int64 block_number = parse(entry.substr(strlen("block-"))); - String zk_path = queue.zookeeper_path + "/block_numbers/" + partitions[i] + "/" + entry; + String zk_path = fs::path(queue.zookeeper_path) / "block_numbers" / partitions[i] / entry; block_infos.emplace_back( BlockInfoInZooKeeper{partitions[i], block_number, zk_path, zookeeper->asyncTryGet(zk_path)}); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 6b7fb3bf17f..7288396b58b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -238,7 +238,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() auto zookeeper = storage.getZooKeeper(); String quorum_str; - if (zookeeper->tryGet(storage.zookeeper_path + "/quorum/status", quorum_str)) + if (zookeeper->tryGet(fs::path(storage.zookeeper_path) / "quorum" / "status", quorum_str)) { ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str); @@ -251,12 +251,12 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() } Strings part_names; - String parallel_quorum_parts_path = storage.zookeeper_path + "/quorum/parallel"; + String parallel_quorum_parts_path = fs::path(storage.zookeeper_path) / "quorum" / "parallel"; if (zookeeper->tryGetChildren(parallel_quorum_parts_path, part_names) == Coordination::Error::ZOK) { for (auto & part_name : part_names) { - if (zookeeper->tryGet(parallel_quorum_parts_path + "/" + part_name, quorum_str)) + if (zookeeper->tryGet(fs::path(parallel_quorum_parts_path) / part_name, quorum_str)) { ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str); if (!quorum_entry.replicas.count(storage.replica_name) @@ -278,7 +278,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() /// How other replicas can access this one. ReplicatedMergeTreeAddress address = storage.getReplicatedMergeTreeAddress(); - String is_active_path = storage.replica_path + "/is_active"; + String is_active_path = fs::path(storage.replica_path) / "is_active"; /** If the node is marked as active, but the mark is made in the same instance, delete it. * This is possible only when session in ZooKeeper expires. @@ -302,7 +302,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() /// Simultaneously declare that this replica is active, and update the host. Coordination::Requests ops; ops.emplace_back(zkutil::makeCreateRequest(is_active_path, active_node_identifier, zkutil::CreateMode::Ephemeral)); - ops.emplace_back(zkutil::makeSetRequest(storage.replica_path + "/host", address.toString(), -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(storage.replica_path) / "host", address.toString(), -1)); try { @@ -311,7 +311,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() catch (const Coordination::Exception & e) { String existing_replica_host; - zookeeper->tryGet(storage.replica_path + "/host", existing_replica_host); + zookeeper->tryGet(fs::path(storage.replica_path) / "host", existing_replica_host); if (existing_replica_host.empty()) existing_replica_host = "without host node"; diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index e8b39c8e28c..dff864bc58c 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -41,10 +41,7 @@ public: query_info, context, max_block_size, - num_streams, - nullptr, - query_info.projection ? query_info.projection->merge_tree_data_select_base_cache.get() - : query_info.merge_tree_data_select_cache.get())); + num_streams)); return query_plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context)); diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 8f0847fc106..8a234833da7 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include @@ -70,7 +69,7 @@ IMergeTreeDataPart::Checksums checkDataPart( NamesAndTypesList columns_txt; { - auto buf = disk->readFile(path + "columns.txt"); + auto buf = disk->readFile(fs::path(path) / "columns.txt"); columns_txt.readText(*buf); assertEOF(*buf); } @@ -231,9 +230,9 @@ IMergeTreeDataPart::Checksums checkDataPart( /// Checksums from the rest files listed in checksums.txt. May be absent. If present, they are subsequently compared with the actual data checksums. IMergeTreeDataPart::Checksums checksums_txt; - if (require_checksums || disk->exists(path + "checksums.txt")) + if (require_checksums || disk->exists(fs::path(path) / "checksums.txt")) { - auto buf = disk->readFile(path + "checksums.txt"); + auto buf = disk->readFile(fs::path(path) / "checksums.txt"); checksums_txt.read(*buf); assertEOF(*buf); } diff --git a/src/Storages/MergeTree/localBackup.cpp b/src/Storages/MergeTree/localBackup.cpp index 7d7dacaeaf1..2e02481e065 100644 --- a/src/Storages/MergeTree/localBackup.cpp +++ b/src/Storages/MergeTree/localBackup.cpp @@ -29,7 +29,7 @@ static void localBackupImpl(const DiskPtr & disk, const String & source_path, co for (auto it = disk->iterateDirectory(source_path); it->isValid(); it->next()) { auto source = it->path(); - auto destination = destination_path + "/" + it->name(); + auto destination = fs::path(destination_path) / it->name(); if (!disk->isDirectory(source)) { @@ -74,13 +74,16 @@ void localBackup(const DiskPtr & disk, const String & source_path, const String continue; } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - ++try_no; - if (try_no == max_tries) - throw; - - continue; + if (e.code() == std::errc::no_such_file_or_directory) + { + ++try_no; + if (try_no == max_tries) + throw; + continue; + } + throw; } break; diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index 434071f9092..dd48b23ecc3 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -172,15 +172,15 @@ ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const metadata.sorting_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {}); metadata.primary_key = KeyDescription::getKeyFromAST({}, metadata.columns, query_context); } - if (query_select.orderBy()) + if (query.orderBy()) throw Exception( "When aggregation is used in projection, ORDER BY cannot be specified", ErrorCodes::ILLEGAL_PROJECTION); } else { result.type = ProjectionDescription::Type::Normal; - metadata.sorting_key = KeyDescription::getSortingKeyFromAST(query_select.orderBy(), metadata.columns, query_context, {}); - metadata.primary_key = KeyDescription::getKeyFromAST(query_select.orderBy(), metadata.columns, query_context); + metadata.sorting_key = KeyDescription::getSortingKeyFromAST(query.orderBy(), metadata.columns, query_context, {}); + metadata.primary_key = KeyDescription::getKeyFromAST(query.orderBy(), metadata.columns, query_context); } metadata.primary_key.definition_ast = nullptr; result.metadata = std::make_shared(metadata); @@ -265,11 +265,15 @@ void ProjectionsDescription::add(ProjectionDescription && projection, const Stri map[it->name] = it; } -void ProjectionsDescription::remove(const String & projection_name) +void ProjectionsDescription::remove(const String & projection_name, bool if_exists) { auto it = map.find(projection_name); if (it == map.end()) + { + if (if_exists) + return; throw Exception("There is no projection " + projection_name + " in table.", ErrorCodes::NO_SUCH_PROJECTION_IN_TABLE); + } projections.erase(it->second); map.erase(it); diff --git a/src/Storages/ProjectionsDescription.h b/src/Storages/ProjectionsDescription.h index 01a6f42ceed..fd505c4fe06 100644 --- a/src/Storages/ProjectionsDescription.h +++ b/src/Storages/ProjectionsDescription.h @@ -118,7 +118,7 @@ struct ProjectionsDescription void add(ProjectionDescription && projection, const String & after_projection = String(), bool first = false, bool if_not_exists = false); - void remove(const String & projection_name); + void remove(const String & projection_name, bool if_exists); private: /// Keep the sequence of columns and allow to lookup by name. diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 6f1724c6c2e..129e231e46a 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -79,7 +79,7 @@ protected: std::unique_ptr rabbitmq_settings_); private: - ContextPtr rabbitmq_context; + ContextMutablePtr rabbitmq_context; std::unique_ptr rabbitmq_settings; const String exchange_name; diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index af6f01a0ab5..70cc173e38a 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -27,14 +27,16 @@ #include #include -#include -#include #include #include #include #include +#include + + +namespace fs = std::filesystem; namespace DB { @@ -254,7 +256,7 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, rocksdb_dir = context_->getPath() + relative_data_path_; if (!attach) { - Poco::File(rocksdb_dir).createDirectories(); + fs::create_directories(rocksdb_dir); } initDb(); } @@ -262,8 +264,8 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, void StorageEmbeddedRocksDB::truncate(const ASTPtr &, const StorageMetadataPtr & , ContextPtr, TableExclusiveLockHolder &) { rocksdb_ptr->Close(); - Poco::File(rocksdb_dir).remove(true); - Poco::File(rocksdb_dir).createDirectories(); + fs::remove_all(rocksdb_dir); + fs::create_directories(rocksdb_dir); initDb(); } diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index afed41189c2..73cf3893a89 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -137,8 +137,8 @@ struct ProjectionCandidate ReadInOrderOptimizerPtr order_optimizer; InputOrderInfoPtr input_order_info; ManyExpressionActions group_by_elements_actions; - std::shared_ptr merge_tree_data_select_base_cache; - std::shared_ptr merge_tree_data_select_projection_cache; + // std::shared_ptr merge_tree_data_select_base_cache; + // std::shared_ptr merge_tree_data_select_projection_cache; }; /** Query along with some additional data, diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 1ce74b16f46..5aeaff590e1 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -185,7 +185,7 @@ void StorageDictionary::startup() bool lazy_load = global_context->getConfigRef().getBool("dictionaries_lazy_load", true); if (!lazy_load) { - auto & external_dictionaries_loader = global_context->getExternalDictionariesLoader(); + const auto & external_dictionaries_loader = global_context->getExternalDictionariesLoader(); /// reloadConfig() is called here to force loading the dictionary. external_dictionaries_loader.reloadConfig(getStorageID().getInternalDictionaryName()); @@ -220,7 +220,7 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id) configuration->setString("dictionary.database", new_table_id.database_name); configuration->setString("dictionary.name", new_table_id.table_name); - auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); + const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); external_dictionaries_loader.reloadConfig(getStorageID().getInternalDictionaryName()); auto result = external_dictionaries_loader.getLoadResult(getStorageID().getInternalDictionaryName()); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3a3291c6c48..256875454db 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -76,6 +76,8 @@ #include +namespace fs = std::filesystem; + namespace { const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_HAS_SHARDING_KEY = 1; @@ -864,7 +866,7 @@ StoragePolicyPtr StorageDistributed::getStoragePolicy() const void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk) { const std::string path(disk->getPath() + relative_data_path); - Poco::File{path}.createDirectories(); + fs::create_directories(path); std::filesystem::directory_iterator begin(path); std::filesystem::directory_iterator end; diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 61e83c611a7..9849ce7625c 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -51,6 +51,7 @@ public: bool supportsSampling() const override { return true; } bool supportsFinal() const override { return true; } bool supportsPrewhere() const override { return true; } + bool supportsSubcolumns() const override { return true; } StoragePolicyPtr getStoragePolicy() const override; bool isRemote() const override { return true; } diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 836d8bae7f3..5ca423b449a 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -31,7 +31,7 @@ static void checkAllTypesAreAllowedInTable(const NamesAndTypesList & names_and_t } -ContextPtr StorageFactory::Arguments::getContext() const +ContextMutablePtr StorageFactory::Arguments::getContext() const { auto ptr = context.lock(); if (!ptr) @@ -39,7 +39,7 @@ ContextPtr StorageFactory::Arguments::getContext() const return ptr; } -ContextPtr StorageFactory::Arguments::getLocalContext() const +ContextMutablePtr StorageFactory::Arguments::getLocalContext() const { auto ptr = local_context.lock(); if (!ptr) @@ -59,8 +59,8 @@ void StorageFactory::registerStorage(const std::string & name, CreatorFn creator StoragePtr StorageFactory::get( const ASTCreateQuery & query, const String & relative_data_path, - ContextPtr local_context, - ContextPtr context, + ContextMutablePtr local_context, + ContextMutablePtr context, const ColumnsDescription & columns, const ConstraintsDescription & constraints, bool has_force_restore_data_flag) const diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index e04e5459dda..bdc57bfdc6d 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -39,16 +39,16 @@ public: /// Relative to from server config (possibly of some of some for *MergeTree) const String & relative_data_path; const StorageID & table_id; - ContextWeakPtr local_context; - ContextWeakPtr context; + ContextWeakMutablePtr local_context; + ContextWeakMutablePtr context; const ColumnsDescription & columns; const ConstraintsDescription & constraints; bool attach; bool has_force_restore_data_flag; const String & comment; - ContextPtr getContext() const; - ContextPtr getLocalContext() const; + ContextMutablePtr getContext() const; + ContextMutablePtr getLocalContext() const; }; /// Analog of the IStorage::supports*() helpers @@ -81,8 +81,8 @@ public: StoragePtr get( const ASTCreateQuery & query, const String & relative_data_path, - ContextPtr local_context, - ContextPtr context, + ContextMutablePtr local_context, + ContextMutablePtr context, const ColumnsDescription & columns, const ConstraintsDescription & constraints, bool has_force_restore_data_flag) const; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index cfb4934dae0..aac1b708567 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -28,9 +28,6 @@ #include #include -#include -#include - #include #include #include @@ -39,6 +36,7 @@ #include #include + namespace fs = std::filesystem; namespace DB @@ -78,10 +76,9 @@ std::vector listFilesWithRegexpMatching(const std::string & path_fo std::vector result; const std::string prefix_without_globs = path_for_ls + for_match.substr(1, end_of_path_without_globs); - if (!fs::exists(fs::path(prefix_without_globs))) - { + if (!fs::exists(prefix_without_globs)) return result; - } + const fs::directory_iterator end; for (fs::directory_iterator it(prefix_without_globs); it != end; ++it) { @@ -102,7 +99,7 @@ std::vector listFilesWithRegexpMatching(const std::string & path_fo if (re2::RE2::FullMatch(file_name, matcher)) { /// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check. - Strings result_part = listFilesWithRegexpMatching(full_path + "/", suffix_with_globs.substr(next_slash)); + Strings result_part = listFilesWithRegexpMatching(fs::path(full_path) / "", suffix_with_globs.substr(next_slash)); std::move(result_part.begin(), result_part.end(), std::back_inserter(result)); } } @@ -125,21 +122,20 @@ void checkCreationIsAllowed(ContextPtr context_global, const std::string & db_di if (!startsWith(table_path, db_dir_path) && table_path != "/dev/null") throw Exception("File is not inside " + db_dir_path, ErrorCodes::DATABASE_ACCESS_DENIED); - Poco::File table_path_poco_file = Poco::File(table_path); - if (table_path_poco_file.exists() && table_path_poco_file.isDirectory()) + if (fs::exists(table_path) && fs::is_directory(table_path)) throw Exception("File must not be a directory", ErrorCodes::INCORRECT_FILE_NAME); } } Strings StorageFile::getPathsList(const String & table_path, const String & user_files_path, ContextPtr context) { - String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); - Poco::Path poco_path = Poco::Path(table_path); - if (poco_path.isRelative()) - poco_path = Poco::Path(user_files_absolute_path, poco_path); + fs::path user_files_absolute_path = fs::weakly_canonical(user_files_path); + fs::path fs_table_path(table_path); + if (fs_table_path.is_relative()) + fs_table_path = user_files_absolute_path / fs_table_path; Strings paths; - const String path = poco_path.absolute().toString(); + const String path = fs::weakly_canonical(fs_table_path); if (path.find_first_of("*?{") == std::string::npos) paths.push_back(path); else @@ -204,8 +200,8 @@ StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArgu if (args.format_name == "Distributed") throw Exception("Distributed format is allowed only with explicit file path", ErrorCodes::INCORRECT_FILE_NAME); - String table_dir_path = base_path + relative_table_dir_path + "/"; - Poco::File(table_dir_path).createDirectories(); + String table_dir_path = fs::path(base_path) / relative_table_dir_path / ""; + fs::create_directories(table_dir_path); paths = {getTablePath(table_dir_path, format_name)}; } @@ -456,7 +452,7 @@ Pipe StorageFile::read( if (use_table_fd) /// need to call ctr BlockInputStream paths = {""}; /// when use fd, paths are empty else - if (paths.size() == 1 && !Poco::File(paths[0]).exists()) + if (paths.size() == 1 && !fs::exists(paths[0])) { if (context->getSettingsRef().engine_file_empty_if_not_exists) return Pipe(std::make_shared(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()))); @@ -599,7 +595,7 @@ BlockOutputStreamPtr StorageFile::write( if (!paths.empty()) { path = paths[0]; - Poco::File(Poco::Path(path).makeParent()).createDirectories(); + fs::create_directories(fs::path(path).parent_path()); } return std::make_shared( @@ -636,8 +632,8 @@ void StorageFile::rename(const String & new_path_to_table_data, const StorageID if (path_new == paths[0]) return; - Poco::File(Poco::Path(path_new).parent()).createDirectories(); - Poco::File(paths[0]).renameTo(path_new); + fs::create_directories(fs::path(path_new).parent_path()); + fs::rename(paths[0], path_new); paths[0] = std::move(path_new); renameInMemory(new_table_id); @@ -659,7 +655,7 @@ void StorageFile::truncate( } else { - if (!Poco::File(paths[0]).exists()) + if (!fs::exists(paths[0])) return; if (0 != ::truncate(paths[0].c_str(), 0)) diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 5be8378b53e..91758931ed1 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -1,10 +1,6 @@ #pragma once #include - -#include -#include - #include #include diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index f0129e95495..fd3ca00520a 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -1,24 +1,25 @@ #include #include +#include #include #include #include -#include #include #include #include #include #include #include +#include #include #include -#include #include +#include -#include /// toLower -#include +#include #include #include +#include /// toLower namespace DB @@ -67,10 +68,18 @@ StorageJoin::StorageJoin( restore(); } +BlockOutputStreamPtr StorageJoin::write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) +{ + std::lock_guard mutate_lock(mutate_mutex); + return StorageSetOrJoinBase::write(query, metadata_snapshot, context); +} void StorageJoin::truncate( const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr, TableExclusiveLockHolder&) { + std::lock_guard mutate_lock(mutate_mutex); + std::unique_lock lock(rwlock); + disk->removeRecursive(path); disk->createDirectories(path); disk->createDirectories(path + "tmp/"); @@ -79,6 +88,70 @@ void StorageJoin::truncate( join = std::make_shared(table_join, metadata_snapshot->getSampleBlock().sortColumns(), overwrite); } +void StorageJoin::checkMutationIsPossible(const MutationCommands & commands, const Settings & /* settings */) const +{ + for (const auto & command : commands) + if (command.type != MutationCommand::DELETE) + throw Exception("Table engine Join supports only DELETE mutations", ErrorCodes::NOT_IMPLEMENTED); +} + +void StorageJoin::mutate(const MutationCommands & commands, ContextPtr context) +{ + /// Firstly accuire lock for mutation, that locks changes of data. + /// We cannot accuire rwlock here, because read lock is needed + /// for execution of mutation interpreter. + std::lock_guard mutate_lock(mutate_mutex); + + constexpr auto tmp_backup_file_name = "tmp/mut.bin"; + auto metadata_snapshot = getInMemoryMetadataPtr(); + + auto backup_buf = disk->writeFile(path + tmp_backup_file_name); + auto compressed_backup_buf = CompressedWriteBuffer(*backup_buf); + auto backup_stream = NativeBlockOutputStream(compressed_backup_buf, 0, metadata_snapshot->getSampleBlock()); + + auto new_data = std::make_shared(table_join, metadata_snapshot->getSampleBlock().sortColumns(), overwrite); + + // New scope controls lifetime of InputStream. + { + auto storage_ptr = DatabaseCatalog::instance().getTable(getStorageID(), context); + auto interpreter = std::make_unique(storage_ptr, metadata_snapshot, commands, context, true); + auto in = interpreter->execute(); + in->readPrefix(); + + while (const Block & block = in->read()) + { + new_data->addJoinedBlock(block, true); + if (persistent) + backup_stream.write(block); + } + + in->readSuffix(); + } + + /// Now accuire exclusive lock and modify storage. + std::unique_lock lock(rwlock); + + join = std::move(new_data); + increment = 1; + + if (persistent) + { + backup_stream.flush(); + compressed_backup_buf.next(); + backup_buf->next(); + backup_buf->finalize(); + + std::vector files; + disk->listFiles(path, files); + for (const auto & file_name: files) + { + if (file_name.ends_with(".bin")) + disk->removeFileIfExists(path + file_name); + } + + disk->replaceFile(path + tmp_backup_file_name, path + std::to_string(increment) + ".bin"); + } +} HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join) const { @@ -93,7 +166,9 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join) /// TODO: check key columns - /// Some HACK to remove wrong names qualifiers: table.column -> column. + /// Set names qualifiers: table.column -> column + /// It's required because storage join stores non-qualified names + /// Qualifies will be added by join implementation (HashJoin) analyzed_join->setRightKeys(key_names); HashJoinPtr join_clone = std::make_shared(analyzed_join, metadata_snapshot->getSampleBlock().sortColumns()); diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index b89780a0f75..78d8b9768e9 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -29,6 +29,10 @@ public: void truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr, TableExclusiveLockHolder &) override; + /// Only delete is supported. + void checkMutationIsPossible(const MutationCommands & commands, const Settings & settings) const override; + void mutate(const MutationCommands & commands, ContextPtr context) override; + /// Return instance of HashJoin holding lock that protects from insertions to StorageJoin. /// HashJoin relies on structure of hash table that's why we need to return it with locked mutex. HashJoinPtr getJoinLocked(std::shared_ptr analyzed_join) const; @@ -41,6 +45,8 @@ public: /// (but not during processing whole query, it's safe for joinGet that doesn't involve `used_flags` from HashJoin) ColumnWithTypeAndName joinGet(const Block & block, const Block & block_with_columns_to_add) const; + BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) override; + Pipe read( const Names & column_names, const StorageMetadataPtr & /*metadata_snapshot*/, @@ -68,6 +74,7 @@ private: /// Protect state for concurrent use in insertFromBlock and joinBlock. /// Lock is stored in HashJoin instance during query and blocks concurrent insertions. mutable std::shared_mutex rwlock; + mutable std::mutex mutate_mutex; void insertBlock(const Block & block) override; void finishInsert() override {} diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 67bd6b21c3f..a66b24ff931 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -54,7 +54,7 @@ StorageMaterializedView::StorageMaterializedView( const ASTCreateQuery & query, const ColumnsDescription & columns_, bool attach_) - : IStorage(table_id_), WithContext(local_context->getGlobalContext()) + : IStorage(table_id_), WithMutableContext(local_context->getGlobalContext()) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); @@ -228,7 +228,7 @@ static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, if (auto txn = current_context->getZooKeeperMetadataTransaction()) { /// For Replicated database - drop_context->setQueryContext(current_context); + drop_context->setQueryContext(std::const_pointer_cast(current_context)); drop_context->initZooKeeperMetadataTransaction(txn, true); } InterpreterDropQuery drop_interpreter(ast_drop_query, drop_context); @@ -431,7 +431,12 @@ Strings StorageMaterializedView::getDataPaths() const ActionLock StorageMaterializedView::getActionLock(StorageActionBlockType type) { - return has_inner_table ? getTargetTable()->getActionLock(type) : ActionLock{}; + if (has_inner_table) + { + if (auto target_table = tryGetTargetTable()) + return target_table->getActionLock(type); + } + return ActionLock{}; } void registerStorageMaterializedView(StorageFactory & factory) diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 8f9c8a9d3f1..8e2b8bdc9f5 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -12,7 +12,7 @@ namespace DB { -class StorageMaterializedView final : public ext::shared_ptr_helper, public IStorage, WithContext +class StorageMaterializedView final : public ext::shared_ptr_helper, public IStorage, WithMutableContext { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5a84e0c3901..15d520c13aa 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -332,7 +332,7 @@ Pipe StorageMerge::createSources( const Block & header, const StorageWithLockAndName & storage_with_lock, Names & real_column_names, - ContextPtr modified_context, + ContextMutablePtr modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams) diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 7c5a50af486..b9d44bfa27e 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -92,7 +92,7 @@ protected: const Block & header, const StorageWithLockAndName & storage_with_lock, Names & real_column_names, - ContextPtr modified_context, + ContextMutablePtr modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams = false); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index d656c7c088d..3d3530825d9 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -62,7 +62,7 @@ StorageMergeTree::StorageMergeTree( const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, bool attach, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr storage_settings_, @@ -1240,56 +1240,66 @@ MergeTreeDataPartPtr StorageMergeTree::outdatePart(const String & part_name, boo } } -void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr local_context, bool throw_if_noop) +void StorageMergeTree::dropPartNoWaitNoThrow(const String & part_name) { + if (auto part = outdatePart(part_name, /*force=*/ false)) + dropPartsImpl({part}, /*detach=*/ false); + + /// Else nothing to do, part was removed in some different way +} + +void StorageMergeTree::dropPart(const String & part_name, bool detach, ContextPtr /*query_context*/) +{ + if (auto part = outdatePart(part_name, /*force=*/ true)) + dropPartsImpl({part}, detach); +} + +void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, ContextPtr local_context) +{ + DataPartsVector parts_to_remove; + /// New scope controls lifetime of merge_blocker. { - MergeTreeData::DataPartsVector parts_to_remove; - auto metadata_snapshot = getInMemoryMetadataPtr(); + /// Asks to complete merges and does not allow them to start. + /// This protects against "revival" of data for a removed partition after completion of merge. + auto merge_blocker = stopMergesAndWait(); + String partition_id = getPartitionIDFromQuery(partition, local_context); + parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); - if (drop_part) - { - auto part = outdatePart(partition->as().value.safeGet(), throw_if_noop); - /// Nothing to do, part was removed in some different way - if (!part) - return; - - parts_to_remove.push_back(part); - } - else - { - /// Asks to complete merges and does not allow them to start. - /// This protects against "revival" of data for a removed partition after completion of merge. - auto merge_blocker = stopMergesAndWait(); - String partition_id = getPartitionIDFromQuery(partition, local_context); - parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); - - /// TODO should we throw an exception if parts_to_remove is empty? - removePartsFromWorkingSet(parts_to_remove, true); - } - - if (detach) - { - /// If DETACH clone parts to detached/ directory - /// NOTE: no race with background cleanup until we hold pointers to parts - for (const auto & part : parts_to_remove) - { - LOG_INFO(log, "Detaching {}", part->relative_path); - part->makeCloneInDetached("", metadata_snapshot); - } - } - - if (deduplication_log) - { - for (const auto & part : parts_to_remove) - deduplication_log->dropPart(part->info); - } - - if (detach) - LOG_INFO(log, "Detached {} parts.", parts_to_remove.size()); - else - LOG_INFO(log, "Removed {} parts.", parts_to_remove.size()); + /// TODO should we throw an exception if parts_to_remove is empty? + removePartsFromWorkingSet(parts_to_remove, true); } + dropPartsImpl(std::move(parts_to_remove), detach); +} + +void StorageMergeTree::dropPartsImpl(DataPartsVector && parts_to_remove, bool detach) +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + + if (detach) + { + /// If DETACH clone parts to detached/ directory + /// NOTE: no race with background cleanup until we hold pointers to parts + for (const auto & part : parts_to_remove) + { + LOG_INFO(log, "Detaching {}", part->relative_path); + part->makeCloneInDetached("", metadata_snapshot); + } + } + + if (deduplication_log) + { + for (const auto & part : parts_to_remove) + deduplication_log->dropPart(part->info); + } + + if (detach) + LOG_INFO(log, "Detached {} parts.", parts_to_remove.size()); + else + LOG_INFO(log, "Removed {} parts.", parts_to_remove.size()); + + /// Need to destroy part objects before clearing them from filesystem. + parts_to_remove.clear(); clearOldPartsFromFilesystem(); } @@ -1509,8 +1519,8 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_ auto disk = part->volume->getDisk(); String part_path = part->getFullRelativePath(); /// If the checksums file is not present, calculate the checksums and write them to disk. - String checksums_path = part_path + "checksums.txt"; - String tmp_checksums_path = part_path + "checksums.txt.tmp"; + String checksums_path = fs::path(part_path) / "checksums.txt"; + String tmp_checksums_path = fs::path(part_path) / "checksums.txt.tmp"; if (part->isStoredOnDisk() && !disk->exists(checksums_path)) { try diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 53199e1595a..72fbb37bdd9 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -211,7 +211,10 @@ private: void clearOldMutations(bool truncate = false); // Partition helpers - void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr context, bool throw_if_noop) override; + void dropPartNoWaitNoThrow(const String & part_name) override; + void dropPart(const String & part_name, bool detach, ContextPtr context) override; + void dropPartition(const ASTPtr & partition, bool detach, ContextPtr context) override; + void dropPartsImpl(DataPartsVector && parts_to_remove, bool detach); PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr context) override; void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr context) override; @@ -250,7 +253,7 @@ protected: const String & relative_data_path_, const StorageInMemoryMetadata & metadata, bool attach, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b3da3e2287b..cb48d08b64e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -66,9 +66,12 @@ #include #include #include +#include #include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event ReplicatedPartMerges; @@ -253,7 +256,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, @@ -272,7 +275,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , zookeeper_name(extractZooKeeperName(zookeeper_path_)) , zookeeper_path(extractZooKeeperPath(zookeeper_path_)) , replica_name(replica_name_) - , replica_path(zookeeper_path + "/replicas/" + replica_name_) + , replica_path(fs::path(zookeeper_path) / "replicas" / replica_name_) , reader(*this) , writer(*this) , merger_mutator(*this, getContext()->getSettingsRef().background_pool_size) @@ -493,14 +496,14 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Mutation maybe killed or whole replica was deleted. /// Wait event will unblock at this moment. Coordination::Stat exists_stat; - if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id, &exists_stat, wait_event)) + if (!getZooKeeper()->exists(fs::path(zookeeper_path) / "mutations" / mutation_id, &exists_stat, wait_event)) { throw Exception(ErrorCodes::UNFINISHED, "Mutation {} was killed, manually removed or table was dropped", mutation_id); } auto zookeeper = getZooKeeper(); /// Replica could be inactive. - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (!zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) { LOG_WARNING(log, "Replica {} is not active during mutation. Mutation will be done asynchronously when replica becomes active.", replica); @@ -508,7 +511,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( break; } - String mutation_pointer = zookeeper_path + "/replicas/" + replica + "/mutation_pointer"; + String mutation_pointer = fs::path(zookeeper_path) / "replicas" / replica / "mutation_pointer"; std::string mutation_pointer_value; /// Replica could be removed if (!zookeeper->tryGet(mutation_pointer, mutation_pointer_value, nullptr, wait_event)) @@ -523,16 +526,17 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( if (wait_event->tryWait(1000)) continue; - /// Here we check mutation for errors or kill on local replica. If they happen on this replica + /// Here we check mutation for errors on local replica. If they happen on this replica /// they will happen on each replica, so we can check only in-memory info. auto mutation_status = queue.getIncompleteMutationsStatus(mutation_id); - if (!mutation_status || !mutation_status->latest_fail_reason.empty()) + /// If mutation status is empty, than local replica may just not loaded it into memory. + if (mutation_status && !mutation_status->latest_fail_reason.empty()) break; } /// It maybe already removed from zk, but local in-memory mutations /// state was not updated. - if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id)) + if (!getZooKeeper()->exists(fs::path(zookeeper_path) / "mutations" / mutation_id)) { throw Exception(ErrorCodes::UNFINISHED, "Mutation {} was killed, manually removed or table was dropped", mutation_id); } @@ -877,12 +881,12 @@ bool StorageReplicatedMergeTree::removeTableNodesFromZooKeeper(zkutil::ZooKeeper for (const auto & child : children) if (child != "dropped") - zookeeper->tryRemoveRecursive(zookeeper_path + "/" + child); + zookeeper->tryRemoveRecursive(fs::path(zookeeper_path) / child); Coordination::Requests ops; Coordination::Responses responses; ops.emplace_back(zkutil::makeRemoveRequest(metadata_drop_lock->getPath(), -1)); - ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/dropped", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(zookeeper_path) / "dropped", -1)); ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path, -1)); code = zookeeper->tryMulti(ops, responses); @@ -921,12 +925,12 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr ReplicatedMergeTreeTableMetadata old_metadata(*this, metadata_snapshot); Coordination::Stat metadata_stat; - String metadata_str = zookeeper->get(zookeeper_prefix + "/metadata", &metadata_stat); + String metadata_str = zookeeper->get(fs::path(zookeeper_prefix) / "metadata", &metadata_stat); auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); old_metadata.checkEquals(metadata_from_zk, metadata_snapshot->getColumns(), getContext()); Coordination::Stat columns_stat; - auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_prefix + "/columns", &columns_stat)); + auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(fs::path(zookeeper_prefix) / "columns", &columns_stat)); const ColumnsDescription & old_columns = metadata_snapshot->getColumns(); if (columns_from_zk != old_columns) @@ -1071,7 +1075,7 @@ static time_t tryGetPartCreateTime(zkutil::ZooKeeperPtr & zookeeper, const Strin /// We get creation time of part, if it still exists (was not merged, for example). Coordination::Stat stat; String unused; - if (zookeeper->tryGet(replica_path + "/parts/" + part_name, unused, &stat)) + if (zookeeper->tryGet(fs::path(replica_path) / "parts" / part_name, unused, &stat)) res = stat.ctime / 1000; return res; @@ -1082,7 +1086,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) { auto zookeeper = getZooKeeper(); - Strings expected_parts_vec = zookeeper->getChildren(replica_path + "/parts"); + Strings expected_parts_vec = zookeeper->getChildren(fs::path(replica_path) / "parts"); /// Parts in ZK. NameSet expected_parts(expected_parts_vec.begin(), expected_parts_vec.end()); @@ -1185,7 +1189,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) exists_futures.reserve(parts_to_fetch.size()); for (const String & part_name : parts_to_fetch) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -1214,7 +1218,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) /// We assume that this occurs before the queue is loaded (queue.initialize). ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(replica_path) / "queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); enqueue_futures.emplace_back(zookeeper->asyncMulti(ops)); } @@ -1261,13 +1265,13 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( part->getColumns(), part->checksums); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); bool has_been_already_added = false; for (const String & replica : replicas) { - String current_part_path = zookeeper_path + "/replicas/" + replica + "/parts/" + part_name; + String current_part_path = fs::path(zookeeper_path) / "replicas" / replica / "parts" / part_name; String part_zk_str; if (!zookeeper->tryGet(current_part_path, part_zk_str)) @@ -1288,9 +1292,9 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: String checksums_str; /// Let's check that the node's version with the columns did not change while we were reading the checksums. /// This ensures that the columns and the checksum refer to the same - if (!zookeeper->tryGet(current_part_path + "/columns", columns_str, &columns_stat_before) || - !zookeeper->tryGet(current_part_path + "/checksums", checksums_str) || - !zookeeper->exists(current_part_path + "/columns", &columns_stat_after) || + if (!zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str, &columns_stat_before) || + !zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str) || + !zookeeper->exists(fs::path(current_part_path) / "columns", &columns_stat_after) || columns_stat_before.version != columns_stat_after.version) { LOG_INFO(log, "Not checking checksums of part {} with replica {} because part changed while we were reading its checksums", part_name, replica); @@ -1324,7 +1328,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (!has_been_already_added) { const auto storage_settings_ptr = getSettings(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; //ops.emplace_back(zkutil::makeCheckRequest( // zookeeper_path + "/columns", expected_columns_version)); @@ -1339,14 +1343,15 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - part_path + "/columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); + fs::path(part_path) / "columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - part_path + "/checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); + fs::path(part_path) / "checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); } } else { - LOG_WARNING(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", replica_path + "/parts/" + part_name); + LOG_WARNING(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", + (fs::path(replica_path) / "parts" / part_name).string()); } } @@ -1414,7 +1419,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo const String part_new_name = actual_part_info.getPartName(); for (const DiskPtr & disk : getStoragePolicy()->getDisks()) - for (const auto it = disk->iterateDirectory(relative_data_path + "detached/"); it->isValid(); it->next()) + for (const auto it = disk->iterateDirectory(fs::path(relative_data_path) / "detached/"); it->isValid(); it->next()) { MergeTreePartInfo part_info; @@ -1423,7 +1428,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo continue; const String part_old_name = part_info.getPartName(); - const String part_path = "detached/" + part_old_name; + const String part_path = fs::path("detached") / part_old_name; const VolumePtr volume = std::make_shared("volume_" + part_old_name, disk); @@ -1478,7 +1483,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) existing_part = getActiveContainingPart(entry.new_part_name); /// Even if the part is local, it (in exceptional cases) may not be in ZooKeeper. Let's check that it is there. - if (existing_part && getZooKeeper()->exists(replica_path + "/parts/" + existing_part->name)) + if (existing_part && getZooKeeper()->exists(fs::path(replica_path) / "parts" / existing_part->name)) { if (!is_get_or_attach || entry.source_replica != replica_name) LOG_DEBUG(log, "Skipping action for part {} because part {} already exists.", @@ -1513,7 +1518,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) /// Perhaps we don't need this part, because during write with quorum, the quorum has failed /// (see below about `/quorum/failed_parts`). - if (entry.quorum && getZooKeeper()->exists(zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name)) + if (entry.quorum && getZooKeeper()->exists(fs::path(zookeeper_path) / "quorum" / "failed_parts" / entry.new_part_name)) { LOG_DEBUG(log, "Skipping action for part {} because quorum for that part was failed.", entry.new_part_name); return true; /// NOTE Deletion from `virtual_parts` is not done, but it is only necessary for merge. @@ -1983,14 +1988,14 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); Coordination::Requests ops; for (const auto & path_part : replicas) { Coordination::Stat stat; - String path = zookeeper_path + "/replicas/" + path_part + "/host"; + String path = fs::path(zookeeper_path) / "replicas" / path_part / "host"; zookeeper->get(path, &stat); ops.emplace_back(zkutil::makeCheckRequest(path, stat.version)); } @@ -2004,8 +2009,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (replica.empty()) { Coordination::Stat quorum_stat; - const String quorum_unparallel_path = zookeeper_path + "/quorum/status"; - const String quorum_parallel_path = zookeeper_path + "/quorum/parallel/" + entry.new_part_name; + const String quorum_unparallel_path = fs::path(zookeeper_path) / "quorum" / "status"; + const String quorum_parallel_path = fs::path(zookeeper_path) / "quorum" / "parallel" / entry.new_part_name; String quorum_str, quorum_path; ReplicatedMergeTreeQuorumEntry quorum_entry; @@ -2029,13 +2034,13 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) ErrorCodes::LOGICAL_ERROR); ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name, + fs::path(zookeeper_path) / "quorum" / "failed_parts" / entry.new_part_name, "", zkutil::CreateMode::Persistent)); /// Deleting from `blocks`. - if (!entry.block_id.empty() && zookeeper->exists(zookeeper_path + "/blocks/" + entry.block_id)) - ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/blocks/" + entry.block_id, -1)); + if (!entry.block_id.empty() && zookeeper->exists(fs::path(zookeeper_path) / "blocks" / entry.block_id)) + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(zookeeper_path) / "blocks" / entry.block_id, -1)); Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); @@ -2048,7 +2053,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else if (code == Coordination::Error::ZBADVERSION || code == Coordination::Error::ZNONODE || code == Coordination::Error::ZNODEEXISTS) { - LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", entry.new_part_name, Coordination::errorMessage(code)); + LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", + entry.new_part_name, Coordination::errorMessage(code)); } else throw Coordination::Exception(code); @@ -2070,7 +2076,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) try { String part_name = entry.actual_new_part_name.empty() ? entry.new_part_name : entry.actual_new_part_name; - if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum)) + if (!fetchPart(part_name, metadata_snapshot, fs::path(zookeeper_path) / "replicas" / replica, false, entry.quorum)) return false; } catch (Exception & e) @@ -2154,7 +2160,7 @@ bool StorageReplicatedMergeTree::executeFetchShared( try { - if (!fetchExistsPart(new_part_name, metadata_snapshot, zookeeper_path + "/replicas/" + source_replica, disk, path)) + if (!fetchExistsPart(new_part_name, metadata_snapshot, fs::path(zookeeper_path) / "replicas" / source_replica, disk, path)) return false; } catch (Exception & e) @@ -2489,8 +2495,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } else if (!part_desc->replica.empty()) { - String source_replica_path = zookeeper_path + "/replicas/" + part_desc->replica; - ReplicatedMergeTreeAddress address(getZooKeeper()->get(source_replica_path + "/host")); + String source_replica_path = fs::path(zookeeper_path) / "replicas" / part_desc->replica; + ReplicatedMergeTreeAddress address(getZooKeeper()->get(fs::path(source_replica_path) / "host")); auto timeouts = getFetchPartHTTPTimeouts(getContext()); auto credentials = getContext()->getInterserverCredentials(); @@ -2632,18 +2638,18 @@ void StorageReplicatedMergeTree::executeClonePartFromShard(const LogEntry & entr void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coordination::Stat source_is_lost_stat, zkutil::ZooKeeperPtr & zookeeper) { - String source_path = zookeeper_path + "/replicas/" + source_replica; + String source_path = fs::path(zookeeper_path) / "replicas" / source_replica; /** TODO: it will be deleted! (It is only to support old version of CH server). * In current code, the replica is created in single transaction. * If the reference/master replica is not yet fully created, let's wait. */ - while (!zookeeper->exists(source_path + "/columns")) + while (!zookeeper->exists(fs::path(source_path) / "columns")) { LOG_INFO(log, "Waiting for replica {} to be fully created", source_path); zkutil::EventPtr event = std::make_shared(); - if (zookeeper->exists(source_path + "/columns", nullptr, event)) + if (zookeeper->exists(fs::path(source_path) / "columns", nullptr, event)) { LOG_WARNING(log, "Oops, a watch has leaked"); break; @@ -2660,29 +2666,29 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo while (true) { Coordination::Stat log_pointer_stat; - String raw_log_pointer = zookeeper->get(source_path + "/log_pointer", &log_pointer_stat); + String raw_log_pointer = zookeeper->get(fs::path(source_path) / "log_pointer", &log_pointer_stat); Coordination::Requests ops; - ops.push_back(zkutil::makeSetRequest(replica_path + "/log_pointer", raw_log_pointer, -1)); + ops.push_back(zkutil::makeSetRequest(fs::path(replica_path) / "log_pointer", raw_log_pointer, -1)); /// For support old versions CH. if (source_is_lost_stat.version == -1) { /// We check that it was not suddenly upgraded to new version. /// Otherwise it can be upgraded and instantly become lost, but we cannot notice that. - ops.push_back(zkutil::makeCreateRequest(source_path + "/is_lost", "0", zkutil::CreateMode::Persistent)); - ops.push_back(zkutil::makeRemoveRequest(source_path + "/is_lost", -1)); + ops.push_back(zkutil::makeCreateRequest(fs::path(source_path) / "is_lost", "0", zkutil::CreateMode::Persistent)); + ops.push_back(zkutil::makeRemoveRequest(fs::path(source_path) / "is_lost", -1)); } else /// The replica we clone should not suddenly become lost. - ops.push_back(zkutil::makeCheckRequest(source_path + "/is_lost", source_is_lost_stat.version)); + ops.push_back(zkutil::makeCheckRequest(fs::path(source_path) / "is_lost", source_is_lost_stat.version)); Coordination::Responses responses; /// Let's remember the queue of the reference/master replica. - source_queue_names = zookeeper->getChildren(source_path + "/queue"); + source_queue_names = zookeeper->getChildren(fs::path(source_path) / "queue"); /// Check that log pointer of source replica didn't changed while we read queue entries - ops.push_back(zkutil::makeCheckRequest(source_path + "/log_pointer", log_pointer_stat.version)); + ops.push_back(zkutil::makeCheckRequest(fs::path(source_path) / "log_pointer", log_pointer_stat.version)); auto rc = zookeeper->tryMulti(ops, responses); @@ -2722,7 +2728,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo for (const String & entry_name : source_queue_names) { String entry; - if (!zookeeper->tryGet(source_path + "/queue/" + entry_name, entry)) + if (!zookeeper->tryGet(fs::path(source_path) / "queue" / entry_name, entry)) continue; source_queue.push_back(entry); } @@ -2731,13 +2737,13 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo cloneMetadataIfNeeded(source_replica, source_path, zookeeper); /// Add to the queue jobs to receive all the active parts that the reference/master replica has. - Strings source_replica_parts = zookeeper->getChildren(source_path + "/parts"); + Strings source_replica_parts = zookeeper->getChildren(fs::path(source_path) / "parts"); ActiveDataPartSet active_parts_set(format_version, source_replica_parts); Strings active_parts = active_parts_set.getParts(); /// Remove local parts if source replica does not have them, because such parts will never be fetched by other replicas. - Strings local_parts_in_zk = zookeeper->getChildren(replica_path + "/parts"); + Strings local_parts_in_zk = zookeeper->getChildren(fs::path(replica_path) / "parts"); Strings parts_to_remove_from_zk; for (const auto & part : local_parts_in_zk) { @@ -2781,7 +2787,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo log_entry.new_part_name = name; log_entry.create_time = tryGetPartCreateTime(zookeeper, source_path, name); - zookeeper->create(replica_path + "/queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); + zookeeper->create(fs::path(replica_path) / "queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); } LOG_DEBUG(log, "Queued {} parts to be fetched", active_parts.size()); @@ -2789,7 +2795,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo /// Add content of the reference/master replica queue to the queue. for (const String & entry : source_queue) { - zookeeper->create(replica_path + "/queue/queue-", entry, zkutil::CreateMode::PersistentSequential); + zookeeper->create(fs::path(replica_path) / "queue/queue-", entry, zkutil::CreateMode::PersistentSequential); } LOG_DEBUG(log, "Copied {} queue entries", source_queue.size()); @@ -2876,7 +2882,7 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke Coordination::Stat is_lost_stat; bool is_new_replica = true; String res; - if (zookeeper->tryGet(replica_path + "/is_lost", res, &is_lost_stat)) + if (zookeeper->tryGet(fs::path(replica_path) / "is_lost", res, &is_lost_stat)) { if (res == "0") return; @@ -2888,14 +2894,14 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke /// Replica was created by old version of CH, so me must create "/is_lost". /// Note that in old version of CH there was no "lost" replicas possible. /// TODO is_lost node should always exist since v18.12, maybe we can replace `tryGet` with `get` and remove old code? - zookeeper->create(replica_path + "/is_lost", "0", zkutil::CreateMode::Persistent); + zookeeper->create(fs::path(replica_path) / "is_lost", "0", zkutil::CreateMode::Persistent); return; } /// is_lost is "1": it means that we are in repair mode. /// Try choose source replica to clone. /// Source replica must not be lost and should have minimal queue size and maximal log pointer. - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); std::vector futures; for (const String & source_replica_name : replicas) { @@ -2903,20 +2909,20 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke if (source_replica_name == replica_name) continue; - String source_replica_path = zookeeper_path + "/replicas/" + source_replica_name; + String source_replica_path = fs::path(zookeeper_path) / "replicas" / source_replica_name; /// Obviously the following get operations are not atomic, but it's ok to choose good enough replica, not the best one. /// NOTE: We may count some entries twice if log_pointer is moved. - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/is_lost")); - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/log_pointer")); - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/queue")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "is_lost")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "log_pointer")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "queue")); } /// Wait for results before getting log entries for (auto & future : futures) future.wait(); - Strings log_entries = zookeeper->getChildren(zookeeper_path + "/log"); + Strings log_entries = zookeeper->getChildren(fs::path(zookeeper_path) / "log"); size_t max_log_entry = 0; if (!log_entries.empty()) { @@ -2987,14 +2993,14 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke LOG_WARNING(log, "Will mimic {}", source_replica); /// Clear obsolete queue that we no longer need. - zookeeper->removeChildren(replica_path + "/queue"); + zookeeper->removeChildren(fs::path(replica_path) / "queue"); /// Will do repair from the selected replica. cloneReplica(source_replica, source_is_lost_stat, zookeeper); /// If repair fails to whatever reason, the exception is thrown, is_lost will remain "1" and the replica will be repaired later. /// If replica is repaired successfully, we remove is_lost flag. - zookeeper->set(replica_path + "/is_lost", "0"); + zookeeper->set(fs::path(replica_path) / "is_lost", "0"); } @@ -3321,7 +3327,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c std::vector> exists_futures; exists_futures.reserve(parts.size()); for (const auto & part : parts) - exists_futures.emplace_back(zookeeper->asyncExists(replica_path + "/parts/" + part->name)); + exists_futures.emplace_back(zookeeper->asyncExists(fs::path(replica_path) / "parts" / part->name)); bool all_in_zk = true; for (size_t i = 0; i < parts.size(); ++i) @@ -3361,11 +3367,11 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c Coordination::Responses responses; ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry.toString(), + fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeSetRequest( - zookeeper_path + "/log", "", log_version)); /// Check and update version. + fs::path(zookeeper_path) / "log", "", log_version)); /// Check and update version. Coordination::Error code = zookeeper->tryMulti(ops, responses); @@ -3401,7 +3407,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c auto zookeeper = getZooKeeper(); /// If there is no information about part in ZK, we will not mutate it. - if (!zookeeper->exists(replica_path + "/parts/" + part.name)) + if (!zookeeper->exists(fs::path(replica_path) / "parts" / part.name)) { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { @@ -3431,11 +3437,11 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c Coordination::Responses responses; ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry.toString(), + fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeSetRequest( - zookeeper_path + "/log", "", log_version)); /// Check and update version. + fs::path(zookeeper_path) / "log", "", log_version)); /// Check and update version. Coordination::Error code = zookeeper->tryMulti(ops, responses); @@ -3456,12 +3462,12 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_name, Coordination::Requests & ops, bool has_children) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; if (has_children) { - ops.emplace_back(zkutil::makeRemoveRequest(part_path + "/checksums", -1)); - ops.emplace_back(zkutil::makeRemoveRequest(part_path + "/columns", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(part_path) / "checksums", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(part_path) / "columns", -1)); } ops.emplace_back(zkutil::makeRemoveRequest(part_path, -1)); } @@ -3469,7 +3475,7 @@ void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_nam void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_name) { auto zookeeper = getZooKeeper(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; Coordination::Stat stat; /// Part doesn't exist, nothing to remove @@ -3486,7 +3492,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n { auto zookeeper = getZooKeeper(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; Coordination::Requests ops; @@ -3505,7 +3511,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n log_entry->new_part_name = part_name; ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", log_entry->toString(), + fs::path(replica_path) / "queue/queue-", log_entry->toString(), zkutil::CreateMode::PersistentSequential)); auto results = zookeeper->multi(ops); @@ -3530,7 +3536,7 @@ void StorageReplicatedMergeTree::enterLeaderElection() { leader_election = std::make_shared( getContext()->getSchedulePool(), - zookeeper_path + "/leader_election", + fs::path(zookeeper_path) / "leader_election", *current_zookeeper, /// current_zookeeper lives for the lifetime of leader_election, /// since before changing `current_zookeeper`, `leader_election` object is destroyed in `partialShutdown` method. callback, @@ -3585,13 +3591,13 @@ ConnectionTimeouts StorageReplicatedMergeTree::getFetchPartHTTPTimeouts(ContextP bool StorageReplicatedMergeTree::checkReplicaHavePart(const String & replica, const String & part_name) { auto zookeeper = getZooKeeper(); - return zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name); + return zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "parts" / part_name); } String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_name, bool active) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3607,7 +3613,7 @@ String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_nam LOG_TRACE(log, "Candidate replica: {}", replica); if (checkReplicaHavePart(replica, part_name) && - (!active || zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + (!active || zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active"))) return replica; /// Obviously, replica could become inactive or even vanish after return from this method. @@ -3619,7 +3625,7 @@ String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_nam String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entry, bool active) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3629,11 +3635,11 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entr if (replica == replica_name) continue; - if (active && !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (active && !zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; String largest_part_found; - Strings parts = zookeeper->getChildren(zookeeper_path + "/replicas/" + replica + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas" / replica / "parts"); for (const String & part_on_replica : parts) { if (part_on_replica == entry.new_part_name @@ -3674,7 +3680,7 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( const String & part_name, bool active, String & found_part_name) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3687,10 +3693,10 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( if (replica == replica_name) continue; - if (active && !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (active && !zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; - Strings parts = zookeeper->getChildren(zookeeper_path + "/replicas/" + replica + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas" / replica / "parts"); for (const String & part_on_replica : parts) { if (part_on_replica == part_name @@ -3718,11 +3724,11 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, bool is_ auto zookeeper = getZooKeeper(); /// Information on which replicas a part has been added, if the quorum has not yet been reached. - String quorum_status_path = zookeeper_path + "/quorum/status"; + String quorum_status_path = fs::path(zookeeper_path) / "quorum" / "status"; if (is_parallel) - quorum_status_path = zookeeper_path + "/quorum/parallel/" + part_name; + quorum_status_path = fs::path(zookeeper_path) / "quorum" / "parallel" / part_name; /// The name of the previous part for which the quorum was reached. - const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + const String quorum_last_part_path = fs::path(zookeeper_path) / "quorum" / "last_part"; String value; Coordination::Stat stat; @@ -3824,7 +3830,7 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) auto zookeeper = getZooKeeper(); /// The name of the previous part for which the quorum was reached. - const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + const String quorum_last_part_path = fs::path(zookeeper_path) / "quorum" / "last_part"; /// Delete information from "last_part" node. @@ -3874,7 +3880,7 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) bool StorageReplicatedMergeTree::partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const { auto zookeeper = getZooKeeper(); - return zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_info.getPartName()); + return zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "parallel" / part_info.getPartName()); } @@ -3882,7 +3888,7 @@ bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & { auto zookeeper = getZooKeeper(); - const String parts_with_quorum_path = zookeeper_path + "/quorum/last_part"; + const String parts_with_quorum_path = fs::path(zookeeper_path) / "quorum" / "last_part"; String parts_with_quorum_str = zookeeper->get(parts_with_quorum_path); @@ -3966,13 +3972,13 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora source_part_checksums.computeTotalChecksums(source_part->checksums); MinimalisticDataPartChecksums desired_checksums; - String part_path = source_replica_path + "/parts/" + part_name; + String part_path = fs::path(source_replica_path) / "parts" / part_name; String part_znode = zookeeper->get(part_path); if (!part_znode.empty()) desired_checksums = ReplicatedMergeTreePartHeader::fromString(part_znode).getChecksums(); else { - String desired_checksums_str = zookeeper->get(part_path + "/checksums"); + String desired_checksums_str = zookeeper->get(fs::path(part_path) / "checksums"); desired_checksums = MinimalisticDataPartChecksums::deserializeFrom(desired_checksums_str); } @@ -4001,7 +4007,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora } else { - address.fromString(zookeeper->get(source_replica_path + "/host")); + address.fromString(zookeeper->get(fs::path(source_replica_path) / "host")); timeouts = getFetchPartHTTPTimeouts(getContext()); credentials = getContext()->getInterserverCredentials(); @@ -4049,16 +4055,16 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (quorum) { /// Check if this quorum insert is parallel or not - if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) + if (zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "parallel" / part_name)) updateQuorum(part_name, true); - else if (zookeeper->exists(zookeeper_path + "/quorum/status")) + else if (zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "status")) updateQuorum(part_name, false); } /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before if (part_info.level != 0 || part_info.mutation != 0) { - Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); + Strings quorum_parts = zookeeper->getChildren(fs::path(zookeeper_path) / "quorum" / "parallel"); for (const String & quorum_part : quorum_parts) { auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); @@ -4081,7 +4087,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora { // The fetched part is valuable and should not be cleaned like a temp part. part->is_temp = false; - part->renameTo("detached/" + part_name, true); + part->renameTo(fs::path("detached") / part_name, true); } } catch (const Exception & e) @@ -4159,7 +4165,7 @@ bool StorageReplicatedMergeTree::fetchExistsPart(const String & part_name, const std::function get_part; - ReplicatedMergeTreeAddress address(zookeeper->get(source_replica_path + "/host")); + ReplicatedMergeTreeAddress address(zookeeper->get(fs::path(source_replica_path) / "host")); auto timeouts = ConnectionTimeouts::getHTTPTimeouts(getContext()); auto credentials = getContext()->getInterserverCredentials(); String interserver_scheme = getContext()->getInterserverScheme(); @@ -4320,7 +4326,7 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg auto zookeeper = getZooKeeper(); - const String quorum_status_path = zookeeper_path + "/quorum/status"; + const String quorum_status_path = fs::path(zookeeper_path) / "quorum" / "status"; String value; Coordination::Stat stat; @@ -4336,7 +4342,7 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg } String added_parts_str; - if (zookeeper->tryGet(zookeeper_path + "/quorum/last_part", added_parts_str)) + if (zookeeper->tryGet(fs::path(zookeeper_path) / "quorum" / "last_part", added_parts_str)) { if (!added_parts_str.empty()) { @@ -4378,9 +4384,9 @@ void StorageReplicatedMergeTree::read( */ if (local_context->getSettingsRef().select_sequential_consistency) { - auto max_added_blocks = getMaxAddedBlocks(); + auto max_added_blocks = std::make_shared(getMaxAddedBlocks()); if (auto plan = reader.read( - column_names, metadata_snapshot, query_info, local_context, max_block_size, num_streams, processed_stage, &max_added_blocks)) + column_names, metadata_snapshot, query_info, local_context, max_block_size, num_streams, processed_stage, std::move(max_added_blocks))) query_plan = std::move(*plan); return; } @@ -4663,8 +4669,8 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer /// If metadata nodes have changed, we will update table structure locally. Coordination::Requests requests; - requests.emplace_back(zkutil::makeSetRequest(replica_path + "/columns", entry.columns_str, -1)); - requests.emplace_back(zkutil::makeSetRequest(replica_path + "/metadata", entry.metadata_str, -1)); + requests.emplace_back(zkutil::makeSetRequest(fs::path(replica_path) / "columns", entry.columns_str, -1)); + requests.emplace_back(zkutil::makeSetRequest(fs::path(replica_path) / "metadata", entry.metadata_str, -1)); zookeeper->multi(requests); @@ -4680,7 +4686,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer } /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node - zookeeper->createOrUpdate(replica_path + "/metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); + zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); return true; } @@ -4726,7 +4732,7 @@ PartitionBlockNumbersHolder StorageReplicatedMergeTree::allocateBlockNumbersInAf { /// TODO: Implement optimal block number aqcuisition algorithm in multiple (but not all) partitions EphemeralLocksInAllPartitions lock_holder( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + fs::path(zookeeper_path) / "block_numbers", "block-", fs::path(zookeeper_path) / "temp", *zookeeper); PartitionBlockNumbersHolder::BlockNumbersType block_numbers; for (const auto & lock : lock_holder.getLocks()) @@ -4826,10 +4832,10 @@ void StorageReplicatedMergeTree::alter( size_t mutation_path_idx = std::numeric_limits::max(); String new_metadata_str = future_metadata_in_zk.toString(); - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, metadata_version)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "metadata", new_metadata_str, metadata_version)); String new_columns_str = future_metadata.columns.toString(); - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "columns", new_columns_str, -1)); if (ast_to_str(current_metadata->settings_changes) != ast_to_str(future_metadata.settings_changes)) { @@ -4858,12 +4864,12 @@ void StorageReplicatedMergeTree::alter( alter_path_idx = ops.size(); ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(zookeeper_path) / "log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); PartitionBlockNumbersHolder partition_block_numbers_holder; if (have_mutation) { - const String mutations_path(zookeeper_path + "/mutations"); + const String mutations_path(fs::path(zookeeper_path) / "mutations"); ReplicatedMergeTreeMutationEntry mutation_entry; mutation_entry.alter_version = new_metadata_version; @@ -4874,7 +4880,7 @@ void StorageReplicatedMergeTree::alter( zookeeper->get(mutations_path, &mutations_stat); partition_block_numbers_holder = - allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper); + allocateBlockNumbersInAffectedPartitions(mutation_entry.commands, query_context, zookeeper); mutation_entry.block_numbers = partition_block_numbers_holder.getBlockNumbers(); mutation_entry.create_time = time(nullptr); @@ -4882,7 +4888,7 @@ void StorageReplicatedMergeTree::alter( ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); mutation_path_idx = ops.size(); ops.emplace_back( - zkutil::makeCreateRequest(mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); + zkutil::makeCreateRequest(fs::path(mutations_path) / "", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); } if (auto txn = query_context->getZooKeeperMetadataTransaction()) @@ -4890,7 +4896,7 @@ void StorageReplicatedMergeTree::alter( txn->moveOpsTo(ops); /// NOTE: IDatabase::alterTable(...) is called when executing ALTER_METADATA queue entry without query context, /// so we have to update metadata of DatabaseReplicated here. - String metadata_zk_path = txn->getDatabaseZooKeeperPath() + "/metadata/" + escapeForFileName(table_id.table_name); + String metadata_zk_path = fs::path(txn->getDatabaseZooKeeperPath()) / "metadata" / escapeForFileName(table_id.table_name); auto ast = DatabaseCatalog::instance().getDatabase(table_id.database_name)->getCreateTableQuery(table_id.table_name, query_context); applyMetadataChangesToCreateQuery(ast, future_metadata); ops.emplace_back(zkutil::makeSetRequest(metadata_zk_path, getObjectDefinitionFromCreateQuery(ast), -1)); @@ -5028,44 +5034,56 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition(const St return true; } - -void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr query_context, bool throw_if_noop) +void StorageReplicatedMergeTree::dropPartNoWaitNoThrow(const String & part_name) { assertNotReadonly(); if (!is_leader) - throw Exception("DROP PART|PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); + throw Exception("DROP PART cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - LogEntry entry; - bool did_drop; - if (drop_part) - { - String part_name = partition->as().value.safeGet(); - did_drop = dropPart(zookeeper, part_name, entry, detach, throw_if_noop); - } - else - { - String partition_id = getPartitionIDFromQuery(partition, query_context); - did_drop = dropAllPartsInPartition(*zookeeper, partition_id, entry, query_context, detach); - } + dropPartImpl(zookeeper, part_name, entry, /*detach=*/ false, /*throw_if_noop=*/ false); +} + +void StorageReplicatedMergeTree::dropPart(const String & part_name, bool detach, ContextPtr query_context) +{ + assertNotReadonly(); + if (!is_leader) + throw Exception("DROP PART cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); + + zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + LogEntry entry; + + dropPartImpl(zookeeper, part_name, entry, detach, /*throw_if_noop=*/ true); + + /// If necessary, wait until the operation is performed on itself or on all replicas. + if (query_context->getSettingsRef().replication_alter_partitions_sync == 1) + waitForReplicaToProcessLogEntry(replica_name, entry); + else if (query_context->getSettingsRef().replication_alter_partitions_sync == 2) + waitForAllReplicasToProcessLogEntry(entry); +} + +void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool detach, ContextPtr query_context) +{ + assertNotReadonly(); + if (!is_leader) + throw Exception("DROP PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); + + zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + LogEntry entry; + + String partition_id = getPartitionIDFromQuery(partition, query_context); + bool did_drop = dropAllPartsInPartition(*zookeeper, partition_id, entry, query_context, detach); if (did_drop) { /// If necessary, wait until the operation is performed on itself or on all replicas. - if (query_context->getSettingsRef().replication_alter_partitions_sync != 0) - { - if (query_context->getSettingsRef().replication_alter_partitions_sync == 1) - waitForReplicaToProcessLogEntry(replica_name, entry); - else - waitForAllReplicasToProcessLogEntry(entry); - } - } + if (query_context->getSettingsRef().replication_alter_partitions_sync == 1) + waitForReplicaToProcessLogEntry(replica_name, entry); + else if (query_context->getSettingsRef().replication_alter_partitions_sync == 2) + waitForAllReplicasToProcessLogEntry(entry); - if (!drop_part) - { - String partition_id = getPartitionIDFromQuery(partition, query_context); cleanLastPartNode(partition_id); } } @@ -5082,7 +5100,7 @@ void StorageReplicatedMergeTree::truncate( zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - Strings partitions = zookeeper->getChildren(zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(zookeeper_path) / "block_numbers"); for (String & partition_id : partitions) { @@ -5157,7 +5175,7 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_table_data, c try { auto zookeeper = getZooKeeper(); - zookeeper->set(replica_path + "/host", getReplicatedMergeTreeAddress().toString()); + zookeeper->set(fs::path(replica_path) / "host", getReplicatedMergeTreeAddress().toString()); } catch (Coordination::Exception & e) { @@ -5207,8 +5225,8 @@ StorageReplicatedMergeTree::allocateBlockNumber( deduplication_check_ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_block_id_path, -1)); } - String block_numbers_path = zookeeper_table_path + "/block_numbers"; - String partition_path = block_numbers_path + "/" + partition_id; + String block_numbers_path = fs::path(zookeeper_table_path) / "block_numbers"; + String partition_path = fs::path(block_numbers_path) / partition_id; if (!existsNodeCached(partition_path)) { @@ -5230,7 +5248,7 @@ StorageReplicatedMergeTree::allocateBlockNumber( try { lock = EphemeralLockInZooKeeper( - partition_path + "/block-", zookeeper_table_path + "/temp", *zookeeper, &deduplication_check_ops); + fs::path(partition_path) / "block-", fs::path(zookeeper_table_path) / "temp", *zookeeper, &deduplication_check_ops); } catch (const zkutil::KeeperMultiException & e) { @@ -5254,11 +5272,11 @@ Strings StorageReplicatedMergeTree::waitForAllTableReplicasToProcessLogEntry( LOG_DEBUG(log, "Waiting for all replicas to process {}", entry.znode_name); auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(table_zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(table_zookeeper_path) / "replicas"); Strings unwaited; for (const String & replica : replicas) { - if (wait_for_non_active || zookeeper->exists(table_zookeeper_path + "/replicas/" + replica + "/is_active")) + if (wait_for_non_active || zookeeper->exists(fs::path(table_zookeeper_path) / "replicas" / replica / "is_active")) { if (!waitForTableReplicaToProcessLogEntry(table_zookeeper_path, replica, entry, wait_for_non_active)) unwaited.push_back(replica); @@ -5308,7 +5326,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( const auto & stop_waiting = [&]() { bool stop_waiting_itself = waiting_itself && (partial_shutdown_called || is_dropped); - bool stop_waiting_non_active = !wait_for_non_active && !getZooKeeper()->exists(table_zookeeper_path + "/replicas/" + replica + "/is_active"); + bool stop_waiting_non_active = !wait_for_non_active && !getZooKeeper()->exists(fs::path(table_zookeeper_path) / "replicas" / replica / "is_active"); return stop_waiting_itself || stop_waiting_non_active; }; @@ -5330,7 +5348,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( { zkutil::EventPtr event = std::make_shared(); - String log_pointer = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + String log_pointer = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer", nullptr, event); if (!log_pointer.empty() && parse(log_pointer) > log_index) break; @@ -5347,9 +5365,9 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( * looking for a node with the same content. And if we do not find it - then the replica has already taken this entry in its queue. */ - String log_pointer = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer"); + String log_pointer = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer"); - Strings log_entries = getZooKeeper()->getChildren(table_zookeeper_path + "/log"); + Strings log_entries = getZooKeeper()->getChildren(fs::path(table_zookeeper_path) / "log"); UInt64 log_index = 0; bool found = false; @@ -5361,7 +5379,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( continue; String log_entry_str; - bool exists = getZooKeeper()->tryGet(table_zookeeper_path + "/log/" + log_entry_name, log_entry_str); + bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "log" / log_entry_name, log_entry_str); if (exists && entry_str == log_entry_str) { found = true; @@ -5379,7 +5397,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( { zkutil::EventPtr event = std::make_shared(); - String log_pointer_new = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + String log_pointer_new = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer", nullptr, event); if (!log_pointer_new.empty() && parse(log_pointer_new) > log_index) break; @@ -5404,13 +5422,13 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( * Therefore, we search by comparing the content. */ - Strings queue_entries = getZooKeeper()->getChildren(table_zookeeper_path + "/replicas/" + replica + "/queue"); + Strings queue_entries = getZooKeeper()->getChildren(fs::path(table_zookeeper_path) / "replicas" / replica / "queue"); String queue_entry_to_wait_for; for (const String & entry_name : queue_entries) { String queue_entry_str; - bool exists = getZooKeeper()->tryGet(table_zookeeper_path + "/replicas/" + replica + "/queue/" + entry_name, queue_entry_str); + bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "replicas" / replica / "queue" / entry_name, queue_entry_str); if (exists && queue_entry_str == entry_str) { queue_entry_to_wait_for = entry_name; @@ -5428,7 +5446,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( LOG_DEBUG(log, "Waiting for {} to disappear from {} queue", queue_entry_to_wait_for, replica); /// Third - wait until the entry disappears from the replica queue or replica become inactive. - String path_to_wait_on = table_zookeeper_path + "/replicas/" + replica + "/queue/" + queue_entry_to_wait_for; + String path_to_wait_on = fs::path(table_zookeeper_path) / "replicas" / replica / "queue" / queue_entry_to_wait_for; return getZooKeeper()->waitForDisappear(path_to_wait_on, stop_waiting); } @@ -5470,7 +5488,7 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) { try { - auto log_entries = zookeeper->getChildren(zookeeper_path + "/log"); + auto log_entries = zookeeper->getChildren(fs::path(zookeeper_path) / "log"); if (!log_entries.empty()) { @@ -5478,14 +5496,14 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) res.log_max_index = parse(last_log_entry.substr(strlen("log-"))); } - String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); res.log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); - auto all_replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + auto all_replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); res.total_replicas = all_replicas.size(); for (const String & replica : all_replicas) - if (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) ++res.active_replicas; } catch (const Coordination::Exception &) @@ -5568,7 +5586,7 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t time_t max_replicas_unprocessed_insert_time = 0; bool have_replica_with_nothing_unprocessed = false; - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); for (const auto & replica : replicas) { @@ -5576,11 +5594,11 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t continue; /// Skip dead replicas. - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (!zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; String value; - if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/min_unprocessed_insert_time", value)) + if (!zookeeper->tryGet(fs::path(zookeeper_path) / "replicas" / replica / "min_unprocessed_insert_time", value)) continue; time_t replica_time = value.empty() ? 0 : parse(value); @@ -5687,13 +5705,13 @@ void StorageReplicatedMergeTree::fetchPartition( { /// List of replicas of source shard. - replicas = zookeeper->getChildren(from + "/replicas"); + replicas = zookeeper->getChildren(fs::path(from) / "replicas"); /// Leave only active replicas. active_replicas.reserve(replicas.size()); for (const String & replica : replicas) - if (zookeeper->exists(from + "/replicas/" + replica + "/is_active")) + if (zookeeper->exists(fs::path(from) / "replicas" / replica / "is_active")) active_replicas.push_back(replica); if (active_replicas.empty()) @@ -5710,13 +5728,13 @@ void StorageReplicatedMergeTree::fetchPartition( for (const String & replica : active_replicas) { - String current_replica_path = from + "/replicas/" + replica; + String current_replica_path = fs::path(from) / "replicas" / replica; - String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(current_replica_path) / "log_pointer"); Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); Coordination::Stat stat; - zookeeper->get(current_replica_path + "/queue", &stat); + zookeeper->get(fs::path(current_replica_path) / "queue", &stat); size_t queue_size = stat.numChildren; if (log_pointer > max_log_pointer @@ -5734,7 +5752,7 @@ void StorageReplicatedMergeTree::fetchPartition( LOG_INFO(log, "Found {} replicas, {} of them are active. Selected {} to fetch from.", replicas.size(), active_replicas.size(), best_replica); - String best_replica_path = from + "/replicas/" + best_replica; + String best_replica_path = fs::path(from) / "replicas" / best_replica; /// Let's find out which parts are on the best replica. @@ -5753,7 +5771,7 @@ void StorageReplicatedMergeTree::fetchPartition( if (try_no >= query_context->getSettings().max_fetch_partition_retries_count) throw Exception("Too many retries to fetch parts from " + best_replica_path, ErrorCodes::TOO_MANY_RETRIES_TO_FETCH_PARTS); - Strings parts = zookeeper->getChildren(best_replica_path + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(best_replica_path) / "parts"); ActiveDataPartSet active_parts_set(format_version, parts); Strings parts_to_fetch; @@ -5874,7 +5892,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte mutation_entry.source_replica = replica_name; mutation_entry.commands = commands; - const String mutations_path = zookeeper_path + "/mutations"; + const String mutations_path = fs::path(zookeeper_path) / "mutations"; const auto zookeeper = getZooKeeper(); /// Update the mutations_path node when creating the mutation and check its version to ensure that @@ -5897,7 +5915,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte Coordination::Requests requests; requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); requests.emplace_back(zkutil::makeCreateRequest( - mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(mutations_path) / "", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); if (auto txn = query_context->getZooKeeperMetadataTransaction()) txn->moveOpsTo(requests); @@ -5936,7 +5954,7 @@ void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t auto zookeeper = getZooKeeper(); Strings replicas; if (mutations_sync == 2) /// wait for all replicas - replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); else if (mutations_sync == 1) /// just wait for ourself replicas.push_back(replica_name); @@ -6131,7 +6149,7 @@ bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(const St exists_futures.reserve(part_names.size()); for (const String & part_name : part_names) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -6144,7 +6162,7 @@ bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(const St { Coordination::Requests ops; removePartFromZooKeeper(part_names[i], ops, exists_resp.stat.numChildren > 0); - remove_futures.emplace_back(zookeeper->tryAsyncMulti(ops)); + remove_futures.emplace_back(zookeeper->asyncTryMultiNoThrow(ops)); } } @@ -6194,7 +6212,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( /// if zk session will be dropped for (const String & part_name : part_names) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -6205,7 +6223,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( { Coordination::Requests ops; removePartFromZooKeeper(part_names[i], ops, exists_resp.stat.numChildren > 0); - remove_futures.emplace_back(zookeeper->tryAsyncMulti(ops)); + remove_futures.emplace_back(zookeeper->asyncTryMultiNoThrow(ops)); } else { @@ -6253,7 +6271,7 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num) { Strings blocks; - if (Coordination::Error::ZOK != zookeeper.tryGetChildren(zookeeper_path + "/blocks", blocks)) + if (Coordination::Error::ZOK != zookeeper.tryGetChildren(fs::path(zookeeper_path) / "blocks", blocks)) throw Exception(zookeeper_path + "/blocks doesn't exist", ErrorCodes::NOT_FOUND_NODE); String partition_prefix = partition_id + "_"; @@ -6262,7 +6280,7 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( { if (startsWith(block_id, partition_prefix)) { - String path = zookeeper_path + "/blocks/" + block_id; + String path = fs::path(zookeeper_path) / "blocks" / block_id; get_futures.emplace_back(path, zookeeper.asyncTryGet(path)); } } @@ -6372,7 +6390,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( else LOG_INFO(log, "Trying to attach {} with hash_hex {}", src_part->name, hash_hex); - String block_id_path = replace ? "" : (zookeeper_path + "/blocks/" + partition_id + "_replace_from_" + hash_hex); + String block_id_path = replace ? "" : (fs::path(zookeeper_path) / "blocks" / (partition_id + "_replace_from_" + hash_hex)); auto lock = allocateBlockNumber(partition_id, zookeeper, block_id_path); if (!lock) @@ -6444,8 +6462,8 @@ void StorageReplicatedMergeTree::replacePartitionFrom( ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); Transaction transaction(*this); { @@ -6632,8 +6650,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(dest_table_storage->zookeeper_path + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest(dest_table_storage->zookeeper_path + "/log/log-", + ops.emplace_back(zkutil::makeSetRequest(fs::path(dest_table_storage->zookeeper_path) / "log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(dest_table_storage->zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); { @@ -6689,12 +6707,12 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Coordination::Requests ops_src; ops_src.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(zookeeper_path) / "log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); /// Check and update version to avoid race with REPLACE_RANGE ops_src.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops_src.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops_src.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops_src.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); delimiting_block_lock->getUnlockOps(ops_src); Coordination::Error code = zookeeper->tryMulti(ops_src, op_results); @@ -6834,22 +6852,22 @@ void StorageReplicatedMergeTree::getCommitPartOps( if (storage_settings_ptr->use_minimalistic_part_header_in_zookeeper) { ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name, + fs::path(replica_path) / "parts" / part->name, ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(), zkutil::CreateMode::Persistent)); } else { ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name, + fs::path(replica_path) / "parts" / part->name, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name + "/columns", + fs::path(replica_path) / "parts" / part->name / "columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name + "/checksums", + fs::path(replica_path) / "parts" / part->name / "checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); } @@ -6937,7 +6955,7 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI return true; } -bool StorageReplicatedMergeTree::dropPart( +bool StorageReplicatedMergeTree::dropPartImpl( zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop) { LOG_TRACE(log, "Will try to insert a log entry to DROP_RANGE for part: " + part_name); @@ -7000,10 +7018,10 @@ bool StorageReplicatedMergeTree::dropPart( entry.detach = detach; entry.create_time = time(nullptr); - ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/log", merge_pred.getVersion())); /// Make sure no new events were added to the log. - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCheckRequest(fs::path(zookeeper_path) / "log", merge_pred.getVersion())); /// Make sure no new events were added to the log. + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); Coordination::Responses responses; Coordination::Error rc = zookeeper->tryMulti(ops, responses); @@ -7057,14 +7075,14 @@ bool StorageReplicatedMergeTree::dropAllPartsInPartition( entry.create_time = time(nullptr); Coordination::Requests ops; - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); /// Check and update version to avoid race with REPLACE_RANGE. /// Otherwise new parts covered by drop_range_info may appear after execution of current DROP_RANGE entry /// as a result of execution of concurrently created REPLACE_RANGE entry. ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); delimiting_block_lock->getUnlockOps(ops); if (auto txn = query_context->getZooKeeperMetadataTransaction()) txn->moveOpsTo(ops); @@ -7150,7 +7168,7 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name + "/" + id + "/" + replica_name; + String zookeeper_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name / id / replica_name; LOG_TRACE(log, "Set zookeeper lock {}", zookeeper_node); @@ -7191,9 +7209,9 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; - String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; - String zookeeper_node = zookeeper_part_uniq_node + "/" + replica_name; + String zookeeper_part_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name; + String zookeeper_part_uniq_node = fs::path(zookeeper_part_node) / id; + String zookeeper_node = fs::path(zookeeper_part_uniq_node) / replica_name; LOG_TRACE(log, "Remove zookeeper lock {}", zookeeper_node); @@ -7252,7 +7270,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( if (!zookeeper) return best_replica; - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; + String zookeeper_part_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name; Strings ids; zookeeper->tryGetChildren(zookeeper_part_node, ids); @@ -7260,7 +7278,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( Strings replicas; for (const auto & id : ids) { - String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; + String zookeeper_part_uniq_node = fs::path(zookeeper_part_node) / id; Strings id_replicas; zookeeper->tryGetChildren(zookeeper_part_uniq_node, id_replicas); LOG_TRACE(log, "Found zookeper replicas for {}: {}", zookeeper_part_uniq_node, id_replicas.size()); @@ -7277,7 +7295,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( active_replicas.reserve(replicas.size()); for (const String & replica : replicas) - if ((replica != replica_name) && (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + if ((replica != replica_name) && (zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active"))) active_replicas.push_back(replica); LOG_TRACE(log, "Found zookeper active replicas for part {}: {}", part.name, active_replicas.size()); @@ -7296,13 +7314,13 @@ String StorageReplicatedMergeTree::getSharedDataReplica( for (const String & replica : active_replicas) { - String current_replica_path = zookeeper_path + "/replicas/" + replica; + String current_replica_path = fs::path(zookeeper_path) / "replicas" / replica; - String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(current_replica_path) / "log_pointer"); Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); Coordination::Stat stat; - zookeeper->get(current_replica_path + "/queue", &stat); + zookeeper->get(fs::path(current_replica_path) / "queue", &stat); size_t queue_size = stat.numChildren; if (log_pointer > max_log_pointer @@ -7320,16 +7338,16 @@ String StorageReplicatedMergeTree::getSharedDataReplica( String StorageReplicatedMergeTree::findReplicaHavingPart( const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_) { - Strings replicas = zookeeper_->getChildren(zookeeper_path_ + "/replicas"); + Strings replicas = zookeeper_->getChildren(fs::path(zookeeper_path_) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); for (const String & replica : replicas) { - if (zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/parts/" + part_name) - && zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/is_active")) - return zookeeper_path_ + "/replicas/" + replica; + if (zookeeper_->exists(fs::path(zookeeper_path_) / "replicas" / replica / "parts" / part_name) + && zookeeper_->exists(fs::path(zookeeper_path_) / "replicas" / replica / "is_active")) + return fs::path(zookeeper_path_) / "replicas" / replica; } return {}; @@ -7337,23 +7355,23 @@ String StorageReplicatedMergeTree::findReplicaHavingPart( bool StorageReplicatedMergeTree::checkIfDetachedPartExists(const String & part_name) { - Poco::DirectoryIterator dir_end; + fs::directory_iterator dir_end; for (const std::string & path : getDataPaths()) - for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) - if (dir_it.name() == part_name) + for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) + if (dir_it->path().filename().string() == part_name) return true; return false; } bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & partition_name) { - Poco::DirectoryIterator dir_end; + fs::directory_iterator dir_end; for (const std::string & path : getDataPaths()) { - for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) + for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) { MergeTreePartInfo part_info; - if (MergeTreePartInfo::tryParsePartName(dir_it.name(), &part_info, format_version) && part_info.partition_id == partition_name) + if (MergeTreePartInfo::tryParsePartName(dir_it->path().filename(), &part_info, format_version) && part_info.partition_id == partition_name) return true; } } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 4e697f2d1f2..ec542acc275 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -642,12 +642,14 @@ private: /// Info about how other replicas can access this one. ReplicatedMergeTreeAddress getReplicatedMergeTreeAddress() const; - bool dropPart(zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop); bool dropAllPartsInPartition( zkutil::ZooKeeper & zookeeper, String & partition_id, LogEntry & entry, ContextPtr query_context, bool detach); + void dropPartNoWaitNoThrow(const String & part_name) override; + void dropPart(const String & part_name, bool detach, ContextPtr query_context) override; + // Partition helpers - void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr query_context, bool throw_if_noop) override; + void dropPartition(const ASTPtr & partition, bool detach, ContextPtr query_context) override; PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr query_context) override; void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr query_context) override; void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr query_context) override; @@ -659,6 +661,8 @@ private: bool fetch_part, ContextPtr query_context) override; + bool dropPartImpl(zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop); + /// Check granularity of already existing replicated table in zookeeper if it exists /// return true if it's fixed bool checkFixedGranualrityInZookeeper(); @@ -685,7 +689,7 @@ protected: const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 36d078e7bf3..fd9ff596329 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -35,8 +35,10 @@ #include #include #include - #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -197,7 +199,7 @@ bool StorageS3Source::initialize() if (current_key.empty()) return false; - file_path = bucket + "/" + current_key; + file_path = fs::path(bucket) / current_key; read_buf = wrapReadBufferWithCompressionMethod( std::make_unique(client, bucket, current_key, s3_max_single_read_retries), chooseCompressionMethod(current_key, compression_hint)); diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index 88d54669889..9cffc32fda1 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/examples/get_current_inserts_in_replicated.cpp b/src/Storages/examples/get_current_inserts_in_replicated.cpp index fa998d20f66..56364997643 100644 --- a/src/Storages/examples/get_current_inserts_in_replicated.cpp +++ b/src/Storages/examples/get_current_inserts_in_replicated.cpp @@ -4,11 +4,12 @@ #include #include #include - #include +#include #include +namespace fs = std::filesystem; using namespace DB; @@ -36,10 +37,10 @@ try Stopwatch stage; /// Load current inserts std::unordered_set lock_holder_paths; - for (const String & entry : zookeeper->getChildren(zookeeper_path + "/temp")) + for (const String & entry : zookeeper->getChildren(fs::path(zookeeper_path) / "temp")) { if (startsWith(entry, "abandonable_lock-")) - lock_holder_paths.insert(zookeeper_path + "/temp/" + entry); + lock_holder_paths.insert(fs::path(zookeeper_path) / "temp" / entry); } std::cerr << "Stage 1 (get lock holders): " << lock_holder_paths.size() << " lock holders, elapsed: " << stage.elapsedSeconds() << "s." << std::endl; @@ -47,14 +48,14 @@ try if (!lock_holder_paths.empty()) { - Strings partitions = zookeeper->getChildren(zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(zookeeper_path) / "block_numbers"); std::cerr << "Stage 2 (get partitions): " << partitions.size() << " partitions, elapsed: " << stage.elapsedSeconds() << "s." << std::endl; stage.restart(); std::vector> lock_futures; for (const String & partition : partitions) - lock_futures.push_back(zookeeper->asyncGetChildren(zookeeper_path + "/block_numbers/" + partition)); + lock_futures.push_back(zookeeper->asyncGetChildren(fs::path(zookeeper_path) / "block_numbers" / partition)); struct BlockInfo { @@ -71,7 +72,7 @@ try for (const String & entry : partition_block_numbers) { Int64 block_number = parse(entry.substr(strlen("block-"))); - String zk_path = zookeeper_path + "/block_numbers/" + partitions[i] + "/" + entry; + String zk_path = fs::path(zookeeper_path) / "block_numbers" / partitions[i] / entry; block_infos.push_back( BlockInfo{partitions[i], block_number, zk_path, zookeeper->asyncTryGet(zk_path)}); } diff --git a/src/Storages/examples/remove_symlink_directory.cpp b/src/Storages/examples/remove_symlink_directory.cpp index ae5fa72fa66..db436c0a608 100644 --- a/src/Storages/examples/remove_symlink_directory.cpp +++ b/src/Storages/examples/remove_symlink_directory.cpp @@ -1,9 +1,10 @@ #include #include -#include -#include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -16,18 +17,15 @@ namespace DB int main(int, char **) try { - Poco::File dir("./test_dir/"); - dir.createDirectories(); - - Poco::File("./test_dir/file").createFile(); + fs::path dir("./test_dir/"); + fs::create_directories(dir); + FS::createFile("./test_dir/file"); if (0 != symlink("./test_dir", "./test_link")) DB::throwFromErrnoWithPath("Cannot create symlink", "./test_link", DB::ErrorCodes::SYSTEM_ERROR); - Poco::File link("./test_link"); - link.renameTo("./test_link2"); - - Poco::File("./test_link2").remove(true); + fs::rename("./test_link", "./test_link2"); + fs::remove_all("./test_link2"); return 0; } catch (...) diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 6735206862d..e61123a3aa8 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -22,7 +22,7 @@ struct State { State(const State&) = delete; - ContextPtr context; + ContextMutablePtr context; static const State & instance() { diff --git a/tests/executable_pool_dictionary.xml b/tests/executable_pool_dictionary.xml new file mode 120000 index 00000000000..83f18b95f20 --- /dev/null +++ b/tests/executable_pool_dictionary.xml @@ -0,0 +1 @@ +config/executable_pool_dictionary.xml \ No newline at end of file diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 6287064b616..733af0cb8f9 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -126,7 +126,7 @@ class ClickHouseCluster: """ def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None, - odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, + odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None): for param in list(os.environ.keys()): print("ENV %40s %s" % (param, os.environ[param])) @@ -455,7 +455,10 @@ class ClickHouseCluster: run_and_check(self.base_cmd + ["up", "--force-recreate", "--no-deps", "-d", node.name]) node.ip_address = self.get_instance_ip(node.name) node.client = Client(node.ip_address, command=self.client_bin_path) - node.wait_for_start(start_timeout=20.0, connection_timeout=600.0) # seconds + print("Restart node with ip change") + # In builds with sanitizer the server can take a long time to start + node.wait_for_start(start_timeout=60.0, connection_timeout=600.0) # seconds + print("Restarted") return node def get_instance_ip(self, instance_name): @@ -1524,7 +1527,7 @@ class ClickHouseInstance: if self.stay_alive: entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND.replace("{main_config_file}", self.main_config_name) - + print("Entrypoint cmd: {}".format(entrypoint_cmd)) networks = app_net = ipv4_address = ipv6_address = net_aliases = net_alias1 = "" diff --git a/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.xml b/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.xml index 2a937b5444d..da848355660 100644 --- a/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.xml +++ b/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.xml @@ -8,7 +8,7 @@ TabSeparated - 1 + 5 key aInt32 diff --git a/tests/integration/test_keeper_auth/__init__.py b/tests/integration/test_keeper_auth/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_keeper_auth/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_keeper_auth/configs/keeper_config.xml b/tests/integration/test_keeper_auth/configs/keeper_config.xml new file mode 100644 index 00000000000..bb3c9a5d94a --- /dev/null +++ b/tests/integration/test_keeper_auth/configs/keeper_config.xml @@ -0,0 +1,24 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + super:0DPiKuNIrrVmD8IUCuw1hQxNqZc= + + + 5000 + 10000 + trace + 75 + + + + + 1 + localhost + 44444 + + + + diff --git a/tests/integration/test_keeper_auth/configs/logs_conf.xml b/tests/integration/test_keeper_auth/configs/logs_conf.xml new file mode 100644 index 00000000000..3384320992a --- /dev/null +++ b/tests/integration/test_keeper_auth/configs/logs_conf.xml @@ -0,0 +1,13 @@ + + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + diff --git a/tests/integration/test_keeper_auth/test.py b/tests/integration/test_keeper_auth/test.py new file mode 100644 index 00000000000..5f60d5b8bdb --- /dev/null +++ b/tests/integration/test_keeper_auth/test.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +import pytest +from helpers.cluster import ClickHouseCluster +from kazoo.client import KazooClient, KazooState +from kazoo.security import ACL, make_digest_acl, make_acl +from kazoo.exceptions import AuthFailedError, InvalidACLError, NoAuthError, KazooException + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', main_configs=['configs/keeper_config.xml', 'configs/logs_conf.xml'], with_zookeeper=True, use_keeper=False, stay_alive=True) + +SUPERAUTH = "super:admin" + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + +def get_fake_zk(timeout=30.0): + _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip('node') + ":9181", timeout=timeout) + _fake_zk_instance.start() + return _fake_zk_instance + +def get_genuine_zk(): + print("Zoo1", cluster.get_instance_ip("zoo1")) + return cluster.get_kazoo_client('zoo1') + +@pytest.mark.parametrize( + ('get_zk'), + [ + get_genuine_zk, + get_fake_zk + ] +) +def test_digest_auth_basic(started_cluster, get_zk): + auth_connection = get_zk() + + auth_connection.add_auth('digest', 'user1:password1') + + auth_connection.create("/test_no_acl", b"") + auth_connection.create("/test_all_acl", b"data", acl=[make_acl("auth", "", all=True)]) + # for some reason original zookeeper accepts this ACL, but doesn't allow to do anything with this node + # even with correct credentials. + auth_connection.create("/test_all_digest_acl", b"dataX", acl=[make_acl("digest", "user1:password1", all=True)]) + + assert auth_connection.get("/test_all_acl")[0] == b"data" + #assert auth_connection.get("/test_all_digest_acl")[0] == b"dataX" + + no_auth_connection = get_zk() + no_auth_connection.set("/test_no_acl", b"hello") + + # no ACL, so cannot access these nodes + assert no_auth_connection.get("/test_no_acl")[0] == b"hello" + + with pytest.raises(NoAuthError): + no_auth_connection.set("/test_all_acl", b"hello") + + with pytest.raises(NoAuthError): + no_auth_connection.get("/test_all_acl") + + with pytest.raises(NoAuthError): + no_auth_connection.get("/test_all_digest_acl") + + # still doesn't help + with pytest.raises(AuthFailedError): + no_auth_connection.add_auth("world", "anyone") + + # session became broken, reconnect + no_auth_connection = get_zk() + + # wrong auth + no_auth_connection.add_auth('digest', 'user2:password2') + + with pytest.raises(NoAuthError): + no_auth_connection.set("/test_all_acl", b"hello") + + with pytest.raises(NoAuthError): + no_auth_connection.set("/test_all_acl", b"hello") + + with pytest.raises(NoAuthError): + no_auth_connection.get("/test_all_acl") + + with pytest.raises(NoAuthError): + no_auth_connection.get("/test_all_digest_acl") + + # but can access some non restricted nodes + no_auth_connection.create("/some_allowed_node", b"data") + + # auth added, go on + no_auth_connection.add_auth('digest', 'user1:password1') + for path in ["/test_no_acl", "/test_all_acl"]: + no_auth_connection.set(path, b"auth_added") + assert no_auth_connection.get(path)[0] == b"auth_added" + + +def test_super_auth(started_cluster): + auth_connection = get_fake_zk() + + auth_connection.add_auth('digest', 'user1:password1') + + auth_connection.create("/test_super_no_acl", b"") + auth_connection.create("/test_super_all_acl", b"data", acl=[make_acl("auth", "", all=True)]) + + super_connection = get_fake_zk() + super_connection.add_auth('digest', 'super:admin') + + for path in ["/test_super_no_acl", "/test_super_all_acl"]: + super_connection.set(path, b"value") + assert super_connection.get(path)[0] == b"value" + + +@pytest.mark.parametrize( + ('get_zk'), + [ + get_genuine_zk, + get_fake_zk + ] +) +def test_digest_auth_multiple(started_cluster, get_zk): + auth_connection = get_zk() + auth_connection.add_auth('digest', 'user1:password1') + auth_connection.add_auth('digest', 'user2:password2') + auth_connection.add_auth('digest', 'user3:password3') + + auth_connection.create("/test_multi_all_acl", b"data", acl=[make_acl("auth", "", all=True)]) + + one_auth_connection = get_zk() + one_auth_connection.add_auth('digest', 'user1:password1') + + one_auth_connection.set("/test_multi_all_acl", b"X") + assert one_auth_connection.get("/test_multi_all_acl")[0] == b"X" + + other_auth_connection = get_zk() + other_auth_connection.add_auth('digest', 'user2:password2') + + other_auth_connection.set("/test_multi_all_acl", b"Y") + + assert other_auth_connection.get("/test_multi_all_acl")[0] == b"Y" + +@pytest.mark.parametrize( + ('get_zk'), + [ + get_genuine_zk, + get_fake_zk + ] +) +def test_partial_auth(started_cluster, get_zk): + auth_connection = get_zk() + auth_connection.add_auth('digest', 'user1:password1') + + auth_connection.create("/test_partial_acl", b"data", acl=[make_acl("auth", "", read=False, write=True, create=True, delete=True, admin=True)]) + + auth_connection.set("/test_partial_acl", b"X") + auth_connection.create("/test_partial_acl/subnode", b"X", acl=[make_acl("auth", "", read=False, write=True, create=True, delete=True, admin=True)]) + + with pytest.raises(NoAuthError): + auth_connection.get("/test_partial_acl") + + with pytest.raises(NoAuthError): + auth_connection.get_children("/test_partial_acl") + + # exists works without read perm + assert auth_connection.exists("/test_partial_acl") is not None + + auth_connection.create("/test_partial_acl_create", b"data", acl=[make_acl("auth", "", read=True, write=True, create=False, delete=True, admin=True)]) + with pytest.raises(NoAuthError): + auth_connection.create("/test_partial_acl_create/subnode") + + auth_connection.create("/test_partial_acl_set", b"data", acl=[make_acl("auth", "", read=True, write=False, create=True, delete=True, admin=True)]) + with pytest.raises(NoAuthError): + auth_connection.set("/test_partial_acl_set", b"X") + + # not allowed to delete child node + auth_connection.create("/test_partial_acl_delete", b"data", acl=[make_acl("auth", "", read=True, write=True, create=True, delete=False, admin=True)]) + auth_connection.create("/test_partial_acl_delete/subnode") + with pytest.raises(NoAuthError): + auth_connection.delete("/test_partial_acl_delete/subnode") + + +def test_bad_auth(started_cluster): + auth_connection = get_fake_zk() + + with pytest.raises(AuthFailedError): + auth_connection.add_auth('world', 'anyone') + + auth_connection = get_fake_zk() + with pytest.raises(AuthFailedError): + print("Sending 1") + auth_connection.add_auth('adssagf', 'user1:password1') + + auth_connection = get_fake_zk() + with pytest.raises(AuthFailedError): + print("Sending 2") + auth_connection.add_auth('digest', '') + + auth_connection = get_fake_zk() + with pytest.raises(AuthFailedError): + print("Sending 3") + auth_connection.add_auth('', 'user1:password1') + + auth_connection = get_fake_zk() + with pytest.raises(AuthFailedError): + print("Sending 4") + auth_connection.add_auth('digest', 'user1') + + auth_connection = get_fake_zk() + with pytest.raises(AuthFailedError): + print("Sending 5") + auth_connection.add_auth('digest', 'user1:password:otherpassword') + + auth_connection = get_fake_zk() + with pytest.raises(AuthFailedError): + print("Sending 6") + auth_connection.add_auth('auth', 'user1:password') + + auth_connection = get_fake_zk() + with pytest.raises(AuthFailedError): + print("Sending 7") + auth_connection.add_auth('world', 'somebody') + + auth_connection = get_fake_zk() + with pytest.raises(InvalidACLError): + print("Sending 8") + auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("dasd", "", read=True, write=False, create=True, delete=True, admin=True)]) + + auth_connection = get_fake_zk() + with pytest.raises(InvalidACLError): + print("Sending 9") + auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("digest", "", read=True, write=False, create=True, delete=True, admin=True)]) + + auth_connection = get_fake_zk() + with pytest.raises(InvalidACLError): + print("Sending 10") + auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("", "", read=True, write=False, create=True, delete=True, admin=True)]) + + auth_connection = get_fake_zk() + with pytest.raises(InvalidACLError): + print("Sending 11") + auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("digest", "dsdasda", read=True, write=False, create=True, delete=True, admin=True)]) + + auth_connection = get_fake_zk() + with pytest.raises(InvalidACLError): + print("Sending 12") + auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("digest", "dsad:DSAa:d", read=True, write=False, create=True, delete=True, admin=True)]) + +def test_auth_snapshot(started_cluster): + connection = get_fake_zk() + connection.add_auth('digest', 'user1:password1') + + connection.create("/test_snapshot_acl", b"data", acl=[make_acl("auth", "", all=True)]) + + connection1 = get_fake_zk() + connection1.add_auth('digest', 'user2:password2') + + connection1.create("/test_snapshot_acl1", b"data", acl=[make_acl("auth", "", all=True)]) + + connection2 = get_fake_zk() + + connection2.create("/test_snapshot_acl2", b"data") + + for i in range(100): + connection.create(f"/test_snapshot_acl/path{i}", b"data", acl=[make_acl("auth", "", all=True)]) + + node.restart_clickhouse() + + connection = get_fake_zk() + + with pytest.raises(NoAuthError): + connection.get("/test_snapshot_acl") + + connection.add_auth('digest', 'user1:password1') + + assert connection.get("/test_snapshot_acl")[0] == b"data" + + with pytest.raises(NoAuthError): + connection.get("/test_snapshot_acl1") + + assert connection.get("/test_snapshot_acl2")[0] == b"data" + + for i in range(100): + assert connection.get(f"/test_snapshot_acl/path{i}")[0] == b"data" + + connection1 = get_fake_zk() + connection1.add_auth('digest', 'user2:password2') + + assert connection1.get("/test_snapshot_acl1")[0] == b"data" + + with pytest.raises(NoAuthError): + connection1.get("/test_snapshot_acl") + + + connection2 = get_fake_zk() + assert connection2.get("/test_snapshot_acl2")[0] == b"data" + with pytest.raises(NoAuthError): + connection2.get("/test_snapshot_acl") + + with pytest.raises(NoAuthError): + connection2.get("/test_snapshot_acl1") diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index 8abd4321fd9..4cb539e448c 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -7,7 +7,7 @@ import time from multiprocessing.dummy import Pool cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/enable_keeper.xml', 'configs/logs_conf.xml'], with_zookeeper=True) +node = cluster.add_instance('node', main_configs=['configs/enable_keeper.xml', 'configs/logs_conf.xml'], with_zookeeper=True, use_keeper=False) from kazoo.client import KazooClient, KazooState, KeeperState def get_genuine_zk(): diff --git a/tests/integration/test_keeper_internal_secure/test.py b/tests/integration/test_keeper_internal_secure/test.py index d9fbca624e1..b4bf62f9a37 100644 --- a/tests/integration/test_keeper_internal_secure/test.py +++ b/tests/integration/test_keeper_internal_secure/test.py @@ -26,13 +26,6 @@ def started_cluster(): def get_fake_zk(nodename, timeout=30.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) - def reset_listener(state): - nonlocal _fake_zk_instance - print("Fake zk callback called for state", state) - if state != KazooState.CONNECTED: - _fake_zk_instance._reset() - - _fake_zk_instance.add_listener(reset_listener) _fake_zk_instance.start() return _fake_zk_instance diff --git a/tests/integration/test_keeper_multinode_blocade_leader/test.py b/tests/integration/test_keeper_multinode_blocade_leader/test.py index e74ccc80fe8..9ae81ccdcc6 100644 --- a/tests/integration/test_keeper_multinode_blocade_leader/test.py +++ b/tests/integration/test_keeper_multinode_blocade_leader/test.py @@ -66,12 +66,6 @@ def wait_nodes(): def get_fake_zk(nodename, timeout=30.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) - def reset_listener(state): - nonlocal _fake_zk_instance - if state != KazooState.CONNECTED: - _fake_zk_instance._reset() - - _fake_zk_instance.add_listener(reset_listener) _fake_zk_instance.start() return _fake_zk_instance diff --git a/tests/integration/test_keeper_multinode_simple/test.py b/tests/integration/test_keeper_multinode_simple/test.py index 18420c9910d..9e57567d8b1 100644 --- a/tests/integration/test_keeper_multinode_simple/test.py +++ b/tests/integration/test_keeper_multinode_simple/test.py @@ -54,13 +54,6 @@ def wait_nodes(): def get_fake_zk(nodename, timeout=30.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) - def reset_listener(state): - nonlocal _fake_zk_instance - print("Fake zk callback called for state", state) - if state != KazooState.CONNECTED: - _fake_zk_instance._reset() - - _fake_zk_instance.add_listener(reset_listener) _fake_zk_instance.start() return _fake_zk_instance diff --git a/tests/integration/test_keeper_persistent_log/test.py b/tests/integration/test_keeper_persistent_log/test.py index 36ee20ae9e7..b0cd9155afb 100644 --- a/tests/integration/test_keeper_persistent_log/test.py +++ b/tests/integration/test_keeper_persistent_log/test.py @@ -33,13 +33,6 @@ def started_cluster(): def get_connection_zk(nodename, timeout=30.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) - def reset_listener(state): - nonlocal _fake_zk_instance - print("Fake zk callback called for state", state) - if state != KazooState.CONNECTED: - _fake_zk_instance._reset() - - _fake_zk_instance.add_listener(reset_listener) _fake_zk_instance.start() return _fake_zk_instance diff --git a/tests/integration/test_keeper_persistent_log_multinode/test.py b/tests/integration/test_keeper_persistent_log_multinode/test.py index 052f38b0bff..306139369fb 100644 --- a/tests/integration/test_keeper_persistent_log_multinode/test.py +++ b/tests/integration/test_keeper_persistent_log_multinode/test.py @@ -25,13 +25,6 @@ def started_cluster(): def get_fake_zk(nodename, timeout=30.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) - def reset_listener(state): - nonlocal _fake_zk_instance - print("Fake zk callback called for state", state) - if state != KazooState.CONNECTED: - _fake_zk_instance._reset() - - _fake_zk_instance.add_listener(reset_listener) _fake_zk_instance.start() return _fake_zk_instance diff --git a/tests/integration/test_keeper_restore_from_snapshot/test.py b/tests/integration/test_keeper_restore_from_snapshot/test.py index eee49f8a319..5f6156800bb 100644 --- a/tests/integration/test_keeper_restore_from_snapshot/test.py +++ b/tests/integration/test_keeper_restore_from_snapshot/test.py @@ -25,13 +25,6 @@ def started_cluster(): def get_fake_zk(nodename, timeout=30.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) - def reset_listener(state): - nonlocal _fake_zk_instance - print("Fake zk callback called for state", state) - if state != KazooState.CONNECTED: - _fake_zk_instance._reset() - - _fake_zk_instance.add_listener(reset_listener) _fake_zk_instance.start() return _fake_zk_instance @@ -77,6 +70,7 @@ def test_recover_from_snapshot(started_cluster): # stale node should recover from leader's snapshot # with some sanitizers can start longer than 5 seconds node3.start_clickhouse(20) + print("Restarted") try: node1_zk = node2_zk = node3_zk = None diff --git a/tests/integration/test_keeper_snapshots/test.py b/tests/integration/test_keeper_snapshots/test.py index 9879c0003be..7d5b69bf5d1 100644 --- a/tests/integration/test_keeper_snapshots/test.py +++ b/tests/integration/test_keeper_snapshots/test.py @@ -35,13 +35,6 @@ def started_cluster(): def get_connection_zk(nodename, timeout=30.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) - def reset_listener(state): - nonlocal _fake_zk_instance - print("Fake zk callback called for state", state) - if state != KazooState.CONNECTED: - _fake_zk_instance._reset() - - _fake_zk_instance.add_listener(reset_listener) _fake_zk_instance.start() return _fake_zk_instance diff --git a/tests/integration/test_keeper_snapshots_multinode/test.py b/tests/integration/test_keeper_snapshots_multinode/test.py index b4e82d62f09..96d19592d29 100644 --- a/tests/integration/test_keeper_snapshots_multinode/test.py +++ b/tests/integration/test_keeper_snapshots_multinode/test.py @@ -25,13 +25,6 @@ def started_cluster(): def get_fake_zk(nodename, timeout=30.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) - def reset_listener(state): - nonlocal _fake_zk_instance - print("Fake zk callback called for state", state) - if state != KazooState.CONNECTED: - _fake_zk_instance._reset() - - _fake_zk_instance.add_listener(reset_listener) _fake_zk_instance.start() return _fake_zk_instance diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index 24ee6c0493b..c01289c8114 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -335,9 +335,9 @@ def get_random_string(length): def get_used_disks_for_table(node, table_name): - return node.query( + return tuple(node.query( "select disk_name from system.parts where table == '{}' and active=1 order by modification_time".format( - table_name)).strip().split('\n') + table_name)).strip().split('\n')) def get_used_parts_for_table(node, table_name): @@ -450,7 +450,7 @@ def test_jbod_overflow(start_cluster, name, engine): node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) used_disks = get_used_disks_for_table(node1, name) - assert all(disk == 'jbod1' for disk in used_disks) + assert used_disks == tuple('jbod1' for _ in used_disks) # should go to the external disk (jbod is overflown) data = [] # 10MB in total @@ -469,11 +469,11 @@ def test_jbod_overflow(start_cluster, name, engine): node1.query("OPTIMIZE TABLE {} FINAL".format(name)) time.sleep(2) - disks_for_merges = node1.query( + disks_for_merges = tuple(node1.query( "SELECT disk_name FROM system.parts WHERE table == '{}' AND level >= 1 and active = 1 ORDER BY modification_time".format( - name)).strip().split('\n') + name)).strip().split('\n')) - assert all(disk == 'external' for disk in disks_for_merges) + assert disks_for_merges == tuple('external' for _ in disks_for_merges) finally: node1.query(f"DROP TABLE IF EXISTS {name} SYNC") diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py index 3f88f719fe4..ab52879c7c3 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test.py @@ -24,7 +24,7 @@ def started_node(): def test_send_segfault(started_node): - if started_node.is_built_with_thread_sanitizer(): + if started_node.is_built_with_thread_sanitizer() or started_node.is_built_with_memory_sanitizer(): pytest.skip("doesn't fit in timeouts for stacktrace generation") started_node.copy_file_to_container(os.path.join(SCRIPT_DIR, "fake_sentry_server.py"), "/fake_sentry_server.py") diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index ea8341aebde..432765e6dd2 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -100,12 +100,11 @@ def test_identity(): cluster_1 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_with_password.xml') cluster_2 = ClickHouseCluster(__file__) - # TODO ACL not implemented in Keeper. node1 = cluster_1.add_instance('node1', main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_with_password.xml"], - with_zookeeper=True, zookeeper_use_tmpfs=False, use_keeper=False) + with_zookeeper=True, zookeeper_use_tmpfs=False) node2 = cluster_2.add_instance('node2', main_configs=["configs/remote_servers.xml"], with_zookeeper=True, - zookeeper_use_tmpfs=False, use_keeper=False) + zookeeper_use_tmpfs=False) try: cluster_1.start() @@ -164,12 +163,12 @@ def test_secure_connection(): "configs_secure/conf.d/remote_servers.xml", "configs_secure/conf.d/ssl_conf.xml"], with_zookeeper=True, zookeeper_docker_compose_path=docker_compose.name, - zookeeper_use_tmpfs=False) + zookeeper_use_tmpfs=False, use_keeper=False) node2 = cluster.add_instance('node2', main_configs=["configs_secure/client.crt", "configs_secure/client.key", "configs_secure/conf.d/remote_servers.xml", "configs_secure/conf.d/ssl_conf.xml"], with_zookeeper=True, zookeeper_docker_compose_path=docker_compose.name, - zookeeper_use_tmpfs=False) + zookeeper_use_tmpfs=False, use_keeper=False) try: cluster.start() diff --git a/tests/performance/string_to_int.xml b/tests/performance/string_to_int.xml new file mode 100644 index 00000000000..f231a4554d4 --- /dev/null +++ b/tests/performance/string_to_int.xml @@ -0,0 +1,12 @@ + + DROP TABLE IF EXISTS numeric_strings + CREATE TABLE numeric_strings(num String) ENGINE Memory + + + INSERT INTO numeric_strings SELECT number FROM numbers(30000000) + + + SELECT count(num::Int64) FROM numeric_strings FORMAT Null + + DROP TABLE IF EXISTS numeric_strings + diff --git a/tests/queries/0_stateless/00118_storage_join.reference b/tests/queries/0_stateless/00118_storage_join.reference index dad5b94e47b..56920b290e6 100644 --- a/tests/queries/0_stateless/00118_storage_join.reference +++ b/tests/queries/0_stateless/00118_storage_join.reference @@ -18,3 +18,49 @@ 7 8 9 +0 0 +1 1 abc +2 2 def +3 3 +4 4 +5 5 +6 6 ghi +7 7 +8 8 +9 9 +0 0 0 +1 1 1 abc +2 2 2 def +3 0 3 +4 0 4 +5 0 5 +6 6 6 ghi +7 0 7 +8 0 8 +9 0 9 +0 3 +3 9 +2 21 def +1 12 abc + +0 45 +0 0 +1 1 abc +2 2 def +3 3 +4 4 +5 5 +6 6 ghi +7 7 +8 8 +9 9 +0 0 0 +1 1 1 abc +2 2 2 def +3 0 3 +4 0 4 +5 0 5 +6 6 6 ghi +7 0 7 +8 0 8 +9 0 9 diff --git a/tests/queries/0_stateless/00118_storage_join.sql b/tests/queries/0_stateless/00118_storage_join.sql index 9da8829e5c7..47896d3316c 100644 --- a/tests/queries/0_stateless/00118_storage_join.sql +++ b/tests/queries/0_stateless/00118_storage_join.sql @@ -1,11 +1,19 @@ -DROP TABLE IF EXISTS join; +DROP TABLE IF EXISTS t2; -CREATE TABLE join (k UInt64, s String) ENGINE = Join(ANY, LEFT, k); +CREATE TABLE t2 (k UInt64, s String) ENGINE = Join(ANY, LEFT, k); -INSERT INTO join VALUES (1, 'abc'), (2, 'def'); -SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; +INSERT INTO t2 VALUES (1, 'abc'), (2, 'def'); +SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; -INSERT INTO join VALUES (6, 'ghi'); -SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; +INSERT INTO t2 VALUES (6, 'ghi'); +SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; -DROP TABLE join; +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number as s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number as s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; + +SELECT k, js1.s, t2.s FROM (SELECT toUInt64(number / 3) AS k, sum(number) as s FROM numbers(10) GROUP BY toUInt64(number / 3) WITH TOTALS) js1 ANY LEFT JOIN t2 USING k; + +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k; +SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k; + +DROP TABLE t2; diff --git a/tests/queries/0_stateless/00119_storage_join.sql b/tests/queries/0_stateless/00119_storage_join.sql index e1cc7a67588..2569a64d2c3 100644 --- a/tests/queries/0_stateless/00119_storage_join.sql +++ b/tests/queries/0_stateless/00119_storage_join.sql @@ -1,18 +1,18 @@ -DROP TABLE IF EXISTS join; +DROP TABLE IF EXISTS t2; -CREATE TABLE join (s String, x Array(UInt8), k UInt64) ENGINE = Join(ANY, LEFT, k); +CREATE TABLE t2 (s String, x Array(UInt8), k UInt64) ENGINE = Join(ANY, LEFT, k); -INSERT INTO join VALUES ('abc', [0], 1), ('def', [1, 2], 2); -INSERT INTO join (k, s) VALUES (3, 'ghi'); -INSERT INTO join (x, k) VALUES ([3, 4, 5], 4); +INSERT INTO t2 VALUES ('abc', [0], 1), ('def', [1, 2], 2); +INSERT INTO t2 (k, s) VALUES (3, 'ghi'); +INSERT INTO t2 (x, k) VALUES ([3, 4, 5], 4); -SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; -SELECT s, x FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; -SELECT x, s, k FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; -SELECT 1, x, 2, s, 3, k, 4 FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; +SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT s, x FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT x, s, k FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT 1, x, 2, s, 3, k, 4 FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; SELECT t1.k, t1.s, t2.x FROM ( SELECT number AS k, 'a' AS s FROM numbers(2) GROUP BY number WITH TOTALS ) AS t1 -ANY LEFT JOIN join AS t2 USING(k); +ANY LEFT JOIN t2 AS t2 USING(k); -DROP TABLE join; +DROP TABLE t2; diff --git a/tests/queries/0_stateless/00304_http_external_data.reference b/tests/queries/0_stateless/00304_http_external_data.reference index 93428f8768e..85909107584 100644 --- a/tests/queries/0_stateless/00304_http_external_data.reference +++ b/tests/queries/0_stateless/00304_http_external_data.reference @@ -2,3 +2,5 @@ 2 World 1 Hello 2 World +1 +2 diff --git a/tests/queries/0_stateless/00304_http_external_data.sh b/tests/queries/0_stateless/00304_http_external_data.sh index 41a9dea1ebb..4a097249cca 100755 --- a/tests/queries/0_stateless/00304_http_external_data.sh +++ b/tests/queries/0_stateless/00304_http_external_data.sh @@ -6,3 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo -ne '1,Hello\n2,World\n' | ${CLICKHOUSE_CURL} -sSF 'file=@-' "${CLICKHOUSE_URL}&query=SELECT+*+FROM+file&file_format=CSV&file_types=UInt8,String"; echo -ne '1@Hello\n2@World\n' | ${CLICKHOUSE_CURL} -sSF 'file=@-' "${CLICKHOUSE_URL}&query=SELECT+*+FROM+file&file_format=CSV&file_types=UInt8,String&format_csv_delimiter=@"; +echo -ne '\x01\x00\x00\x00\x02\x00\x00\x00' | ${CLICKHOUSE_CURL} -sSF "tmp=@-" "${CLICKHOUSE_URL}&query=SELECT+*+FROM+tmp&tmp_structure=TaskID+UInt32&tmp_format=RowBinary"; diff --git a/tests/queries/0_stateless/00574_empty_strings_deserialization.sh b/tests/queries/0_stateless/00574_empty_strings_deserialization.sh index 1cbc9456ab0..fecb6ee17e6 100755 --- a/tests/queries/0_stateless/00574_empty_strings_deserialization.sh +++ b/tests/queries/0_stateless/00574_empty_strings_deserialization.sh @@ -8,8 +8,8 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS empty_strings_deserialization" $CLICKHOUSE_CLIENT -q "CREATE TABLE empty_strings_deserialization(s String, i Int32, f Float32) ENGINE Memory" echo ',,' | $CLICKHOUSE_CLIENT -q "INSERT INTO empty_strings_deserialization FORMAT CSV" -echo 'aaa,-,' | $CLICKHOUSE_CLIENT -q "INSERT INTO empty_strings_deserialization FORMAT CSV" -echo 'bbb,,-' | $CLICKHOUSE_CLIENT -q "INSERT INTO empty_strings_deserialization FORMAT CSV" +echo 'aaa,,' | $CLICKHOUSE_CLIENT -q "INSERT INTO empty_strings_deserialization FORMAT CSV" +echo 'bbb,,-0' | $CLICKHOUSE_CLIENT -q "INSERT INTO empty_strings_deserialization FORMAT CSV" $CLICKHOUSE_CLIENT -q "SELECT * FROM empty_strings_deserialization ORDER BY s" diff --git a/tests/queries/0_stateless/00910_zookeeper_test_alter_compression_codecs_long.sql b/tests/queries/0_stateless/00910_zookeeper_test_alter_compression_codecs_long.sql index 085a79485fb..548f26eadd0 100644 --- a/tests/queries/0_stateless/00910_zookeeper_test_alter_compression_codecs_long.sql +++ b/tests/queries/0_stateless/00910_zookeeper_test_alter_compression_codecs_long.sql @@ -7,12 +7,12 @@ DROP TABLE IF EXISTS alter_compression_codec2; CREATE TABLE alter_compression_codec1 ( somedate Date CODEC(LZ4), id UInt64 CODEC(NONE) -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/'||currentDatabase()||'alter_compression_codecs', '1') PARTITION BY somedate ORDER BY id; +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/'||currentDatabase()||'alter_compression_codecs/{shard}', '1_{replica}') PARTITION BY somedate ORDER BY id; CREATE TABLE alter_compression_codec2 ( somedate Date CODEC(LZ4), id UInt64 CODEC(NONE) -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/'||currentDatabase()||'alter_compression_codecs', '2') PARTITION BY somedate ORDER BY id; +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00910/'||currentDatabase()||'alter_compression_codecs/{shard}', '2_{replica}') PARTITION BY somedate ORDER BY id; INSERT INTO alter_compression_codec1 VALUES('2018-01-01', 1); INSERT INTO alter_compression_codec1 VALUES('2018-01-01', 2); diff --git a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh index 71ca29bfd96..ca3f4efe743 100755 --- a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh +++ b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh @@ -6,6 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}" + $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS elog;" diff --git a/tests/queries/0_stateless/01019_materialized_view_select_extra_columns.sql b/tests/queries/0_stateless/01019_materialized_view_select_extra_columns.sql index 10bda34e6b9..4b7ea127190 100644 --- a/tests/queries/0_stateless/01019_materialized_view_select_extra_columns.sql +++ b/tests/queries/0_stateless/01019_materialized_view_select_extra_columns.sql @@ -28,7 +28,7 @@ FROM mv_extra_columns_src; INSERT INTO mv_extra_columns_src VALUES (0, 0), (1, 1), (2, 2); SELECT * FROM mv_extra_columns_dst ORDER by v; -SELECT * FROM mv_extra_columns_view; -- { serverError 16 } +SELECT * FROM mv_extra_columns_view; -- { serverError 10 } DROP TABLE mv_extra_columns_view; DROP TABLE mv_extra_columns_src; diff --git a/tests/queries/0_stateless/01154_move_partition_long.sh b/tests/queries/0_stateless/01154_move_partition_long.sh index f666cc929cc..66ebbacee42 100755 --- a/tests/queries/0_stateless/01154_move_partition_long.sh +++ b/tests/queries/0_stateless/01154_move_partition_long.sh @@ -6,8 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) declare -A engines engines[0]="MergeTree" -engines[1]="ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src', toString(randConstant()))" -engines[2]="ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src_' || toString(randConstant()), 'single_replica')" +engines[1]="ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{shard}/src', '{replica}_' || toString(randConstant()))" +engines[2]="ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{shard}/src_' || toString(randConstant()), '{replica}')" for ((i=0; i<16; i++)) do $CLICKHOUSE_CLIENT -q "CREATE TABLE dst_$i (p UInt64, k UInt64, v UInt64) diff --git a/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.sql b/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.sql index 7f276d1f8d4..7d0993c1bfc 100644 --- a/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.sql +++ b/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.sql @@ -1,4 +1,4 @@ -SELECT '-1E9-1E9-1E9-1E9' AS x, toDecimal32(x, 0); -- { serverError 6 } +SELECT '-1E9-1E9-1E9-1E9' AS x, toDecimal32(x, 0); -- { serverError 72 } SELECT '-1E9' AS x, toDecimal32(x, 0); -- { serverError 69 } SELECT '1E-9' AS x, toDecimal32(x, 0); SELECT '1E-8' AS x, toDecimal32(x, 0); diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference index 835e2af269a..2843b305f0a 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.reference @@ -7,22 +7,11 @@ ExpressionTransform (Expression) ExpressionTransform × 3 (SettingQuotaAndLimits) - (Expression) - ExpressionTransform × 3 - (Union) - (MergingSorted) - (Expression) - ExpressionTransform - (ReadFromMergeTree) - MergeTree 0 → 1 - (MergingSorted) + (ReadFromMergeTree) + ExpressionTransform × 4 + MergeTree 0 → 1 MergingSortedTransform 2 → 1 - (Expression) ExpressionTransform × 2 - (ReadFromMergeTree) MergeTree × 2 0 → 1 - (MergingSorted) - (Expression) - ExpressionTransform - (ReadFromMergeTree) - MergeTree 0 → 1 + ExpressionTransform + MergeTree 0 → 1 diff --git a/tests/queries/0_stateless/01568_window_functions_distributed.reference b/tests/queries/0_stateless/01568_window_functions_distributed.reference index 7d5a95046f7..483e84a2bee 100644 --- a/tests/queries/0_stateless/01568_window_functions_distributed.reference +++ b/tests/queries/0_stateless/01568_window_functions_distributed.reference @@ -10,7 +10,9 @@ select max(identity(dummy + 1)) over () from remote('127.0.0.{1,2}', system, one 1 1 drop table if exists t_01568; -create table t_01568 engine Memory as select intDiv(number, 3) p, number from numbers(9); +create table t_01568 engine Memory as +select intDiv(number, 3) p, modulo(number, 3) o, number +from numbers(9); select sum(number) over w, max(number) over w from t_01568 window w as (partition by p); 3 2 3 2 @@ -57,4 +59,26 @@ select groupArray(groupArray(number)) over (rows unbounded preceding) from remot [[0,3,6],[1,4,7]] [[0,3,6],[1,4,7],[2,5,8]] select groupArray(groupArray(number)) over (rows unbounded preceding) from remote('127.0.0.{1,2}', '', t_01568) group by mod(number, 3) settings distributed_group_by_no_merge=2; -- { serverError 48 } +-- proper ORDER BY w/window functions +select p, o, count() over (partition by p) +from remote('127.0.0.{1,2}', '', t_01568) +order by p, o; +0 0 6 +0 0 6 +0 1 6 +0 1 6 +0 2 6 +0 2 6 +1 0 6 +1 0 6 +1 1 6 +1 1 6 +1 2 6 +1 2 6 +2 0 6 +2 0 6 +2 1 6 +2 1 6 +2 2 6 +2 2 6 drop table t_01568; diff --git a/tests/queries/0_stateless/01568_window_functions_distributed.sql b/tests/queries/0_stateless/01568_window_functions_distributed.sql index bc82e1ed6ac..6f38597a7a3 100644 --- a/tests/queries/0_stateless/01568_window_functions_distributed.sql +++ b/tests/queries/0_stateless/01568_window_functions_distributed.sql @@ -9,7 +9,9 @@ select max(identity(dummy + 1)) over () from remote('127.0.0.{1,2}', system, one drop table if exists t_01568; -create table t_01568 engine Memory as select intDiv(number, 3) p, number from numbers(9); +create table t_01568 engine Memory as +select intDiv(number, 3) p, modulo(number, 3) o, number +from numbers(9); select sum(number) over w, max(number) over w from t_01568 window w as (partition by p); @@ -22,4 +24,9 @@ select groupArray(groupArray(number)) over (rows unbounded preceding) from remot select groupArray(groupArray(number)) over (rows unbounded preceding) from remote('127.0.0.{1,2}', '', t_01568) group by mod(number, 3) settings distributed_group_by_no_merge=1; select groupArray(groupArray(number)) over (rows unbounded preceding) from remote('127.0.0.{1,2}', '', t_01568) group by mod(number, 3) settings distributed_group_by_no_merge=2; -- { serverError 48 } +-- proper ORDER BY w/window functions +select p, o, count() over (partition by p) +from remote('127.0.0.{1,2}', '', t_01568) +order by p, o; + drop table t_01568; diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.reference b/tests/queries/0_stateless/01576_alias_column_rewrite.reference index c5679544e1d..9d3db8c1d00 100644 --- a/tests/queries/0_stateless/01576_alias_column_rewrite.reference +++ b/tests/queries/0_stateless/01576_alias_column_rewrite.reference @@ -34,19 +34,13 @@ Expression (Projection) FinishSorting Expression ((Before ORDER BY + Add table aliases)) SettingQuotaAndLimits (Set limits and quota after reading from storage) - Union - ReadFromMergeTree - ReadFromMergeTree - ReadFromMergeTree + ReadFromMergeTree Expression (Projection) Limit (preliminary LIMIT) FinishSorting Expression (Before ORDER BY) SettingQuotaAndLimits (Set limits and quota after reading from storage) - Union - ReadFromMergeTree - ReadFromMergeTree - ReadFromMergeTree + ReadFromMergeTree optimize_aggregation_in_order Expression ((Projection + Before ORDER BY)) Aggregating @@ -57,18 +51,12 @@ Expression ((Projection + Before ORDER BY)) Aggregating Expression ((Before GROUP BY + Add table aliases)) SettingQuotaAndLimits (Set limits and quota after reading from storage) - Union - ReadFromMergeTree - ReadFromMergeTree - ReadFromMergeTree + ReadFromMergeTree Expression ((Projection + Before ORDER BY)) Aggregating Expression (Before GROUP BY) SettingQuotaAndLimits (Set limits and quota after reading from storage) - Union - ReadFromMergeTree - ReadFromMergeTree - ReadFromMergeTree + ReadFromMergeTree second-index 1 1 diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index 2b54328688d..fa909972678 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -1054,6 +1054,12 @@ settings max_block_size = 3; 13 2 10 12 10 10 143 14 2 10 13 10 10 154 15 3 15 0 15 15 15 +-- careful with auto-application of Null combinator +select lagInFrame(toNullable(1)) over (); +0 +select lagInFrameOrNull(1) over (); -- { serverError 36 } +select intDiv(1, NULL) x, toTypeName(x), max(x) over (); +\N Nullable(Nothing) \N -- case-insensitive SQL-standard synonyms for any and anyLast select number, @@ -1114,3 +1120,31 @@ select count() over (rows between 2147483648 preceding and 2147493648 following) select count() over () from (select 1 a) l inner join (select 2 a) r using a; -- This case works as expected, one empty input chunk marked as input end. select count() over () where null; +-- Inheriting another window. +select number, count() over (w1 rows unbounded preceding) from numbers(10) +window + w0 as (partition by intDiv(number, 5) as p), + w1 as (w0 order by mod(number, 3) as o) +order by p, o, number +; +0 1 +3 2 +1 3 +4 4 +2 5 +6 1 +9 2 +7 3 +5 4 +8 5 +-- can't redefine PARTITION BY +select count() over (w partition by number) from numbers(1) window w as (partition by intDiv(number, 5)); -- { serverError 36 } +-- can't redefine existing ORDER BY +select count() over (w order by number) from numbers(1) window w as (partition by intDiv(number, 5) order by mod(number, 3)); -- { serverError 36 } +-- parent window can't have frame +select count() over (w range unbounded preceding) from numbers(1) window w as (partition by intDiv(number, 5) order by mod(number, 3) rows unbounded preceding); -- { serverError 36 } +-- looks weird but probably should work -- this is a window that inherits and changes nothing +select count() over (w) from numbers(1) window w as (); +1 +-- nonexistent parent window +select count() over (w2 rows unbounded preceding); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index 9c6f1a7401c..05f4cb49252 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -376,6 +376,11 @@ order by number settings max_block_size = 3; ; +-- careful with auto-application of Null combinator +select lagInFrame(toNullable(1)) over (); +select lagInFrameOrNull(1) over (); -- { serverError 36 } +select intDiv(1, NULL) x, toTypeName(x), max(x) over (); + -- case-insensitive SQL-standard synonyms for any and anyLast select number, @@ -422,3 +427,26 @@ select count() over (rows between 2147483648 preceding and 2147493648 following) select count() over () from (select 1 a) l inner join (select 2 a) r using a; -- This case works as expected, one empty input chunk marked as input end. select count() over () where null; + +-- Inheriting another window. +select number, count() over (w1 rows unbounded preceding) from numbers(10) +window + w0 as (partition by intDiv(number, 5) as p), + w1 as (w0 order by mod(number, 3) as o) +order by p, o, number +; + +-- can't redefine PARTITION BY +select count() over (w partition by number) from numbers(1) window w as (partition by intDiv(number, 5)); -- { serverError 36 } + +-- can't redefine existing ORDER BY +select count() over (w order by number) from numbers(1) window w as (partition by intDiv(number, 5) order by mod(number, 3)); -- { serverError 36 } + +-- parent window can't have frame +select count() over (w range unbounded preceding) from numbers(1) window w as (partition by intDiv(number, 5) order by mod(number, 3) rows unbounded preceding); -- { serverError 36 } + +-- looks weird but probably should work -- this is a window that inherits and changes nothing +select count() over (w) from numbers(1) window w as (); + +-- nonexistent parent window +select count() over (w2 rows unbounded preceding); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01601_detach_permanently.reference b/tests/queries/0_stateless/01601_detach_permanently.reference index 6683603c972..0232c1694e4 100644 --- a/tests/queries/0_stateless/01601_detach_permanently.reference +++ b/tests/queries/0_stateless/01601_detach_permanently.reference @@ -50,7 +50,7 @@ After database reattachement the table is still absent (it was detached permamen And we can not detach it permanently But we can attach it back And detach permanently again to check how database drop will behave -DROP database - Directory not empty error, but database deteched +DROP database - Directory not empty error, but database detached DROP database - now success ----------------------- database lazy tests diff --git a/tests/queries/0_stateless/01601_detach_permanently.sql b/tests/queries/0_stateless/01601_detach_permanently.sql index 0e21fd8f463..3af8ed573ef 100644 --- a/tests/queries/0_stateless/01601_detach_permanently.sql +++ b/tests/queries/0_stateless/01601_detach_permanently.sql @@ -128,8 +128,8 @@ ATTACH TABLE test1601_detach_permanently_ordinary.test_name_reuse; SELECT 'And detach permanently again to check how database drop will behave'; DETACH table test1601_detach_permanently_ordinary.test_name_reuse PERMANENTLY; -SELECT 'DROP database - Directory not empty error, but database deteched'; -DROP DATABASE test1601_detach_permanently_ordinary; -- { serverError 1000 } +SELECT 'DROP database - Directory not empty error, but database detached'; +DROP DATABASE test1601_detach_permanently_ordinary; -- { serverError 1001 } ATTACH DATABASE test1601_detach_permanently_ordinary; @@ -203,7 +203,7 @@ SELECT 'And detach permanently again to check how database drop will behave'; DETACH table test1601_detach_permanently_lazy.test_name_reuse PERMANENTLY; SELECT 'DROP database - Directory not empty error, but database deteched'; -DROP DATABASE test1601_detach_permanently_lazy; -- { serverError 1000 } +DROP DATABASE test1601_detach_permanently_lazy; -- { serverError 1001 } ATTACH DATABASE test1601_detach_permanently_lazy; diff --git a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference index 87659c32e39..1d0901cf9f6 100644 --- a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference +++ b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference @@ -8,11 +8,11 @@ ccccccccc aaaaaaaaa bbbbbbbbb aaaaaaaaa bbbbbbbbb ccccccccc -:107 +:233 :79 :35 :35 -:35 +:233 699415 aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb @@ -21,5 +21,5 @@ ccccccccc aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb 699415 0 :0 -:107 +:70 :79 diff --git a/tests/queries/0_stateless/01710_force_use_projection.reference b/tests/queries/0_stateless/01710_force_use_projection.reference new file mode 100644 index 00000000000..f1ad7de51f0 --- /dev/null +++ b/tests/queries/0_stateless/01710_force_use_projection.reference @@ -0,0 +1 @@ +3 1 diff --git a/tests/queries/0_stateless/01710_force_use_projection.sql b/tests/queries/0_stateless/01710_force_use_projection.sql new file mode 100644 index 00000000000..8931c65e34e --- /dev/null +++ b/tests/queries/0_stateless/01710_force_use_projection.sql @@ -0,0 +1,17 @@ +drop table if exists tp; + +create table tp (d1 Int32, d2 Int32, eventcnt Int64, projection p (select sum(eventcnt) group by d1)) engine = MergeTree order by (d1, d2); + +set allow_experimental_projection_optimization = 1, force_optimize_projection = 1; + +select sum(eventcnt) eventcnt, d1 from tp group by d1; + +select avg(eventcnt) eventcnt, d1 from tp group by d1; + +insert into tp values (1, 2, 3); + +select sum(eventcnt) eventcnt, d1 from tp group by d1; + +select avg(eventcnt) eventcnt, d1 from tp group by d1; -- { serverError 584 } + +drop table tp; diff --git a/tests/queries/0_stateless/01710_projection_drop_if_exists.reference b/tests/queries/0_stateless/01710_projection_drop_if_exists.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01710_projection_drop_if_exists.sql b/tests/queries/0_stateless/01710_projection_drop_if_exists.sql new file mode 100644 index 00000000000..f21092e5491 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_drop_if_exists.sql @@ -0,0 +1,11 @@ +drop table if exists tp; + +create table tp (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y; + +alter table tp drop projection pp; -- { serverError 582 } +alter table tp drop projection if exists pp; +alter table tp drop projection if exists p; +alter table tp drop projection p; -- { serverError 582 } +alter table tp drop projection if exists p; + +drop table tp; diff --git a/tests/queries/0_stateless/01710_projection_fetch.reference b/tests/queries/0_stateless/01710_projection_fetch.reference index 54e5bff80a9..abce5410b26 100644 --- a/tests/queries/0_stateless/01710_projection_fetch.reference +++ b/tests/queries/0_stateless/01710_projection_fetch.reference @@ -10,8 +10,8 @@ 3 3 4 4 0 -CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01710_projection_fetch_default\', \'2\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 +CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 2 -CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01710_projection_fetch_default\', \'2\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 -CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01710_projection_fetch_default\', \'2\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 -CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01710_projection_fetch_default\', \'2\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 +CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 +CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 +CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/{shard}/01710_projection_fetch_default\', \'2_{replica}\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 diff --git a/tests/queries/0_stateless/01710_projection_fetch.sql b/tests/queries/0_stateless/01710_projection_fetch.sql index 06790317808..7e4f6cc1d9a 100644 --- a/tests/queries/0_stateless/01710_projection_fetch.sql +++ b/tests/queries/0_stateless/01710_projection_fetch.sql @@ -1,9 +1,9 @@ drop table if exists tp_1; drop table if exists tp_2; -create table tp_1 (x Int32, y Int32, projection p (select x, y order by x)) engine = ReplicatedMergeTree('/clickhouse/tables/01710_projection_fetch_' || currentDatabase(), '1') order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32; +create table tp_1 (x Int32, y Int32, projection p (select x, y order by x)) engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/01710_projection_fetch_' || currentDatabase(), '1_{replica}') order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32; -create table tp_2 (x Int32, y Int32, projection p (select x, y order by x)) engine = ReplicatedMergeTree('/clickhouse/tables/01710_projection_fetch_' || currentDatabase(), '2') order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32; +create table tp_2 (x Int32, y Int32, projection p (select x, y order by x)) engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/01710_projection_fetch_' || currentDatabase(), '2_{replica}') order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32; insert into tp_1 select number, number from numbers(3); diff --git a/tests/queries/0_stateless/01710_projection_group_by_order_by.reference b/tests/queries/0_stateless/01710_projection_group_by_order_by.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01710_projection_group_by_order_by.sql b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql new file mode 100644 index 00000000000..9370e9d36ce --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql @@ -0,0 +1,5 @@ +drop table if exists tp; + +create table tp (type Int32, eventcnt UInt64, projection p (select sum(eventcnt), type group by type order by sum(eventcnt))) engine = MergeTree order by type; -- { serverError 583 } + +drop table if exists tp; diff --git a/tests/queries/0_stateless/01821_join_table_mutation.reference b/tests/queries/0_stateless/01821_join_table_mutation.reference new file mode 100644 index 00000000000..8c446c806b5 --- /dev/null +++ b/tests/queries/0_stateless/01821_join_table_mutation.reference @@ -0,0 +1,7 @@ +100 +10 +99 +m10 +50 +48 +0 diff --git a/tests/queries/0_stateless/01821_join_table_mutation.sql b/tests/queries/0_stateless/01821_join_table_mutation.sql new file mode 100644 index 00000000000..78903ebd6ec --- /dev/null +++ b/tests/queries/0_stateless/01821_join_table_mutation.sql @@ -0,0 +1,35 @@ +DROP TABLE IF EXISTS join_table_mutation; + +CREATE TABLE join_table_mutation(id Int32, name String) ENGINE = Join(ANY, LEFT, id); + +INSERT INTO join_table_mutation select number, toString(number) from numbers(100); + +SELECT count() FROM join_table_mutation; + +SELECT name FROM join_table_mutation WHERE id = 10; + +ALTER TABLE join_table_mutation DELETE WHERE id = 10; + +SELECT count() FROM join_table_mutation; + +SELECT name FROM join_table_mutation WHERE id = 10; + +INSERT INTO join_table_mutation VALUES (10, 'm10'); + +SELECT name FROM join_table_mutation WHERE id = 10; + +ALTER TABLE join_table_mutation DELETE WHERE id % 2 = 0; + +ALTER TABLE join_table_mutation UPDATE name = 'some' WHERE 1; -- {serverError 48} + +SELECT count() FROM join_table_mutation; + +ALTER TABLE join_table_mutation DELETE WHERE name IN ('1', '2', '3', '4'); + +SELECT count() FROM join_table_mutation; + +ALTER TABLE join_table_mutation DELETE WHERE 1; + +SELECT count() FROM join_table_mutation; + +DROP TABLE join_table_mutation; diff --git a/tests/queries/0_stateless/01821_join_table_race_long.reference b/tests/queries/0_stateless/01821_join_table_race_long.reference new file mode 100644 index 00000000000..f2018833bc6 --- /dev/null +++ b/tests/queries/0_stateless/01821_join_table_race_long.reference @@ -0,0 +1 @@ +1 foo diff --git a/tests/queries/0_stateless/01821_join_table_race_long.sh b/tests/queries/0_stateless/01821_join_table_race_long.sh new file mode 100755 index 00000000000..7c56bf77bfd --- /dev/null +++ b/tests/queries/0_stateless/01821_join_table_race_long.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS join_table_race" +$CLICKHOUSE_CLIENT -q "CREATE TABLE join_table_race(id Int32, name String) ENGINE = Join(ANY, LEFT, id)" + +for _ in {0..100}; do $CLICKHOUSE_CLIENT -q "INSERT INTO join_table_race VALUES ($RANDOM, '$RANDOM')" > /dev/null 2> /dev/null; done & + +for _ in {0..200}; do $CLICKHOUSE_CLIENT -q "SELECT count() FROM join_table_race FORMAT Null" > /dev/null 2> /dev/null; done & + +for _ in {0..100}; do $CLICKHOUSE_CLIENT -q "TRUNCATE TABLE join_table_race" > /dev/null 2> /dev/null; done & + +for _ in {0..100}; do $CLICKHOUSE_CLIENT -q "ALTER TABLE join_table_race DELETE WHERE id % 2 = 0" > /dev/null 2> /dev/null; done & + +wait + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE join_table_race" +$CLICKHOUSE_CLIENT -q "INSERT INTO join_table_race VALUES (1, 'foo')" +$CLICKHOUSE_CLIENT -q "SELECT id, name FROM join_table_race" + +$CLICKHOUSE_CLIENT -q "DROP TABLE join_table_race" diff --git a/tests/queries/0_stateless/01861_explain_pipeline.reference b/tests/queries/0_stateless/01861_explain_pipeline.reference index 8d755f807c0..9d62fb9f6b8 100644 --- a/tests/queries/0_stateless/01861_explain_pipeline.reference +++ b/tests/queries/0_stateless/01861_explain_pipeline.reference @@ -1,13 +1,10 @@ (Expression) ExpressionTransform (SettingQuotaAndLimits) - (Expression) + (ReadFromMergeTree) ExpressionTransform - (MergingFinal) ReplacingSorted 2 → 1 - (Expression) ExpressionTransform × 2 - (ReadFromMergeTree) MergeTree × 2 0 → 1 0 0 1 1 @@ -19,13 +16,10 @@ ExpressionTransform (Expression) ExpressionTransform × 2 (SettingQuotaAndLimits) - (Expression) + (ReadFromMergeTree) ExpressionTransform × 2 - (MergingFinal) ReplacingSorted × 2 2 → 1 Copy × 2 1 → 2 AddingSelector × 2 - (Expression) ExpressionTransform × 2 - (ReadFromMergeTree) MergeTree × 2 0 → 1 diff --git a/tests/queries/0_stateless/01881_create_as_tuple.reference b/tests/queries/0_stateless/01881_create_as_tuple.reference new file mode 100644 index 00000000000..76109462c59 --- /dev/null +++ b/tests/queries/0_stateless/01881_create_as_tuple.reference @@ -0,0 +1,6 @@ +0 [('string',0)] +1 [('string',1)] +2 [('string',2)] +0 [('string',0)] +1 [('string',1)] +2 [('string',2)] diff --git a/tests/queries/0_stateless/01881_create_as_tuple.sql b/tests/queries/0_stateless/01881_create_as_tuple.sql new file mode 100644 index 00000000000..cf370fe4633 --- /dev/null +++ b/tests/queries/0_stateless/01881_create_as_tuple.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS t_create_as_tuple; + +CREATE TABLE t_create_as_tuple ENGINE = MergeTree() +ORDER BY number AS +SELECT number, [('string',number)] AS array FROM numbers(3); + +SELECT * FROM t_create_as_tuple ORDER BY number; + +DETACH TABLE t_create_as_tuple; +ATTACH TABLE t_create_as_tuple; + +SELECT * FROM t_create_as_tuple ORDER BY number; + +DROP TABLE t_create_as_tuple; diff --git a/tests/queries/0_stateless/01882_scalar_subquery_exception.reference b/tests/queries/0_stateless/01882_scalar_subquery_exception.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01882_scalar_subquery_exception.sql b/tests/queries/0_stateless/01882_scalar_subquery_exception.sql new file mode 100644 index 00000000000..c7a4555d876 --- /dev/null +++ b/tests/queries/0_stateless/01882_scalar_subquery_exception.sql @@ -0,0 +1,16 @@ +drop table if exists nums_in_mem; +drop table if exists nums_in_mem_dist; + +create table nums_in_mem(v UInt64) engine=Memory; +insert into nums_in_mem select * from system.numbers limit 1000000; + +create table nums_in_mem_dist as nums_in_mem engine=Distributed('test_shard_localhost', currentDatabase(), nums_in_mem); + +set prefer_localhost_replica = 0; +set max_rows_to_read = 100; + +select + count() + / + (select count() from nums_in_mem_dist where rand() > 0) +from system.one; -- { serverError 158 } diff --git a/tests/queries/0_stateless/01883_subcolumns_distributed.reference b/tests/queries/0_stateless/01883_subcolumns_distributed.reference new file mode 100644 index 00000000000..459f90ada98 --- /dev/null +++ b/tests/queries/0_stateless/01883_subcolumns_distributed.reference @@ -0,0 +1,2 @@ +3 0 bbb ccc +3 0 bbb ccc diff --git a/tests/queries/0_stateless/01883_subcolumns_distributed.sql b/tests/queries/0_stateless/01883_subcolumns_distributed.sql new file mode 100644 index 00000000000..5593dc60251 --- /dev/null +++ b/tests/queries/0_stateless/01883_subcolumns_distributed.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS t_subcolumns_local; +DROP TABLE IF EXISTS t_subcolumns_dist; + +CREATE TABLE t_subcolumns_local (arr Array(UInt32), n Nullable(String), t Tuple(s1 String, s2 String)) +ENGINE = MergeTree ORDER BY tuple(); + +CREATE TABLE t_subcolumns_dist AS t_subcolumns_local ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_subcolumns_local); + +INSERT INTO t_subcolumns_local VALUES ([1, 2, 3], 'aaa', ('bbb', 'ccc')); + +SELECT arr.size0, n.null, t.s1, t.s2 FROM t_subcolumns_dist; + +DROP TABLE t_subcolumns_local; + +-- StripeLog doesn't support subcolumns. +CREATE TABLE t_subcolumns_local (arr Array(UInt32), n Nullable(String), t Tuple(s1 String, s2 String)) ENGINE = StripeLog; + +SELECT arr.size0, n.null, t.s1, t.s2 FROM t_subcolumns_dist; -- { serverError 47 } + +DROP TABLE t_subcolumns_local; +DROP TABLE t_subcolumns_dist; diff --git a/tests/queries/0_stateless/01888_read_int_safe.reference b/tests/queries/0_stateless/01888_read_int_safe.reference new file mode 100644 index 00000000000..94d36aad185 --- /dev/null +++ b/tests/queries/0_stateless/01888_read_int_safe.reference @@ -0,0 +1,2 @@ +1 +-1 diff --git a/tests/queries/0_stateless/01888_read_int_safe.sql b/tests/queries/0_stateless/01888_read_int_safe.sql new file mode 100644 index 00000000000..3caa4878aba --- /dev/null +++ b/tests/queries/0_stateless/01888_read_int_safe.sql @@ -0,0 +1,10 @@ +select toInt64('--1'); -- { serverError 72; } +select toInt64('+-1'); -- { serverError 72; } +select toInt64('++1'); -- { serverError 72; } +select toInt64('++'); -- { serverError 72; } +select toInt64('+'); -- { serverError 72; } +select toInt64('1+1'); -- { serverError 72; } +select toInt64('1-1'); -- { serverError 72; } +select toInt64(''); -- { serverError 32; } +select toInt64('1'); +select toInt64('-1'); diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 2b03f32dccc..a59c1a72a52 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -238,4 +238,5 @@ 01850_dist_INSERT_preserve_error 01870_modulo_partition_key 01880_remote_ipv6 +01882_scalar_subquery_exception 01882_check_max_parts_to_merge_at_once diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 6a278316387..a9a41b1ac1f 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -109,6 +109,7 @@ "01153_attach_mv_uuid" ], "database-replicated": [ + /// Unclassified "memory_tracking", "memory_usage", "live_view", @@ -167,8 +168,9 @@ /// Does not support renaming of multiple tables in single query "00634_rename_view", "00140_rename", - "01783_http_chunk_size", - "01710_projection_fetch" + /// Requires investigation + "00953_zookeeper_suetin_deduplication_bug", + "01783_http_chunk_size" ], "polymorphic-parts": [ "01508_partition_pruning_long", /// bug, shoud be fixed diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 269b96e2143..e08a6ad0ffe 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v21.5.6.6-stable 2021-05-29 v21.5.5.12-stable 2021-05-20 v21.4.7.3-stable 2021-05-19 v21.4.6.55-stable 2021-04-30 diff --git a/utils/zookeeper-test/main.cpp b/utils/zookeeper-test/main.cpp index 102c30e59a1..39a282ce258 100644 --- a/utils/zookeeper-test/main.cpp +++ b/utils/zookeeper-test/main.cpp @@ -73,8 +73,7 @@ void testCreateList(zkutil::ZooKeeper & zk) void testCreateSetVersionRequest(zkutil::ZooKeeper & zk) { zk.create("/data/check_data", "d", zkutil::CreateMode::Persistent); - Coordination::Stat stat; - std::string result = zk.get("/data/check_data", &stat); + Coordination::Stat stat{}; try { zk.set("/data/check_data", "e", stat.version + 2); @@ -224,7 +223,7 @@ std::string random_string(size_t length) "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; const size_t max_index = (sizeof(charset) - 1); - return charset[rand() % max_index]; + return charset[rand() % max_index]; /// NOLINT }; std::string str(length, 0); std::generate_n(str.begin(), length, randchar); @@ -259,9 +258,9 @@ void createOnPrefix(const std::string & zkhost, const String & path_prefix, size holder_futures.push_back(zk.asyncCreate(path, random_string(datasize), zkutil::CreateMode::Persistent)); } - for (size_t i = 0; i < holder_futures.size(); ++i) - holder_futures[i].get(); - } + for (auto & future : holder_futures) + future.get(); + } catch (...) { ::exit(-1);