diff --git a/.github/ISSUE_TEMPLATE/40_bug-report.md b/.github/ISSUE_TEMPLATE/40_bug-report.md index 97137366189..5c8611d47e6 100644 --- a/.github/ISSUE_TEMPLATE/40_bug-report.md +++ b/.github/ISSUE_TEMPLATE/40_bug-report.md @@ -10,12 +10,26 @@ assignees: '' You have to provide the following information whenever possible. **Describe the bug** + A clear and concise description of what works not as it is supposed to. **Does it reproduce on recent release?** + [The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv) +**Enable crash reporting** + +If possible, change "enabled" to true in "send_crash_reports" section in `config.xml`: + +``` + + + + false +``` + **How to reproduce** + * Which ClickHouse server version to use * Which interface to use, if matters * Non-default settings, if any @@ -24,10 +38,13 @@ A clear and concise description of what works not as it is supposed to. * Queries to run that lead to unexpected result **Expected behavior** + A clear and concise description of what you expected to happen. **Error message and/or stacktrace** + If applicable, add screenshots to help explain your problem. **Additional context** + Add any other context about the problem here. diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 01e700ebba3..6aa65942445 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -26,8 +26,6 @@ #include #include #include -#include -#include #include #include #include @@ -59,6 +57,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include @@ -70,6 +69,7 @@ #endif #include +namespace fs = std::filesystem; DB::PipeFDs signal_pipe; @@ -437,11 +437,11 @@ static void sanitizerDeathCallback() static std::string createDirectory(const std::string & file) { - auto path = Poco::Path(file).makeParent(); - if (path.toString().empty()) + fs::path path = fs::path(file).parent_path(); + if (path.empty()) return ""; - Poco::File(path).createDirectories(); - return path.toString(); + fs::create_directories(path); + return path; }; @@ -449,7 +449,7 @@ static bool tryCreateDirectories(Poco::Logger * logger, const std::string & path { try { - Poco::File(path).createDirectories(); + fs::create_directories(path); return true; } catch (...) @@ -470,7 +470,7 @@ void BaseDaemon::reloadConfiguration() */ config_path = config().getString("config-file", getDefaultConfigFileName()); DB::ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); + config_processor.setConfigPath(fs::path(config_path).parent_path()); loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true); if (last_configuration != nullptr) @@ -524,18 +524,20 @@ std::string BaseDaemon::getDefaultConfigFileName() const void BaseDaemon::closeFDs() { #if defined(OS_FREEBSD) || defined(OS_DARWIN) - Poco::File proc_path{"/dev/fd"}; + fs::path proc_path{"/dev/fd"}; #else - Poco::File proc_path{"/proc/self/fd"}; + fs::path proc_path{"/proc/self/fd"}; #endif - if (proc_path.isDirectory()) /// Hooray, proc exists + if (fs::is_directory(proc_path)) /// Hooray, proc exists { - std::vector fds; - /// in /proc/self/fd directory filenames are numeric file descriptors - proc_path.list(fds); - for (const auto & fd_str : fds) + /// in /proc/self/fd directory filenames are numeric file descriptors. + /// Iterate directory separately from closing fds to avoid closing iterated directory fd. + std::vector fds; + for (const auto & path : fs::directory_iterator(proc_path)) + fds.push_back(DB::parse(path.path().filename())); + + for (const auto & fd : fds) { - int fd = DB::parse(fd_str); if (fd > 2 && fd != signal_pipe.fds_rw[0] && fd != signal_pipe.fds_rw[1]) ::close(fd); } @@ -597,7 +599,7 @@ void BaseDaemon::initialize(Application & self) { /** When creating pid file and looking for config, will search for paths relative to the working path of the program when started. */ - std::string path = Poco::Path(config().getString("application.path")).setFileName("").toString(); + std::string path = fs::path(config().getString("application.path")).replace_filename(""); if (0 != chdir(path.c_str())) throw Poco::Exception("Cannot change directory to " + path); } @@ -645,7 +647,7 @@ void BaseDaemon::initialize(Application & self) std::string log_path = config().getString("logger.log", ""); if (!log_path.empty()) - log_path = Poco::Path(log_path).setFileName("").toString(); + log_path = fs::path(log_path).replace_filename(""); /** Redirect stdout, stderr to separate files in the log directory (or in the specified file). * Some libraries write to stderr in case of errors in debug mode, @@ -708,8 +710,7 @@ void BaseDaemon::initialize(Application & self) tryCreateDirectories(&logger(), core_path); - Poco::File cores = core_path; - if (!(cores.exists() && cores.isDirectory())) + if (!(fs::exists(core_path) && fs::is_directory(core_path))) { core_path = !log_path.empty() ? log_path : "/opt/"; tryCreateDirectories(&logger(), core_path); diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 1028dc7d2dc..3571c64edd6 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -1,6 +1,5 @@ #include -#include #include #include @@ -25,6 +24,7 @@ # include # include +namespace fs = std::filesystem; namespace { @@ -53,8 +53,7 @@ void setExtras() sentry_set_extra("physical_cpu_cores", sentry_value_new_int32(getNumberOfPhysicalCPUCores())); if (!server_data_path.empty()) - sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix( - Poco::File(server_data_path).freeSpace()).c_str())); + sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix(fs::space(server_data_path).free).c_str())); } void sentry_logger(sentry_level_e level, const char * message, va_list args, void *) @@ -110,12 +109,12 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) if (enabled) { server_data_path = config.getString("path", ""); - const std::filesystem::path & default_tmp_path = std::filesystem::path(config.getString("tmp_path", Poco::Path::temp())) / "sentry"; + const std::filesystem::path & default_tmp_path = fs::path(config.getString("tmp_path", fs::temp_directory_path())) / "sentry"; const std::string & endpoint = config.getString("send_crash_reports.endpoint"); const std::string & temp_folder_path = config.getString("send_crash_reports.tmp_path", default_tmp_path); - Poco::File(temp_folder_path).createDirectories(); + fs::create_directories(temp_folder_path); sentry_options_t * options = sentry_options_new(); /// will be freed by sentry_init or sentry_shutdown sentry_options_set_release(options, VERSION_STRING_SHORT); diff --git a/base/loggers/Loggers.cpp b/base/loggers/Loggers.cpp index 913deaf1eb8..80e62d0a6d6 100644 --- a/base/loggers/Loggers.cpp +++ b/base/loggers/Loggers.cpp @@ -6,10 +6,11 @@ #include "OwnFormattingChannel.h" #include "OwnPatternFormatter.h" #include -#include #include #include -#include +#include + +namespace fs = std::filesystem; namespace DB { @@ -20,11 +21,11 @@ namespace DB // TODO: move to libcommon static std::string createDirectory(const std::string & file) { - auto path = Poco::Path(file).makeParent(); - if (path.toString().empty()) + auto path = fs::path(file).parent_path(); + if (path.empty()) return ""; - Poco::File(path).createDirectories(); - return path.toString(); + fs::create_directories(path); + return path; }; void Loggers::setTextLog(std::shared_ptr log, int max_priority) @@ -70,7 +71,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log // Set up two channel chains. log_file = new Poco::FileChannel; - log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString()); + log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(log_path)); log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); @@ -102,7 +103,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log std::cerr << "Logging errors to " << errorlog_path << std::endl; error_log_file = new Poco::FileChannel; - error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString()); + error_log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(errorlog_path)); error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true")); diff --git a/contrib/cassandra b/contrib/cassandra index c097fb5c7e6..eb9b68dadbb 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit c097fb5c7e63cc430016d9a8b240d8e63fbefa52 +Subproject commit eb9b68dadbb4417a2c132ad4a1c2fa76e65e6fc1 diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile index 253ca1b729a..b08b1b0bf0d 100644 --- a/docker/test/sqlancer/Dockerfile +++ b/docker/test/sqlancer/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:20.04 RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git openjdk-14-jdk maven python3 --yes --no-install-recommends RUN wget https://github.com/sqlancer/sqlancer/archive/master.zip -O /sqlancer.zip + RUN mkdir /sqlancer && \ cd /sqlancer && \ unzip /sqlancer.zip diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index f5c85289171..835e3c6e98a 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -42,6 +42,8 @@ toc_title: Client Libraries - Ruby - [ClickHouse (Ruby)](https://github.com/shlima/click_house) - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) +- Rust + - [Klickhouse](https://github.com/Protryon/klickhouse) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickHouse](https://github.com/IMSMWU/RClickHouse) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md index 32d174136e0..70f30f3a480 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md @@ -18,10 +18,10 @@ When using multiple `quantile*` functions with different levels in a query, the **Syntax** ``` sql -quantileTDigest(level)(expr) +quantileTDigestWeighted(level)(expr, weight) ``` -Alias: `medianTDigest`. +Alias: `medianTDigestWeighted`. **Arguments** diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 69e2caeb322..5eaf0558d7b 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -31,15 +31,17 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engin If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key). !!! note "Note" - Notice that `*` behaves just like in `SELECT`: `MATERIALIZED` and `ALIAS` columns are not used for expansion. - Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an ALIAS column. + Notice that `*` behaves just like in `SELECT`: [MATERIALIZED](../../sql-reference/statements/create/table.md#materialized) and [ALIAS](../../sql-reference/statements/create/table.md#alias) columns are not used for expansion. + Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an `ALIAS` column. + +**Syntax** ``` sql -OPTIMIZE TABLE table DEDUPLICATE; -- the old one -OPTIMIZE TABLE table DEDUPLICATE BY *; -- not the same as the old one, excludes MATERIALIZED columns (see the note above) +OPTIMIZE TABLE table DEDUPLICATE; -- all columns +OPTIMIZE TABLE table DEDUPLICATE BY *; -- excludes MATERIALIZED and ALIAS columns +OPTIMIZE TABLE table DEDUPLICATE BY colX,colY,colZ; OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT colX; OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT (colX, colY); -OPTIMIZE TABLE table DEDUPLICATE BY col1,col2,col3; OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex'); OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT colX; OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (colX, colY); @@ -47,7 +49,7 @@ OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (c **Examples** -Create a table: +Consider the table: ``` sql CREATE TABLE example ( @@ -62,32 +64,129 @@ CREATE TABLE example ( PARTITION BY partition_key ORDER BY (primary_key, secondary_key); ``` +``` sql +INSERT INTO example (primary_key, secondary_key, value, partition_key) +VALUES (0, 0, 0, 0), (0, 0, 0, 0), (1, 1, 2, 2), (1, 1, 2, 3), (1, 1, 3, 3); +``` +``` sql +SELECT * FROM example; +``` +Result: +``` -The 'old' deduplicate, all columns are taken into account, i.e. row is removed only if all values in all columns are equal to corresponding values in previous row. +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +When columns for deduplication are not specified, all of them are taken into account. Row is removed only if all values in all columns are equal to corresponding values in previous row: ``` sql OPTIMIZE TABLE example FINAL DEDUPLICATE; ``` - -Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key`, and `materialized_value` columns. - ``` sql +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +When columns are specified implicitly, the table is deduplicated by all columns that are not `ALIAS` or `MATERIALIZED`. Considering the table above, these are `primary_key`, `secondary_key`, `value`, and `partition_key` columns: +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY *; ``` - -Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED` and explicitly not `materialized_value`: `primary_key`, `secondary_key`, `value`, and `partition_key` columns. - ``` sql -OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT materialized_value; +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ ``` -Deduplicate explicitly by `primary_key`, `secondary_key`, and `partition_key` columns. +Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED` and explicitly not `value`: `primary_key`, `secondary_key`, and `partition_key` columns. + ``` sql +OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT value; +``` +``` sql +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +Deduplicate explicitly by `primary_key`, `secondary_key`, and `partition_key` columns: +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY primary_key, secondary_key, partition_key; ``` - -Deduplicate by any column matching a regex: `primary_key`, `secondary_key`, and `partition_key` columns. - ``` sql +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +Deduplicate by any column matching a regex: `primary_key`, `secondary_key`, and `partition_key` columns: +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY COLUMNS('.*_key'); ``` +``` sql +SELECT * FROM example; +``` +Result: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index e1a9d613537..70503ec4de9 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -32,10 +32,22 @@ ClickHouse не оповещает клиента. Чтобы включить Список столбцов для дедупликации должен включать все столбцы, указанные в условиях сортировки (первичный ключ и ключ сортировки), а также в условиях партиционирования (ключ партиционирования). - !!! note "Примечание" - Обратите внимание, что символ подстановки `*` обрабатывается так же, как и в запросах `SELECT`: столбцы `MATERIALIZED` и `ALIAS` не включаются в результат. - Если указать пустой список или выражение, которое возвращает пустой список, или дедуплицировать столбец по псевдониму (`ALIAS`), то сервер вернет ошибку. +!!! note "Примечание" + Обратите внимание, что символ подстановки `*` обрабатывается так же, как и в запросах `SELECT`: столбцы [MATERIALIZED](../../sql-reference/statements/create/table.md#materialized) и [ALIAS](../../sql-reference/statements/create/table.md#alias) не включаются в результат. + Если указать пустой список или выражение, которое возвращает пустой список, то сервер вернет ошибку. Запрос вида `DEDUPLICATE BY aliased_value` также вернет ошибку. +**Синтаксис** + +``` sql +OPTIMIZE TABLE table DEDUPLICATE; -- по всем столбцам +OPTIMIZE TABLE table DEDUPLICATE BY *; -- исключаются столбцы MATERIALIZED и ALIAS +OPTIMIZE TABLE table DEDUPLICATE BY colX,colY,colZ; +OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT colX; +OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT (colX, colY); +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex'); +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT colX; +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (colX, colY); +``` **Примеры** @@ -50,38 +62,131 @@ CREATE TABLE example ( materialized_value UInt32 MATERIALIZED 12345, aliased_value UInt32 ALIAS 2, PRIMARY KEY primary_key -) ENGINE=MergeTree -PARTITION BY partition_key; +) ENGINE=MergeTree　 +PARTITION BY partition_key +ORDER BY (primary_key, secondary_key); +``` +``` sql +INSERT INTO example (primary_key, secondary_key, value, partition_key) +VALUES (0, 0, 0, 0), (0, 0, 0, 0), (1, 1, 2, 2), (1, 1, 2, 3), (1, 1, 3, 3); +``` +``` sql +SELECT * FROM example; +``` +Результат: ``` -Прежний способ дедупликации, когда учитываются все столбцы. Строка удаляется только в том случае, если все значения во всех столбцах равны соответствующим значениям в предыдущей строке. +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` +Если в запросе не указаны столбцы, по которым нужно дедуплицировать, то учитываются все столбцы таблицы. Строка удаляется только в том случае, если все значения во всех столбцах равны соответствующим значениям в другой строке. ``` sql OPTIMIZE TABLE example FINAL DEDUPLICATE; ``` - -Дедупликация по всем столбцам, кроме `ALIAS` и `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key` и `materialized_value`. - - ``` sql -OPTIMIZE TABLE example FINAL DEDUPLICATE BY *; +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ ``` -Дедупликация по всем столбцам, кроме `ALIAS`, `MATERIALIZED` и `materialized_value`: столбцы `primary_key`, `secondary_key`, `value` и `partition_key`. - - +Если столбцы в запросе указаны через `*`, то дедупликация пройдет по всем столбцам, кроме `ALIAS` и `MATERIALIZED`. Для таблицы `example` будут учтены: `primary_key`, `secondary_key`, `value` и `partition_key`. +```sql +OPTIMIZE TABLE example FINAL DEDUPLICATE BY *; +``` ``` sql -OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT materialized_value; +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +│ 1 │ 1 │ 3 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +Дедупликация по всем столбцам, кроме `ALIAS` и `MATERIALIZED` (`BY *`), и с исключением столбца `value`: `primary_key`, `secondary_key` и `partition_key`. +``` sql +OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT value; +``` +``` sql +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ ``` Дедупликация по столбцам `primary_key`, `secondary_key` и `partition_key`. - -``` sql +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY primary_key, secondary_key, partition_key; ``` - -Дедупликация по любому столбцу, соответствующему регулярному выражению: столбцам `primary_key`, `secondary_key` и `partition_key`. - ``` sql +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` + +Дедупликация по любому столбцу, который соответствует регулярному выражению `.*_key`: `primary_key`, `secondary_key` и `partition_key`. +```sql OPTIMIZE TABLE example FINAL DEDUPLICATE BY COLUMNS('.*_key'); ``` +``` sql +SELECT * FROM example; +``` +Результат: +``` +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 0 │ 0 │ 0 │ 0 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 2 │ +└─────────────┴───────────────┴───────┴───────────────┘ +┌─primary_key─┬─secondary_key─┬─value─┬─partition_key─┐ +│ 1 │ 1 │ 2 │ 3 │ +└─────────────┴───────────────┴───────┴───────────────┘ +``` \ No newline at end of file diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 1d2b579db3a..2e48c5d20c5 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -36,7 +35,9 @@ #include #include #include +#include +namespace fs = std::filesystem; /** A tool for evaluating ClickHouse performance. * The tool emulates a case with fixed amount of simultaneously executing queries. @@ -119,8 +120,8 @@ public: int main(const std::vector &) override { - if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results - Poco::File(json_path).remove(); + if (!json_path.empty() && fs::exists(json_path)) /// Clear file with previous results + fs::remove(json_path); readQueries(); runBenchmark(); @@ -159,7 +160,7 @@ private: bool print_stacktrace; const Settings & settings; SharedContextHolder shared_context; - ContextPtr global_context; + ContextMutablePtr global_context; QueryProcessingStage::Enum query_processing_stage; /// Don't execute new queries after timelimit or SIGINT or exception diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 098f7e689c5..f268d2b5cdc 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -87,6 +86,8 @@ #include #include #include +#include +#include #if !defined(ARCADIA_BUILD) # include @@ -96,6 +97,7 @@ #pragma GCC optimize("-fno-var-tracking-assignments") #endif +namespace fs = std::filesystem; namespace DB { @@ -181,7 +183,7 @@ private: bool has_vertical_output_suffix = false; /// Is \G present at the end of the query string? SharedContextHolder shared_context = Context::createShared(); - ContextPtr context = Context::createGlobal(shared_context.get()); + ContextMutablePtr context = Context::createGlobal(shared_context.get()); /// Buffer that reads from stdin in batch mode. ReadBufferFromFileDescriptor std_in{STDIN_FILENO}; @@ -276,7 +278,7 @@ private: /// Set path for format schema files if (config().has("format_schema_path")) - context->setFormatSchemaPath(Poco::Path(config().getString("format_schema_path")).toString()); + context->setFormatSchemaPath(fs::weakly_canonical(config().getString("format_schema_path"))); /// Initialize query_id_formats if any if (config().has("query_id_formats")) @@ -633,8 +635,8 @@ private: history_file = home_path + "/.clickhouse-client-history"; } - if (!history_file.empty() && !Poco::File(history_file).exists()) - Poco::File(history_file).createFile(); + if (!history_file.empty() && !fs::exists(history_file)) + FS::createFile(history_file); LineReader::Patterns query_extenders = {"\\"}; LineReader::Patterns query_delimiters = {";", "\\G"}; diff --git a/programs/copier/ClusterCopier.h b/programs/copier/ClusterCopier.h index e875ca7df2e..085fa2ece06 100644 --- a/programs/copier/ClusterCopier.h +++ b/programs/copier/ClusterCopier.h @@ -12,14 +12,14 @@ namespace DB { -class ClusterCopier : WithContext +class ClusterCopier : WithMutableContext { public: ClusterCopier(const String & task_path_, const String & host_id_, const String & proxy_database_name_, - ContextPtr context_) - : WithContext(context_), + ContextMutablePtr context_) + : WithMutableContext(context_), task_zookeeper_path(task_path_), host_id(host_id_), working_database_name(proxy_database_name_), diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index d3fff616b65..8925ab63f99 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -5,7 +5,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -26,7 +28,7 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) copy_fault_probability = std::max(std::min(config().getDouble("copy-fault-probability"), 1.0), 0.0); if (config().has("move-fault-probability")) move_fault_probability = std::max(std::min(config().getDouble("move-fault-probability"), 1.0), 0.0); - base_dir = (config().has("base-dir")) ? config().getString("base-dir") : Poco::Path::current(); + base_dir = (config().has("base-dir")) ? config().getString("base-dir") : fs::current_path().string(); if (config().has("experimental-use-sample-offset")) @@ -38,18 +40,18 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) process_id = std::to_string(DateLUT::instance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid); host_id = escapeForFileName(getFQDNOrHostName()) + '#' + process_id; - process_path = Poco::Path(base_dir + "/clickhouse-copier_" + process_id).absolute().toString(); - Poco::File(process_path).createDirectories(); + process_path = fs::weakly_canonical(fs::path(base_dir) / ("clickhouse-copier_" + process_id)); + fs::create_directories(process_path); /// Override variables for BaseDaemon if (config().has("log-level")) config().setString("logger.level", config().getString("log-level")); if (config().has("base-dir") || !config().has("logger.log")) - config().setString("logger.log", process_path + "/log.log"); + config().setString("logger.log", fs::path(process_path) / "log.log"); if (config().has("base-dir") || !config().has("logger.errorlog")) - config().setString("logger.errorlog", process_path + "/log.err.log"); + config().setString("logger.errorlog", fs::path(process_path) / "log.err.log"); Base::initialize(self); } diff --git a/programs/copier/Internals.h b/programs/copier/Internals.h index 7e45c0ea2ee..9e40d7ebd7b 100644 --- a/programs/copier/Internals.h +++ b/programs/copier/Internals.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 8c187978106..a7f566a78b8 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -288,7 +288,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv) bool is_symlink = fs::is_symlink(symlink_path); fs::path points_to; if (is_symlink) - points_to = fs::absolute(fs::read_symlink(symlink_path)); + points_to = fs::weakly_canonical(fs::read_symlink(symlink_path)); if (is_symlink && points_to == main_bin_path) { diff --git a/programs/keeper/Keeper.h b/programs/keeper/Keeper.h index e80fe10b61c..f5b97dacf7d 100644 --- a/programs/keeper/Keeper.h +++ b/programs/keeper/Keeper.h @@ -32,7 +32,7 @@ public: return BaseDaemon::logger(); } - ContextPtr context() const override + ContextMutablePtr context() const override { return global_context; } @@ -58,7 +58,7 @@ protected: std::string getDefaultConfigFileName() const override; private: - ContextPtr global_context; + ContextMutablePtr global_context; Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) const; diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 043cc596e2b..f48e8d4d0a0 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -42,9 +42,9 @@ #include #include #include - #include +namespace fs = std::filesystem; namespace DB { @@ -72,11 +72,11 @@ void LocalServer::initialize(Poco::Util::Application & self) Poco::Util::Application::initialize(self); /// Load config files if exists - if (config().has("config-file") || Poco::File("config.xml").exists()) + if (config().has("config-file") || fs::exists("config.xml")) { const auto config_path = config().getString("config-file", "config.xml"); ConfigProcessor config_processor(config_path, false, true); - config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); + config_processor.setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", ".")); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); @@ -100,7 +100,7 @@ void LocalServer::initialize(Poco::Util::Application & self) } } -void LocalServer::applyCmdSettings(ContextPtr context) +void LocalServer::applyCmdSettings(ContextMutablePtr context) { context->applySettingsChanges(cmd_settings.changes()); } @@ -287,8 +287,8 @@ try status.emplace(path + "status", StatusFile::write_full_info); LOG_DEBUG(log, "Loading metadata from {}", path); - Poco::File(path + "data/").createDirectories(); - Poco::File(path + "metadata/").createDirectories(); + fs::create_directories(fs::path(path) / "data/"); + fs::create_directories(fs::path(path) / "metadata/"); loadMetadataSystem(global_context); attachSystemTables(global_context); loadMetadata(global_context); @@ -479,7 +479,7 @@ void LocalServer::setupUsers() { ConfigurationPtr users_config; - if (config().has("users_config") || config().has("config-file") || Poco::File("config.xml").exists()) + if (config().has("users_config") || config().has("config-file") || fs::exists("config.xml")) { const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml")); ConfigProcessor config_processor(users_config_path); @@ -645,7 +645,7 @@ void LocalServer::init(int argc, char ** argv) argsToConfig(arguments, config(), 100); } -void LocalServer::applyCmdOptions(ContextPtr context) +void LocalServer::applyCmdOptions(ContextMutablePtr context) { context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV"))); applyCmdSettings(context); diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index c5e9d5716dd..cf8886d9652 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -36,8 +36,8 @@ private: std::string getInitialCreateTableQuery(); void tryInitPath(); - void applyCmdOptions(ContextPtr context); - void applyCmdSettings(ContextPtr context); + void applyCmdOptions(ContextMutablePtr context); + void applyCmdSettings(ContextMutablePtr context); void processQueries(); void setupUsers(); void cleanup(); @@ -45,7 +45,7 @@ private: protected: SharedContextHolder shared_context; - ContextPtr global_context; + ContextMutablePtr global_context; /// Settings specified via command line args Settings cmd_settings; diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index fb6817fbf80..f68b255158c 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -1133,7 +1133,7 @@ try } SharedContextHolder shared_context = Context::createShared(); - ContextPtr context = Context::createGlobal(shared_context.get()); + auto context = Context::createGlobal(shared_context.get()); context->makeGlobalContext(); ReadBufferFromFileDescriptor file_in(STDIN_FILENO); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 6b3136dc200..f169d793ee9 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -74,6 +74,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) @@ -117,6 +118,8 @@ namespace CurrentMetrics extern const Metric MaxDDLEntryID; } +namespace fs = std::filesystem; + #if USE_JEMALLOC static bool jemallocOptionEnabled(const char *name) { @@ -183,19 +186,19 @@ void setupTmpPath(Poco::Logger * log, const std::string & path) { LOG_DEBUG(log, "Setting up {} to store temporary data in it", path); - Poco::File(path).createDirectories(); + fs::create_directories(path); /// Clearing old temporary files. - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(path); it != dir_end; ++it) + fs::directory_iterator dir_end; + for (fs::directory_iterator it(path); it != dir_end; ++it) { - if (it->isFile() && startsWith(it.name(), "tmp")) + if (it->is_regular_file() && startsWith(it->path().filename(), "tmp")) { - LOG_DEBUG(log, "Removing old temporary file {}", it->path()); - it->remove(); + LOG_DEBUG(log, "Removing old temporary file {}", it->path().string()); + fs::remove(it->path()); } else - LOG_DEBUG(log, "Skipped file in temporary path {}", it->path()); + LOG_DEBUG(log, "Skipped file in temporary path {}", it->path().string()); } } @@ -678,37 +681,38 @@ int Server::main(const std::vector & /*args*/) * Examples: do repair of local data; clone all replicated tables from replica. */ { - Poco::File(path + "flags/").createDirectories(); - global_context->setFlagsPath(path + "flags/"); + auto flags_path = fs::path(path) / "flags/"; + fs::create_directories(flags_path); + global_context->setFlagsPath(flags_path); } /** Directory with user provided files that are usable by 'file' table function. */ { - std::string user_files_path = config().getString("user_files_path", path + "user_files/"); + std::string user_files_path = config().getString("user_files_path", fs::path(path) / "user_files/"); global_context->setUserFilesPath(user_files_path); - Poco::File(user_files_path).createDirectories(); + fs::create_directories(user_files_path); } { - std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", path + "dictionaries_lib/"); + std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", fs::path(path) / "dictionaries_lib/"); global_context->setDictionariesLibPath(dictionaries_lib_path); - Poco::File(dictionaries_lib_path).createDirectories(); + fs::create_directories(dictionaries_lib_path); } /// top_level_domains_lists { - const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/") + "/"; - TLDListsHolder::getInstance().parseConfig(top_level_domains_path, config()); + const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/"); + TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config()); } { - Poco::File(path + "data/").createDirectories(); - Poco::File(path + "metadata/").createDirectories(); + fs::create_directories(fs::path(path) / "data/"); + fs::create_directories(fs::path(path) / "metadata/"); /// Directory with metadata of tables, which was marked as dropped by Atomic database - Poco::File(path + "metadata_dropped/").createDirectories(); + fs::create_directories(fs::path(path) / "metadata_dropped/"); } if (config().has("interserver_http_port") && config().has("interserver_https_port")) @@ -891,9 +895,9 @@ int Server::main(const std::vector & /*args*/) #endif /// Set path for format schema files - auto format_schema_path = Poco::File(config().getString("format_schema_path", path + "format_schemas/")); - global_context->setFormatSchemaPath(format_schema_path.path()); - format_schema_path.createDirectories(); + fs::path format_schema_path(config().getString("format_schema_path", fs::path(path) / "format_schemas/")); + global_context->setFormatSchemaPath(format_schema_path); + fs::create_directories(format_schema_path); /// Check sanity of MergeTreeSettings on server startup global_context->getMergeTreeSettings().sanityCheck(settings); diff --git a/programs/server/Server.h b/programs/server/Server.h index c698108767c..45e5fccd51d 100644 --- a/programs/server/Server.h +++ b/programs/server/Server.h @@ -40,7 +40,7 @@ public: return BaseDaemon::logger(); } - ContextPtr context() const override + ContextMutablePtr context() const override { return global_context; } @@ -64,7 +64,7 @@ protected: std::string getDefaultCorePath() const override; private: - ContextPtr global_context; + ContextMutablePtr global_context; Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) const; using CreateServerFunc = std::function; diff --git a/src/Access/AllowedClientHosts.h b/src/Access/AllowedClientHosts.h index a6895b120e0..7b21fd7e236 100644 --- a/src/Access/AllowedClientHosts.h +++ b/src/Access/AllowedClientHosts.h @@ -7,7 +7,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -198,9 +200,9 @@ inline String AllowedClientHosts::IPSubnet::toString() const if (isMaskAllBitsOne()) return prefix.toString(); else if (IPAddress{prefix_length, mask.family()} == mask) - return prefix.toString() + "/" + std::to_string(prefix_length); + return fs::path(prefix.toString()) / std::to_string(prefix_length); else - return prefix.toString() + "/" + mask.toString(); + return fs::path(prefix.toString()) / mask.toString(); } inline bool AllowedClientHosts::IPSubnet::isMaskAllBitsOne() const diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index 92439342168..512ccff1d1b 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -51,7 +51,7 @@ private: void setAllNoLock(const std::vector> & all_entities, Notifications & notifications); void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const; - mutable std::mutex mutex; + mutable std::recursive_mutex mutex; std::unordered_map entries_by_id; /// We want to search entries both by ID and by the pair of name and type. std::unordered_map entries_by_name_and_type[static_cast(EntityType::MAX)]; mutable std::list handlers_by_type[static_cast(EntityType::MAX)]; diff --git a/src/Bridge/IBridgeHelper.cpp b/src/Bridge/IBridgeHelper.cpp index 16886bae4fe..b6f3446d0a6 100644 --- a/src/Bridge/IBridgeHelper.cpp +++ b/src/Bridge/IBridgeHelper.cpp @@ -3,9 +3,10 @@ #include #include #include -#include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -87,10 +88,10 @@ std::unique_ptr IBridgeHelper::startBridgeCommand() const const auto & config = getConfig(); /// Path to executable folder - Poco::Path path{config.getString("application.dir", "/usr/bin")}; + fs::path path(config.getString("application.dir", "/usr/bin")); std::vector cmd_args; - path.setFileName(serviceFileName()); + path /= serviceFileName(); cmd_args.push_back("--http-port"); cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", getDefaultPort()))); @@ -126,7 +127,7 @@ std::unique_ptr IBridgeHelper::startBridgeCommand() const LOG_TRACE(getLog(), "Starting {}", serviceAlias()); - return ShellCommand::executeDirect(path.toString(), cmd_args, ShellCommandDestructorStrategy(true)); + return ShellCommand::executeDirect(path.string(), cmd_args, ShellCommandDestructorStrategy(true)); } } diff --git a/src/Bridge/LibraryBridgeHelper.cpp b/src/Bridge/LibraryBridgeHelper.cpp index 8234b5d755f..8967b191b73 100644 --- a/src/Bridge/LibraryBridgeHelper.cpp +++ b/src/Bridge/LibraryBridgeHelper.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Bridge/XDBCBridgeHelper.h b/src/Bridge/XDBCBridgeHelper.h index ee6a178037d..97a6251a9a0 100644 --- a/src/Bridge/XDBCBridgeHelper.h +++ b/src/Bridge/XDBCBridgeHelper.h @@ -5,10 +5,8 @@ #include #include #include -#include #include #include -#include #include #include #include diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index fa9e9b72087..7bbb4798d6f 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -62,7 +62,7 @@ static std::string numberFromHost(const std::string & s) bool ConfigProcessor::isPreprocessedFile(const std::string & path) { - return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX); + return endsWith(fs::path(path).stem(), PREPROCESSED_SUFFIX); } @@ -416,34 +416,32 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string & { Files files; - Poco::Path merge_dir_path(config_path); + fs::path merge_dir_path(config_path); std::set merge_dirs; /// Add path_to_config/config_name.d dir - merge_dir_path.setExtension("d"); - merge_dirs.insert(merge_dir_path.toString()); + merge_dir_path.replace_extension("d"); + merge_dirs.insert(merge_dir_path); /// Add path_to_config/conf.d dir - merge_dir_path.setBaseName("conf"); - merge_dirs.insert(merge_dir_path.toString()); + merge_dir_path.replace_filename("conf.d"); + merge_dirs.insert(merge_dir_path); for (const std::string & merge_dir_name : merge_dirs) { - Poco::File merge_dir(merge_dir_name); - if (!merge_dir.exists() || !merge_dir.isDirectory()) + if (!fs::exists(merge_dir_name) || !fs::is_directory(merge_dir_name)) continue; - for (Poco::DirectoryIterator it(merge_dir_name); it != Poco::DirectoryIterator(); ++it) + for (fs::directory_iterator it(merge_dir_name); it != fs::directory_iterator(); ++it) { - Poco::File & file = *it; - Poco::Path path(file.path()); - std::string extension = path.getExtension(); - std::string base_name = path.getBaseName(); + fs::path path(it->path()); + std::string extension = path.extension(); + std::string base_name = path.stem(); // Skip non-config and temporary files - if (file.isFile() && (extension == "xml" || extension == "conf" || extension == "yaml" || extension == "yml") && !startsWith(base_name, ".")) - { - files.push_back(file.path()); - } + if (fs::is_regular_file(path) + && (extension == ".xml" || extension == ".conf" || extension == ".yaml" || extension == ".yml") + && !startsWith(base_name, ".")) + files.push_back(it->path()); } } @@ -548,7 +546,7 @@ XMLDocumentPtr ConfigProcessor::processConfig( else { std::string default_path = "/etc/metrika.xml"; - if (Poco::File(default_path).exists()) + if (fs::exists(default_path)) include_from_path = default_path; } if (!include_from_path.empty()) @@ -660,11 +658,11 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, if (!loaded_config.configuration->has("path")) { // Will use current directory - auto parent_path = Poco::Path(loaded_config.config_path).makeParent(); - preprocessed_dir = parent_path.toString(); - Poco::Path poco_new_path(new_path); - poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX); - new_path = poco_new_path.toString(); + fs::path parent_path = fs::path(loaded_config.config_path).parent_path(); + preprocessed_dir = parent_path.string(); + fs::path fs_new_path(new_path); + fs_new_path.replace_filename(fs_new_path.stem().string() + PREPROCESSED_SUFFIX + fs_new_path.extension().string()); + new_path = fs_new_path.string(); } else { @@ -679,9 +677,9 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, } preprocessed_path = (fs::path(preprocessed_dir) / fs::path(new_path)).string(); - auto preprocessed_path_parent = Poco::Path(preprocessed_path).makeParent(); - if (!preprocessed_path_parent.toString().empty()) - Poco::File(preprocessed_path_parent).createDirectories(); + auto preprocessed_path_parent = fs::path(preprocessed_path).parent_path(); + if (!preprocessed_path_parent.empty()) + fs::create_directories(preprocessed_path_parent); } DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml); LOG_DEBUG(log, "Saved preprocessed configuration to '{}'.", preprocessed_path); diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h index 5b16bc0cb1b..e13e66547a3 100644 --- a/src/Common/Config/ConfigProcessor.h +++ b/src/Common/Config/ConfigProcessor.h @@ -15,12 +15,9 @@ #include #include #include -#include -#include #include #include #include - #include diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index afff08e82bb..01bb4f556d6 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -1,12 +1,15 @@ #include "ConfigReloader.h" #include -#include #include #include #include "ConfigProcessor.h" +#include +#include +namespace fs = std::filesystem; + namespace DB { @@ -167,8 +170,8 @@ struct ConfigReloader::FileWithTimestamp void ConfigReloader::FilesChangesTracker::addIfExists(const std::string & path_to_add) { - if (!path_to_add.empty() && Poco::File(path_to_add).exists()) - files.emplace(path_to_add, Poco::File(path_to_add).getLastModified().epochTime()); + if (!path_to_add.empty() && fs::exists(path_to_add)) + files.emplace(path_to_add, FS::getModificationTime(path_to_add)); } bool ConfigReloader::FilesChangesTracker::isDifferOrNewerThan(const FilesChangesTracker & rhs) diff --git a/src/Common/Config/configReadClient.cpp b/src/Common/Config/configReadClient.cpp index 61d8e507c05..8ce2a8b03e9 100644 --- a/src/Common/Config/configReadClient.cpp +++ b/src/Common/Config/configReadClient.cpp @@ -1,8 +1,10 @@ #include "configReadClient.h" #include -#include #include "ConfigProcessor.h" +#include + +namespace fs = std::filesystem; namespace DB { @@ -11,11 +13,11 @@ bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::stri std::string config_path; if (config.has("config-file")) config_path = config.getString("config-file"); - else if (Poco::File("./clickhouse-client.xml").exists()) + else if (fs::exists("./clickhouse-client.xml")) config_path = "./clickhouse-client.xml"; - else if (!home_path.empty() && Poco::File(home_path + "/.clickhouse-client/config.xml").exists()) + else if (!home_path.empty() && fs::exists(home_path + "/.clickhouse-client/config.xml")) config_path = home_path + "/.clickhouse-client/config.xml"; - else if (Poco::File("/etc/clickhouse-client/config.xml").exists()) + else if (fs::exists("/etc/clickhouse-client/config.xml")) config_path = "/etc/clickhouse-client/config.xml"; if (!config_path.empty()) diff --git a/src/Common/CounterInFile.h b/src/Common/CounterInFile.h index 8cd4534d413..6326261234d 100644 --- a/src/Common/CounterInFile.h +++ b/src/Common/CounterInFile.h @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -59,7 +58,7 @@ public: Int64 res = -1; - bool file_doesnt_exists = !Poco::File(path).exists(); + bool file_doesnt_exists = !fs::exists(path); if (file_doesnt_exists && !create_if_need) { throw Poco::Exception("File " + path + " does not exist. " @@ -138,7 +137,7 @@ public: // Not thread-safe and not synchronized between processes. void fixIfBroken(UInt64 value) { - bool file_exists = Poco::File(path).exists(); + bool file_exists = fs::exists(path); int fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); if (-1 == fd) diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index 069f9cf2af7..c6ac7e9933a 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -87,7 +87,7 @@ public: /// Initializes query with current thread as master thread in constructor, and detaches it in destructor struct QueryScope { - explicit QueryScope(ContextPtr query_context); + explicit QueryScope(ContextMutablePtr query_context); ~QueryScope(); void logPeakMemoryUsage(); diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 330a193212b..d840830bf28 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -553,6 +553,7 @@ M(583, ILLEGAL_PROJECTION) \ M(584, PROJECTION_NOT_USED) \ M(585, CANNOT_PARSE_YAML) \ + M(586, CANNOT_CREATE_FILE) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index ac1780a45a8..e98cd3c3046 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -21,6 +21,8 @@ # include #endif +namespace fs = std::filesystem; + namespace DB { @@ -177,7 +179,7 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_ tryLogCurrentExceptionImpl(logger, start_of_message); } -static void getNoSpaceLeftInfoMessage(std::filesystem::path path, std::string & msg) +static void getNoSpaceLeftInfoMessage(std::filesystem::path path, String & msg) { path = std::filesystem::absolute(path); /// It's possible to get ENOSPC for non existent file (e.g. if there are no free inodes and creat() fails) @@ -264,22 +266,12 @@ static std::string getExtraExceptionInfo(const std::exception & e) String msg; try { - if (const auto * file_exception = dynamic_cast(&e)) + if (const auto * file_exception = dynamic_cast(&e)) { - if (file_exception->code() == ENOSPC) - { - /// See Poco::FileImpl::handleLastErrorImpl(...) - constexpr const char * expected_error_message = "no space left on device: "; - if (startsWith(file_exception->message(), expected_error_message)) - { - String path = file_exception->message().substr(strlen(expected_error_message)); - getNoSpaceLeftInfoMessage(path, msg); - } - else - { - msg += "\nCannot print extra info for Poco::Exception"; - } - } + if (file_exception->code() == std::errc::no_space_on_device) + getNoSpaceLeftInfoMessage(file_exception->path1(), msg); + else + msg += "\nCannot print extra info for Poco::Exception"; } else if (const auto * errno_exception = dynamic_cast(&e)) { diff --git a/src/Common/FileUpdatesTracker.h b/src/Common/FileUpdatesTracker.h index 0914c75693c..9fe37c2a75d 100644 --- a/src/Common/FileUpdatesTracker.h +++ b/src/Common/FileUpdatesTracker.h @@ -1,10 +1,11 @@ #pragma once -#include #include - #include +#include +#include +namespace fs = std::filesystem; class FileUpdatesTracker { @@ -31,6 +32,6 @@ public: private: Poco::Timestamp getLastModificationTime() const { - return Poco::File(path).getLastModified(); + return FS::getModificationTimestamp(path); } }; diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index b21454c9ed8..ceedf518608 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -14,7 +13,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -45,7 +46,7 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_) : path(std::move(path_)), fill(std::move(fill_)) { /// If file already exists. NOTE Minor race condition. - if (Poco::File(path).exists()) + if (fs::exists(path)) { std::string contents; { diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 43092f719ec..f3aecd4e76b 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -4,6 +4,7 @@ #include "TestKeeper.h" #include +#include #include #include @@ -17,6 +18,7 @@ #define ZOOKEEPER_CONNECTION_TIMEOUT_MS 1000 +namespace fs = std::filesystem; namespace DB { @@ -612,7 +614,7 @@ void ZooKeeper::removeChildren(const std::string & path) Coordination::Requests ops; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1)); + ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); children.pop_back(); } multi(ops); @@ -628,9 +630,9 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path, const String & Coordination::Requests ops; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - removeChildrenRecursive(path + "/" + children.back()); + removeChildrenRecursive(fs::path(path) / children.back()); if (likely(keep_child_node.empty() || keep_child_node != children.back())) - ops.emplace_back(makeRemoveRequest(path + "/" + children.back(), -1)); + ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); children.pop_back(); } multi(ops); @@ -648,7 +650,7 @@ void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, const Strin Strings batch; for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) { - String child_path = path + "/" + children.back(); + String child_path = fs::path(path) / children.back(); tryRemoveChildrenRecursive(child_path); if (likely(keep_child_node.empty() || keep_child_node != children.back())) { diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 6f0e3155cbe..4855500b776 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -6,10 +6,15 @@ # include #endif #include -#include -#include #include +#include +#include +#include +#include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -20,6 +25,8 @@ namespace ErrorCodes extern const int SYSTEM_ERROR; extern const int NOT_IMPLEMENTED; extern const int CANNOT_STATVFS; + extern const int PATH_ACCESS_DENIED; + extern const int CANNOT_CREATE_FILE; } @@ -38,17 +45,13 @@ struct statvfs getStatVFS(const String & path) bool enoughSpaceInDirectory(const std::string & path [[maybe_unused]], size_t data_size [[maybe_unused]]) { -#if POCO_VERSION >= 0x01090000 - auto free_space = Poco::File(path).freeSpace(); + auto free_space = fs::space(path).free; return data_size <= free_space; -#else - return true; -#endif } std::unique_ptr createTemporaryFile(const std::string & path) { - Poco::File(path).createDirectories(); + fs::create_directories(path); /// NOTE: std::make_shared cannot use protected constructors return std::make_unique(path); @@ -128,3 +131,73 @@ bool pathStartsWith(const String & path, const String & prefix_path) } } + + +/// Copied from Poco::File +namespace FS +{ + +bool createFile(const std::string & path) +{ + int n = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + if (n != -1) + { + close(n); + return true; + } + DB::throwFromErrnoWithPath("Cannot create file: " + path, path, DB::ErrorCodes::CANNOT_CREATE_FILE); +} + +bool canRead(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + { + if (st.st_uid == geteuid()) + return (st.st_mode & S_IRUSR) != 0; + else if (st.st_gid == getegid()) + return (st.st_mode & S_IRGRP) != 0; + else + return (st.st_mode & S_IROTH) != 0 || geteuid() == 0; + } + DB::throwFromErrnoWithPath("Cannot check read access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + + +bool canWrite(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + { + if (st.st_uid == geteuid()) + return (st.st_mode & S_IWUSR) != 0; + else if (st.st_gid == getegid()) + return (st.st_mode & S_IWGRP) != 0; + else + return (st.st_mode & S_IWOTH) != 0 || geteuid() == 0; + } + DB::throwFromErrnoWithPath("Cannot check write access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + +time_t getModificationTime(const std::string & path) +{ + struct stat st; + if (stat(path.c_str(), &st) == 0) + return st.st_mtime; + DB::throwFromErrnoWithPath("Cannot check modification time for file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} + +Poco::Timestamp getModificationTimestamp(const std::string & path) +{ + return Poco::Timestamp::fromEpochTime(getModificationTime(path)); +} + +void setModificationTime(const std::string & path, time_t time) +{ + struct utimbuf tb; + tb.actime = time; + tb.modtime = time; + if (utime(path.c_str(), &tb) != 0) + DB::throwFromErrnoWithPath("Cannot set modification time for file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED); +} +} diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index 0ffbf19643d..b7525a64fae 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -36,3 +36,15 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p bool pathStartsWith(const String & path, const String & prefix_path); } + +namespace FS +{ +bool createFile(const std::string & path); + +bool canRead(const std::string & path); +bool canWrite(const std::string & path); + +time_t getModificationTime(const std::string & path); +Poco::Timestamp getModificationTimestamp(const std::string & path); +void setModificationTime(const std::string & path, time_t time); +} diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index a735a9d72d4..26d90427889 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #if defined(linux) || defined(__linux) || defined(__linux__) #include @@ -10,6 +10,8 @@ #include #endif +namespace fs = std::filesystem; + namespace DB { @@ -93,9 +95,9 @@ static bool renameat2(const std::string &, const std::string &, int) static void renameNoReplaceFallback(const std::string & old_path, const std::string & new_path) { /// NOTE it's unsafe - if (Poco::File{new_path}.exists()) + if (fs::exists(new_path)) throw Exception("File " + new_path + " exists", ErrorCodes::FILE_ALREADY_EXISTS); - Poco::File{old_path}.renameTo(new_path); + fs::rename(old_path, new_path); } /// Do not use [[noreturn]] to avoid warnings like "code will never be executed" in other places diff --git a/src/Common/tests/gtest_global_context.h b/src/Common/tests/gtest_global_context.h index 30ebf1dbca9..9bd7c2490d6 100644 --- a/src/Common/tests/gtest_global_context.h +++ b/src/Common/tests/gtest_global_context.h @@ -5,7 +5,7 @@ struct ContextHolder { DB::SharedContextHolder shared_context; - DB::ContextPtr context; + DB::ContextMutablePtr context; ContextHolder() : shared_context(DB::Context::createShared()) diff --git a/src/Core/ExternalTable.h b/src/Core/ExternalTable.h index fcefa3d7fe3..f682bf9b27f 100644 --- a/src/Core/ExternalTable.h +++ b/src/Core/ExternalTable.h @@ -80,10 +80,10 @@ public: /// Parsing of external table used when sending tables via http /// The `handlePart` function will be called for each table passed, /// so it's also necessary to call `clean` at the end of the `handlePart`. -class ExternalTablesHandler : public HTMLForm::PartHandler, BaseExternalTable, WithContext +class ExternalTablesHandler : public HTMLForm::PartHandler, BaseExternalTable, WithMutableContext { public: - ExternalTablesHandler(ContextPtr context_, const Poco::Net::NameValueCollection & params_) : WithContext(context_), params(params_) {} + ExternalTablesHandler(ContextMutablePtr context_, const Poco::Net::NameValueCollection & params_) : WithMutableContext(context_), params(params_) {} void handlePart(const Poco::Net::MessageHeader & header, ReadBuffer & stream) override; diff --git a/src/Core/MySQL/Authentication.cpp b/src/Core/MySQL/Authentication.cpp index b8dff9972c7..0258f9f94af 100644 --- a/src/Core/MySQL/Authentication.cpp +++ b/src/Core/MySQL/Authentication.cpp @@ -73,7 +73,7 @@ Native41::Native41(const String & password, const String & auth_plugin_data) } void Native41::authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool, const Poco::Net::SocketAddress & address) { if (!auth_response) @@ -136,7 +136,7 @@ Sha256Password::Sha256Password(RSA & public_key_, RSA & private_key_, Poco::Logg } void Sha256Password::authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) { if (!auth_response) diff --git a/src/Core/MySQL/Authentication.h b/src/Core/MySQL/Authentication.h index 5358e2da737..acbda2bdb58 100644 --- a/src/Core/MySQL/Authentication.h +++ b/src/Core/MySQL/Authentication.h @@ -32,7 +32,7 @@ public: virtual String getAuthPluginData() = 0; virtual void authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) = 0; }; @@ -49,7 +49,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool /* is_secure_connection */, const Poco::Net::SocketAddress & address) override; private: @@ -69,7 +69,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, ContextPtr context, + const String & user_name, std::optional auth_response, ContextMutablePtr context, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) override; private: diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 9e1afcd187c..7c5800f5a8f 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -802,7 +802,7 @@ protected: static void setPassword( const String & user_name, const String & password, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { @@ -821,7 +821,7 @@ protected: public: virtual void authenticate( const String & user_name, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) = 0; @@ -835,7 +835,7 @@ class NoPasswordAuth : public AuthenticationMethod public: void authenticate( const String & user_name, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { @@ -853,7 +853,7 @@ class CleartextPasswordAuth : public AuthenticationMethod public: void authenticate( const String & user_name, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { @@ -896,7 +896,7 @@ public: void authenticate( const String & user_name, - ContextPtr context, + ContextMutablePtr context, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 321957e2c90..bc308b56599 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -415,7 +415,6 @@ class IColumn; M(Bool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \ M(String, function_implementation, "", "Choose function implementation for specific target or variant (experimental). If empty enable all of them.", 0) \ M(Bool, allow_experimental_geo_types, false, "Allow geo data types such as Point, Ring, Polygon, MultiPolygon", 0) \ - M(Bool, allow_experimental_bigint_types, false, "Allow Int128, Int256, UInt256 and Decimal256 types", 0) \ M(Bool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ M(Bool, cast_keep_nullable, false, "CAST operator keep Nullable for result data type", 0) \ M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ @@ -458,13 +457,6 @@ class IColumn; M(UInt64, distributed_ddl_entry_format_version, 1, "Version of DDL entry to write into ZooKeeper", 0) \ M(UInt64, external_storage_max_read_rows, 0, "Limit maximum number of rows when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializeMySQL. If equal to 0, this setting is disabled", 0) \ M(UInt64, external_storage_max_read_bytes, 0, "Limit maximum number of bytes when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializeMySQL. If equal to 0, this setting is disabled", 0) \ - \ - /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ - \ - M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ - M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing. Will be removed after 2021-03-31", 0) \ - M(Bool, enable_debug_queries, false, "Enabled debug queries, but now is obsolete", 0) \ - M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing. Will be removed after 2021-02-12", 0) \ M(UnionMode, union_default_mode, UnionMode::Unspecified, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) \ M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ @@ -473,11 +465,20 @@ class IColumn; M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \ M(Bool, query_plan_filter_push_down, true, "Allow to push down filter by predicate query plan step", 0) \ \ - M(Bool, database_replicated_ddl_output, true, "Obsolete setting, does nothing. Will be removed after 2021-09-08", 0) \ - M(HandleKafkaErrorMode, handle_kafka_error_mode, HandleKafkaErrorMode::DEFAULT, "How to handle errors for Kafka engine. Passible values: default, stream.", 0) \ M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ - M(Bool, allow_experimental_funnel_functions, true, "Enable experimental functions for funnel analysis.", 0) \ + \ + \ + /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ + M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete setting, does nothing.", 0) \ + M(UInt64, multiple_joins_rewriter_version, 0, "Obsolete setting, does nothing.", 0) \ + M(Bool, enable_debug_queries, false, "Obsolete setting, does nothing.", 0) \ + M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing.", 0) \ + M(Bool, allow_experimental_funnel_functions, true, "Obsolete setting, does nothing.", 0) \ + M(Bool, allow_experimental_bigint_types, true, "Obsolete setting, does nothing.", 0) \ + M(HandleKafkaErrorMode, handle_kafka_error_mode, HandleKafkaErrorMode::DEFAULT, "Obsolete setting, does nothing.", 0) \ + M(Bool, database_replicated_ddl_output, true, "Obsolete setting, does nothing.", 0) \ + /** The section above is for obsolete settings. Do not add anything there. */ // End of COMMON_SETTINGS diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.h b/src/DataStreams/PushingToViewsBlockOutputStream.h index 2ae941efc2e..552a0a3452a 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -54,8 +54,8 @@ private: }; std::vector views; - ContextPtr select_context; - ContextPtr insert_context; + ContextMutablePtr select_context; + ContextMutablePtr insert_context; void process(const Block & block, ViewInfo & view); }; diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 85526cd98de..e0f5d3302df 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -341,8 +341,6 @@ struct WhichDataType constexpr bool isNullable() const { return idx == TypeIndex::Nullable; } constexpr bool isFunction() const { return idx == TypeIndex::Function; } constexpr bool isAggregateFunction() const { return idx == TypeIndex::AggregateFunction; } - - constexpr bool IsBigIntOrDeimal() const { return isInt128() || isUInt128() || isInt256() || isUInt256() || isDecimal256(); } }; /// IDataType helpers (alternative for IDataType virtual methods with single point of truth) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index e939f76f372..6b8c470861d 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -1,7 +1,5 @@ #include #include -#include -#include #include #include #include @@ -13,6 +11,8 @@ #include #include +namespace fs = std::filesystem; + namespace DB { namespace ErrorCodes @@ -37,12 +37,12 @@ public: DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, const String & logger_name, ContextPtr context_) : DatabaseOrdinary(name_, std::move(metadata_path_), "store/", logger_name, context_) - , path_to_table_symlinks(getContext()->getPath() + "data/" + escapeForFileName(name_) + "/") - , path_to_metadata_symlink(getContext()->getPath() + "metadata/" + escapeForFileName(name_)) + , path_to_table_symlinks(fs::path(getContext()->getPath()) / "data" / escapeForFileName(name_) / "") + , path_to_metadata_symlink(fs::path(getContext()->getPath()) / "metadata" / escapeForFileName(name_)) , db_uuid(uuid) { assert(db_uuid != UUIDHelpers::Nil); - Poco::File(path_to_table_symlinks).createDirectories(); + fs::create_directories(path_to_table_symlinks); tryCreateMetadataSymlink(); } @@ -73,14 +73,14 @@ void DatabaseAtomic::drop(ContextPtr) assert(tables.empty()); try { - Poco::File(path_to_metadata_symlink).remove(); - Poco::File(path_to_table_symlinks).remove(true); + fs::remove(path_to_metadata_symlink); + fs::remove_all(path_to_table_symlinks); } catch (...) { LOG_WARNING(log, getCurrentExceptionMessage(true)); } - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } void DatabaseAtomic::attachTable(const String & name, const StoragePtr & table, const String & relative_table_path) @@ -132,8 +132,8 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na /// (it's more likely to lost connection, than to fail before applying local changes). /// TODO better detection and recovery - Poco::File(table_metadata_path).renameTo(table_metadata_path_drop); /// Mark table as dropped - DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw + fs::rename(table_metadata_path, table_metadata_path_drop); /// Mark table as dropped + DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw table_name_to_path.erase(table_name); } @@ -316,7 +316,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); if (locked_uuid) DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; @@ -416,11 +416,11 @@ UUID DatabaseAtomic::tryGetTableUUID(const String & table_name) const return UUIDHelpers::Nil; } -void DatabaseAtomic::loadStoredObjects(ContextPtr local_context, bool has_force_restore_data_flag, bool force_attach) +void DatabaseAtomic::loadStoredObjects(ContextMutablePtr local_context, bool has_force_restore_data_flag, bool force_attach) { /// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken if (has_force_restore_data_flag) - Poco::File(path_to_table_symlinks).remove(true); + fs::remove_all(path_to_table_symlinks); DatabaseOrdinary::loadStoredObjects(local_context, has_force_restore_data_flag, force_attach); @@ -432,7 +432,7 @@ void DatabaseAtomic::loadStoredObjects(ContextPtr local_context, bool has_force_ table_names = table_name_to_path; } - Poco::File(path_to_table_symlinks).createDirectories(); + fs::create_directories(path_to_table_symlinks); for (const auto & table : table_names) tryCreateSymlink(table.first, table.second, true); } @@ -443,9 +443,9 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String & try { String link = path_to_table_symlinks + escapeForFileName(table_name); - Poco::File data = Poco::Path(getContext()->getPath()).makeAbsolute().toString() + actual_data_path; - if (!if_data_path_exist || data.exists()) - data.linkTo(link, Poco::File::LINK_SYMBOLIC); + fs::path data = fs::canonical(getContext()->getPath()) / actual_data_path; + if (!if_data_path_exist || fs::exists(data)) + fs::create_directory_symlink(data, link); } catch (...) { @@ -458,7 +458,7 @@ void DatabaseAtomic::tryRemoveSymlink(const String & table_name) try { String path = path_to_table_symlinks + escapeForFileName(table_name); - Poco::File{path}.remove(); + fs::remove(path); } catch (...) { @@ -471,17 +471,17 @@ void DatabaseAtomic::tryCreateMetadataSymlink() /// Symlinks in data/db_name/ directory and metadata/db_name/ are not used by ClickHouse, /// it's needed only for convenient introspection. assert(path_to_metadata_symlink != metadata_path); - Poco::File metadata_symlink(path_to_metadata_symlink); - if (metadata_symlink.exists()) + fs::path metadata_symlink(path_to_metadata_symlink); + if (fs::exists(metadata_symlink)) { - if (!metadata_symlink.isLink()) + if (!fs::is_symlink(metadata_symlink)) throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Directory {} exists", path_to_metadata_symlink); } else { try { - Poco::File{metadata_path}.linkTo(path_to_metadata_symlink, Poco::File::LINK_SYMBOLIC); + fs::create_directory_symlink(metadata_path, path_to_metadata_symlink); } catch (...) { @@ -495,7 +495,7 @@ void DatabaseAtomic::renameDatabase(const String & new_name) /// CREATE, ATTACH, DROP, DETACH and RENAME DATABASE must hold DDLGuard try { - Poco::File(path_to_metadata_symlink).remove(); + fs::remove(path_to_metadata_symlink); } catch (...) { @@ -526,7 +526,7 @@ void DatabaseAtomic::renameDatabase(const String & new_name) path_to_table_symlinks = getContext()->getPath() + "data/" + new_name_escaped + "/"; } - Poco::File(old_path_to_table_symlinks).renameTo(path_to_table_symlinks); + fs::rename(old_path_to_table_symlinks, path_to_table_symlinks); tryCreateMetadataSymlink(); } diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h index a6acd10b656..c5dedf00d23 100644 --- a/src/Databases/DatabaseAtomic.h +++ b/src/Databases/DatabaseAtomic.h @@ -47,7 +47,7 @@ public: DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) override; - void loadStoredObjects(ContextPtr context, bool has_force_restore_data_flag, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr context, bool has_force_restore_data_flag, bool force_attach) override; /// Atomic database cannot be detached if there is detached table which still in use void assertCanBeDetached(bool cleanup) override; diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 4626d4f47b6..e09860e79b6 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -11,10 +11,9 @@ #include #include #include -#include -#include #include #include +#include #if !defined(ARCADIA_BUILD) # include "config_core.h" @@ -40,6 +39,8 @@ #include #endif +namespace fs = std::filesystem; + namespace DB { @@ -58,11 +59,12 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m try { /// Creates store/xxx/ for Atomic - Poco::File(Poco::Path(metadata_path).makeParent()).createDirectories(); + fs::create_directories(fs::path(metadata_path).parent_path()); + /// Before 20.7 it's possible that .sql metadata file does not exist for some old database. /// In this case Ordinary database is created on server startup if the corresponding metadata directory exists. /// So we should remove metadata directory if database creation failed. - created = Poco::File(metadata_path).createDirectory(); + created = fs::create_directory(metadata_path); DatabasePtr impl = getImpl(create, metadata_path, context); @@ -74,11 +76,8 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m } catch (...) { - Poco::File metadata_dir(metadata_path); - - if (created && metadata_dir.exists()) - metadata_dir.remove(true); - + if (created && fs::exists(metadata_path)) + fs::remove_all(metadata_path); throw; } } diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 80179de104a..22afaa971a1 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -12,8 +12,9 @@ #include #include #include -#include +#include +namespace fs = std::filesystem; namespace DB { @@ -35,7 +36,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, void DatabaseLazy::loadStoredObjects( - ContextPtr local_context, + ContextMutablePtr local_context, bool /* has_force_restore_data_flag */, bool /*force_attach*/) { @@ -43,8 +44,8 @@ void DatabaseLazy::loadStoredObjects( { const std::string table_name = file_name.substr(0, file_name.size() - 4); - auto detached_permanently_flag = Poco::File(getMetadataPath() + "/" + file_name + detached_suffix); - if (detached_permanently_flag.exists()) + fs::path detached_permanently_flag = fs::path(getMetadataPath()) / (file_name + detached_suffix); + if (fs::exists(detached_permanently_flag)) { LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); return; @@ -228,7 +229,7 @@ StoragePtr DatabaseLazy::loadTable(const String & table_name) const LOG_DEBUG(log, "Load table {} to cache.", backQuote(table_name)); - const String table_metadata_path = getMetadataPath() + "/" + escapeForFileName(table_name) + ".sql"; + const String table_metadata_path = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + ".sql"); try { diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 99a71b342fa..949869f4509 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -27,7 +27,7 @@ public: bool canContainDistributedTables() const override { return false; } void loadStoredObjects( - ContextPtr context, + ContextMutablePtr context, bool has_force_restore_data_flag, bool force_attach) override; void createTable( diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index f21a145df55..c0af027e027 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -4,9 +4,9 @@ #include #include #include -#include #include +namespace fs = std::filesystem; namespace DB { @@ -42,9 +42,9 @@ void DatabaseMemory::dropTable( try { table->drop(); - Poco::File table_data_dir{getTableDataPath(table_name)}; - if (table_data_dir.exists()) - table_data_dir.remove(true); + fs::path table_data_dir{getTableDataPath(table_name)}; + if (fs::exists(table_data_dir)) + fs::remove_all(table_data_dir); } catch (...) { diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 3db5314bbd3..620e560b64c 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -14,14 +14,14 @@ #include #include #include - #include -#include - #include #include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -45,7 +45,7 @@ std::pair createTableFromAST( ASTCreateQuery ast_create_query, const String & database_name, const String & table_data_path_relative, - ContextPtr context, + ContextMutablePtr context, bool has_force_restore_data_flag) { ast_create_query.attach = true; @@ -201,8 +201,8 @@ DatabaseOnDisk::DatabaseOnDisk( , metadata_path(metadata_path_) , data_path(data_path_) { - Poco::File(local_context->getPath() + data_path).createDirectories(); - Poco::File(metadata_path).createDirectories(); + fs::create_directories(local_context->getPath() + data_path); + fs::create_directories(metadata_path); } @@ -245,7 +245,7 @@ void DatabaseOnDisk::createTable( if (!create.attach) checkMetadataFilenameAvailability(table_name); - if (create.attach && Poco::File(table_metadata_path).exists()) + if (create.attach && fs::exists(table_metadata_path)) { ASTPtr ast_detached = parseQueryFromMetadata(log, local_context, table_metadata_path); auto & create_detached = ast_detached->as(); @@ -285,10 +285,10 @@ void DatabaseOnDisk::removeDetachedPermanentlyFlag(ContextPtr, const String & ta { try { - auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix); + fs::path detached_permanently_flag(table_metadata_path + detached_suffix); - if (detached_permanently_flag.exists()) - detached_permanently_flag.remove(); + if (fs::exists(detached_permanently_flag)) + fs::remove(detached_permanently_flag); } catch (Exception & e) { @@ -308,11 +308,11 @@ void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const Stora /// If it was ATTACH query and file with table metadata already exist /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one. - Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); + fs::rename(table_metadata_tmp_path, table_metadata_path); } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); throw; } } @@ -321,10 +321,10 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr, const String & table_nam { auto table = detachTable(table_name); - Poco::File detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix); + fs::path detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix); try { - detached_permanently_flag.createFile(); + FS::createFile(detached_permanently_flag); } catch (Exception & e) { @@ -350,25 +350,25 @@ void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_na bool renamed = false; try { - Poco::File(table_metadata_path).renameTo(table_metadata_path_drop); + fs::rename(table_metadata_path, table_metadata_path_drop); renamed = true; table->drop(); table->is_dropped = true; - Poco::File table_data_dir{local_context->getPath() + table_data_path_relative}; - if (table_data_dir.exists()) - table_data_dir.remove(true); + fs::path table_data_dir(local_context->getPath() + table_data_path_relative); + if (fs::exists(table_data_dir)) + fs::remove_all(table_data_dir); } catch (...) { LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); attachTable(table_name, table, table_data_path_relative); if (renamed) - Poco::File(table_metadata_path_drop).renameTo(table_metadata_path); + fs::rename(table_metadata_path_drop, table_metadata_path); throw; } - Poco::File(table_metadata_path_drop).remove(); + fs::remove(table_metadata_path_drop); } void DatabaseOnDisk::checkMetadataFilenameAvailability(const String & to_table_name) const @@ -381,11 +381,11 @@ void DatabaseOnDisk::checkMetadataFilenameAvailabilityUnlocked(const String & to { String table_metadata_path = getObjectMetadataPath(to_table_name); - if (Poco::File(table_metadata_path).exists()) + if (fs::exists(table_metadata_path)) { - auto detached_permanently_flag = Poco::File(table_metadata_path + detached_suffix); + fs::path detached_permanently_flag(table_metadata_path + detached_suffix); - if (detached_permanently_flag.exists()) + if (fs::exists(detached_permanently_flag)) throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached permanently)", backQuote(database_name), backQuote(to_table_name)); else throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached)", backQuote(database_name), backQuote(to_table_name)); @@ -463,7 +463,7 @@ void DatabaseOnDisk::renameTable( /// Now table data are moved to new database, so we must add metadata and attach table to new database to_database.createTable(local_context, to_table_name, table, attach_query); - Poco::File(table_metadata_path).remove(); + fs::remove(table_metadata_path); if (from_atomic_to_ordinary) { @@ -528,8 +528,8 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const void DatabaseOnDisk::drop(ContextPtr local_context) { assert(tables.empty()); - Poco::File(local_context->getPath() + getDataPath()).remove(false); - Poco::File(getMetadataPath()).remove(false); + fs::remove(local_context->getPath() + getDataPath()); + fs::remove(getMetadataPath()); } String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const @@ -540,10 +540,9 @@ String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_name) const { String table_metadata_path = getObjectMetadataPath(object_name); - Poco::File meta_file(table_metadata_path); - if (meta_file.exists()) - return meta_file.getLastModified().epochTime(); + if (fs::exists(table_metadata_path)) + return FS::getModificationTime(table_metadata_path); else return static_cast(0); } @@ -555,56 +554,57 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat assert(getUUID() == UUIDHelpers::Nil); static const char * tmp_drop_ext = ".sql.tmp_drop"; const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext)); - if (Poco::File(local_context->getPath() + getDataPath() + '/' + object_name).exists()) + + if (fs::exists(local_context->getPath() + getDataPath() + '/' + object_name)) { - Poco::File(getMetadataPath() + file_name).renameTo(getMetadataPath() + object_name + ".sql"); + fs::rename(getMetadataPath() + file_name, getMetadataPath() + object_name + ".sql"); LOG_WARNING(log, "Object {} was not dropped previously and will be restored", backQuote(object_name)); process_metadata_file(object_name + ".sql"); } else { LOG_INFO(log, "Removing file {}", getMetadataPath() + file_name); - Poco::File(getMetadataPath() + file_name).remove(); + fs::remove(getMetadataPath() + file_name); } }; /// Metadata files to load: name and flag for .tmp_drop files std::set> metadata_files; - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) + fs::directory_iterator dir_end; + for (fs::directory_iterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) { + String file_name = dir_it->path().filename(); /// For '.svn', '.gitignore' directory and similar. - if (dir_it.name().at(0) == '.') + if (file_name.at(0) == '.') continue; /// There are .sql.bak files - skip them. - if (endsWith(dir_it.name(), ".sql.bak")) + if (endsWith(file_name, ".sql.bak")) continue; /// Permanently detached table flag - if (endsWith(dir_it.name(), ".sql.detached")) + if (endsWith(file_name, ".sql.detached")) continue; - if (endsWith(dir_it.name(), ".sql.tmp_drop")) + if (endsWith(file_name, ".sql.tmp_drop")) { /// There are files that we tried to delete previously - metadata_files.emplace(dir_it.name(), false); + metadata_files.emplace(file_name, false); } - else if (endsWith(dir_it.name(), ".sql.tmp")) + else if (endsWith(file_name, ".sql.tmp")) { /// There are files .sql.tmp - delete - LOG_INFO(log, "Removing file {}", dir_it->path()); - Poco::File(dir_it->path()).remove(); + LOG_INFO(log, "Removing file {}", dir_it->path().string()); + fs::remove(dir_it->path()); } - else if (endsWith(dir_it.name(), ".sql")) + else if (endsWith(file_name, ".sql")) { /// The required files have names like `table_name.sql` - metadata_files.emplace(dir_it.name(), true); + metadata_files.emplace(file_name, true); } else - throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(), - ErrorCodes::INCORRECT_FILE_NAME); + throw Exception(ErrorCodes::INCORRECT_FILE_NAME, "Incorrect file extension: {} in metadata directory {}", file_name, getMetadataPath()); } /// Read and parse metadata in parallel @@ -651,7 +651,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata( { if (logger) LOG_ERROR(logger, "File {} is empty. Removing.", metadata_file_path); - Poco::File(metadata_file_path).remove(); + fs::remove(metadata_file_path); return nullptr; } @@ -670,8 +670,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata( auto & create = ast->as(); if (!create.table.empty() && create.uuid != UUIDHelpers::Nil) { - String table_name = Poco::Path(metadata_file_path).makeFile().getBaseName(); - table_name = unescapeForFileName(table_name); + String table_name = unescapeForFileName(fs::path(metadata_file_path).stem()); if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER && logger) LOG_WARNING( diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index 677465e306e..73b69dd43a5 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -16,7 +16,7 @@ std::pair createTableFromAST( ASTCreateQuery ast_create_query, const String & database_name, const String & table_data_path_relative, - ContextPtr context, + ContextMutablePtr context, bool has_force_restore_data_flag); /** Get the string with the table definition based on the CREATE query. diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 4a150972573..7758841afc2 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -35,7 +34,7 @@ static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; namespace { void tryAttachTable( - ContextPtr context, + ContextMutablePtr context, const ASTCreateQuery & query, DatabaseOrdinary & database, const String & database_name, @@ -84,7 +83,7 @@ DatabaseOrdinary::DatabaseOrdinary( { } -void DatabaseOrdinary::loadStoredObjects(ContextPtr local_context, bool has_force_restore_data_flag, bool /*force_attach*/) +void DatabaseOrdinary::loadStoredObjects(ContextMutablePtr local_context, bool has_force_restore_data_flag, bool /*force_attach*/) { /** Tables load faster if they are loaded in sorted (by name) order. * Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order, @@ -111,8 +110,7 @@ void DatabaseOrdinary::loadStoredObjects(ContextPtr local_context, bool has_forc auto * create_query = ast->as(); create_query->database = database_name; - auto detached_permanently_flag = Poco::File(full_path.string() + detached_suffix); - if (detached_permanently_flag.exists()) + if (fs::exists(full_path.string() + detached_suffix)) { /// FIXME: even if we don't load the table we can still mark the uuid of it as taken. /// if (create_query->uuid != UUIDHelpers::Nil) @@ -281,11 +279,11 @@ void DatabaseOrdinary::commitAlterTable(const StorageID &, const String & table_ try { /// rename atomically replaces the old file with the new one. - Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); + fs::rename(table_metadata_tmp_path, table_metadata_path); } catch (...) { - Poco::File(table_metadata_tmp_path).remove(); + fs::remove(table_metadata_tmp_path); throw; } } diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 8a32bb10991..4d68890cc2b 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -20,7 +20,7 @@ public: String getEngineName() const override { return "Ordinary"; } - void loadStoredObjects(ContextPtr context, bool has_force_restore_data_flag, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr context, bool has_force_restore_data_flag, bool force_attach) override; void alterTable( ContextPtr context, diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index d36fe45f748..5d3c156f665 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -212,7 +212,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach) createDatabaseNodesInZooKeeper(current_zookeeper); } - replica_path = zookeeper_path + "/replicas/" + getFullReplicaName(); + replica_path = fs::path(zookeeper_path) / "replicas" / getFullReplicaName(); String replica_host_id; if (current_zookeeper->tryGet(replica_path, replica_host_id)) @@ -300,7 +300,7 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt current_zookeeper->multi(ops); } -void DatabaseReplicated::loadStoredObjects(ContextPtr local_context, bool has_force_restore_data_flag, bool force_attach) +void DatabaseReplicated::loadStoredObjects(ContextMutablePtr local_context, bool has_force_restore_data_flag, bool force_attach) { tryConnectToZooKeeperAndInitDatabase(force_attach); diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index b930d27c19b..b1fe0e58b0e 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -57,7 +57,7 @@ public: void drop(ContextPtr /*context*/) override; - void loadStoredObjects(ContextPtr context, bool has_force_restore_data_flag, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr context, bool has_force_restore_data_flag, bool force_attach) override; void shutdown() override; friend struct DatabaseReplicatedTask; diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 9ae4d026bf0..760300d6750 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -1,6 +1,9 @@ #include #include #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -156,7 +159,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na } } - UInt32 our_log_ptr = parse(current_zookeeper->get(database->replica_path + "/log_ptr")); + UInt32 our_log_ptr = parse(current_zookeeper->get(fs::path(database->replica_path) / "log_ptr")); UInt32 entry_num = DatabaseReplicatedTask::getLogEntryNumber(entry_name); if (entry_num <= our_log_ptr) @@ -165,13 +168,13 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na return {}; } - String entry_path = queue_dir + "/" + entry_name; + String entry_path = fs::path(queue_dir) / entry_name; auto task = std::make_unique(entry_name, entry_path, database); String initiator_name; zkutil::EventPtr wait_committed_or_failed = std::make_shared(); - String try_node_path = entry_path + "/try"; + String try_node_path = fs::path(entry_path) / "try"; if (zookeeper->tryGet(try_node_path, initiator_name, nullptr, wait_committed_or_failed)) { task->is_initial_query = initiator_name == task->host_id_str; @@ -203,7 +206,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) throw Coordination::Exception(code, try_node_path); - if (!zookeeper->exists(entry_path + "/committed")) + if (!zookeeper->exists(fs::path(entry_path) / "committed")) { out_reason = fmt::format("Entry {} was forcefully cancelled due to timeout", entry_name); return {}; @@ -212,7 +215,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na } } - if (!zookeeper->exists(entry_path + "/committed")) + if (!zookeeper->exists(fs::path(entry_path) / "committed")) { out_reason = fmt::format("Entry {} hasn't been committed", entry_name); return {}; @@ -220,8 +223,8 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na if (task->is_initial_query) { - assert(!zookeeper->exists(entry_path + "/try")); - assert(zookeeper->exists(entry_path + "/committed") == (zookeeper->get(task->getFinishedNodePath()) == ExecutionStatus(0).serializeText())); + assert(!zookeeper->exists(fs::path(entry_path) / "try")); + assert(zookeeper->exists(fs::path(entry_path) / "committed") == (zookeeper->get(task->getFinishedNodePath()) == ExecutionStatus(0).serializeText())); out_reason = fmt::format("Entry {} has been executed as initial query", entry_name); return {}; } @@ -257,7 +260,7 @@ DDLTaskPtr DatabaseReplicatedDDLWorker::initAndCheckTask(const String & entry_na bool DatabaseReplicatedDDLWorker::canRemoveQueueEntry(const String & entry_name, const Coordination::Stat &) { UInt32 entry_number = DDLTaskBase::getLogEntryNumber(entry_name); - UInt32 max_log_ptr = parse(getAndSetZooKeeper()->get(database->zookeeper_path + "/max_log_ptr")); + UInt32 max_log_ptr = parse(getAndSetZooKeeper()->get(fs::path(database->zookeeper_path) / "max_log_ptr")); return entry_number + logs_to_keep < max_log_ptr; } diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index ea07e51bffd..ba5fa974d5c 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -122,7 +122,7 @@ public: /// Load a set of existing tables. /// You can call only once, right after the object is created. - virtual void loadStoredObjects(ContextPtr /*context*/, bool /*has_force_restore_data_flag*/, bool /*force_attach*/ = false) {} + virtual void loadStoredObjects(ContextMutablePtr /*context*/, bool /*has_force_restore_data_flag*/, bool /*force_attach*/ = false) {} /// Check the existence of the table. virtual bool isTableExist(const String & name, ContextPtr context) const = 0; diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp index 9b71fe537ec..866b03e3bd6 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp @@ -24,10 +24,10 @@ # include # include # include +# include +# include -# include -# include - +namespace fs = std::filesystem; namespace DB { @@ -326,7 +326,7 @@ void DatabaseConnectionMySQL::shutdown() void DatabaseConnectionMySQL::drop(ContextPtr /*context*/) { - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } void DatabaseConnectionMySQL::cleanOutdatedTables() @@ -372,10 +372,10 @@ void DatabaseConnectionMySQL::attachTable(const String & table_name, const Stora local_tables_cache[table_name].second = storage; remove_or_detach_tables.erase(table_name); - Poco::File remove_flag(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); + fs::path remove_flag = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); - if (remove_flag.exists()) - remove_flag.remove(); + if (fs::exists(remove_flag)) + fs::remove(remove_flag); } StoragePtr DatabaseConnectionMySQL::detachTable(const String & table_name) @@ -399,17 +399,17 @@ String DatabaseConnectionMySQL::getMetadataPath() const return metadata_path; } -void DatabaseConnectionMySQL::loadStoredObjects(ContextPtr, bool, bool /*force_attach*/) +void DatabaseConnectionMySQL::loadStoredObjects(ContextMutablePtr, bool, bool /*force_attach*/) { std::lock_guard lock{mutex}; - Poco::DirectoryIterator iterator(getMetadataPath()); + fs::directory_iterator iter(getMetadataPath()); - for (Poco::DirectoryIterator end; iterator != end; ++iterator) + for (fs::directory_iterator end; iter != end; ++iter) { - if (iterator->isFile() && endsWith(iterator.name(), suffix)) + if (fs::is_regular_file(iter->path()) && endsWith(iter->path().filename(), suffix)) { - const auto & filename = iterator.name(); + const auto & filename = iter->path().filename().string(); const auto & table_name = unescapeForFileName(filename.substr(0, filename.size() - strlen(suffix))); remove_or_detach_tables.emplace(table_name); } @@ -420,27 +420,25 @@ void DatabaseConnectionMySQL::detachTablePermanently(ContextPtr, const String & { std::lock_guard lock{mutex}; - Poco::File remove_flag(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); + fs::path remove_flag = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); if (remove_or_detach_tables.count(table_name)) - throw Exception("Table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " is dropped", - ErrorCodes::TABLE_IS_DROPPED); + throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); - if (remove_flag.exists()) - throw Exception("The remove flag file already exists but the " + backQuoteIfNeed(database_name) + - "." + backQuoteIfNeed(table_name) + " does not exists remove tables, it is bug.", ErrorCodes::LOGICAL_ERROR); + if (fs::exists(remove_flag)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The remove flag file already exists but the {}.{} does not exists remove tables, it is bug.", + backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); auto table_iter = local_tables_cache.find(table_name); if (table_iter == local_tables_cache.end()) - throw Exception("Table " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name) + " doesn't exist.", - ErrorCodes::UNKNOWN_TABLE); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); remove_or_detach_tables.emplace(table_name); try { table_iter->second.second->drop(); - remove_flag.createFile(); + FS::createFile(remove_flag); } catch (...) { diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.h b/src/Databases/MySQL/DatabaseConnectionMySQL.h index c1936e51999..32741185cfa 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.h +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.h @@ -74,7 +74,7 @@ public: void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override; - void loadStoredObjects(ContextPtr, bool, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr, bool, bool force_attach) override; StoragePtr detachTable(const String & table_name) override; diff --git a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp index 62a66b22c93..bf9d6cdfbfa 100644 --- a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp @@ -13,9 +13,11 @@ # include # include # include -# include # include # include +# include + +namespace fs = std::filesystem; namespace DB { @@ -92,7 +94,7 @@ void DatabaseMaterializeMySQL::setException(const std::exception_ptr & exc } template -void DatabaseMaterializeMySQL::loadStoredObjects(ContextPtr context_, bool has_force_restore_data_flag, bool force_attach) +void DatabaseMaterializeMySQL::loadStoredObjects(ContextMutablePtr context_, bool has_force_restore_data_flag, bool force_attach) { Base::loadStoredObjects(context_, has_force_restore_data_flag, force_attach); if (!force_attach) @@ -158,10 +160,10 @@ template void DatabaseMaterializeMySQL::drop(ContextPtr context_) { /// Remove metadata info - Poco::File metadata(Base::getMetadataPath() + "/.metadata"); + fs::path metadata(Base::getMetadataPath() + "/.metadata"); - if (metadata.exists()) - metadata.remove(false); + if (fs::exists(metadata)) + fs::remove(metadata); Base::drop(context_); } diff --git a/src/Databases/MySQL/DatabaseMaterializeMySQL.h b/src/Databases/MySQL/DatabaseMaterializeMySQL.h index cd9fe640239..74a3c06e6f0 100644 --- a/src/Databases/MySQL/DatabaseMaterializeMySQL.h +++ b/src/Databases/MySQL/DatabaseMaterializeMySQL.h @@ -43,7 +43,7 @@ protected: public: String getEngineName() const override { return "MaterializeMySQL"; } - void loadStoredObjects(ContextPtr context_, bool has_force_restore_data_flag, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr context_, bool has_force_restore_data_flag, bool force_attach) override; void createTable(ContextPtr context_, const String & name, const StoragePtr & table, const ASTPtr & query) override; diff --git a/src/Databases/MySQL/MaterializeMetadata.cpp b/src/Databases/MySQL/MaterializeMetadata.cpp index c389ab5a1b0..4fd99ca3438 100644 --- a/src/Databases/MySQL/MaterializeMetadata.cpp +++ b/src/Databases/MySQL/MaterializeMetadata.cpp @@ -8,11 +8,13 @@ #include #include #include -#include #include #include #include #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -193,12 +195,11 @@ void commitMetadata(const std::function & function, const String & persi try { function(); - - Poco::File(persistent_tmp_path).renameTo(persistent_path); + fs::rename(persistent_tmp_path, persistent_path); } catch (...) { - Poco::File(persistent_tmp_path).remove(); + fs::remove(persistent_tmp_path); throw; } } @@ -231,7 +232,7 @@ void MaterializeMetadata::transaction(const MySQLReplication::Position & positio MaterializeMetadata::MaterializeMetadata(const String & path_, const Settings & settings_) : persistent_path(path_), settings(settings_) { - if (Poco::File(persistent_path).exists()) + if (fs::exists(persistent_path)) { ReadBufferFromFile in(persistent_path, DBMS_DEFAULT_BUFFER_SIZE); assertString("Version:\t" + toString(meta_version), in); diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp index a7ebcb0d3f2..e5783be4624 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp @@ -41,7 +41,7 @@ namespace ErrorCodes static constexpr auto MYSQL_BACKGROUND_THREAD_NAME = "MySQLDBSync"; -static ContextPtr createQueryContext(ContextPtr context) +static ContextMutablePtr createQueryContext(ContextPtr context) { Settings new_query_settings = context->getSettings(); new_query_settings.insert_allow_materialized_columns = true; @@ -59,7 +59,7 @@ static ContextPtr createQueryContext(ContextPtr context) return query_context; } -static BlockIO tryToExecuteQuery(const String & query_to_execute, ContextPtr query_context, const String & database, const String & comment) +static BlockIO tryToExecuteQuery(const String & query_to_execute, ContextMutablePtr query_context, const String & database, const String & comment) { try { @@ -281,7 +281,7 @@ static inline void cleanOutdatedTables(const String & database_name, ContextPtr } static inline BlockOutputStreamPtr -getTableOutput(const String & database_name, const String & table_name, ContextPtr query_context, bool insert_materialized = false) +getTableOutput(const String & database_name, const String & table_name, ContextMutablePtr query_context, bool insert_materialized = false) { const StoragePtr & storage = DatabaseCatalog::instance().getTable(StorageID(database_name, table_name), query_context); diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 55813e519e4..243ab9166be 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -12,11 +12,12 @@ #include #include #include -#include -#include #include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -209,9 +210,9 @@ void DatabasePostgreSQL::attachTable(const String & table_name, const StoragePtr detached_or_dropped.erase(table_name); - Poco::File table_marked_as_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - if (table_marked_as_removed.exists()) - table_marked_as_removed.remove(); + fs::path table_marked_as_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + if (fs::exists(table_marked_as_removed)) + fs::remove(table_marked_as_removed); } @@ -256,16 +257,8 @@ void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool / if (detached_or_dropped.count(table_name)) throw Exception(fmt::format("Table {}.{} is already dropped/detached", database_name, table_name), ErrorCodes::TABLE_IS_DROPPED); - Poco::File mark_table_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - - try - { - mark_table_removed.createFile(); - } - catch (...) - { - throw; - } + fs::path mark_table_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + FS::createFile(mark_table_removed); if (cache_tables) cached_tables.erase(table_name); @@ -276,22 +269,22 @@ void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool / void DatabasePostgreSQL::drop(ContextPtr /*context*/) { - Poco::File(getMetadataPath()).remove(true); + fs::remove_all(getMetadataPath()); } -void DatabasePostgreSQL::loadStoredObjects(ContextPtr /* context */, bool, bool /*force_attach*/) +void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, bool, bool /*force_attach*/) { { std::lock_guard lock{mutex}; - Poco::DirectoryIterator iterator(getMetadataPath()); + fs::directory_iterator iter(getMetadataPath()); /// Check for previously dropped tables - for (Poco::DirectoryIterator end; iterator != end; ++iterator) + for (fs::directory_iterator end; iter != end; ++iter) { - if (iterator->isFile() && endsWith(iterator.name(), suffix)) + if (fs::is_regular_file(iter->path()) && endsWith(iter->path().filename(), suffix)) { - const auto & file_name = iterator.name(); + const auto & file_name = iter->path().filename().string(); const auto & table_name = unescapeForFileName(file_name.substr(0, file_name.size() - strlen(suffix))); detached_or_dropped.emplace(table_name); } @@ -325,9 +318,9 @@ void DatabasePostgreSQL::removeOutdatedTables() { auto table_name = *iter; iter = detached_or_dropped.erase(iter); - Poco::File table_marked_as_removed(getMetadataPath() + '/' + escapeForFileName(table_name) + suffix); - if (table_marked_as_removed.exists()) - table_marked_as_removed.remove(); + fs::path table_marked_as_removed = fs::path(getMetadataPath()) / (escapeForFileName(table_name) + suffix); + if (fs::exists(table_marked_as_removed)) + fs::remove(table_marked_as_removed); } else ++iter; diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index c5054883f2c..3c3a67ec9d3 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -47,7 +47,7 @@ public: bool empty() const override; - void loadStoredObjects(ContextPtr, bool, bool force_attach) override; + void loadStoredObjects(ContextMutablePtr, bool, bool force_attach) override; DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) override; diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index a16b2f18680..cf954a9620d 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -79,7 +79,7 @@ private: mutable std::string invalidate_query_response; ExternalQueryBuilder query_builder; Block sample_block; - ContextPtr context; + ContextMutablePtr context; ConnectionPoolWithFailoverPtr pool; const std::string load_all_query; Poco::Logger * log = &Poco::Logger::get("ClickHouseDictionarySource"); diff --git a/src/Dictionaries/DictionarySourceHelpers.cpp b/src/Dictionaries/DictionarySourceHelpers.cpp index 6ba4f63be66..723a0c338fd 100644 --- a/src/Dictionaries/DictionarySourceHelpers.cpp +++ b/src/Dictionaries/DictionarySourceHelpers.cpp @@ -67,7 +67,7 @@ Block blockForKeys( return block; } -ContextPtr copyContextAndApplySettings( +ContextMutablePtr copyContextAndApplySettings( const std::string & config_prefix, ContextConstPtr context, const Poco::Util::AbstractConfiguration & config) diff --git a/src/Dictionaries/DictionarySourceHelpers.h b/src/Dictionaries/DictionarySourceHelpers.h index aa0219449e3..1066574f8e9 100644 --- a/src/Dictionaries/DictionarySourceHelpers.h +++ b/src/Dictionaries/DictionarySourceHelpers.h @@ -3,14 +3,13 @@ #include #include - -#include #include #include #include #include #include + namespace DB { @@ -37,7 +36,7 @@ Block blockForKeys( const std::vector & requested_rows); /// Used for applying settings to copied context in some register[...]Source functions -ContextPtr copyContextAndApplySettings( +ContextMutablePtr copyContextAndApplySettings( const std::string & config_prefix, ContextConstPtr context, const Poco::Util::AbstractConfiguration & config); diff --git a/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp b/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp index 3b8848ab19b..210459da0be 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp +++ b/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp @@ -5,7 +5,9 @@ #include #include #include "HierarchyFormatReader.h" +#include +namespace fs = std::filesystem; bool RegionsHierarchyDataSource::isModified() const { @@ -27,14 +29,13 @@ RegionsHierarchiesDataProvider::RegionsHierarchiesDataProvider(const std::string void RegionsHierarchiesDataProvider::discoverFilesWithCustomHierarchies() { - std::string basename = Poco::Path(path).getBaseName(); + std::string basename = fs::path(path).stem(); + fs::path dir_path = fs::canonical(path).parent_path(); - Poco::Path dir_path = Poco::Path(path).absolute().parent(); - - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(dir_path); dir_it != dir_end; ++dir_it) + fs::directory_iterator dir_end; + for (fs::directory_iterator dir_it(dir_path); dir_it != dir_end; ++dir_it) { - std::string candidate_basename = dir_it.path().getBaseName(); + std::string candidate_basename = dir_it->path().stem(); if (candidate_basename.starts_with(basename) && (candidate_basename.size() > basename.size() + 1) diff --git a/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp b/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp index f3e49545481..5f79fda070f 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp +++ b/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp @@ -2,7 +2,9 @@ #include #include "NamesFormatReader.h" +#include +namespace fs = std::filesystem; bool LanguageRegionsNamesDataSource::isModified() const { @@ -11,7 +13,7 @@ bool LanguageRegionsNamesDataSource::isModified() const size_t LanguageRegionsNamesDataSource::estimateTotalSize() const { - return Poco::File(path).getSize(); + return fs::file_size(path); } ILanguageRegionsNamesReaderPtr LanguageRegionsNamesDataSource::createReader() @@ -39,7 +41,7 @@ RegionsNamesDataProvider::RegionsNamesDataProvider(const std::string & directory ILanguageRegionsNamesDataSourcePtr RegionsNamesDataProvider::getLanguageRegionsNamesSource(const std::string & language) const { const auto data_file = getDataFilePath(language); - if (Poco::File(data_file).exists()) + if (fs::exists(data_file)) return std::make_unique(data_file, language); else return {}; diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index cf722aba68f..49e87e24543 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -1,9 +1,4 @@ #include "FileDictionarySource.h" - -#include - -#include - #include #include #include @@ -15,6 +10,7 @@ #include "registerDictionaries.h" #include "DictionarySourceHelpers.h" + namespace DB { static const UInt64 max_block_size = 8192; @@ -68,9 +64,10 @@ std::string FileDictionarySource::toString() const Poco::Timestamp FileDictionarySource::getLastModification() const { - return Poco::File{filepath}.getLastModified(); + return FS::getModificationTimestamp(filepath); } + void registerDictionarySourceFile(DictionarySourceFactory & factory) { auto create_table_source = [=](const DictionaryStructure & dict_struct, diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 3fa61408ba1..f52d774f279 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -1,19 +1,19 @@ #include "LibraryDictionarySource.h" -#include - +#include +#include #include #include #include #include -#include -#include +#include #include #include #include #include +namespace fs = std::filesystem; namespace DB { @@ -44,8 +44,8 @@ LibraryDictionarySource::LibraryDictionarySource( if (created_from_ddl && !pathStartsWith(path, context->getDictionariesLibPath())) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, context->getDictionariesLibPath()); - if (!Poco::File(path).exists()) - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", Poco::File(path).path()); + if (!fs::exists(path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", path); description.init(sample_block); bridge_helper = std::make_shared(context, description.sample_block, dictionary_id); diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index d0cf6a00344..768d1c2354a 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -6,9 +6,9 @@ #include #include #include - #include +#include #include @@ -60,27 +60,28 @@ class DiskLocalDirectoryIterator : public IDiskDirectoryIterator { public: explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_) - : dir_path(dir_path_), iter(disk_path_ + dir_path_) + : dir_path(dir_path_), entry(fs::path(disk_path_) / dir_path_) { } - void next() override { ++iter; } + void next() override { ++entry; } - bool isValid() const override { return iter != Poco::DirectoryIterator(); } + bool isValid() const override { return entry != fs::directory_iterator(); } String path() const override { - if (iter->isDirectory()) - return dir_path + iter.name() + '/'; + if (entry->is_directory()) + return dir_path / entry->path().filename() / ""; else - return dir_path + iter.name(); + return dir_path / entry->path().filename(); } - String name() const override { return iter.name(); } + + String name() const override { return entry->path().filename(); } private: - String dir_path; - Poco::DirectoryIterator iter; + fs::path dir_path; + fs::directory_iterator entry; }; @@ -118,7 +119,7 @@ UInt64 DiskLocal::getTotalSpace() const { struct statvfs fs; if (name == "default") /// for default disk we get space from path/data/ - fs = getStatVFS(disk_path + "data/"); + fs = getStatVFS((fs::path(disk_path) / "data/").string()); else fs = getStatVFS(disk_path); UInt64 total_size = fs.f_blocks * fs.f_bsize; @@ -133,7 +134,7 @@ UInt64 DiskLocal::getAvailableSpace() const /// available for superuser only and for system purposes struct statvfs fs; if (name == "default") /// for default disk we get space from path/data/ - fs = getStatVFS(disk_path + "data/"); + fs = getStatVFS((fs::path(disk_path) / "data/").string()); else fs = getStatVFS(disk_path); UInt64 total_size = fs.f_bavail * fs.f_bsize; @@ -152,45 +153,43 @@ UInt64 DiskLocal::getUnreservedSpace() const bool DiskLocal::exists(const String & path) const { - return Poco::File(disk_path + path).exists(); + return fs::exists(fs::path(disk_path) / path); } bool DiskLocal::isFile(const String & path) const { - return Poco::File(disk_path + path).isFile(); + return fs::is_regular_file(fs::path(disk_path) / path); } bool DiskLocal::isDirectory(const String & path) const { - return Poco::File(disk_path + path).isDirectory(); + return fs::is_directory(fs::path(disk_path) / path); } size_t DiskLocal::getFileSize(const String & path) const { - return Poco::File(disk_path + path).getSize(); + return fs::file_size(fs::path(disk_path) / path); } void DiskLocal::createDirectory(const String & path) { - Poco::File(disk_path + path).createDirectory(); + fs::create_directory(fs::path(disk_path) / path); } void DiskLocal::createDirectories(const String & path) { - Poco::File(disk_path + path).createDirectories(); + fs::create_directories(fs::path(disk_path) / path); } void DiskLocal::clearDirectory(const String & path) { - std::vector files; - Poco::File(disk_path + path).list(files); - for (auto & file : files) - file.remove(); + for (const auto & entry : fs::directory_iterator(fs::path(disk_path) / path)) + fs::remove(entry.path()); } void DiskLocal::moveDirectory(const String & from_path, const String & to_path) { - Poco::File(disk_path + from_path).renameTo(disk_path + to_path); + fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path); } DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) @@ -200,99 +199,95 @@ DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) void DiskLocal::moveFile(const String & from_path, const String & to_path) { - Poco::File(disk_path + from_path).renameTo(disk_path + to_path); + fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path); } void DiskLocal::replaceFile(const String & from_path, const String & to_path) { - Poco::File from_file(disk_path + from_path); - Poco::File to_file(disk_path + to_path); - if (to_file.exists()) - { - Poco::File tmp_file(disk_path + to_path + ".old"); - to_file.renameTo(tmp_file.path()); - from_file.renameTo(disk_path + to_path); - tmp_file.remove(); - } - else - from_file.renameTo(to_file.path()); + fs::path from_file = fs::path(disk_path) / from_path; + fs::path to_file = fs::path(disk_path) / to_path; + fs::rename(from_file, to_file); } std::unique_ptr DiskLocal::readFile( const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const { - return createReadBufferFromFileBase(disk_path + path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size); + return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size); } std::unique_ptr DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode) { int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1; - return std::make_unique(disk_path + path, buf_size, flags); + return std::make_unique(fs::path(disk_path) / path, buf_size, flags); } void DiskLocal::removeFile(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != unlink(fs_path.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK); + throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK); } void DiskLocal::removeFileIfExists(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != unlink(fs_path.c_str()) && errno != ENOENT) - throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK); + throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK); } void DiskLocal::removeDirectory(const String & path) { - auto fs_path = disk_path + path; + auto fs_path = fs::path(disk_path) / path; if (0 != rmdir(fs_path.c_str())) - throwFromErrnoWithPath("Cannot rmdir " + fs_path, fs_path, ErrorCodes::CANNOT_RMDIR); + throwFromErrnoWithPath("Cannot rmdir " + fs_path.string(), fs_path, ErrorCodes::CANNOT_RMDIR); } void DiskLocal::removeRecursive(const String & path) { - Poco::File(disk_path + path).remove(true); + fs::remove_all(fs::path(disk_path) / path); } void DiskLocal::listFiles(const String & path, std::vector & file_names) { - Poco::File(disk_path + path).list(file_names); + file_names.clear(); + for (const auto & entry : fs::directory_iterator(fs::path(disk_path) / path)) + file_names.emplace_back(entry.path().filename()); } void DiskLocal::setLastModified(const String & path, const Poco::Timestamp & timestamp) { - Poco::File(disk_path + path).setLastModified(timestamp); + FS::setModificationTime(fs::path(disk_path) / path, timestamp.epochTime()); } Poco::Timestamp DiskLocal::getLastModified(const String & path) { - return Poco::File(disk_path + path).getLastModified(); + return FS::getModificationTimestamp(fs::path(disk_path) / path); } void DiskLocal::createHardLink(const String & src_path, const String & dst_path) { - DB::createHardLink(disk_path + src_path, disk_path + dst_path); + DB::createHardLink(fs::path(disk_path) / src_path, fs::path(disk_path) / dst_path); } void DiskLocal::truncateFile(const String & path, size_t size) { - int res = truncate((disk_path + path).c_str(), size); + int res = truncate((fs::path(disk_path) / path).string().data(), size); if (-1 == res) throwFromErrnoWithPath("Cannot truncate file " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE); } void DiskLocal::createFile(const String & path) { - Poco::File(disk_path + path).createFile(); + FS::createFile(fs::path(disk_path) / path); } void DiskLocal::setReadOnly(const String & path) { - Poco::File(disk_path + path).setReadOnly(true); + fs::permissions(fs::path(disk_path) / path, + fs::perms::owner_write | fs::perms::group_write | fs::perms::others_write, + fs::perm_options::remove); } bool inline isSameDiskType(const IDisk & one, const IDisk & another) @@ -303,14 +298,23 @@ bool inline isSameDiskType(const IDisk & one, const IDisk & another) void DiskLocal::copy(const String & from_path, const std::shared_ptr & to_disk, const String & to_path) { if (isSameDiskType(*this, *to_disk)) - Poco::File(disk_path + from_path).copyTo(to_disk->getPath() + to_path); /// Use more optimal way. + { + fs::path to = fs::path(to_disk->getPath()) / to_path; + fs::path from = fs::path(disk_path) / from_path; + if (from_path.ends_with('/')) + from = from.parent_path(); + if (fs::is_directory(from)) + to /= from.filename(); + + fs::copy(from, to, fs::copy_options::recursive | fs::copy_options::overwrite_existing); /// Use more optimal way. + } else IDisk::copy(from_path, to_disk, to_path); /// Copy files through buffers. } SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const { - return std::make_unique(disk_path + path); + return std::make_unique(fs::path(disk_path) / path); } DiskPtr DiskLocalReservation::getDisk(size_t i) const @@ -381,10 +385,8 @@ void registerDiskLocal(DiskFactory & factory) throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); } - if (Poco::File disk{path}; !disk.canRead() || !disk.canWrite()) - { + if (!FS::canRead(path) || !FS::canWrite(path)) throw Exception("There is no RW access to the disk " + name + " (" + path + ")", ErrorCodes::PATH_ACCESS_DENIED); - } bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio"); diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 567ca24eb50..47482ad8d67 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -6,8 +6,6 @@ #include #include -#include -#include namespace DB { @@ -27,7 +25,7 @@ public: : name(name_), disk_path(path_), keep_free_space_bytes(keep_free_space_bytes_) { if (disk_path.back() != '/') - throw Exception("Disk path must ends with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Disk path must end with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR); } const String & getName() const override { return name; } diff --git a/src/Disks/DiskMemory.cpp b/src/Disks/DiskMemory.cpp index 68257ec4948..423dba6bed6 100644 --- a/src/Disks/DiskMemory.cpp +++ b/src/Disks/DiskMemory.cpp @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB @@ -24,7 +23,7 @@ namespace ErrorCodes class DiskMemoryDirectoryIterator final : public IDiskDirectoryIterator { public: - explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) + explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) : dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin()) { } @@ -33,13 +32,13 @@ public: bool isValid() const override { return iter != dir_file_paths.end(); } - String path() const override { return (*iter).toString(); } + String path() const override { return iter->string(); } - String name() const override { return (*iter).getFileName(); } + String name() const override { return iter->filename(); } private: - std::vector dir_file_paths; - std::vector::iterator iter; + std::vector dir_file_paths; + std::vector::iterator iter; }; @@ -268,7 +267,7 @@ DiskDirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path) if (!path.empty() && files.find(path) == files.end()) throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); - std::vector dir_file_paths; + std::vector dir_file_paths; for (const auto & file : files) if (parentPath(file.first) == path) dir_file_paths.emplace_back(file.first); diff --git a/src/Disks/HDFS/DiskHDFS.cpp b/src/Disks/HDFS/DiskHDFS.cpp index 0648fd9f08c..da6ccb024c7 100644 --- a/src/Disks/HDFS/DiskHDFS.cpp +++ b/src/Disks/HDFS/DiskHDFS.cpp @@ -172,8 +172,8 @@ void registerDiskHDFS(DiskFactory & factory) const String & config_prefix, ContextConstPtr context_) -> DiskPtr { - Poco::File disk{context_->getPath() + "disks/" + name}; - disk.createDirectories(); + fs::path disk = fs::path(context_->getPath()) / "disks" / name; + fs::create_directories(disk); String uri{config.getString(config_prefix + ".endpoint")}; diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index ee7f57af771..82705b5dcc8 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -22,7 +22,8 @@ bool IDisk::isDirectoryEmpty(const String & path) void copyFile(IDisk & from_disk, const String & from_path, IDisk & to_disk, const String & to_path) { - LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} {} to {} {}.", from_disk.getName(), from_path, to_disk.getName(), to_path); + LOG_DEBUG(&Poco::Logger::get("IDisk"), "Copying from {} (path: {}) {} to {} (path: {}) {}.", + from_disk.getName(), from_disk.getPath(), from_path, to_disk.getName(), to_disk.getPath(), to_path); auto in = from_disk.readFile(from_path); auto out = to_disk.writeFile(to_path); @@ -41,16 +42,15 @@ void asyncCopy(IDisk & from_disk, String from_path, IDisk & to_disk, String to_p [&from_disk, from_path, &to_disk, to_path]() { setThreadName("DiskCopier"); - DB::copyFile(from_disk, from_path, to_disk, to_path + fileName(from_path)); + DB::copyFile(from_disk, from_path, to_disk, fs::path(to_path) / fileName(from_path)); }); results.push_back(std::move(result)); } else { - Poco::Path path(from_path); - const String & dir_name = path.directory(path.depth() - 1); - const String dest = to_path + dir_name + "/"; + fs::path dir_name = fs::path(from_path).parent_path().filename(); + fs::path dest(fs::path(to_path) / dir_name); to_disk.createDirectories(dest); for (auto it = from_disk.iterateDirectory(from_path); it->isValid(); it->next()) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index a42a60959c5..cb718605dd6 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -7,16 +7,16 @@ #include #include #include -#include "Disks/Executor.h" #include #include #include #include -#include #include +#include #include "Poco/Util/AbstractConfiguration.h" +namespace fs = std::filesystem; namespace CurrentMetrics { @@ -212,10 +212,10 @@ public: virtual DiskType::Type getType() const = 0; /// Invoked when Global Context is shutdown. - virtual void shutdown() { } + virtual void shutdown() {} /// Performs action on disk startup. - virtual void startup() { } + virtual void startup() {} /// Return some uniq string for file, overrode for S3 /// Required for distinguish different copies of the same part on S3 @@ -233,7 +233,7 @@ public: virtual SyncGuardPtr getDirectorySyncGuard(const String & path) const; /// Applies new settings for disk in runtime. - virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextConstPtr) { } + virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextConstPtr) {} protected: friend class DiskDecorator; @@ -294,25 +294,27 @@ public: /// Return full path to a file on disk. inline String fullPath(const DiskPtr & disk, const String & path) { - return disk->getPath() + path; + return fs::path(disk->getPath()) / path; } /// Return parent path for the specified path. inline String parentPath(const String & path) { - return Poco::Path(path).parent().toString(); + if (path.ends_with('/')) + return fs::path(path).parent_path().parent_path() / ""; + return fs::path(path).parent_path() / ""; } /// Return file name for the specified path. inline String fileName(const String & path) { - return Poco::Path(path).getFileName(); + return fs::path(path).filename(); } /// Return directory path for the specified path. inline String directoryPath(const String & path) { - return Poco::Path(path).setFileName("").toString(); + return fs::path(path).parent_path() / ""; } } diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index bcb399f5d07..b30e9613ed8 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -6,12 +6,12 @@ #include #include #include -#include #include #include #include #include #include +#include namespace DB @@ -179,9 +179,9 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths { LOG_DEBUG(log, "Remove file by path: {}", backQuote(metadata_path + path)); - Poco::File file(metadata_path + path); + fs::path file(metadata_path + path); - if (!file.isFile()) + if (!fs::is_regular_file(file)) throw Exception(ErrorCodes::CANNOT_DELETE_DIRECTORY, "Path '{}' is a directory", path); try @@ -191,7 +191,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths /// If there is no references - delete content from remote FS. if (metadata.ref_count == 0) { - file.remove(); + fs::remove(file); for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects) fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path); } @@ -199,7 +199,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths { --metadata.ref_count; metadata.save(); - file.remove(); + fs::remove(file); } } catch (const Exception & e) @@ -210,7 +210,7 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths LOG_WARNING(log, "Metadata file {} can't be read by reason: {}. Removing it forcibly.", backQuote(path), e.nested() ? e.nested()->message() : e.message()); - file.remove(); + fs::remove(file); } else throw; @@ -222,8 +222,8 @@ void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. - Poco::File file(metadata_path + path); - if (file.isFile()) + fs::path file = fs::path(metadata_path) / path; + if (fs::is_regular_file(file)) { removeMeta(path, fs_paths_keeper); } @@ -231,7 +231,7 @@ void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr { for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) removeMetaRecursive(it->path(), fs_paths_keeper); - file.remove(); + fs::remove(file); } } @@ -296,13 +296,13 @@ IDiskRemote::IDiskRemote( bool IDiskRemote::exists(const String & path) const { - return Poco::File(metadata_path + path).exists(); + return fs::exists(fs::path(metadata_path) / path); } bool IDiskRemote::isFile(const String & path) const { - return Poco::File(metadata_path + path).isFile(); + return fs::is_regular_file(fs::path(metadata_path) / path); } @@ -326,7 +326,7 @@ void IDiskRemote::moveFile(const String & from_path, const String & to_path) if (exists(to_path)) throw Exception("File already exists: " + to_path, ErrorCodes::FILE_ALREADY_EXISTS); - Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path); + fs::rename(fs::path(metadata_path) / from_path, fs::path(metadata_path) / to_path); } @@ -347,7 +347,7 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path) void IDiskRemote::removeFileIfExists(const String & path) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); - if (Poco::File(metadata_path + path).exists()) + if (fs::exists(fs::path(metadata_path) / path)) { removeMeta(path, fs_paths_keeper); removeFromRemoteFS(fs_paths_keeper); @@ -385,19 +385,19 @@ void IDiskRemote::setReadOnly(const String & path) bool IDiskRemote::isDirectory(const String & path) const { - return Poco::File(metadata_path + path).isDirectory(); + return fs::is_directory(fs::path(metadata_path) / path); } void IDiskRemote::createDirectory(const String & path) { - Poco::File(metadata_path + path).createDirectory(); + fs::create_directory(fs::path(metadata_path) / path); } void IDiskRemote::createDirectories(const String & path) { - Poco::File(metadata_path + path).createDirectories(); + fs::create_directories(fs::path(metadata_path) / path); } @@ -411,7 +411,7 @@ void IDiskRemote::clearDirectory(const String & path) void IDiskRemote::removeDirectory(const String & path) { - Poco::File(metadata_path + path).remove(); + fs::remove(fs::path(metadata_path) / path); } @@ -430,13 +430,13 @@ void IDiskRemote::listFiles(const String & path, std::vector & file_name void IDiskRemote::setLastModified(const String & path, const Poco::Timestamp & timestamp) { - Poco::File(metadata_path + path).setLastModified(timestamp); + FS::setModificationTime(fs::path(metadata_path) / path, timestamp.epochTime()); } Poco::Timestamp IDiskRemote::getLastModified(const String & path) { - return Poco::File(metadata_path + path).getLastModified(); + return FS::getModificationTimestamp(fs::path(metadata_path) / path); } diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h index b32258331a7..a90621443e4 100644 --- a/src/Disks/IDiskRemote.h +++ b/src/Disks/IDiskRemote.h @@ -4,11 +4,12 @@ #include #include "Disks/DiskFactory.h" #include "Disks/Executor.h" -#include #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -193,21 +194,21 @@ public: void next() override { ++iter; } - bool isValid() const override { return iter != Poco::DirectoryIterator(); } + bool isValid() const override { return iter != fs::directory_iterator(); } String path() const override { - if (iter->isDirectory()) - return folder_path + iter.name() + '/'; + if (fs::is_directory(iter->path())) + return folder_path / iter->path().filename().string() / ""; else - return folder_path + iter.name(); + return folder_path / iter->path().filename().string(); } - String name() const override { return iter.name(); } + String name() const override { return iter->path().filename(); } private: - Poco::DirectoryIterator iter; - String folder_path; + fs::directory_iterator iter; + fs::path folder_path; }; diff --git a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp index 955986e5259..6d4764e4392 100644 --- a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp @@ -21,6 +21,7 @@ ReadIndirectBufferFromRemoteFS::ReadIndirectBufferFromRemoteFS( { } + template off_t ReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence) { diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index a3f5fe89870..89c2d20db9f 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -215,7 +214,7 @@ void DiskS3::moveFile(const String & from_path, const String & to_path, bool sen createFileOperationObject("rename", revision, object_metadata); } - Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path); + fs::rename(fs::path(metadata_path) / from_path, fs::path(metadata_path) / to_path); } std::unique_ptr DiskS3::readFile(const String & path, size_t buf_size, size_t, size_t, size_t, MMappedFileCache *) const @@ -675,8 +674,8 @@ void DiskS3::restore() restoreFiles(information); restoreFileOperations(information); - Poco::File restore_file(metadata_path + RESTORE_FILE_NAME); - restore_file.remove(); + fs::path restore_file = fs::path(metadata_path) / RESTORE_FILE_NAME; + fs::remove(restore_file); saveSchemaVersion(RESTORABLE_SCHEMA_VERSION); @@ -863,8 +862,9 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio continue; /// Skip not finished parts. They shouldn't be in 'detached' directory, because CH wouldn't be able to finish processing them. - Poco::Path directory_path (path); - auto directory_name = directory_path.directory(directory_path.depth() - 1); + fs::path directory_path(path); + auto directory_name = directory_path.parent_path().filename().string(); + auto predicate = [&directory_name](String & prefix) { return directory_name.starts_with(prefix); }; if (std::any_of(not_finished_prefixes.begin(), not_finished_prefixes.end(), predicate)) continue; @@ -873,7 +873,14 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio LOG_DEBUG(log, "Move directory to 'detached' {} -> {}", path, detached_path); - Poco::File(metadata_path + path).moveTo(metadata_path + detached_path); + fs::path from_path = fs::path(metadata_path) / path; + fs::path to_path = fs::path(metadata_path) / detached_path; + if (path.ends_with('/')) + to_path /= from_path.parent_path().filename(); + else + to_path /= from_path.filename(); + fs::copy(from_path, to_path, fs::copy_options::recursive | fs::copy_options::overwrite_existing); + fs::remove_all(from_path); } } @@ -905,7 +912,9 @@ String DiskS3::revisionToString(UInt64 revision) String DiskS3::pathToDetached(const String & source_path) { - return Poco::Path(source_path).parent().append(Poco::Path("detached")).toString() + '/'; + if (source_path.ends_with('/')) + return fs::path(source_path).parent_path().parent_path() / "detached/"; + return fs::path(source_path).parent_path() / "detached/"; } void DiskS3::onFreeze(const String & path) diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index e02f413c65e..21eff5f3d31 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -174,7 +174,7 @@ void registerDiskS3(DiskFactory & factory) throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS); String metadata_path = config.getString(config_prefix + ".metadata_path", context->getPath() + "disks/" + name + "/"); - Poco::File (metadata_path).createDirectories(); + fs::create_directories(metadata_path); std::shared_ptr s3disk = std::make_shared( name, diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index cff2685ca24..efedff9929f 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -8,8 +8,6 @@ #include -#include - namespace { diff --git a/src/Disks/WriteIndirectBufferFromRemoteFS.cpp b/src/Disks/WriteIndirectBufferFromRemoteFS.cpp index adc711608d7..6951b9fa92e 100644 --- a/src/Disks/WriteIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/WriteIndirectBufferFromRemoteFS.cpp @@ -8,7 +8,6 @@ namespace DB { -/// Stores data in S3/HDFS and adds the object key (S3 path) and object size to metadata file on local FS. template WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( std::unique_ptr impl_, diff --git a/src/Disks/WriteIndirectBufferFromRemoteFS.h b/src/Disks/WriteIndirectBufferFromRemoteFS.h index cda7523e19e..ece7b9d5871 100644 --- a/src/Disks/WriteIndirectBufferFromRemoteFS.h +++ b/src/Disks/WriteIndirectBufferFromRemoteFS.h @@ -10,7 +10,7 @@ namespace DB { -/// Stores data in S3/HDFS and adds the object key (S3 path) and object size to metadata file on local FS. +/// Stores data in S3/HDFS and adds the object path and object size to metadata file on local FS. template class WriteIndirectBufferFromRemoteFS final : public WriteBufferFromFileDecorator { diff --git a/src/Disks/tests/gtest_disk.cpp b/src/Disks/tests/gtest_disk.cpp index 714abf485ee..36f91249391 100644 --- a/src/Disks/tests/gtest_disk.cpp +++ b/src/Disks/tests/gtest_disk.cpp @@ -2,6 +2,9 @@ #include #include #include "gtest_disk.h" +#include + +namespace fs = std::filesystem; #if !defined(__clang__) @@ -22,7 +25,7 @@ DB::DiskPtr createDisk() template <> DB::DiskPtr createDisk() { - Poco::File("tmp/").createDirectory(); + fs::create_directory("tmp/"); return std::make_shared("local_disk", "tmp/", 0); } @@ -43,7 +46,7 @@ template <> void destroyDisk(DB::DiskPtr & disk) { disk.reset(); - Poco::File("tmp/").remove(true); + fs::remove_all("tmp/"); } diff --git a/src/Formats/FormatSchemaInfo.cpp b/src/Formats/FormatSchemaInfo.cpp index 707f9babe8d..2605c0bdf04 100644 --- a/src/Formats/FormatSchemaInfo.cpp +++ b/src/Formats/FormatSchemaInfo.cpp @@ -1,7 +1,7 @@ #include -#include #include #include +#include namespace DB @@ -11,6 +11,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +namespace fs = std::filesystem; namespace { @@ -34,55 +35,66 @@ FormatSchemaInfo::FormatSchemaInfo(const String & format_schema, const String & String default_file_extension = getFormatSchemaDefaultFileExtension(format); - Poco::Path path; + fs::path path; if (require_message) { size_t colon_pos = format_schema.find(':'); - if ((colon_pos == String::npos) || (colon_pos == 0) || (colon_pos == format_schema.length() - 1) - || path.assign(format_schema.substr(0, colon_pos)).makeFile().getFileName().empty()) + if ((colon_pos == String::npos) || (colon_pos == 0) || (colon_pos == format_schema.length() - 1)) { throw Exception( "Format schema requires the 'format_schema' setting to have the 'schema_file:message_name' format" + (default_file_extension.empty() ? "" : ", e.g. 'schema." + default_file_extension + ":Message'") + - ". Got '" + format_schema - + "'", - ErrorCodes::BAD_ARGUMENTS); + ". Got '" + format_schema + "'", ErrorCodes::BAD_ARGUMENTS); + } + else + { + path = fs::path(format_schema.substr(0, colon_pos)); + String filename = path.has_filename() ? path.filename() : path.parent_path().filename(); + if (filename.empty()) + throw Exception( + "Format schema requires the 'format_schema' setting to have the 'schema_file:message_name' format" + + (default_file_extension.empty() ? "" : ", e.g. 'schema." + default_file_extension + ":Message'") + + ". Got '" + format_schema + "'", ErrorCodes::BAD_ARGUMENTS); } - message_name = format_schema.substr(colon_pos + 1); } else - path.assign(format_schema).makeFile().getFileName(); + { + path = fs::path(format_schema); + if (!path.has_filename()) + path = path.parent_path() / ""; + } auto default_schema_directory = [&format_schema_path]() { - static const String str = Poco::Path(format_schema_path).makeAbsolute().makeDirectory().toString(); + static const String str = fs::canonical(format_schema_path) / ""; return str; }; - if (path.getExtension().empty() && !default_file_extension.empty()) - path.setExtension(default_file_extension); + if (!path.has_extension() && !default_file_extension.empty()) + path = path.parent_path() / (path.stem().string() + '.' + default_file_extension); - if (path.isAbsolute()) + fs::path default_schema_directory_path(default_schema_directory()); + if (path.is_absolute()) { if (is_server) - throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.toString(), ErrorCodes::BAD_ARGUMENTS); - schema_path = path.getFileName(); - schema_directory = path.makeParent().toString(); + throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.string(), ErrorCodes::BAD_ARGUMENTS); + schema_path = path.filename(); + schema_directory = path.parent_path() / ""; } - else if (path.depth() >= 1 && path.directory(0) == "..") + else if (path.has_parent_path() && !fs::weakly_canonical(default_schema_directory_path / path).string().starts_with(fs::weakly_canonical(default_schema_directory_path).string())) { if (is_server) - throw Exception( - "Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: " + path.toString(), - ErrorCodes::BAD_ARGUMENTS); - path = Poco::Path(default_schema_directory()).resolve(path).toString(); - schema_path = path.getFileName(); - schema_directory = path.makeParent().toString(); + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: {} ({} not in {})", + path.string()); + path = default_schema_directory_path / path; + schema_path = path.filename(); + schema_directory = path.parent_path() / ""; } else { - schema_path = path.toString(); + schema_path = path; schema_directory = default_schema_directory(); } } diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index 4e81a4497b7..cd81f2b4e0b 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -3,10 +3,11 @@ #include #include #include -#include -#include #include #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -68,21 +69,19 @@ public: { const char * filename = reinterpret_cast(&chars[source_offset]); - const String user_files_path = getContext()->getUserFilesPath(); - String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); - Poco::Path poco_filepath = Poco::Path(filename); - if (poco_filepath.isRelative()) - poco_filepath = Poco::Path(user_files_absolute_path, poco_filepath); - const String file_absolute_path = poco_filepath.absolute().toString(); - checkReadIsAllowedOrThrow(user_files_absolute_path, file_absolute_path); + fs::path user_files_absolute_path = fs::canonical(fs::path(getContext()->getUserFilesPath())); + fs::path file_path(filename); + if (file_path.is_relative()) + file_path = user_files_absolute_path / file_path; + fs::path file_absolute_path = fs::canonical(file_path); + checkReadIsAllowedOrThrow(user_files_absolute_path.string(), file_absolute_path); - checked_filenames[row] = file_absolute_path; - auto file = Poco::File(file_absolute_path); + checked_filenames[row] = file_absolute_path.string(); - if (!file.exists()) - throw Exception(fmt::format("File {} doesn't exist.", file_absolute_path), ErrorCodes::FILE_DOESNT_EXIST); + if (!fs::exists(file_absolute_path)) + throw Exception(fmt::format("File {} doesn't exist.", file_absolute_path.string()), ErrorCodes::FILE_DOESNT_EXIST); - const auto current_file_size = Poco::File(file_absolute_path).getSize(); + const auto current_file_size = fs::file_size(file_absolute_path); result_offset += current_file_size + 1; res_offsets[row] = result_offset; @@ -117,8 +116,8 @@ private: if (file_absolute_path.find(user_files_absolute_path) != 0) throw Exception("File is not inside " + user_files_absolute_path, ErrorCodes::DATABASE_ACCESS_DENIED); - Poco::File path_poco_file = Poco::File(file_absolute_path); - if (path_poco_file.exists() && path_poco_file.isDirectory()) + fs::path fs_path(file_absolute_path); + if (fs::exists(fs_path) && fs::is_directory(fs_path)) throw Exception("File can't be a directory", ErrorCodes::INCORRECT_FILE_NAME); } }; diff --git a/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp b/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp index 53248467418..656e40b2ad7 100644 --- a/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp +++ b/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include #include @@ -9,7 +8,9 @@ #include #include #include +#include +namespace fs = std::filesystem; using namespace DB; @@ -235,7 +236,7 @@ try buf.reset(); reread_buf.reset(); - ASSERT_TRUE(!Poco::File(tmp_filename).exists()); + ASSERT_TRUE(!fs::exists(tmp_filename)); } } catch (...) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 5284756a4ff..2af65e0fd87 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -24,7 +24,7 @@ namespace ErrorCodes namespace ClusterProxy { -ContextPtr updateSettingsForCluster(const Cluster & cluster, ContextPtr context, const Settings & settings, Poco::Logger * log) +ContextMutablePtr updateSettingsForCluster(const Cluster & cluster, ContextPtr context, const Settings & settings, Poco::Logger * log) { Settings new_settings = settings; new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time); diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index 46525335803..c9efedfc422 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -31,7 +31,7 @@ class IStreamFactory; /// - optimize_skip_unused_shards_nesting /// /// @return new Context with adjusted settings -ContextPtr updateSettingsForCluster(const Cluster & cluster, ContextPtr context, const Settings & settings, Poco::Logger * log = nullptr); +ContextMutablePtr updateSettingsForCluster(const Cluster & cluster, ContextPtr context, const Settings & settings, Poco::Logger * log = nullptr); /// Execute a distributed query, creating a vector of BlockInputStreams, from which the result can be read. /// `stream_factory` object encapsulates the logic of creating streams for a different type of query diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7d9c2c27ee2..a920a26e4c4 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -74,8 +74,11 @@ #include #include #include +#include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event ContextLock; @@ -140,7 +143,7 @@ public: /// Find existing session or create a new. std::shared_ptr acquireSession( const String & session_id, - ContextPtr context, + ContextMutablePtr context, std::chrono::steady_clock::duration timeout, bool throw_if_not_found) { @@ -513,7 +516,7 @@ SharedContextHolder::SharedContextHolder(std::unique_ptr shar void SharedContextHolder::reset() { shared.reset(); } -ContextPtr Context::createGlobal(ContextSharedPart * shared) +ContextMutablePtr Context::createGlobal(ContextSharedPart * shared) { auto res = std::shared_ptr(new Context); res->shared = shared; @@ -530,19 +533,19 @@ SharedContextHolder Context::createShared() return SharedContextHolder(std::make_unique()); } -ContextPtr Context::createCopy(const ContextConstPtr & other) +ContextMutablePtr Context::createCopy(const ContextPtr & other) { return std::shared_ptr(new Context(*other)); } -ContextPtr Context::createCopy(const ContextWeakConstPtr & other) +ContextMutablePtr Context::createCopy(const ContextWeakConstPtr & other) { auto ptr = other.lock(); if (!ptr) throw Exception("Can't copy an expired context", ErrorCodes::LOGICAL_ERROR); return createCopy(ptr); } -ContextPtr Context::createCopy(const ContextPtr & other) +ContextMutablePtr Context::createCopy(const ContextMutablePtr & other) { return createCopy(std::const_pointer_cast(other)); } @@ -1079,7 +1082,7 @@ void Context::addViewSource(const StoragePtr & storage) } -StoragePtr Context::getViewSource() +StoragePtr Context::getViewSource() const { return view_source; } @@ -1313,7 +1316,7 @@ void Context::setMacros(std::unique_ptr && macros) shared->macros.set(std::move(macros)); } -ContextPtr Context::getQueryContext() const +ContextMutablePtr Context::getQueryContext() const { auto ptr = query_context.lock(); if (!ptr) throw Exception("There is no query or query context has expired", ErrorCodes::THERE_IS_NO_QUERY); @@ -1326,21 +1329,21 @@ bool Context::isInternalSubquery() const return ptr && ptr.get() != this; } -ContextPtr Context::getSessionContext() const +ContextMutablePtr Context::getSessionContext() const { auto ptr = session_context.lock(); if (!ptr) throw Exception("There is no session or session context has expired", ErrorCodes::THERE_IS_NO_SESSION); return ptr; } -ContextPtr Context::getGlobalContext() const +ContextMutablePtr Context::getGlobalContext() const { auto ptr = global_context.lock(); if (!ptr) throw Exception("There is no global context or global context has expired", ErrorCodes::LOGICAL_ERROR); return ptr; } -ContextPtr Context::getBufferContext() const +ContextMutablePtr Context::getBufferContext() const { if (!buffer_context) throw Exception("There is no buffer context", ErrorCodes::LOGICAL_ERROR); return buffer_context; @@ -1871,7 +1874,7 @@ std::shared_ptr Context::tryGetCluster(const std::string & cluster_name } -void Context::reloadClusterConfig() +void Context::reloadClusterConfig() const { while (true) { @@ -1958,7 +1961,7 @@ bool Context::hasTraceCollector() const } -std::shared_ptr Context::getQueryLog() +std::shared_ptr Context::getQueryLog() const { auto lock = getLock(); @@ -1969,7 +1972,7 @@ std::shared_ptr Context::getQueryLog() } -std::shared_ptr Context::getQueryThreadLog() +std::shared_ptr Context::getQueryThreadLog() const { auto lock = getLock(); @@ -1980,7 +1983,7 @@ std::shared_ptr Context::getQueryThreadLog() } -std::shared_ptr Context::getPartLog(const String & part_database) +std::shared_ptr Context::getPartLog(const String & part_database) const { auto lock = getLock(); @@ -1998,7 +2001,7 @@ std::shared_ptr Context::getPartLog(const String & part_database) } -std::shared_ptr Context::getTraceLog() +std::shared_ptr Context::getTraceLog() const { auto lock = getLock(); @@ -2009,7 +2012,7 @@ std::shared_ptr Context::getTraceLog() } -std::shared_ptr Context::getTextLog() +std::shared_ptr Context::getTextLog() const { auto lock = getLock(); @@ -2020,7 +2023,7 @@ std::shared_ptr Context::getTextLog() } -std::shared_ptr Context::getMetricLog() +std::shared_ptr Context::getMetricLog() const { auto lock = getLock(); @@ -2042,7 +2045,7 @@ std::shared_ptr Context::getAsynchronousMetricLog() const } -std::shared_ptr Context::getOpenTelemetrySpanLog() +std::shared_ptr Context::getOpenTelemetrySpanLog() const { auto lock = getLock(); @@ -2212,14 +2215,14 @@ void Context::checkCanBeDropped(const String & database, const String & table, c if (!max_size_to_drop || size <= max_size_to_drop) return; - Poco::File force_file(getFlagsPath() + "force_drop_table"); - bool force_file_exists = force_file.exists(); + fs::path force_file(getFlagsPath() + "force_drop_table"); + bool force_file_exists = fs::exists(force_file); if (force_file_exists) { try { - force_file.remove(); + fs::remove(force_file); return; } catch (...) @@ -2241,9 +2244,9 @@ void Context::checkCanBeDropped(const String & database, const String & table, c "Example:\nsudo touch '{}' && sudo chmod 666 '{}'", backQuoteIfNeed(database), backQuoteIfNeed(table), size_str, max_size_to_drop_str, - force_file.path(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist", - force_file.path(), - force_file.path(), force_file.path()); + force_file.string(), force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist", + force_file.string(), + force_file.string(), force_file.string()); } @@ -2640,11 +2643,14 @@ ZooKeeperMetadataTransactionPtr Context::getZooKeeperMetadataTransaction() const return metadata_transaction; } -PartUUIDsPtr Context::getPartUUIDs() +PartUUIDsPtr Context::getPartUUIDs() const { auto lock = getLock(); if (!part_uuids) - part_uuids = std::make_shared(); + /// For context itself, only this initialization is not const. + /// We could have done in constructor. + /// TODO: probably, remove this from Context. + const_cast(part_uuids) = std::make_shared(); return part_uuids; } @@ -2663,11 +2669,11 @@ void Context::setReadTaskCallback(ReadTaskCallback && callback) next_task_callback = callback; } -PartUUIDsPtr Context::getIgnoredPartUUIDs() +PartUUIDsPtr Context::getIgnoredPartUUIDs() const { auto lock = getLock(); if (!ignored_part_uuids) - ignored_part_uuids = std::make_shared(); + const_cast(ignored_part_uuids) = std::make_shared(); return ignored_part_uuids; } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 5089d2c0288..97cb1b980f1 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -252,12 +252,12 @@ private: StoragePtr view_source; /// Temporary StorageValues used to generate alias columns for materialized views Tables table_function_results; /// Temporary tables obtained by execution of table functions. Keyed by AST tree id. - ContextWeakPtr query_context; - ContextWeakPtr session_context; /// Session context or nullptr. Could be equal to this. - ContextWeakPtr global_context; /// Global context. Could be equal to this. + ContextWeakMutablePtr query_context; + ContextWeakMutablePtr session_context; /// Session context or nullptr. Could be equal to this. + ContextWeakMutablePtr global_context; /// Global context. Could be equal to this. /// XXX: move this stuff to shared part instead. - ContextPtr buffer_context; /// Buffer context. Could be equal to this. + ContextMutablePtr buffer_context; /// Buffer context. Could be equal to this. public: // Top-level OpenTelemetry trace context for the query. Makes sense only for a query context. @@ -293,10 +293,10 @@ private: public: /// Create initial Context with ContextShared and etc. - static ContextPtr createGlobal(ContextSharedPart * shared); - static ContextPtr createCopy(const ContextWeakConstPtr & other); - static ContextPtr createCopy(const ContextConstPtr & other); - static ContextPtr createCopy(const ContextPtr & other); + static ContextMutablePtr createGlobal(ContextSharedPart * shared); + static ContextMutablePtr createCopy(const ContextWeakConstPtr & other); + static ContextMutablePtr createCopy(const ContextMutablePtr & other); + static ContextMutablePtr createCopy(const ContextPtr & other); static SharedContextHolder createShared(); void copyFrom(const ContextPtr & other); @@ -459,7 +459,7 @@ public: StoragePtr executeTableFunction(const ASTPtr & table_expression); void addViewSource(const StoragePtr & storage); - StoragePtr getViewSource(); + StoragePtr getViewSource() const; String getCurrentDatabase() const; String getCurrentQueryId() const { return client_info.current_query_id; } @@ -555,14 +555,14 @@ public: /// For methods below you may need to acquire the context lock by yourself. - ContextPtr getQueryContext() const; + ContextMutablePtr getQueryContext() const; bool hasQueryContext() const { return !query_context.expired(); } bool isInternalSubquery() const; - ContextPtr getSessionContext() const; + ContextMutablePtr getSessionContext() const; bool hasSessionContext() const { return !session_context.expired(); } - ContextPtr getGlobalContext() const; + ContextMutablePtr getGlobalContext() const; bool hasGlobalContext() const { return !global_context.expired(); } bool isGlobalContext() const { @@ -570,10 +570,10 @@ public: return ptr && ptr.get() == this; } - ContextPtr getBufferContext() const; + ContextMutablePtr getBufferContext() const; - void setQueryContext(ContextPtr context_) { query_context = context_; } - void setSessionContext(ContextPtr context_) { session_context = context_; } + void setQueryContext(ContextMutablePtr context_) { query_context = context_; } + void setSessionContext(ContextMutablePtr context_) { session_context = context_; } void makeQueryContext() { query_context = shared_from_this(); } void makeSessionContext() { session_context = shared_from_this(); } @@ -668,7 +668,7 @@ public: void setClustersConfig(const ConfigurationPtr & config, const String & config_name = "remote_servers"); /// Sets custom cluster, but doesn't update configuration void setCluster(const String & cluster_name, const std::shared_ptr & cluster); - void reloadClusterConfig(); + void reloadClusterConfig() const; Compiler & getCompiler(); @@ -681,17 +681,17 @@ public: bool hasTraceCollector() const; /// Nullptr if the query log is not ready for this moment. - std::shared_ptr getQueryLog(); - std::shared_ptr getQueryThreadLog(); - std::shared_ptr getTraceLog(); - std::shared_ptr getTextLog(); - std::shared_ptr getMetricLog(); + std::shared_ptr getQueryLog() const; + std::shared_ptr getQueryThreadLog() const; + std::shared_ptr getTraceLog() const; + std::shared_ptr getTextLog() const; + std::shared_ptr getMetricLog() const; std::shared_ptr getAsynchronousMetricLog() const; - std::shared_ptr getOpenTelemetrySpanLog(); + std::shared_ptr getOpenTelemetrySpanLog() const; /// Returns an object used to log operations with parts if it possible. /// Provide table name to make required checks. - std::shared_ptr getPartLog(const String & part_database); + std::shared_ptr getPartLog(const String & part_database) const; const MergeTreeSettings & getMergeTreeSettings() const; const MergeTreeSettings & getReplicatedMergeTreeSettings() const; @@ -778,8 +778,8 @@ public: MySQLWireContext mysql; - PartUUIDsPtr getPartUUIDs(); - PartUUIDsPtr getIgnoredPartUUIDs(); + PartUUIDsPtr getPartUUIDs() const; + PartUUIDsPtr getIgnoredPartUUIDs() const; ReadTaskCallback getReadTaskCallback() const; void setReadTaskCallback(ReadTaskCallback && callback); @@ -820,7 +820,7 @@ struct NamedSession { NamedSessionKey key; UInt64 close_cycle = 0; - ContextPtr context; + ContextMutablePtr context; std::chrono::steady_clock::duration timeout; NamedSessions & parent; diff --git a/src/Interpreters/Context_fwd.h b/src/Interpreters/Context_fwd.h index d7232717bb2..af18e5a7fdd 100644 --- a/src/Interpreters/Context_fwd.h +++ b/src/Interpreters/Context_fwd.h @@ -22,10 +22,12 @@ class Context; /// Most used types have shorter names /// TODO: in the first part of refactoring all the context pointers are non-const. -using ContextPtr = std::shared_ptr; +using ContextPtr = std::shared_ptr; using ContextConstPtr = std::shared_ptr; -using ContextWeakPtr = std::weak_ptr; +using ContextMutablePtr = std::shared_ptr; +using ContextWeakPtr = std::weak_ptr; using ContextWeakConstPtr = std::weak_ptr; +using ContextWeakMutablePtr = std::weak_ptr; template struct WithContextImpl @@ -50,5 +52,6 @@ protected: using WithContext = WithContextImpl<>; using WithConstContext = WithContextImpl; +using WithMutableContext = WithContextImpl; } diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 087e5504672..4fb44738d8d 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -145,7 +145,7 @@ void DDLTaskBase::parseQueryFromEntry(ContextPtr context) query = parseQuery(parser_query, begin, end, description, 0, context->getSettingsRef().max_parser_depth); } -ContextPtr DDLTaskBase::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & /*zookeeper*/) +ContextMutablePtr DDLTaskBase::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & /*zookeeper*/) { auto query_context = Context::createCopy(from_context); query_context->makeQueryContext(); @@ -355,7 +355,7 @@ void DatabaseReplicatedTask::parseQueryFromEntry(ContextPtr context) } } -ContextPtr DatabaseReplicatedTask::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper) +ContextMutablePtr DatabaseReplicatedTask::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper) { auto query_context = DDLTaskBase::makeQueryContext(from_context, zookeeper); query_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index 5fafb42bfd2..703d691a358 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace Poco { @@ -14,6 +15,8 @@ namespace zkutil class ZooKeeper; } +namespace fs = std::filesystem; + namespace DB { @@ -98,11 +101,11 @@ struct DDLTaskBase virtual String getShardID() const = 0; - virtual ContextPtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper); + virtual ContextMutablePtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper); - inline String getActiveNodePath() const { return entry_path + "/active/" + host_id_str; } - inline String getFinishedNodePath() const { return entry_path + "/finished/" + host_id_str; } - inline String getShardNodePath() const { return entry_path + "/shards/" + getShardID(); } + inline String getActiveNodePath() const { return fs::path(entry_path) / "active" / host_id_str; } + inline String getFinishedNodePath() const { return fs::path(entry_path) / "finished" / host_id_str; } + inline String getShardNodePath() const { return fs::path(entry_path) / "shards" / getShardID(); } static String getLogEntryName(UInt32 log_entry_number); static UInt32 getLogEntryNumber(const String & log_entry_name); @@ -136,7 +139,7 @@ struct DatabaseReplicatedTask : public DDLTaskBase String getShardID() const override; void parseQueryFromEntry(ContextPtr context) override; - ContextPtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper) override; + ContextMutablePtr makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & zookeeper) override; DatabaseReplicated * database; }; diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index e92644f0f99..45218226fee 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -110,7 +110,7 @@ protected: void runMainThread(); void runCleanupThread(); - ContextPtr context; + ContextMutablePtr context; Poco::Logger * log; std::string host_fqdn; /// current host domain name diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 09166103155..a70cdb2e054 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,8 @@ #include #include #include +#include +#include #if !defined(ARCADIA_BUILD) # include "config_core.h" @@ -27,8 +28,7 @@ # include #endif -#include - +namespace fs = std::filesystem; namespace CurrentMetrics { @@ -354,10 +354,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, bool d db->drop(getContext()); /// Old ClickHouse versions did not store database.sql files - Poco::File database_metadata_file( - getContext()->getPath() + "metadata/" + escapeForFileName(database_name) + ".sql"); - if (database_metadata_file.exists()) - database_metadata_file.remove(false); + fs::path database_metadata_file = fs::path(getContext()->getPath()) / "metadata" / (escapeForFileName(database_name) + ".sql"); + if (fs::exists(database_metadata_file)) + fs::remove(database_metadata_file); } return db; @@ -528,13 +527,13 @@ void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, std::unique_ptr DatabaseCatalog::database_catalog; -DatabaseCatalog::DatabaseCatalog(ContextPtr global_context_) - : WithContext(global_context_), log(&Poco::Logger::get("DatabaseCatalog")) +DatabaseCatalog::DatabaseCatalog(ContextMutablePtr global_context_) + : WithMutableContext(global_context_), log(&Poco::Logger::get("DatabaseCatalog")) { TemporaryLiveViewCleaner::init(global_context_); } -DatabaseCatalog & DatabaseCatalog::init(ContextPtr global_context_) +DatabaseCatalog & DatabaseCatalog::init(ContextMutablePtr global_context_) { if (database_catalog) { @@ -783,7 +782,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr } addUUIDMapping(table_id.uuid); - drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); + drop_time = FS::getModificationTime(dropped_metadata_path); } std::lock_guard lock(tables_marked_dropped_mutex); @@ -892,16 +891,15 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) /// Even if table is not loaded, try remove its data from disk. /// TODO remove data from all volumes - String data_path = getContext()->getPath() + "store/" + getPathForUUID(table.table_id.uuid); - Poco::File table_data_dir{data_path}; - if (table_data_dir.exists()) + fs::path data_path = fs::path(getContext()->getPath()) / "store" / getPathForUUID(table.table_id.uuid); + if (fs::exists(data_path)) { - LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path, table.table_id.getNameForLogs()); - table_data_dir.remove(true); + LOG_INFO(log, "Removing data directory {} of dropped table {}", data_path.string(), table.table_id.getNameForLogs()); + fs::remove_all(data_path); } LOG_INFO(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); - Poco::File(table.metadata_path).remove(); + fs::remove(fs::path(table.metadata_path)); removeUUIDMappingFinally(table.table_id.uuid); CurrentMetrics::sub(CurrentMetrics::TablesToDropQueueSize, 1); diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 783c511fa08..e20259fb973 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -117,13 +117,13 @@ using TemporaryTablesMapping = std::map database_catalog; - explicit DatabaseCatalog(ContextPtr global_context_); + explicit DatabaseCatalog(ContextMutablePtr global_context_); void assertDatabaseExistsUnlocked(const String & database_name) const; void assertDatabaseDoesntExistUnlocked(const String & database_name) const; diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 21326947213..b9fb1ae89d8 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -286,15 +286,13 @@ static FunctionBasePtr compile( return nullptr; } - LOG_TRACE(getLogger(), "Try to compile expression {}", dag.dump()); - auto llvm_function = std::make_shared(dag); if (auto * compilation_cache = CompiledExpressionCacheFactory::instance().tryGetCache()) { - - auto [compiled_function_cache_entry, was_inserted] = compilation_cache->getOrSet(hash_key, [&] () + auto [compiled_function_cache_entry, _] = compilation_cache->getOrSet(hash_key, [&] () { + LOG_TRACE(getLogger(), "Compile expression {}", llvm_function->getName()); CHJIT::CompiledModuleInfo compiled_module_info = compileFunction(getJITInstance(), *llvm_function); auto * compiled_jit_function = getJITInstance().findCompiledFunction(compiled_module_info, llvm_function->getName()); auto compiled_function = std::make_shared(compiled_jit_function, compiled_module_info); @@ -302,27 +300,17 @@ static FunctionBasePtr compile( return std::make_shared(std::move(compiled_function), compiled_module_info.size); }); - if (was_inserted) - LOG_TRACE(getLogger(), - "Put compiled expression {} in cache used cache size {} total cache size {}", - llvm_function->getName(), - compilation_cache->weight(), - compilation_cache->maxSize()); - else - LOG_TRACE(getLogger(), "Get compiled expression {} from cache", llvm_function->getName()); - llvm_function->setCompiledFunction(compiled_function_cache_entry->getCompiledFunction()); } else { + LOG_TRACE(getLogger(), "Compile expression {}", llvm_function->getName()); CHJIT::CompiledModuleInfo compiled_module_info = compileFunction(getJITInstance(), *llvm_function); auto * compiled_jit_function = getJITInstance().findCompiledFunction(compiled_module_info, llvm_function->getName()); auto compiled_function = std::make_shared(compiled_jit_function, compiled_module_info); llvm_function->setCompiledFunction(compiled_function); } - LOG_TRACE(getLogger(), "Use compiled expression {}", llvm_function->getName()); - return llvm_function; } diff --git a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp index 63755ee1839..00c31110ea4 100644 --- a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp @@ -3,12 +3,13 @@ #include #include #include - #include -#include -#include +#include +#include +namespace fs = std::filesystem; + namespace DB { ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository( @@ -19,7 +20,7 @@ ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository( Poco::Timestamp ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name) { - return Poco::File(definition_entity_name).getLastModified(); + return FS::getModificationTimestamp(definition_entity_name); } std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefinitionNames() @@ -36,8 +37,8 @@ std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti if (pattern[0] != '/') { const auto app_config_path = main_config.getString("config-file", "config.xml"); - const auto config_dir = Poco::Path{app_config_path}.parent().toString(); - const auto absolute_path = config_dir + pattern; + const String config_dir = fs::path(app_config_path).parent_path(); + const String absolute_path = fs::path(config_dir) / pattern; Poco::Glob::glob(absolute_path, files, 0); if (!files.empty()) continue; @@ -59,7 +60,7 @@ std::set ExternalLoaderXMLConfigRepository::getAllLoadablesDefiniti bool ExternalLoaderXMLConfigRepository::exists(const std::string & definition_entity_name) { - return Poco::File(definition_entity_name).exists(); + return fs::exists(fs::path(definition_entity_name)); } Poco::AutoPtr ExternalLoaderXMLConfigRepository::load( diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 51373ea49d4..09c4c538347 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -680,7 +680,19 @@ namespace class AddedColumns { public: - using TypeAndNames = std::vector>; + + struct TypeAndName + { + DataTypePtr type; + String name; + String qualified_name; + + TypeAndName(DataTypePtr type_, const String & name_, const String & qualified_name_) + : type(type_) + , name(name_) + , qualified_name(qualified_name_) + {} + }; AddedColumns(const Block & block_with_columns_to_add, const Block & block, @@ -705,27 +717,30 @@ public: for (const auto & src_column : block_with_columns_to_add) { + /// Column names `src_column.name` and `qualified_name` can differ for StorageJoin, + /// because it uses not qualified right block column names + auto qualified_name = join.getTableJoin().renamedRightColumnName(src_column.name); /// Don't insert column if it's in left block - if (!block.has(src_column.name)) - addColumn(src_column); + if (!block.has(qualified_name)) + addColumn(src_column, qualified_name); } if (is_asof_join) { const ColumnWithTypeAndName & right_asof_column = join.rightAsofKeyColumn(); - addColumn(right_asof_column); + addColumn(right_asof_column, right_asof_column.name); left_asof_key = key_columns.back(); } for (auto & tn : type_name) - right_indexes.push_back(saved_block_sample.getPositionByName(tn.second)); + right_indexes.push_back(saved_block_sample.getPositionByName(tn.name)); } size_t size() const { return columns.size(); } ColumnWithTypeAndName moveColumn(size_t i) { - return ColumnWithTypeAndName(std::move(columns[i]), type_name[i].first, type_name[i].second); + return ColumnWithTypeAndName(std::move(columns[i]), type_name[i].type, type_name[i].qualified_name); } template @@ -748,7 +763,7 @@ public: if (lazy_defaults_count) { for (size_t j = 0, size = right_indexes.size(); j < size; ++j) - JoinCommon::addDefaultValues(*columns[j], type_name[j].first, lazy_defaults_count); + JoinCommon::addDefaultValues(*columns[j], type_name[j].type, lazy_defaults_count); lazy_defaults_count = 0; } } @@ -764,7 +779,7 @@ public: bool need_filter = false; private: - TypeAndNames type_name; + std::vector type_name; MutableColumns columns; std::vector right_indexes; size_t lazy_defaults_count = 0; @@ -773,11 +788,11 @@ private: ASOF::Inequality asof_inequality; const IColumn * left_asof_key = nullptr; - void addColumn(const ColumnWithTypeAndName & src_column) + void addColumn(const ColumnWithTypeAndName & src_column, const std::string & qualified_name) { columns.push_back(src_column.column->cloneEmpty()); columns.back()->reserve(src_column.column->size()); - type_name.emplace_back(src_column.type, src_column.name); + type_name.emplace_back(src_column.type, src_column.name, qualified_name); } }; @@ -1081,7 +1096,8 @@ void HashJoin::joinBlockImpl( const auto & col = block.getByName(left_name); bool is_nullable = nullable_right_side || right_key.type->isNullable(); - block.insert(correctNullability({col.column, col.type, right_key.name}, is_nullable)); + auto right_col_name = getTableJoin().renamedRightColumnName(right_key.name); + block.insert(correctNullability({col.column, col.type, right_col_name}, is_nullable)); } } else if (has_required_right_keys) @@ -1106,7 +1122,8 @@ void HashJoin::joinBlockImpl( bool is_nullable = nullable_right_side || right_key.type->isNullable(); ColumnPtr thin_column = filterWithBlanks(col.column, filter); - block.insert(correctNullability({thin_column, col.type, right_key.name}, is_nullable, null_map_filter)); + auto right_col_name = getTableJoin().renamedRightColumnName(right_key.name); + block.insert(correctNullability({thin_column, col.type, right_col_name}, is_nullable, null_map_filter)); if constexpr (need_replication) right_keys_to_replicate.push_back(block.getPositionByName(right_key.name)); @@ -1311,7 +1328,12 @@ void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) void HashJoin::joinTotals(Block & block) const { - JoinCommon::joinTotals(totals, sample_block_with_columns_to_add, *table_join, block); + Block sample_right_block = sample_block_with_columns_to_add.cloneEmpty(); + /// For StorageJoin column names isn't qualified in sample_block_with_columns_to_add + for (auto & col : sample_right_block) + col.name = getTableJoin().renamedRightColumnName(col.name); + + JoinCommon::joinTotals(totals, sample_right_block, *table_join, block); } diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.h b/src/Interpreters/IInterpreterUnionOrSelectQuery.h index 72723d68161..0b07f27e14a 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.h +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.h @@ -32,7 +32,7 @@ public: protected: ASTPtr query_ptr; - ContextPtr context; + ContextMutablePtr context; Block result_header; SelectQueryOptions options; size_t max_streams = 1; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 6cb823aae3c..dc3fc0458b9 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -89,8 +89,8 @@ namespace ErrorCodes namespace fs = std::filesystem; -InterpreterCreateQuery::InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_) +InterpreterCreateQuery::InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) { } @@ -645,22 +645,6 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat } } - if (!create.attach && !settings.allow_experimental_bigint_types) - { - for (const auto & name_and_type_pair : properties.columns.getAllPhysical()) - { - WhichDataType which(*name_and_type_pair.type); - if (which.IsBigIntOrDeimal()) - { - const auto & type_name = name_and_type_pair.type->getName(); - String message = "Cannot create table with column '" + name_and_type_pair.name + "' which type is '" - + type_name + "' because experimental bigint types are not allowed. " - + "Set 'allow_experimental_bigint_types' setting to enable."; - throw Exception(message, ErrorCodes::ILLEGAL_COLUMN); - } - } - } - if (!create.attach && !settings.allow_experimental_map_type) { for (const auto & name_and_type_pair : properties.columns.getAllPhysical()) diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 990b87f02be..45f0bbd7cf8 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -22,10 +22,10 @@ using DatabasePtr = std::shared_ptr; /** Allows to create new table or database, * or create an object for existing table or database. */ -class InterpreterCreateQuery : public IInterpreter, WithContext +class InterpreterCreateQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateQuotaQuery.h b/src/Interpreters/InterpreterCreateQuotaQuery.h index d8edd24b2d9..7fb60cde1f6 100644 --- a/src/Interpreters/InterpreterCreateQuotaQuery.h +++ b/src/Interpreters/InterpreterCreateQuotaQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateQuotaQuery; struct Quota; -class InterpreterCreateQuotaQuery : public IInterpreter, WithContext +class InterpreterCreateQuotaQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateQuotaQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateQuotaQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateRoleQuery.h b/src/Interpreters/InterpreterCreateRoleQuery.h index 18b3f946837..ee8ff8ff7fb 100644 --- a/src/Interpreters/InterpreterCreateRoleQuery.h +++ b/src/Interpreters/InterpreterCreateRoleQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateRoleQuery; struct Role; -class InterpreterCreateRoleQuery : public IInterpreter, WithContext +class InterpreterCreateRoleQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateRoleQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateRoleQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.h b/src/Interpreters/InterpreterCreateRowPolicyQuery.h index 10167bac669..8adfe6b0855 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.h +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateRowPolicyQuery; struct RowPolicy; -class InterpreterCreateRowPolicyQuery : public IInterpreter, WithContext +class InterpreterCreateRowPolicyQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateRowPolicyQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateRowPolicyQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateSettingsProfileQuery.h b/src/Interpreters/InterpreterCreateSettingsProfileQuery.h index 9ef1f0354a9..aa9264fbf22 100644 --- a/src/Interpreters/InterpreterCreateSettingsProfileQuery.h +++ b/src/Interpreters/InterpreterCreateSettingsProfileQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateSettingsProfileQuery; struct SettingsProfile; -class InterpreterCreateSettingsProfileQuery : public IInterpreter, WithContext +class InterpreterCreateSettingsProfileQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateSettingsProfileQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateSettingsProfileQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterCreateUserQuery.h b/src/Interpreters/InterpreterCreateUserQuery.h index e9f4e82e767..7d357924d35 100644 --- a/src/Interpreters/InterpreterCreateUserQuery.h +++ b/src/Interpreters/InterpreterCreateUserQuery.h @@ -10,10 +10,10 @@ namespace DB class ASTCreateUserQuery; struct User; -class InterpreterCreateUserQuery : public IInterpreter, WithContext +class InterpreterCreateUserQuery : public IInterpreter, WithMutableContext { public: - InterpreterCreateUserQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterCreateUserQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.h b/src/Interpreters/InterpreterDropAccessEntityQuery.h index 7f0f6348610..0ee478e904e 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.h +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.h @@ -9,10 +9,10 @@ namespace DB class AccessRightsElements; -class InterpreterDropAccessEntityQuery : public IInterpreter, WithContext +class InterpreterDropAccessEntityQuery : public IInterpreter, WithMutableContext { public: - InterpreterDropAccessEntityQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterDropAccessEntityQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 24cff90caea..3c515d58e0c 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -1,5 +1,3 @@ -#include - #include #include #include @@ -42,7 +40,7 @@ static DatabasePtr tryGetDatabase(const String & database_name, bool if_exists) } -InterpreterDropQuery::InterpreterDropQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) +InterpreterDropQuery::InterpreterDropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) { } diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 8e8d577deec..16a08e95a3b 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -16,10 +16,10 @@ class AccessRightsElements; * or remove information about table (just forget) from server (DETACH), * or just clear all data in table (TRUNCATE). */ -class InterpreterDropQuery : public IInterpreter, WithContext +class InterpreterDropQuery : public IInterpreter, WithMutableContext { public: - InterpreterDropQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterDropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); /// Drop table or database. BlockIO execute() override; diff --git a/src/Interpreters/InterpreterExternalDDLQuery.cpp b/src/Interpreters/InterpreterExternalDDLQuery.cpp index 8f9f0cf9ddb..e91b95a988d 100644 --- a/src/Interpreters/InterpreterExternalDDLQuery.cpp +++ b/src/Interpreters/InterpreterExternalDDLQuery.cpp @@ -26,8 +26,8 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -InterpreterExternalDDLQuery::InterpreterExternalDDLQuery(const ASTPtr & query_, ContextPtr context_) - : WithContext(context_), query(query_) +InterpreterExternalDDLQuery::InterpreterExternalDDLQuery(const ASTPtr & query_, ContextMutablePtr context_) + : WithMutableContext(context_), query(query_) { } diff --git a/src/Interpreters/InterpreterExternalDDLQuery.h b/src/Interpreters/InterpreterExternalDDLQuery.h index 15a842a2611..d084e68d7c8 100644 --- a/src/Interpreters/InterpreterExternalDDLQuery.h +++ b/src/Interpreters/InterpreterExternalDDLQuery.h @@ -6,10 +6,10 @@ namespace DB { -class InterpreterExternalDDLQuery : public IInterpreter, WithContext +class InterpreterExternalDDLQuery : public IInterpreter, WithMutableContext { public: - InterpreterExternalDDLQuery(const ASTPtr & query_, ContextPtr context_); + InterpreterExternalDDLQuery(const ASTPtr & query_, ContextMutablePtr context_); BlockIO execute() override; diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 4af8b6ffa7d..79cda364c42 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -92,7 +92,7 @@ namespace ErrorCodes } -std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextPtr context, const SelectQueryOptions & options) +std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMutablePtr context, const SelectQueryOptions & options) { OpenTelemetrySpanHolder span("InterpreterFactory::get()"); diff --git a/src/Interpreters/InterpreterFactory.h b/src/Interpreters/InterpreterFactory.h index c122fe11b7d..774cbd1cb0f 100644 --- a/src/Interpreters/InterpreterFactory.h +++ b/src/Interpreters/InterpreterFactory.h @@ -16,7 +16,7 @@ class InterpreterFactory public: static std::unique_ptr get( ASTPtr & query, - ContextPtr context, + ContextMutablePtr context, const SelectQueryOptions & options = {}); }; diff --git a/src/Interpreters/InterpreterGrantQuery.h b/src/Interpreters/InterpreterGrantQuery.h index f5939ff3cb7..abaddcc599b 100644 --- a/src/Interpreters/InterpreterGrantQuery.h +++ b/src/Interpreters/InterpreterGrantQuery.h @@ -12,10 +12,10 @@ class ASTGrantQuery; struct User; struct Role; -class InterpreterGrantQuery : public IInterpreter, WithContext +class InterpreterGrantQuery : public IInterpreter, WithMutableContext { public: - InterpreterGrantQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterGrantQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterKillQueryQuery.h b/src/Interpreters/InterpreterKillQueryQuery.h index 5ffd9a525a2..9284d777ca7 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.h +++ b/src/Interpreters/InterpreterKillQueryQuery.h @@ -10,10 +10,10 @@ namespace DB class AccessRightsElements; -class InterpreterKillQueryQuery final : public IInterpreter, WithContext +class InterpreterKillQueryQuery final : public IInterpreter, WithMutableContext { public: - InterpreterKillQueryQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) { } + InterpreterKillQueryQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) { } BlockIO execute() override; diff --git a/src/Interpreters/InterpreterSetQuery.h b/src/Interpreters/InterpreterSetQuery.h index 31519be6f29..9bd49708421 100644 --- a/src/Interpreters/InterpreterSetQuery.h +++ b/src/Interpreters/InterpreterSetQuery.h @@ -11,10 +11,10 @@ class ASTSetQuery; /** Change one or several settings for the session or just for the current context. */ -class InterpreterSetQuery : public IInterpreter, WithContext +class InterpreterSetQuery : public IInterpreter, WithMutableContext { public: - InterpreterSetQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterSetQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} /** Usual SET query. Set setting for the session. */ diff --git a/src/Interpreters/InterpreterSetRoleQuery.h b/src/Interpreters/InterpreterSetRoleQuery.h index 70ba3c381ab..0a489f45fcd 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.h +++ b/src/Interpreters/InterpreterSetRoleQuery.h @@ -11,10 +11,10 @@ class ASTSetRoleQuery; struct RolesOrUsersSet; struct User; -class InterpreterSetRoleQuery : public IInterpreter, WithContext +class InterpreterSetRoleQuery : public IInterpreter, WithMutableContext { public: - InterpreterSetRoleQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterSetRoleQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp index c2c2305f976..41b986e43a2 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -17,8 +17,8 @@ namespace ErrorCodes using EntityType = IAccessEntity::Type; -InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_) +InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) { } diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h index 7224f0d593b..35511a38d8f 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h @@ -7,10 +7,10 @@ namespace DB { -class InterpreterShowAccessEntitiesQuery : public IInterpreter, WithContext +class InterpreterShowAccessEntitiesQuery : public IInterpreter, WithMutableContext { public: - InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; diff --git a/src/Interpreters/InterpreterShowPrivilegesQuery.cpp b/src/Interpreters/InterpreterShowPrivilegesQuery.cpp index c566d31e2fc..201c1cfece8 100644 --- a/src/Interpreters/InterpreterShowPrivilegesQuery.cpp +++ b/src/Interpreters/InterpreterShowPrivilegesQuery.cpp @@ -4,7 +4,7 @@ namespace DB { -InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, ContextPtr context_) +InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : query_ptr(query_ptr_), context(context_) { } diff --git a/src/Interpreters/InterpreterShowPrivilegesQuery.h b/src/Interpreters/InterpreterShowPrivilegesQuery.h index 75989263405..852d5173eb1 100644 --- a/src/Interpreters/InterpreterShowPrivilegesQuery.h +++ b/src/Interpreters/InterpreterShowPrivilegesQuery.h @@ -11,7 +11,7 @@ class Context; class InterpreterShowPrivilegesQuery : public IInterpreter { public: - InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; @@ -20,7 +20,7 @@ public: private: ASTPtr query_ptr; - ContextPtr context; + ContextMutablePtr context; }; } diff --git a/src/Interpreters/InterpreterShowProcesslistQuery.h b/src/Interpreters/InterpreterShowProcesslistQuery.h index 5eedb67595e..31454882a89 100644 --- a/src/Interpreters/InterpreterShowProcesslistQuery.h +++ b/src/Interpreters/InterpreterShowProcesslistQuery.h @@ -9,11 +9,11 @@ namespace DB /** Return list of currently executing queries. */ -class InterpreterShowProcesslistQuery : public IInterpreter, WithContext +class InterpreterShowProcesslistQuery : public IInterpreter, WithMutableContext { public: - InterpreterShowProcesslistQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_) {} + InterpreterShowProcesslistQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) {} BlockIO execute() override; diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 901999f004f..609df1404ca 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -18,8 +18,8 @@ namespace ErrorCodes } -InterpreterShowTablesQuery::InterpreterShowTablesQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_) +InterpreterShowTablesQuery::InterpreterShowTablesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_) { } diff --git a/src/Interpreters/InterpreterShowTablesQuery.h b/src/Interpreters/InterpreterShowTablesQuery.h index b61be568e35..16fc9ef2cf4 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.h +++ b/src/Interpreters/InterpreterShowTablesQuery.h @@ -13,10 +13,10 @@ class Context; /** Return a list of tables or databases meets specified conditions. * Interprets a query through replacing it to SELECT query from system.tables or system.databases. */ -class InterpreterShowTablesQuery : public IInterpreter, WithContext +class InterpreterShowTablesQuery : public IInterpreter, WithMutableContext { public: - InterpreterShowTablesQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterShowTablesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index e7babb9b83f..f97001883bd 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -192,8 +192,8 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type, } -InterpreterSystemQuery::InterpreterSystemQuery(const ASTPtr & query_ptr_, ContextPtr context_) - : WithContext(context_), query_ptr(query_ptr_->clone()), log(&Poco::Logger::get("InterpreterSystemQuery")) +InterpreterSystemQuery::InterpreterSystemQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_), query_ptr(query_ptr_->clone()), log(&Poco::Logger::get("InterpreterSystemQuery")) { } @@ -424,7 +424,7 @@ BlockIO InterpreterSystemQuery::execute() } -StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextPtr system_context, bool need_ddl_guard) +StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context, bool need_ddl_guard) { getContext()->checkAccess(AccessType::SYSTEM_RESTART_REPLICA, replica); @@ -469,7 +469,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, return table; } -void InterpreterSystemQuery::restartReplicas(ContextPtr system_context) +void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context) { std::vector replica_names; auto & catalog = DatabaseCatalog::instance(); @@ -542,7 +542,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) else if (!query.replica_zk_path.empty()) { getContext()->checkAccess(AccessType::SYSTEM_DROP_REPLICA); - auto remote_replica_path = query.replica_zk_path + "/replicas/" + query.replica; + String remote_replica_path = fs::path(query.replica_zk_path) / "replicas" / query.replica; /// This check is actually redundant, but it may prevent from some user mistakes for (auto & elem : DatabaseCatalog::instance().getDatabases()) diff --git a/src/Interpreters/InterpreterSystemQuery.h b/src/Interpreters/InterpreterSystemQuery.h index 341611e0af1..297f7225a92 100644 --- a/src/Interpreters/InterpreterSystemQuery.h +++ b/src/Interpreters/InterpreterSystemQuery.h @@ -30,10 +30,10 @@ class ASTSystemQuery; * - start/stop actions for all existing tables. * Note that the actions for tables that will be created after this query will not be affected. */ -class InterpreterSystemQuery : public IInterpreter, WithContext +class InterpreterSystemQuery : public IInterpreter, WithMutableContext { public: - InterpreterSystemQuery(const ASTPtr & query_ptr_, ContextPtr context_); + InterpreterSystemQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_); BlockIO execute() override; @@ -45,9 +45,9 @@ private: /// Tries to get a replicated table and restart it /// Returns pointer to a newly created table if the restart was successful - StoragePtr tryRestartReplica(const StorageID & replica, ContextPtr context, bool need_ddl_guard = true); + StoragePtr tryRestartReplica(const StorageID & replica, ContextMutablePtr context, bool need_ddl_guard = true); - void restartReplicas(ContextPtr system_context); + void restartReplicas(ContextMutablePtr system_context); void syncReplica(ASTSystemQuery & query); void dropReplica(ASTSystemQuery & query); bool dropReplicaImpl(ASTSystemQuery & query, const StoragePtr & table); diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 2332dada770..ca0b8257f6e 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -179,7 +179,7 @@ bool isStorageTouchedByMutations( const StoragePtr & storage, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, - ContextPtr context_copy) + ContextMutablePtr context_copy) { if (commands.empty()) return false; diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 7e2f910466b..0d91da5613c 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -23,7 +23,7 @@ bool isStorageTouchedByMutations( const StoragePtr & storage, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, - ContextPtr context_copy + ContextMutablePtr context_copy ); ASTPtr getPartitionAndPredicateExpressionForMutationCommand( diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h index 3202612ac94..1efbe8ee744 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h @@ -55,12 +55,12 @@ namespace MySQLInterpreter }; template -class InterpreterMySQLDDLQuery : public IInterpreter, WithContext +class InterpreterMySQLDDLQuery : public IInterpreter, WithMutableContext { public: InterpreterMySQLDDLQuery( - const ASTPtr & query_ptr_, ContextPtr context_, const String & mapped_to_database_, const String & mysql_database_) - : WithContext(context_), query_ptr(query_ptr_), mapped_to_database(mapped_to_database_), mysql_database(mysql_database_) + const ASTPtr & query_ptr_, ContextMutablePtr context_, const String & mapped_to_database_, const String & mysql_database_) + : WithMutableContext(context_), query_ptr(query_ptr_), mapped_to_database(mapped_to_database_), mysql_database(mysql_database_) { } diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index f547e011a73..122e2cd6479 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -156,9 +156,12 @@ NameSet TableJoin::requiredRightKeys() const { NameSet required; for (const auto & name : key_names_right) + { + auto rename = renamedRightColumnName(name); for (const auto & column : columns_added_by_join) - if (name == column.name) + if (rename == column.name) required.insert(name); + } return required; } @@ -464,4 +467,11 @@ ActionsDAGPtr TableJoin::applyKeyConvertToTable( return dag; } +String TableJoin::renamedRightColumnName(const String & name) const +{ + if (const auto it = renames.find(name); it != renames.end()) + return it->second; + return name; +} + } diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index b75ef848f13..dc8ebeb7413 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -203,6 +203,8 @@ public: /// Split key and other columns by keys name list void splitAdditionalColumns(const Block & sample_block, Block & block_keys, Block & block_others) const; Block getRequiredRightKeys(const Block & right_table_keys, std::vector & keys_sources) const; + + String renamedRightColumnName(const String & name) const; }; } diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index cff6fb9cbcd..8590b3c94f3 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -508,7 +508,7 @@ void CurrentThread::detachQueryIfNotDetached() } -CurrentThread::QueryScope::QueryScope(ContextPtr query_context) +CurrentThread::QueryScope::QueryScope(ContextMutablePtr query_context) { CurrentThread::initializeQuery(); CurrentThread::attachQueryContext(query_context); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 542ed7ca0f9..a78c810d5d4 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -335,7 +335,7 @@ static void onExceptionBeforeStart(const String & query_for_logging, ContextPtr } } -static void setQuerySpecificSettings(ASTPtr & ast, ContextPtr context) +static void setQuerySpecificSettings(ASTPtr & ast, ContextMutablePtr context) { if (auto * ast_insert_into = dynamic_cast(ast.get())) { @@ -347,7 +347,7 @@ static void setQuerySpecificSettings(ASTPtr & ast, ContextPtr context) static std::tuple executeQueryImpl( const char * begin, const char * end, - ContextPtr context, + ContextMutablePtr context, bool internal, QueryProcessingStage::Enum stage, bool has_query_tail, @@ -910,7 +910,7 @@ static std::tuple executeQueryImpl( BlockIO executeQuery( const String & query, - ContextPtr context, + ContextMutablePtr context, bool internal, QueryProcessingStage::Enum stage, bool may_have_embedded_data) @@ -935,7 +935,7 @@ BlockIO executeQuery( BlockIO executeQuery( const String & query, - ContextPtr context, + ContextMutablePtr context, bool internal, QueryProcessingStage::Enum stage, bool may_have_embedded_data, @@ -954,7 +954,7 @@ void executeQuery( ReadBuffer & istr, WriteBuffer & ostr, bool allow_into_outfile, - ContextPtr context, + ContextMutablePtr context, std::function set_result_details) { PODArray parse_buf; diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index bdb1f877ce3..6448b26a652 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -16,7 +16,7 @@ void executeQuery( ReadBuffer & istr, /// Where to read query from (and data for INSERT, if present). WriteBuffer & ostr, /// Where to write query output to. bool allow_into_outfile, /// If true and the query contains INTO OUTFILE section, redirect output to that file. - ContextPtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... + ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... std::function set_result_details /// If a non-empty callback is passed, it will be called with the query id, the content-type, the format, and the timezone. ); @@ -37,7 +37,7 @@ void executeQuery( /// must be done separately. BlockIO executeQuery( const String & query, /// Query text without INSERT data. The latter must be written to BlockIO::out. - ContextPtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... + ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... bool internal = false, /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, /// To which stage the query must be executed. bool may_have_embedded_data = false /// If insert query may have embedded data @@ -46,7 +46,7 @@ BlockIO executeQuery( /// Old interface with allow_processors flag. For compatibility. BlockIO executeQuery( const String & query, - ContextPtr context, + ContextMutablePtr context, bool internal, QueryProcessingStage::Enum stage, bool may_have_embedded_data, diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 79076e57328..43f9727c355 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -18,14 +18,16 @@ #include #include +#include +namespace fs = std::filesystem; namespace DB { static void executeCreateQuery( const String & query, - ContextPtr context, + ContextMutablePtr context, const String & database, const String & file_name, bool has_force_restore_data_flag) @@ -46,7 +48,7 @@ static void executeCreateQuery( static void loadDatabase( - ContextPtr context, + ContextMutablePtr context, const String & database, const String & database_path, bool force_restore_data) @@ -54,13 +56,13 @@ static void loadDatabase( String database_attach_query; String database_metadata_file = database_path + ".sql"; - if (Poco::File(database_metadata_file).exists()) + if (fs::exists(fs::path(database_metadata_file))) { /// There is .sql file with database creation statement. ReadBufferFromFile in(database_metadata_file, 1024); readStringUntilEOF(database_attach_query, in); } - else if (Poco::File(database_path).exists()) + else if (fs::exists(fs::path(database_path))) { /// Database exists, but .sql file is absent. It's old-style Ordinary database (e.g. system or default) database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database) + " ENGINE = Ordinary"; @@ -84,7 +86,7 @@ static void loadDatabase( } -void loadMetadata(ContextPtr context, const String & default_database_name) +void loadMetadata(ContextMutablePtr context, const String & default_database_name) { Poco::Logger * log = &Poco::Logger::get("loadMetadata"); @@ -95,34 +97,35 @@ void loadMetadata(ContextPtr context, const String & default_database_name) * This file is deleted after successful loading of tables. * (flag is "one-shot") */ - Poco::File force_restore_data_flag_file(context->getFlagsPath() + "force_restore_data"); - bool has_force_restore_data_flag = force_restore_data_flag_file.exists(); + auto force_restore_data_flag_file = fs::path(context->getFlagsPath()) / "force_restore_data"; + bool has_force_restore_data_flag = fs::exists(force_restore_data_flag_file); /// Loop over databases. std::map databases; - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(path); it != dir_end; ++it) + fs::directory_iterator dir_end; + for (fs::directory_iterator it(path); it != dir_end; ++it) { - if (it->isLink()) + if (it->is_symlink()) continue; - if (!it->isDirectory()) + const auto current_file = it->path().filename().string(); + if (!it->is_directory()) { /// TODO: DETACH DATABASE PERMANENTLY ? - if (endsWith(it.name(), ".sql")) + if (fs::path(current_file).extension() == ".sql") { - String db_name = it.name().substr(0, it.name().size() - 4); + String db_name = fs::path(current_file).stem(); if (db_name != DatabaseCatalog::SYSTEM_DATABASE) - databases.emplace(unescapeForFileName(db_name), path + "/" + db_name); + databases.emplace(unescapeForFileName(db_name), fs::path(path) / db_name); } /// Temporary fails may be left from previous server runs. - if (endsWith(it.name(), ".tmp")) + if (fs::path(current_file).extension() == ".tmp") { - LOG_WARNING(log, "Removing temporary file {}", it->path()); + LOG_WARNING(log, "Removing temporary file {}", it->path().string()); try { - it->remove(); + fs::remove(it->path()); } catch (...) { @@ -135,13 +138,13 @@ void loadMetadata(ContextPtr context, const String & default_database_name) } /// For '.svn', '.gitignore' directory and similar. - if (it.name().at(0) == '.') + if (current_file.at(0) == '.') continue; - if (it.name() == DatabaseCatalog::SYSTEM_DATABASE) + if (current_file == DatabaseCatalog::SYSTEM_DATABASE) continue; - databases.emplace(unescapeForFileName(it.name()), it.path().toString()); + databases.emplace(unescapeForFileName(current_file), it->path().string()); } /// clickhouse-local creates DatabaseMemory as default database by itself @@ -158,7 +161,7 @@ void loadMetadata(ContextPtr context, const String & default_database_name) { try { - force_restore_data_flag_file.remove(); + fs::remove(force_restore_data_flag_file); } catch (...) { @@ -168,11 +171,11 @@ void loadMetadata(ContextPtr context, const String & default_database_name) } -void loadMetadataSystem(ContextPtr context) +void loadMetadataSystem(ContextMutablePtr context) { String path = context->getPath() + "metadata/" + DatabaseCatalog::SYSTEM_DATABASE; String metadata_file = path + ".sql"; - if (Poco::File(path).exists() || Poco::File(metadata_file).exists()) + if (fs::exists(fs::path(path)) || fs::exists(fs::path(metadata_file))) { /// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted. loadDatabase(context, DatabaseCatalog::SYSTEM_DATABASE, path, true); diff --git a/src/Interpreters/loadMetadata.h b/src/Interpreters/loadMetadata.h index 047def84bba..cf038a42855 100644 --- a/src/Interpreters/loadMetadata.h +++ b/src/Interpreters/loadMetadata.h @@ -8,9 +8,9 @@ namespace DB /// Load tables from system database. Only real tables like query_log, part_log. /// You should first load system database, then attach system tables that you need into it, then load other databases. -void loadMetadataSystem(ContextPtr context); +void loadMetadataSystem(ContextMutablePtr context); /// Load tables from databases and add them to context. Database 'system' is ignored. Use separate function to load system tables. -void loadMetadata(ContextPtr context, const String & default_database_name = {}); +void loadMetadata(ContextMutablePtr context, const String & default_database_name = {}); } diff --git a/src/Parsers/ASTProjectionSelectQuery.cpp b/src/Parsers/ASTProjectionSelectQuery.cpp index 58943ed0430..cc0d387b933 100644 --- a/src/Parsers/ASTProjectionSelectQuery.cpp +++ b/src/Parsers/ASTProjectionSelectQuery.cpp @@ -134,8 +134,6 @@ ASTPtr ASTProjectionSelectQuery::cloneToASTSelect() const if (groupBy()) select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, groupBy()->clone()); // Get rid of orderBy. It's used for projection definition only - if (orderBy()) - select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, orderBy()->clone()); return node; } diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h index f69c2562d7f..dc3be3289fc 100644 --- a/src/Processors/Pipe.h +++ b/src/Processors/Pipe.h @@ -110,7 +110,7 @@ public: /// Do not allow to change the table while the processors of pipe are alive. void addTableLock(TableLockHolder lock) { holder.table_locks.emplace_back(std::move(lock)); } /// This methods are from QueryPipeline. Needed to make conversion from pipeline to pipe possible. - void addInterpreterContext(std::shared_ptr context) { holder.interpreter_context.emplace_back(std::move(context)); } + void addInterpreterContext(std::shared_ptr context) { holder.interpreter_context.emplace_back(std::move(context)); } void addStorageHolder(StoragePtr storage) { holder.storage_holders.emplace_back(std::move(storage)); } void addQueryIdHolder(std::shared_ptr query_id_holder) { holder.query_id_holder = std::move(query_id_holder); } /// For queries with nested interpreters (i.e. StorageDistributed) @@ -129,7 +129,7 @@ private: /// Some processors may implicitly use Context or temporary Storage created by Interpreter. /// But lifetime of Streams is not nested in lifetime of Interpreters, so we have to store it here, /// because QueryPipeline is alive until query is finished. - std::vector> interpreter_context; + std::vector> interpreter_context; std::vector storage_holders; std::vector table_locks; std::vector> query_plans; diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 90c7e370880..1585f2532ff 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -119,7 +119,7 @@ public: const Block & getHeader() const { return pipe.getHeader(); } void addTableLock(TableLockHolder lock) { pipe.addTableLock(std::move(lock)); } - void addInterpreterContext(std::shared_ptr context) { pipe.addInterpreterContext(std::move(context)); } + void addInterpreterContext(std::shared_ptr context) { pipe.addInterpreterContext(std::move(context)); } void addStorageHolder(StoragePtr storage) { pipe.addStorageHolder(std::move(storage)); } void addQueryPlan(std::unique_ptr plan) { pipe.addQueryPlan(std::move(plan)); } void setLimits(const StreamLocalLimits & limits) { pipe.setLimits(limits); } diff --git a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp index d7711abb3e1..0d1a0fdc619 100644 --- a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp +++ b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp @@ -4,7 +4,7 @@ namespace DB { -ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_) +ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_) : ISourceStep(DataStream{.header = pipe_.getHeader()}) , pipe(std::move(pipe_)) , context(std::move(context_)) diff --git a/src/Processors/QueryPlan/ReadFromPreparedSource.h b/src/Processors/QueryPlan/ReadFromPreparedSource.h index 69297c3c9ae..0d4e9653806 100644 --- a/src/Processors/QueryPlan/ReadFromPreparedSource.h +++ b/src/Processors/QueryPlan/ReadFromPreparedSource.h @@ -9,7 +9,7 @@ namespace DB class ReadFromPreparedSource : public ISourceStep { public: - explicit ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_ = nullptr); + explicit ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_ = nullptr); String getName() const override { return "ReadFromPreparedSource"; } @@ -17,7 +17,7 @@ public: private: Pipe pipe; - std::shared_ptr context; + std::shared_ptr context; }; class ReadFromStorageStep : public ReadFromPreparedSource diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 6f0f2d30123..2d6de423c5b 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -521,7 +521,7 @@ namespace Poco::Logger * log = nullptr; std::shared_ptr session; - ContextPtr query_context; + ContextMutablePtr query_context; std::optional query_scope; String query_text; ASTPtr ast; @@ -932,8 +932,8 @@ namespace String format = external_table.format(); if (format.empty()) format = "TabSeparated"; - ContextPtr external_table_context = query_context; - ContextPtr temp_context; + ContextMutablePtr external_table_context = query_context; + ContextMutablePtr temp_context; if (!external_table.settings().empty()) { temp_context = Context::createCopy(query_context); diff --git a/src/Server/HTTP/HTTPServerResponse.cpp b/src/Server/HTTP/HTTPServerResponse.cpp index db5cfb132e3..25e7604a515 100644 --- a/src/Server/HTTP/HTTPServerResponse.cpp +++ b/src/Server/HTTP/HTTPServerResponse.cpp @@ -1,11 +1,8 @@ #include - #include - #include #include #include -#include #include #include #include @@ -13,6 +10,7 @@ #include #include + namespace DB { diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 8aed5d20f74..6eab6cdda5e 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -39,7 +39,6 @@ #include #include -#include #include #include #include @@ -277,7 +276,7 @@ HTTPHandler::~HTTPHandler() bool HTTPHandler::authenticateUser( - ContextPtr context, + ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response) @@ -441,7 +440,7 @@ bool HTTPHandler::authenticateUser( void HTTPHandler::processQuery( - ContextPtr context, + ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response, @@ -937,7 +936,7 @@ DynamicQueryHandler::DynamicQueryHandler(IServer & server_, const std::string & { } -bool DynamicQueryHandler::customizeQueryParam(ContextPtr context, const std::string & key, const std::string & value) +bool DynamicQueryHandler::customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) { if (key == param_name) return true; /// do nothing @@ -953,7 +952,7 @@ bool DynamicQueryHandler::customizeQueryParam(ContextPtr context, const std::str return false; } -std::string DynamicQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) +std::string DynamicQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) { if (likely(!startsWith(request.getContentType(), "multipart/form-data"))) { @@ -992,7 +991,7 @@ PredefinedQueryHandler::PredefinedQueryHandler( { } -bool PredefinedQueryHandler::customizeQueryParam(ContextPtr context, const std::string & key, const std::string & value) +bool PredefinedQueryHandler::customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) { if (receive_params.count(key)) { @@ -1003,7 +1002,7 @@ bool PredefinedQueryHandler::customizeQueryParam(ContextPtr context, const std:: return false; } -void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, ContextPtr context) +void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, ContextMutablePtr context) { /// If in the configuration file, the handler's header is regex and contains named capture group /// We will extract regex named capture groups as query parameters @@ -1039,7 +1038,7 @@ void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, Conte } } -std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) +std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) { if (unlikely(startsWith(request.getContentType(), "multipart/form-data"))) { diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index 4715949cb87..2149a7ca55c 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -33,11 +33,11 @@ public: void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; /// This method is called right before the query execution. - virtual void customizeContext(HTTPServerRequest & /* request */, ContextPtr /* context */) {} + virtual void customizeContext(HTTPServerRequest & /* request */, ContextMutablePtr /* context */) {} - virtual bool customizeQueryParam(ContextPtr context, const std::string & key, const std::string & value) = 0; + virtual bool customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) = 0; - virtual std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) = 0; + virtual std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) = 0; private: struct Output @@ -73,7 +73,7 @@ private: // The request_context and the request_credentials instances may outlive a single request/response loop. // This happens only when the authentication mechanism requires more than a single request/response exchange (e.g., SPNEGO). - ContextPtr request_context; + ContextMutablePtr request_context; std::unique_ptr request_credentials; // Returns true when the user successfully authenticated, @@ -82,14 +82,14 @@ private: // the request_context and request_credentials instances are preserved. // Throws an exception if authentication failed. bool authenticateUser( - ContextPtr context, + ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response); /// Also initializes 'used_output'. void processQuery( - ContextPtr context, + ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response, @@ -113,9 +113,9 @@ private: public: explicit DynamicQueryHandler(IServer & server_, const std::string & param_name_ = "query"); - std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) override; + std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) override; - bool customizeQueryParam(ContextPtr context, const std::string &key, const std::string &value) override; + bool customizeQueryParam(ContextMutablePtr context, const std::string &key, const std::string &value) override; }; class PredefinedQueryHandler : public HTTPHandler @@ -130,11 +130,11 @@ public: IServer & server_, const NameSet & receive_params_, const std::string & predefined_query_ , const CompiledRegexPtr & url_regex_, const std::unordered_map & header_name_with_regex_); - virtual void customizeContext(HTTPServerRequest & request, ContextPtr context) override; + virtual void customizeContext(HTTPServerRequest & request, ContextMutablePtr context) override; - std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextPtr context) override; + std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) override; - bool customizeQueryParam(ContextPtr context, const std::string & key, const std::string & value) override; + bool customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) override; }; } diff --git a/src/Server/IServer.h b/src/Server/IServer.h index 80736fda3ea..c55b045d2a5 100644 --- a/src/Server/IServer.h +++ b/src/Server/IServer.h @@ -28,7 +28,7 @@ public: virtual Poco::Logger & logger() const = 0; /// Returns global application's context. - virtual ContextPtr context() const = 0; + virtual ContextMutablePtr context() const = 0; /// Returns true if shutdown signaled. virtual bool isCancelled() const = 0; diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index f5fb82b5bef..e681ad2e6f6 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -56,7 +56,7 @@ private: protected: Poco::Logger * log; - ContextPtr connection_context; + ContextMutablePtr connection_context; std::shared_ptr packet_endpoint; diff --git a/src/Server/PostgreSQLHandler.h b/src/Server/PostgreSQLHandler.h index 0f114d388fb..9aaad1d7aa7 100644 --- a/src/Server/PostgreSQLHandler.h +++ b/src/Server/PostgreSQLHandler.h @@ -37,7 +37,7 @@ private: Poco::Logger * log = &Poco::Logger::get("PostgreSQLHandler"); IServer & server; - ContextPtr connection_context; + ContextMutablePtr connection_context; bool ssl_enabled = false; Int32 connection_id = 0; Int32 secret_key = 0; diff --git a/src/Server/StaticRequestHandler.cpp b/src/Server/StaticRequestHandler.cpp index 169d6859b43..19b91ae9c42 100644 --- a/src/Server/StaticRequestHandler.cpp +++ b/src/Server/StaticRequestHandler.cpp @@ -14,14 +14,15 @@ #include -#include -#include #include #include #include #include +#include +namespace fs = std::filesystem; + namespace DB { @@ -137,11 +138,14 @@ void StaticRequestHandler::writeResponse(WriteBuffer & out) if (startsWith(response_expression, file_prefix)) { - const auto & user_files_absolute_path = Poco::Path(server.context()->getUserFilesPath()).makeAbsolute().makeDirectory().toString(); - const auto & file_name = response_expression.substr(file_prefix.size(), response_expression.size() - file_prefix.size()); + auto file_name = response_expression.substr(file_prefix.size(), response_expression.size() - file_prefix.size()); + if (file_name.starts_with('/')) + file_name = file_name.substr(1); - const auto & file_path = Poco::Path(user_files_absolute_path, file_name).makeAbsolute().toString(); - if (!Poco::File(file_path).exists()) + fs::path user_files_absolute_path = fs::canonical(fs::path(server.context()->getUserFilesPath())); + String file_path = fs::weakly_canonical(user_files_absolute_path / file_name); + + if (!fs::exists(file_path)) throw Exception("Invalid file name " + file_path + " for static HTTPHandler. ", ErrorCodes::INCORRECT_FILE_NAME); ReadBufferFromFile in(file_path); diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 8387ca5f254..086c1f7d5e5 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -119,7 +119,7 @@ public: void run() override; /// This method is called right before the query execution. - virtual void customizeContext(ContextPtr /*context*/) {} + virtual void customizeContext(ContextMutablePtr /*context*/) {} private: IServer & server; @@ -132,8 +132,8 @@ private: UInt64 client_version_patch = 0; UInt64 client_tcp_protocol_version = 0; - ContextPtr connection_context; - ContextPtr query_context; + ContextMutablePtr connection_context; + ContextMutablePtr query_context; size_t unknown_packet_in_send_data = 0; diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 55ee884646e..e8835132f8f 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -25,11 +25,9 @@ #include #include #include - #include #include - -#include +#include namespace CurrentMetrics @@ -39,6 +37,8 @@ namespace CurrentMetrics extern const Metric BrokenDistributedFilesToInsert; } +namespace fs = std::filesystem; + namespace DB { @@ -293,7 +293,7 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( , pool(std::move(pool_)) , disk(disk_) , relative_path(relative_path_) - , path(disk->getPath() + relative_path + '/') + , path(fs::path(disk->getPath()) / relative_path / "") , should_batch_inserts(storage.getContext()->getSettingsRef().distributed_directory_monitor_batch_inserts) , dir_fsync(storage.getDistributedSettingsRef().fsync_directories) , min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows) @@ -347,7 +347,7 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData() } auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - Poco::File(path).remove(true); + fs::remove_all(path); } @@ -490,16 +490,14 @@ std::map StorageDistributedDirectoryMonitor::getFiles() std::map files; size_t new_bytes_count = 0; - Poco::DirectoryIterator end; - for (Poco::DirectoryIterator it{path}; it != end; ++it) + fs::directory_iterator end; + for (fs::directory_iterator it{path}; it != end; ++it) { const auto & file_path_str = it->path(); - Poco::Path file_path{file_path_str}; - - if (!it->isDirectory() && startsWith(file_path.getExtension(), "bin")) + if (!it->is_directory() && startsWith(fs::path(file_path_str).extension(), ".bin")) { - files[parse(file_path.getBaseName())] = file_path_str; - new_bytes_count += Poco::File(file_path).getSize(); + files[parse(fs::path(file_path_str).stem())] = file_path_str; + new_bytes_count += fs::file_size(fs::path(file_path_str)); } } @@ -663,8 +661,7 @@ struct StorageDistributedDirectoryMonitor::Batch String tmp_file{parent.current_batch_file_path + ".tmp"}; auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path); - - if (Poco::File{tmp_file}.exists()) + if (fs::exists(tmp_file)) LOG_ERROR(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); { @@ -676,7 +673,7 @@ struct StorageDistributedDirectoryMonitor::Batch out.sync(); } - Poco::File{tmp_file}.renameTo(parent.current_batch_file_path); + fs::rename(tmp_file, parent.current_batch_file_path); } auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(parent.storage.getContext()->getSettingsRef()); auto connection = parent.pool->get(timeouts); @@ -757,7 +754,7 @@ struct StorageDistributedDirectoryMonitor::Batch total_bytes = 0; recovered = false; - Poco::File{parent.current_batch_file_path}.setSize(0); + fs::resize_file(parent.current_batch_file_path, 0); } void writeText(WriteBuffer & out) @@ -850,7 +847,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map { std::unordered_set file_indices_to_skip; - if (Poco::File{current_batch_file_path}.exists()) + if (fs::exists(current_batch_file_path)) { /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch. Batch batch(*this, files); @@ -951,8 +948,8 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map /// current_batch.txt will not exist if there was no send /// (this is the case when all batches that was pending has been marked as pending) - if (Poco::File{current_batch_file_path}.exists()) - Poco::File{current_batch_file_path}.remove(); + if (fs::exists(current_batch_file_path)) + fs::remove(current_batch_file_path); } } @@ -961,20 +958,18 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p const auto last_path_separator_pos = file_path.rfind('/'); const auto & base_path = file_path.substr(0, last_path_separator_pos + 1); const auto & file_name = file_path.substr(last_path_separator_pos + 1); - const auto & broken_path = base_path + "broken/"; - const auto & broken_file_path = broken_path + file_name; + const String & broken_path = fs::path(base_path) / "broken/"; + const String & broken_file_path = fs::path(broken_path) / file_name; - Poco::File{broken_path}.createDirectory(); + fs::create_directory(broken_path); auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path); - auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path + "/broken/"); - - Poco::File file(file_path); + auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, fs::path(relative_path) / "broken/"); { std::lock_guard status_lock(status_mutex); - size_t file_size = file.getSize(); + size_t file_size = fs::file_size(file_path); --status.files_count; status.bytes_count -= file_size; @@ -985,15 +980,13 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p metric_broken_files.add(); } - file.renameTo(broken_file_path); - + fs::rename(file_path, broken_file_path); LOG_ERROR(log, "Renamed `{}` to `{}`", file_path, broken_file_path); } + void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_path) { - Poco::File file(file_path); - - size_t file_size = file.getSize(); + size_t file_size = fs::file_size(file_path); { std::lock_guard status_lock(status_mutex); @@ -1002,7 +995,7 @@ void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_pat status.bytes_count -= file_size; } - file.remove(); + fs::remove(file_path); } bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) @@ -1030,7 +1023,7 @@ void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_rela { std::lock_guard status_lock(status_mutex); relative_path = new_relative_path; - path = disk->getPath() + relative_path + '/'; + path = fs::path(disk->getPath()) / relative_path / ""; } current_batch_file_path = path + "current_batch.txt"; diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index a4aa2779771..ea694fb0cfe 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -33,11 +33,10 @@ #include #include -#include - #include #include #include +#include namespace CurrentMetrics @@ -50,10 +49,11 @@ namespace ProfileEvents extern const Event DistributedSyncInsertionTimeoutExceeded; } +namespace fs = std::filesystem; + namespace DB { - namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -660,10 +660,10 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: /// hardlinking to ensure the inode is not freed until we're done { const std::string path(disk_path + data_path + *it); - Poco::File(path).createDirectory(); - const std::string tmp_path(path + "/tmp/"); - Poco::File(tmp_path).createDirectory(); + + fs::create_directory(path); + fs::create_directory(tmp_path); const std::string file_name(toString(storage.file_names_increment.get()) + ".bin"); @@ -717,7 +717,7 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: } // Create hardlink here to reuse increment number - const std::string block_file_path(path + '/' + file_name); + const std::string block_file_path(fs::path(path) / file_name); createHardLink(first_file_tmp_path, block_file_path); auto dir_sync_guard = make_directory_sync_guard(*it); } @@ -726,18 +726,18 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: /// Make hardlinks for (; it != dir_names.end(); ++it) { - const std::string path(disk_path + data_path + *it); - Poco::File(path).createDirectory(); + const std::string path(fs::path(disk_path) / (data_path + *it)); + fs::create_directory(path); - const std::string block_file_path(path + '/' + toString(storage.file_names_increment.get()) + ".bin"); + const std::string block_file_path(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin")); createHardLink(first_file_tmp_path, block_file_path); auto dir_sync_guard = make_directory_sync_guard(*it); } - auto file_size = Poco::File(first_file_tmp_path).getSize(); + auto file_size = fs::file_size(first_file_tmp_path); /// remove the temporary file, enabling the OS to reclaim inode after all threads /// have removed their corresponding files - Poco::File(first_file_tmp_path).remove(); + fs::remove(first_file_tmp_path); /// Notify auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms; diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.h b/src/Storages/Distributed/DistributedBlockOutputStream.h index f574702f35f..a17e7d6565b 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.h +++ b/src/Storages/Distributed/DistributedBlockOutputStream.h @@ -84,7 +84,7 @@ private: /// Returns the number of blocks was written for each cluster node. Uses during exception handling. std::string getCurrentStateDescription(); - ContextPtr context; + ContextMutablePtr context; StorageDistributed & storage; StorageMetadataPtr metadata_snapshot; ASTPtr query_ast; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 1428e80f9af..e3fd287bad8 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -18,7 +18,6 @@ #include #include #include - #include #include #include @@ -26,7 +25,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -257,7 +258,7 @@ Strings LSWithRegexpMatching(const String & path_for_ls, const HDFSFSPtr & fs, c { if (re2::RE2::FullMatch(file_name, matcher)) { - Strings result_part = LSWithRegexpMatching(full_path + "/", fs, suffix_with_globs.substr(next_slash)); + Strings result_part = LSWithRegexpMatching(fs::path(full_path) / "", fs, suffix_with_globs.substr(next_slash)); /// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check. std::move(result_part.begin(), result_part.end(), std::back_inserter(result)); } diff --git a/src/Storages/Kafka/KafkaBlockOutputStream.h b/src/Storages/Kafka/KafkaBlockOutputStream.h index 715ed39b8d6..9f413ae527f 100644 --- a/src/Storages/Kafka/KafkaBlockOutputStream.h +++ b/src/Storages/Kafka/KafkaBlockOutputStream.h @@ -12,7 +12,7 @@ public: explicit KafkaBlockOutputStream( StorageKafka & storage_, const StorageMetadataPtr & metadata_snapshot_, - const std::shared_ptr & context_); + const std::shared_ptr & context_); Block getHeader() const override; @@ -25,7 +25,7 @@ public: private: StorageKafka & storage; StorageMetadataPtr metadata_snapshot; - const std::shared_ptr context; + const std::shared_ptr context; ProducerBufferPtr buffer; BlockOutputStreamPtr child; }; diff --git a/src/Storages/LiveView/StorageLiveView.h b/src/Storages/LiveView/StorageLiveView.h index df09316f333..e787b7bf939 100644 --- a/src/Storages/LiveView/StorageLiveView.h +++ b/src/Storages/LiveView/StorageLiveView.h @@ -191,7 +191,7 @@ private: ASTPtr inner_query; /// stored query : SELECT * FROM ( SELECT a FROM A) ASTPtr inner_subquery; /// stored query's innermost subquery if any ASTPtr inner_blocks_query; /// query over the mergeable blocks to produce final result - ContextPtr live_view_context; + ContextMutablePtr live_view_context; Poco::Logger * log; diff --git a/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp b/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp index 7294b82f10d..69369cbc1a3 100644 --- a/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp +++ b/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes namespace { - void executeDropQuery(const StorageID & storage_id, ContextPtr context) + void executeDropQuery(const StorageID & storage_id, ContextMutablePtr context) { if (!DatabaseCatalog::instance().isTableExist(storage_id, context)) return; @@ -42,7 +42,7 @@ namespace std::unique_ptr TemporaryLiveViewCleaner::the_instance; -void TemporaryLiveViewCleaner::init(ContextPtr global_context_) +void TemporaryLiveViewCleaner::init(ContextMutablePtr global_context_) { if (the_instance) throw Exception("TemporaryLiveViewCleaner already initialized", ErrorCodes::LOGICAL_ERROR); @@ -63,7 +63,7 @@ void TemporaryLiveViewCleaner::shutdown() the_instance.reset(); } -TemporaryLiveViewCleaner::TemporaryLiveViewCleaner(ContextPtr global_context_) : WithContext(global_context_) +TemporaryLiveViewCleaner::TemporaryLiveViewCleaner(ContextMutablePtr global_context_) : WithMutableContext(global_context_) { } diff --git a/src/Storages/LiveView/TemporaryLiveViewCleaner.h b/src/Storages/LiveView/TemporaryLiveViewCleaner.h index 9b31bf9c999..3fe0079a46f 100644 --- a/src/Storages/LiveView/TemporaryLiveViewCleaner.h +++ b/src/Storages/LiveView/TemporaryLiveViewCleaner.h @@ -14,7 +14,7 @@ struct StorageID; /// This class removes temporary live views in the background thread when it's possible. /// There should only a single instance of this class. -class TemporaryLiveViewCleaner : WithContext +class TemporaryLiveViewCleaner : WithMutableContext { public: static TemporaryLiveViewCleaner & instance() { return *the_instance; } @@ -23,7 +23,7 @@ public: void addView(const std::shared_ptr & view); /// Should be called once. - static void init(ContextPtr global_context_); + static void init(ContextMutablePtr global_context_); static void shutdown(); void startup(); @@ -31,7 +31,7 @@ public: private: friend std::unique_ptr::deleter_type; - TemporaryLiveViewCleaner(ContextPtr global_context_); + TemporaryLiveViewCleaner(ContextMutablePtr global_context_); ~TemporaryLiveViewCleaner(); void backgroundThreadFunc(); diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 5ccb2ca8c27..529fc9f54b8 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -13,11 +13,11 @@ #include #include #include - -#include #include +namespace fs = std::filesystem; + namespace CurrentMetrics { extern const Metric ReplicatedSend; @@ -289,7 +289,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( { String file_name = it.first; - String path = part->getFullRelativePath() + file_name; + String path = fs::path(part->getFullRelativePath()) / file_name; UInt64 size = disk->getFileSize(path); @@ -339,15 +339,15 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB { String file_name = it.first; - String metadata_file = disk->getPath() + part->getFullRelativePath() + file_name; + String metadata_file = fs::path(disk->getPath()) / part->getFullRelativePath() / file_name; - Poco::File metadata(metadata_file); + fs::path metadata(metadata_file); - if (!metadata.exists()) + if (!fs::exists(metadata)) throw Exception("S3 metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA); - if (!metadata.isFile()) + if (!fs::is_regular_file(metadata)) throw Exception("S3 metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA); - UInt64 file_size = metadata.getSize(); + UInt64 file_size = fs::file_size(metadata); writeStringBinary(it.first, out); writeBinary(file_size, out); @@ -570,7 +570,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( readUUIDText(part_uuid, in); auto storage_id = data.getStorageID(); - String new_part_path = part_type == "InMemory" ? "memory" : data.getFullPathOnDisk(reservation->getDisk()) + part_name + "/"; + String new_part_path = part_type == "InMemory" ? "memory" : fs::path(data.getFullPathOnDisk(reservation->getDisk())) / part_name / ""; auto entry = data.getContext()->getReplicatedFetchList().insert( storage_id.getDatabaseName(), storage_id.getTableName(), part_info.partition_id, part_name, new_part_path, @@ -681,13 +681,13 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( /// File must be inside "absolute_part_path" directory. /// Otherwise malicious ClickHouse replica may force us to write to arbitrary path. - String absolute_file_path = Poco::Path(part_download_path + file_name).absolute().toString(); - if (!startsWith(absolute_file_path, Poco::Path(part_download_path).absolute().toString())) + String absolute_file_path = fs::weakly_canonical(fs::path(part_download_path) / file_name); + if (!startsWith(absolute_file_path, fs::weakly_canonical(part_download_path).string())) throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + part_download_path + ")." " This may happen if we are trying to download part from malicious replica or logical error.", ErrorCodes::INSECURE_PATH); - auto file_out = disk->writeFile(part_download_path + file_name); + auto file_out = disk->writeFile(fs::path(part_download_path) / file_name); HashingWriteBuffer hashing_out(*file_out); copyData(in, hashing_out, file_size, blocker.getCounter()); @@ -704,7 +704,7 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( readPODBinary(expected_hash, in); if (expected_hash != hashing_out.getHash()) - throw Exception("Checksum mismatch for file " + fullPath(disk, part_download_path + file_name) + " transferred from " + replica_path, + throw Exception("Checksum mismatch for file " + fullPath(disk, (fs::path(part_download_path) / file_name).string()) + " transferred from " + replica_path, ErrorCodes::CHECKSUM_DOESNT_MATCH); if (file_name != "checksums.txt" && @@ -811,7 +811,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; String part_relative_path = String(to_detached ? "detached/" : "") + tmp_prefix + part_name; - String part_download_path = data.getRelativeDataPath() + part_relative_path + "/"; + String part_download_path = fs::path(data.getRelativeDataPath()) / part_relative_path / ""; if (disk->exists(part_download_path)) throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); @@ -833,7 +833,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( readStringBinary(file_name, in); readBinary(file_size, in); - String data_path = part_download_path + file_name; + String data_path = fs::path(part_download_path) / file_name; String metadata_file = fullPath(disk, data_path); { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 8a35ce3dcb8..703cf32f743 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -69,7 +69,7 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Dis hyperrectangle.reserve(minmax_idx_size); for (size_t i = 0; i < minmax_idx_size; ++i) { - String file_name = part_path + "minmax_" + escapeForFileName(minmax_column_names[i]) + ".idx"; + String file_name = fs::path(part_path) / ("minmax_" + escapeForFileName(minmax_column_names[i]) + ".idx"); auto file = openForReading(disk_, file_name); auto serialization = minmax_column_types[i]->getDefaultSerialization(); @@ -111,7 +111,7 @@ void IMergeTreeDataPart::MinMaxIndex::store( String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; auto serialization = data_types.at(i)->getDefaultSerialization(); - auto out = disk_->writeFile(part_path + file_name); + auto out = disk_->writeFile(fs::path(part_path) / file_name); HashingWriteBuffer out_hashing(*out); serialization->serializeBinary(hyperrectangle[i].left, out_hashing); serialization->serializeBinary(hyperrectangle[i].right, out_hashing); @@ -560,7 +560,7 @@ String IMergeTreeDataPart::getFullPath() const if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return storage.getFullPathOnDisk(volume->getDisk()) + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + return fs::path(storage.getFullPathOnDisk(volume->getDisk())) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; } String IMergeTreeDataPart::getFullRelativePath() const @@ -568,7 +568,7 @@ String IMergeTreeDataPart::getFullRelativePath() const if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return storage.relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + return fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; } void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) @@ -643,7 +643,7 @@ void IMergeTreeDataPart::loadIndex() loaded_index[i]->reserve(index_granularity.getMarksCount()); } - String index_path = getFullRelativePath() + "primary.idx"; + String index_path = fs::path(getFullRelativePath()) / "primary.idx"; auto index_file = openForReading(volume->getDisk(), index_path); size_t marks_count = index_granularity.getMarksCount(); @@ -678,7 +678,7 @@ NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const return {}; NameSet result = {"checksums.txt", "columns.txt"}; - String default_codec_path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME; + String default_codec_path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; if (volume->getDisk()->exists(default_codec_path)) result.emplace(DEFAULT_COMPRESSION_CODEC_FILE_NAME); @@ -695,7 +695,7 @@ void IMergeTreeDataPart::loadDefaultCompressionCodec() return; } - String path = getFullRelativePath() + DEFAULT_COMPRESSION_CODEC_FILE_NAME; + String path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; if (!volume->getDisk()->exists(path)) { default_codec = detectDefaultCompressionCodec(); @@ -756,7 +756,7 @@ CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const { if (path_to_data_file.empty()) { - String candidate_path = getFullRelativePath() + ISerialization::getFileNameForStream(part_column, substream_path) + ".bin"; + String candidate_path = fs::path(getFullRelativePath()) / (ISerialization::getFileNameForStream(part_column, substream_path) + ".bin"); /// We can have existing, but empty .bin files. Example: LowCardinality(Nullable(...)) columns and column_name.dict.null.bin file. if (volume->getDisk()->exists(candidate_path) && volume->getDisk()->getFileSize(candidate_path) != 0) @@ -822,7 +822,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() void IMergeTreeDataPart::loadChecksums(bool require) { - const String path = getFullRelativePath() + "checksums.txt"; + const String path = fs::path(getFullRelativePath()) / "checksums.txt"; if (volume->getDisk()->exists(path)) { @@ -847,11 +847,11 @@ void IMergeTreeDataPart::loadChecksums(bool require) checksums = checkDataPart(shared_from_this(), false); { - auto out = volume->getDisk()->writeFile(getFullRelativePath() + "checksums.txt.tmp", 4096); + auto out = volume->getDisk()->writeFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", 4096); checksums.write(*out); } - volume->getDisk()->moveFile(getFullRelativePath() + "checksums.txt.tmp", getFullRelativePath() + "checksums.txt"); + volume->getDisk()->moveFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", fs::path(getFullRelativePath()) / "checksums.txt"); bytes_on_disk = checksums.getTotalSizeOnDisk(); } @@ -859,7 +859,7 @@ void IMergeTreeDataPart::loadChecksums(bool require) void IMergeTreeDataPart::loadRowsCount() { - String path = getFullRelativePath() + "count.txt"; + String path = fs::path(getFullRelativePath()) / "count.txt"; if (index_granularity.empty()) { rows_count = 0; @@ -960,7 +960,7 @@ void IMergeTreeDataPart::loadRowsCount() void IMergeTreeDataPart::loadTTLInfos() { - String path = getFullRelativePath() + "ttl.txt"; + String path = fs::path(getFullRelativePath()) / "ttl.txt"; if (volume->getDisk()->exists(path)) { auto in = openForReading(volume->getDisk(), path); @@ -987,7 +987,7 @@ void IMergeTreeDataPart::loadTTLInfos() void IMergeTreeDataPart::loadUUID() { - String path = getFullRelativePath() + UUID_FILE_NAME; + String path = fs::path(getFullRelativePath()) / UUID_FILE_NAME; if (volume->getDisk()->exists(path)) { @@ -1000,7 +1000,7 @@ void IMergeTreeDataPart::loadUUID() void IMergeTreeDataPart::loadColumns(bool require) { - String path = getFullRelativePath() + "columns.txt"; + String path = fs::path(getFullRelativePath()) / "columns.txt"; auto metadata_snapshot = storage.getInMemoryMetadataPtr(); if (parent_part) metadata_snapshot = metadata_snapshot->projections.get(name).metadata; @@ -1015,7 +1015,7 @@ void IMergeTreeDataPart::loadColumns(bool require) /// If there is no file with a list of columns, write it down. for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAllPhysical()) - if (volume->getDisk()->exists(getFullRelativePath() + getFileNameForColumn(column) + ".bin")) + if (volume->getDisk()->exists(fs::path(getFullRelativePath()) / (getFileNameForColumn(column) + ".bin"))) loaded_columns.push_back(column); if (columns.empty()) @@ -1053,7 +1053,7 @@ UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DiskPtr & disk_, const disk_->listFiles(from, files); UInt64 res = 0; for (const auto & file : files) - res += calculateTotalSizeOnDisk(disk_, from + "/" + file); + res += calculateTotalSizeOnDisk(disk_, fs::path(from) / file); return res; } @@ -1063,7 +1063,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ assertOnDisk(); String from = getFullRelativePath(); - String to = storage.relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + new_relative_path + "/"; + String to = fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / new_relative_path / ""; if (!volume->getDisk()->exists(from)) throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); @@ -1124,8 +1124,8 @@ void IMergeTreeDataPart::remove(bool keep_s3) const * And a race condition can happen that will lead to "File not found" error here. */ - String from = storage.relative_data_path + relative_path; - String to = storage.relative_data_path + "delete_tmp_" + name; + fs::path from = fs::path(storage.relative_data_path) / relative_path; + fs::path to = fs::path(storage.relative_data_path) / ("delete_tmp_" + name); // TODO directory delete_tmp_ is never removed if server crashes before returning from this function auto disk = volume->getDisk(); @@ -1134,7 +1134,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_WARNING(storage.log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it.", fullPath(disk, to)); try { - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } catch (...) { @@ -1147,11 +1147,14 @@ void IMergeTreeDataPart::remove(bool keep_s3) const { disk->moveDirectory(from, to); } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - LOG_ERROR(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(disk, to)); - - return; + if (e.code() == std::errc::no_such_file_or_directory) + { + LOG_ERROR(storage.log, "Directory {} (part to remove) doesn't exist or one of nested files has gone. Most likely this is due to manual removing. This should be discouraged. Ignoring.", fullPath(disk, to)); + return; + } + throw; } // Record existing projection directories so we don't remove them twice @@ -1166,7 +1169,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const if (checksums.empty()) { /// If the part is not completely written, we cannot use fast path by listing files. - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } else { @@ -1181,17 +1184,17 @@ void IMergeTreeDataPart::remove(bool keep_s3) const for (const auto & [file, _] : checksums.files) { if (projection_directories.find(file) == projection_directories.end()) - disk->removeSharedFile(to + "/" + file, keep_s3); + disk->removeSharedFile(fs::path(to) / file, keep_s3); } #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(to + "/" + file, keep_s3); + disk->removeSharedFile(fs::path(to) / file, keep_s3); - disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); - disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); + disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); + disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); disk->removeDirectory(to); } @@ -1201,7 +1204,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false)); - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", keep_s3); } } } @@ -1268,7 +1271,7 @@ String IMergeTreeDataPart::getRelativePathForPrefix(const String & prefix) const { res = (prefix.empty() ? "" : prefix + "_") + name + (try_no ? "_try" + DB::toString(try_no) : ""); - if (!volume->getDisk()->exists(getFullRelativePath() + res)) + if (!volume->getDisk()->exists(fs::path(getFullRelativePath()) / res)) return res; LOG_WARNING(storage.log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res); @@ -1291,11 +1294,11 @@ void IMergeTreeDataPart::renameToDetached(const String & prefix) const void IMergeTreeDataPart::makeCloneInDetached(const String & prefix, const StorageMetadataPtr & /*metadata_snapshot*/) const { - String destination_path = storage.relative_data_path + getRelativePathForDetachedPart(prefix); + String destination_path = fs::path(storage.relative_data_path) / getRelativePathForDetachedPart(prefix); /// Backup is not recursive (max_level is 0), so do not copy inner directories localBackup(volume->getDisk(), getFullRelativePath(), destination_path, 0); - volume->getDisk()->removeFileIfExists(destination_path + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME); + volume->getDisk()->removeFileIfExists(fs::path(destination_path) / DELETE_ON_DESTROY_MARKER_FILE_NAME); } void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & directory_name) const @@ -1307,16 +1310,16 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di if (directory_name.empty()) throw Exception("Can not clone data part " + name + " to empty directory.", ErrorCodes::LOGICAL_ERROR); - String path_to_clone = storage.relative_data_path + directory_name + '/'; + String path_to_clone = fs::path(storage.relative_data_path) / directory_name / ""; - if (disk->exists(path_to_clone + relative_path)) + if (disk->exists(fs::path(path_to_clone) / relative_path)) { LOG_WARNING(storage.log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); - disk->removeRecursive(path_to_clone + relative_path + '/'); + disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone); - volume->getDisk()->removeFileIfExists(path_to_clone + '/' + DELETE_ON_DESTROY_MARKER_FILE_NAME); + volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / DELETE_ON_DESTROY_MARKER_FILE_NAME); } void IMergeTreeDataPart::checkConsistencyBase() const @@ -1370,19 +1373,19 @@ void IMergeTreeDataPart::checkConsistencyBase() const /// Check that the primary key index is not empty. if (!pk.column_names.empty()) - check_file_not_empty(volume->getDisk(), path + "primary.idx"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - check_file_not_empty(volume->getDisk(), path + "count.txt"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "count.txt"); if (metadata_snapshot->hasPartitionKey()) - check_file_not_empty(volume->getDisk(), path + "partition.dat"); + check_file_not_empty(volume->getDisk(), fs::path(path) / "partition.dat"); if (!parent_part) { for (const String & col_name : storage.getMinMaxColumnsNames(partition_key)) - check_file_not_empty(volume->getDisk(), path + "minmax_" + escapeForFileName(col_name) + ".idx"); + check_file_not_empty(volume->getDisk(), fs::path(path) / ("minmax_" + escapeForFileName(col_name) + ".idx")); } } } @@ -1477,7 +1480,7 @@ String IMergeTreeDataPart::getUniqueId() const auto disk = volume->getDisk(); if (disk->getType() == DB::DiskType::Type::S3) - id = disk->getUniqueId(getFullRelativePath() + "checksums.txt"); + id = disk->getUniqueId(fs::path(getFullRelativePath()) / "checksums.txt"); if (id.empty()) throw Exception("Can't get unique S3 object", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index a8a49680dd7..53640b41507 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -17,8 +17,6 @@ #include #include -#include - #include namespace zkutil diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 52d3e7ca9ab..14187564536 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index 7cdfd8e566d..2810385e9e5 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -83,7 +83,7 @@ private: void createNode() { shutdown_called = false; - node = EphemeralNodeHolder::createSequential(path + "/leader_election-", zookeeper, identifier); + node = EphemeralNodeHolder::createSequential(fs::path(path) / "leader_election-", zookeeper, identifier); std::string node_path = node->getPath(); node_name = node_path.substr(node_path.find_last_of('/') + 1); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0ed3f89b2c3..ef25828b574 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -52,8 +52,6 @@ #include #include -#include - #include #include @@ -67,8 +65,11 @@ #include #include #include +#include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event RejectedInserts; @@ -136,7 +137,7 @@ MergeTreeData::MergeTreeData( const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr storage_settings_, @@ -144,7 +145,7 @@ MergeTreeData::MergeTreeData( bool attach, BrokenPartCallback broken_part_callback_) : IStorage(table_id_) - , WithContext(context_->getGlobalContext()) + , WithMutableContext(context_->getGlobalContext()) , merging_params(merging_params_) , require_part_metadata(require_part_metadata_) , relative_data_path(relative_data_path_) @@ -210,8 +211,8 @@ MergeTreeData::MergeTreeData( for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { disk->createDirectories(path); - disk->createDirectories(path + MergeTreeData::DETACHED_DIR_NAME); - auto current_version_file_path = path + MergeTreeData::FORMAT_VERSION_FILE_NAME; + disk->createDirectories(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); + String current_version_file_path = fs::path(path) / MergeTreeData::FORMAT_VERSION_FILE_NAME; if (disk->exists(current_version_file_path)) { if (!version_file.first.empty()) @@ -225,7 +226,7 @@ MergeTreeData::MergeTreeData( /// If not choose any if (version_file.first.empty()) - version_file = {relative_data_path + MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()}; + version_file = {fs::path(relative_data_path) / MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()}; bool version_file_exists = version_file.second->exists(version_file.first); @@ -927,8 +928,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) auto part = createPart(part_name, part_info, single_disk_volume, part_name); bool broken = false; - String part_path = relative_data_path + "/" + part_name; - String marker_path = part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; + String part_path = fs::path(relative_data_path) / part_name; + String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; if (part_disk_ptr->exists(marker_path)) { LOG_WARNING(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name); @@ -1016,7 +1017,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) else has_adaptive_parts.store(true, std::memory_order_relaxed); - part->modification_time = part_disk_ptr->getLastModified(relative_data_path + part_name).epochTime(); + part->modification_time = part_disk_ptr->getLastModified(fs::path(relative_data_path) / part_name).epochTime(); /// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later part->setState(DataPartState::Committed); @@ -1160,9 +1161,14 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life disk->removeRecursive(it->path()); } } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - /// If the file is already deleted, do nothing. + if (e.code() == std::errc::no_such_file_or_directory) + { + /// If the file is already deleted, do nothing. + } + else + throw; } } } @@ -1387,11 +1393,7 @@ void MergeTreeData::clearEmptyParts() for (const auto & part : parts) { if (part->rows_count == 0) - { - ASTPtr literal = std::make_shared(part->name); - /// If another replica has already started drop, it's ok, no need to throw. - dropPartition(literal, /* detach = */ false, /*drop_part = */ true, getContext(), /* throw_if_noop = */ false); - } + dropPartNoWaitNoThrow(part->name); } } @@ -1448,10 +1450,15 @@ void MergeTreeData::dropAllData() { disk->removeRecursive(path); } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - /// If the file is already deleted, log the error message and do nothing. - tryLogCurrentException(__PRETTY_FUNCTION__); + if (e.code() == std::errc::no_such_file_or_directory) + { + /// If the file is already deleted, log the error message and do nothing. + tryLogCurrentException(__PRETTY_FUNCTION__); + } + else + throw; } } @@ -1474,8 +1481,8 @@ void MergeTreeData::dropIfEmpty() for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { /// Non recursive, exception is thrown if there are more files. - disk->removeFileIfExists(path + MergeTreeData::FORMAT_VERSION_FILE_NAME); - disk->removeDirectory(path + MergeTreeData::DETACHED_DIR_NAME); + disk->removeFileIfExists(fs::path(path) / MergeTreeData::FORMAT_VERSION_FILE_NAME); + disk->removeDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); disk->removeDirectory(path); } } @@ -1925,7 +1932,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part) const { MergeTreeDataPartType type; - auto full_path = relative_data_path + (parent_part ? parent_part->relative_path + "/" : "") + relative_path + "/"; + auto full_path = fs::path(relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(volume->getDisk(), full_path); if (mrk_ext) @@ -1978,7 +1985,7 @@ void MergeTreeData::changeSettings( { auto disk = new_storage_policy->getDiskByName(disk_name); disk->createDirectories(relative_data_path); - disk->createDirectories(relative_data_path + MergeTreeData::DETACHED_DIR_NAME); + disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME); } /// FIXME how would that be done while reloading configuration??? @@ -2007,7 +2014,7 @@ void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const old_and_new_names.push_back({old_name, new_name}); for (const auto & [path, disk] : storage.getRelativeDataPathsWithDisks()) { - for (auto it = disk->iterateDirectory(path + source_dir); it->isValid(); it->next()) + for (auto it = disk->iterateDirectory(fs::path(path) / source_dir); it->isValid(); it->next()) { if (it->name() == old_name) { @@ -2029,8 +2036,8 @@ void MergeTreeData::PartsTemporaryRename::tryRenameAll() if (old_name.empty() || new_name.empty()) throw DB::Exception("Empty part name. Most likely it's a bug.", ErrorCodes::INCORRECT_FILE_NAME); const auto & [path, disk] = old_part_name_to_path_and_disk[old_name]; - const auto full_path = path + source_dir; /// for old_name - disk->moveFile(full_path + old_name, full_path + new_name); + const auto full_path = fs::path(path) / source_dir; /// for old_name + disk->moveFile(fs::path(full_path) / old_name, fs::path(full_path) / new_name); } catch (...) { @@ -2054,8 +2061,8 @@ MergeTreeData::PartsTemporaryRename::~PartsTemporaryRename() try { const auto & [path, disk] = old_part_name_to_path_and_disk[old_name]; - const auto full_path = path + source_dir; /// for old_name - disk->moveFile(full_path + new_name, full_path + old_name); + const String full_path = fs::path(path) / source_dir; /// for old_name + disk->moveFile(fs::path(full_path) / new_name, fs::path(full_path) / old_name); } catch (...) { @@ -2752,7 +2759,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) addPartContributionToDataVolume(part_copy); auto disk = original_active_part->volume->getDisk(); - String marker_path = original_active_part->getFullRelativePath() + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; + String marker_path = fs::path(original_active_part->getFullRelativePath()) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; try { disk->createFile(marker_path); @@ -2904,9 +2911,8 @@ void MergeTreeData::checkPartitionCanBeDropped(const ASTPtr & partition) getContext()->checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, partition_size); } -void MergeTreeData::checkPartCanBeDropped(const ASTPtr & part_ast) +void MergeTreeData::checkPartCanBeDropped(const String & part_name) { - String part_name = part_ast->as().value.safeGet(); auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Committed}); if (!part) throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in committed state", part_name); @@ -3041,12 +3047,20 @@ Pipe MergeTreeData::alterPartition( switch (command.type) { case PartitionCommand::DROP_PARTITION: + { if (command.part) - checkPartCanBeDropped(command.partition); + { + auto part_name = command.partition->as().value.safeGet(); + checkPartCanBeDropped(part_name); + dropPart(part_name, command.detach, query_context); + } else + { checkPartitionCanBeDropped(command.partition); - dropPartition(command.partition, command.detach, command.part, query_context); - break; + dropPartition(command.partition, command.detach, query_context); + } + } + break; case PartitionCommand::DROP_DETACHED_PARTITION: dropDetached(command.partition, command.part, query_context); @@ -3315,7 +3329,7 @@ MergeTreeData::getDetachedParts() const for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { - for (auto it = disk->iterateDirectory(path + MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next()) + for (auto it = disk->iterateDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next()) { res.emplace_back(); auto & part = res.back(); @@ -3370,7 +3384,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr for (auto & [old_name, new_name] : renamed_parts.old_and_new_names) { const auto & [path, disk] = renamed_parts.old_part_name_to_path_and_disk[old_name]; - disk->removeRecursive(path + "detached/" + new_name + "/"); + disk->removeRecursive(fs::path(path) / "detached" / new_name / ""); LOG_DEBUG(log, "Dropped detached part {}", old_name); old_name.clear(); } @@ -3431,8 +3445,8 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const if (!containing_part.empty() && containing_part != name) // TODO maybe use PartsTemporaryRename here? - disk->moveDirectory(relative_data_path + source_dir + name, - relative_data_path + source_dir + "inactive_" + name); + disk->moveDirectory(fs::path(relative_data_path) / source_dir / name, + fs::path(relative_data_path) / source_dir / ("inactive_" + name)); else renamed_parts.addPart(name, "attaching_" + name); } @@ -4294,12 +4308,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( const auto & src_relative_data_path = src_part_in_memory->storage.relative_data_path; auto flushed_part_path = src_part_in_memory->getRelativePathForPrefix(tmp_part_prefix); src_part_in_memory->flushToDisk(src_relative_data_path, flushed_part_path, metadata_snapshot); - src_part_path = src_relative_data_path + flushed_part_path + "/"; + src_part_path = fs::path(src_relative_data_path) / flushed_part_path / ""; } LOG_DEBUG(log, "Cloning part {} to {}", fullPath(disk, src_part_path), fullPath(disk, dst_part_path)); localBackup(disk, src_part_path, dst_part_path); - disk->removeFileIfExists(dst_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + disk->removeFileIfExists(fs::path(dst_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); auto dst_data_part = createPart(dst_part_name, dst_part_info, single_disk_volume, tmp_dst_part_name); @@ -4411,10 +4425,10 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( const String & with_name, ContextPtr local_context) { - String clickhouse_path = Poco::Path(local_context->getPath()).makeAbsolute().toString(); - String default_shadow_path = clickhouse_path + "shadow/"; - Poco::File(default_shadow_path).createDirectories(); - auto increment = Increment(default_shadow_path + "increment.txt").get(true); + String clickhouse_path = fs::canonical(local_context->getPath()); + String default_shadow_path = fs::path(clickhouse_path) / "shadow/"; + fs::create_directories(default_shadow_path); + auto increment = Increment(fs::path(default_shadow_path) / "increment.txt").get(true); const String shadow_path = "shadow/"; @@ -4422,7 +4436,7 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( const auto data_parts = getDataParts(); String backup_name = (!with_name.empty() ? escapeForFileName(with_name) : toString(increment)); - String backup_path = shadow_path + backup_name + "/"; + String backup_path = fs::path(shadow_path) / backup_name / ""; for (const auto & disk : getStoragePolicy()->getDisks()) disk->onFreeze(backup_path); @@ -4439,20 +4453,20 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( part->volume->getDisk()->createDirectories(backup_path); - String backup_part_path = backup_path + relative_data_path + part->relative_path; + String backup_part_path = fs::path(backup_path) / relative_data_path / part->relative_path; if (auto part_in_memory = asInMemoryPart(part)) - part_in_memory->flushToDisk(backup_path + relative_data_path, part->relative_path, metadata_snapshot); + part_in_memory->flushToDisk(fs::path(backup_path) / relative_data_path, part->relative_path, metadata_snapshot); else localBackup(part->volume->getDisk(), part->getFullRelativePath(), backup_part_path); - part->volume->getDisk()->removeFileIfExists(backup_part_path + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + part->volume->getDisk()->removeFileIfExists(fs::path(backup_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); part->is_frozen.store(true, std::memory_order_relaxed); result.push_back(PartitionCommandResultInfo{ .partition_id = part->info.partition_id, .part_name = part->name, - .backup_path = part->volume->getDisk()->getPath() + backup_path, - .part_backup_path = part->volume->getDisk()->getPath() + backup_part_path, + .backup_path = fs::path(part->volume->getDisk()->getPath()) / backup_path, + .part_backup_path = fs::path(part->volume->getDisk()->getPath()) / backup_part_path, .backup_name = backup_name, }); ++parts_processed; @@ -4481,7 +4495,7 @@ PartitionCommandsResultInfo MergeTreeData::unfreezeAll( PartitionCommandsResultInfo MergeTreeData::unfreezePartitionsByMatcher(MatcherFn matcher, const String & backup_name, ContextPtr) { - auto backup_path = std::filesystem::path("shadow") / escapeForFileName(backup_name) / relative_data_path; + auto backup_path = fs::path("shadow") / escapeForFileName(backup_name) / relative_data_path; LOG_DEBUG(log, "Unfreezing parts by path {}", backup_path.generic_string()); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index a80d7eae139..4f33aa30bdc 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -113,7 +113,7 @@ namespace ErrorCodes /// - MergeTreeDataWriter /// - MergeTreeDataMergerMutator -class MergeTreeData : public IStorage, public WithContext +class MergeTreeData : public IStorage, public WithMutableContext { public: /// Function to call if the part is suspected to contain corrupt data. @@ -353,7 +353,7 @@ public: MergeTreeData(const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, @@ -613,7 +613,7 @@ public: void checkPartitionCanBeDropped(const ASTPtr & partition) override; - void checkPartCanBeDropped(const ASTPtr & part); + void checkPartCanBeDropped(const String & part_name); Pipe alterPartition( const StorageMetadataPtr & metadata_snapshot, @@ -993,7 +993,11 @@ protected: // Partition helpers bool canReplacePartition(const DataPartPtr & src_part) const; - virtual void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr context, bool throw_if_noop = true) = 0; + /// Tries to drop part in background without any waits or throwing exceptions in case of errors. + virtual void dropPartNoWaitNoThrow(const String & part_name) = 0; + + virtual void dropPart(const String & part_name, bool detach, ContextPtr context) = 0; + virtual void dropPartition(const ASTPtr & partition, bool detach, ContextPtr context) = 0; virtual PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr context) = 0; virtual void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr context) = 0; virtual void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr context) = 0; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index f5edde01478..846ad7b026d 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -2268,7 +2268,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( if (need_remove_expired_values) { /// Write a file with ttl infos in json format. - auto out_ttl = disk->writeFile(new_data_part->getFullRelativePath() + "ttl.txt", 4096); + auto out_ttl = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "ttl.txt", 4096); HashingWriteBuffer out_hashing(*out_ttl); new_data_part->ttl_infos.write(out_hashing); new_data_part->checksums.files["ttl.txt"].file_size = out_hashing.count(); @@ -2277,7 +2277,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( { /// Write file with checksums. - auto out_checksums = disk->writeFile(new_data_part->getFullRelativePath() + "checksums.txt", 4096); + auto out_checksums = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "checksums.txt", 4096); new_data_part->checksums.write(*out_checksums); } /// close fd @@ -2288,7 +2288,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( { /// Write a file with a description of columns. - auto out_columns = disk->writeFile(new_data_part->getFullRelativePath() + "columns.txt", 4096); + auto out_columns = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "columns.txt", 4096); new_data_part->getColumns().writeText(*out_columns); } /// close fd diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 8f5c4b6a848..1df97dc9241 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -7,7 +7,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 5c1a3b01804..7b1641a0537 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 473f2e598a9..e929bfc6862 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 84378faa513..4eebd729dd8 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -1,5 +1,4 @@ #include "MergeTreeDataPartWide.h" -#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index b1d2f33c659..2d8d4146197 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -3,8 +3,6 @@ #include #include -#include - #include #include #include @@ -144,11 +142,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( const auto & settings = context->getSettingsRef(); if (!query_info.projection) { - if (settings.allow_experimental_projection_optimization && settings.force_optimize_projection - && !metadata_snapshot->projections.empty()) - throw Exception("No projection is used when allow_experimental_projection_optimization = 1", ErrorCodes::PROJECTION_NOT_USED); - - return readFromParts( + auto plan = readFromParts( data.getDataPartsVector(), column_names_to_return, metadata_snapshot, @@ -159,6 +153,14 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( num_streams, max_block_numbers_to_read, query_info.merge_tree_data_select_cache.get()); + + if (plan->isInitialized() && settings.allow_experimental_projection_optimization && settings.force_optimize_projection + && !metadata_snapshot->projections.empty()) + throw Exception( + "No projection is used when allow_experimental_projection_optimization = 1 and force_optimize_projection = 1", + ErrorCodes::PROJECTION_NOT_USED); + + return plan; } LOG_DEBUG( diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 3bf33d45802..0b05650b42c 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index ed0e7e55fc8..6da0a822f7f 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -1,6 +1,8 @@ #include #include -#include + + +namespace fs = std::filesystem; namespace DB { @@ -17,8 +19,7 @@ std::optional MergeTreeIndexGranularityInfo::getMarksExtensionFromF { for (DiskDirectoryIteratorPtr it = disk->iterateDirectory(path_to_part); it->isValid(); it->next()) { - Poco::Path path(it->path()); - const auto & ext = "." + path.getExtension(); + const auto & ext = fs::path(it->path()).extension(); if (ext == getNonAdaptiveMrkExtension() || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE) || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT)) diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index c5a99b128e9..717179e5f26 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp index 49c4e93eb1d..2aefb3df2be 100644 --- a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp @@ -4,9 +4,6 @@ #include #include -#include -#include - #include diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index f9e3883d5e2..5b77ac9ec4a 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -206,18 +206,18 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt /// Try to fetch part from S3 without copy and fallback to default copy /// if it's not possible moving_part.part->assertOnDisk(); - String path_to_clone = data->getRelativeDataPath() + directory_to_move + "/"; + String path_to_clone = fs::path(data->getRelativeDataPath()) / directory_to_move / ""; String relative_path = part->relative_path; if (disk->exists(path_to_clone + relative_path)) { LOG_WARNING(log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); - disk->removeRecursive(path_to_clone + relative_path + "/"); + disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); - bool is_fetched = data->tryToFetchIfShared(*part, disk, path_to_clone + "/" + part->name); + bool is_fetched = data->tryToFetchIfShared(*part, disk, fs::path(path_to_clone) / part->name); if (!is_fetched) - part->volume->getDisk()->copy(data->getRelativeDataPath() + relative_path + "/", disk, path_to_clone); - part->volume->getDisk()->removeFileIfExists(path_to_clone + "/" + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + part->volume->getDisk()->copy(fs::path(data->getRelativeDataPath()) / relative_path / "", disk, path_to_clone); + part->volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); } else { @@ -226,7 +226,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt auto single_disk_volume = std::make_shared("volume_" + part->name, moving_part.reserved_space->getDisk(), 0); MergeTreeData::MutableDataPartPtr cloned_part = - data->createPart(part->name, single_disk_volume, directory_to_move + '/' + part->name); + data->createPart(part->name, single_disk_volume, fs::path(directory_to_move) / part->name); LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index da28f75b57f..783a37cce60 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp index 5ee4aa555e6..cf5c850daa1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp @@ -3,7 +3,6 @@ #include #include #include -#include namespace DB { diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 3655743d41a..6de01caa833 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -1,6 +1,5 @@ #include #include -#include #include @@ -123,7 +122,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { if (new_part->uuid != UUIDHelpers::Nil) { - auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::UUID_FILE_NAME, 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / IMergeTreeDataPart::UUID_FILE_NAME, 4096); HashingWriteBuffer out_hashing(*out); writeUUIDText(new_part->uuid, out_hashing); checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count(); @@ -142,7 +141,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); - auto count_out = volume->getDisk()->writeFile(part_path + "count.txt", 4096); + auto count_out = volume->getDisk()->writeFile(fs::path(part_path) / "count.txt", 4096); HashingWriteBuffer count_out_hashing(*count_out); writeIntText(rows_count, count_out_hashing); count_out_hashing.next(); @@ -157,7 +156,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( if (!new_part->ttl_infos.empty()) { /// Write a file with ttl infos in json format. - auto out = volume->getDisk()->writeFile(part_path + "ttl.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "ttl.txt", 4096); HashingWriteBuffer out_hashing(*out); new_part->ttl_infos.write(out_hashing); checksums.files["ttl.txt"].file_size = out_hashing.count(); @@ -171,7 +170,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { /// Write a file with a description of columns. - auto out = volume->getDisk()->writeFile(part_path + "columns.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "columns.txt", 4096); part_columns.writeText(*out); out->finalize(); if (sync) @@ -192,7 +191,7 @@ void MergedBlockOutputStream::finalizePartOnDisk( { /// Write file with checksums. - auto out = volume->getDisk()->writeFile(part_path + "checksums.txt", 4096); + auto out = volume->getDisk()->writeFile(fs::path(part_path) / "checksums.txt", 4096); checksums.write(*out); out->finalize(); if (sync) diff --git a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp index 7a4c3d530a7..4c187109ac6 100644 --- a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp +++ b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp @@ -347,7 +347,7 @@ void PartMovesBetweenShardsOrchestrator::stepEntry(const Entry & entry, zkutil:: { { ReplicatedMergeTreeLogEntry log_entry; - if (storage.dropPart(zk, entry.part_name, log_entry,false, false)) + if (storage.dropPartImpl(zk, entry.part_name, log_entry, false, false)) storage.waitForAllReplicasToProcessLogEntry(log_entry, true); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 30569e53f64..439d4970aa8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -62,7 +62,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr & bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { - auto queue_path = replica_path + "/queue"; + String queue_path = fs::path(replica_path) / "queue"; LOG_DEBUG(log, "Loading queue from {}", queue_path); bool updated = false; @@ -74,7 +74,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) /// Reset batch size on initialization to recover from possible errors of too large batch size. current_multi_batch_size = 1; - String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); std::unordered_set already_loaded_paths; @@ -101,7 +101,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) futures.reserve(children.size()); for (const String & child : children) - futures.emplace_back(child, zookeeper->asyncGet(queue_path + "/" + child)); + futures.emplace_back(child, zookeeper->asyncGet(fs::path(queue_path) / child)); for (auto & future : futures) { @@ -116,7 +116,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) updated = true; } - zookeeper->tryGet(replica_path + "/mutation_pointer", mutation_pointer); + zookeeper->tryGet(fs::path(replica_path) / "mutation_pointer", mutation_pointer); } updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, {}); @@ -399,7 +399,7 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep if (!need_remove_from_zk) return; - auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); + auto code = zookeeper->tryRemove(fs::path(replica_path) / "queue" / entry->znode_name); if (code != Coordination::Error::ZOK) LOG_ERROR(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, Coordination::errorMessage(code)); @@ -455,7 +455,7 @@ bool ReplicatedMergeTreeQueue::remove(zkutil::ZooKeeperPtr zookeeper, const Stri notifySubscribers(queue_size); - zookeeper->tryRemove(replica_path + "/queue/" + found->znode_name); + zookeeper->tryRemove(fs::path(replica_path) / "queue" / found->znode_name); updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); return true; @@ -474,14 +474,14 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper if (pull_log_blocker.isCancelled()) throw Exception("Log pulling is cancelled", ErrorCodes::ABORTED); - String index_str = zookeeper->get(replica_path + "/log_pointer"); + String index_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); UInt64 index; /// The version of "/log" is modified when new entries to merge/mutate/drop appear. Coordination::Stat stat; - zookeeper->get(zookeeper_path + "/log", &stat); + zookeeper->get(fs::path(zookeeper_path) / "log", &stat); - Strings log_entries = zookeeper->getChildrenWatch(zookeeper_path + "/log", nullptr, watch_callback); + Strings log_entries = zookeeper->getChildrenWatch(fs::path(zookeeper_path) / "log", nullptr, watch_callback); /// We update mutations after we have loaded the list of log entries, but before we insert them /// in the queue. @@ -494,7 +494,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper /// If we do not already have a pointer to the log, put a pointer to the first entry in it. index = log_entries.empty() ? 0 : parse(std::min_element(log_entries.begin(), log_entries.end())->substr(strlen("log-"))); - zookeeper->set(replica_path + "/log_pointer", toString(index)); + zookeeper->set(fs::path(replica_path) / "log_pointer", toString(index)); } else { @@ -541,7 +541,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper futures.reserve(end - begin); for (auto it = begin; it != end; ++it) - futures.emplace_back(*it, zookeeper->asyncGet(zookeeper_path + "/log/" + *it)); + futures.emplace_back(*it, zookeeper->asyncGet(fs::path(zookeeper_path) / "log" / *it)); /// Simultaneously add all new entries to the queue and move the pointer to the log. @@ -558,7 +558,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper copied_entries.emplace_back(LogEntry::parse(res.data, res.stat)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", res.data, zkutil::CreateMode::PersistentSequential)); + fs::path(replica_path) / "queue/queue-", res.data, zkutil::CreateMode::PersistentSequential)); const auto & entry = *copied_entries.back(); if (entry.type == LogEntry::GET_PART || entry.type == LogEntry::ATTACH_PART) @@ -573,11 +573,11 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper } ops.emplace_back(zkutil::makeSetRequest( - replica_path + "/log_pointer", toString(last_entry_index + 1), -1)); + fs::path(replica_path) / "log_pointer", toString(last_entry_index + 1), -1)); if (min_unprocessed_insert_time_changed) ops.emplace_back(zkutil::makeSetRequest( - replica_path + "/min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1)); + fs::path(replica_path) / "min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1)); auto responses = zookeeper->multi(ops); @@ -655,7 +655,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C { std::lock_guard lock(update_mutations_mutex); - Strings entries_in_zk = zookeeper->getChildrenWatch(zookeeper_path + "/mutations", nullptr, watch_callback); + Strings entries_in_zk = zookeeper->getChildrenWatch(fs::path(zookeeper_path) / "mutations", nullptr, watch_callback); StringSet entries_in_zk_set(entries_in_zk.begin(), entries_in_zk.end()); /// Compare with the local state, delete obsolete entries and determine which new entries to load. @@ -712,7 +712,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C std::vector> futures; for (const String & entry : entries_to_load) - futures.emplace_back(zookeeper->asyncGet(zookeeper_path + "/mutations/" + entry)); + futures.emplace_back(zookeeper->asyncGet(fs::path(zookeeper_path) / "mutations" / entry)); std::vector new_mutations; for (size_t i = 0; i < entries_to_load.size(); ++i) @@ -796,7 +796,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( { std::lock_guard lock(update_mutations_mutex); - auto rc = zookeeper->tryRemove(zookeeper_path + "/mutations/" + mutation_id); + auto rc = zookeeper->tryRemove(fs::path(zookeeper_path) / "mutations" / mutation_id); if (rc == Coordination::Error::ZOK) LOG_DEBUG(log, "Removed mutation {} from ZooKeeper.", mutation_id); @@ -940,12 +940,12 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( { if ((*it)->currently_executing) to_wait.push_back(*it); - auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name); + auto code = zookeeper->tryRemove(fs::path(replica_path) / "queue" / (*it)->znode_name); /// FIXME it's probably unsafe to remove entries non-atomically /// when this method called directly from alter query (not from replication queue task), /// because entries will be lost if ALTER fails. if (code != Coordination::Error::ZOK) - LOG_INFO(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, Coordination::errorMessage(code)); + LOG_INFO(log, "Couldn't remove {}: {}", (fs::path(replica_path) / "queue" / (*it)->znode_name).string(), Coordination::errorMessage(code)); updateStateOnQueueEntryRemoval( *it, /* is_successful = */ false, @@ -1554,7 +1554,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep if (!finished.empty()) { - zookeeper->set(replica_path + "/mutation_pointer", finished.back()->znode_name); + zookeeper->set(fs::path(replica_path) / "mutation_pointer", finished.back()->znode_name); std::lock_guard lock(state_mutex); @@ -1750,22 +1750,22 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( } /// Load current quorum status. - auto quorum_status_future = zookeeper->asyncTryGet(queue.zookeeper_path + "/quorum/status"); + auto quorum_status_future = zookeeper->asyncTryGet(fs::path(queue.zookeeper_path) / "quorum" / "status"); /// Load current inserts std::unordered_set lock_holder_paths; - for (const String & entry : zookeeper->getChildren(queue.zookeeper_path + "/temp")) + for (const String & entry : zookeeper->getChildren(fs::path(queue.zookeeper_path) / "temp")) { if (startsWith(entry, "abandonable_lock-")) - lock_holder_paths.insert(queue.zookeeper_path + "/temp/" + entry); + lock_holder_paths.insert(fs::path(queue.zookeeper_path) / "temp" / entry); } if (!lock_holder_paths.empty()) { - Strings partitions = zookeeper->getChildren(queue.zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(queue.zookeeper_path) / "block_numbers"); std::vector> lock_futures; for (const String & partition : partitions) - lock_futures.push_back(zookeeper->asyncGetChildren(queue.zookeeper_path + "/block_numbers/" + partition)); + lock_futures.push_back(zookeeper->asyncGetChildren(fs::path(queue.zookeeper_path) / "block_numbers" / partition)); struct BlockInfoInZooKeeper { @@ -1786,7 +1786,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( if (startsWith(entry, "block-")) { Int64 block_number = parse(entry.substr(strlen("block-"))); - String zk_path = queue.zookeeper_path + "/block_numbers/" + partitions[i] + "/" + entry; + String zk_path = fs::path(queue.zookeeper_path) / "block_numbers" / partitions[i] / entry; block_infos.emplace_back( BlockInfoInZooKeeper{partitions[i], block_number, zk_path, zookeeper->asyncTryGet(zk_path)}); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 6b7fb3bf17f..7288396b58b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -238,7 +238,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() auto zookeeper = storage.getZooKeeper(); String quorum_str; - if (zookeeper->tryGet(storage.zookeeper_path + "/quorum/status", quorum_str)) + if (zookeeper->tryGet(fs::path(storage.zookeeper_path) / "quorum" / "status", quorum_str)) { ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str); @@ -251,12 +251,12 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() } Strings part_names; - String parallel_quorum_parts_path = storage.zookeeper_path + "/quorum/parallel"; + String parallel_quorum_parts_path = fs::path(storage.zookeeper_path) / "quorum" / "parallel"; if (zookeeper->tryGetChildren(parallel_quorum_parts_path, part_names) == Coordination::Error::ZOK) { for (auto & part_name : part_names) { - if (zookeeper->tryGet(parallel_quorum_parts_path + "/" + part_name, quorum_str)) + if (zookeeper->tryGet(fs::path(parallel_quorum_parts_path) / part_name, quorum_str)) { ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str); if (!quorum_entry.replicas.count(storage.replica_name) @@ -278,7 +278,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() /// How other replicas can access this one. ReplicatedMergeTreeAddress address = storage.getReplicatedMergeTreeAddress(); - String is_active_path = storage.replica_path + "/is_active"; + String is_active_path = fs::path(storage.replica_path) / "is_active"; /** If the node is marked as active, but the mark is made in the same instance, delete it. * This is possible only when session in ZooKeeper expires. @@ -302,7 +302,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() /// Simultaneously declare that this replica is active, and update the host. Coordination::Requests ops; ops.emplace_back(zkutil::makeCreateRequest(is_active_path, active_node_identifier, zkutil::CreateMode::Ephemeral)); - ops.emplace_back(zkutil::makeSetRequest(storage.replica_path + "/host", address.toString(), -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(storage.replica_path) / "host", address.toString(), -1)); try { @@ -311,7 +311,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() catch (const Coordination::Exception & e) { String existing_replica_host; - zookeeper->tryGet(storage.replica_path + "/host", existing_replica_host); + zookeeper->tryGet(fs::path(storage.replica_path) / "host", existing_replica_host); if (existing_replica_host.empty()) existing_replica_host = "without host node"; diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 8f0847fc106..8a234833da7 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include @@ -70,7 +69,7 @@ IMergeTreeDataPart::Checksums checkDataPart( NamesAndTypesList columns_txt; { - auto buf = disk->readFile(path + "columns.txt"); + auto buf = disk->readFile(fs::path(path) / "columns.txt"); columns_txt.readText(*buf); assertEOF(*buf); } @@ -231,9 +230,9 @@ IMergeTreeDataPart::Checksums checkDataPart( /// Checksums from the rest files listed in checksums.txt. May be absent. If present, they are subsequently compared with the actual data checksums. IMergeTreeDataPart::Checksums checksums_txt; - if (require_checksums || disk->exists(path + "checksums.txt")) + if (require_checksums || disk->exists(fs::path(path) / "checksums.txt")) { - auto buf = disk->readFile(path + "checksums.txt"); + auto buf = disk->readFile(fs::path(path) / "checksums.txt"); checksums_txt.read(*buf); assertEOF(*buf); } diff --git a/src/Storages/MergeTree/localBackup.cpp b/src/Storages/MergeTree/localBackup.cpp index 7d7dacaeaf1..2e02481e065 100644 --- a/src/Storages/MergeTree/localBackup.cpp +++ b/src/Storages/MergeTree/localBackup.cpp @@ -29,7 +29,7 @@ static void localBackupImpl(const DiskPtr & disk, const String & source_path, co for (auto it = disk->iterateDirectory(source_path); it->isValid(); it->next()) { auto source = it->path(); - auto destination = destination_path + "/" + it->name(); + auto destination = fs::path(destination_path) / it->name(); if (!disk->isDirectory(source)) { @@ -74,13 +74,16 @@ void localBackup(const DiskPtr & disk, const String & source_path, const String continue; } - catch (const Poco::FileNotFoundException &) + catch (const fs::filesystem_error & e) { - ++try_no; - if (try_no == max_tries) - throw; - - continue; + if (e.code() == std::errc::no_such_file_or_directory) + { + ++try_no; + if (try_no == max_tries) + throw; + continue; + } + throw; } break; diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index b2a06018054..dd48b23ecc3 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -172,15 +172,15 @@ ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const metadata.sorting_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {}); metadata.primary_key = KeyDescription::getKeyFromAST({}, metadata.columns, query_context); } - if (query_select.orderBy()) + if (query.orderBy()) throw Exception( "When aggregation is used in projection, ORDER BY cannot be specified", ErrorCodes::ILLEGAL_PROJECTION); } else { result.type = ProjectionDescription::Type::Normal; - metadata.sorting_key = KeyDescription::getSortingKeyFromAST(query_select.orderBy(), metadata.columns, query_context, {}); - metadata.primary_key = KeyDescription::getKeyFromAST(query_select.orderBy(), metadata.columns, query_context); + metadata.sorting_key = KeyDescription::getSortingKeyFromAST(query.orderBy(), metadata.columns, query_context, {}); + metadata.primary_key = KeyDescription::getKeyFromAST(query.orderBy(), metadata.columns, query_context); } metadata.primary_key.definition_ast = nullptr; result.metadata = std::make_shared(metadata); diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 6f1724c6c2e..129e231e46a 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -79,7 +79,7 @@ protected: std::unique_ptr rabbitmq_settings_); private: - ContextPtr rabbitmq_context; + ContextMutablePtr rabbitmq_context; std::unique_ptr rabbitmq_settings; const String exchange_name; diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index af6f01a0ab5..70cc173e38a 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -27,14 +27,16 @@ #include #include -#include -#include #include #include #include #include +#include + + +namespace fs = std::filesystem; namespace DB { @@ -254,7 +256,7 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, rocksdb_dir = context_->getPath() + relative_data_path_; if (!attach) { - Poco::File(rocksdb_dir).createDirectories(); + fs::create_directories(rocksdb_dir); } initDb(); } @@ -262,8 +264,8 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, void StorageEmbeddedRocksDB::truncate(const ASTPtr &, const StorageMetadataPtr & , ContextPtr, TableExclusiveLockHolder &) { rocksdb_ptr->Close(); - Poco::File(rocksdb_dir).remove(true); - Poco::File(rocksdb_dir).createDirectories(); + fs::remove_all(rocksdb_dir); + fs::create_directories(rocksdb_dir); initDb(); } diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 1ce74b16f46..5aeaff590e1 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -185,7 +185,7 @@ void StorageDictionary::startup() bool lazy_load = global_context->getConfigRef().getBool("dictionaries_lazy_load", true); if (!lazy_load) { - auto & external_dictionaries_loader = global_context->getExternalDictionariesLoader(); + const auto & external_dictionaries_loader = global_context->getExternalDictionariesLoader(); /// reloadConfig() is called here to force loading the dictionary. external_dictionaries_loader.reloadConfig(getStorageID().getInternalDictionaryName()); @@ -220,7 +220,7 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id) configuration->setString("dictionary.database", new_table_id.database_name); configuration->setString("dictionary.name", new_table_id.table_name); - auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); + const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); external_dictionaries_loader.reloadConfig(getStorageID().getInternalDictionaryName()); auto result = external_dictionaries_loader.getLoadResult(getStorageID().getInternalDictionaryName()); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3a3291c6c48..256875454db 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -76,6 +76,8 @@ #include +namespace fs = std::filesystem; + namespace { const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_HAS_SHARDING_KEY = 1; @@ -864,7 +866,7 @@ StoragePolicyPtr StorageDistributed::getStoragePolicy() const void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk) { const std::string path(disk->getPath() + relative_data_path); - Poco::File{path}.createDirectories(); + fs::create_directories(path); std::filesystem::directory_iterator begin(path); std::filesystem::directory_iterator end; diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 836d8bae7f3..5ca423b449a 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -31,7 +31,7 @@ static void checkAllTypesAreAllowedInTable(const NamesAndTypesList & names_and_t } -ContextPtr StorageFactory::Arguments::getContext() const +ContextMutablePtr StorageFactory::Arguments::getContext() const { auto ptr = context.lock(); if (!ptr) @@ -39,7 +39,7 @@ ContextPtr StorageFactory::Arguments::getContext() const return ptr; } -ContextPtr StorageFactory::Arguments::getLocalContext() const +ContextMutablePtr StorageFactory::Arguments::getLocalContext() const { auto ptr = local_context.lock(); if (!ptr) @@ -59,8 +59,8 @@ void StorageFactory::registerStorage(const std::string & name, CreatorFn creator StoragePtr StorageFactory::get( const ASTCreateQuery & query, const String & relative_data_path, - ContextPtr local_context, - ContextPtr context, + ContextMutablePtr local_context, + ContextMutablePtr context, const ColumnsDescription & columns, const ConstraintsDescription & constraints, bool has_force_restore_data_flag) const diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index e04e5459dda..bdc57bfdc6d 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -39,16 +39,16 @@ public: /// Relative to from server config (possibly of some of some for *MergeTree) const String & relative_data_path; const StorageID & table_id; - ContextWeakPtr local_context; - ContextWeakPtr context; + ContextWeakMutablePtr local_context; + ContextWeakMutablePtr context; const ColumnsDescription & columns; const ConstraintsDescription & constraints; bool attach; bool has_force_restore_data_flag; const String & comment; - ContextPtr getContext() const; - ContextPtr getLocalContext() const; + ContextMutablePtr getContext() const; + ContextMutablePtr getLocalContext() const; }; /// Analog of the IStorage::supports*() helpers @@ -81,8 +81,8 @@ public: StoragePtr get( const ASTCreateQuery & query, const String & relative_data_path, - ContextPtr local_context, - ContextPtr context, + ContextMutablePtr local_context, + ContextMutablePtr context, const ColumnsDescription & columns, const ConstraintsDescription & constraints, bool has_force_restore_data_flag) const; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index cfb4934dae0..aac1b708567 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -28,9 +28,6 @@ #include #include -#include -#include - #include #include #include @@ -39,6 +36,7 @@ #include #include + namespace fs = std::filesystem; namespace DB @@ -78,10 +76,9 @@ std::vector listFilesWithRegexpMatching(const std::string & path_fo std::vector result; const std::string prefix_without_globs = path_for_ls + for_match.substr(1, end_of_path_without_globs); - if (!fs::exists(fs::path(prefix_without_globs))) - { + if (!fs::exists(prefix_without_globs)) return result; - } + const fs::directory_iterator end; for (fs::directory_iterator it(prefix_without_globs); it != end; ++it) { @@ -102,7 +99,7 @@ std::vector listFilesWithRegexpMatching(const std::string & path_fo if (re2::RE2::FullMatch(file_name, matcher)) { /// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check. - Strings result_part = listFilesWithRegexpMatching(full_path + "/", suffix_with_globs.substr(next_slash)); + Strings result_part = listFilesWithRegexpMatching(fs::path(full_path) / "", suffix_with_globs.substr(next_slash)); std::move(result_part.begin(), result_part.end(), std::back_inserter(result)); } } @@ -125,21 +122,20 @@ void checkCreationIsAllowed(ContextPtr context_global, const std::string & db_di if (!startsWith(table_path, db_dir_path) && table_path != "/dev/null") throw Exception("File is not inside " + db_dir_path, ErrorCodes::DATABASE_ACCESS_DENIED); - Poco::File table_path_poco_file = Poco::File(table_path); - if (table_path_poco_file.exists() && table_path_poco_file.isDirectory()) + if (fs::exists(table_path) && fs::is_directory(table_path)) throw Exception("File must not be a directory", ErrorCodes::INCORRECT_FILE_NAME); } } Strings StorageFile::getPathsList(const String & table_path, const String & user_files_path, ContextPtr context) { - String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); - Poco::Path poco_path = Poco::Path(table_path); - if (poco_path.isRelative()) - poco_path = Poco::Path(user_files_absolute_path, poco_path); + fs::path user_files_absolute_path = fs::weakly_canonical(user_files_path); + fs::path fs_table_path(table_path); + if (fs_table_path.is_relative()) + fs_table_path = user_files_absolute_path / fs_table_path; Strings paths; - const String path = poco_path.absolute().toString(); + const String path = fs::weakly_canonical(fs_table_path); if (path.find_first_of("*?{") == std::string::npos) paths.push_back(path); else @@ -204,8 +200,8 @@ StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArgu if (args.format_name == "Distributed") throw Exception("Distributed format is allowed only with explicit file path", ErrorCodes::INCORRECT_FILE_NAME); - String table_dir_path = base_path + relative_table_dir_path + "/"; - Poco::File(table_dir_path).createDirectories(); + String table_dir_path = fs::path(base_path) / relative_table_dir_path / ""; + fs::create_directories(table_dir_path); paths = {getTablePath(table_dir_path, format_name)}; } @@ -456,7 +452,7 @@ Pipe StorageFile::read( if (use_table_fd) /// need to call ctr BlockInputStream paths = {""}; /// when use fd, paths are empty else - if (paths.size() == 1 && !Poco::File(paths[0]).exists()) + if (paths.size() == 1 && !fs::exists(paths[0])) { if (context->getSettingsRef().engine_file_empty_if_not_exists) return Pipe(std::make_shared(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()))); @@ -599,7 +595,7 @@ BlockOutputStreamPtr StorageFile::write( if (!paths.empty()) { path = paths[0]; - Poco::File(Poco::Path(path).makeParent()).createDirectories(); + fs::create_directories(fs::path(path).parent_path()); } return std::make_shared( @@ -636,8 +632,8 @@ void StorageFile::rename(const String & new_path_to_table_data, const StorageID if (path_new == paths[0]) return; - Poco::File(Poco::Path(path_new).parent()).createDirectories(); - Poco::File(paths[0]).renameTo(path_new); + fs::create_directories(fs::path(path_new).parent_path()); + fs::rename(paths[0], path_new); paths[0] = std::move(path_new); renameInMemory(new_table_id); @@ -659,7 +655,7 @@ void StorageFile::truncate( } else { - if (!Poco::File(paths[0]).exists()) + if (!fs::exists(paths[0])) return; if (0 != ::truncate(paths[0].c_str(), 0)) diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 5be8378b53e..91758931ed1 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -1,10 +1,6 @@ #pragma once #include - -#include -#include - #include #include diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index f0129e95495..4064bc83d23 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -14,9 +14,7 @@ #include #include #include - #include /// toLower -#include #include #include @@ -93,7 +91,9 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join) /// TODO: check key columns - /// Some HACK to remove wrong names qualifiers: table.column -> column. + /// Set names qualifiers: table.column -> column + /// It's required because storage join stores non-qualified names + /// Qualifies will be added by join implementation (HashJoin) analyzed_join->setRightKeys(key_names); HashJoinPtr join_clone = std::make_shared(analyzed_join, metadata_snapshot->getSampleBlock().sortColumns()); diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 67bd6b21c3f..a66b24ff931 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -54,7 +54,7 @@ StorageMaterializedView::StorageMaterializedView( const ASTCreateQuery & query, const ColumnsDescription & columns_, bool attach_) - : IStorage(table_id_), WithContext(local_context->getGlobalContext()) + : IStorage(table_id_), WithMutableContext(local_context->getGlobalContext()) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); @@ -228,7 +228,7 @@ static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, if (auto txn = current_context->getZooKeeperMetadataTransaction()) { /// For Replicated database - drop_context->setQueryContext(current_context); + drop_context->setQueryContext(std::const_pointer_cast(current_context)); drop_context->initZooKeeperMetadataTransaction(txn, true); } InterpreterDropQuery drop_interpreter(ast_drop_query, drop_context); @@ -431,7 +431,12 @@ Strings StorageMaterializedView::getDataPaths() const ActionLock StorageMaterializedView::getActionLock(StorageActionBlockType type) { - return has_inner_table ? getTargetTable()->getActionLock(type) : ActionLock{}; + if (has_inner_table) + { + if (auto target_table = tryGetTargetTable()) + return target_table->getActionLock(type); + } + return ActionLock{}; } void registerStorageMaterializedView(StorageFactory & factory) diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 8f9c8a9d3f1..8e2b8bdc9f5 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -12,7 +12,7 @@ namespace DB { -class StorageMaterializedView final : public ext::shared_ptr_helper, public IStorage, WithContext +class StorageMaterializedView final : public ext::shared_ptr_helper, public IStorage, WithMutableContext { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5a84e0c3901..15d520c13aa 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -332,7 +332,7 @@ Pipe StorageMerge::createSources( const Block & header, const StorageWithLockAndName & storage_with_lock, Names & real_column_names, - ContextPtr modified_context, + ContextMutablePtr modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams) diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 7c5a50af486..b9d44bfa27e 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -92,7 +92,7 @@ protected: const Block & header, const StorageWithLockAndName & storage_with_lock, Names & real_column_names, - ContextPtr modified_context, + ContextMutablePtr modified_context, size_t streams_num, bool has_table_virtual_column, bool concat_streams = false); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index d656c7c088d..3d3530825d9 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -62,7 +62,7 @@ StorageMergeTree::StorageMergeTree( const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, bool attach, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr storage_settings_, @@ -1240,56 +1240,66 @@ MergeTreeDataPartPtr StorageMergeTree::outdatePart(const String & part_name, boo } } -void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr local_context, bool throw_if_noop) +void StorageMergeTree::dropPartNoWaitNoThrow(const String & part_name) { + if (auto part = outdatePart(part_name, /*force=*/ false)) + dropPartsImpl({part}, /*detach=*/ false); + + /// Else nothing to do, part was removed in some different way +} + +void StorageMergeTree::dropPart(const String & part_name, bool detach, ContextPtr /*query_context*/) +{ + if (auto part = outdatePart(part_name, /*force=*/ true)) + dropPartsImpl({part}, detach); +} + +void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, ContextPtr local_context) +{ + DataPartsVector parts_to_remove; + /// New scope controls lifetime of merge_blocker. { - MergeTreeData::DataPartsVector parts_to_remove; - auto metadata_snapshot = getInMemoryMetadataPtr(); + /// Asks to complete merges and does not allow them to start. + /// This protects against "revival" of data for a removed partition after completion of merge. + auto merge_blocker = stopMergesAndWait(); + String partition_id = getPartitionIDFromQuery(partition, local_context); + parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); - if (drop_part) - { - auto part = outdatePart(partition->as().value.safeGet(), throw_if_noop); - /// Nothing to do, part was removed in some different way - if (!part) - return; - - parts_to_remove.push_back(part); - } - else - { - /// Asks to complete merges and does not allow them to start. - /// This protects against "revival" of data for a removed partition after completion of merge. - auto merge_blocker = stopMergesAndWait(); - String partition_id = getPartitionIDFromQuery(partition, local_context); - parts_to_remove = getDataPartsVectorInPartition(MergeTreeDataPartState::Committed, partition_id); - - /// TODO should we throw an exception if parts_to_remove is empty? - removePartsFromWorkingSet(parts_to_remove, true); - } - - if (detach) - { - /// If DETACH clone parts to detached/ directory - /// NOTE: no race with background cleanup until we hold pointers to parts - for (const auto & part : parts_to_remove) - { - LOG_INFO(log, "Detaching {}", part->relative_path); - part->makeCloneInDetached("", metadata_snapshot); - } - } - - if (deduplication_log) - { - for (const auto & part : parts_to_remove) - deduplication_log->dropPart(part->info); - } - - if (detach) - LOG_INFO(log, "Detached {} parts.", parts_to_remove.size()); - else - LOG_INFO(log, "Removed {} parts.", parts_to_remove.size()); + /// TODO should we throw an exception if parts_to_remove is empty? + removePartsFromWorkingSet(parts_to_remove, true); } + dropPartsImpl(std::move(parts_to_remove), detach); +} + +void StorageMergeTree::dropPartsImpl(DataPartsVector && parts_to_remove, bool detach) +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + + if (detach) + { + /// If DETACH clone parts to detached/ directory + /// NOTE: no race with background cleanup until we hold pointers to parts + for (const auto & part : parts_to_remove) + { + LOG_INFO(log, "Detaching {}", part->relative_path); + part->makeCloneInDetached("", metadata_snapshot); + } + } + + if (deduplication_log) + { + for (const auto & part : parts_to_remove) + deduplication_log->dropPart(part->info); + } + + if (detach) + LOG_INFO(log, "Detached {} parts.", parts_to_remove.size()); + else + LOG_INFO(log, "Removed {} parts.", parts_to_remove.size()); + + /// Need to destroy part objects before clearing them from filesystem. + parts_to_remove.clear(); clearOldPartsFromFilesystem(); } @@ -1509,8 +1519,8 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_ auto disk = part->volume->getDisk(); String part_path = part->getFullRelativePath(); /// If the checksums file is not present, calculate the checksums and write them to disk. - String checksums_path = part_path + "checksums.txt"; - String tmp_checksums_path = part_path + "checksums.txt.tmp"; + String checksums_path = fs::path(part_path) / "checksums.txt"; + String tmp_checksums_path = fs::path(part_path) / "checksums.txt.tmp"; if (part->isStoredOnDisk() && !disk->exists(checksums_path)) { try diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 53199e1595a..72fbb37bdd9 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -211,7 +211,10 @@ private: void clearOldMutations(bool truncate = false); // Partition helpers - void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr context, bool throw_if_noop) override; + void dropPartNoWaitNoThrow(const String & part_name) override; + void dropPart(const String & part_name, bool detach, ContextPtr context) override; + void dropPartition(const ASTPtr & partition, bool detach, ContextPtr context) override; + void dropPartsImpl(DataPartsVector && parts_to_remove, bool detach); PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr context) override; void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr context) override; @@ -250,7 +253,7 @@ protected: const String & relative_data_path_, const StorageInMemoryMetadata & metadata, bool attach, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 319e04f2424..3702cd0ce21 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -66,9 +66,12 @@ #include #include #include +#include #include +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event ReplicatedPartMerges; @@ -253,7 +256,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, @@ -272,7 +275,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , zookeeper_name(extractZooKeeperName(zookeeper_path_)) , zookeeper_path(extractZooKeeperPath(zookeeper_path_)) , replica_name(replica_name_) - , replica_path(zookeeper_path + "/replicas/" + replica_name_) + , replica_path(fs::path(zookeeper_path) / "replicas" / replica_name_) , reader(*this) , writer(*this) , merger_mutator(*this, getContext()->getSettingsRef().background_pool_size) @@ -493,14 +496,14 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( /// Mutation maybe killed or whole replica was deleted. /// Wait event will unblock at this moment. Coordination::Stat exists_stat; - if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id, &exists_stat, wait_event)) + if (!getZooKeeper()->exists(fs::path(zookeeper_path) / "mutations" / mutation_id, &exists_stat, wait_event)) { throw Exception(ErrorCodes::UNFINISHED, "Mutation {} was killed, manually removed or table was dropped", mutation_id); } auto zookeeper = getZooKeeper(); /// Replica could be inactive. - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (!zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) { LOG_WARNING(log, "Replica {} is not active during mutation. Mutation will be done asynchronously when replica becomes active.", replica); @@ -508,7 +511,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( break; } - String mutation_pointer = zookeeper_path + "/replicas/" + replica + "/mutation_pointer"; + String mutation_pointer = fs::path(zookeeper_path) / "replicas" / replica / "mutation_pointer"; std::string mutation_pointer_value; /// Replica could be removed if (!zookeeper->tryGet(mutation_pointer, mutation_pointer_value, nullptr, wait_event)) @@ -523,16 +526,17 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( if (wait_event->tryWait(1000)) continue; - /// Here we check mutation for errors or kill on local replica. If they happen on this replica + /// Here we check mutation for errors on local replica. If they happen on this replica /// they will happen on each replica, so we can check only in-memory info. auto mutation_status = queue.getIncompleteMutationsStatus(mutation_id); - if (!mutation_status || !mutation_status->latest_fail_reason.empty()) + /// If mutation status is empty, than local replica may just not loaded it into memory. + if (mutation_status && !mutation_status->latest_fail_reason.empty()) break; } /// It maybe already removed from zk, but local in-memory mutations /// state was not updated. - if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id)) + if (!getZooKeeper()->exists(fs::path(zookeeper_path) / "mutations" / mutation_id)) { throw Exception(ErrorCodes::UNFINISHED, "Mutation {} was killed, manually removed or table was dropped", mutation_id); } @@ -877,12 +881,12 @@ bool StorageReplicatedMergeTree::removeTableNodesFromZooKeeper(zkutil::ZooKeeper for (const auto & child : children) if (child != "dropped") - zookeeper->tryRemoveRecursive(zookeeper_path + "/" + child); + zookeeper->tryRemoveRecursive(fs::path(zookeeper_path) / child); Coordination::Requests ops; Coordination::Responses responses; ops.emplace_back(zkutil::makeRemoveRequest(metadata_drop_lock->getPath(), -1)); - ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/dropped", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(zookeeper_path) / "dropped", -1)); ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path, -1)); code = zookeeper->tryMulti(ops, responses); @@ -921,12 +925,12 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr ReplicatedMergeTreeTableMetadata old_metadata(*this, metadata_snapshot); Coordination::Stat metadata_stat; - String metadata_str = zookeeper->get(zookeeper_prefix + "/metadata", &metadata_stat); + String metadata_str = zookeeper->get(fs::path(zookeeper_prefix) / "metadata", &metadata_stat); auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); old_metadata.checkEquals(metadata_from_zk, metadata_snapshot->getColumns(), getContext()); Coordination::Stat columns_stat; - auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_prefix + "/columns", &columns_stat)); + auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(fs::path(zookeeper_prefix) / "columns", &columns_stat)); const ColumnsDescription & old_columns = metadata_snapshot->getColumns(); if (columns_from_zk != old_columns) @@ -1071,7 +1075,7 @@ static time_t tryGetPartCreateTime(zkutil::ZooKeeperPtr & zookeeper, const Strin /// We get creation time of part, if it still exists (was not merged, for example). Coordination::Stat stat; String unused; - if (zookeeper->tryGet(replica_path + "/parts/" + part_name, unused, &stat)) + if (zookeeper->tryGet(fs::path(replica_path) / "parts" / part_name, unused, &stat)) res = stat.ctime / 1000; return res; @@ -1082,7 +1086,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) { auto zookeeper = getZooKeeper(); - Strings expected_parts_vec = zookeeper->getChildren(replica_path + "/parts"); + Strings expected_parts_vec = zookeeper->getChildren(fs::path(replica_path) / "parts"); /// Parts in ZK. NameSet expected_parts(expected_parts_vec.begin(), expected_parts_vec.end()); @@ -1185,7 +1189,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) exists_futures.reserve(parts_to_fetch.size()); for (const String & part_name : parts_to_fetch) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -1214,7 +1218,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) /// We assume that this occurs before the queue is loaded (queue.initialize). ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(replica_path) / "queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); enqueue_futures.emplace_back(zookeeper->asyncMulti(ops)); } @@ -1261,13 +1265,13 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( part->getColumns(), part->checksums); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); bool has_been_already_added = false; for (const String & replica : replicas) { - String current_part_path = zookeeper_path + "/replicas/" + replica + "/parts/" + part_name; + String current_part_path = fs::path(zookeeper_path) / "replicas" / replica / "parts" / part_name; String part_zk_str; if (!zookeeper->tryGet(current_part_path, part_zk_str)) @@ -1288,9 +1292,9 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: String checksums_str; /// Let's check that the node's version with the columns did not change while we were reading the checksums. /// This ensures that the columns and the checksum refer to the same - if (!zookeeper->tryGet(current_part_path + "/columns", columns_str, &columns_stat_before) || - !zookeeper->tryGet(current_part_path + "/checksums", checksums_str) || - !zookeeper->exists(current_part_path + "/columns", &columns_stat_after) || + if (!zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str, &columns_stat_before) || + !zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str) || + !zookeeper->exists(fs::path(current_part_path) / "columns", &columns_stat_after) || columns_stat_before.version != columns_stat_after.version) { LOG_INFO(log, "Not checking checksums of part {} with replica {} because part changed while we were reading its checksums", part_name, replica); @@ -1324,7 +1328,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (!has_been_already_added) { const auto storage_settings_ptr = getSettings(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; //ops.emplace_back(zkutil::makeCheckRequest( // zookeeper_path + "/columns", expected_columns_version)); @@ -1339,14 +1343,15 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - part_path + "/columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); + fs::path(part_path) / "columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - part_path + "/checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); + fs::path(part_path) / "checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); } } else { - LOG_WARNING(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", replica_path + "/parts/" + part_name); + LOG_WARNING(log, "checkPartAndAddToZooKeeper: node {} already exists. Will not commit any nodes.", + (fs::path(replica_path) / "parts" / part_name).string()); } } @@ -1414,7 +1419,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo const String part_new_name = actual_part_info.getPartName(); for (const DiskPtr & disk : getStoragePolicy()->getDisks()) - for (const auto it = disk->iterateDirectory(relative_data_path + "detached/"); it->isValid(); it->next()) + for (const auto it = disk->iterateDirectory(fs::path(relative_data_path) / "detached/"); it->isValid(); it->next()) { MergeTreePartInfo part_info; @@ -1423,7 +1428,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo continue; const String part_old_name = part_info.getPartName(); - const String part_path = "detached/" + part_old_name; + const String part_path = fs::path("detached") / part_old_name; const VolumePtr volume = std::make_shared("volume_" + part_old_name, disk); @@ -1478,7 +1483,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) existing_part = getActiveContainingPart(entry.new_part_name); /// Even if the part is local, it (in exceptional cases) may not be in ZooKeeper. Let's check that it is there. - if (existing_part && getZooKeeper()->exists(replica_path + "/parts/" + existing_part->name)) + if (existing_part && getZooKeeper()->exists(fs::path(replica_path) / "parts" / existing_part->name)) { if (!is_get_or_attach || entry.source_replica != replica_name) LOG_DEBUG(log, "Skipping action for part {} because part {} already exists.", @@ -1513,7 +1518,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) /// Perhaps we don't need this part, because during write with quorum, the quorum has failed /// (see below about `/quorum/failed_parts`). - if (entry.quorum && getZooKeeper()->exists(zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name)) + if (entry.quorum && getZooKeeper()->exists(fs::path(zookeeper_path) / "quorum" / "failed_parts" / entry.new_part_name)) { LOG_DEBUG(log, "Skipping action for part {} because quorum for that part was failed.", entry.new_part_name); return true; /// NOTE Deletion from `virtual_parts` is not done, but it is only necessary for merge. @@ -1983,14 +1988,14 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); Coordination::Requests ops; for (const auto & path_part : replicas) { Coordination::Stat stat; - String path = zookeeper_path + "/replicas/" + path_part + "/host"; + String path = fs::path(zookeeper_path) / "replicas" / path_part / "host"; zookeeper->get(path, &stat); ops.emplace_back(zkutil::makeCheckRequest(path, stat.version)); } @@ -2004,8 +2009,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (replica.empty()) { Coordination::Stat quorum_stat; - const String quorum_unparallel_path = zookeeper_path + "/quorum/status"; - const String quorum_parallel_path = zookeeper_path + "/quorum/parallel/" + entry.new_part_name; + const String quorum_unparallel_path = fs::path(zookeeper_path) / "quorum" / "status"; + const String quorum_parallel_path = fs::path(zookeeper_path) / "quorum" / "parallel" / entry.new_part_name; String quorum_str, quorum_path; ReplicatedMergeTreeQuorumEntry quorum_entry; @@ -2029,13 +2034,13 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) ErrorCodes::LOGICAL_ERROR); ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name, + fs::path(zookeeper_path) / "quorum" / "failed_parts" / entry.new_part_name, "", zkutil::CreateMode::Persistent)); /// Deleting from `blocks`. - if (!entry.block_id.empty() && zookeeper->exists(zookeeper_path + "/blocks/" + entry.block_id)) - ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/blocks/" + entry.block_id, -1)); + if (!entry.block_id.empty() && zookeeper->exists(fs::path(zookeeper_path) / "blocks" / entry.block_id)) + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(zookeeper_path) / "blocks" / entry.block_id, -1)); Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); @@ -2048,7 +2053,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } else if (code == Coordination::Error::ZBADVERSION || code == Coordination::Error::ZNONODE || code == Coordination::Error::ZNODEEXISTS) { - LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", entry.new_part_name, Coordination::errorMessage(code)); + LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", + entry.new_part_name, Coordination::errorMessage(code)); } else throw Coordination::Exception(code); @@ -2070,7 +2076,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) try { String part_name = entry.actual_new_part_name.empty() ? entry.new_part_name : entry.actual_new_part_name; - if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum)) + if (!fetchPart(part_name, metadata_snapshot, fs::path(zookeeper_path) / "replicas" / replica, false, entry.quorum)) return false; } catch (Exception & e) @@ -2154,7 +2160,7 @@ bool StorageReplicatedMergeTree::executeFetchShared( try { - if (!fetchExistsPart(new_part_name, metadata_snapshot, zookeeper_path + "/replicas/" + source_replica, disk, path)) + if (!fetchExistsPart(new_part_name, metadata_snapshot, fs::path(zookeeper_path) / "replicas" / source_replica, disk, path)) return false; } catch (Exception & e) @@ -2489,8 +2495,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } else if (!part_desc->replica.empty()) { - String source_replica_path = zookeeper_path + "/replicas/" + part_desc->replica; - ReplicatedMergeTreeAddress address(getZooKeeper()->get(source_replica_path + "/host")); + String source_replica_path = fs::path(zookeeper_path) / "replicas" / part_desc->replica; + ReplicatedMergeTreeAddress address(getZooKeeper()->get(fs::path(source_replica_path) / "host")); auto timeouts = getFetchPartHTTPTimeouts(getContext()); auto credentials = getContext()->getInterserverCredentials(); @@ -2632,18 +2638,18 @@ void StorageReplicatedMergeTree::executeClonePartFromShard(const LogEntry & entr void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coordination::Stat source_is_lost_stat, zkutil::ZooKeeperPtr & zookeeper) { - String source_path = zookeeper_path + "/replicas/" + source_replica; + String source_path = fs::path(zookeeper_path) / "replicas" / source_replica; /** TODO: it will be deleted! (It is only to support old version of CH server). * In current code, the replica is created in single transaction. * If the reference/master replica is not yet fully created, let's wait. */ - while (!zookeeper->exists(source_path + "/columns")) + while (!zookeeper->exists(fs::path(source_path) / "columns")) { LOG_INFO(log, "Waiting for replica {} to be fully created", source_path); zkutil::EventPtr event = std::make_shared(); - if (zookeeper->exists(source_path + "/columns", nullptr, event)) + if (zookeeper->exists(fs::path(source_path) / "columns", nullptr, event)) { LOG_WARNING(log, "Oops, a watch has leaked"); break; @@ -2660,29 +2666,29 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo while (true) { Coordination::Stat log_pointer_stat; - String raw_log_pointer = zookeeper->get(source_path + "/log_pointer", &log_pointer_stat); + String raw_log_pointer = zookeeper->get(fs::path(source_path) / "log_pointer", &log_pointer_stat); Coordination::Requests ops; - ops.push_back(zkutil::makeSetRequest(replica_path + "/log_pointer", raw_log_pointer, -1)); + ops.push_back(zkutil::makeSetRequest(fs::path(replica_path) / "log_pointer", raw_log_pointer, -1)); /// For support old versions CH. if (source_is_lost_stat.version == -1) { /// We check that it was not suddenly upgraded to new version. /// Otherwise it can be upgraded and instantly become lost, but we cannot notice that. - ops.push_back(zkutil::makeCreateRequest(source_path + "/is_lost", "0", zkutil::CreateMode::Persistent)); - ops.push_back(zkutil::makeRemoveRequest(source_path + "/is_lost", -1)); + ops.push_back(zkutil::makeCreateRequest(fs::path(source_path) / "is_lost", "0", zkutil::CreateMode::Persistent)); + ops.push_back(zkutil::makeRemoveRequest(fs::path(source_path) / "is_lost", -1)); } else /// The replica we clone should not suddenly become lost. - ops.push_back(zkutil::makeCheckRequest(source_path + "/is_lost", source_is_lost_stat.version)); + ops.push_back(zkutil::makeCheckRequest(fs::path(source_path) / "is_lost", source_is_lost_stat.version)); Coordination::Responses responses; /// Let's remember the queue of the reference/master replica. - source_queue_names = zookeeper->getChildren(source_path + "/queue"); + source_queue_names = zookeeper->getChildren(fs::path(source_path) / "queue"); /// Check that log pointer of source replica didn't changed while we read queue entries - ops.push_back(zkutil::makeCheckRequest(source_path + "/log_pointer", log_pointer_stat.version)); + ops.push_back(zkutil::makeCheckRequest(fs::path(source_path) / "log_pointer", log_pointer_stat.version)); auto rc = zookeeper->tryMulti(ops, responses); @@ -2722,7 +2728,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo for (const String & entry_name : source_queue_names) { String entry; - if (!zookeeper->tryGet(source_path + "/queue/" + entry_name, entry)) + if (!zookeeper->tryGet(fs::path(source_path) / "queue" / entry_name, entry)) continue; source_queue.push_back(entry); } @@ -2731,13 +2737,13 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo cloneMetadataIfNeeded(source_replica, source_path, zookeeper); /// Add to the queue jobs to receive all the active parts that the reference/master replica has. - Strings source_replica_parts = zookeeper->getChildren(source_path + "/parts"); + Strings source_replica_parts = zookeeper->getChildren(fs::path(source_path) / "parts"); ActiveDataPartSet active_parts_set(format_version, source_replica_parts); Strings active_parts = active_parts_set.getParts(); /// Remove local parts if source replica does not have them, because such parts will never be fetched by other replicas. - Strings local_parts_in_zk = zookeeper->getChildren(replica_path + "/parts"); + Strings local_parts_in_zk = zookeeper->getChildren(fs::path(replica_path) / "parts"); Strings parts_to_remove_from_zk; for (const auto & part : local_parts_in_zk) { @@ -2781,7 +2787,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo log_entry.new_part_name = name; log_entry.create_time = tryGetPartCreateTime(zookeeper, source_path, name); - zookeeper->create(replica_path + "/queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); + zookeeper->create(fs::path(replica_path) / "queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); } LOG_DEBUG(log, "Queued {} parts to be fetched", active_parts.size()); @@ -2789,7 +2795,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo /// Add content of the reference/master replica queue to the queue. for (const String & entry : source_queue) { - zookeeper->create(replica_path + "/queue/queue-", entry, zkutil::CreateMode::PersistentSequential); + zookeeper->create(fs::path(replica_path) / "queue/queue-", entry, zkutil::CreateMode::PersistentSequential); } LOG_DEBUG(log, "Copied {} queue entries", source_queue.size()); @@ -2876,7 +2882,7 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke Coordination::Stat is_lost_stat; bool is_new_replica = true; String res; - if (zookeeper->tryGet(replica_path + "/is_lost", res, &is_lost_stat)) + if (zookeeper->tryGet(fs::path(replica_path) / "is_lost", res, &is_lost_stat)) { if (res == "0") return; @@ -2888,14 +2894,14 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke /// Replica was created by old version of CH, so me must create "/is_lost". /// Note that in old version of CH there was no "lost" replicas possible. /// TODO is_lost node should always exist since v18.12, maybe we can replace `tryGet` with `get` and remove old code? - zookeeper->create(replica_path + "/is_lost", "0", zkutil::CreateMode::Persistent); + zookeeper->create(fs::path(replica_path) / "is_lost", "0", zkutil::CreateMode::Persistent); return; } /// is_lost is "1": it means that we are in repair mode. /// Try choose source replica to clone. /// Source replica must not be lost and should have minimal queue size and maximal log pointer. - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); std::vector futures; for (const String & source_replica_name : replicas) { @@ -2903,20 +2909,20 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke if (source_replica_name == replica_name) continue; - String source_replica_path = zookeeper_path + "/replicas/" + source_replica_name; + String source_replica_path = fs::path(zookeeper_path) / "replicas" / source_replica_name; /// Obviously the following get operations are not atomic, but it's ok to choose good enough replica, not the best one. /// NOTE: We may count some entries twice if log_pointer is moved. - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/is_lost")); - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/log_pointer")); - futures.emplace_back(zookeeper->asyncTryGet(source_replica_path + "/queue")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "is_lost")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "log_pointer")); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(source_replica_path) / "queue")); } /// Wait for results before getting log entries for (auto & future : futures) future.wait(); - Strings log_entries = zookeeper->getChildren(zookeeper_path + "/log"); + Strings log_entries = zookeeper->getChildren(fs::path(zookeeper_path) / "log"); size_t max_log_entry = 0; if (!log_entries.empty()) { @@ -2987,14 +2993,14 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke LOG_WARNING(log, "Will mimic {}", source_replica); /// Clear obsolete queue that we no longer need. - zookeeper->removeChildren(replica_path + "/queue"); + zookeeper->removeChildren(fs::path(replica_path) / "queue"); /// Will do repair from the selected replica. cloneReplica(source_replica, source_is_lost_stat, zookeeper); /// If repair fails to whatever reason, the exception is thrown, is_lost will remain "1" and the replica will be repaired later. /// If replica is repaired successfully, we remove is_lost flag. - zookeeper->set(replica_path + "/is_lost", "0"); + zookeeper->set(fs::path(replica_path) / "is_lost", "0"); } @@ -3321,7 +3327,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c std::vector> exists_futures; exists_futures.reserve(parts.size()); for (const auto & part : parts) - exists_futures.emplace_back(zookeeper->asyncExists(replica_path + "/parts/" + part->name)); + exists_futures.emplace_back(zookeeper->asyncExists(fs::path(replica_path) / "parts" / part->name)); bool all_in_zk = true; for (size_t i = 0; i < parts.size(); ++i) @@ -3361,11 +3367,11 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c Coordination::Responses responses; ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry.toString(), + fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeSetRequest( - zookeeper_path + "/log", "", log_version)); /// Check and update version. + fs::path(zookeeper_path) / "log", "", log_version)); /// Check and update version. Coordination::Error code = zookeeper->tryMulti(ops, responses); @@ -3401,7 +3407,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c auto zookeeper = getZooKeeper(); /// If there is no information about part in ZK, we will not mutate it. - if (!zookeeper->exists(replica_path + "/parts/" + part.name)) + if (!zookeeper->exists(fs::path(replica_path) / "parts" / part.name)) { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { @@ -3431,11 +3437,11 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c Coordination::Responses responses; ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry.toString(), + fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeSetRequest( - zookeeper_path + "/log", "", log_version)); /// Check and update version. + fs::path(zookeeper_path) / "log", "", log_version)); /// Check and update version. Coordination::Error code = zookeeper->tryMulti(ops, responses); @@ -3456,12 +3462,12 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_name, Coordination::Requests & ops, bool has_children) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; if (has_children) { - ops.emplace_back(zkutil::makeRemoveRequest(part_path + "/checksums", -1)); - ops.emplace_back(zkutil::makeRemoveRequest(part_path + "/columns", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(part_path) / "checksums", -1)); + ops.emplace_back(zkutil::makeRemoveRequest(fs::path(part_path) / "columns", -1)); } ops.emplace_back(zkutil::makeRemoveRequest(part_path, -1)); } @@ -3469,7 +3475,7 @@ void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_nam void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_name) { auto zookeeper = getZooKeeper(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; Coordination::Stat stat; /// Part doesn't exist, nothing to remove @@ -3486,7 +3492,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n { auto zookeeper = getZooKeeper(); - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; Coordination::Requests ops; @@ -3505,7 +3511,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n log_entry->new_part_name = part_name; ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/queue/queue-", log_entry->toString(), + fs::path(replica_path) / "queue/queue-", log_entry->toString(), zkutil::CreateMode::PersistentSequential)); auto results = zookeeper->multi(ops); @@ -3530,7 +3536,7 @@ void StorageReplicatedMergeTree::enterLeaderElection() { leader_election = std::make_shared( getContext()->getSchedulePool(), - zookeeper_path + "/leader_election", + fs::path(zookeeper_path) / "leader_election", *current_zookeeper, /// current_zookeeper lives for the lifetime of leader_election, /// since before changing `current_zookeeper`, `leader_election` object is destroyed in `partialShutdown` method. callback, @@ -3585,13 +3591,13 @@ ConnectionTimeouts StorageReplicatedMergeTree::getFetchPartHTTPTimeouts(ContextP bool StorageReplicatedMergeTree::checkReplicaHavePart(const String & replica, const String & part_name) { auto zookeeper = getZooKeeper(); - return zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name); + return zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "parts" / part_name); } String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_name, bool active) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3607,7 +3613,7 @@ String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_nam LOG_TRACE(log, "Candidate replica: {}", replica); if (checkReplicaHavePart(replica, part_name) && - (!active || zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + (!active || zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active"))) return replica; /// Obviously, replica could become inactive or even vanish after return from this method. @@ -3619,7 +3625,7 @@ String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_nam String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entry, bool active) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3629,11 +3635,11 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entr if (replica == replica_name) continue; - if (active && !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (active && !zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; String largest_part_found; - Strings parts = zookeeper->getChildren(zookeeper_path + "/replicas/" + replica + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas" / replica / "parts"); for (const String & part_on_replica : parts) { if (part_on_replica == entry.new_part_name @@ -3674,7 +3680,7 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( const String & part_name, bool active, String & found_part_name) { auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -3687,10 +3693,10 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( if (replica == replica_name) continue; - if (active && !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (active && !zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; - Strings parts = zookeeper->getChildren(zookeeper_path + "/replicas/" + replica + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas" / replica / "parts"); for (const String & part_on_replica : parts) { if (part_on_replica == part_name @@ -3718,11 +3724,11 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, bool is_ auto zookeeper = getZooKeeper(); /// Information on which replicas a part has been added, if the quorum has not yet been reached. - String quorum_status_path = zookeeper_path + "/quorum/status"; + String quorum_status_path = fs::path(zookeeper_path) / "quorum" / "status"; if (is_parallel) - quorum_status_path = zookeeper_path + "/quorum/parallel/" + part_name; + quorum_status_path = fs::path(zookeeper_path) / "quorum" / "parallel" / part_name; /// The name of the previous part for which the quorum was reached. - const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + const String quorum_last_part_path = fs::path(zookeeper_path) / "quorum" / "last_part"; String value; Coordination::Stat stat; @@ -3824,7 +3830,7 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) auto zookeeper = getZooKeeper(); /// The name of the previous part for which the quorum was reached. - const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; + const String quorum_last_part_path = fs::path(zookeeper_path) / "quorum" / "last_part"; /// Delete information from "last_part" node. @@ -3874,7 +3880,7 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) bool StorageReplicatedMergeTree::partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const { auto zookeeper = getZooKeeper(); - return zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_info.getPartName()); + return zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "parallel" / part_info.getPartName()); } @@ -3882,7 +3888,7 @@ bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & { auto zookeeper = getZooKeeper(); - const String parts_with_quorum_path = zookeeper_path + "/quorum/last_part"; + const String parts_with_quorum_path = fs::path(zookeeper_path) / "quorum" / "last_part"; String parts_with_quorum_str = zookeeper->get(parts_with_quorum_path); @@ -3966,13 +3972,13 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora source_part_checksums.computeTotalChecksums(source_part->checksums); MinimalisticDataPartChecksums desired_checksums; - String part_path = source_replica_path + "/parts/" + part_name; + String part_path = fs::path(source_replica_path) / "parts" / part_name; String part_znode = zookeeper->get(part_path); if (!part_znode.empty()) desired_checksums = ReplicatedMergeTreePartHeader::fromString(part_znode).getChecksums(); else { - String desired_checksums_str = zookeeper->get(part_path + "/checksums"); + String desired_checksums_str = zookeeper->get(fs::path(part_path) / "checksums"); desired_checksums = MinimalisticDataPartChecksums::deserializeFrom(desired_checksums_str); } @@ -4001,7 +4007,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora } else { - address.fromString(zookeeper->get(source_replica_path + "/host")); + address.fromString(zookeeper->get(fs::path(source_replica_path) / "host")); timeouts = getFetchPartHTTPTimeouts(getContext()); credentials = getContext()->getInterserverCredentials(); @@ -4049,16 +4055,16 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (quorum) { /// Check if this quorum insert is parallel or not - if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) + if (zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "parallel" / part_name)) updateQuorum(part_name, true); - else if (zookeeper->exists(zookeeper_path + "/quorum/status")) + else if (zookeeper->exists(fs::path(zookeeper_path) / "quorum" / "status")) updateQuorum(part_name, false); } /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before if (part_info.level != 0 || part_info.mutation != 0) { - Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); + Strings quorum_parts = zookeeper->getChildren(fs::path(zookeeper_path) / "quorum" / "parallel"); for (const String & quorum_part : quorum_parts) { auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); @@ -4081,7 +4087,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora { // The fetched part is valuable and should not be cleaned like a temp part. part->is_temp = false; - part->renameTo("detached/" + part_name, true); + part->renameTo(fs::path("detached") / part_name, true); } } catch (const Exception & e) @@ -4159,7 +4165,7 @@ bool StorageReplicatedMergeTree::fetchExistsPart(const String & part_name, const std::function get_part; - ReplicatedMergeTreeAddress address(zookeeper->get(source_replica_path + "/host")); + ReplicatedMergeTreeAddress address(zookeeper->get(fs::path(source_replica_path) / "host")); auto timeouts = ConnectionTimeouts::getHTTPTimeouts(getContext()); auto credentials = getContext()->getInterserverCredentials(); String interserver_scheme = getContext()->getInterserverScheme(); @@ -4320,7 +4326,7 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg auto zookeeper = getZooKeeper(); - const String quorum_status_path = zookeeper_path + "/quorum/status"; + const String quorum_status_path = fs::path(zookeeper_path) / "quorum" / "status"; String value; Coordination::Stat stat; @@ -4336,7 +4342,7 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg } String added_parts_str; - if (zookeeper->tryGet(zookeeper_path + "/quorum/last_part", added_parts_str)) + if (zookeeper->tryGet(fs::path(zookeeper_path) / "quorum" / "last_part", added_parts_str)) { if (!added_parts_str.empty()) { @@ -4663,8 +4669,8 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer /// If metadata nodes have changed, we will update table structure locally. Coordination::Requests requests; - requests.emplace_back(zkutil::makeSetRequest(replica_path + "/columns", entry.columns_str, -1)); - requests.emplace_back(zkutil::makeSetRequest(replica_path + "/metadata", entry.metadata_str, -1)); + requests.emplace_back(zkutil::makeSetRequest(fs::path(replica_path) / "columns", entry.columns_str, -1)); + requests.emplace_back(zkutil::makeSetRequest(fs::path(replica_path) / "metadata", entry.metadata_str, -1)); zookeeper->multi(requests); @@ -4680,7 +4686,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer } /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node - zookeeper->createOrUpdate(replica_path + "/metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); + zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); return true; } @@ -4726,7 +4732,7 @@ PartitionBlockNumbersHolder StorageReplicatedMergeTree::allocateBlockNumbersInAf { /// TODO: Implement optimal block number aqcuisition algorithm in multiple (but not all) partitions EphemeralLocksInAllPartitions lock_holder( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + fs::path(zookeeper_path) / "block_numbers", "block-", fs::path(zookeeper_path) / "temp", *zookeeper); PartitionBlockNumbersHolder::BlockNumbersType block_numbers; for (const auto & lock : lock_holder.getLocks()) @@ -4826,10 +4832,10 @@ void StorageReplicatedMergeTree::alter( size_t mutation_path_idx = std::numeric_limits::max(); String new_metadata_str = future_metadata_in_zk.toString(); - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, metadata_version)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "metadata", new_metadata_str, metadata_version)); String new_columns_str = future_metadata.columns.toString(); - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "columns", new_columns_str, -1)); if (ast_to_str(current_metadata->settings_changes) != ast_to_str(future_metadata.settings_changes)) { @@ -4858,12 +4864,12 @@ void StorageReplicatedMergeTree::alter( alter_path_idx = ops.size(); ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(zookeeper_path) / "log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); PartitionBlockNumbersHolder partition_block_numbers_holder; if (have_mutation) { - const String mutations_path(zookeeper_path + "/mutations"); + const String mutations_path(fs::path(zookeeper_path) / "mutations"); ReplicatedMergeTreeMutationEntry mutation_entry; mutation_entry.alter_version = new_metadata_version; @@ -4882,7 +4888,7 @@ void StorageReplicatedMergeTree::alter( ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); mutation_path_idx = ops.size(); ops.emplace_back( - zkutil::makeCreateRequest(mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); + zkutil::makeCreateRequest(fs::path(mutations_path) / "", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); } if (auto txn = query_context->getZooKeeperMetadataTransaction()) @@ -4890,7 +4896,7 @@ void StorageReplicatedMergeTree::alter( txn->moveOpsTo(ops); /// NOTE: IDatabase::alterTable(...) is called when executing ALTER_METADATA queue entry without query context, /// so we have to update metadata of DatabaseReplicated here. - String metadata_zk_path = txn->getDatabaseZooKeeperPath() + "/metadata/" + escapeForFileName(table_id.table_name); + String metadata_zk_path = fs::path(txn->getDatabaseZooKeeperPath()) / "metadata" / escapeForFileName(table_id.table_name); auto ast = DatabaseCatalog::instance().getDatabase(table_id.database_name)->getCreateTableQuery(table_id.table_name, query_context); applyMetadataChangesToCreateQuery(ast, future_metadata); ops.emplace_back(zkutil::makeSetRequest(metadata_zk_path, getObjectDefinitionFromCreateQuery(ast), -1)); @@ -5028,44 +5034,56 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition(const St return true; } - -void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr query_context, bool throw_if_noop) +void StorageReplicatedMergeTree::dropPartNoWaitNoThrow(const String & part_name) { assertNotReadonly(); if (!is_leader) - throw Exception("DROP PART|PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); + throw Exception("DROP PART cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - LogEntry entry; - bool did_drop; - if (drop_part) - { - String part_name = partition->as().value.safeGet(); - did_drop = dropPart(zookeeper, part_name, entry, detach, throw_if_noop); - } - else - { - String partition_id = getPartitionIDFromQuery(partition, query_context); - did_drop = dropAllPartsInPartition(*zookeeper, partition_id, entry, query_context, detach); - } + dropPartImpl(zookeeper, part_name, entry, /*detach=*/ false, /*throw_if_noop=*/ false); +} + +void StorageReplicatedMergeTree::dropPart(const String & part_name, bool detach, ContextPtr query_context) +{ + assertNotReadonly(); + if (!is_leader) + throw Exception("DROP PART cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); + + zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + LogEntry entry; + + dropPartImpl(zookeeper, part_name, entry, detach, /*throw_if_noop=*/ true); + + /// If necessary, wait until the operation is performed on itself or on all replicas. + if (query_context->getSettingsRef().replication_alter_partitions_sync == 1) + waitForReplicaToProcessLogEntry(replica_name, entry); + else if (query_context->getSettingsRef().replication_alter_partitions_sync == 2) + waitForAllReplicasToProcessLogEntry(entry); +} + +void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool detach, ContextPtr query_context) +{ + assertNotReadonly(); + if (!is_leader) + throw Exception("DROP PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); + + zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + LogEntry entry; + + String partition_id = getPartitionIDFromQuery(partition, query_context); + bool did_drop = dropAllPartsInPartition(*zookeeper, partition_id, entry, query_context, detach); if (did_drop) { /// If necessary, wait until the operation is performed on itself or on all replicas. - if (query_context->getSettingsRef().replication_alter_partitions_sync != 0) - { - if (query_context->getSettingsRef().replication_alter_partitions_sync == 1) - waitForReplicaToProcessLogEntry(replica_name, entry); - else - waitForAllReplicasToProcessLogEntry(entry); - } - } + if (query_context->getSettingsRef().replication_alter_partitions_sync == 1) + waitForReplicaToProcessLogEntry(replica_name, entry); + else if (query_context->getSettingsRef().replication_alter_partitions_sync == 2) + waitForAllReplicasToProcessLogEntry(entry); - if (!drop_part) - { - String partition_id = getPartitionIDFromQuery(partition, query_context); cleanLastPartNode(partition_id); } } @@ -5082,7 +5100,7 @@ void StorageReplicatedMergeTree::truncate( zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - Strings partitions = zookeeper->getChildren(zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(zookeeper_path) / "block_numbers"); for (String & partition_id : partitions) { @@ -5157,7 +5175,7 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_table_data, c try { auto zookeeper = getZooKeeper(); - zookeeper->set(replica_path + "/host", getReplicatedMergeTreeAddress().toString()); + zookeeper->set(fs::path(replica_path) / "host", getReplicatedMergeTreeAddress().toString()); } catch (Coordination::Exception & e) { @@ -5207,8 +5225,8 @@ StorageReplicatedMergeTree::allocateBlockNumber( deduplication_check_ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_block_id_path, -1)); } - String block_numbers_path = zookeeper_table_path + "/block_numbers"; - String partition_path = block_numbers_path + "/" + partition_id; + String block_numbers_path = fs::path(zookeeper_table_path) / "block_numbers"; + String partition_path = fs::path(block_numbers_path) / partition_id; if (!existsNodeCached(partition_path)) { @@ -5230,7 +5248,7 @@ StorageReplicatedMergeTree::allocateBlockNumber( try { lock = EphemeralLockInZooKeeper( - partition_path + "/block-", zookeeper_table_path + "/temp", *zookeeper, &deduplication_check_ops); + fs::path(partition_path) / "block-", fs::path(zookeeper_table_path) / "temp", *zookeeper, &deduplication_check_ops); } catch (const zkutil::KeeperMultiException & e) { @@ -5254,11 +5272,11 @@ Strings StorageReplicatedMergeTree::waitForAllTableReplicasToProcessLogEntry( LOG_DEBUG(log, "Waiting for all replicas to process {}", entry.znode_name); auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(table_zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(table_zookeeper_path) / "replicas"); Strings unwaited; for (const String & replica : replicas) { - if (wait_for_non_active || zookeeper->exists(table_zookeeper_path + "/replicas/" + replica + "/is_active")) + if (wait_for_non_active || zookeeper->exists(fs::path(table_zookeeper_path) / "replicas" / replica / "is_active")) { if (!waitForTableReplicaToProcessLogEntry(table_zookeeper_path, replica, entry, wait_for_non_active)) unwaited.push_back(replica); @@ -5308,7 +5326,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( const auto & stop_waiting = [&]() { bool stop_waiting_itself = waiting_itself && (partial_shutdown_called || is_dropped); - bool stop_waiting_non_active = !wait_for_non_active && !getZooKeeper()->exists(table_zookeeper_path + "/replicas/" + replica + "/is_active"); + bool stop_waiting_non_active = !wait_for_non_active && !getZooKeeper()->exists(fs::path(table_zookeeper_path) / "replicas" / replica / "is_active"); return stop_waiting_itself || stop_waiting_non_active; }; @@ -5330,7 +5348,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( { zkutil::EventPtr event = std::make_shared(); - String log_pointer = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + String log_pointer = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer", nullptr, event); if (!log_pointer.empty() && parse(log_pointer) > log_index) break; @@ -5347,9 +5365,9 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( * looking for a node with the same content. And if we do not find it - then the replica has already taken this entry in its queue. */ - String log_pointer = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer"); + String log_pointer = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer"); - Strings log_entries = getZooKeeper()->getChildren(table_zookeeper_path + "/log"); + Strings log_entries = getZooKeeper()->getChildren(fs::path(table_zookeeper_path) / "log"); UInt64 log_index = 0; bool found = false; @@ -5361,7 +5379,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( continue; String log_entry_str; - bool exists = getZooKeeper()->tryGet(table_zookeeper_path + "/log/" + log_entry_name, log_entry_str); + bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "log" / log_entry_name, log_entry_str); if (exists && entry_str == log_entry_str) { found = true; @@ -5379,7 +5397,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( { zkutil::EventPtr event = std::make_shared(); - String log_pointer_new = getZooKeeper()->get(table_zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + String log_pointer_new = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer", nullptr, event); if (!log_pointer_new.empty() && parse(log_pointer_new) > log_index) break; @@ -5404,13 +5422,13 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( * Therefore, we search by comparing the content. */ - Strings queue_entries = getZooKeeper()->getChildren(table_zookeeper_path + "/replicas/" + replica + "/queue"); + Strings queue_entries = getZooKeeper()->getChildren(fs::path(table_zookeeper_path) / "replicas" / replica / "queue"); String queue_entry_to_wait_for; for (const String & entry_name : queue_entries) { String queue_entry_str; - bool exists = getZooKeeper()->tryGet(table_zookeeper_path + "/replicas/" + replica + "/queue/" + entry_name, queue_entry_str); + bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "replicas" / replica / "queue" / entry_name, queue_entry_str); if (exists && queue_entry_str == entry_str) { queue_entry_to_wait_for = entry_name; @@ -5428,7 +5446,7 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( LOG_DEBUG(log, "Waiting for {} to disappear from {} queue", queue_entry_to_wait_for, replica); /// Third - wait until the entry disappears from the replica queue or replica become inactive. - String path_to_wait_on = table_zookeeper_path + "/replicas/" + replica + "/queue/" + queue_entry_to_wait_for; + String path_to_wait_on = fs::path(table_zookeeper_path) / "replicas" / replica / "queue" / queue_entry_to_wait_for; return getZooKeeper()->waitForDisappear(path_to_wait_on, stop_waiting); } @@ -5470,7 +5488,7 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) { try { - auto log_entries = zookeeper->getChildren(zookeeper_path + "/log"); + auto log_entries = zookeeper->getChildren(fs::path(zookeeper_path) / "log"); if (!log_entries.empty()) { @@ -5478,14 +5496,14 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) res.log_max_index = parse(last_log_entry.substr(strlen("log-"))); } - String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(replica_path) / "log_pointer"); res.log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); - auto all_replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + auto all_replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); res.total_replicas = all_replicas.size(); for (const String & replica : all_replicas) - if (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) ++res.active_replicas; } catch (const Coordination::Exception &) @@ -5568,7 +5586,7 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t time_t max_replicas_unprocessed_insert_time = 0; bool have_replica_with_nothing_unprocessed = false; - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); for (const auto & replica : replicas) { @@ -5576,11 +5594,11 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t continue; /// Skip dead replicas. - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + if (!zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active")) continue; String value; - if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/min_unprocessed_insert_time", value)) + if (!zookeeper->tryGet(fs::path(zookeeper_path) / "replicas" / replica / "min_unprocessed_insert_time", value)) continue; time_t replica_time = value.empty() ? 0 : parse(value); @@ -5687,13 +5705,13 @@ void StorageReplicatedMergeTree::fetchPartition( { /// List of replicas of source shard. - replicas = zookeeper->getChildren(from + "/replicas"); + replicas = zookeeper->getChildren(fs::path(from) / "replicas"); /// Leave only active replicas. active_replicas.reserve(replicas.size()); for (const String & replica : replicas) - if (zookeeper->exists(from + "/replicas/" + replica + "/is_active")) + if (zookeeper->exists(fs::path(from) / "replicas" / replica / "is_active")) active_replicas.push_back(replica); if (active_replicas.empty()) @@ -5710,13 +5728,13 @@ void StorageReplicatedMergeTree::fetchPartition( for (const String & replica : active_replicas) { - String current_replica_path = from + "/replicas/" + replica; + String current_replica_path = fs::path(from) / "replicas" / replica; - String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(current_replica_path) / "log_pointer"); Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); Coordination::Stat stat; - zookeeper->get(current_replica_path + "/queue", &stat); + zookeeper->get(fs::path(current_replica_path) / "queue", &stat); size_t queue_size = stat.numChildren; if (log_pointer > max_log_pointer @@ -5734,7 +5752,7 @@ void StorageReplicatedMergeTree::fetchPartition( LOG_INFO(log, "Found {} replicas, {} of them are active. Selected {} to fetch from.", replicas.size(), active_replicas.size(), best_replica); - String best_replica_path = from + "/replicas/" + best_replica; + String best_replica_path = fs::path(from) / "replicas" / best_replica; /// Let's find out which parts are on the best replica. @@ -5753,7 +5771,7 @@ void StorageReplicatedMergeTree::fetchPartition( if (try_no >= query_context->getSettings().max_fetch_partition_retries_count) throw Exception("Too many retries to fetch parts from " + best_replica_path, ErrorCodes::TOO_MANY_RETRIES_TO_FETCH_PARTS); - Strings parts = zookeeper->getChildren(best_replica_path + "/parts"); + Strings parts = zookeeper->getChildren(fs::path(best_replica_path) / "parts"); ActiveDataPartSet active_parts_set(format_version, parts); Strings parts_to_fetch; @@ -5874,7 +5892,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte mutation_entry.source_replica = replica_name; mutation_entry.commands = commands; - const String mutations_path = zookeeper_path + "/mutations"; + const String mutations_path = fs::path(zookeeper_path) / "mutations"; const auto zookeeper = getZooKeeper(); /// Update the mutations_path node when creating the mutation and check its version to ensure that @@ -5897,7 +5915,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte Coordination::Requests requests; requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); requests.emplace_back(zkutil::makeCreateRequest( - mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(mutations_path) / "", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); if (auto txn = query_context->getZooKeeperMetadataTransaction()) txn->moveOpsTo(requests); @@ -5936,7 +5954,7 @@ void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t auto zookeeper = getZooKeeper(); Strings replicas; if (mutations_sync == 2) /// wait for all replicas - replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + replicas = zookeeper->getChildren(fs::path(zookeeper_path) / "replicas"); else if (mutations_sync == 1) /// just wait for ourself replicas.push_back(replica_name); @@ -6131,7 +6149,7 @@ bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(const St exists_futures.reserve(part_names.size()); for (const String & part_name : part_names) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -6194,7 +6212,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( /// if zk session will be dropped for (const String & part_name : part_names) { - String part_path = replica_path + "/parts/" + part_name; + String part_path = fs::path(replica_path) / "parts" / part_name; exists_futures.emplace_back(zookeeper->asyncExists(part_path)); } @@ -6253,7 +6271,7 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( Coordination::Requests & ops, zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num) { Strings blocks; - if (Coordination::Error::ZOK != zookeeper.tryGetChildren(zookeeper_path + "/blocks", blocks)) + if (Coordination::Error::ZOK != zookeeper.tryGetChildren(fs::path(zookeeper_path) / "blocks", blocks)) throw Exception(zookeeper_path + "/blocks doesn't exist", ErrorCodes::NOT_FOUND_NODE); String partition_prefix = partition_id + "_"; @@ -6262,7 +6280,7 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( { if (startsWith(block_id, partition_prefix)) { - String path = zookeeper_path + "/blocks/" + block_id; + String path = fs::path(zookeeper_path) / "blocks" / block_id; get_futures.emplace_back(path, zookeeper.asyncTryGet(path)); } } @@ -6372,7 +6390,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( else LOG_INFO(log, "Trying to attach {} with hash_hex {}", src_part->name, hash_hex); - String block_id_path = replace ? "" : (zookeeper_path + "/blocks/" + partition_id + "_replace_from_" + hash_hex); + String block_id_path = replace ? "" : (fs::path(zookeeper_path) / "blocks" / (partition_id + "_replace_from_" + hash_hex)); auto lock = allocateBlockNumber(partition_id, zookeeper, block_id_path); if (!lock) @@ -6444,8 +6462,8 @@ void StorageReplicatedMergeTree::replacePartitionFrom( ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); Transaction transaction(*this); { @@ -6632,8 +6650,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(dest_table_storage->zookeeper_path + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest(dest_table_storage->zookeeper_path + "/log/log-", + ops.emplace_back(zkutil::makeSetRequest(fs::path(dest_table_storage->zookeeper_path) / "log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(dest_table_storage->zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); { @@ -6689,12 +6707,12 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Coordination::Requests ops_src; ops_src.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); + fs::path(zookeeper_path) / "log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); /// Check and update version to avoid race with REPLACE_RANGE ops_src.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops_src.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops_src.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops_src.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); delimiting_block_lock->getUnlockOps(ops_src); Coordination::Error code = zookeeper->tryMulti(ops_src, op_results); @@ -6834,22 +6852,22 @@ void StorageReplicatedMergeTree::getCommitPartOps( if (storage_settings_ptr->use_minimalistic_part_header_in_zookeeper) { ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name, + fs::path(replica_path) / "parts" / part->name, ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(), zkutil::CreateMode::Persistent)); } else { ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name, + fs::path(replica_path) / "parts" / part->name, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name + "/columns", + fs::path(replica_path) / "parts" / part->name / "columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - replica_path + "/parts/" + part->name + "/checksums", + fs::path(replica_path) / "parts" / part->name / "checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); } @@ -6937,7 +6955,7 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI return true; } -bool StorageReplicatedMergeTree::dropPart( +bool StorageReplicatedMergeTree::dropPartImpl( zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop) { LOG_TRACE(log, "Will try to insert a log entry to DROP_RANGE for part: " + part_name); @@ -7000,10 +7018,10 @@ bool StorageReplicatedMergeTree::dropPart( entry.detach = detach; entry.create_time = time(nullptr); - ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/log", merge_pred.getVersion())); /// Make sure no new events were added to the log. - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCheckRequest(fs::path(zookeeper_path) / "log", merge_pred.getVersion())); /// Make sure no new events were added to the log. + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); Coordination::Responses responses; Coordination::Error rc = zookeeper->tryMulti(ops, responses); @@ -7057,14 +7075,14 @@ bool StorageReplicatedMergeTree::dropAllPartsInPartition( entry.create_time = time(nullptr); Coordination::Requests ops; - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); /// Check and update version to avoid race with REPLACE_RANGE. /// Otherwise new parts covered by drop_range_info may appear after execution of current DROP_RANGE entry /// as a result of execution of concurrently created REPLACE_RANGE entry. ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version)); ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1)); /// Just update version, because merges assignment relies on it - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1)); delimiting_block_lock->getUnlockOps(ops); if (auto txn = query_context->getZooKeeperMetadataTransaction()) txn->moveOpsTo(ops); @@ -7150,7 +7168,7 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name + "/" + id + "/" + replica_name; + String zookeeper_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name / id / replica_name; LOG_TRACE(log, "Set zookeeper lock {}", zookeeper_node); @@ -7191,9 +7209,9 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; - String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; - String zookeeper_node = zookeeper_part_uniq_node + "/" + replica_name; + String zookeeper_part_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name; + String zookeeper_part_uniq_node = fs::path(zookeeper_part_node) / id; + String zookeeper_node = fs::path(zookeeper_part_uniq_node) / replica_name; LOG_TRACE(log, "Remove zookeeper lock {}", zookeeper_node); @@ -7252,7 +7270,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( if (!zookeeper) return best_replica; - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; + String zookeeper_part_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name; Strings ids; zookeeper->tryGetChildren(zookeeper_part_node, ids); @@ -7260,7 +7278,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( Strings replicas; for (const auto & id : ids) { - String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; + String zookeeper_part_uniq_node = fs::path(zookeeper_part_node) / id; Strings id_replicas; zookeeper->tryGetChildren(zookeeper_part_uniq_node, id_replicas); LOG_TRACE(log, "Found zookeper replicas for {}: {}", zookeeper_part_uniq_node, id_replicas.size()); @@ -7277,7 +7295,7 @@ String StorageReplicatedMergeTree::getSharedDataReplica( active_replicas.reserve(replicas.size()); for (const String & replica : replicas) - if ((replica != replica_name) && (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + if ((replica != replica_name) && (zookeeper->exists(fs::path(zookeeper_path) / "replicas" / replica / "is_active"))) active_replicas.push_back(replica); LOG_TRACE(log, "Found zookeper active replicas for part {}: {}", part.name, active_replicas.size()); @@ -7296,13 +7314,13 @@ String StorageReplicatedMergeTree::getSharedDataReplica( for (const String & replica : active_replicas) { - String current_replica_path = zookeeper_path + "/replicas/" + replica; + String current_replica_path = fs::path(zookeeper_path) / "replicas" / replica; - String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); + String log_pointer_str = zookeeper->get(fs::path(current_replica_path) / "log_pointer"); Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); Coordination::Stat stat; - zookeeper->get(current_replica_path + "/queue", &stat); + zookeeper->get(fs::path(current_replica_path) / "queue", &stat); size_t queue_size = stat.numChildren; if (log_pointer > max_log_pointer @@ -7320,16 +7338,16 @@ String StorageReplicatedMergeTree::getSharedDataReplica( String StorageReplicatedMergeTree::findReplicaHavingPart( const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_) { - Strings replicas = zookeeper_->getChildren(zookeeper_path_ + "/replicas"); + Strings replicas = zookeeper_->getChildren(fs::path(zookeeper_path_) / "replicas"); /// Select replicas in uniformly random order. std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); for (const String & replica : replicas) { - if (zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/parts/" + part_name) - && zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/is_active")) - return zookeeper_path_ + "/replicas/" + replica; + if (zookeeper_->exists(fs::path(zookeeper_path_) / "replicas" / replica / "parts" / part_name) + && zookeeper_->exists(fs::path(zookeeper_path_) / "replicas" / replica / "is_active")) + return fs::path(zookeeper_path_) / "replicas" / replica; } return {}; @@ -7337,23 +7355,23 @@ String StorageReplicatedMergeTree::findReplicaHavingPart( bool StorageReplicatedMergeTree::checkIfDetachedPartExists(const String & part_name) { - Poco::DirectoryIterator dir_end; + fs::directory_iterator dir_end; for (const std::string & path : getDataPaths()) - for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) - if (dir_it.name() == part_name) + for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) + if (dir_it->path().filename().string() == part_name) return true; return false; } bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & partition_name) { - Poco::DirectoryIterator dir_end; + fs::directory_iterator dir_end; for (const std::string & path : getDataPaths()) { - for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) + for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) { MergeTreePartInfo part_info; - if (MergeTreePartInfo::tryParsePartName(dir_it.name(), &part_info, format_version) && part_info.partition_id == partition_name) + if (MergeTreePartInfo::tryParsePartName(dir_it->path().filename(), &part_info, format_version) && part_info.partition_id == partition_name) return true; } } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 4e697f2d1f2..ec542acc275 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -642,12 +642,14 @@ private: /// Info about how other replicas can access this one. ReplicatedMergeTreeAddress getReplicatedMergeTreeAddress() const; - bool dropPart(zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop); bool dropAllPartsInPartition( zkutil::ZooKeeper & zookeeper, String & partition_id, LogEntry & entry, ContextPtr query_context, bool detach); + void dropPartNoWaitNoThrow(const String & part_name) override; + void dropPart(const String & part_name, bool detach, ContextPtr query_context) override; + // Partition helpers - void dropPartition(const ASTPtr & partition, bool detach, bool drop_part, ContextPtr query_context, bool throw_if_noop) override; + void dropPartition(const ASTPtr & partition, bool detach, ContextPtr query_context) override; PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr query_context) override; void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr query_context) override; void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr query_context) override; @@ -659,6 +661,8 @@ private: bool fetch_part, ContextPtr query_context) override; + bool dropPartImpl(zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop); + /// Check granularity of already existing replicated table in zookeeper if it exists /// return true if it's fixed bool checkFixedGranualrityInZookeeper(); @@ -685,7 +689,7 @@ protected: const StorageID & table_id_, const String & relative_data_path_, const StorageInMemoryMetadata & metadata_, - ContextPtr context_, + ContextMutablePtr context_, const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 36d078e7bf3..fd9ff596329 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -35,8 +35,10 @@ #include #include #include - #include +#include + +namespace fs = std::filesystem; namespace DB { @@ -197,7 +199,7 @@ bool StorageS3Source::initialize() if (current_key.empty()) return false; - file_path = bucket + "/" + current_key; + file_path = fs::path(bucket) / current_key; read_buf = wrapReadBufferWithCompressionMethod( std::make_unique(client, bucket, current_key, s3_max_single_read_retries), chooseCompressionMethod(current_key, compression_hint)); diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index 88d54669889..9cffc32fda1 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/examples/get_current_inserts_in_replicated.cpp b/src/Storages/examples/get_current_inserts_in_replicated.cpp index fa998d20f66..56364997643 100644 --- a/src/Storages/examples/get_current_inserts_in_replicated.cpp +++ b/src/Storages/examples/get_current_inserts_in_replicated.cpp @@ -4,11 +4,12 @@ #include #include #include - #include +#include #include +namespace fs = std::filesystem; using namespace DB; @@ -36,10 +37,10 @@ try Stopwatch stage; /// Load current inserts std::unordered_set lock_holder_paths; - for (const String & entry : zookeeper->getChildren(zookeeper_path + "/temp")) + for (const String & entry : zookeeper->getChildren(fs::path(zookeeper_path) / "temp")) { if (startsWith(entry, "abandonable_lock-")) - lock_holder_paths.insert(zookeeper_path + "/temp/" + entry); + lock_holder_paths.insert(fs::path(zookeeper_path) / "temp" / entry); } std::cerr << "Stage 1 (get lock holders): " << lock_holder_paths.size() << " lock holders, elapsed: " << stage.elapsedSeconds() << "s." << std::endl; @@ -47,14 +48,14 @@ try if (!lock_holder_paths.empty()) { - Strings partitions = zookeeper->getChildren(zookeeper_path + "/block_numbers"); + Strings partitions = zookeeper->getChildren(fs::path(zookeeper_path) / "block_numbers"); std::cerr << "Stage 2 (get partitions): " << partitions.size() << " partitions, elapsed: " << stage.elapsedSeconds() << "s." << std::endl; stage.restart(); std::vector> lock_futures; for (const String & partition : partitions) - lock_futures.push_back(zookeeper->asyncGetChildren(zookeeper_path + "/block_numbers/" + partition)); + lock_futures.push_back(zookeeper->asyncGetChildren(fs::path(zookeeper_path) / "block_numbers" / partition)); struct BlockInfo { @@ -71,7 +72,7 @@ try for (const String & entry : partition_block_numbers) { Int64 block_number = parse(entry.substr(strlen("block-"))); - String zk_path = zookeeper_path + "/block_numbers/" + partitions[i] + "/" + entry; + String zk_path = fs::path(zookeeper_path) / "block_numbers" / partitions[i] / entry; block_infos.push_back( BlockInfo{partitions[i], block_number, zk_path, zookeeper->asyncTryGet(zk_path)}); } diff --git a/src/Storages/examples/remove_symlink_directory.cpp b/src/Storages/examples/remove_symlink_directory.cpp index ae5fa72fa66..db436c0a608 100644 --- a/src/Storages/examples/remove_symlink_directory.cpp +++ b/src/Storages/examples/remove_symlink_directory.cpp @@ -1,9 +1,10 @@ #include #include -#include -#include #include +#include +#include +namespace fs = std::filesystem; namespace DB { @@ -16,18 +17,15 @@ namespace DB int main(int, char **) try { - Poco::File dir("./test_dir/"); - dir.createDirectories(); - - Poco::File("./test_dir/file").createFile(); + fs::path dir("./test_dir/"); + fs::create_directories(dir); + FS::createFile("./test_dir/file"); if (0 != symlink("./test_dir", "./test_link")) DB::throwFromErrnoWithPath("Cannot create symlink", "./test_link", DB::ErrorCodes::SYSTEM_ERROR); - Poco::File link("./test_link"); - link.renameTo("./test_link2"); - - Poco::File("./test_link2").remove(true); + fs::rename("./test_link", "./test_link2"); + fs::remove_all("./test_link2"); return 0; } catch (...) diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 6735206862d..e61123a3aa8 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -22,7 +22,7 @@ struct State { State(const State&) = delete; - ContextPtr context; + ContextMutablePtr context; static const State & instance() { diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 6287064b616..733af0cb8f9 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -126,7 +126,7 @@ class ClickHouseCluster: """ def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None, - odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, + odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None): for param in list(os.environ.keys()): print("ENV %40s %s" % (param, os.environ[param])) @@ -455,7 +455,10 @@ class ClickHouseCluster: run_and_check(self.base_cmd + ["up", "--force-recreate", "--no-deps", "-d", node.name]) node.ip_address = self.get_instance_ip(node.name) node.client = Client(node.ip_address, command=self.client_bin_path) - node.wait_for_start(start_timeout=20.0, connection_timeout=600.0) # seconds + print("Restart node with ip change") + # In builds with sanitizer the server can take a long time to start + node.wait_for_start(start_timeout=60.0, connection_timeout=600.0) # seconds + print("Restarted") return node def get_instance_ip(self, instance_name): @@ -1524,7 +1527,7 @@ class ClickHouseInstance: if self.stay_alive: entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND.replace("{main_config_file}", self.main_config_name) - + print("Entrypoint cmd: {}".format(entrypoint_cmd)) networks = app_net = ipv4_address = ipv6_address = net_aliases = net_alias1 = "" diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py index 3f88f719fe4..ab52879c7c3 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test.py @@ -24,7 +24,7 @@ def started_node(): def test_send_segfault(started_node): - if started_node.is_built_with_thread_sanitizer(): + if started_node.is_built_with_thread_sanitizer() or started_node.is_built_with_memory_sanitizer(): pytest.skip("doesn't fit in timeouts for stacktrace generation") started_node.copy_file_to_container(os.path.join(SCRIPT_DIR, "fake_sentry_server.py"), "/fake_sentry_server.py") diff --git a/tests/queries/0_stateless/00118_storage_join.reference b/tests/queries/0_stateless/00118_storage_join.reference index dad5b94e47b..56920b290e6 100644 --- a/tests/queries/0_stateless/00118_storage_join.reference +++ b/tests/queries/0_stateless/00118_storage_join.reference @@ -18,3 +18,49 @@ 7 8 9 +0 0 +1 1 abc +2 2 def +3 3 +4 4 +5 5 +6 6 ghi +7 7 +8 8 +9 9 +0 0 0 +1 1 1 abc +2 2 2 def +3 0 3 +4 0 4 +5 0 5 +6 6 6 ghi +7 0 7 +8 0 8 +9 0 9 +0 3 +3 9 +2 21 def +1 12 abc + +0 45 +0 0 +1 1 abc +2 2 def +3 3 +4 4 +5 5 +6 6 ghi +7 7 +8 8 +9 9 +0 0 0 +1 1 1 abc +2 2 2 def +3 0 3 +4 0 4 +5 0 5 +6 6 6 ghi +7 0 7 +8 0 8 +9 0 9 diff --git a/tests/queries/0_stateless/00118_storage_join.sql b/tests/queries/0_stateless/00118_storage_join.sql index 9da8829e5c7..47896d3316c 100644 --- a/tests/queries/0_stateless/00118_storage_join.sql +++ b/tests/queries/0_stateless/00118_storage_join.sql @@ -1,11 +1,19 @@ -DROP TABLE IF EXISTS join; +DROP TABLE IF EXISTS t2; -CREATE TABLE join (k UInt64, s String) ENGINE = Join(ANY, LEFT, k); +CREATE TABLE t2 (k UInt64, s String) ENGINE = Join(ANY, LEFT, k); -INSERT INTO join VALUES (1, 'abc'), (2, 'def'); -SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; +INSERT INTO t2 VALUES (1, 'abc'), (2, 'def'); +SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; -INSERT INTO join VALUES (6, 'ghi'); -SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; +INSERT INTO t2 VALUES (6, 'ghi'); +SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; -DROP TABLE join; +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number as s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number as s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; + +SELECT k, js1.s, t2.s FROM (SELECT toUInt64(number / 3) AS k, sum(number) as s FROM numbers(10) GROUP BY toUInt64(number / 3) WITH TOTALS) js1 ANY LEFT JOIN t2 USING k; + +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k; +SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k; + +DROP TABLE t2; diff --git a/tests/queries/0_stateless/00119_storage_join.sql b/tests/queries/0_stateless/00119_storage_join.sql index e1cc7a67588..2569a64d2c3 100644 --- a/tests/queries/0_stateless/00119_storage_join.sql +++ b/tests/queries/0_stateless/00119_storage_join.sql @@ -1,18 +1,18 @@ -DROP TABLE IF EXISTS join; +DROP TABLE IF EXISTS t2; -CREATE TABLE join (s String, x Array(UInt8), k UInt64) ENGINE = Join(ANY, LEFT, k); +CREATE TABLE t2 (s String, x Array(UInt8), k UInt64) ENGINE = Join(ANY, LEFT, k); -INSERT INTO join VALUES ('abc', [0], 1), ('def', [1, 2], 2); -INSERT INTO join (k, s) VALUES (3, 'ghi'); -INSERT INTO join (x, k) VALUES ([3, 4, 5], 4); +INSERT INTO t2 VALUES ('abc', [0], 1), ('def', [1, 2], 2); +INSERT INTO t2 (k, s) VALUES (3, 'ghi'); +INSERT INTO t2 (x, k) VALUES ([3, 4, 5], 4); -SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; -SELECT s, x FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; -SELECT x, s, k FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; -SELECT 1, x, 2, s, 3, k, 4 FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN join USING k; +SELECT k, s FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT s, x FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT x, s, k FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; +SELECT 1, x, 2, s, 3, k, 4 FROM (SELECT number AS k FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 USING k; SELECT t1.k, t1.s, t2.x FROM ( SELECT number AS k, 'a' AS s FROM numbers(2) GROUP BY number WITH TOTALS ) AS t1 -ANY LEFT JOIN join AS t2 USING(k); +ANY LEFT JOIN t2 AS t2 USING(k); -DROP TABLE join; +DROP TABLE t2; diff --git a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh index 71ca29bfd96..ca3f4efe743 100755 --- a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh +++ b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh @@ -6,6 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}" + $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS elog;" diff --git a/tests/queries/0_stateless/01601_detach_permanently.reference b/tests/queries/0_stateless/01601_detach_permanently.reference index 6683603c972..0232c1694e4 100644 --- a/tests/queries/0_stateless/01601_detach_permanently.reference +++ b/tests/queries/0_stateless/01601_detach_permanently.reference @@ -50,7 +50,7 @@ After database reattachement the table is still absent (it was detached permamen And we can not detach it permanently But we can attach it back And detach permanently again to check how database drop will behave -DROP database - Directory not empty error, but database deteched +DROP database - Directory not empty error, but database detached DROP database - now success ----------------------- database lazy tests diff --git a/tests/queries/0_stateless/01601_detach_permanently.sql b/tests/queries/0_stateless/01601_detach_permanently.sql index 0e21fd8f463..3af8ed573ef 100644 --- a/tests/queries/0_stateless/01601_detach_permanently.sql +++ b/tests/queries/0_stateless/01601_detach_permanently.sql @@ -128,8 +128,8 @@ ATTACH TABLE test1601_detach_permanently_ordinary.test_name_reuse; SELECT 'And detach permanently again to check how database drop will behave'; DETACH table test1601_detach_permanently_ordinary.test_name_reuse PERMANENTLY; -SELECT 'DROP database - Directory not empty error, but database deteched'; -DROP DATABASE test1601_detach_permanently_ordinary; -- { serverError 1000 } +SELECT 'DROP database - Directory not empty error, but database detached'; +DROP DATABASE test1601_detach_permanently_ordinary; -- { serverError 1001 } ATTACH DATABASE test1601_detach_permanently_ordinary; @@ -203,7 +203,7 @@ SELECT 'And detach permanently again to check how database drop will behave'; DETACH table test1601_detach_permanently_lazy.test_name_reuse PERMANENTLY; SELECT 'DROP database - Directory not empty error, but database deteched'; -DROP DATABASE test1601_detach_permanently_lazy; -- { serverError 1000 } +DROP DATABASE test1601_detach_permanently_lazy; -- { serverError 1001 } ATTACH DATABASE test1601_detach_permanently_lazy; diff --git a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference index 87659c32e39..1d0901cf9f6 100644 --- a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference +++ b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference @@ -8,11 +8,11 @@ ccccccccc aaaaaaaaa bbbbbbbbb aaaaaaaaa bbbbbbbbb ccccccccc -:107 +:233 :79 :35 :35 -:35 +:233 699415 aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb @@ -21,5 +21,5 @@ ccccccccc aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb 699415 0 :0 -:107 +:70 :79 diff --git a/tests/queries/0_stateless/01710_force_use_projection.reference b/tests/queries/0_stateless/01710_force_use_projection.reference new file mode 100644 index 00000000000..f1ad7de51f0 --- /dev/null +++ b/tests/queries/0_stateless/01710_force_use_projection.reference @@ -0,0 +1 @@ +3 1 diff --git a/tests/queries/0_stateless/01710_force_use_projection.sql b/tests/queries/0_stateless/01710_force_use_projection.sql new file mode 100644 index 00000000000..8931c65e34e --- /dev/null +++ b/tests/queries/0_stateless/01710_force_use_projection.sql @@ -0,0 +1,17 @@ +drop table if exists tp; + +create table tp (d1 Int32, d2 Int32, eventcnt Int64, projection p (select sum(eventcnt) group by d1)) engine = MergeTree order by (d1, d2); + +set allow_experimental_projection_optimization = 1, force_optimize_projection = 1; + +select sum(eventcnt) eventcnt, d1 from tp group by d1; + +select avg(eventcnt) eventcnt, d1 from tp group by d1; + +insert into tp values (1, 2, 3); + +select sum(eventcnt) eventcnt, d1 from tp group by d1; + +select avg(eventcnt) eventcnt, d1 from tp group by d1; -- { serverError 584 } + +drop table tp; diff --git a/tests/queries/0_stateless/01710_projection_group_by_order_by.reference b/tests/queries/0_stateless/01710_projection_group_by_order_by.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01710_projection_group_by_order_by.sql b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql new file mode 100644 index 00000000000..9370e9d36ce --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql @@ -0,0 +1,5 @@ +drop table if exists tp; + +create table tp (type Int32, eventcnt UInt64, projection p (select sum(eventcnt), type group by type order by sum(eventcnt))) engine = MergeTree order by type; -- { serverError 583 } + +drop table if exists tp;